From 7a0b67192140ab307b911719189c52f4fa87033d Mon Sep 17 00:00:00 2001 From: Syakur Rahman Date: Wed, 12 Aug 2026 12:39:48 +0700 Subject: [PATCH 001/117] fix(proxy): durably recover hard HTTP bridge operations (#1657) * fix(proxy): durably recover HTTP bridge operations * fix(proxy): close durable bridge recovery review findings * fix(proxy): harden transcript retention and replay spools * fix(proxy): preserve operation ambiguity across bridge failures * fix(proxy): fence cross-session operation retention * fix(proxy): gate indefinite recovery streams * fix(proxy): preserve recovery event continuity * fix(proxy): harden durable recovery fencing * fix(proxy): finish terminal recovery handling * fix(proxy): fence cleanup health and frame size * fix(proxy): fence active same-session recovery * fix(http-bridge): close recovery ledger review gaps * test(http-bridge): cover terminal spools and migration round trips * fix(http-bridge): gate recovery and preflight metadata * fix(http-bridge): clean up pre-dispatch and abandoned spools * fix(http-bridge): settle replay reservations * fix(http-bridge): reserve operation metadata * fix(http-bridge): normalize reserved identity before hashing * fix(http-bridge): fence local recovery replay * fix(http-bridge): retain durable recovery fences * fix(http-bridge): discard detached operation contexts * fix(http-bridge): preserve same-request recovery operation * fix(http-bridge): complete recovery validation paths * fix(http-bridge): normalize account-neutral operation fingerprints * fix(http-bridge): retain rebind fences across retries * fix(http-bridge): hand off recovery operation leases * fix(http-bridge): fence ambiguous recovery attempts * fix(http-bridge): claim one-shot recovery operations * fix(http-bridge): spool detached terminal events * fix(http-bridge): re-fence recovery race paths * fix(http-bridge): restore security retry fences * fix(cleanup): retain operation transcripts when sticky cleanup is disabled * fix(http-bridge): mark security retry dispatch after preflight * fix(ci): restore hard bridge recovery pipeline * fix(websocket): keep downstream alive during terminal relay * test(websocket): leave margin for delayed upstream events * fix(db): converge recovery migration with latest main * fix(proxy): detach retained sessions across process epochs * fix(proxy): detach retained rows from prior process epoch * fix(proxy): normalize lease timestamps during recovery handoff * fix(http-bridge): persist one-shot recovery budget * fix(bridge): recover startup disconnects safely * fix(bridge): preserve rollback before frame dispatch * fix(bridge): normalize lease timestamps at lookup boundary * fix(bridge): retain recovery owner through replay handoff * fix(bridge): fence ambiguous operation before cleanup * fix(bridge): refund stale replay claims before dispatch * fix(bridge): recover consumed checkpoints safely * fix(bridge): terminalize recovery admission failures * fix(bridge): fence pre-admission terminal recovery * fix(bridge): keep failed rows fenced when spool overflows * fix(bridge): recover stale owners before continuation admission * fix(bridge): fence hard turn-state recovery retries * fix(websocket): bound upstream cleanup close * fix(bridge): advance repeated hard turns * fix(bridge): walk repeated hard-turn chain * test(bridge): cover repeated hard turns * fix(bridge): preserve incomplete hard turns * test(bridge): isolate reconnect affinity key * test(bridge): align reconnect fixture with ring identity * fix(bridge): continue race-path hard-turn chain * fix(bridge): retain anchored hard-turn retry body * fix(bridge): mark fallback anchor as injected * test(bridge): keep quarantine fixture on ring instance * test(bridge): align unsafe resend fixture identity * fix(proxy): damp stale websocket anchor retries * fix(proxy): isolate stale anchor cache per service * fix(proxy): revalidate stale owners across replicas * fix(bridge): settle terminal recovery outcomes * fix(bridge): fence discarded event spools * fix(bridge): preserve consumed recovery checkpoints * ci: rerun flaky integration bridge * fix(shutdown): bound recovery settlement pre-drain * fix(bridge): close remaining Codex review gaps * fix(websocket): cancel timed-out upstream closes * fix(websocket): cancel closes with no drain budget * fix(bridge): preserve acknowledged recovery state * fix(bridge): defer recovery until parent proof * test(bridge): cover recovery parent proof path * test(bridge): exercise durable parent advancement * fix(bridge): roll back unsent durable operations * fix(bridge): clear rolled back operation identity * fix(bridge): retain advanced body across capacity retries * fix(bridge): clear restored operation identity * test(bridge): cover unknown reclaim capacity retry * fix(db): merge recovery and upstream migration heads * test(bridge): assert reclaim precedes dispatch * fix(bridge): persist terminal state before spool finalization * fix(bridge): make takeover and terminal spooling recoverable * fix(bridge): mark atomic terminal spools replayable * fix(bridge): preserve dropped spool safety on takeover * fix(bridge): settle dropped terminal operations * test(bridge): cover dropped failure settlement --------- Co-authored-by: Syakur Rahman Co-authored-by: shaqman Co-authored-by: Codex Co-authored-by: Darafei Praliaskouski --- app/core/config/settings.py | 33 + ...60804_000000_add_http_bridge_operations.py | 63 + ...lobal_http_bridge_operation_fingerprint.py | 53 + ..._000000_add_http_bridge_operation_spool.py | 75 + ...01_finalize_http_bridge_operation_spool.py | 65 + ..._bridge_operation_spool_and_latest_main.py | 24 + ...add_http_bridge_recovery_dispatch_count.py | 44 + ...ery_dispatch_and_hourly_cancelled_heads.py | 25 + app/db/models.py | 81 + app/main.py | 56 +- .../proxy/_service/http_bridge/helpers.py | 8 + .../proxy/_service/http_bridge/mixin.py | 4 + .../proxy/_service/http_bridge/protocol.py | 2 +- .../_service/http_bridge/request_submit.py | 968 ++++++++++- .../_service/http_bridge/retry_circuit.py | 15 +- .../proxy/_service/http_bridge/streaming.py | 365 ++++- .../_service/http_bridge/upstream_events.py | 359 +++- app/modules/proxy/_service/support.py | 19 + .../proxy/_service/websocket/helpers.py | 86 + app/modules/proxy/_service/websocket/mixin.py | 147 +- app/modules/proxy/api.py | 320 +++- app/modules/proxy/continuity.py | 12 + .../proxy/durable_bridge_coordinator.py | 286 ++++ .../proxy/durable_bridge_repository.py | 1076 +++++++++++- .../proxy/http_bridge_event_batcher.py | 341 ++++ app/modules/proxy/service.py | 17 +- .../sticky_sessions/cleanup_scheduler.py | 106 +- docs/reference/settings.md | 12 +- .../.openspec.yaml | 2 + .../context.md | 41 + .../proposal.md | 30 + .../specs/responses-api-compat/spec.md | 358 ++++ .../tasks.md | 49 + scripts/generate_settings_reference.py | 2 + .../integration/test_http_responses_bridge.py | 23 +- .../test_proxy_websocket_responses.py | 5 +- tests/unit/test_bridge_ring_lifecycle.py | 840 ++++++++++ tests/unit/test_db_migrate.py | 94 ++ tests/unit/test_durable_bridge_sessions.py | 74 +- tests/unit/test_graceful_shutdown.py | 31 +- tests/unit/test_http_bridge_event_batcher.py | 166 ++ tests/unit/test_proxy_api_websocket_auth.py | 29 + tests/unit/test_proxy_errors.py | 229 +++ tests/unit/test_proxy_http_bridge.py | 1445 ++++++++++++++++- tests/unit/test_proxy_utils.py | 317 ++++ tests/unit/test_settings_reference.py | 11 +- .../test_sticky_session_cleanup_scheduler.py | 47 + .../test_websocket_terminal_cancellation.py | 73 + 48 files changed, 8206 insertions(+), 322 deletions(-) create mode 100644 app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py create mode 100644 app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py create mode 100644 app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py create mode 100644 app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py create mode 100644 app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py create mode 100644 app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py create mode 100644 app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py create mode 100644 app/modules/proxy/http_bridge_event_batcher.py create mode 100644 openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml create mode 100644 openspec/changes/durable-http-bridge-operation-recovery/context.md create mode 100644 openspec/changes/durable-http-bridge-operation-recovery/proposal.md create mode 100644 openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/durable-http-bridge-operation-recovery/tasks.md create mode 100644 tests/unit/test_http_bridge_event_batcher.py diff --git a/app/core/config/settings.py b/app/core/config/settings.py index e38723400d..d4692d7925 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -306,6 +306,39 @@ class Settings(BaseSettings): le=30.0, ) http_responses_session_bridge_gateway_safe_mode: bool = False + # Attach the durable operation identity to response.create client metadata. + # The upstream must explicitly support/deduplicate this value before any + # automatic replay is enabled; metadata-only propagation is safe by default. + http_responses_session_bridge_operation_ledger_enabled: bool = True + # Bound durable replay storage per operation so a long response cannot + # exhaust the database. An incomplete spool is never replayed. + http_responses_session_bridge_operation_event_spool_max_bytes: int = Field(default=2 * 1024 * 1024, gt=0) + http_responses_session_bridge_operation_event_spool_batch_size: int = Field(default=32, gt=0, le=256) + http_responses_session_bridge_operation_event_spool_flush_interval_seconds: float = Field( + default=0.1, + ge=0.01, + le=5.0, + ) + http_responses_session_bridge_operation_event_spool_max_pending_events: int = Field(default=2048, gt=0) + http_responses_session_bridge_operation_event_spool_max_pending_bytes: int = Field( + default=32 * 1024 * 1024, + gt=0, + ) + # Keep durable transcript material short-lived by default. The transcript + # is sensitive prompt/output data and is only a recovery aid. + http_responses_session_bridge_operation_spool_retention_seconds: float = Field( + default=7 * 24 * 60 * 60, + gt=0, + ) + # Recovery-first mode can either ask the client to drop an ambiguous anchor + # or let the bridge retry that anchored request once on a fresh upstream + # socket. Both are at-least-once strategies; fail-closed remains default. + http_responses_session_bridge_ambiguous_continuation_recovery_mode: Literal[ + "fail_closed", + "client_full_history_once", + "server_anchored_replay_once", + "server_indefinite_recovery", + ] = "fail_closed" http_responses_session_bridge_instance_id: str = Field(default_factory=_default_http_bridge_instance_id) http_responses_session_bridge_instance_ring: Annotated[list[str], NoDecode] = Field(default_factory=list) http_responses_session_bridge_advertise_base_url: str | None = None diff --git a/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py b/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py new file mode 100644 index 0000000000..2022dd0c08 --- /dev/null +++ b/app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py @@ -0,0 +1,63 @@ +"""add durable HTTP bridge operation identities and outcomes + +Revision ID: 20260804_000000_add_http_bridge_operations +Revises: 20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads +Create Date: 2026-08-04 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260804_000000_add_http_bridge_operations" +down_revision = "20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def upgrade() -> None: + bind = op.get_bind() + if _has_table(bind): + return + op.create_table( + _TABLE, + sa.Column("operation_id", sa.String(80), primary_key=True), + sa.Column("session_id", sa.String(36), nullable=False), + sa.Column("request_fingerprint", sa.String(64), nullable=False), + sa.Column("account_id", sa.String(), nullable=True), + sa.Column("model", sa.String(), nullable=True), + sa.Column("parent_response_id", sa.Text(), nullable=True), + sa.Column("state", sa.String(32), nullable=False, server_default="submitted"), + sa.Column("response_id", sa.Text(), nullable=True), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint(["session_id"], ["http_bridge_sessions.id"], ondelete="CASCADE"), + sa.UniqueConstraint( + "session_id", + "request_fingerprint", + name="uq_http_bridge_operations_session_fingerprint", + ), + ) + op.create_index( + "idx_http_bridge_operations_session_parent_state", + _TABLE, + ["session_id", "parent_response_id", "state"], + ) + op.create_index("idx_http_bridge_operations_state_updated", _TABLE, ["state", "updated_at"]) + + +def downgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + op.drop_index("idx_http_bridge_operations_state_updated", table_name=_TABLE) + op.drop_index("idx_http_bridge_operations_session_parent_state", table_name=_TABLE) + op.drop_table(_TABLE) diff --git a/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py b/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py new file mode 100644 index 0000000000..016641815d --- /dev/null +++ b/app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py @@ -0,0 +1,53 @@ +"""fence HTTP bridge operations across durable sessions + +Revision ID: 20260804_000001_add_global_http_bridge_operation_fingerprint +Revises: 20260804_000000_add_http_bridge_operations +Create Date: 2026-08-04 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" +down_revision = "20260804_000000_add_http_bridge_operations" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_FINGERPRINT_INDEX = "uq_http_bridge_operations_request_fingerprint" +_PARENT_INDEX = "idx_http_bridge_operations_parent_state" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_index(connection: Connection, name: str) -> bool: + return any(index.get("name") == name for index in sa.inspect(connection).get_indexes(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + # A request fingerprint includes the parent response anchor, so it is a + # global operation identity even when a client reconnects to another + # durable bridge session. The unique index also closes the race where two + # workers observe a miss and try to dispatch the same continuation. + if not _has_index(bind, _FINGERPRINT_INDEX): + op.create_index(_FINGERPRINT_INDEX, _TABLE, ["request_fingerprint"], unique=True) + if not _has_index(bind, _PARENT_INDEX): + op.create_index(_PARENT_INDEX, _TABLE, ["parent_response_id", "state", "updated_at"]) + + +def downgrade() -> None: + bind = op.get_bind() + if not _has_table(bind): + return + if _has_index(bind, _PARENT_INDEX): + op.drop_index(_PARENT_INDEX, table_name=_TABLE) + if _has_index(bind, _FINGERPRINT_INDEX): + op.drop_index(_FINGERPRINT_INDEX, table_name=_TABLE) diff --git a/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py b/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py new file mode 100644 index 0000000000..a9f23893c2 --- /dev/null +++ b/app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py @@ -0,0 +1,75 @@ +"""add durable HTTP bridge operation request and event spool + +Revision ID: 20260805_000000_add_http_bridge_operation_spool +Revises: 20260804_000001_add_global_http_bridge_operation_fingerprint +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260805_000000_add_http_bridge_operation_spool" +down_revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_EVENTS_TABLE = "http_bridge_operation_events" + + +def _has_table(connection: Connection, table: str) -> bool: + return sa.inspect(connection).has_table(table) + + +def _has_column(connection: Connection, table: str, column: str) -> bool: + return any(item["name"] == column for item in sa.inspect(connection).get_columns(table)) + + +def upgrade() -> None: + bind = op.get_bind() + if _has_table(bind, _TABLE): + if not _has_column(bind, _TABLE, "request_text"): + op.add_column(_TABLE, sa.Column("request_text", sa.Text(), nullable=True)) + if not _has_column(bind, _TABLE, "event_bytes"): + op.add_column(_TABLE, sa.Column("event_bytes", sa.Integer(), nullable=False, server_default="0")) + if not _has_column(bind, _TABLE, "event_spool_complete"): + op.add_column( + _TABLE, + sa.Column("event_spool_complete", sa.Boolean(), nullable=False, server_default=sa.text("true")), + ) + if _has_table(bind, _EVENTS_TABLE): + return + op.create_table( + _EVENTS_TABLE, + sa.Column("event_id", sa.String(36), primary_key=True), + sa.Column("operation_id", sa.String(80), nullable=False), + sa.Column("sequence_number", sa.Integer(), nullable=False), + sa.Column("event_fingerprint", sa.String(64), nullable=False), + sa.Column("event_text", sa.Text(), nullable=False), + sa.Column("created_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint(["operation_id"], [f"{_TABLE}.operation_id"], ondelete="CASCADE"), + sa.UniqueConstraint( + "operation_id", + "event_fingerprint", + name="uq_http_bridge_operation_events_operation_fingerprint", + ), + ) + op.create_index( + "idx_http_bridge_operation_events_operation_sequence", + _EVENTS_TABLE, + ["operation_id", "sequence_number"], + ) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind, _EVENTS_TABLE): + op.drop_index("idx_http_bridge_operation_events_operation_sequence", table_name=_EVENTS_TABLE) + op.drop_table(_EVENTS_TABLE) + if _has_table(bind, _TABLE): + for column in ("event_spool_complete", "event_bytes", "request_text"): + if _has_column(bind, _TABLE, column): + op.drop_column(_TABLE, column) diff --git a/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py b/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py new file mode 100644 index 0000000000..2194bd9029 --- /dev/null +++ b/app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py @@ -0,0 +1,65 @@ +"""make operation event spool completion conservative + +Revision ID: 20260805_000001_finalize_http_bridge_operation_spool +Revises: 20260805_000000_add_http_bridge_operation_spool +Create Date: 2026-08-05 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260805_000001_finalize_http_bridge_operation_spool" +down_revision = "20260805_000000_add_http_bridge_operation_spool" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_column(connection: Connection, column: str) -> bool: + return any(item["name"] == column for item in sa.inspect(connection).get_columns(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind) or not _has_column(bind, "event_spool_complete"): + return + # Rows written by older releases used true as the implicit value. They + # cannot be replayed safely unless their event queue is drained again. + op.execute(sa.text(f"UPDATE {_TABLE} SET event_spool_complete = false")) + # SQLite has no direct ALTER COLUMN syntax. Alembic's batch operation + # rebuilds the table and preserves the false default for future inserts; + # merely changing the ORM declaration would leave the old true default in + # sqlite_master. + if bind.dialect.name == "sqlite": + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.alter_column( + "event_spool_complete", + existing_type=sa.Boolean(), + existing_nullable=False, + server_default=sa.text("false"), + ) + else: + op.alter_column(_TABLE, "event_spool_complete", server_default=sa.text("false")) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind) and _has_column(bind, "event_spool_complete"): + if bind.dialect.name == "sqlite": + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.alter_column( + "event_spool_complete", + existing_type=sa.Boolean(), + existing_nullable=False, + server_default=sa.text("true"), + ) + else: + op.alter_column(_TABLE, "event_spool_complete", server_default=sa.text("true")) diff --git a/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py b/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py new file mode 100644 index 0000000000..1a19cc4299 --- /dev/null +++ b/app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py @@ -0,0 +1,24 @@ +"""merge the durable HTTP bridge ledger with the current release head. + +The operation-ledger revisions were authored on the recovery branch while +main continued to receive additive schema revisions. This no-op merge keeps +Alembic at one head without rewriting either already-applied lineage. +""" + +from __future__ import annotations + +revision = "20260807_000000_merge_http_bridge_operation_spool_and_latest_main" +down_revision = ( + "20260806_120000_add_http_bridge_owner_process_epoch", + "20260805_000001_finalize_http_bridge_operation_spool", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py b/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py new file mode 100644 index 0000000000..4b08cf493e --- /dev/null +++ b/app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py @@ -0,0 +1,44 @@ +"""persist the HTTP bridge ambiguous recovery dispatch budget + +Revision ID: 20260810_000000_add_http_bridge_recovery_dispatch_count +Revises: 20260807_000000_merge_http_bridge_operation_spool_and_latest_main +Create Date: 2026-08-10 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260810_000000_add_http_bridge_recovery_dispatch_count" +down_revision = "20260807_000000_merge_http_bridge_operation_spool_and_latest_main" +branch_labels = None +depends_on = None + +_TABLE = "http_bridge_operations" +_COLUMN = "recovery_dispatch_count" + + +def _has_table(connection: Connection) -> bool: + return sa.inspect(connection).has_table(_TABLE) + + +def _has_column(connection: Connection) -> bool: + return any(item["name"] == _COLUMN for item in sa.inspect(connection).get_columns(_TABLE)) + + +def upgrade() -> None: + bind = op.get_bind() + if not _has_table(bind) or _has_column(bind): + return + op.add_column( + _TABLE, + sa.Column(_COLUMN, sa.Integer(), nullable=False, server_default=sa.text("0")), + ) + + +def downgrade() -> None: + bind = op.get_bind() + if _has_table(bind) and _has_column(bind): + op.drop_column(_TABLE, _COLUMN) diff --git a/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py b/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py new file mode 100644 index 0000000000..78c4105b29 --- /dev/null +++ b/app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py @@ -0,0 +1,25 @@ +"""merge the recovery-dispatch and hourly-rollup migration heads. + +The durable HTTP bridge recovery branch and upstream's cancelled-count rollup +landed as independent additive revisions. This no-op merge keeps startup and +CI migration checks at one canonical Alembic head without rewriting either +already-applied lineage. +""" + +from __future__ import annotations + +revision = "20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads" +down_revision = ( + "20260810_000000_add_http_bridge_recovery_dispatch_count", + "20260811_000000_add_hourly_rollup_cancelled_count", +) +branch_labels = None +depends_on = None + + +def upgrade() -> None: + pass + + +def downgrade() -> None: + pass diff --git a/app/db/models.py b/app/db/models.py index 1989d82216..ae6d71eecd 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -1804,6 +1804,14 @@ class HttpBridgeRecoveryAttemptState(str, Enum): REPLAYED = "replayed" +class HttpBridgeOperationState(str, Enum): + SUBMITTED = "submitted" + UNKNOWN = "unknown" + ACKNOWLEDGED = "acknowledged" + COMPLETED = "completed" + FAILED = "failed" + + class HttpBridgeSessionRecord(Base): __tablename__ = "http_bridge_sessions" @@ -1917,6 +1925,79 @@ class HttpBridgeRecoveryAttemptRecord(Base): ) +class HttpBridgeOperationRecord(Base): + """Durable identity and outcome for a continuity-bound response.create.""" + + __tablename__ = "http_bridge_operations" + + operation_id: Mapped[str] = mapped_column(String(80), primary_key=True) + session_id: Mapped[str] = mapped_column( + String(36), + ForeignKey("http_bridge_sessions.id", ondelete="CASCADE"), + nullable=False, + ) + request_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + account_id: Mapped[str | None] = mapped_column(String, nullable=True) + model: Mapped[str | None] = mapped_column(String, nullable=True) + parent_response_id: Mapped[str | None] = mapped_column(Text, nullable=True) + request_text: Mapped[str | None] = mapped_column(Text, nullable=True) + state: Mapped[str] = mapped_column(String(32), nullable=False, server_default=text("'submitted'")) + response_id: Mapped[str | None] = mapped_column(Text, nullable=True) + recovery_dispatch_count: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + event_bytes: Mapped[int] = mapped_column(Integer, nullable=False, server_default=text("0")) + event_spool_complete: Mapped[bool] = mapped_column(Boolean, nullable=False, server_default=text("false")) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now() + ) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now(), onupdate=func.now() + ) + + __table_args__ = ( + UniqueConstraint( + "session_id", + "request_fingerprint", + name="uq_http_bridge_operations_session_fingerprint", + ), + Index( + "uq_http_bridge_operations_request_fingerprint", + "request_fingerprint", + unique=True, + ), + Index("idx_http_bridge_operations_session_parent_state", "session_id", "parent_response_id", "state"), + Index("idx_http_bridge_operations_parent_state", "parent_response_id", "state", "updated_at"), + Index("idx_http_bridge_operations_state_updated", "state", "updated_at"), + ) + + +class HttpBridgeOperationEvent(Base): + """Replayable upstream SSE blocks for a durable bridge operation.""" + + __tablename__ = "http_bridge_operation_events" + + event_id: Mapped[str] = mapped_column(String(36), primary_key=True, default=lambda: str(uuid.uuid4())) + operation_id: Mapped[str] = mapped_column( + String(80), + ForeignKey("http_bridge_operations.operation_id", ondelete="CASCADE"), + nullable=False, + ) + sequence_number: Mapped[int] = mapped_column(Integer, nullable=False) + event_fingerprint: Mapped[str] = mapped_column(String(64), nullable=False) + event_text: Mapped[str] = mapped_column(Text, nullable=False) + created_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), nullable=False, default=func.now(), server_default=func.now() + ) + + __table_args__ = ( + UniqueConstraint( + "operation_id", + "event_fingerprint", + name="uq_http_bridge_operation_events_operation_fingerprint", + ), + Index("idx_http_bridge_operation_events_operation_sequence", "operation_id", "sequence_number"), + ) + + class HttpBridgeSessionAlias(Base): __tablename__ = "http_bridge_session_aliases" diff --git a/app/main.py b/app/main.py index 00f3c78786..0ef01d28c5 100644 --- a/app/main.py +++ b/app/main.py @@ -142,6 +142,26 @@ async def _release_leader_lease_within(timeout: float) -> None: logger.warning("Failed to release scheduler leader lease during shutdown", exc_info=exc) +async def _drain_proxy_persistence_tasks( + proxy_service: Any, + timeout_seconds: float, + *, + task_name_prefixes: tuple[str, ...] | None = None, + failure_message: str, +) -> bool: + """Drain proxy persistence work within the caller's committed deadline.""" + if proxy_service is None or not hasattr(proxy_service, "drain_persistence_tasks"): + return True + try: + kwargs: dict[str, Any] = {"timeout_seconds": timeout_seconds} + if task_name_prefixes is not None: + kwargs["task_name_prefixes"] = task_name_prefixes + return bool(await proxy_service.drain_persistence_tasks(**kwargs)) + except Exception: + logger.warning(failure_message, exc_info=True) + return False + + async def _drain_detached_control_plane_tasks(timeout_seconds: float) -> None: # Closing admission is synchronous with producer checks on the event loop, # so no task can appear after the stable drain passes complete. @@ -288,6 +308,15 @@ async def lifespan(app: FastAPI): "deleted": deleted_bridge_rows, }, ) + purged_operation_rows = await DurableBridgeSessionCoordinator(SessionLocal).purge_operation_spool( + cutoff=utcnow() + - timedelta(seconds=settings.http_responses_session_bridge_operation_spool_retention_seconds), + ) + if purged_operation_rows > 0: + logger.info( + "Purged expired durable HTTP bridge operation transcript rows", + extra={"deleted": purged_operation_rows}, + ) from app.core.auth.api_key_cache import get_api_key_cache from app.core.cache.invalidation import ( NAMESPACE_ACCOUNT_ROUTING, @@ -537,14 +566,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N recovery_settlements_drained = True # Settle detached recovery journals while their origin leases are # still held; bridge teardown below may release those owner fences. - if proxy_service is not None and hasattr(proxy_service, "drain_persistence_tasks"): - try: - recovery_settlements_drained = await proxy_service.drain_persistence_tasks( - timeout_seconds=settings.shutdown_drain_timeout_seconds, - task_name_prefixes=("http-bridge-recovery-settlement-",), - ) - except Exception: - logger.warning("Failed to pre-drain proxy settlement tasks during shutdown", exc_info=True) + remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 + recovery_settlements_drained = await _drain_proxy_persistence_tasks( + proxy_service, + remaining_drain_seconds, + task_name_prefixes=("http-bridge-recovery-settlement-",), + failure_message="Failed to pre-drain proxy settlement tasks during shutdown", + ) if ( recovery_settlements_drained and proxy_service is not None @@ -562,12 +590,12 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N # Drain AFTER the bridge teardown: failing a bridge's pending # requests writes their request logs, which enqueues more # persistence tasks that this drain must cover. - if proxy_service is not None and hasattr(proxy_service, "drain_persistence_tasks"): - try: - remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 - await proxy_service.drain_persistence_tasks(timeout_seconds=remaining_drain_seconds) - except Exception: - logger.warning("Failed to drain proxy persistence tasks during shutdown", exc_info=True) + remaining_drain_seconds = shutdown_state.remaining_drain_timeout_seconds() or 0.0 + await _drain_proxy_persistence_tasks( + proxy_service, + remaining_drain_seconds, + failure_message="Failed to drain proxy persistence tasks during shutdown", + ) # Cancel heartbeat and age the shared ring row near expiry. if heartbeat_task is not None: diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index d98842b5c8..fd5e0383ff 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -2399,6 +2399,14 @@ def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyRespo "bridge_instance_mismatch", }: return True + if code in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout"}: + # Recovery-first server mode permits exactly one anchored retry on a + # fresh upstream socket. This keeps Codex unchanged; delivery remains + # at-least-once because upstream acceptance is ambiguous. + return _service_get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode in { + "server_anchored_replay_once", + "server_indefinite_recovery", + } param_value = error.get("param") param = param_value.strip() if isinstance(param_value, str) and param_value.strip() else None message_value = error.get("message") diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index e164edc072..bddeab2592 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -1611,6 +1611,10 @@ async def close_all_http_bridge_sessions(self) -> None: for session in sessions_to_close: await self._close_http_bridge_session(session) await self._drain_http_bridge_background_cleanup_tasks(reason="shutdown") + event_batcher = getattr(self, "_http_bridge_operation_event_batcher", None) + close_batcher = getattr(event_batcher, "close", None) + if callable(close_batcher): + await close_batcher() async def mark_http_bridge_draining(self) -> None: try: diff --git a/app/modules/proxy/_service/http_bridge/protocol.py b/app/modules/proxy/_service/http_bridge/protocol.py index 841d02c014..a15bac84cc 100644 --- a/app/modules/proxy/_service/http_bridge/protocol.py +++ b/app/modules/proxy/_service/http_bridge/protocol.py @@ -43,7 +43,7 @@ def _raise_for_unsupported_input_image_references(self, payload: ResponsesReques async def _resolve_file_account_for_responses( self, payload: ResponsesRequest, headers: Mapping[str, str] ) -> str | None: ... - async def _fail_pending_websocket_requests(self, *args: Any, **kwargs: Any) -> None: ... + async def _fail_pending_websocket_requests(self, *args: Any, **kwargs: Any) -> bool: ... async def _finalize_websocket_request_state(self, *args: Any, **kwargs: Any) -> None: ... async def _next_websocket_receive_timeout(self, *args: Any, **kwargs: Any) -> Any: ... async def _close_http_bridge_session_bounded(self, session: _HTTPBridgeSession, *, reason: str) -> None: ... diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index a84ea4aec6..d987ea3ecc 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -6,6 +6,7 @@ import math import random from collections import deque +from collections.abc import Callable from dataclasses import replace from typing import Any, Literal, Mapping, cast from uuid import uuid4 @@ -14,8 +15,9 @@ from app.core.clients.files import create_file as core_create_file # noqa: F401 from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401 -from app.core.clients.proxy import CodexControlResponse as CodexControlResponse from app.core.clients.proxy import ( # noqa: F401 + CODEX_INSTALLATION_ID_HEADER, + CODEX_TURN_METADATA_HEADER, ImageFetchSession, ProxyResponseError, UpstreamProxyRouteTrace, @@ -35,6 +37,7 @@ push_stream_timeout_overrides, push_transcribe_timeout_overrides, ) +from app.core.clients.proxy import CodexControlResponse as CodexControlResponse from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401 from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401 from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 @@ -43,9 +46,7 @@ UpstreamWebSocketTransportError, is_account_neutral_websocket_error_code, ) -from app.core.errors import ( - openai_error, -) +from app.core.errors import OpenAIErrorEnvelope, openai_error from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ( ResponsesRequest, @@ -192,7 +193,10 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, + durable_bridge_api_key_scope, durable_bridge_hash, + durable_bridge_operation_fingerprint, + durable_bridge_operation_id, ) from app.modules.proxy.fair_share import ( API_KEY_STREAM_FAIR_SHARE_ERROR_CODE, @@ -222,6 +226,109 @@ ) +def _http_bridge_client_full_history_recovery_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether an ambiguous send failure may ask the client to replay.""" + settings = _service_get_settings() + return ( + request_state.propagate_http_errors + and getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + == "client_full_history_once" + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + ) + + +def _http_bridge_server_anchored_replay_enabled(request_state: _WebSocketRequestState) -> bool: + settings = _service_get_settings() + return ( + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and ( + request_state.replay_count == 0 + or getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "") + == "server_indefinite_recovery" + ) + ) + + +def _http_bridge_operation_fence_for_hard_continuity_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether a hard turn-state request may use the durable replay fence.""" + if not request_state.hard_continuity_anchor: + return False + return getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) in {"server_anchored_replay_once", "server_indefinite_recovery"} + + +def _http_bridge_operation_fingerprint( + *, + session_id: str, + api_key_scope: str, + request_state: _WebSocketRequestState, + text_data: str, +) -> str: + fingerprint_text = _text_without_account_installation_id(text_data) + if request_state.previous_response_id is None and _http_bridge_operation_fence_for_hard_continuity_enabled( + request_state + ): + # Hard turn-state requests do not carry previous_response_id. Scope + # their operation identity to the durable session so identical prompts + # in two conversations cannot collide in the global fingerprint fence. + fingerprint_text = f"session:{session_id}\n{fingerprint_text}" + return durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=fingerprint_text, + ) + + +def _http_bridge_terminal_hard_turn_response_id( + request_state: _WebSocketRequestState, + operation: Any, + *, + allow_anchored_continuation: bool = False, +) -> str | None: + """Return a completed hard-turn anchor when this is a new client turn. + + Hard turn-state requests can omit ``previous_response_id``. Their durable + operation fingerprint is therefore otherwise identical for repeated + prompts. A terminal operation with a response id represents the prior turn, + not an in-flight retry, so the next request must advance from its response + rather than replaying that transcript. Recovery/rebind states retain the + operation identity and are intentionally excluded here. Spool completeness + is required only when replaying the stored transcript. + """ + if ( + (request_state.previous_response_id is not None and not allow_anchored_continuation) + or not request_state.hard_continuity_anchor + or request_state.operation_id is not None + or request_state.operation_rebind_required + or request_state.replay_count != 0 + ): + return None + operation_state = getattr(operation, "state", None) + operation_state = getattr(operation_state, "value", operation_state) + if operation_state != "completed": + return None + response_id = getattr(operation, "response_id", None) + return response_id if isinstance(response_id, str) and response_id else None + + +def _http_bridge_client_full_history_recovery_error() -> OpenAIErrorEnvelope: + payload = openai_error( + "previous_response_not_found", + "Previous response was not found; retry without previous_response_id.", + error_type="invalid_request_error", + ) + payload["error"]["param"] = "previous_response_id" + return payload + + async def _rollback_http_bridge_recovery_turn_state_registration( service: Any, receipt: DurableBridgeAliasRegistrationReceipt, @@ -236,7 +343,16 @@ async def _send_http_bridge_request_text_with_archive_id( session: "_HTTPBridgeSession", request_state: _WebSocketRequestState, text_data: str, + *, + on_send_started: Callable[[], None] | None = None, ) -> None: + text_data = _text_with_operation_id(text_data, request_state.operation_id) + # Operation metadata is added after the initial payload sizing pass. Check + # the exact frame that will cross the websocket so the metadata cannot + # push an otherwise-valid response.create over the upstream limit. + _enforce_http_bridge_response_create_text_size(request_state, text_data) + if on_send_started is not None: + on_send_started() token = set_request_id(request_state.archive_request_id) try: request_state.response_create_sent_at = _service_time().monotonic() @@ -282,6 +398,89 @@ def _text_with_account_installation_id(text_data: str, codex_installation_id: st return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) +def _text_with_operation_id(text_data: str, operation_id: str | None) -> str: + """Attach a stable operation identity without changing the request contract.""" + if not operation_id: + return text_data + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + metadata = dict(raw_metadata) if isinstance(raw_metadata, dict) else {} + # This namespace is reserved by the bridge; never trust a caller-supplied + # value to stand in for the durable operation identity. + metadata["codex_lb_operation_id"] = operation_id + payload["client_metadata"] = metadata + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_without_operation_id(text_data: str) -> str: + """Remove caller-supplied bridge identity before durable fingerprinting.""" + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + if not isinstance(raw_metadata, dict) or "codex_lb_operation_id" not in raw_metadata: + return text_data + metadata = dict(raw_metadata) + metadata.pop("codex_lb_operation_id", None) + if metadata: + payload["client_metadata"] = metadata + else: + payload.pop("client_metadata", None) + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_without_account_installation_id(text_data: str) -> str: + """Normalize account-specific installation metadata out of a fingerprint.""" + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict): + return text_data + raw_metadata = payload.get("client_metadata") + if not isinstance(raw_metadata, dict): + return text_data + metadata: dict[str, JsonValue] = {} + for key, value in raw_metadata.items(): + if not isinstance(key, str) or key.lower() == CODEX_INSTALLATION_ID_HEADER: + continue + if key.lower() == CODEX_TURN_METADATA_HEADER and isinstance(value, str): + try: + turn_metadata = json.loads(value) + except json.JSONDecodeError: + turn_metadata = None + if isinstance(turn_metadata, dict) and "installation_id" in turn_metadata: + turn_metadata.pop("installation_id", None) + value = json.dumps(turn_metadata, ensure_ascii=True, separators=(",", ":")) + metadata[key] = value + if metadata: + payload["client_metadata"] = metadata + else: + payload.pop("client_metadata", None) + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + +def _text_with_previous_response_id(text_data: str, response_id: str | None) -> str: + if not response_id: + return text_data + try: + payload = json.loads(text_data) + except (TypeError, json.JSONDecodeError): + return text_data + if not isinstance(payload, dict) or not response_id: + return text_data + payload["previous_response_id"] = response_id + return json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + + def _enforce_http_bridge_response_create_text_size( request_state: _WebSocketRequestState, text_data: str, @@ -617,6 +816,51 @@ async def _submit_http_bridge_request( request_scope_id=request_scope_id, ) + async def _http_bridge_operation_fenced_continuity_replay_allowed( + self: Any, + session: "_HTTPBridgeSession", + *, + request_state: _WebSocketRequestState, + text_data: str, + ) -> bool: + """Allow a cooldown bypass only for an already-fenced hard turn.""" + if ( + not _http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + or request_state.previous_response_id is not None + or session.durable_session_id is None + or session.durable_owner_epoch is None + ): + return False + get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None) + if not callable(get_operation_by_fingerprint): + return False + api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id) + request_fingerprint = _http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope=api_key_scope, + request_state=request_state, + text_data=text_data, + ) + try: + operation = await _call_with_supported_optional_kwargs( + get_operation_by_fingerprint, + optional_kwargs={"api_key_scope": api_key_scope}, + request_fingerprint=request_fingerprint, + ) + except Exception: + logger.warning( + "Failed to inspect hard-continuity operation fence before retry request_id=%s", + request_state.request_id, + exc_info=True, + ) + return False + if operation is None or operation.session_id != session.durable_session_id: + return False + operation_state = getattr(operation.state, "value", operation.state) + return operation_state == "unknown" or ( + operation_state in {"completed", "incomplete"} and bool(getattr(operation, "event_spool_complete", False)) + ) + async def _submit_http_bridge_request_with_handoff( self: Any, session: "_HTTPBridgeSession", @@ -627,6 +871,18 @@ async def _submit_http_bridge_request_with_handoff( request_scope_id: str, recovery_turn_state: str | None = None, ) -> None: + recovery_attempt_consumed = False + allow_operation_fenced_continuity_replay = False + if _http_bridge_operation_fence_for_hard_continuity_enabled(request_state): + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) + if retry_cooldown_seconds > 0: + allow_operation_fenced_continuity_replay = ( + await self._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data=text_data, + ) + ) # Eventless upstream timeouts retire the current socket. A client # reconnect can otherwise create a fresh socket for the same hard key # and submit the identical request repeatedly while the retry circuit @@ -642,9 +898,11 @@ async def _submit_http_bridge_request_with_handoff( and request_state.response_event_count == 0 and request_state.replay_count == 0 ) + allow_server_anchored_replay = _http_bridge_server_anchored_replay_enabled(request_state) if not await self._http_bridge_precreated_retry_allowed( session, - allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay, + allow_proof_gated_continuity_replay=allow_proof_gated_continuity_replay or allow_server_anchored_replay, + allow_operation_fenced_continuity_replay=allow_operation_fenced_continuity_replay, ): retry_after_seconds = max( 1, @@ -714,14 +972,24 @@ async def _submit_http_bridge_request_with_handoff( ), ) if getattr(attempt.state, "value", attempt.state) != "unknown": - raise ProxyResponseError( - 502, - openai_error( - "bridge_continuity_persistence_failed", - "The recovery checkpoint was already consumed; retry the request.", - ), - ) - if getattr(attempt, "request_id", request_state.request_id) != request_state.request_id: + if getattr(attempt.state, "value", attempt.state) != "replayed": + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + # A REPLAYED checkpoint may belong to a completed + # operation whose finalized transcript is safe to replay. + # Defer rejection until the operation ledger lookup below + # can decide that terminal-spool case; nonterminal rows + # remain fail-closed after that lookup. + recovery_attempt_consumed = True + if ( + not recovery_attempt_consumed + and getattr(attempt, "request_id", request_state.request_id) != request_state.request_id + ): raise ProxyResponseError( 502, openai_error( @@ -756,8 +1024,371 @@ async def _submit_http_bridge_request_with_handoff( "Recovered response continuity could not be persisted; retry the request.", ), ) from exc + # Account installation metadata is part of the final upstream frame. + # Apply and size-check it before recording the operation so a local + # payload-too-large rejection cannot leave a submitted retry fence. + text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data) + operation_ledger_enabled = bool( + getattr(_service_get_settings(), "http_responses_session_bridge_operation_ledger_enabled", True) + ) + operation_ledger_for_hard_continuity = _http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + record_operation = getattr(self._durable_bridge, "record_operation", None) + if ( + operation_ledger_enabled + and callable(record_operation) + and ( + request_state.previous_response_id is not None + or operation_ledger_for_hard_continuity + or request_state.operation_rebind_required + or recovery_attempt_consumed + ) + and (request_state.operation_id is None or request_state.operation_rebind_required) + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + text_data = _text_without_operation_id(text_data) + api_key_scope = durable_bridge_api_key_scope(session.key.api_key_id) + operation_fingerprint = ( + request_state.operation_fingerprint + if request_state.operation_rebind_required and request_state.operation_fingerprint is not None + else _http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope=api_key_scope, + request_state=request_state, + text_data=text_data, + ) + ) + operation_id = ( + request_state.operation_id + if request_state.operation_rebind_required and request_state.operation_id is not None + else durable_bridge_operation_id(session.durable_session_id, operation_fingerprint) + ) + operation_parent_response_id = ( + request_state.operation_parent_response_id + if request_state.operation_rebind_required + else request_state.previous_response_id + ) + # The operation row must not be committed until the exact + # operation-tagged frame is known to fit. Otherwise a local size + # rejection before ``send_text`` leaves a submitted ledger row + # that fences every identical retry as an unknown in-flight turn. + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + try: + get_operation_by_fingerprint = getattr(self._durable_bridge, "get_operation_by_fingerprint", None) + get_operation = getattr(self._durable_bridge, "get_operation", None) + + async def lookup_operation() -> Any: + operation = None + if callable(get_operation_by_fingerprint): + operation = await _call_with_supported_optional_kwargs( + get_operation_by_fingerprint, + optional_kwargs={"api_key_scope": api_key_scope}, + request_fingerprint=operation_fingerprint, + ) + if operation is None and callable(get_operation): + operation = await get_operation(operation_id=operation_id) + return operation + + existing_operation = await lookup_operation() + if recovery_attempt_consumed and existing_operation is None: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + hard_turn_chain_advanced = False + seen_hard_turn_response_ids: set[str] = set() + while not recovery_attempt_consumed: + terminal_hard_turn_response_id = _http_bridge_terminal_hard_turn_response_id( + request_state, + existing_operation, + allow_anchored_continuation=hard_turn_chain_advanced, + ) + if ( + terminal_hard_turn_response_id is not None + and terminal_hard_turn_response_id not in seen_hard_turn_response_ids + ): + # A completed operation with the same body is the + # prior hard turn, not a replay request: advance from + # its response instead of replaying that transcript. + # Keep walking the chain because repeated identical + # turns can have several terminal operations with + # successive response anchors. + seen_hard_turn_response_ids.add(terminal_hard_turn_response_id) + hard_turn_chain_advanced = True + text_data = _text_with_previous_response_id(text_data, terminal_hard_turn_response_id) + request_state.request_text = text_data + request_state.previous_response_id = terminal_hard_turn_response_id + request_state.proxy_injected_previous_response_id = True + request_state.hard_continuity_anchor = True + operation_parent_response_id = terminal_hard_turn_response_id + operation_fingerprint = durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=_text_without_account_installation_id(text_data), + ) + operation_id = durable_bridge_operation_id( + session.durable_session_id, + operation_fingerprint, + ) + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + existing_operation = await lookup_operation() + continue + + # If another worker durably observed the previous turn's + # completion, advance a new continuation to that response + # anchor instead of replaying the timed-out turn. Re-run + # the operation lookup after this race-path advancement so + # two completions observed back-to-back are both walked. + if existing_operation is None: + get_latest_completed = getattr(self._durable_bridge, "get_latest_completed_operation", None) + if callable(get_latest_completed): + completed_operation = await _call_with_supported_optional_kwargs( + get_latest_completed, + optional_kwargs={"request_fingerprint": operation_fingerprint}, + session_id=session.durable_session_id, + parent_response_id=operation_parent_response_id, + ) + if completed_operation is None: + get_latest_completed_any_session = getattr( + self._durable_bridge, + "get_latest_completed_operation_any_session", + None, + ) + if callable(get_latest_completed_any_session): + completed_operation = await _call_with_supported_optional_kwargs( + get_latest_completed_any_session, + optional_kwargs={ + "api_key_scope": api_key_scope, + "request_fingerprint": operation_fingerprint, + }, + parent_response_id=request_state.previous_response_id, + ) + completed_response_id = getattr(completed_operation, "response_id", None) + if completed_response_id and completed_response_id != request_state.previous_response_id: + text_data = _text_with_previous_response_id(text_data, completed_response_id) + request_state.request_text = text_data + request_state.previous_response_id = completed_response_id + request_state.proxy_injected_previous_response_id = True + operation_parent_response_id = completed_response_id + hard_turn_chain_advanced = True + seen_hard_turn_response_ids.add(completed_response_id) + request_state.hard_continuity_anchor = True + operation_fingerprint = durable_bridge_operation_fingerprint( + api_key_scope=api_key_scope, + request_text=_text_without_account_installation_id(text_data), + ) + operation_id = durable_bridge_operation_id( + session.durable_session_id, + operation_fingerprint, + ) + operation_tagged_text = _text_with_operation_id(text_data, operation_id) + _enforce_http_bridge_response_create_text_size(request_state, operation_tagged_text) + existing_operation = await lookup_operation() + continue + break + operation = await _call_with_supported_optional_kwargs( + record_operation, + optional_kwargs={ + "recovery_attempt_session_id": request_state.recovery_attempt_session_id + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_owner_epoch": request_state.recovery_attempt_owner_epoch + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_fingerprint": request_state.recovery_attempt_fingerprint + if request_state.recovery_attempt_claimed + else None, + "recovery_attempt_consumed": recovery_attempt_consumed, + }, + operation_id=operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=operation_fingerprint, + api_key_scope=api_key_scope, + account_id=session.account.id, + model=request_state.model, + parent_response_id=operation_parent_response_id, + request_text=text_data, + ) + except Exception as exc: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + _record_continuity_fail_closed( + surface="http_bridge", + reason="operation_persistence_failed", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code="bridge_continuity_persistence_failed", + ) + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Response operation continuity could not be persisted; retry the request.", + ), + ) from exc + if operation is None: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership changed; retry the request.", + ), + ) + if recovery_attempt_consumed and operation.created: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + if not operation.created: + if operation.state in {"completed", "incomplete"}: + if getattr(operation, "event_spool_complete", False): + get_operation_events = getattr(self._durable_bridge, "get_operation_events", None) + replay_events = ( + await get_operation_events(operation_id=operation.operation_id) + if callable(get_operation_events) + else [] + ) + if replay_events and request_state.event_queue is not None: + request_state.operation_replay = True + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_registered = True + for replay_event in replay_events: + await request_state.event_queue.put(replay_event) + await request_state.event_queue.put(None) + return + if recovery_attempt_consumed: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "The recovery checkpoint was already consumed; retry the request.", + ), + ) + recovery_mode = getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) + indefinite_recovery = recovery_mode == "server_indefinite_recovery" + one_shot_recovery = recovery_mode == "server_anchored_replay_once" and request_state.replay_count == 0 + async with session.pending_lock: + same_operation_pending = any( + pending_request is not request_state + and getattr(pending_request, "operation_id", None) == operation.operation_id + for pending_request in session.pending_requests + ) + if ( + (indefinite_recovery or one_shot_recovery) + and operation.state == "unknown" + and not same_operation_pending + ): + # A previous owner may have persisted a partial sequence + # before its socket died. Claim UNKNOWN atomically with + # the transcript reset so concurrent reconnects cannot + # both pass admission and submit the same operation. + claim_unknown_operation = getattr( + self._durable_bridge, + "claim_unknown_operation_for_recovery", + None, + ) + if not callable(claim_unknown_operation): + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery could not claim the previous operation; retry the request.", + ), + ) + claimed = await _call_with_supported_optional_kwargs( + claim_unknown_operation, + optional_kwargs={"max_recovery_dispatches": 1} if one_shot_recovery else {}, + operation_id=operation.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + if not claimed: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 503, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery ownership changed; retry the request.", + ), + ) + request_state.operation_recovery_claimed = True + # The operation remains fenced to one durable identity. + # One-shot mode consumes its existing replay-count budget; + # indefinite mode may make further serialized attempts + # after cooldown because upstream has no idempotency or + # status endpoint. + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_registered = True + else: + # A prior dispatch with the same parent and body may have + # been accepted by upstream, or another request may still + # be using the same operation in this session. Without + # upstream idempotency/status proof, never submit it a + # second time. + _record_continuity_fail_closed( + surface="http_bridge", + reason="operation_already_recorded_no_status_proof", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code="upstream_operation_status_unknown", + ) + retry_after_seconds = max( + 1, + math.ceil(await self._http_bridge_precreated_retry_cooldown_seconds(session)), + ) + raise ProxyResponseError( + 503, + openai_error( + "upstream_operation_status_unknown", + "The previous response operation may still be running; retry after the cooldown.", + ), + retry_after_seconds=retry_after_seconds, + ) + request_state.operation_id = operation.operation_id + request_state.operation_fingerprint = operation_fingerprint + request_state.operation_parent_response_id = operation_parent_response_id + request_state.operation_registered = True + request_state.operation_rebind_required = False + request_state.operation_created = operation.created + + async def _cleanup_unsubmitted_recovery_claim() -> None: + if ( + not request_state.operation_recovery_claimed and not request_state.operation_created + ) or request_state.operation_dispatched: + return + await self._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + ) + text_data = self._http_bridge_text_with_account_installation_id(session, request_state, text_data) if request_state.response_id is not None or request_state.response_event_count > 0: + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_after_response_event", session.key, @@ -779,6 +1410,7 @@ async def _submit_http_bridge_request_with_handoff( ), ) if session.upstream_control.retire_after_drain: + await _cleanup_unsubmitted_recovery_claim() if not session.upstream_close_attempted: await self._retire_http_bridge_after_drain_if_ready(session) raise ProxyResponseError( @@ -798,6 +1430,7 @@ async def _submit_http_bridge_request_with_handoff( elif http_bridge_sessions is not None: current_session = http_bridge_sessions.get(session.key) if current_session is None and _http_bridge_key_strength(session.key) == "hard": + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_on_closed", session.key, @@ -829,16 +1462,21 @@ async def _submit_http_bridge_request_with_handoff( # receiving 400 previous_response_not_found (which causes the # CLI to drop previous_response_id and resend the full # conversation history, inflating per-turn context by ~20x). - recovered = await self._retry_http_bridge_request_on_fresh_upstream( - session, - request_state=request_state, - text_data=text_data, - send_request=False, - require_same_account=_http_bridge_key_strength(session.key) == "hard", - ) + try: + recovered = await self._retry_http_bridge_request_on_fresh_upstream( + session, + request_state=request_state, + text_data=text_data, + send_request=False, + require_same_account=_http_bridge_key_strength(session.key) == "hard", + ) + except BaseException: + await _cleanup_unsubmitted_recovery_claim() + raise if recovered: session.closed = False else: + await _cleanup_unsubmitted_recovery_claim() _log_http_bridge_event( "submit_on_closed", session.key, @@ -856,14 +1494,34 @@ async def _submit_http_bridge_request_with_handoff( gate_acquired = False request_enqueued = False admission_waiter_registered = False - async with session.pending_lock: - await self._ensure_http_bridge_session_stream_lease_locked(session, request_state=request_state) - # Register the submit as an admission waiter atomically with the - # reacquire so a previous turn's finalizer unwinding concurrently - # cannot see an apparently idle session and release this lease - # before the turn is counted into the session queue. - session.admission_waiter_count += 1 - admission_waiter_registered = True + try: + async with session.pending_lock: + await self._ensure_http_bridge_session_stream_lease_locked(session, request_state=request_state) + # Register the submit as an admission waiter atomically with the + # reacquire so a previous turn's finalizer unwinding concurrently + # cannot see an apparently idle session and release this lease + # before the turn is counted into the session queue. + session.admission_waiter_count += 1 + admission_waiter_registered = True + except BaseException: + # Recovery claims are made before admission. If reacquiring an + # idle session's stream lease fails, no upstream frame can have + # been sent; restore that claim before propagating the admission + # error so a later reconnect is not fenced as already dispatched. + if getattr(session, "unanchored_reservation_id", None) == request_scope_id: + session.unanchored_reservation_id = None + cleanup_task = asyncio.create_task( + self._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + admission_waiter_registered=admission_waiter_registered, + ) + ) + await _await_task_deferring_cancellation(cleanup_task) + raise try: await self._maybe_prewarm_http_bridge_session( session, @@ -1181,11 +1839,27 @@ async def _submit_http_bridge_request_with_handoff( session.admission_waiter_count = max(0, session.admission_waiter_count - 1) admission_waiter_registered = False request_enqueued = True - upstream_send_started = True + + def mark_upstream_send_started() -> None: + nonlocal upstream_send_started + # The helper invokes this only after the final frame + # size preflight. A payload_too_large rejection must + # therefore remain proven pre-dispatch so cleanup can + # roll back a newly-created operation. + upstream_send_started = True + try: - await _send_http_bridge_request_text_with_archive_id(session, request_state, text_data) + await _send_http_bridge_request_text_with_archive_id( + session, + request_state, + text_data, + on_send_started=mark_upstream_send_started, + ) except BaseException as exc: - request_state.recovery_attempt_dispatched = True + request_state.recovery_attempt_dispatched = upstream_send_started + request_state.operation_dispatched = ( + request_state.operation_id is not None and upstream_send_started + ) # Publish retirement while lifecycle ownership is still # held; a gate waiter must never reuse an ambiguously sent # response.create socket between unlock and cleanup. @@ -1203,6 +1877,7 @@ async def _submit_http_bridge_request_with_handoff( session.claim_liveness_settlement() raise request_state.recovery_attempt_dispatched = True + request_state.operation_dispatched = request_state.operation_id is not None session.last_used_at = _service_time().monotonic() except asyncio.CancelledError: if recovery_receipt is not None and not upstream_send_started: @@ -1281,6 +1956,7 @@ async def _submit_http_bridge_request_with_handoff( # handed to the kernel. Never reconnect-and-resend from this path; # only failures proven to precede dispatch may be replayed. error_code = exc.error_code if isinstance(exc, UpstreamWebSocketTransportError) else "stream_incomplete" + failure_error_message = str(exc) or "Upstream websocket closed before response.completed" # Liveness expiry and local network loss are transport failures, # not evidence against the selected account. Keep this in sync # with the reader path's shared provenance classification. @@ -1305,6 +1981,48 @@ async def _submit_http_bridge_request_with_handoff( if settlement_cancellation is not None: raise settlement_cancellation else: + # Once the operation-tagged frame has been handed to the + # socket, the transport exception is ambiguous: upstream may + # have accepted it even though this worker saw no + # acknowledgement. Persist UNKNOWN under the owner fence + # before cleanup can retire the closed session and release + # that fence. + if ( + request_state.operation_dispatched + and request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None) + marked_unknown = False + if callable(mark_operation_unknown): + try: + marked_unknown = await mark_operation_unknown( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + logger.warning( + "Failed to mark ambiguous HTTP bridge operation UNKNOWN operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if not marked_unknown: + request_state.operation_registered = False + error_code = "bridge_continuity_persistence_failed" + failure_error_message = ( + "Ambiguous response operation could not be persisted; retry the request." + ) + _record_continuity_fail_closed( + surface="http_bridge", + reason="ambiguous_operation_unknown_persistence_failed", + previous_response_id=request_state.previous_response_id, + session_id=request_state.session_id, + upstream_error_code=error_code, + ) await self._cleanup_http_bridge_submit_interruption( session, request_state=request_state, @@ -1319,7 +2037,7 @@ async def _submit_http_bridge_request_with_handoff( pending_requests=deque([request_state]), pending_lock=anyio.Lock(), error_code=error_code, - error_message=str(exc) or "Upstream websocket closed before response.completed", + error_message=failure_error_message, api_key=None, response_create_gate=session.response_create_gate, penalize_account=not account_neutral, @@ -1334,9 +2052,14 @@ async def _submit_http_bridge_request_with_handoff( # previous_response_not_found causes the client to drop # previous_response_id and resend the full conversation # history, inflating per-turn context by ~20x. + if _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) from exc raise ProxyResponseError( 502, - openai_error(error_code, str(exc) or "Upstream websocket closed"), + openai_error(error_code, failure_error_message), ) from exc async def _maybe_prewarm_http_bridge_session( @@ -1576,6 +2299,92 @@ async def _cleanup_http_bridge_submit_interruption( if admission_waiter_registered: session.admission_waiter_count = max(0, session.admission_waiter_count - 1) retire_closed_session = session.closed and session.admission_waiter_count == 0 + if ( + request_state.recovery_attempt_fingerprint is not None + and not request_state.recovery_attempt_claimed + and not request_state.recovery_attempt_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + rollback_recovery_attempt = getattr(self._durable_bridge, "rollback_recovery_attempt_before_dispatch", None) + if callable(rollback_recovery_attempt): + try: + await _call_with_supported_optional_kwargs( + rollback_recovery_attempt, + optional_kwargs={}, + session_id=session.durable_session_id, + api_key_id=session.key.api_key_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=request_state.recovery_attempt_fingerprint, + ) + except Exception: + logger.warning( + "Failed to roll back pre-dispatch HTTP bridge recovery checkpoint request_id=%s", + request_state.request_id, + exc_info=True, + ) + if ( + request_state.operation_recovery_claimed + and request_state.operation_registered + and request_state.operation_id is not None + and not request_state.operation_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + mark_operation_unknown = getattr(self._durable_bridge, "mark_operation_unknown", None) + restored = False + if callable(mark_operation_unknown): + try: + restored = await _call_with_supported_optional_kwargs( + mark_operation_unknown, + optional_kwargs={"restore_recovery_dispatch_claim": True}, + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + logger.warning( + "Failed to restore pre-dispatch HTTP bridge recovery operation UNKNOWN operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if restored: + request_state.operation_recovery_claimed = False + request_state.operation_id = None + request_state.operation_fingerprint = None + request_state.operation_parent_response_id = None + elif ( + request_state.operation_created + and request_state.operation_registered + and request_state.operation_id is not None + and not request_state.operation_dispatched + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + rollback_operation = getattr(self._durable_bridge, "rollback_operation_before_dispatch", None) + if callable(rollback_operation): + try: + rolled_back = await rollback_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + except Exception: + rolled_back = False + logger.warning( + "Failed to roll back pre-dispatch HTTP bridge operation operation_id=%s", + request_state.operation_id, + exc_info=True, + ) + if rolled_back: + request_state.operation_registered = False + request_state.operation_created = False + request_state.operation_id = None + request_state.operation_fingerprint = None + request_state.operation_parent_response_id = None self._cancel_request_state_api_key_reservation_heartbeat(request_state) if request_state.response_create_gate is not None: if gate_acquired or request_state.response_create_gate_acquired: @@ -1761,6 +2570,15 @@ async def _detach_http_bridge_request( request_state.event_queue = None await _release_websocket_response_create_gate(request_state, session.response_create_gate) if not detached: + if request_state.operation_replay: + # Replay requests are delivered from the durable transcript + # without entering pending ownership, so the normal detach + # branch cannot settle their API-key reservation. + self._cancel_request_state_api_key_reservation_heartbeat(request_state) + await self._release_websocket_request_state_reservation(request_state) + request_state.api_key_reservation = None + request_state.operation_replay = False + return False if request_state.terminal_settlement_phase == "abandoned": # Belt-and-braces for issue #1594: terminal bookkeeping # claimed this request out of pending ownership, aborted, and @@ -2027,6 +2845,7 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: fresh_hard_request_account_switch_candidate = False proof_gated_continuity_replay_candidate = False + server_anchored_replay_candidate = False if session.key.strength == "hard": async with session.pending_lock: retryable_candidates = [ @@ -2051,10 +2870,13 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: and candidate.response_event_count == 0 and candidate.replay_count == 0 ) + server_anchored_replay_candidate = _http_bridge_server_anchored_replay_enabled(candidate) if not await self._http_bridge_precreated_retry_allowed( session, allow_fresh_hard_account_switch=fresh_hard_request_account_switch_candidate, - allow_proof_gated_continuity_replay=proof_gated_continuity_replay_candidate, + allow_proof_gated_continuity_replay=( + proof_gated_continuity_replay_candidate or server_anchored_replay_candidate + ), ): return False @@ -2516,6 +3338,13 @@ async def _retry_http_bridge_security_work_request( model_class=_extract_model_class(session.request_model) if session.request_model else None, ) reconnected = False + operation_rebound_for_retry = False + security_retry_send_started = False + + def mark_security_retry_send_started() -> None: + nonlocal security_retry_send_started + security_retry_send_started = True + try: request_state.precreated_replay_account_id = session.account.id await self._release_request_state_account_response_create_lease(request_state) @@ -2550,14 +3379,83 @@ async def _retry_http_bridge_security_work_request( session.account.id, kind=previous_session_affinity.kind, ) + if ( + request_state.operation_registered + and request_state.operation_id is not None + and request_state.operation_fingerprint is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + record_operation = getattr(self._durable_bridge, "record_operation", None) + if not callable(record_operation): + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Security-work recovery operation could not be re-fenced; retry the request.", + ), + ) + rebound_operation = await record_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + request_fingerprint=request_state.operation_fingerprint, + api_key_scope=durable_bridge_api_key_scope(session.key.api_key_id), + account_id=session.account.id, + model=request_state.model, + parent_response_id=request_state.operation_parent_response_id or request_state.previous_response_id, + ) + if rebound_operation is None or getattr(rebound_operation, "state", None) != "submitted": + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "Security-work recovery operation could not be re-fenced; retry the request.", + ), + ) + operation_rebound_for_retry = True retry_text = self._http_bridge_text_with_account_installation_id(session, request_state, retry_text) - await _send_http_bridge_request_text_with_archive_id(session, request_state, retry_text) + await _send_http_bridge_request_text_with_archive_id( + session, + request_state, + retry_text, + on_send_started=mark_security_retry_send_started, + ) session.last_used_at = _service_time().monotonic() return True except UpstreamWebSocketTransportError: raise except Exception as exc: logger.warning("HTTP bridge security-work retry failed", exc_info=True) + if ( + operation_rebound_for_retry + and not security_retry_send_started + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + update_operation = getattr(self._durable_bridge, "update_operation", None) + if callable(update_operation): + try: + restored = await update_operation( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + state="failed", + ) + if not restored: + logger.info( + "HTTP bridge security retry failed to restore operation fence operation_id=%s", + request_state.operation_id, + ) + except Exception: + logger.warning( + "Failed to restore HTTP bridge security retry operation operation_id=%s", + request_state.operation_id, + exc_info=True, + ) if isinstance(exc, ProxyResponseError): error = _parse_openai_error(exc.payload) code = _normalize_error_code(error.code if error else None, error.type if error else None) diff --git a/app/modules/proxy/_service/http_bridge/retry_circuit.py b/app/modules/proxy/_service/http_bridge/retry_circuit.py index 51047cc168..891bf49dca 100644 --- a/app/modules/proxy/_service/http_bridge/retry_circuit.py +++ b/app/modules/proxy/_service/http_bridge/retry_circuit.py @@ -48,7 +48,9 @@ class _HTTPBridgeRetryCircuitState: half_open_until: float = 0.0 -def _initialize_http_bridge_retry_circuit(service: Any) -> None: +def _initialize_http_bridge_retry_circuit(service: Any, reset_transient_cache: Any = None) -> None: + if reset_transient_cache is not None: + reset_transient_cache() service._http_bridge_retry_circuits = {} service._http_bridge_retry_circuit_loaded_keys = set() service._http_bridge_retry_circuit_persisted_keys = set() @@ -259,6 +261,7 @@ async def _http_bridge_precreated_retry_allowed( *, allow_fresh_hard_account_switch: bool = False, allow_proof_gated_continuity_replay: bool = False, + allow_operation_fenced_continuity_replay: bool = False, ) -> bool: """Avoid replaying a repeatedly failing hard-affinity request in a tight loop.""" if session.key.strength != "hard": @@ -311,6 +314,16 @@ async def _http_bridge_precreated_retry_allowed( retry_after, ) return True + if allow_operation_fenced_continuity_replay: + logger.info( + "http_bridge_retry_circuit event=bypass_operation_fenced_continuity_replay bridge_kind=%s " + "bridge_key=%s failures=%s retry_after_seconds=%.1f", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + state.consecutive_failures, + retry_after, + ) + return True if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: http_bridge_retry_circuit_total.labels(outcome="suppressed").inc() logger.info( diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 98cb86a568..d7f2d8f599 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -36,6 +36,7 @@ from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401 from app.core.clients.proxy_websocket import UpstreamWebSocketTransportError from app.core.errors import ( + OpenAIErrorEnvelope, openai_error, response_failed_event, ) @@ -84,6 +85,7 @@ _http_bridge_request_budget_seconds, _http_bridge_request_needs_unanchored_handoff, _http_bridge_request_stage, + _http_bridge_requires_cluster_registration, _http_bridge_runtime_config, _http_bridge_should_attempt_local_bootstrap_rebind, _http_bridge_should_attempt_local_previous_response_recovery, @@ -244,6 +246,11 @@ def _http_bridge_continuity_bound_without_safe_replay(request_state: _WebSocketR ) +def _http_bridge_durable_recovery_predecessor_proven(request_state: _WebSocketRequestState) -> bool: + """Return whether the operation has a durable predecessor anchor.""" + return request_state.previous_response_id is not None or request_state.operation_parent_response_id is not None + + class _VerifiedDurableFullResend: """Immutable proof that one payload contains a durable turn's complete context.""" @@ -399,6 +406,52 @@ def _verify_durable_full_resend( return _VerifiedDurableFullResend._verify(payload, durable_lookup) +def _http_bridge_client_full_history_recovery_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether an ambiguous anchored turn may fall back to client replay. + + The client can recover an unknown upstream handoff by dropping + ``previous_response_id`` and resending its full local history. This is + intentionally opt-in: upstream acceptance is still ambiguous and the + fallback therefore has at-least-once (possible duplicate) semantics. + """ + settings = _service_get_settings() + return ( + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + == "client_full_history_once" + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and not request_state.fresh_upstream_request_is_retry_safe + ) + + +def _http_bridge_server_anchored_replay_enabled(request_state: _WebSocketRequestState) -> bool: + """Return whether the one permitted server-side anchored replay is unused.""" + settings = _service_get_settings() + return ( + getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "fail_closed") + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and request_state.previous_response_id is not None + and request_state.response_id is None + and request_state.response_event_count == 0 + and ( + request_state.replay_count == 0 + or getattr(settings, "http_responses_session_bridge_ambiguous_continuation_recovery_mode", "") + == "server_indefinite_recovery" + ) + ) + + +def _http_bridge_client_full_history_recovery_error() -> OpenAIErrorEnvelope: + payload = openai_error( + "previous_response_not_found", + "Previous response was not found; retry without previous_response_id.", + error_type="invalid_request_error", + ) + payload["error"]["param"] = "previous_response_id" + return payload + + _HTTP_BRIDGE_DEAD_OWNER_NOT_FOUND_DETAIL = "The previous bridge owner is no longer available." @@ -1274,6 +1327,7 @@ async def release_unowned_bridge_lifecycle( durable_recovery_attempt_claimed = False durable_recovery_attempt_session_id: str | None = None durable_recovery_attempt_owner_epoch: int | None = None + durable_recovery_fresh_replay = False durable_full_resend_proof = _verify_durable_full_resend(payload, durable_lookup) durable_full_resend_fresh_bridge_proof: _VerifiedDurableFullResend | None = None force_local_recovery_creation = False @@ -1683,11 +1737,32 @@ def classify_durable_full_resend( session_id=request_state.session_id, surface="http_bridge", ) - request_state.preferred_account_id = resolve_required_account_id( - ("durable bridge", request_state.preferred_account_id), - ("live bridge", local_previous_response_owner), - ("previous-response index", indexed_previous_response_owner), - ) + try: + request_state.preferred_account_id = resolve_required_account_id( + ("durable bridge", request_state.preferred_account_id), + ("live bridge", local_previous_response_owner), + ("previous-response index", indexed_previous_response_owner), + ) + except ProxyResponseError: + # The request-log owner cache is intentionally only a fast + # path. If it conflicts with a durable anchor, re-read the + # authoritative request-log row once before failing closed; + # this repairs stale in-process pins without adding a DB read + # to the normal continuation path. + if durable_lookup is None or indexed_previous_response_owner is None: + raise + indexed_previous_response_owner = await self._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=api_key, + session_id=request_state.session_id, + surface="http_bridge", + force_request_log_lookup=True, + ) + request_state.preferred_account_id = resolve_required_account_id( + ("durable bridge", request_state.preferred_account_id), + ("live bridge", local_previous_response_owner), + ("previous-response index", indexed_previous_response_owner), + ) durable_lookup_requires_owner = durable_lookup is not None and ( request_state.previous_response_id is not None or bridge_session_key.strength == "hard" @@ -1841,6 +1916,15 @@ def switch_to_account_neutral_replay() -> None: nonlocal text_data nonlocal untrimmed_effective_payload + preserve_operation_identity = durable_recovery_attempt_claimed or durable_recovery_fresh_replay + prior_operation_id = request_state.operation_id if preserve_operation_identity else None + prior_operation_fingerprint = request_state.operation_fingerprint if preserve_operation_identity else None + prior_operation_parent_response_id = ( + request_state.operation_parent_response_id or request_state.previous_response_id + if preserve_operation_identity + else None + ) + prior_operation_registered = request_state.operation_registered if preserve_operation_identity else False failed_owner_id = request_state.preferred_account_id _log_http_bridge_event( "owner_unavailable_fresh_resend", @@ -1865,6 +1949,12 @@ def switch_to_account_neutral_replay() -> None: if fresh_payload is None: raise RuntimeError("account-neutral replay projection missing after eligibility check") request_state, text_data = prepare_bridge_request(fresh_payload) + if preserve_operation_identity: + request_state.operation_id = prior_operation_id + request_state.operation_fingerprint = prior_operation_fingerprint + request_state.operation_parent_response_id = prior_operation_parent_response_id + request_state.operation_registered = prior_operation_registered + request_state.operation_rebind_required = True request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity request_state.excluded_account_ids.update(fresh_replay_excluded_account_ids) @@ -1956,6 +2046,16 @@ def switch_to_account_neutral_replay() -> None: allow_forward_to_owner=( not fresh_replay_excluded_account_ids and not force_local_recovery_creation ), + # A single-instance restart can leave an anchored durable + # row owned by the previous process epoch. Once the old + # owner is proven dead, let the initial continuation + # rebind locally; clustered deployments still route or + # fail closed through the normal owner path. + allow_previous_response_recovery_rebind=( + request_state.previous_response_id is not None + and dead_owner_anchor + and not _http_bridge_requires_cluster_registration(settings) + ), forwarded_request=forwarded_request, forwarded_original_request_unanchored=original_request_unanchored, forwarded_affinity_kind=forwarded_affinity_kind, @@ -2732,9 +2832,22 @@ def switch_to_account_neutral_replay() -> None: durable_recovery_fresh_replay = False retry_request_state: _WebSocketRequestState | None = None + async def release_recovery_origin_lease() -> None: + if durable_recovery_attempt_session_id is None or durable_recovery_attempt_owner_epoch is None: + return + try: + await self._durable_bridge.release_live_session( + session_id=durable_recovery_attempt_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=durable_recovery_attempt_owner_epoch, + draining=False, + ) + except Exception: + logger.warning("Failed to release HTTP bridge recovery origin lease", exc_info=True) + async def rollback_pre_dispatch_recovery_claim() -> None: if not ( - durable_recovery_fresh_replay + (durable_recovery_fresh_replay or durable_recovery_attempt_claimed) and (retry_request_state is None or not retry_request_state.recovery_attempt_dispatched) and durable_recovery_attempt_fingerprint is not None and durable_recovery_attempt_session_id is not None @@ -2742,13 +2855,15 @@ async def rollback_pre_dispatch_recovery_claim() -> None: ): return try: - await self._durable_bridge.rollback_recovery_attempt_replayed( + rolled_back = await self._durable_bridge.rollback_recovery_attempt_replayed( session_id=durable_recovery_attempt_session_id, api_key_id=bridge_session_key.api_key_id, instance_id=_service_get_settings().http_responses_session_bridge_instance_id, owner_epoch=durable_recovery_attempt_owner_epoch, request_fingerprint=durable_recovery_attempt_fingerprint, ) + if rolled_back: + await release_recovery_origin_lease() except Exception: logger.warning("Failed to roll back pre-dispatch HTTP bridge recovery claim", exc_info=True) @@ -2756,6 +2871,19 @@ async def rollback_pre_dispatch_recovery_claim() -> None: yield event_block yielded_any = True except ProxyResponseError as exc: + if ( + request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and _http_bridge_durable_recovery_predecessor_proven(request_state) + ): + # The API-level recovery loop must only run when this request + # has an actual durable operation fence. Settings alone are + # insufficient during a rolling migration where the durable + # tables may be unavailable and the bridge falls back to an + # in-memory session. + setattr(exc, "http_bridge_durable_recovery_eligible", True) if yielded_any: yield _partial_output_proxy_error_event_block( exc, @@ -3029,6 +3157,10 @@ async def rollback_pre_dispatch_recovery_claim() -> None: session, error_code="stream_incomplete", error_message="Upstream websocket closed before response.completed", + # Keep the origin lease fenced while the replacement + # session is admitted and the one-shot journal is + # either rolled back or settled. Releasing it here + # would make both transitions fail their owner fence. preserve_durable_lease=True, ) switch_to_account_neutral_replay() @@ -3157,68 +3289,72 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_preferred_account_id = request_state.preferred_account_id allow_previous_response_recovery_rebind = True - while True: - try: - session = await self._get_or_create_http_bridge_session( - bridge_session_key, - headers=dict(session_creation_headers), - affinity=affinity, - api_key=api_key, - request_model=retry_payload.model, - request_service_tier=request_state.requested_service_tier, - idle_ttl_seconds=_effective_http_bridge_idle_ttl_seconds( + try: + while True: + try: + session = await self._get_or_create_http_bridge_session( + bridge_session_key, + headers=dict(session_creation_headers), affinity=affinity, - idle_ttl_seconds=idle_ttl_seconds, - codex_idle_ttl_seconds=codex_idle_ttl_seconds, - prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, - ), - max_sessions=max_sessions, - previous_response_id=retry_previous_response_id, - gateway_safe_mode=runtime_config.gateway_safe_mode, - allow_forward_to_owner=False, - forwarded_request=False, - allow_previous_response_recovery_rebind=allow_previous_response_recovery_rebind, - session_header_fallback_key=session_header_fallback_key, - durable_lookup=durable_lookup, - request_stage=retry_request_stage, - preferred_account_id=retry_preferred_account_id, - preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, - fallback_on_preferred_account_unavailable=not ( - file_required_preferred_account and retry_preferred_account_id is not None - ), - request_usage_budget=estimate_api_key_request_usage(retry_payload), - request_deadline=request_deadline, - exclude_account_ids=request_state.excluded_account_ids or None, - deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, - defer_account_health_writes=request_state.api_key_reservation is not None, - ) - except ProxyResponseError as capacity_exc: - wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) - if wait_plan is None: - raise - bounded_wait_seconds, account_capacity_wait_seconds, message = wait_plan - logger.info( - "Waiting for an account to recover before retrying HTTP bridge local recovery session " - "request_id=%s model=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f path=%s error=%s", - request_id, - retry_payload.model, - bounded_wait_seconds, - account_capacity_wait_seconds, - recovery_path, - message, - ) - async for line in _iter_account_capacity_wait_sse( - request_id=request_id, - reason=message, - sleep_seconds=bounded_wait_seconds, - emit_keepalives=not propagate_http_errors, - request_state=request_state, - ): - yield line - if _service_time().monotonic() >= request_deadline: - raise - continue - break + api_key=api_key, + request_model=retry_payload.model, + request_service_tier=request_state.requested_service_tier, + idle_ttl_seconds=_effective_http_bridge_idle_ttl_seconds( + affinity=affinity, + idle_ttl_seconds=idle_ttl_seconds, + codex_idle_ttl_seconds=codex_idle_ttl_seconds, + prompt_cache_idle_ttl_seconds=prompt_cache_idle_ttl_seconds, + ), + max_sessions=max_sessions, + previous_response_id=retry_previous_response_id, + gateway_safe_mode=runtime_config.gateway_safe_mode, + allow_forward_to_owner=False, + forwarded_request=False, + allow_previous_response_recovery_rebind=allow_previous_response_recovery_rebind, + session_header_fallback_key=session_header_fallback_key, + durable_lookup=durable_lookup, + request_stage=retry_request_stage, + preferred_account_id=retry_preferred_account_id, + preferred_account_has_continuity_provenance=preferred_account_has_continuity_provenance, + fallback_on_preferred_account_unavailable=not ( + file_required_preferred_account and retry_preferred_account_id is not None + ), + request_usage_budget=estimate_api_key_request_usage(retry_payload), + request_deadline=request_deadline, + exclude_account_ids=request_state.excluded_account_ids or None, + deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, + defer_account_health_writes=request_state.api_key_reservation is not None, + ) + except ProxyResponseError as capacity_exc: + wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) + if wait_plan is None: + raise + bounded_wait_seconds, account_capacity_wait_seconds, message = wait_plan + logger.info( + "Waiting for an account to recover before retrying HTTP bridge local recovery session " + "request_id=%s model=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f path=%s error=%s", + request_id, + retry_payload.model, + bounded_wait_seconds, + account_capacity_wait_seconds, + recovery_path, + message, + ) + async for line in _iter_account_capacity_wait_sse( + request_id=request_id, + reason=message, + sleep_seconds=bounded_wait_seconds, + emit_keepalives=not propagate_http_errors, + request_state=request_state, + ): + yield line + if _service_time().monotonic() >= request_deadline: + raise + continue + break + except BaseException: + await rollback_pre_dispatch_recovery_claim() + raise _record_bridge_reattach(path=recovery_path, outcome="success") local_recovery_scope_id = ensure_request_scope_id() if original_request_unanchored else None @@ -3248,6 +3384,48 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_payload, reservation=retry_api_key_reservation, ) + if ( + recovery_path == "local_previous_response_error" + and request_state.operation_registered + and request_state.operation_id is not None + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + ): + reset_operation_event_spool = getattr(self._durable_bridge, "reset_operation_event_spool", None) + if callable(reset_operation_event_spool): + reset_ok = await reset_operation_event_spool( + operation_id=request_state.operation_id, + session_id=session.durable_session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=session.durable_owner_epoch, + ) + if not reset_ok: + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP response recovery spool could not be reset; retry the request.", + ), + ) + # A recovery request is the one bounded server-side replay; + # prevent a second cooldown bypass if this fresh socket also + # fails before response.created. + if recovery_path == "local_previous_response_error": + retry_request_state.replay_count = max(1, request_state.replay_count + 1) + # Keep the durable operation identity attached to the + # server-owned recovery attempt. Re-registering the same + # fingerprint would be interpreted as an already-dispatched + # unknown operation and suppress the intended one-shot replay. + retry_request_state.operation_id = request_state.operation_id + retry_request_state.operation_fingerprint = request_state.operation_fingerprint + retry_request_state.operation_parent_response_id = request_state.operation_parent_response_id + retry_request_state.operation_registered = request_state.operation_registered + retry_request_state.operation_rebind_required = request_state.operation_rebind_required + if recovery_path == "local_previous_response_error": + # The prior response.failed/error made the operation + # terminal. Re-enter record_operation so its owner fence + # atomically moves it back to submitted before send. + retry_request_state.operation_rebind_required = True retry_request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract if durable_recovery_fresh_replay and durable_recovery_attempt_fingerprint is not None: retry_request_state.recovery_attempt_fingerprint = durable_recovery_attempt_fingerprint @@ -3404,6 +3582,23 @@ async def retry_precreated_for_idle_recovery( request_state.request_id, exc.error_code, ) + if getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) == "server_indefinite_recovery" and _http_bridge_server_anchored_replay_enabled(request_state): + # Let the outer server-owned recovery loop classify this + # eventless transport failure as retryable. Returning a + # synthetic response.failed event would make the loop + # believe the attempt completed successfully after one try. + raise ProxyResponseError( + 502, + openai_error( + "stream_idle_timeout", + str(exc), + error_type="server_error", + ), + ) from exc return ( False, format_sse_event( @@ -3420,7 +3615,9 @@ async def retry_precreated_for_idle_recovery( def continuity_bound_without_safe_replay() -> bool: """Do not hold a client stream through a cooldown we cannot use.""" - return _http_bridge_continuity_bound_without_safe_replay(request_state) + return _http_bridge_continuity_bound_without_safe_replay(request_state) and not ( + _http_bridge_server_anchored_replay_enabled(request_state) + ) async def startup_continuity_cooldown_terminal_event() -> str | None: if ( @@ -3453,6 +3650,11 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: # non-streaming collector. await self._release_websocket_request_state_reservation(request_state) request_state.api_key_reservation = None + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) if propagate_http_errors: if request_state.durable_owner_dead: raise _http_bridge_dead_owner_previous_response_not_found_proxy_error( @@ -3565,6 +3767,12 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: raise if gate_contention and session.closed: raise + # Durable hard-turn admission may rewrite the request state + # with a completed predecessor before a pre-dispatch capacity + # failure. Retry the exact rewritten body rather than the + # stale outer-loop payload, or the next attempt could lose the + # injected previous_response_id and its continuity anchor. + text_data = request_state.request_text or text_data continue break event_queue = request_state.event_queue @@ -3612,6 +3820,11 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: # gate, reservation, and pending queue entry while marking the # upstream handoff for retirement. await self._detach_http_bridge_request(session, request_state=request_state) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled(request_state): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) if propagate_http_errors: if request_state.durable_owner_dead: raise _http_bridge_dead_owner_previous_response_not_found_proxy_error( @@ -3850,6 +4063,13 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: retry_cooldown_seconds, continuity_bound, ) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled( + request_state + ): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) yield format_sse_event( cast( Mapping[str, JsonValue], @@ -3893,6 +4113,13 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: retry_cooldown_seconds, retry_cooldown_remaining_budget, ) + if propagate_http_errors and _http_bridge_client_full_history_recovery_enabled( + request_state + ): + raise ProxyResponseError( + 400, + _http_bridge_client_full_history_recovery_error(), + ) yield format_sse_event( cast( Mapping[str, JsonValue], diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index a192c9d3d5..f240b09078 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -2,6 +2,7 @@ import asyncio import logging +import time from dataclasses import replace from typing import Any, TypeVar, cast @@ -210,6 +211,195 @@ ) _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS = 10.0 +# A single missing response.created is not proof that an account is bad: the +# upstream may have accepted the request while the transport was silent. Only +# repeated failures on separate bridge retirements are allowed to influence +# account routing, and the signal expires quickly so a transient upstream +# incident does not permanently drain an account. +_HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS = 300.0 +_HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD = 3 + + +async def _record_http_bridge_account_timeout_signal( + service: Any, + session: "_HTTPBridgeSession", +) -> None: + """Drain an account after repeated eventless upstream timeouts. + + This is deliberately separate from the per-session retry circuit. A + timeout cannot be replayed safely for a continuity-bound turn, but three + independent eventless failures are enough evidence to keep *new* turns + away from that account until its normal health probe succeeds. + """ + + account_id = session.account.id + now = time.monotonic() + async with service._http_bridge_account_timeout_lock: + failures = service._http_bridge_account_timeout_failures.setdefault(account_id, []) + failures[:] = [ + timestamp for timestamp in failures if now - timestamp < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS + ] + failures.append(now) + if len(failures) < _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD: + return + # Start a fresh evidence window after applying one health penalty. A + # continuously failing account should be re-evaluated by normal + # health-tier logic, not receive an unbounded error-count increase from + # every pending request on one broken socket. + failures.clear() + + try: + # Health-tier draining starts at two transient errors. Apply exactly + # that minimum penalty so one threshold event actually removes the + # account from normal routing without over-counting the incident. + await service._load_balancer.record_errors(session.account, 2) + except Exception: + logger.warning( + "Failed to record repeated HTTP bridge account timeout account_id=%s", + account_id, + exc_info=True, + ) + else: + logger.warning( + "HTTP bridge account temporarily drained after repeated eventless upstream timeouts " + "account_id=%s threshold=%s window_seconds=%.0f", + account_id, + _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD, + _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS, + ) + + +async def _update_http_bridge_operation_state( + service: Any, + session: "_HTTPBridgeSession", + request_state: Any, + *, + state: str, + response_id: str | None = None, +) -> None: + """Persist operation outcome without allowing journaling to break streaming.""" + operation_id = getattr(request_state, "operation_id", None) + session_id = getattr(session, "durable_session_id", None) + owner_epoch = getattr(session, "durable_owner_epoch", None) + update_operation = getattr(getattr(service, "_durable_bridge", None), "update_operation", None) + if not operation_id or session_id is None or owner_epoch is None or not callable(update_operation): + return + try: + marked = await update_operation( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + state=state, + response_id=response_id, + ) + if not marked: + logger.info( + "HTTP bridge operation outcome owner fence rejected operation_id=%s state=%s", + operation_id, + state, + ) + except Exception: + logger.warning( + "Failed to persist HTTP bridge operation outcome operation_id=%s state=%s", + operation_id, + state, + exc_info=True, + ) + + +def _http_bridge_operation_state_for_event(event_type: str | None) -> str | None: + return { + "response.created": "acknowledged", + "response.completed": "completed", + "response.incomplete": "incomplete", + "response.failed": "failed", + "error": "failed", + }.get(event_type) + + +async def _persist_http_bridge_operation_event( + service: Any, + session: "_HTTPBridgeSession", + request_state: Any, + event_block: str, + *, + terminal: bool = False, + terminal_state: str | None = None, +) -> None: + """Spool one downstream-visible SSE block for reconnect replay.""" + operation_id = getattr(request_state, "operation_id", None) + session_id = getattr(session, "durable_session_id", None) + owner_epoch = getattr(session, "durable_owner_epoch", None) + batcher_enqueue = getattr(getattr(service, "_http_bridge_operation_event_batcher", None), "enqueue", None) + append_event = getattr(getattr(service, "_durable_bridge", None), "append_operation_event", None) + if not operation_id or session_id is None or owner_epoch is None: + return + try: + batcher = getattr(service, "_http_bridge_operation_event_batcher", None) + append_terminal_batch = getattr(batcher, "append_terminal_event", None) + if terminal and terminal_state is not None and callable(append_terminal_batch): + persisted = await append_terminal_batch( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + max_bytes=int( + getattr( + _service_get_settings(), + "http_responses_session_bridge_operation_event_spool_max_bytes", + 2 * 1024 * 1024, + ) + ), + state=terminal_state, + response_id=_websocket_downstream_response_id(request_state), + ) + if not persisted: + logger.info("HTTP bridge terminal event spool became incomplete operation_id=%s", operation_id) + return + if callable(batcher_enqueue): + await batcher_enqueue( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + terminal=terminal, + ) + return + if not callable(append_event): + return + persisted = await append_event( + operation_id=operation_id, + session_id=session_id, + instance_id=_service_get_settings().http_responses_session_bridge_instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + max_bytes=int( + getattr( + _service_get_settings(), + "http_responses_session_bridge_operation_event_spool_max_bytes", + 2 * 1024 * 1024, + ) + ), + ) + if not persisted: + logger.info("HTTP bridge operation event spool became incomplete operation_id=%s", operation_id) + if terminal and terminal_state is not None: + await _update_http_bridge_operation_state( + service, + session, + request_state, + state=terminal_state, + response_id=_websocket_downstream_response_id(request_state), + ) + except Exception: + # The upstream result is still delivered. A reconnect can only replay + # when every event was durably persisted, so never fail a live stream + # because the optional spool is unavailable. + logger.warning("Failed to persist HTTP bridge operation event operation_id=%s", operation_id, exc_info=True) + async def _wait_for_http_bridge_recovery_settlement_retry( service: Any, @@ -770,6 +960,38 @@ async def _fail_http_bridge_reader_and_maybe_retire( cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) + # Draining-only requests no longer count against the queue, but their + # event-batcher contexts still belong to the disconnected operation + # and must be discarded just like ordinary pending requests. + operation_states: list[Any] = [ + request_state for request_state in pending_request_states if getattr(request_state, "operation_id", None) + ] + # Remove the disconnected attempt's in-memory spool before publishing + # UNKNOWN/ACKNOWLEDGED state. A same-replica reconnect may reclaim the + # operation as soon as that state is visible; discarding afterward + # could then delete the replacement attempt's events. + discard_operation = getattr( + getattr(self, "_http_bridge_operation_event_batcher", None), + "discard_operation", + None, + ) + if callable(discard_operation): + for request_state in operation_states: + operation_id = getattr(request_state, "operation_id", None) + if operation_id: + await discard_operation(operation_id=operation_id) + for request_state in operation_states: + # A shared websocket can carry several logical response.create + # requests. Classify each operation from its own event count; + # using the session-wide maximum would mark an eventless + # sibling as safely retryable after another request streamed. + operation_state = "unknown" if getattr(request_state, "response_event_count", 0) == 0 else "acknowledged" + await _update_http_bridge_operation_state( + self, + session, + request_state, + state=operation_state, + ) if force_retire and retire_detail: _log_http_bridge_event( retire_detail, @@ -782,7 +1004,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( model_class=_extract_model_class(session.request_model) if session.request_model else None, ) try: - await self._fail_pending_websocket_requests( + reservations_settled = await self._fail_pending_websocket_requests( account=session.account, account_id_value=session.account.id, pending_requests=session.pending_requests, @@ -793,6 +1015,16 @@ async def _fail_http_bridge_reader_and_maybe_retire( response_create_gate=session.response_create_gate, penalize_account=penalize_account, ) + if ( + failed_pending_count > 0 + and reservations_settled is not False + and observed_response_events == 0 + and retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + ): + # Only penalize the account after pending-request cleanup has + # settled its API-key reservations. A failed release must not + # be hidden behind an already-recorded timeout health signal. + await _record_http_bridge_account_timeout_signal(self, session) finally: poison_after_deferred_failures = False if session.admission_waiter_count > 0 and not force_retire: @@ -1580,9 +1812,26 @@ async def _process_parsed_http_bridge_upstream_event( grouped_request_state, reason=grouped_error_reason, ) + grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type) + await _persist_http_bridge_operation_event( + self, + session, + grouped_request_state, + grouped_event_block, + terminal=True, + terminal_state=grouped_operation_state, + ) if grouped_request_state.event_queue is not None: await grouped_request_state.event_queue.put(grouped_event_block) await grouped_request_state.event_queue.put(None) + if grouped_operation_state is not None and grouped_operation_state != "failed": + await _update_http_bridge_operation_state( + self, + session, + grouped_request_state, + state=grouped_operation_state, + response_id=_websocket_downstream_response_id(grouped_request_state), + ) await self._finalize_websocket_request_state( grouped_request_state, account=session.account, @@ -1645,6 +1894,7 @@ async def _process_parsed_http_bridge_upstream_event( surface="http_bridge", ) + continuity_persistence_failed_after_ack = False if ( event_type == "response.completed" and terminal_request_state is not None @@ -2039,9 +2289,37 @@ async def _process_parsed_http_bridge_upstream_event( event_block = format_sse_event(payload) event = parse_sse_event_payload(payload) event_type = "response.failed" + # The upstream response was already acknowledged. The local + # alias write failed, so expose a terminal error downstream + # but keep the durable operation acknowledged/ambiguous to + # prevent an identical retry from dispatching it again. + continuity_persistence_failed_after_ack = True completed_usage = None completed_empty_prewarm = False + operation_state = _http_bridge_operation_state_for_event(event_type) + if operation_state is not None: + operation_request_states: list[Any] = [] + for candidate in (matched_request_state, terminal_request_state): + if candidate is not None and candidate not in operation_request_states: + operation_request_states.append(candidate) + for operation_request_state in operation_request_states: + request_operation_state = operation_state + if continuity_persistence_failed_after_ack and operation_request_state is matched_request_state: + request_operation_state = "acknowledged" + if request_operation_state == "failed": + # Failure rows are exposed only by the terminal-event + # persistence path below, which appends the terminal SSE + # block and flips the operation state atomically. + continue + await _update_http_bridge_operation_state( + self, + session, + operation_request_state, + state=request_operation_state, + response_id=response_id, + ) + recovery_attempt_session_id = ( matched_request_state.recovery_attempt_session_id if matched_request_state is not None and matched_request_state.recovery_attempt_session_id is not None @@ -2055,12 +2333,15 @@ async def _process_parsed_http_bridge_upstream_event( if ( isinstance(event_type, str) - and event_type.startswith("response.") + and (event_type.startswith("response.") or event_type == "error") and matched_request_state is not None and matched_request_state.recovery_attempt_fingerprint is not None and recovery_attempt_session_id is not None and recovery_attempt_owner_epoch is not None - and (event_type == "response.completed" or not matched_request_state.recovery_attempt_event_observed) + and ( + event_type in {"response.completed", "response.failed", "response.incomplete", "error"} + or not matched_request_state.recovery_attempt_event_observed + ) ): settlement_marked = False for settlement_attempt in range(3): @@ -2088,7 +2369,8 @@ async def _process_parsed_http_bridge_upstream_event( response_id=response_id, release_origin_lease=( recovery_attempt_session_id != session.durable_session_id - and event_type in {"response.completed", "response.failed"} + and event_type + in {"response.completed", "response.failed", "response.incomplete", "error"} ), ) except Exception: @@ -2105,14 +2387,15 @@ async def _process_parsed_http_bridge_upstream_event( response_id=response_id, release_origin_lease=( recovery_attempt_session_id != session.durable_session_id - and event_type in {"response.completed", "response.failed"} + and event_type + in {"response.completed", "response.failed", "response.incomplete", "error"} ), ) else: await asyncio.sleep(0.05 * (settlement_attempt + 1)) if ( settlement_marked - and event_type in {"response.completed", "response.failed"} + and event_type in {"response.completed", "response.failed", "response.incomplete", "error"} and recovery_attempt_session_id != session.durable_session_id ): try: @@ -2324,8 +2607,38 @@ async def _process_parsed_http_bridge_upstream_event( if matched_request_state is not None else None ) + matched_deferred_texts = ( + _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state) + if matched_request_state is not None and not suppress_downstream_event + else [] + ) + matched_terminal_state = _http_bridge_operation_state_for_event(event_type) + if continuity_persistence_failed_after_ack and matched_request_state is not None: + # The upstream response was already accepted. The downstream + # failure only reports that its durable alias could not be + # persisted, so keep the operation fenced as acknowledged while + # retaining the failure SSE for the client. + matched_terminal_state = "acknowledged" + if matched_request_state is not None and not suppress_downstream_event: + for deferred_text in matched_deferred_texts: + await _persist_http_bridge_operation_event( + self, + session, + matched_request_state, + deferred_text, + terminal=False, + ) + if matched_request_state is not None and not suppress_downstream_event: + await _persist_http_bridge_operation_event( + self, + session, + matched_request_state, + event_block, + terminal=event_type in {"response.completed", "response.failed", "response.incomplete", "error"}, + terminal_state=matched_terminal_state, + ) if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event: - for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(matched_request_state): + for deferred_text in matched_deferred_texts: await matched_event_queue.put(deferred_text) await matched_event_queue.put(event_block) @@ -2335,10 +2648,34 @@ async def _process_parsed_http_bridge_upstream_event( terminal_event_queue = ( completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue ) - if terminal_request_state is not matched_request_state and terminal_event_queue is not None: - for deferred_text in _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state): - await terminal_event_queue.put(deferred_text) - await terminal_event_queue.put(event_block) + if terminal_request_state is not matched_request_state: + deferred_texts = _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state) + for deferred_text in deferred_texts: + if not suppress_downstream_event: + await _persist_http_bridge_operation_event( + self, + session, + terminal_request_state, + deferred_text, + terminal=False, + ) + if terminal_event_queue is not None: + await terminal_event_queue.put(deferred_text) + if not suppress_downstream_event: + await _persist_http_bridge_operation_event( + self, + session, + terminal_request_state, + event_block, + terminal=True, + terminal_state=( + "acknowledged" + if continuity_persistence_failed_after_ack and terminal_request_state is matched_request_state + else _http_bridge_operation_state_for_event(event_type) + ), + ) + if terminal_event_queue is not None: + await terminal_event_queue.put(event_block) if terminal_event_queue is not None: await terminal_event_queue.put(None) if completed_event_queue_claimed and completed_delivery_scope is not None: diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index cfcb250dc4..c92e2681cb 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -881,6 +881,25 @@ class _WebSocketRequestState: # claimed recovery journal; an attempted send must remain consumed. recovery_attempt_dispatched: bool = False recovery_attempt_event_observed: bool = False + # Durable operation identity for a continuity-bound response.create. It is + # stable across client reconnects with the same parent response and body. + operation_id: str | None = None + operation_fingerprint: str | None = None + operation_parent_response_id: str | None = None + operation_registered: bool = False + # Account-neutral durable recovery keeps the original operation identity + # while asking request submission to rebind it to the replacement session. + operation_rebind_required: bool = False + # True after an existing UNKNOWN operation is claimed for this attempt. + # If admission fails before send, cleanup must restore UNKNOWN rather than + # treating the pre-existing row like a newly-created operation. + operation_recovery_claimed: bool = False + # True only when this request created the durable operation row. A + # pre-dispatch admission failure may remove that row; an existing row + # represents an ambiguous upstream attempt and must remain fenced. + operation_created: bool = False + operation_replay: bool = False + operation_dispatched: bool = False # Responses-Lite model advertised by ``fresh_upstream_request_text``. A # fresh replay built from a trusted marker-only frame has the reserved # marker stripped, so swapping to the fresh body must also swap this onto diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 9bb2bc99d9..3a0ee3d4c7 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -345,6 +345,87 @@ def _facade() -> Any: return sys.modules["app.modules.proxy.service"] +# A confirmed stale previous-response anchor can otherwise cause every client +# reconnect to repeat the same owner lookup and doomed upstream connection. +# Keep this local and short-lived: an owner record may still be committed by a +# concurrent request, so discovery always invalidates the negative entry. +_WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS = 60.0 +_WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_LIMIT = 4096 +_websocket_stale_previous_response_index: dict[tuple[str, str | None], float] = {} + + +def _clear_websocket_stale_previous_response_cache() -> None: + """Drop process-local negative entries when a proxy service is created. + + The cache intentionally is not durable: it only suppresses repeated + lookups during a short recovery window. Clearing it with the service + lifecycle prevents entries from one app/test instance from affecting a + later instance that happens to receive the same synthetic response id. + """ + _websocket_stale_previous_response_index.clear() + + +def _prune_websocket_stale_previous_response_cache(now: float | None = None) -> None: + current_time = time.monotonic() if now is None else now + for cache_key, expires_at in tuple(_websocket_stale_previous_response_index.items()): + if expires_at <= current_time: + _websocket_stale_previous_response_index.pop(cache_key, None) + while len(_websocket_stale_previous_response_index) > _WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_LIMIT: + _websocket_stale_previous_response_index.pop(next(iter(_websocket_stale_previous_response_index))) + + +def _remember_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> None: + if previous_response_id is None: + return + response_id = previous_response_id.strip() + if not response_id: + return + now = time.monotonic() + _prune_websocket_stale_previous_response_cache(now) + cache_key = (response_id, api_key_id) + _websocket_stale_previous_response_index.pop(cache_key, None) + _websocket_stale_previous_response_index[cache_key] = now + _WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS + _prune_websocket_stale_previous_response_cache(now) + + +def _forget_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> None: + if previous_response_id is None: + return + response_id = previous_response_id.strip() + if not response_id: + return + _websocket_stale_previous_response_index.pop((response_id, api_key_id), None) + + +def _is_websocket_stale_previous_response( + *, + previous_response_id: str | None, + api_key_id: str | None, +) -> bool: + if previous_response_id is None: + return False + response_id = previous_response_id.strip() + if not response_id: + return False + now = time.monotonic() + _prune_websocket_stale_previous_response_cache(now) + expires_at = _websocket_stale_previous_response_index.get((response_id, api_key_id)) + if expires_at is None: + return False + if expires_at <= now: + _websocket_stale_previous_response_index.pop((response_id, api_key_id), None) + return False + return True + + def _prepare_websocket_request_state_for_visible_output_replay( request_state: "_WebSocketRequestState", ) -> str | None: @@ -1154,6 +1235,11 @@ def _record_websocket_stale_anchor_failure( request_state.failure_phase_override = "upstream" request_state.failure_detail_override = _websocket_stale_anchor_failure_detail(diagnostics) request_state.upstream_error_code_override = upstream_error_code + if not diagnostics.fresh_replay_available: + _remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=request_state.api_key.id if request_state.api_key is not None else None, + ) _record_continuity_fail_closed( surface=surface, reason="previous_response_not_found", diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 371654853b..3b7a1b1e2d 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -377,7 +377,9 @@ _app_error_to_websocket_event, _assign_websocket_response_id, _find_websocket_request_state_by_response_id, + _forget_websocket_stale_previous_response, _is_websocket_response_create, + _is_websocket_stale_previous_response, _match_websocket_request_state_for_anonymous_event, _matching_websocket_request_states_for_missing_tool_output_error, _matching_websocket_request_states_for_previous_response_error, @@ -785,6 +787,59 @@ def _discard_owned_task(_done_task: asyncio.Task[Any]) -> None: task.add_done_callback(_discard_owned_task) +_WEBSOCKET_UPSTREAM_CLOSE_CLEANUP_TIMEOUT_SECONDS = 0.25 + + +async def _close_websocket_upstream_for_cleanup( + proxy: _WebSocketServiceProtocol, + upstream: UpstreamWebSocket, + *, + timeout_seconds: float, +) -> None: + """Close an upstream socket without letting a stuck close block cleanup. + + Some websocket implementations can wait for a close handshake after the + peer has already disappeared. The close operation remains tracked so it + can finish asynchronously, while scope finalization continues releasing + request ownership and leases within its bounded cleanup budget. + """ + + close_task = asyncio.create_task( + upstream.close(), + name="proxy-websocket-upstream-close", + ) + _track_websocket_owned_task(proxy, close_task) + effective_timeout = min( + max(float(timeout_seconds), 0.0), + _WEBSOCKET_UPSTREAM_CLOSE_CLEANUP_TIMEOUT_SECONDS, + ) + + async def cancel_close_task() -> None: + try: + await _facade()._await_cancelled_task( + close_task, + timeout_seconds=effective_timeout, + label="proxy websocket upstream close", + cleanup_tasks=proxy._background_cleanup_tasks, + ) + except Exception: + _facade().logger.debug("Failed to cancel upstream websocket close task", exc_info=True) + + if effective_timeout <= 0: + await cancel_close_task() + return + try: + await asyncio.wait_for(asyncio.shield(close_task), timeout=effective_timeout) + except TimeoutError: + _facade().logger.debug( + "Upstream websocket close continued after cleanup budget timeout_seconds=%.3f", + effective_timeout, + ) + await cancel_close_task() + except Exception: + _facade().logger.debug("Failed to close upstream websocket during scope cleanup", exc_info=True) + + async def _await_owned_websocket_task_after_reader_cancellation( task: asyncio.Task[Any], *, @@ -1457,6 +1512,7 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if not await proxy._downstream_websocket_is_idle( pending_requests, pending_lock=pending_lock, + upstream_control=upstream_control, downstream_activity=downstream_activity, idle_timeout_seconds=downstream_idle_timeout_seconds, ): @@ -1466,6 +1522,7 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if await proxy._downstream_websocket_is_idle( pending_requests, pending_lock=pending_lock, + upstream_control=upstream_control, downstream_activity=downstream_activity, idle_timeout_seconds=downstream_idle_timeout_seconds, ): @@ -2470,16 +2527,18 @@ async def finalize_websocket_scope() -> None: # release that wait. reader_to_await.cancel() if upstream is not None: - try: - await upstream.close() - except Exception: - _facade().logger.debug("Failed to close upstream websocket", exc_info=True) + await _close_websocket_upstream_for_cleanup( + proxy, + upstream, + timeout_seconds=cleanup_timeout, + ) if reader_to_await is not None: try: await _facade()._await_cancelled_task( reader_to_await, label="proxy websocket upstream reader", cancel=False, + cleanup_tasks=proxy._background_cleanup_tasks, ) except Exception: # Reader failure must not skip lease release or the @@ -2609,7 +2668,12 @@ def log_scope_cleanup_failure(done_task: asyncio.Task[None]) -> None: timeout=max(float(cleanup_timeout), 0.0), ) if not done: - _facade().logger.warning("Websocket scope cleanup exceeded its remaining drain budget") + _facade().logger.warning( + "Websocket scope cleanup exceeded its remaining drain budget " + "timeout_seconds=%.3f background_cleanup_tasks=%d", + max(float(cleanup_timeout), 0.0), + sum(1 for task in proxy._background_cleanup_tasks if not task.done()), + ) async def _prepare_websocket_response_create_request( self, @@ -4097,6 +4161,10 @@ def _remember_websocket_previous_response_owner( account_id_value = account_id.strip() if not account_id_value: return + _forget_websocket_stale_previous_response( + previous_response_id=response_id, + api_key_id=api_key_id, + ) cache_keys = [(response_id, api_key_id, None)] normalized_session_id = _facade()._normalize_session_id(session_id) if normalized_session_id is not None: @@ -4133,6 +4201,7 @@ async def _resolve_websocket_previous_response_owner( session_id: str | None = None, surface: str, request_state: _WebSocketRequestState | None = None, + force_request_log_lookup: bool = False, ) -> str | None: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy @@ -4151,6 +4220,24 @@ def _record_lookup_metadata( request_state.previous_response_owner_requested_at = requested_at request_state.previous_response_owner_session_id = owner_session_id + def _raise_stale_response_cache_suppression(*, outcome: str) -> NoReturn: + _record_lookup_metadata(source="stale_response_cache", outcome=outcome) + _record_continuity_owner_resolution( + surface=surface, + source="stale_response_cache", + outcome=outcome, + previous_response_id=response_id, + session_id=session_id_value, + ) + raise ProxyResponseError( + 502, + openai_error( + "stream_incomplete", + "Previous response is temporarily unavailable; retrying is suppressed for the recovery window.", + error_type="server_error", + ), + ) + if previous_response_id is None: return None response_id = previous_response_id.strip() @@ -4158,8 +4245,19 @@ def _record_lookup_metadata( return None api_key_id = api_key.id if api_key is not None else None session_id_value = _facade()._normalize_session_id(session_id) + stale_cache_hit = ( + request_state is not None + and not force_request_log_lookup + and not request_state.fresh_upstream_request_is_retry_safe + and _is_websocket_stale_previous_response( + previous_response_id=response_id, + api_key_id=api_key_id, + ) + ) cache_key = (response_id, api_key_id, session_id_value) - cached_account_id = proxy._websocket_previous_response_account_index.get(cache_key) + cached_account_id = ( + None if force_request_log_lookup else proxy._websocket_previous_response_account_index.get(cache_key) + ) if cached_account_id is not None: _record_lookup_metadata(source="request_cache", outcome="hit") _record_continuity_owner_resolution( @@ -4171,9 +4269,13 @@ def _record_lookup_metadata( ) return cached_account_id fallback_account_id = ( - proxy._websocket_previous_response_account_index.get((response_id, api_key_id, None)) - if session_id_value is not None - else None + None + if force_request_log_lookup + else ( + proxy._websocket_previous_response_account_index.get((response_id, api_key_id, None)) + if session_id_value is not None + else None + ) ) try: async with proxy._repo_factory() as repos: @@ -4183,6 +4285,8 @@ def _record_lookup_metadata( session_id=session_id_value, ) except Exception as exc: + if stale_cache_hit: + _raise_stale_response_cache_suppression(outcome="lookup_failed") if fallback_account_id is not None: _record_lookup_metadata(source="request_cache_fallback", outcome="hit") _record_continuity_owner_resolution( @@ -4217,6 +4321,12 @@ def _record_lookup_metadata( _facade()._previous_response_owner_lookup_failed_error_envelope(), ) from exc if owner_record is None: + if stale_cache_hit: + _raise_stale_response_cache_suppression(outcome="hit") + if force_request_log_lookup: + proxy._websocket_previous_response_account_index.pop(cache_key, None) + if session_id_value is not None: + proxy._websocket_previous_response_account_index.pop((response_id, api_key_id, None), None) if fallback_account_id is not None: _record_lookup_metadata(source="request_cache_fallback", outcome="hit") _record_continuity_owner_resolution( @@ -5329,11 +5439,16 @@ async def _downstream_websocket_is_idle( pending_requests: deque[_WebSocketRequestState], *, pending_lock: anyio.Lock, + upstream_control: _WebSocketUpstreamControl | None = None, downstream_activity: _DownstreamWebSocketActivity, idle_timeout_seconds: float, ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy + if upstream_control is not None: + terminal_task = upstream_control.terminal_message_task + if terminal_task is not None and not terminal_task.done(): + return False async with pending_lock: if pending_requests: return False @@ -5830,10 +5945,10 @@ async def _fail_pending_websocket_requests( status: str = "error", penalize_account: bool = True, suppress_sequenced_downstream_errors: bool = False, - ) -> None: + ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - finalization_task: asyncio.Task[None] | None = None + finalization_task: asyncio.Task[bool] | None = None await pending_lock.acquire() try: remaining = list(pending_requests) @@ -5865,10 +5980,10 @@ async def _fail_pending_websocket_requests( pending_lock.release() if finalization_task is None: - return + return True try: - await asyncio.shield(finalization_task) + settlement_succeeded = await asyncio.shield(finalization_task) except asyncio.CancelledError: remaining_timeout = shutdown_state.remaining_drain_timeout_seconds() timeout_seconds = ( @@ -5881,6 +5996,7 @@ async def _fail_pending_websocket_requests( # the claimed states and remains visible to lifespan draining. await asyncio.wait({finalization_task}, timeout=timeout_seconds) raise + return settlement_succeeded async def _finalize_claimed_websocket_requests( self, @@ -5898,7 +6014,7 @@ async def _finalize_claimed_websocket_requests( status: str, penalize_account: bool, suppress_sequenced_downstream_errors: bool, - ) -> None: + ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy @@ -5967,6 +6083,7 @@ async def _finalize_claimed_websocket_requests( request_state.request_log_id or request_state.request_id, exc_info=True, ) + if response_create_gate is not None: await _release_websocket_response_create_ownership_for_cleanup( request_state, @@ -6133,6 +6250,8 @@ async def _finalize_claimed_websocket_requests( exc_info=True, ) + return reservation_release_succeeded + async def _emit_websocket_terminal_error( self, websocket: WebSocket, diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index c94197eb6b..70e44a1ef7 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -4969,6 +4969,11 @@ async def _stream_responses( ) bridge_active = prefer_http_bridge and proxy_service_module.get_settings().http_responses_session_bridge_enabled effective_headers = forwarded_headers or request.headers + bridge_recovery_eligible = _http_bridge_recovery_request_eligible( + payload, + bridge_active=bridge_active, + headers=effective_headers, + ) client_ip = forwarded_client_ip if forwarded_request else resolve_request_client_host(request) downstream_turn_state = ( forwarded_downstream_turn_state @@ -5044,30 +5049,31 @@ async def _stream_responses( capacity_wait_event = asyncio.Event() capacity_ready_event = _CapacityStartupReadyEvent() payload.stream = True - if prefer_http_bridge: - stream = context.service.stream_http_responses( - payload, - effective_headers, - codex_session_affinity=codex_session_affinity, - propagate_http_errors=True, - openai_cache_affinity=openai_cache_affinity, - api_key=api_key, - api_key_reservation=reservation, - suppress_text_done_events=suppress_text_done_events, - downstream_turn_state=downstream_turn_state, - forwarded_request=forwarded_request, - forwarded_original_request_unanchored=forwarded_original_request_unanchored, - forwarded_legacy_signature=forwarded_legacy_signature, - forwarded_affinity_kind=forwarded_affinity_kind, - forwarded_affinity_key=forwarded_affinity_key, - forwarded_file_owner_account_id=forwarded_file_owner_account_id, - client_ip=client_ip, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - capacity_startup_wait_event=capacity_wait_event, - capacity_startup_ready_event=capacity_ready_event, - ) - else: - stream = context.service.stream_responses( + + def build_response_stream() -> AsyncIterator[str]: + if prefer_http_bridge: + return context.service.stream_http_responses( + payload, + effective_headers, + codex_session_affinity=codex_session_affinity, + propagate_http_errors=True, + openai_cache_affinity=openai_cache_affinity, + api_key=api_key, + api_key_reservation=reservation, + suppress_text_done_events=suppress_text_done_events, + downstream_turn_state=downstream_turn_state, + forwarded_request=forwarded_request, + forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_legacy_signature=forwarded_legacy_signature, + forwarded_affinity_kind=forwarded_affinity_kind, + forwarded_affinity_key=forwarded_affinity_key, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + client_ip=client_ip, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + capacity_startup_wait_event=capacity_wait_event, + capacity_startup_ready_event=capacity_ready_event, + ) + return context.service.stream_responses( payload, request.headers, codex_session_affinity=codex_session_affinity, @@ -5079,6 +5085,56 @@ async def _stream_responses( client_ip=client_ip, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) + + def build_recovery_response_stream() -> AsyncIterator[str]: + """Build a server-owned retry with a fresh API-key reservation. + + The first bridge generator owns and settles the admission reservation + when it terminates. Indefinite recovery must not reuse that object: + each retry gets a new reservation and therefore remains accounted and + bounded even when the client connection stays open for a long time. + """ + + async def _retry() -> AsyncIterator[str]: + retry_reservation = reservation + if prefer_http_bridge and api_key is not None and reservation is not None: + retry_reservation = await _enforce_request_limits( + api_key, + request_model=payload.model, + request_service_tier=( + dict(payload.to_payload()).get("service_tier") + if isinstance(dict(payload.to_payload()).get("service_tier"), str) + else None + ), + request_usage_budget=estimate_api_key_request_usage(payload), + ) + retry_stream = context.service.stream_http_responses( + payload, + effective_headers, + codex_session_affinity=codex_session_affinity, + propagate_http_errors=True, + openai_cache_affinity=openai_cache_affinity, + api_key=api_key, + api_key_reservation=retry_reservation, + suppress_text_done_events=suppress_text_done_events, + downstream_turn_state=downstream_turn_state, + forwarded_request=forwarded_request, + forwarded_original_request_unanchored=forwarded_original_request_unanchored, + forwarded_legacy_signature=forwarded_legacy_signature, + forwarded_affinity_kind=forwarded_affinity_kind, + forwarded_affinity_key=forwarded_affinity_key, + forwarded_file_owner_account_id=forwarded_file_owner_account_id, + client_ip=client_ip, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + capacity_startup_wait_event=capacity_wait_event, + capacity_startup_ready_event=capacity_ready_event, + ) + async for line in retry_stream: + yield line + + return _retry() + + stream = build_response_stream() capacity_wait_token = _bind_propagated_capacity_startup_wait(capacity_wait_event) capacity_ready_token = _bind_propagated_capacity_startup_ready(capacity_ready_event) try: @@ -5095,18 +5151,52 @@ async def _stream_responses( _reset_propagated_capacity_startup_ready(capacity_ready_token) _reset_propagated_capacity_startup_wait(capacity_wait_token) if startup_error is not None: - if owns_reservation: - await _release_reservation(reservation) - return _stream_startup_error_response( - request, - startup_error, - headers=rate_limit_headers, - ) + startup_error_code = ( + _startup_error_details(startup_error)[0] if isinstance(startup_error, ProxyResponseError) else None + ) + startup_recovery_allowed = ( + isinstance(startup_error, ProxyResponseError) + and bridge_recovery_eligible + and get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "server_indefinite_recovery" + and getattr(startup_error, "http_bridge_durable_recovery_eligible", False) + and startup_error_code + in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout", "upstream_unavailable"} + ) + if startup_recovery_allowed: + assert isinstance(startup_error, ProxyResponseError) + + # A durable bridge can fail before the startup probe observes the + # first response.created event. Feed that error through the same + # server-owned recovery loop used for failures after the probe; + # returning JSON here would hand a recoverable disconnect back to + # the client before recovery is even installed. + async def _raise_startup_error() -> AsyncIterator[str]: + raise startup_error + yield "" # pragma: no cover + + stream = _raise_startup_error() + else: + if owns_reservation: + await _release_reservation(reservation) + return _stream_startup_error_response( + request, + startup_error, + headers=rate_limit_headers, + allow_client_full_history_once=bridge_recovery_eligible, + ) + # Server-indefinite recovery is only safe for an explicitly anchored + # continuation. Fresh first-turn requests have no durable parent + # operation to fence, so do not install the recovery loop for them. + recovery_stream_factory = build_recovery_response_stream if bridge_recovery_eligible else None stream = _normalize_public_responses_stream( _stream_response_error_events( stream, owns_reservation=owns_reservation, reservation=reservation, + recovery_stream_factory=recovery_stream_factory, + allow_client_full_history_once=bridge_recovery_eligible, + require_durable_recovery_fence=bridge_recovery_eligible, ), enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) @@ -5174,6 +5264,11 @@ async def _collect_responses( rate_limit_headers = await _rate_limit_headers_with_reservation_cleanup(context, api_key, reservation) bridge_active = prefer_http_bridge and proxy_service_module.get_settings().http_responses_session_bridge_enabled + bridge_recovery_eligible = _http_bridge_recovery_request_eligible( + payload, + bridge_active=bridge_active, + headers=request.headers, + ) downstream_turn_state = ( proxy_affinity_module.ensure_http_downstream_turn_state(request.headers) if bridge_active else None ) @@ -5218,7 +5313,13 @@ async def _collect_responses( except ProxyResponseError as exc: await _release_reservation(reservation) error = _parse_error_envelope(exc.payload) - status_code, error = _mask_previous_response_not_found_error(error, default_status=exc.status_code) + status_code, error = _mask_previous_response_not_found_error( + error, + default_status=exc.status_code, + allow_client_full_history_once=( + bridge_recovery_eligible and getattr(exc, "http_bridge_durable_recovery_eligible", False) + ), + ) return _logged_error_json_response( request, status_code, @@ -5228,7 +5329,10 @@ async def _collect_responses( if isinstance(response_payload, OpenAIResponsePayload): if response_payload.status == "failed": error_payload = _error_envelope_from_response(response_payload.error) - status_code, error_payload = _mask_previous_response_not_found_error(error_payload) + status_code, error_payload = _mask_previous_response_not_found_error( + error_payload, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -5239,7 +5343,10 @@ async def _collect_responses( content=response_payload.model_dump(mode="json", exclude_none=True), headers={**turn_state_headers, **captured_turn_state_headers, **rate_limit_headers}, ) - status_code, response_payload = _mask_previous_response_not_found_error(response_payload) + status_code, response_payload = _mask_previous_response_not_found_error( + response_payload, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -6223,18 +6330,112 @@ async def _stream_response_error_events( *, owns_reservation: bool, reservation: ApiKeyUsageReservationData | None, + recovery_stream_factory: Callable[[], AsyncIterator[str]] | None = None, + allow_client_full_history_once: bool = False, + require_durable_recovery_fence: bool = False, ) -> AsyncIterator[str]: + saw_downstream_event = False try: async for line in stream: + if line.startswith("data:") or line.startswith("event:"): + saw_downstream_event = True yield line except ProxyResponseError as exc: + error_code = exc.payload.get("error", {}).get("code") if isinstance(exc.payload, dict) else None + indefinite_recovery = ( + get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "server_indefinite_recovery" + ) + if ( + recovery_stream_factory is not None + and indefinite_recovery + and (not require_durable_recovery_fence or getattr(exc, "http_bridge_durable_recovery_eligible", False)) + and not saw_downstream_event + and error_code + in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout", "upstream_unavailable"} + ): + # Keep the client stream alive while the server owns recovery. + # The operation remains serialized by the durable operation + # fingerprint; each new upstream attempt is still at-least-once. + retry_delay = max(1.0, min(30.0, float(exc.retry_after_seconds or 5.0))) + while True: + yield ": codex-lb recovery in progress\n\n" + await asyncio.sleep(retry_delay) + try: + retry_stream = recovery_stream_factory() + retry_saw_downstream_event = False + async for line in retry_stream: + if line.startswith("data:") or line.startswith("event:"): + retry_saw_downstream_event = True + saw_downstream_event = True + yield line + return + except ProxyResponseError as retry_exc: + retry_code = ( + retry_exc.payload.get("error", {}).get("code") if isinstance(retry_exc.payload, dict) else None + ) + if ( + retry_code + not in { + "stream_incomplete", + "stream_idle_timeout", + "upstream_request_timeout", + "upstream_unavailable", + } + or retry_saw_downstream_event + or ( + require_durable_recovery_fence + and not getattr(retry_exc, "http_bridge_durable_recovery_eligible", False) + ) + ): + exc = retry_exc + break + retry_delay = max(1.0, min(30.0, float(retry_exc.retry_after_seconds or retry_delay))) + except (ProxyRateLimitError, ProxyAuthError) as retry_limit_exc: + # A quota revocation or limit can happen between recovery + # attempts. Convert it into the same terminal SSE shape + # as other proxy failures instead of aborting an already + # started response stream without a response.failed event. + exc = ProxyResponseError( + retry_limit_exc.status_code, + openai_error( + retry_limit_exc.code, + retry_limit_exc.message, + error_type=getattr(retry_limit_exc, "error_type", "server_error"), + ), + ) + break + except Exception: + # Recovery admission can also fail before a replacement + # stream is created (for example, a transient database + # failure while reserving usage). Do not let that + # unexpected exception truncate an already-started SSE + # response; the outer cleanup still settles the original + # reservation and emits one terminal response.failed event. + logger.warning("HTTP bridge recovery admission failed", exc_info=True) + exc = ProxyResponseError( + 503, + openai_error( + "bridge_recovery_admission_failed", + "Recovery admission failed; retry shortly.", + error_type="server_error", + ), + retry_after_seconds=5, + ) + break if owns_reservation: try: await _release_reservation(reservation) except Exception: logger.warning("Failed to release stream reservation after upstream proxy error", exc_info=True) envelope = _parse_error_envelope(exc.payload) - _, envelope = _mask_previous_response_not_found_error(envelope, default_status=exc.status_code) + _, envelope = _mask_previous_response_not_found_error( + envelope, + default_status=exc.status_code, + allow_client_full_history_once=( + allow_client_full_history_once and getattr(exc, "http_bridge_durable_recovery_eligible", False) + ), + ) error = envelope.error retry_hint = "" if exc.retry_after_seconds is not None and exc.retry_after_seconds > 0: @@ -6259,10 +6460,17 @@ def _stream_startup_error_response( error: ProxyResponseError | OpenAIErrorEnvelopeModel, *, headers: Mapping[str, str], + allow_client_full_history_once: bool = False, ) -> JSONResponse: if isinstance(error, ProxyResponseError): envelope = _parse_error_envelope(error.payload) - status_code, envelope = _mask_previous_response_not_found_error(envelope, default_status=error.status_code) + status_code, envelope = _mask_previous_response_not_found_error( + envelope, + default_status=error.status_code, + allow_client_full_history_once=( + allow_client_full_history_once and getattr(error, "http_bridge_durable_recovery_eligible", False) + ), + ) startup_headers = dict(headers) if error.retry_after_seconds is not None and error.retry_after_seconds > 0: startup_headers.setdefault("Retry-After", str(error.retry_after_seconds)) @@ -6272,7 +6480,10 @@ def _stream_startup_error_response( envelope.model_dump(mode="json", exclude_none=True), headers=startup_headers, ) - status_code, envelope = _mask_previous_response_not_found_error(error) + status_code, envelope = _mask_previous_response_not_found_error( + error, + allow_client_full_history_once=False, + ) return _logged_error_json_response( request, status_code, @@ -7886,13 +8097,48 @@ def _is_previous_response_not_found_public_error(error_value: OpenAIError | None ) +def _http_bridge_recovery_request_eligible( + payload: ResponsesRequest, + *, + bridge_active: bool, + headers: Mapping[str, str] | None = None, +) -> bool: + turn_state_anchor = proxy_affinity_module._sticky_key_from_turn_state_header(headers or {}) + if not bridge_active or (payload.previous_response_id is None and turn_state_anchor is None): + return False + settings = proxy_service_module.get_settings() + if not getattr(settings, "http_responses_session_bridge_operation_ledger_enabled", True): + return False + # Turn-state-only requests are admitted to the recovery-capable stream so + # the submit path can first prove a durable predecessor by advancing its + # operation anchor. The streaming layer marks an exception recovery-safe + # only after that proof; fresh first turns remain fail-closed there. + if proxy_service_module._responses_request_contains_input_image( + payload + ) or proxy_service_module._responses_request_uses_image_generation(payload): + return False + payload_bytes = len(json.dumps(payload.to_payload(), ensure_ascii=True, separators=(",", ":")).encode("utf-8")) + return payload_bytes <= proxy_service_module._ws_transport_payload_budget_bytes(settings) + + def _mask_previous_response_not_found_error( envelope: OpenAIErrorEnvelopeModel, *, default_status: int | None = None, + allow_client_full_history_once: bool = False, ) -> tuple[int, OpenAIErrorEnvelopeModel]: if not _is_previous_response_not_found_public_error(envelope.error): return default_status if default_status is not None else _status_for_error(envelope.error), envelope + # In recovery-first mode, preserve the upstream-shaped 400 so Codex can + # drop the ambiguous previous_response_id anchor and resend full local + # history. This is intentionally opt-in because the resend is at-least-once + # and may duplicate an upstream response that was accepted but not observed. + if ( + allow_client_full_history_once + and get_settings().http_responses_session_bridge_ambiguous_continuation_recovery_mode + == "client_full_history_once" + ): + return default_status if default_status is not None else 400, envelope return ( 502, OpenAIErrorEnvelopeModel( diff --git a/app/modules/proxy/continuity.py b/app/modules/proxy/continuity.py index 463497aeb2..c26ba9a93b 100644 --- a/app/modules/proxy/continuity.py +++ b/app/modules/proxy/continuity.py @@ -2,7 +2,9 @@ from __future__ import annotations +import logging from collections.abc import Mapping +from hashlib import sha256 from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error @@ -20,6 +22,7 @@ "x-codex-turn-state", } ) +logger = logging.getLogger("app.modules.proxy.continuity") def make_http_bridge_account_neutral_replay_key(nonce: str) -> tuple[str, str]: @@ -63,6 +66,15 @@ def resolve_required_account_id(*owners: tuple[str, str | None]) -> str | None: # side would silently abandon the other, so conflicts are never ordered # by caller precedence or softened into ordinary affinity fallback. sources = ", ".join(source for source, _account_id in resolved) + owner_hashes = ", ".join( + f"{source}={sha256(account_id.encode()).hexdigest()[:12]}" for source, account_id in resolved + ) + logger.warning( + "continuity_owner_conflict sources=%s conflicting_sources=%s owner_hashes=%s", + sources, + ", ".join(conflicting_sources), + owner_hashes, + ) raise ProxyResponseError( 502, openai_error( diff --git a/app/modules/proxy/durable_bridge_coordinator.py b/app/modules/proxy/durable_bridge_coordinator.py index 4fdd3e0bc4..d0d98a381a 100644 --- a/app/modules/proxy/durable_bridge_coordinator.py +++ b/app/modules/proxy/durable_bridge_coordinator.py @@ -16,10 +16,13 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, + DurableBridgeOperationEventInput, + DurableBridgeOperationSnapshot, DurableBridgeRecoveryAttemptSnapshot, DurableBridgeRepository, DurableBridgeRetryCircuitSnapshot, DurableBridgeSessionSnapshot, + DurableBridgeTranscriptTurn, durable_bridge_api_key_scope, ) @@ -489,6 +492,289 @@ async def rollback_recovery_attempt_replayed( request_fingerprint=request_fingerprint, ) + async def rollback_recovery_attempt_before_dispatch( + self, + *, + session_id: str, + api_key_id: str | None, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + ) -> bool: + del api_key_id + async with self._session() as session: + return await DurableBridgeRepository(session).rollback_recovery_attempt_before_dispatch( + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + request_fingerprint=request_fingerprint, + ) + + async def record_operation( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + account_id: str | None, + model: str | None, + parent_response_id: str | None, + api_key_scope: str | None = None, + request_text: str | None = None, + recovery_attempt_session_id: str | None = None, + recovery_attempt_owner_epoch: int | None = None, + recovery_attempt_fingerprint: str | None = None, + recovery_attempt_consumed: bool = False, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).record_operation( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + request_fingerprint=request_fingerprint, + api_key_scope=api_key_scope, + account_id=account_id, + model=model, + parent_response_id=parent_response_id, + request_text=request_text, + recovery_attempt_session_id=recovery_attempt_session_id, + recovery_attempt_owner_epoch=recovery_attempt_owner_epoch, + recovery_attempt_fingerprint=recovery_attempt_fingerprint, + recovery_attempt_consumed=recovery_attempt_consumed, + ) + + async def get_operation_events(self, *, operation_id: str) -> list[str]: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation_events(operation_id=operation_id) + + async def get_replayable_transcript( + self, + *, + response_id: str, + max_turns: int = 128, + max_bytes: int = 8 * 1024 * 1024, + ) -> list[DurableBridgeTranscriptTurn] | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_replayable_transcript( + response_id=response_id, + max_turns=max_turns, + max_bytes=max_bytes, + ) + + async def purge_operation_spool(self, *, cutoff: datetime, batch_size: int = 500) -> int: + async with self._session() as session: + return await DurableBridgeRepository(session).purge_operation_spool( + cutoff=cutoff, + batch_size=batch_size, + ) + + async def append_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).append_operation_event( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + max_bytes=max_bytes, + ) + + async def append_terminal_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + state: str, + response_id: str | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).append_terminal_operation_event( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + max_bytes=max_bytes, + state=state, + response_id=response_id, + ) + + async def append_operation_events( + self, + *, + events: Sequence[DurableBridgeOperationEventInput], + max_bytes: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).append_operation_events( + events=events, + max_bytes=max_bytes, + ) + + async def finalize_operation_event_spool( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).finalize_operation_event_spool( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + ) + + async def update_operation( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + response_id: str | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).update_operation( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + response_id=response_id, + ) + + async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation(operation_id=operation_id) + + async def reset_operation_event_spool( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).reset_operation_event_spool( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + ) + + async def claim_unknown_operation_for_recovery( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + max_recovery_dispatches: int | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + max_recovery_dispatches=max_recovery_dispatches, + ) + + async def mark_operation_unknown( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + restore_recovery_dispatch_claim: bool = False, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).mark_operation_unknown( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + restore_recovery_dispatch_claim=restore_recovery_dispatch_claim, + ) + + async def rollback_operation_before_dispatch( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).rollback_operation_before_dispatch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + ) + + async def get_operation_by_fingerprint( + self, + *, + request_fingerprint: str, + api_key_scope: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_operation_by_fingerprint( + request_fingerprint=request_fingerprint, + api_key_scope=api_key_scope, + ) + + async def get_latest_completed_operation( + self, + *, + session_id: str, + parent_response_id: str, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_latest_completed_operation( + session_id=session_id, + parent_response_id=parent_response_id, + request_fingerprint=request_fingerprint, + ) + + async def get_latest_completed_operation_any_session( + self, + *, + parent_response_id: str, + api_key_scope: str | None = None, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + async with self._session() as session: + return await DurableBridgeRepository(session).get_latest_completed_operation_any_session( + parent_response_id=parent_response_id, + api_key_scope=api_key_scope, + request_fingerprint=request_fingerprint, + ) + async def mark_instance_draining(self, *, instance_id: str) -> int: async with self._session() as session: return await DurableBridgeRepository(session).mark_owner_draining(instance_id=instance_id) diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index bcea92b62b..e8227d826a 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -9,7 +9,7 @@ from hashlib import sha256 from typing import Any -from sqlalchemy import Row, and_, case, delete, func, or_, select, text, true, update +from sqlalchemy import Row, and_, case, delete, exists, func, or_, select, text, true, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import IntegrityError @@ -17,6 +17,8 @@ from app.core.utils.time import to_utc_naive, utcnow from app.db.models import ( + HttpBridgeOperationEvent, + HttpBridgeOperationRecord, HttpBridgeRecoveryAttemptRecord, HttpBridgeRecoveryAttemptState, HttpBridgeRetryCircuit, @@ -38,6 +40,8 @@ "http_bridge_session_aliases", "http_bridge_retry_circuits", "http_bridge_recovery_attempts", + "http_bridge_operations", + "http_bridge_operation_events", ) DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS = 3600.0 _PURGE_CLOSED_BATCH_SIZE = 500 @@ -78,6 +82,16 @@ def durable_bridge_hash(value: str) -> str: return sha256(value.encode("utf-8")).hexdigest() +def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str: + """Hash the logical turn together with its authorization namespace.""" + return durable_bridge_hash(f"{api_key_scope}:{request_text}") + + +def durable_bridge_operation_id(session_id: str, request_fingerprint: str) -> str: + """Derive a stable, non-secret operation key for a continuity-bound turn.""" + return f"op_{durable_bridge_hash(f'{session_id}:{request_fingerprint}')[:64]}" + + def _encode_pending_tool_calls(response_id: str, value: Mapping[str, str] | None) -> str | None: if value is None: return None @@ -156,6 +170,37 @@ class DurableBridgeRecoveryAttemptSnapshot: response_id: str | None +@dataclass(frozen=True, slots=True) +class DurableBridgeOperationSnapshot: + operation_id: str + session_id: str + request_fingerprint: str + account_id: str | None + model: str | None + parent_response_id: str | None + state: str + response_id: str | None + recovery_dispatch_count: int = 0 + request_text: str | None = None + event_spool_complete: bool = True + created: bool = False + + +@dataclass(frozen=True, slots=True) +class DurableBridgeTranscriptTurn: + operation: DurableBridgeOperationSnapshot + events: tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class DurableBridgeOperationEventInput: + operation_id: str + session_id: str + instance_id: str + owner_epoch: int + event_text: str + + class DurableBridgeRepository: def __init__(self, session: AsyncSession) -> None: self._session = session @@ -998,6 +1043,907 @@ async def rollback_recovery_attempt_replayed( await self._session.commit() return bool(getattr(result, "rowcount", 0)) + async def rollback_recovery_attempt_before_dispatch( + self, + *, + session_id: str, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + ) -> bool: + """Delete an UNKNOWN checkpoint proven not to have reached upstream.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return False + result = await self._session.execute( + delete(HttpBridgeRecoveryAttemptRecord).where( + HttpBridgeRecoveryAttemptRecord.session_id == session_id, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint, + HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN, + ) + ) + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) + + async def record_operation( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + request_fingerprint: str, + account_id: str | None, + model: str | None, + parent_response_id: str | None, + api_key_scope: str | None = None, + request_text: str | None = None, + recovery_attempt_session_id: str | None = None, + recovery_attempt_owner_epoch: int | None = None, + recovery_attempt_fingerprint: str | None = None, + recovery_attempt_consumed: bool = False, + ) -> DurableBridgeOperationSnapshot | None: + """Create a fenced operation identity, or return the existing one.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return None + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where(HttpBridgeOperationRecord.operation_id == operation_id) + .with_for_update() + ) + if operation is None: + fingerprint_statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + fingerprint_statement = fingerprint_statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(fingerprint_statement.with_for_update()) + if operation is not None: + if recovery_attempt_consumed: + # A REPLAYED recovery checkpoint is immutable. Return the + # existing row for safe transcript replay or fail-closed + # handling; never rebind a failed row and clear its spool. + snapshot = _to_operation_snapshot(operation) + await self._session.rollback() + return snapshot + rebound = False + handoff_allowed = True + if operation.session_id != session_id and operation.state not in {"completed", "incomplete"}: + # A global fingerprint can outlive the durable session + # that first recorded it. Do not steal an operation from + # a still-live owner: its stream may still be dispatching + # the turn, and rebinding would fence its writes while a + # second owner sends a duplicate upstream request. + previous_session = await self._session.scalar( + select(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == operation.session_id) + .with_for_update() + ) + now = utcnow() + recovery_handoff_allowed = False + if ( + previous_session is not None + and recovery_attempt_session_id == operation.session_id + and recovery_attempt_owner_epoch is not None + and recovery_attempt_fingerprint is not None + and previous_session.owner_instance_id == instance_id + and previous_session.owner_epoch == recovery_attempt_owner_epoch + ): + # A fresh account-neutral replay has already fenced + # the one-shot journal on the origin session. That + # journal owner must remain fenced until settlement, + # but the operation itself must move to the + # replacement owner so its transcript and outcome + # writes are accepted there. This is the only + # cross-session handoff allowed while the origin + # lease is still active. + recovery_attempt = await self._session.scalar( + select(HttpBridgeRecoveryAttemptRecord) + .where( + HttpBridgeRecoveryAttemptRecord.session_id == recovery_attempt_session_id, + HttpBridgeRecoveryAttemptRecord.request_fingerprint == recovery_attempt_fingerprint, + HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED, + HttpBridgeRecoveryAttemptRecord.response_id.is_(None), + ) + .with_for_update() + ) + recovery_handoff_allowed = recovery_attempt is not None + handoff_allowed = ( + recovery_handoff_allowed + or previous_session is None + or not ( + previous_session.owner_instance_id is not None + and previous_session.lease_expires_at is not None + # PostgreSQL returns timestamptz values with an + # attached UTC offset, while ``utcnow`` is a + # naive UTC value used by the durable layer. + # Normalize before comparing so cross-session + # recovery remains database-backend agnostic. + and to_utc_naive(previous_session.lease_expires_at) > now + ) + ) + if handoff_allowed: + # Transfer only nonterminal operations to the currently + # fenced owner before the caller resets the attempt + # spool; completed transcripts remain attached to + # their original session for replay. + operation.session_id = session_id + operation.account_id = account_id + operation.model = model + operation.parent_response_id = parent_response_id + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.updated_at = now + if operation.state == "failed" and handoff_allowed: + # An explicit upstream failure is retryable. Rebind the + # durable operation to the current owner while preserving + # its global identity; concurrent reconnects will see the + # submitted state and remain fenced. + operation.session_id = session_id + operation.account_id = account_id + operation.model = model + operation.parent_response_id = parent_response_id + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.state = "submitted" + operation.response_id = None + # A failed attempt is a new replay attempt. Remove the + # previous attempt's SSE spool atomically so a later + # successful retry cannot replay a stale response.failed + # event before its fresh response.created sequence. + await self._session.execute( + delete(HttpBridgeOperationEvent).where( + HttpBridgeOperationEvent.operation_id == operation.operation_id + ) + ) + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + rebound = True + if request_text is not None and operation.request_text is None: + operation.request_text = request_text + operation.updated_at = utcnow() + snapshot = _to_operation_snapshot(operation, created=rebound) + await self._session.commit() + return snapshot + operation = HttpBridgeOperationRecord( + operation_id=operation_id, + session_id=session_id, + request_fingerprint=request_fingerprint, + account_id=account_id, + model=model, + parent_response_id=parent_response_id, + request_text=request_text, + state="submitted", + # A transcript is replayable only after the event batcher has + # drained and finalized it. Set this explicitly rather than + # relying on a backend-specific schema default (notably the + # pre-existing SQLite default on migrated databases). + event_spool_complete=False, + ) + self._session.add(operation) + try: + await self._session.commit() + except IntegrityError: + await self._session.rollback() + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id) + ) + if operation is None: + # A reconnect may derive a different session-scoped + # operation ID for the same anchored request. The global + # fingerprint fence makes that race resolve to the + # already-recorded operation instead of dispatching a + # duplicate. + fingerprint_statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + fingerprint_statement = fingerprint_statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(fingerprint_statement) + if operation is None: + raise + return _to_operation_snapshot(operation) + await self._session.refresh(operation) + return _to_operation_snapshot(operation, created=True) + + async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None: + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def reset_operation_event_spool( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + """Start a fresh transcript for a server-owned ambiguous retry.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state.not_in(("completed", "incomplete")), + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id == operation_id) + ) + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def claim_unknown_operation_for_recovery( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + max_recovery_dispatches: int | None = None, + ) -> bool: + """Atomically claim an UNKNOWN operation for one recovery attempt. + + Recovery admission can be reached by multiple reconnects at once. A + reset followed by a later state transition leaves a window where each + reconnect can observe UNKNOWN and submit the same operation. Keep the + owner fence, state transition, and transcript reset in one serialized + write so exactly one caller can move UNKNOWN back to SUBMITTED. + """ + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state == "unknown", + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + if max_recovery_dispatches is not None and operation.recovery_dispatch_count >= max_recovery_dispatches: + await self._session.rollback() + return False + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id == operation_id) + ) + operation.state = "submitted" + operation.response_id = None + operation.recovery_dispatch_count += 1 + operation.event_bytes = 0 + operation.event_spool_complete = False + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def mark_operation_unknown( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + restore_recovery_dispatch_claim: bool = False, + ) -> bool: + """Fence an ambiguously dispatched SUBMITTED operation as UNKNOWN. + + The operation event reader can race the send-failure cleanup. Lock the + row before changing it and leave an already acknowledged or terminal + operation untouched; those states carry stronger evidence than the + transport exception and must never be downgraded to UNKNOWN. + """ + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + if operation.state == "submitted": + operation.state = "unknown" + if restore_recovery_dispatch_claim and operation.recovery_dispatch_count > 0: + operation.recovery_dispatch_count -= 1 + operation.updated_at = utcnow() + elif ( + restore_recovery_dispatch_claim + and operation.state == "unknown" + and operation.recovery_dispatch_count > 0 + ): + # A concurrent cleanup may have fenced the row first. The + # caller still owns a proven pre-dispatch recovery claim, so + # refund exactly that claim while retaining UNKNOWN. + operation.recovery_dispatch_count -= 1 + operation.updated_at = utcnow() + await self._session.commit() + return True + + async def rollback_operation_before_dispatch( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + """Remove a newly-created operation that never reached upstream.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state == "submitted", + HttpBridgeOperationRecord.response_id.is_(None), + HttpBridgeOperationRecord.event_bytes == 0, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + has_events = await self._session.scalar( + select(HttpBridgeOperationEvent.event_id) + .where(HttpBridgeOperationEvent.operation_id == operation_id) + .limit(1) + ) + if has_events is not None: + await self._session.rollback() + return False + await self._session.delete(operation) + await self._session.commit() + return True + + async def get_operation_by_fingerprint( + self, + *, + request_fingerprint: str, + api_key_scope: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + statement = select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.request_fingerprint == request_fingerprint + ) + if api_key_scope is not None: + statement = statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar(statement) + return _to_operation_snapshot(operation) if operation is not None else None + + async def get_operation_events(self, *, operation_id: str) -> list[str]: + result = await self._session.execute( + select(HttpBridgeOperationEvent.event_text) + .where(HttpBridgeOperationEvent.operation_id == operation_id) + .order_by(HttpBridgeOperationEvent.sequence_number.asc()) + ) + return [str(value) for value in result.scalars().all()] + + async def get_operation_by_response_id(self, *, response_id: str) -> DurableBridgeOperationSnapshot | None: + operation = await self._session.scalar( + select(HttpBridgeOperationRecord).where( + HttpBridgeOperationRecord.response_id == response_id, + HttpBridgeOperationRecord.state.in_(("completed", "incomplete")), + ) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def get_replayable_transcript( + self, + *, + response_id: str, + max_turns: int = 128, + max_bytes: int = 8 * 1024 * 1024, + ) -> list[DurableBridgeTranscriptTurn] | None: + """Return a complete parent-response chain, newest turn last. + + Missing request bodies, truncated event spools, or a broken parent + chain make the transcript ineligible for reconstruction. + """ + turns: list[DurableBridgeTranscriptTurn] = [] + visited: set[str] = set() + total_bytes = 0 + current_response_id: str | None = response_id + while current_response_id is not None: + if current_response_id in visited or len(turns) >= max_turns: + return None + visited.add(current_response_id) + operation = await self.get_operation_by_response_id(response_id=current_response_id) + if operation is None or operation.request_text is None or not operation.event_spool_complete: + return None + events = await self.get_operation_events(operation_id=operation.operation_id) + if not events or not any( + "response.completed" in event or "response.incomplete" in event for event in events + ): + return None + turn_bytes = len(operation.request_text.encode("utf-8")) + sum( + len(event.encode("utf-8")) for event in events + ) + total_bytes += turn_bytes + if total_bytes > max_bytes: + return None + turns.append(DurableBridgeTranscriptTurn(operation=operation, events=tuple(events))) + current_response_id = operation.parent_response_id + turns.reverse() + return turns + + async def purge_operation_spool(self, *, cutoff: datetime, batch_size: int = 500) -> int: + """Delete eligible transcript material past retention. + + Nonterminal rows are purgeable only after their owning session is + ownerless or its lease has expired. Recheck that predicate in the + delete transaction so an in-flight operation cannot lose its + duplicate-suppression ledger between selection and deletion. + """ + terminal_states = ("completed", "incomplete", "failed") + # UNKNOWN is an ambiguous, still-live operation while its owner lease + # is active. Treat it like the other nonterminal states so retention + # cannot delete the duplicate-suppression fence during a long-running + # server-indefinite recovery attempt. + nonterminal_states = ("submitted", "acknowledged", "unknown") + stale_owner = or_( + HttpBridgeSessionRecord.owner_instance_id.is_(None), + HttpBridgeSessionRecord.lease_expires_at.is_(None), + HttpBridgeSessionRecord.lease_expires_at < utcnow(), + ) + stale_nonterminal = and_( + HttpBridgeOperationRecord.state.in_(nonterminal_states), + exists( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + stale_owner, + ) + .correlate(HttpBridgeOperationRecord) + ), + ) + purgeable = or_(HttpBridgeOperationRecord.state.in_(terminal_states), stale_nonterminal) + async with sqlite_writer_section(): + selected = await self._session.execute( + select(HttpBridgeOperationRecord) + .join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ) + .where(HttpBridgeOperationRecord.updated_at < cutoff, purgeable) + .order_by(HttpBridgeOperationRecord.updated_at.asc()) + .limit(batch_size) + .with_for_update() + ) + # The joined FOR UPDATE locks both the operation and owning + # session on PostgreSQL, serializing retention deletion with + # claim_session() on the same continuity row. + operation_ids = [str(operation.operation_id) for operation in selected.scalars().all()] + if not operation_ids: + await self._session.commit() + return 0 + deleted = await self._session.execute( + delete(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id.in_(operation_ids), + HttpBridgeOperationRecord.updated_at < cutoff, + purgeable, + ) + .returning(HttpBridgeOperationRecord.operation_id) + ) + deleted_ids = [str(value) for value in deleted.scalars().all()] + if deleted_ids: + await self._session.execute( + delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id.in_(deleted_ids)) + ) + await self._session.commit() + return len(deleted_ids) + + async def append_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + ) -> bool: + """Append one replayable SSE block under the durable owner fence.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + event_size = len(event_text.encode("utf-8")) + if event_size > max_bytes or int(operation.event_bytes or 0) + event_size > max_bytes: + operation.event_spool_complete = False + await self._session.commit() + return False + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == operation_id, + ) + ) + sequence = int(next_sequence or 1) + self._session.add( + HttpBridgeOperationEvent( + operation_id=operation_id, + sequence_number=sequence, + # Include occurrence position so identical downstream + # blocks remain distinct in replay transcripts. + event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"), + event_text=event_text, + ) + ) + operation.event_bytes = int(operation.event_bytes or 0) + event_size + await self._session.commit() + return True + + async def append_terminal_operation_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + state: str, + response_id: str | None = None, + ) -> bool: + """Append a terminal event and expose its operation state atomically.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + event_size = len(event_text.encode("utf-8")) + persisted = event_size <= max_bytes and int(operation.event_bytes or 0) + event_size <= max_bytes + if persisted: + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == operation_id, + ) + ) + sequence = int(next_sequence or 1) + self._session.add( + HttpBridgeOperationEvent( + operation_id=operation_id, + sequence_number=sequence, + event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"), + event_text=event_text, + ) + ) + operation.event_bytes = int(operation.event_bytes or 0) + event_size + else: + operation.event_spool_complete = False + # The terminal outcome is still authoritative even when the + # transcript block cannot fit in the bounded spool. Expose + # the failed state so an identical retry does not remain + # fenced as an in-flight operation until retention expires. + operation.state = state + if response_id is not None: + operation.response_id = response_id + operation.updated_at = utcnow() + await self._session.commit() + return False + operation.state = state + if response_id is not None: + operation.response_id = response_id + operation.event_spool_complete = True + operation.updated_at = utcnow() + await self._session.commit() + return persisted + + async def append_operation_events( + self, + *, + events: Sequence[DurableBridgeOperationEventInput], + max_bytes: int, + ) -> bool: + """Append a batch of SSE blocks with one fenced transaction.""" + if not events: + return True + first = events[0] + if any( + event.operation_id != first.operation_id + or event.session_id != first.session_id + or event.instance_id != first.instance_id + or event.owner_epoch != first.owner_epoch + for event in events + ): + return False + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == first.session_id, + HttpBridgeSessionRecord.owner_instance_id == first.instance_id, + HttpBridgeSessionRecord.owner_epoch == first.owner_epoch, + ) + .with_for_update() + ) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == first.operation_id, + HttpBridgeOperationRecord.session_id == first.session_id, + ) + .with_for_update() + ) + if owner_exists is None or operation is None: + await self._session.rollback() + return False + next_sequence = await self._session.scalar( + select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where( + HttpBridgeOperationEvent.operation_id == first.operation_id, + ) + ) + sequence = int(next_sequence or 1) + pending: list[tuple[str, int, str, int]] = [] + total_bytes = int(operation.event_bytes or 0) + for event in events: + event_size = len(event.event_text.encode("utf-8")) + if total_bytes + event_size > max_bytes: + operation.event_spool_complete = False + await self._session.commit() + return False + total_bytes += event_size + pending.append( + ( + event.event_text, + sequence, + durable_bridge_hash(f"{sequence}:{event.event_text}"), + event_size, + ) + ) + sequence += 1 + if pending: + for event_text, sequence_number, fingerprint, event_size in pending: + self._session.add( + HttpBridgeOperationEvent( + operation_id=first.operation_id, + sequence_number=sequence_number, + event_fingerprint=fingerprint, + event_text=event_text, + ) + ) + operation.event_bytes = total_bytes + await self._session.commit() + return True + + async def finalize_operation_event_spool( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + ) -> bool: + """Mark a terminal operation replay-complete after its queue drained.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + result = await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.state.in_(("completed", "incomplete")), + HttpBridgeOperationRecord.event_spool_complete.is_(False), + ) + .values(event_spool_complete=True, updated_at=utcnow()) + ) + if owner_exists is None: + await self._session.rollback() + return False + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) + + async def get_latest_completed_operation( + self, + *, + session_id: str, + parent_response_id: str, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + predicates = [ + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.parent_response_id == parent_response_id, + HttpBridgeOperationRecord.state == "completed", + HttpBridgeOperationRecord.response_id.is_not(None), + ] + if request_fingerprint is not None: + predicates.append(HttpBridgeOperationRecord.request_fingerprint == request_fingerprint) + operation = await self._session.scalar( + select(HttpBridgeOperationRecord) + .where(*predicates) + .order_by(HttpBridgeOperationRecord.updated_at.desc()) + .limit(1) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def get_latest_completed_operation_any_session( + self, + *, + parent_response_id: str, + api_key_scope: str | None = None, + request_fingerprint: str | None = None, + ) -> DurableBridgeOperationSnapshot | None: + statement = select(HttpBridgeOperationRecord) + if api_key_scope is not None: + statement = statement.join( + HttpBridgeSessionRecord, + HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id, + ).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope) + operation = await self._session.scalar( + statement.where( + HttpBridgeOperationRecord.parent_response_id == parent_response_id, + HttpBridgeOperationRecord.state == "completed", + HttpBridgeOperationRecord.response_id.is_not(None), + *( + [HttpBridgeOperationRecord.request_fingerprint == request_fingerprint] + if request_fingerprint is not None + else [] + ), + ) + .order_by(HttpBridgeOperationRecord.updated_at.desc()) + .limit(1) + ) + return _to_operation_snapshot(operation) if operation is not None else None + + async def update_operation( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + response_id: str | None = None, + ) -> bool: + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return False + values: dict[str, object] = {"state": state, "updated_at": utcnow()} + if response_id is not None: + values["response_id"] = response_id + result = await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + ) + .values(**values) + ) + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) + async def _execute_fenced_session_update( self, *, @@ -1120,25 +2066,90 @@ async def purge_owned_sessions_on_startup( session_ids = [candidate.id for candidate in candidates] if not session_ids: return deleted_count + # Operation rows are the durable recovery ledger. Never cascade + # delete a session that still owns a retained operation, including + # completed replayable transcripts; detach it so the next instance + # can inspect and take over without losing continuity history. + operation_session_ids = set( + await self._session.scalars( + select(HttpBridgeOperationRecord.session_id).where( + HttpBridgeOperationRecord.session_id.in_(session_ids), + ) + ) + ) retained_recovery_ids = { candidate.id for candidate in candidates - if candidate.owner_instance_id == instance_id - and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch - and (ownerless_cutoff is None or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff)) - and is_http_bridge_account_neutral_replay( - kind=candidate.session_key_kind, - key=candidate.session_key_value, + if candidate.id in operation_session_ids + or ( + candidate.owner_instance_id == instance_id + and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch + and ( + ownerless_cutoff is None + or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff) + ) + and is_http_bridge_account_neutral_replay( + kind=candidate.session_key_kind, + key=candidate.session_key_value, + ) ) } async with sqlite_writer_section(): + ownerless_operation_ids = { + candidate.id + for candidate in candidates + if candidate.id in retained_recovery_ids + and candidate.id in operation_session_ids + and candidate.owner_instance_id is None + } + if ownerless_operation_ids: + # The ownerless-cutoff predicate is part of the same + # startup query. Refresh retained rows so the bounded + # loop cannot select them forever while their operation + # transcript is awaiting normal retention cleanup. + await self._session.execute( + update(HttpBridgeSessionRecord) + .where( + HttpBridgeSessionRecord.id.in_(ownerless_operation_ids), + HttpBridgeSessionRecord.owner_instance_id.is_(None), + ) + .values(last_seen_at=now, lease_expires_at=now) + ) if retained_recovery_ids: + # A process can die after recording a submitted + # operation but before upstream acknowledges it. Once + # startup has fenced and detached that owner's session, + # classify those rows as UNKNOWN so the replacement can + # enter the normal proof-gated recovery path. + operation_retained_session_ids = retained_recovery_ids & operation_session_ids + if operation_retained_session_ids: + eligible_operation_sessions = set( + await self._session.scalars( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id.in_(operation_retained_session_ids), + startup_purge_filter, + ) + .with_for_update() + ) + ) + await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.session_id.in_(eligible_operation_sessions), + HttpBridgeOperationRecord.state == "submitted", + ) + .values(state="unknown", updated_at=now) + ) await self._session.execute( update(HttpBridgeSessionRecord) .where( HttpBridgeSessionRecord.id.in_(retained_recovery_ids), - HttpBridgeSessionRecord.owner_instance_id == instance_id, - HttpBridgeSessionRecord.owner_process_epoch == owner_process_epoch, + # Detach the rows selected as belonging to the + # previous process. With an explicit new epoch, + # matching the new epoch here would leave old + # retained rows selected forever on every loop. + startup_purge_filter, ) .values( owner_instance_id=None, @@ -1232,6 +2243,11 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG .where( HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED, HttpBridgeSessionRecord.last_seen_at < cutoff, + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) .order_by(HttpBridgeSessionRecord.last_seen_at.asc()) .limit(batch_size) @@ -1247,6 +2263,11 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG HttpBridgeSessionRecord.id.in_(session_ids), HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED, HttpBridgeSessionRecord.last_seen_at < cutoff, + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) ) ) @@ -1256,6 +2277,13 @@ async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURG .where(HttpBridgeSessionRecord.id.in_(session_ids)) .where(HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED) .where(HttpBridgeSessionRecord.last_seen_at < cutoff) + .where( + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ) + ) .returning(HttpBridgeSessionRecord.id) ) await self._session.commit() @@ -1274,6 +2302,11 @@ async def purge_abandoned_before(self, cutoff: datetime, *, batch_size: int = _P HttpBridgeSessionRecord.lease_expires_at < now, ), HttpBridgeSessionRecord.last_seen_at < cutoff, + ~exists( + select(HttpBridgeOperationRecord.operation_id).where( + HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id, + ) + ), ) result = await self._session.execute( select(HttpBridgeSessionRecord.id) @@ -1779,7 +2812,7 @@ async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ... "SELECT name FROM sqlite_master " "WHERE type = 'table' " "AND name IN ('http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', " - "'http_bridge_recovery_attempts')" + "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events')" ) ) else: @@ -1789,7 +2822,7 @@ async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ... "WHERE table_schema = 'public' " "AND table_name IN (" "'http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', " - "'http_bridge_recovery_attempts'" + "'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events'" ")" ) ) @@ -1902,6 +2935,27 @@ def _to_recovery_attempt_snapshot( ) +def _to_operation_snapshot( + row: HttpBridgeOperationRecord, + *, + created: bool = False, +) -> DurableBridgeOperationSnapshot: + return DurableBridgeOperationSnapshot( + operation_id=row.operation_id, + session_id=row.session_id, + request_fingerprint=row.request_fingerprint, + account_id=row.account_id, + model=row.model, + parent_response_id=row.parent_response_id, + state=row.state, + response_id=row.response_id, + recovery_dispatch_count=row.recovery_dispatch_count, + request_text=row.request_text, + event_spool_complete=bool(row.event_spool_complete), + created=created, + ) + + def _to_retry_circuit_snapshot(row: HttpBridgeRetryCircuit | None) -> DurableBridgeRetryCircuitSnapshot | None: if row is None: return None diff --git a/app/modules/proxy/http_bridge_event_batcher.py b/app/modules/proxy/http_bridge_event_batcher.py new file mode 100644 index 0000000000..a22bc733d1 --- /dev/null +++ b/app/modules/proxy/http_bridge_event_batcher.py @@ -0,0 +1,341 @@ +from __future__ import annotations + +import asyncio +import logging +from dataclasses import dataclass +from typing import Any + +from app.core.config.settings import get_settings +from app.modules.proxy.durable_bridge_repository import DurableBridgeOperationEventInput + +logger = logging.getLogger("app.modules.proxy.http_bridge_event_batcher") + + +@dataclass(frozen=True, slots=True) +class _PendingOperationEvent: + operation_id: str + session_id: str + instance_id: str + owner_epoch: int + event_text: str + + +class HttpBridgeOperationEventBatcher: + """Best-effort in-memory event buffer for the HTTP bridge. + + Normal stream handling only appends to memory. A short-lived flusher + commits groups of events in one transaction. A terminal event drains its + operation synchronously once, so a completed operation is marked + replayable only after all queued events were persisted. A process crash or + queue overflow therefore loses optional transcript data, never upstream + work safety. + """ + + @classmethod + def from_settings(cls, durable_bridge: Any, settings: Any | None = None) -> "HttpBridgeOperationEventBatcher": + """Build the event spooler from the operator-facing settings surface.""" + settings = settings or get_settings() + return cls( + durable_bridge, + max_bytes=int( + getattr(settings, "http_responses_session_bridge_operation_event_spool_max_bytes", 2 * 1024 * 1024) + ), + batch_size=int(getattr(settings, "http_responses_session_bridge_operation_event_spool_batch_size", 32)), + flush_interval_seconds=float( + getattr(settings, "http_responses_session_bridge_operation_event_spool_flush_interval_seconds", 0.1) + ), + max_pending_events=int( + getattr(settings, "http_responses_session_bridge_operation_event_spool_max_pending_events", 2048) + ), + max_pending_bytes=int( + getattr( + settings, "http_responses_session_bridge_operation_event_spool_max_pending_bytes", 32 * 1024 * 1024 + ) + ), + ) + + def __init__( + self, + durable_bridge: Any, + *, + max_bytes: int, + batch_size: int = 32, + flush_interval_seconds: float = 0.1, + max_pending_events: int = 2048, + max_pending_bytes: int = 32 * 1024 * 1024, + ) -> None: + self._durable_bridge = durable_bridge + self._max_bytes = max_bytes + self._batch_size = batch_size + self._flush_interval_seconds = flush_interval_seconds + self._max_pending_events = max_pending_events + self._max_pending_bytes = max_pending_bytes + self._pending: dict[str, list[_PendingOperationEvent]] = {} + self._contexts: dict[str, _PendingOperationEvent] = {} + self._dropped_operations: set[str] = set() + self._closing_operations: set[str] = set() + self._pending_count = 0 + self._pending_bytes = 0 + self._lock = asyncio.Lock() + # SQLite already serializes writers; this also prevents a background + # flush racing a terminal drain and final marker for one operation. + self._flush_lock = asyncio.Lock() + self._wake = asyncio.Event() + self._task: asyncio.Task[None] | None = None + + async def enqueue( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + terminal: bool = False, + ) -> None: + self._ensure_task() + pending = _PendingOperationEvent( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + ) + async with self._lock: + self._contexts.setdefault(operation_id, pending) + if terminal: + self._closing_operations.add(operation_id) + if operation_id not in self._dropped_operations: + event_bytes = len(event_text.encode("utf-8")) + if ( + self._pending_count >= self._max_pending_events + or self._pending_bytes + event_bytes > self._max_pending_bytes + ): + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + logger.info( + "Dropping HTTP bridge transcript events after queue overflow operation_id=%s", + operation_id, + ) + else: + self._pending.setdefault(operation_id, []).append(pending) + self._pending_count += 1 + self._pending_bytes += event_bytes + self._wake.set() + if terminal: + await self.flush_operation(operation_id=operation_id) + + def _ensure_task(self) -> None: + if self._task is None or self._task.done(): + self._task = asyncio.create_task(self._run(), name="http-bridge-operation-event-flusher") + + async def _run(self) -> None: + while True: + try: + await asyncio.wait_for(self._wake.wait(), timeout=self._flush_interval_seconds) + except TimeoutError: + pass + self._wake.clear() + operation_ids = await self._operation_ids_to_flush() + for operation_id in operation_ids: + await self._flush_one(operation_id) + + async def _operation_ids_to_flush(self) -> list[str]: + async with self._lock: + return [operation_id for operation_id in self._pending if operation_id not in self._closing_operations] + + async def _take_batch(self, operation_id: str) -> list[_PendingOperationEvent]: + async with self._lock: + pending = self._pending.get(operation_id, []) + batch = pending[: self._batch_size] + if batch: + del pending[: len(batch)] + self._pending_count -= len(batch) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in batch) + if not pending: + self._pending.pop(operation_id, None) + return batch + + async def _flush_one(self, operation_id: str) -> None: + async with self._flush_lock: + batch = await self._take_batch(operation_id) + if not batch: + return + async with self._lock: + if operation_id in self._dropped_operations: + return + try: + persisted = await self._durable_bridge.append_operation_events( + events=[ + DurableBridgeOperationEventInput( + operation_id=item.operation_id, + session_id=item.session_id, + instance_id=item.instance_id, + owner_epoch=item.owner_epoch, + event_text=item.event_text, + ) + for item in batch + ], + max_bytes=self._max_bytes, + ) + if not persisted: + async with self._lock: + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + except Exception: + async with self._lock: + self._dropped_operations.add(operation_id) + dropped = self._pending.pop(operation_id, []) + self._pending_count -= len(dropped) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in dropped) + logger.debug( + "Dropping failed HTTP bridge transcript event batch operation_id=%s", + operation_id, + exc_info=True, + ) + + async def flush_operation(self, *, operation_id: str) -> None: + await self.flush_pending_operation(operation_id=operation_id) + async with self._lock: + dropped = operation_id in self._dropped_operations + context = self._contexts.get(operation_id) + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + if dropped or context is None: + return + # A single final marker is the only synchronous database operation on + # the terminal path. If it fails, the operation remains ineligible for + # transcript replay. + try: + finalized = await self._durable_bridge.finalize_operation_event_spool( + operation_id=context.operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + ) + if not finalized: + logger.debug( + "HTTP bridge operation spool finalization was fenced or ineligible operation_id=%s", + operation_id, + ) + except Exception: + logger.debug( + "Failed to finalize HTTP bridge operation event spool operation_id=%s", + operation_id, + exc_info=True, + ) + + async def append_terminal_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + event_text: str, + max_bytes: int, + state: str, + response_id: str | None = None, + ) -> bool: + """Drain queued events and atomically append the terminal outcome.""" + async with self._lock: + self._contexts.setdefault( + operation_id, + _PendingOperationEvent( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_text, + ), + ) + self._closing_operations.add(operation_id) + await self.flush_pending_operation(operation_id=operation_id) + async with self._lock: + context = self._contexts.get(operation_id) + dropped = operation_id in self._dropped_operations + if context is None: + return False + if dropped: + try: + await self._durable_bridge.update_operation( + operation_id=operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + state=state, + response_id=response_id, + ) + except Exception: + logger.debug( + "Failed to settle dropped terminal HTTP bridge operation_id=%s", + operation_id, + exc_info=True, + ) + finally: + async with self._lock: + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + return False + try: + persisted = await self._durable_bridge.append_terminal_operation_event( + operation_id=operation_id, + session_id=context.session_id, + instance_id=context.instance_id, + owner_epoch=context.owner_epoch, + event_text=event_text, + max_bytes=max_bytes, + state=state, + response_id=response_id, + ) + return bool(persisted and not dropped) + except Exception: + logger.debug( + "Failed to append terminal HTTP bridge event operation_id=%s", + operation_id, + exc_info=True, + ) + return False + finally: + async with self._lock: + self._closing_operations.discard(operation_id) + self._contexts.pop(operation_id, None) + self._dropped_operations.discard(operation_id) + + async def flush_pending_operation(self, *, operation_id: str) -> bool: + """Drain queued events while retaining the operation context.""" + while True: + await self._flush_one(operation_id) + async with self._lock: + has_pending = bool(self._pending.get(operation_id)) + if not has_pending: + break + async with self._lock: + return operation_id not in self._dropped_operations + + async def discard_operation(self, *, operation_id: str) -> None: + """Drop an abandoned nonterminal context without finalizing its spool.""" + async with self._flush_lock: + async with self._lock: + pending = self._pending.pop(operation_id, []) + self._pending_count -= len(pending) + self._pending_bytes -= sum(len(item.event_text.encode("utf-8")) for item in pending) + self._contexts.pop(operation_id, None) + self._closing_operations.discard(operation_id) + self._dropped_operations.discard(operation_id) + + async def close(self) -> None: + task = self._task + self._task = None + if task is not None: + task.cancel() + try: + await task + except asyncio.CancelledError: + pass diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 60bc435de9..ed3ed91013 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -650,6 +650,7 @@ from app.modules.proxy._service.websocket.helpers import ( _app_error_to_websocket_event, # noqa: F401 _assign_websocket_response_id, # noqa: F401 + _clear_websocket_stale_previous_response_cache, # noqa: F401 _draining_websocket_request_states, # noqa: F401 _find_websocket_request_state_by_response_id, # noqa: F401 _is_websocket_previous_response_output_item, # noqa: F401 @@ -731,6 +732,7 @@ _parse_openai_error, _upstream_error_from_openai, ) +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher from app.modules.proxy.http_bridge_forwarding import ( HTTPBridgeForwardContext as HTTPBridgeForwardContext, ) @@ -936,9 +938,11 @@ def __init__( self._live_websocket_connector = live_websocket_connector self._ring_membership = RingMembershipService(SessionLocal) self._durable_bridge = DurableBridgeSessionCoordinator(SessionLocal) + self._http_bridge_operation_event_batcher = HttpBridgeOperationEventBatcher.from_settings(self._durable_bridge) self._http_bridge_owner_client = HTTPBridgeOwnerClient() self._http_bridge_sessions: dict[_HTTPBridgeSessionKey, _HTTPBridgeSession] = {} - _initialize_http_bridge_retry_circuit(self) + _initialize_http_bridge_retry_circuit(self, _clear_websocket_stale_previous_response_cache) + self._http_bridge_account_timeout_failures, self._http_bridge_account_timeout_lock = {}, asyncio.Lock() self._http_bridge_inflight_sessions: dict[_HTTPBridgeSessionKey, asyncio.Future[_HTTPBridgeSession]] = {} self._http_bridge_turn_state_index: dict[tuple[str, str | None], _HTTPBridgeSessionKey] = {} self._http_bridge_previous_response_index: dict[tuple[str, str | None], _HTTPBridgeSessionKey] = {} @@ -947,12 +951,11 @@ def __init__( self._background_cleanup_tasks: set[asyncio.Task[None]] = set() self._stream_api_key_release_retry_semaphore = asyncio.Semaphore(_STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY) # In-memory pin from upstream-issued file_id -> codex-lb account_id. - # Used so ``finalize_file`` for a given ``file_id`` is routed to - # the same account that handled ``create_file``. Cross-instance - # routing is best-effort: if the finalize request lands on a - # different replica with no pin, we fall back to a fresh load- - # balancer selection. The TTL is short enough (5 min) that we - # never hold stale pins after the upstream upload window closes. + # Used so ``finalize_file`` for a given ``file_id`` is routed to the + # same account that handled ``create_file``. Cross-instance + # routing is best-effort: if finalize lands on a replica without a pin, we + # fall back to a fresh load-balancer selection. The TTL is short enough + # (5 min) that we never hold stale pins after the upstream upload window closes. self._file_account_pins: dict[str, _FilePinEntry] = {} self._file_account_pin_lock = asyncio.Lock() self._http_bridge_lock = anyio.Lock() diff --git a/app/modules/sticky_sessions/cleanup_scheduler.py b/app/modules/sticky_sessions/cleanup_scheduler.py index b5c17ab3ee..4f95a48dc6 100644 --- a/app/modules/sticky_sessions/cleanup_scheduler.py +++ b/app/modules/sticky_sessions/cleanup_scheduler.py @@ -78,12 +78,15 @@ def _abandoned_bridge_retention_seconds( class StickySessionCleanupScheduler: interval_seconds: int enabled: bool + # Durable bridge transcript retention is a data-safety obligation and must + # continue even when operators disable sticky-session mapping cleanup. + operation_retention_enabled: bool = True _task: asyncio.Task[None] | None = None _stop: asyncio.Event = field(default_factory=asyncio.Event) _lock: asyncio.Lock = field(default_factory=asyncio.Lock) async def start(self) -> None: - if not self.enabled: + if not self.enabled and not self.operation_retention_enabled: return if self._task and not self._task.done(): return @@ -117,49 +120,74 @@ async def _cleanup_as_leader(self) -> None: settings_repo = SettingsRepository(session) bridge_repo = DurableBridgeRepository(session) sticky_repo = StickySessionsRepository(session) - settings = await settings_repo.get_or_create() - - cutoff = utcnow() - timedelta(seconds=settings.openai_cache_affinity_max_age_seconds) - deleted_count = await sticky_repo.purge_prompt_cache_before(cutoff) - if deleted_count > 0: - logger.info("Purged stale prompt-cache sticky sessions deleted_count=%s", deleted_count) - cleanup_now = utcnow() - stale_hard_codex_session_cutoff = cleanup_now - timedelta( - seconds=_STALE_HARD_CODEX_SESSION_UNAVAILABLE_SECONDS - ) - stale_hard_codex_session_deleted_count = await sticky_repo.purge_stale_hard_codex_session_mappings( - stale_hard_codex_session_cutoff, now=cleanup_now - ) - if stale_hard_codex_session_deleted_count > 0: - logger.info( - "Purged stale hard codex_session sticky mappings pinned to a durably unavailable " - "owner deleted_count=%s", - stale_hard_codex_session_deleted_count, + settings = await settings_repo.get_or_create() if self.enabled else None + + if self.enabled: + assert settings is not None + cutoff = utcnow() - timedelta(seconds=settings.openai_cache_affinity_max_age_seconds) + deleted_count = await sticky_repo.purge_prompt_cache_before(cutoff) + if deleted_count > 0: + logger.info("Purged stale prompt-cache sticky sessions deleted_count=%s", deleted_count) + cleanup_now = utcnow() + stale_hard_codex_session_cutoff = cleanup_now - timedelta( + seconds=_STALE_HARD_CODEX_SESSION_UNAVAILABLE_SECONDS ) - if startup_module._bridge_durable_schema_ready or not await missing_durable_bridge_tables(session): - bridge_deleted_count = await bridge_repo.purge_closed_before(cutoff) - if bridge_deleted_count > 0: - logger.info("Purged closed HTTP bridge sessions deleted_count=%s", bridge_deleted_count) - abandoned_cutoff = utcnow() - timedelta( - seconds=_abandoned_bridge_retention_seconds(settings, get_settings()) - ) - abandoned_deleted_count = await bridge_repo.purge_abandoned_before(abandoned_cutoff) - if abandoned_deleted_count > 0: - logger.info( - "Purged abandoned HTTP bridge sessions deleted_count=%s", abandoned_deleted_count + stale_hard_codex_session_deleted_count = ( + await sticky_repo.purge_stale_hard_codex_session_mappings( + stale_hard_codex_session_cutoff, now=cleanup_now ) - retry_circuit_deleted_count = await bridge_repo.purge_retry_circuits_before( - time.time() - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS ) - if retry_circuit_deleted_count > 0: + if stale_hard_codex_session_deleted_count > 0: logger.info( - "Purged expired HTTP bridge retry circuits deleted_count=%s", - retry_circuit_deleted_count, + "Purged stale hard codex_session sticky mappings pinned to a durably unavailable " + "owner deleted_count=%s", + stale_hard_codex_session_deleted_count, + ) + if startup_module._bridge_durable_schema_ready or not await missing_durable_bridge_tables(session): + if self.enabled: + assert settings is not None + bridge_deleted_count = await bridge_repo.purge_closed_before(cutoff) + if bridge_deleted_count > 0: + logger.info("Purged closed HTTP bridge sessions deleted_count=%s", bridge_deleted_count) + abandoned_cutoff = utcnow() - timedelta( + seconds=_abandoned_bridge_retention_seconds(settings, get_settings()) + ) + abandoned_deleted_count = await bridge_repo.purge_abandoned_before(abandoned_cutoff) + if abandoned_deleted_count > 0: + logger.info( + "Purged abandoned HTTP bridge sessions deleted_count=%s", abandoned_deleted_count + ) + retry_circuit_deleted_count = await bridge_repo.purge_retry_circuits_before( + time.time() - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS + ) + if retry_circuit_deleted_count > 0: + logger.info( + "Purged expired HTTP bridge retry circuits deleted_count=%s", + retry_circuit_deleted_count, + ) + if self.operation_retention_enabled: + operation_cutoff = utcnow() - timedelta( + seconds=get_settings().http_responses_session_bridge_operation_spool_retention_seconds ) - ring_cutoff = utcnow() - timedelta(seconds=RING_MEMBER_RETENTION_SECONDS) - ring_deleted_count = await RingMembershipService(SessionLocal).purge_stale_before(ring_cutoff) - if ring_deleted_count > 0: - logger.info("Purged stale bridge ring members deleted_count=%s", ring_deleted_count) + operation_deleted_count = 0 + # Drain all eligible batches. A single startup pass is + # bounded to protect latency, but a long-lived process + # must continue pruning old prompt/output transcripts. + while True: + deleted_batch = await bridge_repo.purge_operation_spool(cutoff=operation_cutoff) + operation_deleted_count += deleted_batch + if deleted_batch < 500: + break + if operation_deleted_count > 0: + logger.info( + "Purged expired HTTP bridge operation transcript rows deleted_count=%s", + operation_deleted_count, + ) + if self.enabled: + ring_cutoff = utcnow() - timedelta(seconds=RING_MEMBER_RETENTION_SECONDS) + ring_deleted_count = await RingMembershipService(SessionLocal).purge_stale_before(ring_cutoff) + if ring_deleted_count > 0: + logger.info("Purged stale bridge ring members deleted_count=%s", ring_deleted_count) except Exception: logger.exception("Sticky session cleanup loop failed") diff --git a/docs/reference/settings.md b/docs/reference/settings.md index e7486b2321..5f570be225 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 118 settings. Every setting is an environment +codex-lb currently exposes 126 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -82,6 +82,7 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_ADVERTISE_BASE_URL` | `str \| None` | `None` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_AMBIGUOUS_CONTINUATION_RECOVERY_MODE` | `'fail_closed' \| 'client_full_history_once' \| 'server_anchored_replay_once' \| 'server_indefinite_recovery'` | `'fail_closed'` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_ANCHOR_POISON_FAILURE_THRESHOLD` | `int` | `7` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_CLEAN_CLOSE_RETRY_JITTER_MAX_SECONDS` | `float` | `2.0` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_CODEX_IDLE_TTL_SECONDS` | `float` | `900.0` | @@ -92,6 +93,13 @@ the host side of the compose `ports` mapping instead. | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_INSTANCE_ID` | `str` | process hostname | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_INSTANCE_RING` | `list[str]` | `[]` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_MAX_SESSIONS` | `int` | `256` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_BATCH_SIZE` | `int` | `32` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_FLUSH_INTERVAL_SECONDS` | `float` | `0.1` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_BYTES` | `int` | `2097152` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_PENDING_BYTES` | `int` | `33554432` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_EVENT_SPOOL_MAX_PENDING_EVENTS` | `int` | `2048` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_LEDGER_ENABLED` | `bool` | `True` | +| `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_OPERATION_SPOOL_RETENTION_SECONDS` | `float` | `604800` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_QUEUE_LIMIT` | `int` | `8` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_REQUEST_BUDGET_SECONDS` | `float` | `7200.0` | | `CODEX_LB_HTTP_RESPONSES_SESSION_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS` | `float` | `300.0` | @@ -310,4 +318,4 @@ issue [#1340](https://github.com/Soju06/codex-lb/issues/1340)): --- -*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* +*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* diff --git a/openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml b/openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml new file mode 100644 index 0000000000..878dc3156e --- /dev/null +++ b/openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-07 diff --git a/openspec/changes/durable-http-bridge-operation-recovery/context.md b/openspec/changes/durable-http-bridge-operation-recovery/context.md new file mode 100644 index 0000000000..1e1128e906 --- /dev/null +++ b/openspec/changes/durable-http-bridge-operation-recovery/context.md @@ -0,0 +1,41 @@ +## Context + +The bridge has no upstream idempotency/status endpoint, so an ambiguous turn +must be fenced locally. The durable operation row is the local proof. SQLite +and PostgreSQL deployments must therefore retain that row and its session while +another instance takes ownership. + +## Decisions + +- Scope the fingerprint hash with the normalized API-key scope instead of + changing the public request contract or exposing a new database column. +- Treat submitted, acknowledged, and unknown operations as recoverable; only + terminal rows may be removed by normal retention. +- Clear event rows and byte accounting in the same transaction that rebinds a + failed operation, so a retry has a fresh transcript. +- Require a matching fingerprint before using a completed sibling as a new + continuation anchor. A different request remains attached to its requested + parent. +- Use a no-op Alembic merge revision to converge the operation-ledger branch + with additive migrations already present on main. +- Treat the event spool as incomplete until the asynchronous batcher drains it; + SQLite's table default is rebuilt explicitly because SQLite does not support + a direct ALTER COLUMN operation. +- Run transcript retention from the existing leader-gated cleanup loop, + draining bounded repository batches without adding a new scheduler process; + transcript retention remains active when sticky mapping cleanup is disabled. +- Reset partial operation events before server-owned indefinite retries and + persist deferred reasoning blocks before the visible block they precede. + +## Failure modes + +- If durable persistence is unavailable, the bridge remains fail-closed rather + than dispatching an untracked duplicate. +- If a transcript is incomplete, recovery may not replay it; the existing + bounded retry policy remains authoritative. + +## Example + +Two API keys submit identical JSON against the same parent response. Their +normalized scopes produce distinct operation fingerprints, so neither request +can consume the other's completion or event spool. diff --git a/openspec/changes/durable-http-bridge-operation-recovery/proposal.md b/openspec/changes/durable-http-bridge-operation-recovery/proposal.md new file mode 100644 index 0000000000..f99b2e901c --- /dev/null +++ b/openspec/changes/durable-http-bridge-operation-recovery/proposal.md @@ -0,0 +1,30 @@ +## Why + +Ambiguous HTTP Responses bridge disconnects can leave an upstream +`response.create` in flight. Recovery needs a durable, tenant-scoped operation +identity and transcript that survives process ownership changes without +replaying stale terminal events or deleting recoverable rows during startup. + +## What Changes + +- Keep durable operation fingerprints isolated by API-key scope. +- Preserve sessions with submitted, acknowledged, or unknown operations during + startup takeover and detach their ownership for recovery. +- Reset an operation's event spool before rebinding an explicit failed retry. +- Only advance a continuation anchor when the completed sibling proves the same + logical request fingerprint. +- Keep the operation-ledger migration lineage converged with the current main + Alembic head. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: ambiguous HTTP bridge operations recover safely across + owner changes and retries. + +## Impact + +The proxy durable repository, HTTP bridge request submission, startup takeover, +Alembic graph, and focused recovery tests are affected. Public request and +response shapes remain unchanged. diff --git a/openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md b/openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..92e2b41de3 --- /dev/null +++ b/openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md @@ -0,0 +1,358 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Scoped operation identity + +The system MUST include the normalized API-key scope in every durable HTTP +bridge operation fingerprint and MUST apply that scope to fingerprint and +completed-operation lookups. + +#### Scenario: Equal requests from different keys remain isolated + +- **WHEN** two API keys submit the same logical request +- **THEN** each key receives an independent durable operation identity + +### Requirement: Recoverable startup takeover + +Startup cleanup MUST retain sessions that own submitted, acknowledged, or +unknown operations and MUST detach ownership before a replacement instance +takes over. + +#### Scenario: Restart preserves an in-flight operation + +- **WHEN** an instance restarts while an operation is nonterminal +- **THEN** cleanup detaches the old owner without deleting the operation spool + +### Requirement: Fresh retry transcript + +When an explicit failed operation is rebound, the system MUST atomically remove +the prior operation events and reset event-byte/spool state before accepting new +events. + +#### Scenario: Failed retry cannot replay stale failure output + +- **WHEN** a failed operation is retried and later completes +- **THEN** replay contains only the new attempt's events + +### Requirement: Proof-gated sibling anchoring + +The system MUST advance a continuation to a completed sibling response only +when the sibling has the same parent and logical request fingerprint in the +same API-key scope. + +#### Scenario: Distinct sibling input keeps its requested parent + +- **WHEN** a request reuses a parent with a different fingerprint +- **THEN** the service does not silently anchor it to another child response + +### Requirement: Single migration head + +The Alembic graph MUST converge the durable operation revisions with the current +release head and MUST expose one canonical head after upgrade. + +#### Scenario: Upgrade resolves one head + +- **WHEN** migrations are upgraded to the release tip +- **THEN** Alembic reports one canonical head + +### Requirement: Conservative spool defaults + +New operation rows MUST start with an incomplete event spool on SQLite and +PostgreSQL. A transcript MUST become replayable only after terminal event drain +and explicit finalization. + +#### Scenario: Nonterminal spool is not replayable + +- **WHEN** an operation has events but no finalized terminal event +- **THEN** recovery does not replay its transcript as complete + +### Requirement: Retain completed recovery transcripts + +Startup ownership cleanup MUST retain sessions with operation transcripts that +remain inside the configured operation retention window, including completed +operations, and MUST let normal spool retention remove the operation rows. + +#### Scenario: Recent completed transcript survives takeover + +- **WHEN** startup cleanup sees a recent completed transcript +- **THEN** it retains the session until normal retention expires it + +### Requirement: Continuous transcript retention + +Operation transcript cleanup MUST run periodically in a leader-gated scheduler +and MUST drain all eligible batches during each pass. Disabling the existing +sticky-session mapping cleanup switch MUST NOT disable operation transcript +retention; that switch MAY skip sticky mapping maintenance while durable +operation retention continues. + +#### Scenario: Retention drains all eligible batches + +- **WHEN** more rows are eligible than one deletion batch +- **THEN** one scheduler pass removes every eligible batch + +#### Scenario: Sticky cleanup toggle does not disable transcript retention + +- **WHEN** sticky-session cleanup is disabled and the durable bridge schema is + available +- **THEN** the leader-gated scheduler still drains expired operation transcript + rows while skipping sticky mapping cleanup + +### Requirement: Fresh indefinite-recovery spool + +Before dispatching a server-owned retry for a nonterminal operation, the system +MUST atomically clear any partial event spool under the durable owner fence. + +#### Scenario: Retry starts with a clean transcript + +- **WHEN** an anchored retry is dispatched after partial persistence +- **THEN** old events and byte counts are cleared before new output is accepted + +### Requirement: Ordered deferred reasoning persistence + +Deferred reasoning events released before a visible event MUST be persisted in +the same order in which they are delivered downstream, before the visible +event is persisted. + +#### Scenario: Deferred events preserve downstream order + +- **WHEN** buffered reasoning is released before visible output +- **THEN** the durable spool stores the reasoning blocks before that output + +### Requirement: Per-operation disconnect classification + +When a shared bridge websocket closes, each pending operation MUST be +classified from that operation's own observed response-event count. Activity +from a sibling request MUST NOT make an eventless operation safely retryable. + +#### Scenario: Sibling output does not acknowledge an eventless request + +- **WHEN** one pending request emitted output and another emitted none +- **THEN** the two operations receive different disconnect classifications + +### Requirement: Abandoned operation retention + +Operation retention MUST expire stale submitted and acknowledged rows in +addition to terminal and ambiguous rows, so a crashed or abandoned operation +cannot retain raw request data indefinitely. + +#### Scenario: Stale abandoned request is purged + +- **WHEN** a submitted operation exceeds retention age +- **THEN** its request data and event spool are removed + +### Requirement: Acknowledged alias persistence failure + +If upstream has acknowledged a response but local continuity-alias persistence +fails, the downstream error MUST NOT transition the durable operation to a +retryable failed state. The operation MUST remain acknowledged/ambiguous so an +identical retry cannot dispatch a duplicate upstream turn. + +#### Scenario: Alias write failure remains fail-closed + +- **WHEN** an acknowledged response cannot publish its continuity alias +- **THEN** the operation remains non-retryable and the client receives a terminal error + +### Requirement: Cross-session nonterminal handoff + +When a scoped operation fingerprint is found under a different durable +session, a nonterminal operation MUST be atomically rebound to the currently +owned session before its event spool is reset or a recovery attempt is sent. +Completed replayable operations MUST remain attached to their original session. +The handoff MUST be refused while the prior session has an unexpired owner +lease, preventing concurrent owners from dispatching the same turn. + +#### Scenario: Active prior owner fences handoff + +- **WHEN** a duplicate request finds a nonterminal operation under another session +- **AND** that session still has an unexpired owner lease +- **THEN** the operation remains with the prior session and no concurrent retry is dispatched + +#### Scenario: Expired prior owner permits handoff + +- **WHEN** the prior session lease is absent or expired +- **THEN** the operation can be atomically rebound before recovery + +### Requirement: Fenced one-shot recovery dispatch + +The durable recovery journal MUST persist a one-shot replay budget for every +recovery-safe request. The budget MUST be consumed atomically when a replay is +claimed for dispatch, and a caller that proves the replay never reached the +upstream send boundary MUST restore that claim under the same session owner +fence. A replacement session MUST retain or transfer a fenced origin owner +until the claim is rolled back or settled; selecting a replacement or failing +preflight MUST NOT permanently consume an unsent replay. + +#### Scenario: Concurrent reconnects consume one replay + +- **WHEN** concurrent reconnects observe the same ambiguous operation +- **THEN** exactly one owner atomically claims the persisted replay budget and + other reconnects fail closed without dispatching a duplicate + +#### Scenario: Pre-dispatch replacement failure restores the budget + +- **WHEN** a replay claim is made but replacement admission or preflight fails + before the exact upstream frame is sent +- **THEN** the claim returns to the available state and the fenced origin + owner is released only after that rollback succeeds + +#### Scenario: Successful replacement settles the origin journal + +- **WHEN** a replacement session dispatches the claimed replay and receives a + terminal response event +- **THEN** settlement uses the retained origin owner fence before releasing it + and the replay budget cannot be claimed again + +### Requirement: Lease-aware operation retention + +Retention MUST NOT delete stale submitted or acknowledged operations while +their session is actively owned with an unexpired lease. The owner/lease +predicate MUST be rechecked in the deletion transaction. + +#### Scenario: Active lease protects stale operation + +- **WHEN** a stale operation belongs to a session with a live lease +- **THEN** retention leaves it intact + +### Requirement: Anchored indefinite recovery gate + +The server-indefinite recovery loop MUST be installed only for an eventless +anchored continuation with a durable parent operation. Fresh first-turn +requests and streams that already emitted downstream response events MUST +terminate normally rather than being resent indefinitely. + +#### Scenario: Fresh request is not held indefinitely + +- **WHEN** a first-turn request loses its upstream connection +- **THEN** the proxy returns its normal error path without an indefinite loop + +### Requirement: Retry reservation terminalization + +If reacquiring API-key usage limits for a recovery attempt fails, the proxy +MUST settle the prior reservation and emit a terminal `response.failed` SSE +event instead of aborting the already-started stream. + +#### Scenario: Quota failure produces terminal SSE + +- **WHEN** a recovery retry cannot reacquire its usage reservation +- **THEN** the client receives `response.failed` and the prior reservation is settled + +#### Scenario: Unexpected admission failure produces terminal SSE + +- **WHEN** recovery admission raises an unexpected infrastructure error before + a replacement stream starts +- **THEN** the client receives `response.failed` and the prior reservation is + settled instead of receiving a truncated stream + +### Requirement: Failure spool/state ordering + +For an explicit deterministic failure, the proxy MUST persist the terminal SSE +block before exposing the durable operation as failed. The event append and +failed-state transition MUST use the same owner fence and transaction when the +durable repository supports it. + +#### Scenario: Concurrent retry cannot reset an unspooled failure + +- **WHEN** a response failure is being settled while an identical reconnect is + admitted +- **THEN** the reconnect observes the terminal operation fence and cannot reset + or mix the previous failure into a new transcript + +### Requirement: Partial disconnect acknowledgement + +When a bridge disconnects after an operation has emitted any response event but +before a terminal event, the durable operation MUST remain acknowledged or +ambiguous. It MUST NOT be classified as retryable failed solely because the +disconnect was non-terminal. + +#### Scenario: Partial output is never resent as a fresh turn + +- **WHEN** the upstream closes after `response.created` but before completion +- **THEN** the operation remains non-retryable + +### Requirement: Retry output stops indefinite recovery + +An indefinite recovery attempt MUST stop retrying once that attempt emits any +downstream response event, even if the attempt later fails with a retryable +transport error. + +#### Scenario: Retry output prevents a second attempt + +- **WHEN** a retry emits a data event and then times out +- **THEN** the server stops the indefinite loop instead of appending another response + +### Requirement: Preserve repeated event occurrences + +The durable event spool MUST preserve repeated identical SSE blocks as distinct +ordered occurrences. Event identity MUST include its operation-local sequence +position rather than content alone. + +#### Scenario: Identical deltas replay twice + +- **WHEN** two consecutive SSE blocks have identical text +- **THEN** both occurrences are present in the replay transcript + +### Requirement: Stop event persistence during shutdown + +Proxy shutdown MUST close the HTTP bridge event batcher and cancel its +background flusher before the process exits. + +#### Scenario: Shutdown cancels the flusher + +- **WHEN** the proxy service begins shutdown after queueing an event +- **THEN** the batcher's background task is cancelled and awaited + +### Requirement: Classify response.incomplete as terminal + +An anchored `response.incomplete` event MUST transition the durable operation to +an explicit terminal state and finalize its transcript so it is not left in an +unknown in-flight state. + +#### Scenario: Incomplete response is replayable as terminal + +- **WHEN** upstream emits `response.incomplete` +- **THEN** the operation is terminalized and its drained transcript is eligible for replay + +### Requirement: Settle reservations before timeout health + +When an eventless timeout retires a keyed bridge, the proxy MUST settle all +pending request reservations before recording the account timeout health signal. +If settlement fails, the health signal MUST NOT claim that cleanup completed. + +#### Scenario: Failed reservation release does not poison health state + +- **WHEN** the timeout cleanup cannot release a pending reservation +- **THEN** the account timeout signal is not recorded before that failure is surfaced + +### Requirement: Replay finalized incomplete operations + +A finalized `incomplete` operation transcript MUST be replayed for an identical +request and MUST NOT be reset or treated as an unknown in-flight operation. + +#### Scenario: Reconnect receives stored incomplete transcript + +- **WHEN** an identical request finds a finalized incomplete operation +- **THEN** the stored terminal transcript is delivered without a new upstream dispatch + +### Requirement: Validate final response.create size + +After adding durable operation metadata, the proxy MUST revalidate the exact +serialized `response.create` frame against the upstream size limit before +sending it. + +#### Scenario: Metadata cannot create an oversized frame + +- **WHEN** operation metadata makes the final frame exceed the configured limit +- **THEN** the request is rejected or slimmed before any upstream send + +### Requirement: Fence same-session active operations + +Server-indefinite recovery MUST NOT reset or redispatch a nonterminal operation +when another pending request in the same durable session still references that +operation. Submitted and acknowledged operations MUST remain fail-closed; +only an inactive `unknown` operation may enter a fresh recovery attempt. + +#### Scenario: Active same-session operation is not duplicated + +- **WHEN** a duplicate request finds a submitted operation still referenced by another pending request +- **THEN** the proxy refuses a second dispatch and preserves the existing spool diff --git a/openspec/changes/durable-http-bridge-operation-recovery/tasks.md b/openspec/changes/durable-http-bridge-operation-recovery/tasks.md new file mode 100644 index 0000000000..8c4b47e7b8 --- /dev/null +++ b/openspec/changes/durable-http-bridge-operation-recovery/tasks.md @@ -0,0 +1,49 @@ +## 1. Implementation + +- [x] 1.1 Scope operation fingerprints and lookups by API-key namespace. +- [x] 1.2 Preserve recoverable operation sessions during startup takeover. +- [x] 1.3 Reset failed-operation event spools atomically. +- [x] 1.4 Gate sibling continuation anchoring on matching fingerprints. +- [x] 1.5 Merge the operation-ledger migration lineage with latest main. +- [x] 1.6 Keep SQLite event-spool defaults conservative and explicit. +- [x] 1.7 Retain completed transcripts through startup takeover and drain + periodic retention batches. +- [x] 1.8 Reset partial spools before indefinite recovery retries. +- [x] 1.9 Persist deferred reasoning events in downstream order. +- [x] 1.10 Classify shared-websocket disconnects per operation event count. +- [x] 1.11 Expire stale submitted and acknowledged operation rows. +- [x] 1.12 Preserve acknowledged state after alias persistence failure. +- [x] 1.13 Rebind nonterminal cross-session operations before recovery reset. +- [x] 1.14 Protect actively leased operations during retention cleanup. +- [x] 1.15 Keep event-spool settings compatible with legacy test doubles. +- [x] 1.16 Gate indefinite recovery to eventless anchored operations. +- [x] 1.17 Convert recovery reservation failures into terminal SSE events. +- [x] 1.18 Preserve acknowledged state after partial response output and disconnect. +- [x] 1.19 Stop indefinite recovery after a retry attempt emits downstream output. +- [x] 1.20 Include sequence position in event fingerprints so repeated SSE blocks survive replay. +- [x] 1.21 Close the event batcher flusher from the proxy shutdown path. +- [x] 1.22 Refuse cross-session handoff while the prior session lease is active. +- [x] 1.23 Keep eventless local transport failures retryable in indefinite recovery. +- [x] 1.24 Terminalize and persist `response.incomplete` operation outcomes. +- [x] 1.25 Place all response-compatibility requirements in the capability delta path. +- [x] 1.26 Record timeout health only after pending reservation settlement. +- [x] 1.27 Replay finalized incomplete operations without resetting their terminal spool. +- [x] 1.28 Return reservation settlement status to timeout health handling. +- [x] 1.29 Revalidate the final response.create frame after operation metadata injection. +- [x] 1.30 Require an inactive unknown operation before same-session recovery reset. +- [x] 1.31 Keep operation transcript retention active when sticky mapping cleanup is disabled. +- [x] 1.32 Persist and fence the one-shot recovery dispatch budget through + replacement-session handoff, rollback, and terminal settlement. +- [x] 1.33 Restore claimed recovery operations on every pre-admission exit and + atomically spool deterministic terminal failures before exposing `failed`. + +## 2. Validation + +- [x] 2.1 Add or update focused repository and request-submit regressions. +- [x] 2.2 Run focused HTTP bridge tests, Ruff, Ty, diff checks, and strict + OpenSpec validation. + - Evidence: focused HTTP bridge/API tests, Ruff, Ty, migration checks, and + strict OpenSpec validation passed after the recovery-budget handoff fix. +- [x] 2.3 Verify disabled sticky cleanup still runs durable transcript retention. +- [x] 2.4 Add regressions for pre-admission claim restoration and terminal + failure spool/state ordering. diff --git a/scripts/generate_settings_reference.py b/scripts/generate_settings_reference.py index d3c23c5222..18ba3abdc3 100644 --- a/scripts/generate_settings_reference.py +++ b/scripts/generate_settings_reference.py @@ -260,6 +260,8 @@ def render_settings_reference() -> str: "", "*Specs: [user-documentation]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · " + "[responses-api-compat]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · " "[deployment-installation]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)*", "", diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 2007188cdc..5a7f1cf7db 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -4,6 +4,7 @@ import base64 import contextlib import json +import socket import time from collections import deque from collections.abc import AsyncGenerator @@ -7061,7 +7062,11 @@ async def fake_submit_http_bridge_request( @pytest.mark.asyncio async def test_v1_responses_http_bridge_reconnects_after_clean_upstream_close(async_client, monkeypatch): - _install_bridge_settings(monkeypatch, enabled=True) + # The app lifespan registers the process hostname in the durable bridge + # ring before this test installs its settings. Keep the test on that same + # instance so the startup heartbeat cannot make the reconnect path look + # like a cross-replica ownership conflict. + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) account_id = await _import_account(async_client, "acc_http_bridge_reconnect", "http-bridge-reconnect@example.com") account = await _get_account(account_id) first_upstream = _ClosingBridgeUpstreamWebSocket() @@ -7137,7 +7142,10 @@ async def fail_legacy_stream(*args, **kwargs): "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", - "prompt_cache_key": "http-bridge-reconnect-thread-1", + # Scope the soft-affinity key to this test's account so a parallel or + # ordered integration run cannot inherit another instance's durable + # owner and turn the reconnect assertion into a 409 race. + "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}", } first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) second = await asyncio.wait_for( @@ -12906,7 +12914,12 @@ async def fake_connect_responses_websocket( ) assert second.status_code == 502 - assert second.json()["error"]["code"] in ("upstream_unavailable", "stream_incomplete", "bridge_owner_unreachable") + assert second.json()["error"]["code"] in ( + "upstream_unavailable", + "stream_incomplete", + "bridge_owner_unreachable", + "bridge_continuity_persistence_failed", + ) assert "previous_response_not_found" not in second.json()["error"].get("code", "") assert connect_count == 1 @@ -14418,7 +14431,7 @@ async def test_v1_responses_http_bridge_quarantines_reattach_that_streams_withou ``response.created`` must quarantine the session so the next request does not rebuild the identical anchored reattach and instead completes on the fresh no-anchor path.""" - _install_bridge_settings(monkeypatch, enabled=True) + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) account_id = await _import_account( async_client, "acc_http_bridge_quarantine_silent", @@ -14587,7 +14600,7 @@ async def test_v1_responses_http_bridge_quarantined_unsafe_full_resend_dispatche hydration restored ``last_completed_response_id`` and the session-level injection re-added the same anchor and trimmed the prefix — rebuilding the wedge despite the ``fresh_reattach_anchor_skipped_quarantined`` log.""" - _install_bridge_settings(monkeypatch, enabled=True) + _install_bridge_settings_with_limits(monkeypatch, enabled=True, instance_id=socket.gethostname()) account_id = await _import_account( async_client, "acc_http_bridge_quarantine_unsafe_suffix", diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index b7c11cebcb..c0761a6b9a 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -8751,7 +8751,10 @@ async def get(self): runtime_settings = _websocket_settings( proxy_downstream_websocket_idle_timeout_seconds=0.1, - stream_idle_timeout_seconds=0.2, + # Keep the upstream stream budget above both delayed messages. The + # assertion targets the downstream idle guard, not an upstream idle + # timeout; a slower CI runner must not turn the fixture into a race. + stream_idle_timeout_seconds=0.5, ) async def allow_firewall(_websocket): diff --git a/tests/unit/test_bridge_ring_lifecycle.py b/tests/unit/test_bridge_ring_lifecycle.py index 744cd6c51f..895c5d2ab8 100644 --- a/tests/unit/test_bridge_ring_lifecycle.py +++ b/tests/unit/test_bridge_ring_lifecycle.py @@ -22,6 +22,7 @@ AccountStatus, Base, BridgeRingMember, + HttpBridgeOperationRecord, HttpBridgeRetryCircuit, HttpBridgeSessionAlias, HttpBridgeSessionRecord, @@ -36,6 +37,7 @@ DurableBridgeAliasRegistration, DurableBridgeRepository, durable_bridge_hash, + durable_bridge_operation_id, ) from app.modules.proxy.ring_membership import RingMembershipService @@ -565,6 +567,844 @@ async def test_recovery_attempt_pre_dispatch_claim_can_be_rolled_back( ) assert restored is not None assert restored.state.value == "unknown" + assert await repository.rollback_recovery_attempt_before_dispatch( + session_id=claim.id, + instance_id="inst-recovery-rollback", + owner_epoch=claim.owner_epoch, + request_fingerprint="fingerprint-recovery-rollback", + ) + assert ( + await repository.lookup_recovery_attempt( + session_id=claim.id, + request_fingerprint="fingerprint-recovery-rollback", + ) + is None + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_ledger_is_fenced_and_idempotent( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-ledger", session_key_value="sid-operation") + fingerprint = durable_bridge_hash("continuation-body") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + created = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + request_text='{"model":"gpt-5.6","input":"turn"}', + ) + assert created is not None + assert created.created is True + assert created.state == "submitted" + assert created.request_text == '{"model":"gpt-5.6","input":"turn"}' + assert created.event_spool_complete is False + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert existing is not None + assert existing.created is False + assert existing.operation_id == operation_id + + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + completed = await repository.get_latest_completed_operation( + session_id=claim.id, + parent_response_id="resp-parent", + ) + assert completed is not None + assert completed.response_id == "resp-completed" + by_fingerprint = await repository.get_operation_by_fingerprint(request_fingerprint=fingerprint) + assert by_fingerprint is not None + assert by_fingerprint.operation_id == operation_id + cross_session_completed = await repository.get_latest_completed_operation_any_session( + parent_response_id="resp-parent", + ) + assert cross_session_completed is not None + assert cross_session_completed.response_id == "resp-completed" + + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.completed"}\n\n', + max_bytes=1024, + ) + # Repeated identical SSE blocks are distinct downstream occurrences, + # so replay must preserve both copies rather than hash-deduplicating. + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-ledger", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.completed"}\n\n', + max_bytes=1024, + ) + assert await repository.get_operation_events(operation_id=operation_id) == [ + 'data: {"type":"response.completed"}\n\n', + 'data: {"type":"response.completed"}\n\n', + ] + # A missing parent turn makes the chain ineligible rather than + # silently constructing an incomplete conversation. + assert await repository.get_replayable_transcript(response_id="resp-completed") is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_retry_reset_clears_partial_spool( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-reset", session_key_value="sid-operation-reset") + fingerprint = durable_bridge_hash("continuation-reset") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.output_text.delta"}\n\n', + max_bytes=1024, + ) + assert await repository.reset_operation_event_spool( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-reset", + owner_epoch=claim.owner_epoch, + ) + assert await repository.get_operation_events(operation_id=operation_id) == [] + reset = await repository.get_operation(operation_id=operation_id) + assert reset is not None + assert reset.event_spool_complete is False + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_terminal_operation_event_exposes_failure_after_spooling( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-terminal-event", session_key_value="sid-terminal-event") + fingerprint = durable_bridge_hash("terminal-event") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-event", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-event", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + event_text = 'data: {"type":"response.failed"}\n\n' + + assert await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-event", + owner_epoch=claim.owner_epoch, + event_text=event_text, + max_bytes=1024, + state="failed", + ) + failed = await repository.get_operation(operation_id=operation_id) + assert failed is not None + assert failed.state == "failed" + assert await repository.get_operation_events(operation_id=operation_id) == [event_text] + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_terminal_failure_exposes_state_when_spool_overflows( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-terminal-overflow", + session_key_value="sid-terminal-overflow", + ) + fingerprint = durable_bridge_hash("terminal-overflow") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-overflow", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-overflow", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + + persisted = await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-overflow", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1, + state="failed", + ) + + assert persisted is False + failed = await repository.get_operation(operation_id=operation_id) + assert failed is not None + assert failed.state == "failed" + assert failed.event_spool_complete is False + assert await repository.get_operation_events(operation_id=operation_id) == [] + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_consumed_recovery_checkpoint_does_not_rebind_failed_operation( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-consumed-original", + session_key_value="sid-consumed-original", + ) + replacement = await _claim( + repository, + instance_id="inst-consumed-replacement", + session_key_value="sid-consumed-replacement", + ) + fingerprint = durable_bridge_hash("consumed-failed-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-consumed-original", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-consumed", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_terminal_operation_event( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-consumed-original", + owner_epoch=original.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + ) + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-consumed-replacement", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + recovery_attempt_consumed=True, + ) + + assert existing is not None + assert existing.created is False + assert existing.session_id == original.id + assert existing.state == "failed" + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.session_id == original.id + assert persisted.state == "failed" + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_unknown_operation_recovery_claim_is_atomic_and_single_use( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-operation-claim", session_key_value="sid-operation-claim") + fingerprint = durable_bridge_hash("continuation-claim") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.output_text.delta"}\n\n', + max_bytes=1024, + ) + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + claimed = await repository.get_operation(operation_id=operation_id) + assert claimed is not None + assert claimed.state == "submitted" + assert claimed.response_id is None + assert claimed.event_spool_complete is False + assert await repository.get_operation_events(operation_id=operation_id) == [] + + # The state transition is the claim: a concurrent reconnect that gets + # the write lock later cannot reset and submit the same operation. + assert not await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-claim", + owner_epoch=claim.owner_epoch, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_one_shot_recovery_budget_survives_unknown_reset( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-one-shot", + session_key_value="sid-operation-one-shot", + ) + fingerprint = durable_bridge_hash("continuation-one-shot") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + ) + + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + # A failed or ambiguous dispatch may return the operation to UNKNOWN, + # but that must not refund the durable one-shot recovery budget. + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + ) + assert not await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-one-shot", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.state == "unknown" + assert persisted.recovery_dispatch_count == 1 + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_pre_dispatch_recovery_claim_restores_one_shot_budget( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-refund", + session_key_value="sid-operation-refund", + ) + fingerprint = durable_bridge_hash("continuation-refund") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + + # A cancellation before send_text() is proven pre-dispatch and must + # refund the claim so the next reconnect can make the one safe retry. + assert await repository.mark_operation_unknown( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + restore_recovery_dispatch_claim=True, + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-refund", + owner_epoch=claim.owner_epoch, + max_recovery_dispatches=1, + ) + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.recovery_dispatch_count == 1 + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_pre_dispatch_operation_rollback_removes_only_empty_new_row( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-operation-rollback", + session_key_value="sid-operation-rollback", + ) + fingerprint = durable_bridge_hash("operation-rollback") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-rollback", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None and operation.created is True + assert await repository.rollback_operation_before_dispatch( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-operation-rollback", + owner_epoch=claim.owner_epoch, + ) + assert await repository.get_operation(operation_id=operation_id) is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_operation_spool_purge_expires_stale_nonterminal_rows( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-stale-operation", session_key_value="sid-stale-operation") + fingerprint = durable_bridge_hash("stale-operation") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-stale-operation", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id=None, + request_text='{"input":"stale"}', + ) + stale_at = datetime.now(timezone.utc).replace(tzinfo=None) - timedelta(days=8) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-stale-operation", + owner_epoch=claim.owner_epoch, + state="unknown", + ) + await session.execute( + update(HttpBridgeOperationRecord) + .where(HttpBridgeOperationRecord.operation_id == operation_id) + .values(updated_at=stale_at) + ) + await session.commit() + + # A stale timestamp alone must not delete an UNKNOWN operation whose + # session is still owned and leased; it may be a long-running recovery + # request whose duplicate-suppression fence must remain intact. + assert await repository.purge_operation_spool(cutoff=datetime.now(timezone.utc).replace(tzinfo=None)) == 0 + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == claim.id) + .values(owner_instance_id=None, lease_expires_at=None) + ) + await session.commit() + assert await repository.purge_operation_spool(cutoff=datetime.now(timezone.utc).replace(tzinfo=None)) == 1 + assert await repository.get_operation(operation_id=operation_id) is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_nonterminal_operation_rebinds_before_cross_session_recovery_reset( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-original-operation", + session_key_value="sid-original-operation", + ) + replacement = await _claim( + repository, + instance_id="inst-replacement-operation", + session_key_value="sid-replacement-operation", + ) + fingerprint = durable_bridge_hash("cross-session-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-original-operation", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + request_text='{"input":"cross-session"}', + ) + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == original.id) + .values(owner_instance_id=None, lease_expires_at=None) + ) + await session.commit() + rebound = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-replacement-operation", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert rebound is not None + assert rebound.session_id == replacement.id + assert await repository.reset_operation_event_spool( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-replacement-operation", + owner_epoch=replacement.owner_epoch, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_nonterminal_operation_does_not_rebind_from_live_prior_owner( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + original = await _claim( + repository, + instance_id="inst-live-original-operation", + session_key_value="sid-live-original-operation", + ) + replacement = await _claim( + repository, + instance_id="inst-live-replacement-operation", + session_key_value="sid-live-replacement-operation", + ) + fingerprint = durable_bridge_hash("live-cross-session-operation") + operation_id = durable_bridge_operation_id(original.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id="inst-live-original-operation", + owner_epoch=original.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + + existing = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id="inst-live-replacement-operation", + owner_epoch=replacement.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + + assert existing is not None + assert existing.session_id == original.id + persisted = await repository.get_operation(operation_id=operation_id) + assert persisted is not None + assert persisted.session_id == original.id + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_recovery_handoff_rebinds_operation_while_origin_journal_stays_fenced( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + instance_id = "inst-recovery-handoff" + original = await _claim( + repository, + instance_id=instance_id, + session_key_value="sid-recovery-origin", + ) + replacement = await _claim( + repository, + instance_id=instance_id, + session_key_value="sid-recovery-replacement", + ) + operation_fingerprint = durable_bridge_hash("recovery-handoff-operation") + operation_id = durable_bridge_operation_id(original.id, operation_fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=operation_fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + recovery_fingerprint = durable_bridge_hash("recovery-handoff-request") + attempt = await repository.record_recovery_attempt( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + request_id="request-recovery-handoff", + account_id="account-operation", + model="gpt-5.6", + replay_safe=True, + ) + assert attempt is not None + assert await repository.mark_recovery_attempt_replayed( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + ) + + rebound = await repository.record_operation( + operation_id=operation_id, + session_id=replacement.id, + instance_id=instance_id, + owner_epoch=replacement.owner_epoch, + request_fingerprint=operation_fingerprint, + account_id="account-replacement", + model="gpt-5.6", + parent_response_id="resp-parent", + recovery_attempt_session_id=original.id, + recovery_attempt_owner_epoch=original.owner_epoch, + recovery_attempt_fingerprint=recovery_fingerprint, + ) + assert rebound is not None + assert rebound.session_id == replacement.id + origin = await repository.get_session_by_id(original.id) + assert origin is not None + assert origin.owner_instance_id == instance_id + assert await repository.rollback_recovery_attempt_replayed( + session_id=original.id, + instance_id=instance_id, + owner_epoch=original.owner_epoch, + request_fingerprint=recovery_fingerprint, + ) + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_startup_retains_completed_operation_session( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-completed-retain", session_key_value="sid-completed-retain") + fingerprint = durable_bridge_hash("completed-retain") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-completed-retain", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-completed-retain", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + assert await repository.purge_owned_sessions_on_startup(instance_id="inst-completed-retain") == 0 + retained = await repository.get_operation(operation_id=operation_id) + assert retained is not None + owner = await repository.get_session_by_id(claim.id) + assert owner is not None + assert owner.owner_instance_id is None + finally: + await session.close() + + +@pytest.mark.asyncio +async def test_startup_retains_completed_operation_session_across_process_epoch( + async_session_factory: Callable[[], AsyncSession], +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim(repository, instance_id="inst-epoch-retain", session_key_value="sid-epoch-retain") + fingerprint = durable_bridge_hash("epoch-retain") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-epoch-retain", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-operation", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-epoch-retain", + owner_epoch=claim.owner_epoch, + state="completed", + response_id="resp-completed", + ) + assert ( + await repository.purge_owned_sessions_on_startup( + instance_id="inst-epoch-retain", + owner_process_epoch="new-process", + ) + == 0 + ) + owner = await repository.get_session_by_id(claim.id) + assert owner is not None + assert owner.owner_instance_id is None + assert owner.owner_process_epoch == "test-process" finally: await session.close() diff --git a/tests/unit/test_db_migrate.py b/tests/unit/test_db_migrate.py index 92ab87f8ed..53493dde30 100644 --- a/tests/unit/test_db_migrate.py +++ b/tests/unit/test_db_migrate.py @@ -2219,6 +2219,100 @@ def test_connection_request_kind_migration_is_additive_without_backfill(tmp_path engine.dispose() +def test_http_bridge_operation_migrations_round_trip_existing_rows_and_rebuild_sqlite_defaults( + tmp_path: Path, +) -> None: + db_path = tmp_path / "http-bridge-operation-round-trip.db" + url = _db_url(db_path) + parent_revision = "20260804_000001_add_global_http_bridge_operation_fingerprint" + spool_revision = "20260805_000001_finalize_http_bridge_operation_spool" + + run_upgrade(url, parent_revision, bootstrap_legacy=False) + config = _build_alembic_config(url) + engine = create_engine(to_sync_database_url(url)) + try: + with engine.begin() as connection: + connection.execute( + text( + """ + INSERT INTO http_bridge_sessions ( + id, session_key_kind, session_key_value, session_key_hash, api_key_scope, + owner_epoch, state, last_seen_at, created_at, updated_at + ) + VALUES ( + 'migration-operation-session', 'session_header', 'migration-operation-key', + 'migration-operation-hash', '__anonymous__', 1, 'active', CURRENT_TIMESTAMP, + CURRENT_TIMESTAMP, CURRENT_TIMESTAMP + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO http_bridge_operations ( + operation_id, session_id, request_fingerprint, account_id, model, + parent_response_id, state, response_id + ) + VALUES ( + 'migration-operation', 'migration-operation-session', 'migration-fingerprint', + NULL, 'gpt-5.6', 'migration-parent', 'submitted', NULL + ) + """ + ) + ) + + command.upgrade(config, spool_revision) + with engine.connect() as connection: + inspector = inspect(connection) + operation_columns = {column["name"]: column for column in inspector.get_columns("http_bridge_operations")} + assert {"request_text", "event_bytes", "event_spool_complete"} <= operation_columns.keys() + row = connection.execute( + text( + """ + SELECT request_text, event_bytes, event_spool_complete + FROM http_bridge_operations + WHERE operation_id = 'migration-operation' + """ + ) + ).one() + assert row == (None, 0, False) + assert inspector.has_table("http_bridge_operation_events") + + command.downgrade(config, parent_revision) + with engine.connect() as connection: + inspector = inspect(connection) + assert inspector.has_table("http_bridge_operations") + assert not inspector.has_table("http_bridge_operation_events") + assert ( + connection.execute( + text( + "SELECT request_fingerprint FROM http_bridge_operations " + "WHERE operation_id = 'migration-operation'" + ) + ).scalar_one() + == "migration-fingerprint" + ) + + command.upgrade(config, "head") + with engine.connect() as connection: + inspector = inspect(connection) + operation_columns = {column["name"] for column in inspector.get_columns("http_bridge_operations")} + assert {"request_text", "event_bytes", "event_spool_complete"} <= operation_columns + assert ( + connection.execute( + text( + "SELECT event_spool_complete FROM http_bridge_operations " + "WHERE operation_id = 'migration-operation'" + ) + ).scalar_one() + == 0 + ) + assert inspector.has_table("http_bridge_operation_events") + finally: + engine.dispose() + + def test_check_schema_drift_detects_missing_dashboard_hot_path_indexes(tmp_path: Path) -> None: db_path = tmp_path / "missing-hot-path-indexes.db" url = _db_url(db_path) diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index dd154d75c1..38935cf149 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -28,10 +28,12 @@ is_http_bridge_account_neutral_replay, make_http_bridge_account_neutral_replay_key, ) -from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeLookup, DurableBridgeSessionCoordinator from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeRepository, + durable_bridge_hash, + durable_bridge_operation_id, ) pytestmark = pytest.mark.unit @@ -2226,6 +2228,24 @@ async def test_durable_bridge_lookup_active_lease_survives_request_lookup( assert lookup.lease_is_active(now=utcnow()) is True +def test_durable_bridge_lookup_lease_accepts_offset_aware_timestamp() -> None: + lookup = DurableBridgeLookup( + session_id="session-aware-lease", + canonical_kind="session_header", + canonical_key="sid-aware-lease", + api_key_scope="anonymous", + account_id="acc-1", + owner_instance_id="instance-a", + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(minutes=1), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state=None, + latest_response_id=None, + ) + + assert lookup.lease_is_active(now=utcnow()) is True + + @pytest.mark.asyncio async def test_durable_bridge_lookup_falls_back_to_latest_turn_state_when_alias_missing( coordinator: DurableBridgeSessionCoordinator, @@ -2434,6 +2454,57 @@ async def test_startup_purges_owned_bridge_rows( assert sticky is not None +@pytest.mark.asyncio +async def test_startup_reclassifies_submitted_operation_for_recovery( + coordinator: DurableBridgeSessionCoordinator, + async_session_factory: Callable[[], AsyncSession], +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="session_header", + session_key_value="sid-submitted-recovery", + api_key_id=None, + instance_id="instance-submitted-recovery", + owner_process_epoch="old-process", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.6", + service_tier=None, + latest_turn_state="turn-state", + latest_response_id=None, + allow_takeover=True, + ) + fingerprint = durable_bridge_hash("submitted-recovery") + operation_id = durable_bridge_operation_id(claimed.session_id, fingerprint) + async with async_session_factory() as session: + repository = DurableBridgeRepository(session) + assert await repository.record_operation( + operation_id=operation_id, + session_id=claimed.session_id, + instance_id="instance-submitted-recovery", + owner_epoch=claimed.owner_epoch, + request_fingerprint=fingerprint, + account_id="acc-1", + model="gpt-5.6", + parent_response_id=None, + ) + await session.execute( + update(HttpBridgeSessionRecord) + .where(HttpBridgeSessionRecord.id == claimed.session_id) + .values(last_seen_at=utcnow() - timedelta(minutes=5)) + ) + await session.commit() + + deleted = await repository.purge_owned_sessions_on_startup( + instance_id="instance-submitted-recovery", + owner_process_epoch="new-process", + ) + + assert deleted == 0 + operation = await repository.get_operation(operation_id=operation_id) + assert operation is not None + assert operation.state == "unknown" + + @pytest.mark.asyncio async def test_startup_closes_same_instance_previous_process_epoch_rows( coordinator: DurableBridgeSessionCoordinator, @@ -2743,6 +2814,7 @@ async def test_startup_retention_normalizes_aware_postgres_timestamps() -> None: exhausted = SimpleNamespace(all=lambda: []) session = SimpleNamespace( execute=AsyncMock(side_effect=[selected, SimpleNamespace(), exhausted]), + scalars=AsyncMock(return_value=[]), commit=AsyncMock(), ) repository = DurableBridgeRepository(cast(AsyncSession, session)) diff --git a/tests/unit/test_graceful_shutdown.py b/tests/unit/test_graceful_shutdown.py index b7046b2751..a99839e7f5 100644 --- a/tests/unit/test_graceful_shutdown.py +++ b/tests/unit/test_graceful_shutdown.py @@ -8,7 +8,12 @@ import pytest from app.core.shutdown import wait_for_tasks_to_drain -from app.main import InFlightMiddleware, _drain_detached_control_plane_tasks, _release_leader_lease_within +from app.main import ( + InFlightMiddleware, + _drain_detached_control_plane_tasks, + _drain_proxy_persistence_tasks, + _release_leader_lease_within, +) app_main = import_module("app.main") shutdown_state = import_module("app.core.shutdown") @@ -124,6 +129,30 @@ async def drain_fleet(_: float) -> bool: assert "Failed to drain audit log tasks during shutdown" in caplog.text +@pytest.mark.asyncio +async def test_lifespan_recovery_settlement_pre_drain_uses_remaining_deadline() -> None: + calls: list[dict[str, object]] = [] + + class _ProxyService: + async def drain_persistence_tasks(self, **kwargs: object) -> bool: + calls.append(kwargs) + return True + + assert await _drain_proxy_persistence_tasks( + _ProxyService(), + 3.25, + task_name_prefixes=("http-bridge-recovery-settlement-",), + failure_message="unused", + ) + + assert calls == [ + { + "timeout_seconds": 3.25, + "task_name_prefixes": ("http-bridge-recovery-settlement-",), + } + ] + + @pytest.mark.asyncio async def test_control_plane_drain_requires_stable_clean_pass( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_http_bridge_event_batcher.py b/tests/unit/test_http_bridge_event_batcher.py new file mode 100644 index 0000000000..bf8eddea23 --- /dev/null +++ b/tests/unit/test_http_bridge_event_batcher.py @@ -0,0 +1,166 @@ +from __future__ import annotations + +import asyncio + +import pytest + +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher + + +class _FakeDurableBridge: + def __init__(self, *, append_result: bool = True) -> None: + self.append_result = append_result + self.batches: list[list[str]] = [] + self.finalized: list[str] = [] + self.updated: list[dict[str, object]] = [] + + async def append_operation_events(self, *, events, max_bytes: int) -> bool: + del max_bytes + self.batches.append([event.event_text for event in events]) + return self.append_result + + async def finalize_operation_event_spool(self, **kwargs) -> bool: + self.finalized.append(kwargs["operation_id"]) + return True + + async def update_operation(self, **kwargs) -> bool: + self.updated.append(kwargs) + return True + + +async def _enqueue( + batcher: HttpBridgeOperationEventBatcher, + text: str, + *, + terminal: bool = False, +) -> None: + await batcher.enqueue( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=1, + event_text=text, + terminal=terminal, + ) + + +@pytest.mark.asyncio +async def test_batches_without_blocking_and_finalizes_terminal_event() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + await _enqueue(batcher, "two") + await _enqueue(batcher, "three", terminal=True) + assert durable.batches == [["one", "two", "three"]] + assert durable.finalized == ["op-1"] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_background_flushes_nonterminal_events_as_one_batch() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + await _enqueue(batcher, "two") + for _ in range(20): + if durable.batches: + break + await asyncio.sleep(0.01) + assert durable.batches == [["one", "two"]] + assert durable.finalized == [] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_dropped_batch_is_never_marked_replayable() -> None: + durable = _FakeDurableBridge(append_result=False) + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=0.01, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "one") + for _ in range(20): + if durable.batches: + break + await asyncio.sleep(0.01) + assert ( + await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=1, + event_text="terminal", + max_bytes=1024, + state="failed", + ) + is False + ) + assert durable.finalized == [] + assert durable.updated[0]["state"] == "failed" + assert batcher._contexts == {} + assert batcher._dropped_operations == set() + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_discard_operation_releases_partial_nonterminal_context() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=60.0, + max_pending_events=32, + ) + try: + await _enqueue(batcher, "partial") + await batcher.discard_operation(operation_id="op-1") + assert batcher._pending == {} + assert batcher._contexts == {} + assert batcher._pending_count == 0 + assert batcher._pending_bytes == 0 + assert durable.batches == [] + assert durable.finalized == [] + finally: + await batcher.close() + + +@pytest.mark.asyncio +async def test_close_cancels_background_flusher() -> None: + durable = _FakeDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + batch_size=8, + flush_interval_seconds=60.0, + max_pending_events=32, + ) + await _enqueue(batcher, "one") + task = batcher._task + assert task is not None + + await batcher.close() + + assert batcher._task is None + assert task.done() diff --git a/tests/unit/test_proxy_api_websocket_auth.py b/tests/unit/test_proxy_api_websocket_auth.py index 7a896cad75..57976701e8 100644 --- a/tests/unit/test_proxy_api_websocket_auth.py +++ b/tests/unit/test_proxy_api_websocket_auth.py @@ -630,6 +630,35 @@ def test_public_previous_response_not_found_error_is_masked_to_stream_incomplete assert "resp_missing" not in masked.model_dump_json() +def test_public_previous_response_not_found_can_enable_client_full_history_recovery( + monkeypatch: pytest.MonkeyPatch, +) -> None: + envelope = proxy_api_module.OpenAIErrorEnvelopeModel( + error=proxy_api_module.OpenAIError( + message="Previous response with id 'resp_missing' not found.", + type="invalid_request_error", + code="previous_response_not_found", + param="previous_response_id", + ) + ) + monkeypatch.setattr( + proxy_api_module, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="client_full_history_once" + ), + ) + + status_code, preserved = proxy_api_module._mask_previous_response_not_found_error( + envelope, + default_status=400, + allow_client_full_history_once=True, + ) + + assert status_code == 400 + assert preserved == envelope + + def test_public_previous_response_invalid_request_param_is_masked_to_stream_incomplete(): envelope = proxy_api_module.OpenAIErrorEnvelopeModel( error=proxy_api_module.OpenAIError( diff --git a/tests/unit/test_proxy_errors.py b/tests/unit/test_proxy_errors.py index 083d6b437f..7d24526fd4 100644 --- a/tests/unit/test_proxy_errors.py +++ b/tests/unit/test_proxy_errors.py @@ -1,16 +1,86 @@ from __future__ import annotations import json +from collections.abc import AsyncIterator +from types import SimpleNamespace import pytest from starlette.requests import Request from app.core.clients.proxy import ProxyResponseError, _error_event_from_response, _error_payload_from_response +from app.core.exceptions import ProxyRateLimitError +from app.core.openai.requests import ResponsesRequest +from app.modules.proxy import api as proxy_api from app.modules.proxy.api import _logged_error_json_response, _stream_response_error_events pytestmark = pytest.mark.unit +def test_http_bridge_recovery_eligibility_accepts_turn_state_anchor_without_previous_response( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + proxy_api.proxy_service_module, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_operation_ledger_enabled=True), + ) + payload = ResponsesRequest(model="gpt-5.6", instructions="", input="retry") + + assert ( + proxy_api._http_bridge_recovery_request_eligible( + payload, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + assert ( + proxy_api._http_bridge_recovery_request_eligible( + payload, + bridge_active=True, + headers={}, + ) + is False + ) + + +def test_http_bridge_indefinite_recovery_defers_predecessor_proof_to_submit_path( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + proxy_api.proxy_service_module, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_operation_ledger_enabled=True, + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + ), + ) + fresh_turn = ResponsesRequest(model="gpt-5.6", instructions="", input="retry") + anchored_turn = ResponsesRequest( + model="gpt-5.6", + instructions="", + input="retry", + previous_response_id="resp_parent", + ) + + assert ( + proxy_api._http_bridge_recovery_request_eligible( + fresh_turn, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + assert ( + proxy_api._http_bridge_recovery_request_eligible( + anchored_turn, + bridge_active=True, + headers={"x-codex-turn-state": "turn-1"}, + ) + is True + ) + + def test_logged_error_json_response_preserves_upstream_diagnostic_markers(): message = "Provider Exception: failed while reading /tmp/upstream-cache" request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) @@ -76,6 +146,165 @@ async def stream(): assert events[0].startswith("retry: 2000\n") +@pytest.mark.asyncio +async def test_indefinite_recovery_does_not_retry_after_downstream_event(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + + async def recovery() -> AsyncIterator[str]: + raise AssertionError("recovery must not run after a downstream event") + yield "" + + async def stream(): + yield 'data: {"type":"response.created"}\n\n' + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=recovery, + ) + ] + + assert len(events) == 2 + assert "response.created" in events[0] + assert "response.failed" in events[1] + + +@pytest.mark.asyncio +async def test_indefinite_recovery_converts_retry_reservation_failure_to_sse(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + raise ProxyRateLimitError("quota exhausted") + yield "" + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert any("rate_limit_exceeded" in event and "response.failed" in event for event in events) + + +@pytest.mark.asyncio +async def test_indefinite_recovery_converts_unexpected_admission_failure_to_sse( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + raise RuntimeError("durable admission database unavailable") + yield "" + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert any("bridge_recovery_admission_failed" in event and "response.failed" in event for event in events) + + +@pytest.mark.asyncio +async def test_indefinite_recovery_stops_after_retry_output_then_transport_error( + monkeypatch: pytest.MonkeyPatch, +): + monkeypatch.setattr( + proxy_api, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + monkeypatch.setattr(proxy_api.asyncio, "sleep", lambda _delay: _completed_asyncio_sleep()) + attempts = 0 + + async def stream(): + if False: + yield "" + raise ProxyResponseError( + 502, + {"error": {"code": "stream_incomplete", "message": "closed", "type": "server_error"}}, + ) + + async def recovery_stream(): + nonlocal attempts + attempts += 1 + yield 'data: {"type":"response.created"}\n\n' + raise ProxyResponseError( + 502, + {"error": {"code": "upstream_request_timeout", "message": "stalled", "type": "server_error"}}, + ) + + events = [ + event + async for event in _stream_response_error_events( + stream(), + owns_reservation=False, + reservation=None, + recovery_stream_factory=lambda: recovery_stream(), + ) + ] + + assert attempts == 1 + assert any('"type":"response.created"' in event for event in events) + + +async def _completed_asyncio_sleep(_delay: float = 0.0) -> None: + return None + + def _payload_error_code(payload) -> str | None: return payload["error"].get("code") diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index d5ef6e5eb6..672bc40935 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -13,7 +13,7 @@ from dataclasses import replace from datetime import UTC, datetime, timedelta, timezone from types import SimpleNamespace -from typing import Any, cast +from typing import Any, Mapping, cast from unittest.mock import AsyncMock, Mock import aiohttp @@ -157,6 +157,601 @@ def _without_installation_metadata(text: str) -> dict[str, Any]: return payload +def test_http_bridge_operation_metadata_is_stable_and_non_destructive() -> None: + payload = {"type": "response.create", "previous_response_id": "resp_parent", "input": "continue"} + text = json.dumps(payload) + with_operation = http_bridge_request_submit_module._text_with_operation_id(text, "op_test") + decoded = json.loads(with_operation) + assert decoded["previous_response_id"] == "resp_parent" + assert decoded["client_metadata"] == {"codex_lb_operation_id": "op_test"} + assert http_bridge_request_submit_module._text_with_operation_id(with_operation, "op_test") == with_operation + supplied = '{"type":"response.create","client_metadata":{"codex_lb_operation_id":"caller-value"}}' + supplied_result = json.loads(http_bridge_request_submit_module._text_with_operation_id(supplied, "op_test")) + assert supplied_result["client_metadata"]["codex_lb_operation_id"] == "op_test" + normalized = http_bridge_request_submit_module._text_without_operation_id(supplied) + assert json.loads(normalized) == {"type": "response.create"} + + +def test_http_bridge_inserts_previous_response_id_for_hard_turn_advance() -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-next", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + ) + completed_operation = SimpleNamespace( + state="completed", + event_spool_complete=True, + response_id="resp-prior-turn", + ) + + response_id = http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + request_state, + completed_operation, + ) + assert response_id == "resp-prior-turn" + assert ( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + request_state, + SimpleNamespace( + state="completed", + event_spool_complete=False, + response_id="resp-completed-but-unsynced", + ), + ) + == "resp-completed-but-unsynced" + ) + assert ( + json.loads( + http_bridge_request_submit_module._text_with_previous_response_id( + '{"type":"response.create","input":"same"}', + response_id, + ) + )["previous_response_id"] + == "resp-prior-turn" + ) + + request_state.replay_count = 1 + assert ( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + request_state, + completed_operation, + ) + is None + ) + + request_state.replay_count = 0 + request_state.previous_response_id = "resp-prior-turn" + assert ( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + request_state, + SimpleNamespace( + state="completed", + event_spool_complete=True, + response_id="resp-second-turn", + ), + allow_anchored_continuation=True, + ) + == "resp-second-turn" + ) + assert ( + http_bridge_request_submit_module._http_bridge_terminal_hard_turn_response_id( + replace(request_state, previous_response_id=None), + SimpleNamespace( + state="incomplete", + event_spool_complete=True, + response_id="resp-incomplete-turn", + ), + ) + is None + ) + + +def test_http_bridge_operation_fingerprint_strips_account_installation_metadata() -> None: + request = ( + '{"type":"response.create","previous_response_id":"resp_parent",' + '"client_metadata":{"x-codex-installation-id":"account-a",' + '"x-codex-turn-metadata":"{\\"installation_id\\":\\"account-a\\",\\"turn_id\\":\\"t1\\"}",' + '"caller":"stable"}}' + ) + normalized = json.loads(http_bridge_request_submit_module._text_without_account_installation_id(request)) + assert normalized == { + "type": "response.create", + "previous_response_id": "resp_parent", + "client_metadata": { + "x-codex-turn-metadata": '{"turn_id":"t1"}', + "caller": "stable", + }, + } + + +@pytest.mark.asyncio +async def test_submit_hard_turn_walks_completed_operation_chain_before_recording( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="hard-turn-chain") + session.durable_session_id = "durable-hard-turn-chain" + session.durable_owner_epoch = 4 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-chain", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + request_text='{"type":"response.create","input":"same"}', + transport="http", + skip_request_log=True, + ) + completed_operations = iter( + [ + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-1"), + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-2"), + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-3"), + None, + ] + ) + recorded: dict[str, Any] = {} + initial_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id=session.durable_session_id, + api_key_scope="api-key-scope", + request_state=request_state, + text_data=request_state.request_text or "{}", + ) + + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return next(completed_operations) + + async def get_operation(**_kwargs: Any) -> None: + return None + + async def record_operation(**kwargs: Any) -> Any: + recorded.update(kwargs) + raise RuntimeError("stop after operation identity assertion") + + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + record_operation=record_operation, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-chain", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._submit_http_bridge_request_with_handoff( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + request_scope_id="scope-hard-turn-chain", + ) + + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + assert json.loads(recorded["request_text"])["previous_response_id"] == "resp-3" + assert recorded["parent_response_id"] == "resp-3" + assert recorded["request_fingerprint"] != initial_fingerprint + assert json.loads(request_state.request_text or "{}")["previous_response_id"] == "resp-3" + + +@pytest.mark.asyncio +async def test_submit_hard_turn_walks_race_path_chain_before_recording( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="hard-turn-race-chain") + session.durable_session_id = "durable-hard-turn-race-chain" + session.durable_owner_epoch = 4 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-race-chain", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + request_text='{"type":"response.create","input":"same"}', + transport="http", + skip_request_log=True, + ) + operation_lookups = iter( + [ + None, + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-2"), + None, + ] + ) + latest_completed = iter( + [ + SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-1"), + None, + ] + ) + recorded: dict[str, Any] = {} + + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return next(operation_lookups) + + async def get_operation(**_kwargs: Any) -> None: + return None + + async def get_latest_completed_operation(**_kwargs: Any) -> Any: + return next(latest_completed) + + async def record_operation(**kwargs: Any) -> Any: + recorded.update(kwargs) + raise RuntimeError("stop after operation identity assertion") + + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + get_latest_completed_operation=get_latest_completed_operation, + record_operation=record_operation, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-race-chain", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + + with pytest.raises(ProxyResponseError) as exc_info: + await service._submit_http_bridge_request_with_handoff( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + request_scope_id="scope-hard-turn-race-chain", + ) + + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + assert json.loads(recorded["request_text"])["previous_response_id"] == "resp-2" + assert recorded["parent_response_id"] == "resp-2" + assert json.loads(request_state.request_text or "{}")["previous_response_id"] == "resp-2" + assert request_state.proxy_injected_previous_response_id is True + + +def test_ambiguous_continuation_recovery_is_opt_in_and_requires_unobserved_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp-parent", + response_event_count=0, + response_id=None, + fresh_upstream_request_is_retry_safe=False, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_ambiguous_continuation_recovery_mode="fail_closed"), + ) + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is False + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="client_full_history_once" + ), + ) + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is True + request_state.propagate_http_errors = True + assert http_bridge_request_submit_module._http_bridge_client_full_history_recovery_enabled(request_state) is True + request_state.response_event_count = 1 + assert http_bridge_streaming_module._http_bridge_client_full_history_recovery_enabled(request_state) is False + assert http_bridge_request_submit_module._http_bridge_client_full_history_recovery_enabled(request_state) is False + + +def test_hard_continuity_operation_fence_requires_server_recovery_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-fence", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_ambiguous_continuation_recovery_mode="fail_closed"), + ) + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is False + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is True + ) + first_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id="durable-a", + api_key_scope="key-scope", + request_state=request_state, + text_data='{"type":"response.create","input":"same"}', + ) + second_fingerprint = http_bridge_request_submit_module._http_bridge_operation_fingerprint( + session_id="durable-b", + api_key_scope="key-scope", + request_state=request_state, + text_data='{"type":"response.create","input":"same"}', + ) + assert first_fingerprint != second_fingerprint + request_state.hard_continuity_anchor = False + assert ( + http_bridge_request_submit_module._http_bridge_operation_fence_for_hard_continuity_enabled(request_state) + is False + ) + + +def test_http_bridge_durable_recovery_requires_predecessor_anchor() -> None: + fresh_turn = proxy_service._WebSocketRequestState( + request_id="req-fresh-recovery-proof", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + operation_registered=True, + operation_id="op-fresh", + ) + anchored_turn = replace( + fresh_turn, + operation_parent_response_id="resp-parent", + ) + + assert http_bridge_streaming_module._http_bridge_durable_recovery_predecessor_proven(fresh_turn) is False + assert http_bridge_streaming_module._http_bridge_durable_recovery_predecessor_proven(anchored_turn) is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize("anchored", [False, True]) +async def test_stream_via_http_bridge_marks_recovery_only_after_parent_proof( + monkeypatch: pytest.MonkeyPatch, + anchored: bool, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + turn_state = "turn-recovery-proof" + payload_data: dict[str, Any] = {"model": "gpt-5.6", "instructions": "", "input": "retry"} + payload = proxy_service.ResponsesRequest.model_validate(payload_data) + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-recovery-proof-{anchored}", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + previous_response_id=None, + hard_continuity_anchor=True, + ) + session = _make_bridge_session( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", turn_state, None), + key_value=turn_state, + ) + session.durable_session_id = "durable-recovery-proof" + session.durable_owner_epoch = 1 + session.closed = True + + def fake_prepare( + _payload: proxy_service.ResponsesRequest, + _headers: Mapping[str, str], + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip + return request_state, '{"type":"response.create"}' + + async def fail_eventlessly(*_args: Any, **_kwargs: Any): + raise AssertionError("submit should fail before the upstream event reader is entered") + yield "" # pragma: no cover + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "release_live_session", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "reset_operation_event_spool", AsyncMock(return_value=True)) + completed_operations: list[Any] = ( + [SimpleNamespace(state="completed", event_spool_complete=True, response_id="resp-parent"), None] + if anchored + else [None] + ) + + async def get_operation_by_fingerprint(**_kwargs: Any) -> Any: + return completed_operations.pop(0) + + async def get_operation(**_kwargs: Any) -> None: + return None + + async def record_operation(**kwargs: Any) -> Any: + request_state.operation_id = kwargs["operation_id"] + return SimpleNamespace( + created=True, + operation_id=kwargs["operation_id"], + state="submitted", + event_spool_complete=False, + response_id=None, + ) + + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=get_operation_by_fingerprint, + get_operation=get_operation, + record_operation=record_operation, + lookup_request_targets=AsyncMock(return_value=None), + release_live_session=AsyncMock(return_value=None), + reset_operation_event_spool=AsyncMock(return_value=True), + ), + ) + service._http_bridge_sessions[session.key] = session + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-bridge")) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", AsyncMock(return_value=session)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + monkeypatch.setattr(service, "_retry_http_bridge_request_on_fresh_upstream", AsyncMock(return_value=False)) + + async def submit_then_fail( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del propagate_http_errors, downstream_turn_state, request_deadline + await service._submit_http_bridge_request_with_handoff( + _session, + request_state=request_state, + text_data=text_data, + queue_limit=queue_limit, + request_scope_id=request_state.request_id, + ) + yield "" + + monkeypatch.setattr(service, "_stream_http_bridge_session_events", submit_then_fail) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_via_http_bridge( + payload, + headers={"x-codex-turn-state": turn_state}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=900.0, + max_sessions=8, + queue_limit=4, + ): + pass + + assert getattr(exc_info.value, "http_bridge_durable_recovery_eligible", False) is anchored + if anchored: + assert request_state.previous_response_id == "resp-parent" + assert request_state.operation_parent_response_id == "resp-parent" + + +@pytest.mark.asyncio +async def test_hard_continuity_operation_replay_requires_matching_unknown_fence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="hard-fence") + session.durable_session_id = "durable-hard-fence" + session.durable_owner_epoch = 3 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-fence-replay", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + hard_continuity_anchor=True, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery" + ), + ) + operation = SimpleNamespace( + session_id="durable-hard-fence", + state="unknown", + event_spool_complete=False, + ) + service._durable_bridge = SimpleNamespace(get_operation_by_fingerprint=AsyncMock(return_value=operation)) + + assert ( + await service._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data='{"type":"response.create","input":"retry"}', + ) + is True + ) + + operation.session_id = "different-session" + assert ( + await service._http_bridge_operation_fenced_continuity_replay_allowed( + session, + request_state=request_state, + text_data='{"type":"response.create","input":"retry"}', + ) + is False + ) + + def _make_app_settings(*, bridge_enabled: bool = True, **overrides: Any) -> Settings: return Settings(http_responses_session_bridge_enabled=bridge_enabled, **overrides) @@ -450,6 +1045,55 @@ async def send_text(_text: str) -> None: ) +@pytest.mark.asyncio +async def test_http_bridge_send_started_callback_runs_after_exact_frame_preflight( + monkeypatch: pytest.MonkeyPatch, +) -> None: + request_state = _make_eventless_http_bridge_owner() + session = _make_bridge_session() + send_started = Mock() + send_text = AsyncMock() + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) + + def fail_preflight(_request_state: object, _text_data: str) -> None: + raise proxy_service.ProxyResponseError(400, {"error": {"code": "payload_too_large"}}) + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_enforce_http_bridge_response_create_text_size", + fail_preflight, + ) + + with pytest.raises(proxy_service.ProxyResponseError): + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "request", + on_send_started=send_started, + ) + + send_started.assert_not_called() + send_text.assert_not_awaited() + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_enforce_http_bridge_response_create_text_size", + lambda _request_state, _text_data: None, + ) + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + session, + request_state, + "request", + on_send_started=send_started, + ) + + send_started.assert_called_once() + send_text.assert_awaited_once() + + def _make_account_neutral_replay_session_key( nonce: str, api_key_id: str | None = None, @@ -3028,9 +3672,74 @@ async def fake_capacity_wait(**kwargs: object): ): pass - assert exc_info.value is capacity_error - assert waited == [1.0] - assert submit.await_count == 1 + assert exc_info.value is capacity_error + assert waited == [1.0] + assert submit.await_count == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_submit_capacity_retry_uses_advanced_request_body( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-submit-advanced-body") + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-advanced-body", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=99.5, + transport="http", + event_queue=asyncio.Queue(), + previous_response_id="resp-parent", + proxy_injected_previous_response_id=True, + request_text='{"type":"response.create","previous_response_id":"resp-parent","input":"same"}', + ) + stale_text_data = '{"type":"response.create","input":"same"}' + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", + ), + ) + submitted_texts: list[str] = [] + clock = [100.0] + + async def submit(_session: Any, *, request_state: Any, text_data: str, **_kwargs: Any) -> None: + submitted_texts.append(text_data) + if len(submitted_texts) == 1: + raise capacity_error + await request_state.event_queue.put(None) + + async def fake_capacity_wait(**kwargs: object): + clock[0] += cast(float, kwargs["sleep_seconds"]) + if False: + yield "" + + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", fake_capacity_wait) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_time", + lambda: SimpleNamespace(monotonic=lambda: clock[0]), + ) + + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data=stale_text_data, + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=101.0, + ): + pass + + assert submitted_texts == [stale_text_data, request_state.request_text] def _make_api_key( @@ -4119,6 +4828,7 @@ async def test_recovery_completed_alias_persistence_failure_fails_response_and_r transport="http", skip_request_log=True, ) + request_state.operation_id = "op-alias-persistence-failure" session = _make_bridge_session( key=_make_account_neutral_replay_session_key("completed-alias-failure"), pending_requests=deque([request_state]), @@ -4126,10 +4836,18 @@ async def test_recovery_completed_alias_persistence_failure_fails_response_and_r ) register_previous = AsyncMock(return_value=False) finalize = AsyncMock() + operation_updates = AsyncMock() + persist_operation_event = AsyncMock() close_session = AsyncMock() monkeypatch.setattr(service, "_register_http_bridge_previous_response_id", register_previous) monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) monkeypatch.setattr(service, "_close_http_bridge_session", close_session) + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", operation_updates) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_persist_http_bridge_operation_event", + persist_operation_event, + ) await service._process_http_bridge_upstream_text( session, @@ -4165,11 +4883,121 @@ async def test_recovery_completed_alias_persistence_failure_fails_response_and_r finalize_call = finalize.await_args assert finalize_call is not None assert finalize_call.kwargs["event_type"] == "response.failed" + operation_updates.assert_awaited_once() + assert operation_updates.await_args is not None + assert operation_updates.await_args.kwargs["state"] == "acknowledged" + persist_operation_event.assert_awaited_once() + assert persist_operation_event.await_args is not None + assert persist_operation_event.await_args.kwargs["terminal_state"] == "acknowledged" assert await service._retire_http_bridge_after_drain_if_ready(session) is True close_session.assert_awaited_once_with(session) +@pytest.mark.asyncio +async def test_http_bridge_incomplete_event_terminalizes_operation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-incomplete-terminal", + response_id="resp-incomplete-terminal", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-incomplete-terminal" + session = _make_bridge_session( + key_value="incomplete-terminal", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + operation_updates = AsyncMock() + finalize = AsyncMock() + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", operation_updates) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize) + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.incomplete", + "response": { + "id": "resp-incomplete-terminal", + "object": "response", + "status": "incomplete", + "output": [], + }, + } + ), + ) + + operation_updates.assert_awaited_once() + assert operation_updates.await_args is not None + assert operation_updates.await_args.kwargs["state"] == "incomplete" + finalize.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_http_bridge_batched_terminal_state_precedes_spool_finalize( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-batched-terminal-order", + response_id="resp-batched-terminal-order", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-batched-terminal-order" + session = _make_bridge_session( + key_value="batched-terminal-order", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-batched-terminal-order" + session.durable_owner_epoch = 1 + order: list[str] = [] + + async def update_state(*args: Any, **kwargs: Any) -> None: + del args, kwargs + order.append("state") + + async def append_terminal_event(*args: Any, **kwargs: Any) -> bool: + del args + assert kwargs["session_id"] == session.durable_session_id + order.append("terminal") + return True + + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", update_state) + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace(append_terminal_event=append_terminal_event), + ) + + await http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + 'data: {"type":"response.completed"}\n\n', + terminal=True, + terminal_state="completed", + ) + + assert order == ["terminal"] + + @pytest.mark.asyncio async def test_ordinary_completed_alias_rejection_preserves_successful_response( monkeypatch: pytest.MonkeyPatch, @@ -4600,6 +5428,34 @@ async def test_http_bridge_startup_cooldown_releases_api_key_reservation( assert request_state.api_key_reservation is None +@pytest.mark.asyncio +async def test_http_bridge_replay_detach_releases_reservation_without_pending_ownership( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-replay-reservation") + reservation = cast(Any, object()) + request_state = proxy_service._WebSocketRequestState( + request_id="req-replay-reservation", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + operation_replay=True, + ) + release = AsyncMock() + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) + + assert await service._detach_http_bridge_request(session, request_state=request_state) is False + + release.assert_awaited_once_with(request_state) + assert request_state.api_key_reservation is None + assert request_state.operation_replay is False + + @pytest.mark.asyncio async def test_http_bridge_post_submit_cooldown_race_detaches_request( monkeypatch: pytest.MonkeyPatch, @@ -17820,108 +18676,406 @@ async def test_http_bridge_retire_after_drain_does_not_cancel_current_upstream_r owner_epoch=7, draining=False, ) - release_account_lease.assert_awaited_once_with(lease) - assert session.account_lease is None + release_account_lease.assert_awaited_once_with(lease) + assert session.account_lease is None + + +@pytest.mark.asyncio +async def test_submit_http_bridge_request_starts_api_key_reservation_heartbeat( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + send_text = AsyncMock() + api_key = _make_api_key(key_id="key-http-heartbeat", assigned_account_ids=[]) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="reservation-http-heartbeat", + key_id=api_key.id, + model="gpt-5.5", + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-http-heartbeat", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + transport="http", + api_key=api_key, + skip_request_log=True, + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_heartbeat", api_key.id), + headers={"x-codex-turn-state": "http_turn_heartbeat"}, + affinity=proxy_service._AffinityPolicy( + key="http_turn_heartbeat", + kind=proxy_service.StickySessionKind.CODEX_SESSION, + ), + request_model="gpt-5.5", + account=cast(Any, SimpleNamespace(id="acc-http-heartbeat", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=120.0, + ) + service._http_bridge_sessions[session.key] = session + started = asyncio.Event() + seen: dict[str, object] = {} + + async def fake_heartbeat(**kwargs: object) -> None: + seen.update(kwargs) + started.set() + stop_event = cast(asyncio.Event, kwargs["stop_event"]) + await stop_event.wait() + + admission_saw_heartbeat = False + + async def fake_acquire_admission( + state: proxy_service._WebSocketRequestState, + *, + response_create_gate: asyncio.Semaphore, + bridge_session: proxy_service._HTTPBridgeSession | None = None, + compact: bool = False, + account_id: str | None = None, + surface: str = "websocket", + apply_gate_timeout: bool = True, + ) -> None: + del bridge_session + del compact + del account_id + del surface + del apply_gate_timeout + nonlocal admission_saw_heartbeat + admission_saw_heartbeat = state.api_key_reservation_heartbeat_task is not None + state.response_create_gate = response_create_gate + await response_create_gate.acquire() + state.response_create_gate_acquired = True + state.awaiting_response_created = True + + monkeypatch.setattr(service, "_run_api_key_reservation_heartbeat", fake_heartbeat) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", fake_acquire_admission) + + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + await asyncio.wait_for(started.wait(), timeout=1.0) + + assert seen["api_key"] is api_key + assert seen["reservation"] is reservation + assert seen["request_id"] == "req-http-heartbeat" + assert seen["surface"] == "http_bridge" + assert admission_saw_heartbeat is True + assert request_state.api_key_reservation_heartbeat_task is not None + send_text.assert_awaited_once_with(request_state.request_text) + + service._cancel_request_state_api_key_reservation_heartbeat(request_state) + + +@pytest.mark.asyncio +async def test_submit_http_bridge_request_restores_recovery_claim_when_stream_lease_reacquire_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + send_text = AsyncMock() + session = _make_bridge_session(key_value="recovery-lease-reacquire") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery-lease-reacquire", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5","input":"retry"}', + transport="http", + skip_request_log=True, + ) + request_state.operation_recovery_claimed = True + service._durable_bridge = cast( + Any, + SimpleNamespace(lookup_retry_circuit=AsyncMock(return_value=None)), + ) + cleanup = AsyncMock() + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + lease_failure = proxy_service.ProxyResponseError( + 429, + openai_error("account_stream_cap", "stream capacity exhausted"), + ) + monkeypatch.setattr( + service, + "_ensure_http_bridge_session_stream_lease_locked", + AsyncMock(side_effect=lease_failure), + ) + + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + ) + + assert exc_info.value is lease_failure + cleanup.assert_awaited_once() + assert cleanup.await_args is not None + assert cleanup.await_args.kwargs["admission_waiter_registered"] is False + assert cleanup.await_args.kwargs["request_enqueued"] is False + send_text.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_cleanup_http_bridge_submit_interruption_clears_restored_operation_identity() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + mark_operation_unknown = AsyncMock(return_value=True) + service._durable_bridge = cast(Any, SimpleNamespace(mark_operation_unknown=mark_operation_unknown)) + session = _make_bridge_session(key_value="restored-operation-identity") + session.durable_session_id = "durable-restored-operation-identity" + session.durable_owner_epoch = 2 + request_state = proxy_service._WebSocketRequestState( + request_id="req-restored-operation-identity", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + operation_id="operation-restored", + operation_fingerprint="fingerprint-restored", + operation_parent_response_id="resp-parent", + operation_registered=True, + operation_recovery_claimed=True, + ) + + await service._cleanup_http_bridge_submit_interruption( + session, + request_state=request_state, + gate_acquired=False, + request_enqueued=False, + counted_in_queue=False, + ) + + mark_operation_unknown.assert_awaited_once() + assert request_state.operation_recovery_claimed is False + assert request_state.operation_id is None + assert request_state.operation_fingerprint is None + assert request_state.operation_parent_response_id is None @pytest.mark.asyncio -async def test_submit_http_bridge_request_starts_api_key_reservation_heartbeat( +async def test_http_bridge_capacity_retry_reclaims_unknown_operation_before_send( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) send_text = AsyncMock() - api_key = _make_api_key(key_id="key-http-heartbeat", assigned_account_ids=[]) - reservation = proxy_service.ApiKeyUsageReservationData( - reservation_id="reservation-http-heartbeat", - key_id=api_key.id, - model="gpt-5.5", + session = _make_bridge_session(key_value="unknown-operation-capacity-retry") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace(send_text=send_text, close=AsyncMock()), ) + session.durable_session_id = "durable-unknown-operation-capacity-retry" + session.durable_owner_epoch = 4 + service._http_bridge_sessions[session.key] = session request_state = proxy_service._WebSocketRequestState( - request_id="req-http-heartbeat", + request_id="req-unknown-operation-capacity-retry", model="gpt-5.5", service_tier=None, reasoning_effort=None, - api_key_reservation=reservation, + api_key_reservation=None, started_at=time.monotonic(), + hard_continuity_anchor=True, + previous_response_id="resp-parent", awaiting_response_created=True, event_queue=asyncio.Queue(), - request_text='{"type":"response.create","model":"gpt-5.5","input":"new"}', + request_text='{"type":"response.create","input":"retry"}', transport="http", - api_key=api_key, skip_request_log=True, ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_heartbeat", api_key.id), - headers={"x-codex-turn-state": "http_turn_heartbeat"}, - affinity=proxy_service._AffinityPolicy( - key="http_turn_heartbeat", - kind=proxy_service.StickySessionKind.CODEX_SESSION, + existing_operation = SimpleNamespace( + operation_id="operation-existing-unknown", + session_id=session.durable_session_id, + state="unknown", + created=False, + event_spool_complete=False, + response_id=None, + ) + lookup = AsyncMock(return_value=existing_operation) + claim_unknown = AsyncMock(return_value=True) + restore_unknown = AsyncMock(return_value=True) + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=lookup, + get_operation=AsyncMock(return_value=existing_operation), + record_operation=AsyncMock(return_value=existing_operation), + claim_unknown_operation_for_recovery=claim_unknown, + mark_operation_unknown=restore_unknown, + release_live_session=AsyncMock(return_value=None), ), - request_model="gpt-5.5", - account=cast(Any, SimpleNamespace(id="acc-http-heartbeat", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(send_text=send_text, close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=1.0, - idle_ttl_seconds=120.0, ) - service._http_bridge_sessions[session.key] = session - started = asyncio.Event() - seen: dict[str, object] = {} - - async def fake_heartbeat(**kwargs: object) -> None: - seen.update(kwargs) - started.set() - stop_event = cast(asyncio.Event, kwargs["stop_event"]) - await stop_event.wait() - - admission_saw_heartbeat = False + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-unknown-operation-capacity-retry", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + capacity_error = ProxyResponseError( + 429, + openai_error( + "account_response_create_cap", + "Account response-create concurrency limit reached", + error_type="rate_limit_error", + ), + ) + admission_calls = 0 - async def fake_acquire_admission( + async def acquire_admission( state: proxy_service._WebSocketRequestState, *, response_create_gate: asyncio.Semaphore, - bridge_session: proxy_service._HTTPBridgeSession | None = None, - compact: bool = False, - account_id: str | None = None, - surface: str = "websocket", - apply_gate_timeout: bool = True, + **_kwargs: Any, ) -> None: - del bridge_session - del compact - del account_id - del surface - del apply_gate_timeout - nonlocal admission_saw_heartbeat - admission_saw_heartbeat = state.api_key_reservation_heartbeat_task is not None + nonlocal admission_calls + admission_calls += 1 + if admission_calls == 1: + raise capacity_error state.response_create_gate = response_create_gate await response_create_gate.acquire() state.response_create_gate_acquired = True state.awaiting_response_created = True - monkeypatch.setattr(service, "_run_api_key_reservation_heartbeat", fake_heartbeat) - monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", fake_acquire_admission) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", acquire_admission) + wait_calls = 0 - await service._submit_http_bridge_request( + async def capacity_wait(**_kwargs: object): + nonlocal wait_calls + wait_calls += 1 + if False: + yield "" + + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_account_capacity_wait_seconds", lambda _exc: 0.001) + monkeypatch.setattr(http_bridge_streaming_module, "_iter_account_capacity_wait_sse", capacity_wait) + + async def send_and_finish(_text: str) -> None: + assert claim_unknown.await_count == 2 + event_queue = request_state.event_queue + assert event_queue is not None + await event_queue.put(None) + + send_text.side_effect = send_and_finish + + async for _ in service._stream_http_bridge_session_events( session, request_state=request_state, text_data=request_state.request_text or "{}", - queue_limit=8, + queue_limit=4, + propagate_http_errors=True, + downstream_turn_state=None, + request_deadline=time.monotonic() + 10.0, + ): + pass + + assert wait_calls == 1 + assert admission_calls == 2 + assert claim_unknown.await_count == 2 + restore_unknown.assert_awaited_once() + send_text.assert_awaited_once() + assert request_state.operation_id == "operation-existing-unknown" + + +@pytest.mark.asyncio +async def test_submit_hard_turn_rolls_back_new_operation_before_retiring_session( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="hard-turn-unsent-operation") + session.durable_session_id = "durable-hard-turn-unsent-operation" + session.durable_owner_epoch = 3 + session.upstream_control.retire_after_drain = True + session.upstream_close_attempted = True + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-unsent-operation", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + hard_continuity_anchor=True, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","input":"same"}', + transport="http", + skip_request_log=True, ) - await asyncio.wait_for(started.wait(), timeout=1.0) + record_operation = AsyncMock( + return_value=SimpleNamespace( + created=True, + operation_id="operation-unsent", + state="submitted", + response_id=None, + event_spool_complete=False, + ) + ) + rollback_operation = AsyncMock(return_value=True) + service._durable_bridge = cast( + Any, + SimpleNamespace( + get_operation_by_fingerprint=AsyncMock(return_value=None), + get_operation=AsyncMock(return_value=None), + record_operation=record_operation, + rollback_operation_before_dispatch=rollback_operation, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_indefinite_recovery", + http_responses_session_bridge_instance_id="instance-hard-turn-unsent-operation", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_allowed", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=0.0)) - assert seen["api_key"] is api_key - assert seen["reservation"] is reservation - assert seen["request_id"] == "req-http-heartbeat" - assert seen["surface"] == "http_bridge" - assert admission_saw_heartbeat is True - assert request_state.api_key_reservation_heartbeat_task is not None - send_text.assert_awaited_once_with(request_state.request_text) + with pytest.raises(proxy_service.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request_with_handoff( + session, + request_state=request_state, + text_data=request_state.request_text or "{}", + queue_limit=8, + request_scope_id="scope-hard-turn-unsent-operation", + ) - service._cancel_request_state_api_key_reservation_heartbeat(request_state) + assert exc_info.value.payload["error"]["code"] == "upstream_unavailable" + record_operation.assert_awaited_once() + rollback_operation.assert_awaited_once_with( + operation_id="operation-unsent", + session_id="durable-hard-turn-unsent-operation", + instance_id="instance-hard-turn-unsent-operation", + owner_epoch=3, + ) + assert request_state.operation_created is False + assert request_state.operation_registered is False + assert request_state.operation_id is None + assert request_state.operation_fingerprint is None + assert request_state.operation_parent_response_id is None @pytest.mark.asyncio @@ -19364,6 +20518,82 @@ async def test_process_http_bridge_upstream_text_marks_text_delta_downstream_vis assert forwarded_payload["delta"] == "I started" +@pytest.mark.asyncio +async def test_http_bridge_recovery_releases_origin_after_terminal_event_following_created( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-recovery-origin-terminal", + model="gpt-5.4", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.4","input":"hello"}', + transport="http", + skip_request_log=True, + recovery_attempt_fingerprint="recovery-origin-terminal-fingerprint", + recovery_attempt_session_id="durable-recovery-origin", + recovery_attempt_owner_epoch=4, + ) + session = _make_bridge_session( + key_value="recovery-origin-terminal", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-replacement" + session.durable_owner_epoch = 9 + mark_replayed = AsyncMock(return_value=True) + release_origin = AsyncMock() + service._durable_bridge = cast( + Any, + SimpleNamespace( + mark_recovery_attempt_replayed=mark_replayed, + release_live_session=release_origin, + ), + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings(http_responses_session_bridge_instance_id="replacement-instance"), + ) + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + {"type": "response.created", "response": {"id": "resp-recovery-origin", "status": "in_progress"}}, + separators=(",", ":"), + ), + ) + release_origin.assert_not_awaited() + + await service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "response.incomplete", + "response": { + "id": "resp-recovery-origin", + "status": "incomplete", + "incomplete_details": {"reason": "max_output_tokens"}, + }, + }, + separators=(",", ":"), + ), + ) + + assert mark_replayed.await_count == 2 + release_origin.assert_awaited_once_with( + session_id="durable-recovery-origin", + instance_id="replacement-instance", + owner_epoch=4, + draining=False, + ) + + @pytest.mark.asyncio async def test_retry_http_bridge_request_on_fresh_upstream_refuses_to_resend_previous_response_id( monkeypatch: pytest.MonkeyPatch, @@ -23372,6 +24602,61 @@ async def fail_replay(target_session: proxy_service._HTTPBridgeSession) -> bool: await submit_task +@pytest.mark.asyncio +async def test_http_bridge_reader_failure_classifies_each_operation_from_its_own_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + eventless = proxy_service._WebSocketRequestState( + request_id="req-eventless-sibling", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + operation_id="op-eventless-sibling", + response_event_count=0, + ) + streamed = proxy_service._WebSocketRequestState( + request_id="req-streamed-sibling", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + operation_id="op-streamed-sibling", + response_event_count=1, + ) + session = _make_bridge_session( + key_value="per-operation-close-classification", + pending_requests=deque([eventless, streamed]), + queued_request_count=2, + ) + event_order: list[str] = [] + update_operation = AsyncMock() + + async def discard_operation(*, operation_id: str) -> None: + event_order.append(f"discard:{operation_id}") + + update_operation.side_effect = lambda *args, **kwargs: event_order.append(f"update:{kwargs['state']}") + service._http_bridge_operation_event_batcher = cast(Any, SimpleNamespace(discard_operation=discard_operation)) + monkeypatch.setattr(http_bridge_upstream_events_module, "_update_http_bridge_operation_state", update_operation) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", AsyncMock()) + + await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="closed", + ) + + assert [call.kwargs["state"] for call in update_operation.await_args_list] == ["unknown", "acknowledged"] + assert event_order[:2] == ["discard:op-eventless-sibling", "discard:op-streamed-sibling"] + assert event_order[2:] == ["update:unknown", "update:acknowledged"] + + @pytest.mark.asyncio async def test_http_bridge_reader_failure_keeps_waiter_count_when_draining_request_is_present( monkeypatch: pytest.MonkeyPatch, @@ -25389,3 +26674,25 @@ async def test_http_bridge_has_live_local_session_treats_quarantined_as_absent() ) is True ) + + +@pytest.mark.asyncio +async def test_http_bridge_eventless_timeout_signal_drains_after_repeated_sessions() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + account = SimpleNamespace(id="acc-eventless-timeouts") + session = cast(proxy_service._HTTPBridgeSession, SimpleNamespace(account=account)) + record_errors = AsyncMock() + service._load_balancer.record_errors = record_errors + + for _ in range(2): + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_not_awaited() + + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_awaited_once_with(account, 2) + + # The evidence window resets after one penalty; another isolated failure + # must not immediately apply a second account health penalty. + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + await http_bridge_upstream_events_module._record_http_bridge_account_timeout_signal(service, session) + record_errors.assert_awaited_once_with(account, 2) diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 7bd4bb817d..d6ec0a27bb 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -2480,6 +2480,156 @@ async def test_resolve_websocket_previous_response_owner_cache_hit_keeps_owner_s assert websocket_helpers_module._websocket_stale_anchor_diagnostics(request_state).same_session is None +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_suppresses_confirmed_stale_anchor(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_stale_anchor_cache", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp_stale_anchor_cache", + session_id="sid-stale-anchor-cache", + ) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=None, + session_id=request_state.session_id, + surface="websocket_stream", + request_state=request_state, + ) + + assert request_logs.lookup_calls == [("resp_stale_anchor_cache", None, "sid-stale-anchor-cache")] + assert request_state.previous_response_owner_lookup_source == "stale_response_cache" + assert request_state.previous_response_owner_lookup_outcome == "hit" + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "stream_incomplete" + + +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_revalidates_shared_owner_after_stale_cache(monkeypatch): + request_logs = _RequestLogsRecorder() + request_logs.response_owner_by_id[("resp_stale_anchor_shared_owner", None, "sid-shared-owner")] = "acc-shared" + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_stale_anchor_shared_owner", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + previous_response_id="resp_stale_anchor_shared_owner", + session_id="sid-shared-owner", + ) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + owner = await service._resolve_websocket_previous_response_owner( + previous_response_id=request_state.previous_response_id, + api_key=None, + session_id=request_state.session_id, + surface="websocket_stream", + request_state=request_state, + ) + + assert owner == "acc-shared" + assert request_logs.lookup_calls == [("resp_stale_anchor_shared_owner", None, "sid-shared-owner")] + assert request_state.previous_response_owner_lookup_source == "request_logs" + assert request_state.previous_response_owner_lookup_outcome == "hit" + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id=request_state.previous_response_id, + api_key_id=None, + ) + + +def test_remember_websocket_previous_response_owner_invalidates_stale_anchor_cache(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + ) + + service._remember_websocket_previous_response_owner( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + account_id="acc-new-owner", + ) + + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_invalidate", + api_key_id="key-stale-anchor", + ) + + +def test_websocket_stale_previous_response_cache_expires(monkeypatch): + clock = {"value": 100.0} + monkeypatch.setattr(websocket_helpers_module.time, "monotonic", lambda: clock["value"]) + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) + websocket_helpers_module._remember_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + + assert websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + clock["value"] += websocket_helpers_module._WEBSOCKET_STALE_PREVIOUS_RESPONSE_CACHE_TTL_SECONDS + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_stale_anchor_expiry", + api_key_id=None, + ) + + +@pytest.mark.asyncio +async def test_resolve_websocket_previous_response_owner_force_refresh_replaces_stale_cache(): + request_logs = _RequestLogsRecorder() + request_logs.response_owner_by_id[("resp_force_refresh", None, "sid-force-refresh")] = "acc_authoritative" + # Avoid constructing the full service here: its unrelated background + # event-spool settings are intentionally omitted by several lightweight + # test settings fixtures. The owner lookup only needs the repository + # factory and the in-process owner index. + service = object.__new__(proxy_service.ProxyService) + service._repo_factory = _repo_factory(request_logs) + service._websocket_previous_response_account_index = {} + service._remember_websocket_previous_response_owner( + previous_response_id="resp_force_refresh", + api_key_id=None, + account_id="acc_stale_cache", + session_id="sid-force-refresh", + ) + + owner = await service._resolve_websocket_previous_response_owner( + previous_response_id="resp_force_refresh", + api_key=None, + session_id="sid-force-refresh", + surface="http_bridge", + force_request_log_lookup=True, + ) + + assert owner == "acc_authoritative" + assert request_logs.lookup_calls == [("resp_force_refresh", None, "sid-force-refresh")] + assert ( + service._websocket_previous_response_account_index[("resp_force_refresh", None, "sid-force-refresh")] + == "acc_authoritative" + ) + + @pytest.mark.asyncio async def test_resolve_websocket_previous_response_owner_fail_closed_records_metric_and_log(monkeypatch, caplog): request_logs = _RequestLogsRecorder() @@ -32451,6 +32601,28 @@ def test_http_bridge_should_attempt_local_previous_response_recovery_invalid_req assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(non_recoverable_error) is False +def test_http_bridge_server_recovery_mode_retries_ambiguous_transport_once(monkeypatch: pytest.MonkeyPatch): + ambiguous_error = proxy_module.ProxyResponseError( + 502, + { + "error": { + "type": "server_error", + "code": "upstream_request_timeout", + "message": "Upstream did not acknowledge response.create", + } + }, + ) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once" + ), + ) + + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(ambiguous_error) is True + + def test_http_bridge_should_rollover_after_context_overflow(): context_overflow_error = proxy_module.ProxyResponseError( 400, @@ -32535,6 +32707,7 @@ def test_maybe_rewrite_websocket_previous_response_not_found_masks_lost_local_an def test_sanitize_websocket_connect_failure_rewrites_previous_response_not_found(monkeypatch, caplog): fixed_now = utcnow() + monkeypatch.setattr(websocket_helpers_module, "_websocket_stale_previous_response_index", {}) request_state = proxy_service._WebSocketRequestState( request_id="ws_req_prev_connect_failure", model="gpt-5.1", @@ -32606,6 +32779,10 @@ def test_sanitize_websocket_connect_failure_rewrites_previous_response_not_found "value": 1.0, } ] + assert not websocket_helpers_module._is_websocket_stale_previous_response( + previous_response_id="resp_prev_anchor", + api_key_id=None, + ) def test_sanitize_websocket_terminal_stale_error_marks_missing_anchor_source_unknown(): @@ -42472,6 +42649,7 @@ def make_state(request_id: str) -> "proxy_service._WebSocketRequestState": @pytest.mark.asyncio async def test_submit_http_bridge_request_reinlines_final_text(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) original_text = json.dumps( { @@ -42560,6 +42738,7 @@ async def capture_send_text(_text: str) -> None: @pytest.mark.asyncio async def test_submit_http_bridge_network_send_failure_is_neutral_and_not_replayed(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) request_state = proxy_service._WebSocketRequestState( request_id="req_submit_network_failure", @@ -42631,9 +42810,147 @@ async def cleanup(*_args: object, **_kwargs: object) -> None: close.assert_awaited_once() +@pytest.mark.asyncio +async def test_submit_http_bridge_marks_ambiguous_operation_before_releasing_owner(monkeypatch): + service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + events: list[str] = [] + service._durable_bridge = SimpleNamespace( + mark_operation_unknown=AsyncMock(side_effect=lambda **_kwargs: events.append("mark") or True), + ) + proxy_service._initialize_http_bridge_retry_circuit(service) + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-owner-fence-order", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5"}', + operation_id="operation-owner-fence-order", + operation_registered=True, + ) + send_error = UpstreamWebSocketTransportError( + "upstream websocket closed after dispatch", + error_code="proxy_network_unavailable", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-owner-fence-order", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-owner-fence-order"), + request_model="gpt-5.5", + account=_make_account("acc-owner-fence-order"), + upstream=cast( + proxy_service.UpstreamWebSocket, + SimpleNamespace(send_text=AsyncMock(side_effect=send_error), close=AsyncMock()), + ), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=0.0, + idle_ttl_seconds=120.0, + durable_session_id="durable-owner-fence-order", + durable_owner_epoch=4, + ) + + async def cleanup(*_args: object, **_kwargs: object) -> None: + events.append("cleanup") + + monkeypatch.setattr(service, "_inline_http_bridge_image_urls", AsyncMock(return_value=request_state.request_text)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", AsyncMock()) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *args, **kwargs: None) + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + + with pytest.raises(proxy_module.ProxyResponseError): + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "", + queue_limit=1, + ) + + assert events == ["mark", "cleanup"] + service._durable_bridge.mark_operation_unknown.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_submit_http_bridge_preflight_failure_keeps_operation_pre_dispatch(monkeypatch): + service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None + proxy_service._initialize_http_bridge_retry_circuit(service) + request_state = proxy_service._WebSocketRequestState( + request_id="req-submit-preflight", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + request_text='{"type":"response.create","model":"gpt-5.5"}', + operation_id="operation-preflight", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("session_header", "sid-submit-preflight", None), + headers={}, + affinity=proxy_service._AffinityPolicy(key="sid-submit-preflight"), + request_model="gpt-5.5", + account=_make_account("acc_submit_preflight"), + upstream=cast( + proxy_service.UpstreamWebSocket, + SimpleNamespace(send_text=AsyncMock(), close=AsyncMock()), + ), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=0.0, + idle_ttl_seconds=120.0, + ) + send_frame = AsyncMock( + side_effect=proxy_module.ProxyResponseError( + 400, + {"error": {"code": "payload_too_large", "message": "response.create is too large"}}, + ) + ) + cleanup_dispatched: list[bool] = [] + + async def cleanup(*_args: object, **_kwargs: object) -> None: + cleanup_dispatched.append(request_state.operation_dispatched) + + monkeypatch.setattr(proxy_http_bridge_request_submit, "_send_http_bridge_request_text_with_archive_id", send_frame) + monkeypatch.setattr(service, "_inline_http_bridge_image_urls", AsyncMock(return_value=request_state.request_text)) + monkeypatch.setattr(service, "_maybe_prewarm_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", AsyncMock()) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *args, **kwargs: None) + monkeypatch.setattr(service, "_cleanup_http_bridge_submit_interruption", cleanup) + monkeypatch.setattr(service, "_retire_http_bridge_after_drain_if_ready", AsyncMock()) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._submit_http_bridge_request( + session, + request_state=request_state, + text_data=request_state.request_text or "", + queue_limit=1, + ) + + assert exc_info.value.status_code == 400 + send_frame.assert_awaited_once() + assert cleanup_dispatched == [False] + assert request_state.recovery_attempt_dispatched is False + assert request_state.operation_dispatched is False + + @pytest.mark.asyncio async def test_submit_http_bridge_request_checks_queue_before_inlining(monkeypatch): service = proxy_service.ProxyService.__new__(proxy_service.ProxyService) + service._durable_bridge = None proxy_service._initialize_http_bridge_retry_circuit(service) request_state = proxy_service._WebSocketRequestState( request_id="req_submit_queue_full_inline", diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index 428b5057b1..328bacdc0d 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -51,10 +51,12 @@ def _isolated_settings(**overrides: Any) -> Settings: # gate, issue #1535). Not a hardcoded default because the right congestion # threshold depends on pool size and workload mix, and 0-means-off is the P1 # default-off switch; the companion min-guarantee constant stayed hardcoded. -# 117 -> 118: http_responses_session_bridge_anchor_poison_failure_threshold -# (bridge restart anchor poisoning). Not hardcoded because operators need a -# bounded deployment-specific poison threshold while recovery telemetry matures. -MAX_SETTINGS_FIELDS = 118 +# 117 -> 126: durable HTTP bridge continuity controls (operation ledger, +# ambiguous-continuation recovery, and best-effort transcript spool, #1657). +# These remain operator-selectable because deployments differ in recovery +# safety policy and available persistence/latency budgets; their conservative +# defaults preserve fail-closed behavior and bound background write work. +MAX_SETTINGS_FIELDS = 126 def test_generated_settings_reference_matches_code() -> None: @@ -68,6 +70,7 @@ def test_generated_settings_reference_matches_code() -> None: def test_settings_reference_page_is_checked_in_under_docs() -> None: assert OUTPUT_PATH == REPO_ROOT / "docs" / "reference" / "settings.md" assert OUTPUT_PATH.is_file() + assert "openspec/specs/responses-api-compat" in render_settings_reference() def test_settings_surface_ratchet() -> None: diff --git a/tests/unit/test_sticky_session_cleanup_scheduler.py b/tests/unit/test_sticky_session_cleanup_scheduler.py index 2a0eb22df6..ae904eee52 100644 --- a/tests/unit/test_sticky_session_cleanup_scheduler.py +++ b/tests/unit/test_sticky_session_cleanup_scheduler.py @@ -53,6 +53,7 @@ async def test_cleanup_once_purges_prompt_cache_only(monkeypatch) -> None: lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -64,6 +65,7 @@ async def test_cleanup_once_purges_prompt_cache_only(monkeypatch) -> None: bridge_repo.purge_closed_before = AsyncMock(return_value=2) bridge_repo.purge_abandoned_before = AsyncMock(return_value=1) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=3) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -95,6 +97,7 @@ async def __aexit__(self, *args): bridge_repo.purge_closed_before.assert_called_once() bridge_repo.purge_abandoned_before.assert_called_once() bridge_repo.purge_retry_circuits_before.assert_called_once() + bridge_repo.purge_operation_spool.assert_called_once() ring_service.purge_stale_before.assert_called_once() sticky_repo.purge_stale_hard_codex_session_mappings.assert_called_once() passed_cutoff = sticky_repo.purge_stale_hard_codex_session_mappings.call_args.args[0] @@ -117,6 +120,7 @@ async def test_cleanup_once_skips_bridge_purge_when_schema_is_not_ready(monkeypa lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -127,6 +131,7 @@ async def test_cleanup_once_skips_bridge_purge_when_schema_is_not_ready(monkeypa bridge_repo.purge_closed_before = AsyncMock(return_value=0) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -180,6 +185,7 @@ async def test_cleanup_once_purges_bridge_when_schema_exists_after_startup_flag_ lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -190,6 +196,7 @@ async def test_cleanup_once_purges_bridge_when_schema_exists_after_startup_flag_ bridge_repo.purge_closed_before = AsyncMock(return_value=1) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=2) @@ -221,6 +228,7 @@ async def __aexit__(self, *args): bridge_repo.purge_closed_before.assert_called_once() bridge_repo.purge_abandoned_before.assert_called_once() bridge_repo.purge_retry_circuits_before.assert_called_once() + bridge_repo.purge_operation_spool.assert_called_once() ring_service.purge_stale_before.assert_called_once() @@ -268,6 +276,7 @@ async def test_cleanup_once_gates_abandoned_purge_on_prompt_cache_reuse_ttl(monk lambda: SimpleNamespace( http_responses_session_bridge_idle_ttl_seconds=120.0, http_responses_session_bridge_codex_idle_ttl_seconds=900.0, + http_responses_session_bridge_operation_spool_retention_seconds=604800.0, ), ) @@ -278,6 +287,7 @@ async def test_cleanup_once_gates_abandoned_purge_on_prompt_cache_reuse_ttl(monk bridge_repo.purge_closed_before = AsyncMock(return_value=0) bridge_repo.purge_abandoned_before = AsyncMock(return_value=0) bridge_repo.purge_retry_circuits_before = AsyncMock(return_value=0) + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) ring_service = AsyncMock() ring_service.purge_stale_before = AsyncMock(return_value=0) @@ -310,3 +320,40 @@ async def __aexit__(self, *args): # must be retained for the full 3600s prompt-cache reuse window. gap_seconds = (closed_cutoff - abandoned_cutoff).total_seconds() assert abs(gap_seconds - 1800.0) < 5.0 + + +@pytest.mark.asyncio +async def test_cleanup_once_retains_operation_purge_when_sticky_cleanup_disabled(monkeypatch) -> None: + settings_repo = AsyncMock() + sticky_repo = AsyncMock() + bridge_repo = AsyncMock() + bridge_repo.purge_operation_spool = AsyncMock(return_value=0) + + class FakeSession: + async def __aenter__(self): + return AsyncMock() + + async def __aexit__(self, *args): + pass + + monkeypatch.setattr( + cleanup_scheduler, + "get_settings", + lambda: SimpleNamespace(http_responses_session_bridge_operation_spool_retention_seconds=604800.0), + ) + scheduler = cleanup_scheduler.StickySessionCleanupScheduler(interval_seconds=60, enabled=False) + + with ( + patch.object(cleanup_scheduler, "get_background_session", FakeSession), + patch.object(cleanup_scheduler, "SettingsRepository", return_value=settings_repo), + patch.object(cleanup_scheduler, "StickySessionsRepository", return_value=sticky_repo), + patch.object(cleanup_scheduler, "DurableBridgeRepository", return_value=bridge_repo), + patch.object(cleanup_scheduler, "_get_leader_election", lambda: _FakeLeader()), + patch.object(cleanup_scheduler.startup_module, "_bridge_durable_schema_ready", True), + ): + await scheduler._cleanup_once() + + settings_repo.get_or_create.assert_not_awaited() + sticky_repo.purge_prompt_cache_before.assert_not_awaited() + bridge_repo.purge_closed_before.assert_not_awaited() + bridge_repo.purge_operation_spool.assert_awaited_once() diff --git a/tests/unit/test_websocket_terminal_cancellation.py b/tests/unit/test_websocket_terminal_cancellation.py index 5d2665c35b..45122c622e 100644 --- a/tests/unit/test_websocket_terminal_cancellation.py +++ b/tests/unit/test_websocket_terminal_cancellation.py @@ -1240,6 +1240,79 @@ async def owned_child() -> None: await asyncio.wait_for(child, timeout=1) +@pytest.mark.asyncio +async def test_stuck_upstream_close_is_cancelled_after_scope_cleanup_timeout() -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + close_started = asyncio.Event() + release_close = asyncio.Event() + close_cancelled = False + + async def close() -> None: + nonlocal close_cancelled + close_started.set() + try: + await release_close.wait() + except asyncio.CancelledError: + close_cancelled = True + raise + + upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close)) + + cleanup = asyncio.create_task( + websocket_mixin._close_websocket_upstream_for_cleanup( + service, + upstream, + timeout_seconds=1.0, + ) + ) + await asyncio.wait_for(close_started.wait(), timeout=1) + await asyncio.wait_for(cleanup, timeout=1) + + assert close_cancelled is True + assert service._background_cleanup_tasks == set() + release_close.set() + + +@pytest.mark.asyncio +async def test_upstream_close_is_cancelled_when_cleanup_budget_is_exhausted() -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + close_started = asyncio.Event() + close_cancelled = False + + async def close() -> None: + nonlocal close_cancelled + close_started.set() + try: + await asyncio.Event().wait() + except asyncio.CancelledError: + close_cancelled = True + raise + + upstream = cast(UpstreamWebSocket, SimpleNamespace(close=close)) + + await websocket_mixin._close_websocket_upstream_for_cleanup( + service, + upstream, + timeout_seconds=0.0, + ) + + await asyncio.wait_for(close_started.wait(), timeout=1) + for _ in range(20): + if close_cancelled and not service._background_cleanup_tasks: + break + await asyncio.sleep(0) + assert close_cancelled is True + assert service._background_cleanup_tasks == set() + + @pytest.mark.asyncio @pytest.mark.parametrize("message_kind", ["text", "transport_end"]) async def test_reader_cancellation_remains_cancelled_when_owned_child_fails( From 7c4671980094135b9094278d2ebb374c7cb22655 Mon Sep 17 00:00:00 2001 From: Roman Leventov Date: Wed, 12 Aug 2026 14:22:02 +0800 Subject: [PATCH 002/117] fix(http-bridge): keep idle retirements out of retry circuit (#1677) * fix(http-bridge): keep idle retirements out of retry circuit Advance a hard-key retry circuit only when bridge retirement owns at least one pending request and that request has emitted no response event. Routine idle socket retirement remains diagnostic but no longer creates phantom failures or premature cooldowns. * fix(http-bridge): derive circuit evidence on retirement --------- Co-authored-by: Soju06 --- .../_service/http_bridge/request_submit.py | 42 +++- .../_service/http_bridge/upstream_events.py | 8 + .../.openspec.yaml | 0 .../design.md | 116 ++++++++++ .../proposal.md | 10 + .../specs/responses-api-compat/spec.md | 79 ++++++- .../tasks.md | 8 + .../specs/responses-api-compat/context.md | 8 + openspec/specs/responses-api-compat/spec.md | 218 +++++++++++++++++- .../integration/test_http_responses_bridge.py | 84 +++++++ tests/unit/test_proxy_http_bridge.py | 121 +++++++++- 11 files changed, 679 insertions(+), 15 deletions(-) rename openspec/changes/{recover-repeated-clean-close => archive/2026-08-10-recover-repeated-clean-close}/.openspec.yaml (100%) create mode 100644 openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md rename openspec/changes/{recover-repeated-clean-close => archive/2026-08-10-recover-repeated-clean-close}/proposal.md (81%) rename openspec/changes/{recover-repeated-clean-close => archive/2026-08-10-recover-repeated-clean-close}/specs/responses-api-compat/spec.md (69%) rename openspec/changes/{recover-repeated-clean-close => archive/2026-08-10-recover-repeated-clean-close}/tasks.md (66%) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index d987ea3ecc..23216a8c36 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -2698,16 +2698,56 @@ async def _retire_stale_pending_http_bridge_session( detail: str, retry_circuit_detail: str | None = None, response_events_seen: int | None = None, + retired_request_count: int | None = None, ) -> None: async with session.pending_lock: retired_request_states = list(session.pending_requests) + if retired_request_count is None: + retired_request_count = sum( + 1 + for request_state in retired_request_states + if _http_bridge_request_counts_against_queue(request_state) + ) + if response_events_seen is None: + # Direct retirement must derive event evidence from the same + # locked ownership snapshot as the pending count. Otherwise an + # eventful stale-gate owner looks eventless merely because its + # caller omitted this optional handoff, creating a false + # circuit strike. Explicit values remain authoritative for + # reader-failure callers whose pending deque was already + # drained before entering this shared boundary. + response_events_seen = max( + ( + max( + request_state.response_event_count, + int( + request_state.response_id is not None + or request_state.latency_response_created_ms is not None + or request_state.downstream_visible + ), + ) + for request_state in retired_request_states + ), + default=0, + ) # Direct retirement (for example the all-stale stuck-gate path, where # the wedged reattach is the only pending request) cancels the reader # and fails the pendings without passing the partial-cleanup hook or # the reader-failure funnel, so evaluate the wedge shape (#1534) here # too; recording is idempotent for callers that already quarantined. _record_http_bridge_quarantine_wedged_pending(self, session, retired_request_states) - if response_events_seen is None or response_events_seen == 0: + # This circuit measures failed request lifecycles, not upstream socket + # churn. ``response_events_seen == 0`` is also true when an idle reader + # closes with an empty pending deque. Charging that idle close creates a + # phantom first strike, so one later response-create timeout opens the + # nominally "repeated" 60-second cooldown and interrupts the client. + # Keep the ownership proof at this shared retirement boundary unless a + # caller already claimed and drained the deque. The reader-failure + # funnel must pass its pre-drain count because terminal notification + # deliberately empties ``pending_requests`` before retirement. Without + # that handoff, genuine pre-response failures disappear from circuit + # accounting while idle closes and request failures look identical. + if retired_request_count > 0 and response_events_seen == 0: await self._record_http_bridge_retry_circuit_failure( session, detail=retry_circuit_detail or detail, diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index f240b09078..431aaea01a 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -1090,12 +1090,20 @@ async def _fail_http_bridge_reader_and_maybe_retire( detail=error_code, retry_circuit_detail="clean_close", response_events_seen=observed_response_events, + retired_request_count=failed_pending_count, ) else: await self._retire_stale_pending_http_bridge_session( session, detail=retire_detail or error_code, response_events_seen=observed_response_events, + # ``_fail_pending_websocket_requests`` has already + # claimed and drained these states. Carry the count + # sampled under ``pending_lock`` across that ownership + # transfer so normal reader failures still consume one + # strike. The deferred/poison branch records its own + # strike above and intentionally does not pass it. + retired_request_count=failed_pending_count, ) return force_retire or session.admission_waiter_count == 0 diff --git a/openspec/changes/recover-repeated-clean-close/.openspec.yaml b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/.openspec.yaml similarity index 100% rename from openspec/changes/recover-repeated-clean-close/.openspec.yaml rename to openspec/changes/archive/2026-08-10-recover-repeated-clean-close/.openspec.yaml diff --git a/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md new file mode 100644 index 0000000000..3922d38c33 --- /dev/null +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/design.md @@ -0,0 +1,116 @@ +## Context + +The HTTP Responses bridge multiplexes downstream requests over a reusable +upstream WebSocket. Recovery can be initiated either by the upstream reader or +by the downstream HTTP stream watchdog, so socket replacement, reader +ownership, pending-request settlement, and retry-circuit accounting cross +several asynchronous lifecycle boundaries. See `proposal.md` for motivation +and `specs/responses-api-compat/spec.md` for the normative contract. + +Hard-affinity retry circuits are durable across replicas. Their evidence must +therefore describe a client-affecting request lifecycle, not merely a socket +lifecycle event, because idle socket retirement is normal bridge maintenance. + +## Goals / Non-Goals + +**Goals:** + +- Transfer reader ownership atomically when a downstream watchdog replaces the + upstream socket. +- Bound pre-visible recovery so it completes before the downstream client + deadline without permitting duplicate visible work. +- Count only request-affecting, pre-response bridge failures toward the durable + hard-key circuit. +- Preserve circuit state across replicas while bounding process-local and + durable stale state. + +**Non-Goals:** + +- Replay work after any response event has become visible. +- Replay delivery-ambiguous liveness failures or continuity-sensitive payloads. +- Suppress a cooldown after two genuine consecutive eventless request + failures. +- Change the Codex client's WebSocket-to-HTTP fallback policy. + +## Decisions + +### Treat the reader and socket as one generation + +When recovery originates outside the reader, the bridge cancels and awaits the +old reader before locally closing its socket, keeps the shared session live +during replacement, and starts exactly one reader for the new socket. The old +reader's finalizer is generation-guarded so it cannot retire pending work that +has moved to the replacement. + +Allowing old and new readers to overlap was rejected because a local close can +wake the old reader after the pending deque has already been transferred. A +simple `closed` flag was also rejected because it cannot distinguish the +superseded socket generation from the shared session lifetime. + +### Keep pre-visible replay bounded and ahead of the client deadline + +The bridge permits one additional clean-close replay only after the existing +first replay, only before any response event, and with bounded jitter. Silent +pre-response recovery starts after no more than six default ten-second +keepalive intervals, leaving headroom before a 120-second client deadline. + +An unbounded reconnect loop was rejected because it can duplicate requests, +hide deterministic input rejection, and outlive the downstream caller. + +### Derive circuit evidence from an owned request lifecycle + +Retirement advances the circuit only when the retiring session still owns a +pending request and that lifecycle has observed zero response events. The +eligibility snapshot is taken while lifecycle ownership is known; an idle +session with no pending request remains visible in diagnostics but is neutral +to the circuit. A request that emitted any event is excluded because the +pre-response circuit cannot safely characterize a midstream failure. + +Counting every socket retirement was rejected because routine idle churn +creates phantom first strikes. Counting only error labels was rejected because +the same transport label can describe idle maintenance, pre-response failure, +or midstream loss. + +### Persist hard-key circuits and merge conservatively + +Circuit rows are scoped by hard-affinity kind, key, and API-key scope. Conflict +updates cannot shorten an existing cooldown, retry decisions refresh durable +state, success clears state, and stale local/durable entries expire. Durable +lookup failures degrade to local state with diagnostics rather than failing the +request. + +Process-local-only state was rejected because another replica could continue +replay during an open cooldown. Treating persistence failure as terminal was +rejected because the circuit is protective metadata, not request continuity +state. + +### Judge stuck gates from upstream activity + +The watchdog uses elapsed upstream inactivity plus the absence of a response +identifier or `response.created` latency. A prior continuity anchor receives a +bounded second threshold, not an indefinite exemption. Admission flags alone +were rejected because they can remain ambiguous while the upstream socket is +silent. + +## Risks / Trade-offs + +- [A replacement is also silent] -> The extra replay remains hard-capped and + the request reaches terminal or circuit handling. +- [Reader cancellation races with pruning] -> Session handoff state keeps the + shared lifecycle live until replacement ownership is established. +- [Concurrent replicas record failures] -> Durable merge semantics preserve + the longest applicable cooldown. +- [A genuine failure occurs after an idle close] -> The idle close contributes + no strike, so the genuine failure is correctly treated as the first one. +- [Database ancestry was stamped before a merge edge existed] -> A separate + forward-only repair reconnects the request-usage rollup history without + rewriting deployed migrations. + +## Migration Plan + +Apply the forward-only database revisions, deploy the revision-labelled image, +and verify bridge create/reuse, timeout, and retry-circuit diagnostics. Health +verification must confirm the expected image revision and current schema. +Rollback is an image replacement; the prior version can ignore the additional +runtime behavior while the durable circuit table and repair revision remain +forward-compatible. diff --git a/openspec/changes/recover-repeated-clean-close/proposal.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md similarity index 81% rename from openspec/changes/recover-repeated-clean-close/proposal.md rename to openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md index 04d3093d7e..45ecce0254 100644 --- a/openspec/changes/recover-repeated-clean-close/proposal.md +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/proposal.md @@ -9,6 +9,11 @@ upstream failure and retires work already moved to the replacement socket. Together these behaviors make a transient handoff issue visible as a reconnect loop and require the Codex client to be restarted. +Post-deploy evidence exposed a related accounting gap: retiring an idle bridge +with no pending request still records a retry-circuit failure. The next real +pre-response timeout can therefore open the repeated-failure cooldown after +only one client-affecting failure. + ## What Changes - Permit one additional pre-visible replay when the replacement upstream @@ -31,6 +36,9 @@ loop and require the Codex client to be restarted. response creation, rather than admission flags alone. Give requests with a prior continuity anchor a bounded two-threshold grace period, and emit diagnostic state when the watchdog skips a candidate. +- Count retirement failures only when the bridge still owns a pending request + that has not emitted a response event; idle no-pending closes remain visible + in lifecycle diagnostics but do not consume retry-circuit strikes. ## Impact @@ -39,6 +47,8 @@ loop and require the Codex client to be restarted. - The retry remains bounded and does not create an unbounded replay loop. - Reader ownership follows the active socket across idle recovery, preventing locally generated close frames from being counted as upstream instability. +- Idle upstream connection churn no longer turns one later request timeout into + an immediate sixty-second hard-key cooldown. - Adds the `http_bridge_retry_circuits` durable table and migration so retry cooldown state survives cross-replica clean-close and incomplete-stream failures. diff --git a/openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md similarity index 69% rename from openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md index d825873a65..61b83d8285 100644 --- a/openspec/changes/recover-repeated-clean-close/specs/responses-api-compat/spec.md +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/specs/responses-api-compat/spec.md @@ -41,6 +41,12 @@ being considered stale. When the watchdog skips a candidate, it MUST emit a low-cardinality diagnostic containing the session-closed state, candidate count, and pending-state verdicts. +#### Scenario: clean close before response.created is not retried + +- **WHEN** the initial upstream HTTP responses bridge closes with `close_code = 1000` before any `response.*` event for the pending request +- **THEN** the proxy returns HTTP 502 with `error.code = "upstream_rejected_input"` +- **AND** does not transparently replay the pre-created request + #### Scenario: clean close before response output receives one bounded additional replay - **GIVEN** an HTTP bridge request has no surfaced `response.*` events @@ -115,6 +121,13 @@ when no API key is present). The proxy MUST record only the documented pre-response failure classes (`stream_incomplete`, `clean_close`, and `stream_idle_timeout`). +A bridge retirement MUST record one of those failures only when the retiring +session still owns at least one pending request and no response event has been +observed for that request lifecycle. Retiring an idle upstream bridge with no +pending request MUST NOT advance the circuit or cause a later request to be +treated as a repeated failure. A pending request that has already emitted a +response event MUST remain excluded from this pre-response circuit. + The default circuit MUST open after two consecutive recorded failures. Once open, it MUST suppress pre-created replay until the persisted cooldown expires, using exponential backoff from sixty seconds up to ten minutes. Clean-close @@ -139,6 +152,25 @@ record the failure for observability. Rows older than one hour MUST be treated as expired and removed. A successful terminal response MUST clear the local and durable circuit state. +#### Scenario: idle bridge retirement does not consume a circuit strike + +- **GIVEN** a hard-affinity HTTP bridge has no pending requests +- **WHEN** its upstream WebSocket closes and the idle bridge is retired +- **THEN** the retry-circuit failure count for that key remains unchanged +- **AND** a later request is not placed in cooldown because of the idle close + +#### Scenario: eventless pending retirement consumes exactly one strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with no observed response event +- **WHEN** the bridge retires because the upstream fails before acknowledging the request +- **THEN** the retry circuit records exactly one failure for that request lifecycle + +#### Scenario: midstream retirement does not consume a pre-response strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with an observed response event +- **WHEN** the bridge retires before completion +- **THEN** the pre-response retry-circuit failure count remains unchanged + #### Scenario: the second hard-key failure opens a durable circuit - **GIVEN** a hard-affinity key has one recorded pre-response failure @@ -174,14 +206,45 @@ When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide -network failure, is a clean close (`close_code = 1000`) before any -`response.*` event, or carries the classified per-socket -`upstream_keepalive_timeout` transport error. Clean pre-response closes and -keepalive timeouts MUST remain account-neutral while using the bounded retry -and retry-circuit handling above. A classified process-wide network failure -MUST remain account neutral and use its network error code. For other closes, -the proxy MUST surface -`stream_incomplete` to affected pending requests. +network failure or upstream WebSocket liveness timeout, is a clean close +(`close_code = 1000`) before any `response.*` event, or carries the classified +per-socket `upstream_keepalive_timeout` transport error. Clean pre-response +closes, keepalive timeouts, process-wide network failures, and liveness +timeouts MUST remain account-neutral and use their classified error and bounded +retry or retry-circuit handling. For other closes, the proxy MUST surface +`stream_incomplete` to affected pending requests except when a direct Responses +WebSocket request has already successfully emitted a finite integer +`sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST +record the request outcome as `stream_incomplete` without emitting a synthetic +terminal frame under the active response id, then MUST close the downstream +WebSocket with code 1011. + +#### Scenario: websocket closes before pending responses complete + +- **GIVEN** a streamed response request is pending on an upstream websocket +- **AND** the direct downstream response has not emitted a numeric sequence, or the request uses another transport +- **WHEN** the websocket closes before a terminal response event is observed +- **AND** the close does not carry a classified process-wide network failure or upstream WebSocket liveness timeout +- **THEN** the pending request fails with `stream_incomplete` +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: sequenced direct websocket closes before completion + +- **GIVEN** a direct Responses WebSocket request has successfully emitted a finite integer `sequence_number` +- **WHEN** the upstream websocket closes before a terminal response event is observed +- **AND** the close does not carry a classified process-wide network failure or upstream WebSocket liveness timeout +- **THEN** the request is recorded as failed with `stream_incomplete` +- **AND** no synthetic terminal frame is emitted under the active response id +- **AND** the downstream WebSocket closes with code 1011 +- **AND** the account receives a transient upstream failure signal for routing + +#### Scenario: websocket liveness timeout remains account neutral + +- **GIVEN** a streamed response request is pending on an upstream websocket +- **WHEN** its transport reports `upstream_websocket_liveness_timeout` +- **THEN** the pending request fails with that classified error code +- **AND** the account receives no failure-health signal +- **AND** the request is not transparently replayed #### Scenario: clean pre-response close does not penalize the account diff --git a/openspec/changes/recover-repeated-clean-close/tasks.md b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md similarity index 66% rename from openspec/changes/recover-repeated-clean-close/tasks.md rename to openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md index aa0246098d..7e1be1873a 100644 --- a/openspec/changes/recover-repeated-clean-close/tasks.md +++ b/openspec/changes/archive/2026-08-10-recover-repeated-clean-close/tasks.md @@ -12,3 +12,11 @@ - [x] Add a forward-only repair for databases stamped before request-usage rollups were connected to the merge head. - [x] Validate the OpenSpec change and run the focused and full test suites. - [x] Build and deploy the validated image, then verify production health and logs. + +## Post-deploy regression: idle retirement accounting + +- [x] Require an owned eventless pending request before retirement advances the retry circuit. +- [x] Add lifecycle coverage proving idle no-pending retirement is neutral and eventless pending retirement records exactly one strike. +- [x] Add routed coverage proving an idle close plus one real timeout does not open the repeated-failure cooldown. +- [x] Run focused bridge suites, lint/type/architecture checks, and strict OpenSpec validation. +- [x] Build and deploy the revised image, then verify health and retry-circuit diagnostics. diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index 9a62f360da..583fc9b4a8 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -35,6 +35,7 @@ See `openspec/specs/responses-api-compat/spec.md` for normative requirements. - Upstream Responses WebSockets use transport ping/pong control frames to detect a black-holed connection without confusing valid application-event silence with an idle turn. Direct and routed connections reuse `proxy_downstream_websocket_idle_timeout_seconds` for this zero-config liveness budget. - A post-send liveness timeout is delivery-ambiguous. It remains account-neutral, is never transparently replayed, and retires the affected upstream socket so a client retry opens a fresh route without risking duplicated model work or tool side effects. - HTTP bridge settlement ownership is explicit: `closed` rejects new work but does not imply that a submitter owns existing siblings. Only a liveness-failed send claims whole-deque settlement under the lifecycle lock; otherwise the reader remains responsible for settling pending requests when the transport dies. +- Hard-affinity retry-circuit evidence is request-lifecycle evidence: retirement counts only while the bridge still owns an eventless pending request. Idle no-pending retirement remains observable but neutral, so routine socket churn cannot manufacture the first strike for a later real timeout. ## Fast Mode and Service Tiers @@ -118,6 +119,7 @@ when upstream reports a different actual tier. - **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are surfaced as `response.failed` / `codex_previous_response_stale` without the raw upstream code or missing `resp_...` id; OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. - **Websocket handshake forbidden/not-found:** Auto transport now fails loud on `403` / `404` instead of silently hiding the websocket regression behind HTTP fallback. - **Upstream websocket stops answering pings:** Pending direct-WebSocket and HTTP-bridge work fails with `upstream_websocket_liveness_timeout`; the account remains healthy and the request is not replayed because upstream acceptance is unknown. +- **Repeated eventless bridge failures:** Two consecutive request-affecting pre-response failures can open the hard-key cooldown. A successful terminal response clears the state; an idle close followed by one real timeout remains only one strike. - **Invalid request payloads:** Return 4xx with `invalid_request_error`. ## Error Envelope Mapping (Reference) @@ -150,6 +152,11 @@ Cursor-style model alias request: This forwards upstream as `model: "gpt-5.4-mini"` with `reasoning.effort: "high"`. +Retry-circuit accounting example: an idle bridge closes with `pending=0`, then +the next request times out before `response.created`. The idle close is logged +but contributes no failure; the timeout is the first strike. Only another +consecutive eventless pending failure may open the repeated-failure cooldown. + ## Known Client Integrations (Reference) Third-party agents that consume the `/v1` Responses surface documented by this @@ -179,5 +186,6 @@ OpenSpec change first. - When tracing compact incidents, confirm that request logs and upstream logs show direct `/codex/responses/compact` usage without surrogate `/codex/responses` fallback. - Post-deploy: monitor `no_accounts`, `stream_incomplete`, and `upstream_unavailable`. - Post-deploy: monitor `upstream_websocket_liveness_timeout`; recurring failures indicate a host route, VPN, proxy, or intermediary that black-holes established WebSockets. +- Post-deploy: correlate retry-circuit `opened`, `half_open`, and `reset` events with bridge `pending` and `response_events_seen` diagnostics. An idle `pending=0` retirement must not precede an immediate two-failure cooldown. - Post-deploy: monitor `codex_previous_response_stale` on `/backend-api/codex/responses`; recurring spikes mean clients are still relying on stale upstream anchors and should perform the documented full-context retry without `previous_response_id`. - Websocket/Codex CLI tier verification runbook: `openspec/specs/responses-api-compat/ops.md` diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 4ddc97a32b..f210efbdaa 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -67,14 +67,204 @@ When `upstream_stream_transport` is `"auto"` and the serialized request payload ### Requirement: Clean upstream close before any response event fails fast -When the HTTP responses bridge observes an upstream websocket close with `close_code = 1000` before any `response.*` event has been surfaced for the pending request, the proxy MUST classify the close as rejected input, surface HTTP 502 `upstream_rejected_input`, and MUST NOT trigger `retry_precreated` or `retry_fresh_upstream`. +When the HTTP Responses bridge observes an upstream WebSocket close with +`close_code = 1000` before any `response.*` event has been surfaced for the +pending request, the proxy MUST preserve its existing pre-visible replay +guards. If the request has already used exactly one eligible pre-visible +replay and the replacement upstream WebSocket also closes cleanly before any +response event, the proxy MAY perform exactly one additional replay. The +additional replay MUST be hard-capped at one per request, and the configured +maximum MUST NOT raise that cap. + +The proxy MUST NOT replay after downstream-visible output, after a terminal +response event, or when continuity-sensitive request state makes replay unsafe. +Before the additional replay, the proxy MAY sleep for bounded configured +jitter. The proxy MUST emit a dedicated low-cardinality diagnostic event for +the additional replay. + +When a downstream HTTP stream task initiates pre-response recovery while the +upstream reader is blocked on the superseded socket, the proxy MUST cancel and +await that reader before locally closing the socket. It MUST then start exactly +one reader for the replacement socket. A close caused by replacing the socket +MUST NOT be recorded as an upstream clean-close failure, MUST NOT increment the +retry circuit, and MUST NOT retire pending work moved to the replacement. The +cancelled reader's socket-generation finalizer MUST NOT leave the shared session +marked closed while the replacement socket is being selected or opened, so idle +pruning MUST NOT evict the handoff in progress. + +The default pre-response idle-recovery window MUST leave bounded headroom +before the downstream client's request timeout. With the default ten-second +keepalive interval, the proxy MUST initiate eligible recovery after no more +than six silent intervals so replacement connection and first output can occur +before a 120-second client deadline. + +The stuck pre-response watchdog MUST judge staleness using elapsed time since +the last upstream activity and the absence of a response identifier or +`response.created` latency, not admission flags alone. A request with a prior +continuity anchor MUST receive at most two retire-thresholds of grace before +being considered stale. When the watchdog skips a candidate, it MUST emit a +low-cardinality diagnostic containing the session-closed state, candidate +count, and pending-state verdicts. #### Scenario: clean close before response.created is not retried -- **WHEN** upstream closes the HTTP responses bridge with `close_code = 1000` before any `response.*` event for the pending request +- **WHEN** the initial upstream HTTP responses bridge closes with `close_code = 1000` before any `response.*` event for the pending request - **THEN** the proxy returns HTTP 502 with `error.code = "upstream_rejected_input"` - **AND** does not transparently replay the pre-created request +#### Scenario: clean close before response output receives one bounded additional replay + +- **GIVEN** an HTTP bridge request has no surfaced `response.*` events +- **AND** its first pre-visible replay has already been used +- **WHEN** the replacement upstream WebSocket closes with code `1000` +- **THEN** the proxy performs one additional pre-visible replay +- **AND** the request replay count increases by one +- **AND** the proxy emits a `retry_precreated_clean_close` diagnostic event + +#### Scenario: repeated clean closes do not create an unbounded replay loop + +- **GIVEN** the additional clean-close replay has already been used +- **WHEN** another upstream WebSocket closes cleanly before response output +- **THEN** the proxy does not replay the request again +- **AND** the existing terminal or circuit handling is used + +#### Scenario: visible output still prevents clean-close replay + +- **GIVEN** the pending request has surfaced any response event downstream +- **WHEN** the upstream WebSocket closes with code `1000` +- **THEN** the proxy does not replay the request + +#### Scenario: clean-close retry jitter is bounded + +- **GIVEN** clean-close retry jitter is configured +- **WHEN** the additional clean-close replay is scheduled +- **THEN** the delay is no greater than the configured jitter maximum +- **AND** the hard replay cap remains one regardless of the configured value + +#### Scenario: downstream idle recovery transfers reader ownership + +- **GIVEN** the upstream reader is blocked on the current bridge socket +- **AND** the downstream HTTP stream task initiates eligible pre-response recovery +- **WHEN** the bridge replaces the upstream socket +- **THEN** the old reader is cancelled and awaited before its socket is closed +- **AND** the shared session remains live while the replacement socket opens +- **AND** idle pruning retains the registered session while the handoff is in progress +- **AND** exactly one reader owns the replacement socket +- **AND** the local close does not open or increment the retry circuit +- **AND** pending work remains attached to the replacement session + +#### Scenario: silent pre-response recovery precedes the client timeout + +- **GIVEN** the upstream has produced no response event +- **AND** the default ten-second keepalive interval is active +- **WHEN** six silent intervals elapse +- **THEN** the proxy initiates eligible pre-response recovery +- **AND** at least sixty seconds remain before a 120-second client request timeout + +#### Scenario: anchored stuck-gate grace is bounded + +- **GIVEN** a pending HTTP bridge request has a prior continuity anchor +- **AND** no response identifier or `response.created` latency has been recorded +- **WHEN** less than two retire thresholds have elapsed since the gate began waiting +- **THEN** the watchdog does not classify the request as stale +- **WHEN** two retire thresholds elapse without upstream activity +- **THEN** the watchdog may classify the request as stale + +#### Scenario: upstream activity resolves admission-flag ambiguity + +- **GIVEN** a pending request has not acquired the response-created gate +- **AND** upstream activity has not produced a response identifier or `response.created` +- **WHEN** the staleness threshold elapses +- **THEN** the watchdog classifies the request as stale +- **AND** emits pending-state verdict inputs when it skips a watchdog pass + +### Requirement: Durable retry-circuit state protects repeated hard-affinity failures + +For a hard-affinity bridge key, the proxy MUST scope retry-circuit state by +affinity kind, affinity key, and API-key scope (using a stable anonymous scope +when no API key is present). The proxy MUST record only the documented +pre-response failure classes (`stream_incomplete`, `clean_close`, and +`stream_idle_timeout`). + +A bridge retirement MUST record one of those failures only when the retiring +session still owns at least one pending request and no response event has been +observed for that request lifecycle. Retiring an idle upstream bridge with no +pending request MUST NOT advance the circuit or cause a later request to be +treated as a repeated failure. A pending request that has already emitted a +response event MUST remain excluded from this pre-response circuit. + +The default circuit MUST open after two consecutive recorded failures. Once +open, it MUST suppress pre-created replay until the persisted cooldown expires, +using exponential backoff from sixty seconds up to ten minutes. Clean-close +failures MUST cap their cooldown at thirty seconds. The proxy MUST persist +failure count, cooldown deadline, last failure detail, and update time in the +`http_bridge_retry_circuits` table and MUST merge conflict updates so concurrent +replicas cannot shorten an existing cooldown. + +The clean-close retry jitter maximum MUST be read from the +`http_responses_session_bridge_clean_close_retry_jitter_max_seconds` runtime +setting and MUST be bounded to the inclusive range 0–30 seconds. + +The proxy MUST evict process-local circuit entries and their loaded/persisted +markers after one hour without use, independently of durable-row cleanup, so +one-shot hard-affinity keys cannot grow the worker's memory without bound. + +Before every hard-affinity retry decision, the proxy MUST refresh the durable +row so a cooldown opened by another replica is observed even when this process +has already loaded the key. A durable lookup or persistence failure MUST NOT +crash the request; the proxy MUST continue using available local state and +record the failure for observability. Rows older than one hour MUST be treated +as expired and removed. A successful terminal response MUST clear the local +and durable circuit state. + +#### Scenario: idle bridge retirement does not consume a circuit strike + +- **GIVEN** a hard-affinity HTTP bridge has no pending requests +- **WHEN** its upstream WebSocket closes and the idle bridge is retired +- **THEN** the retry-circuit failure count for that key remains unchanged +- **AND** a later request is not placed in cooldown because of the idle close + +#### Scenario: eventless pending retirement consumes exactly one strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with no observed response event +- **WHEN** the bridge retires because the upstream fails before acknowledging the request +- **THEN** the retry circuit records exactly one failure for that request lifecycle + +#### Scenario: midstream retirement does not consume a pre-response strike + +- **GIVEN** a hard-affinity HTTP bridge owns a pending request with an observed response event +- **WHEN** the bridge retires before completion +- **THEN** the pre-response retry-circuit failure count remains unchanged + +#### Scenario: the second hard-key failure opens a durable circuit + +- **GIVEN** a hard-affinity key has one recorded pre-response failure +- **WHEN** a second eligible failure is recorded +- **THEN** the proxy opens the retry circuit +- **AND** persists at least two consecutive failures and a cooldown deadline +- **AND** subsequent pre-created replay is suppressed until that deadline + +#### Scenario: retry decisions observe a cooldown opened by another replica + +- **GIVEN** this replica previously looked up a hard-affinity key with no row +- **AND** another replica persists an open cooldown for that same key and API-key scope +- **WHEN** this replica evaluates the next pre-created retry +- **THEN** it refreshes durable state before deciding +- **AND** suppresses the retry for the persisted cooldown + +#### Scenario: circuit state remains isolated by key and API-key scope + +- **GIVEN** one hard-affinity key has an open circuit +- **WHEN** a different affinity key or API-key scope evaluates a retry +- **THEN** that request is not suppressed by the first key's circuit + +#### Scenario: durable circuit lookup failure does not fail the request + +- **GIVEN** durable retry-circuit lookup or persistence is unavailable +- **WHEN** the proxy evaluates or records a retry-circuit event +- **THEN** the request continues using any available local circuit state +- **AND** the failure is logged and exposed through retry-circuit observability + ### Requirement: Long Codex websocket turns tolerate extended upstream silence The default compact request budget MUST be at least 180 seconds, and the default upstream stream idle timeout MUST be at least 600 seconds, so long-running Codex turns can survive expensive compaction or tool execution without a local proxy watchdog ending the turn prematurely. @@ -128,7 +318,22 @@ The proxy MUST configure direct and routed upstream Responses WebSocket transpor - **AND** the submitter cancellation is preserved after settlement completes ### Requirement: Upstream websocket drops penalize affected accounts -When an upstream websocket closes while one or more streamed response requests are pending and have not reached a terminal event, the proxy MUST record a transient upstream error for the account before signaling failure for those pending requests, except when the close carries a classified process-wide network failure or upstream WebSocket liveness timeout. A classified process-wide network failure or upstream WebSocket liveness timeout MUST remain account neutral and use its classified error code. For other closes, the proxy MUST surface `stream_incomplete` to affected pending requests except when a direct Responses WebSocket request has already successfully emitted a finite integer `sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST record the request outcome as `stream_incomplete` without emitting a synthetic terminal frame under the active response id, then MUST close the downstream WebSocket with code 1011. +When an upstream websocket closes while one or more streamed response requests +are pending and have not reached a terminal event, the proxy MUST record a +transient upstream error for the account before signaling failure for those +pending requests, except when the close carries a classified process-wide +network failure or upstream WebSocket liveness timeout, is a clean close +(`close_code = 1000`) before any `response.*` event, or carries the classified +per-socket `upstream_keepalive_timeout` transport error. Clean pre-response +closes, keepalive timeouts, process-wide network failures, and liveness +timeouts MUST remain account-neutral and use their classified error and bounded +retry or retry-circuit handling. For other closes, the proxy MUST surface +`stream_incomplete` to affected pending requests except when a direct Responses +WebSocket request has already successfully emitted a finite integer +`sequence_number`. For that sequenced direct-WebSocket case, the proxy MUST +record the request outcome as `stream_incomplete` without emitting a synthetic +terminal frame under the active response id, then MUST close the downstream +WebSocket with code 1011. #### Scenario: websocket closes before pending responses complete @@ -157,6 +362,13 @@ When an upstream websocket closes while one or more streamed response requests a - **AND** the account receives no failure-health signal - **AND** the request is not transparently replayed +#### Scenario: clean pre-response close does not penalize the account + +- **GIVEN** a hard-affinity HTTP bridge request is pending with no surfaced response event +- **WHEN** the upstream websocket closes cleanly before response output +- **THEN** the proxy records the clean-close retry-circuit outcome +- **AND** the selected account is not penalized + ### Requirement: Single HTTP bridge previous-response misses recover or fail closed When an HTTP bridge session receives an anonymous upstream `previous_response_not_found` error for a single pending follow-up request, the service MUST treat the error as an internal continuity-loss signal. It MUST either recover through the existing previous-response rebind path or rewrite the error to a retryable continuity failure instead of forwarding the raw upstream invalid-request error. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 5a7f1cf7db..6498e5b7d5 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -38,6 +38,7 @@ from app.modules.proxy._service.http_bridge import quarantine as http_bridge_quarantine_module from app.modules.proxy._service.http_bridge import streaming as http_bridge_streaming_module from app.modules.proxy._service.http_bridge.helpers import ( + _make_http_bridge_session_header_fallback_key, _release_http_bridge_unanchored_handoff, _reserve_http_bridge_unanchored_handoff, ) @@ -12668,6 +12669,89 @@ async def fake_connect_responses_websocket( record_retry_circuit_failure.assert_not_awaited() +@pytest.mark.asyncio +async def test_backend_responses_http_bridge_idle_retirement_does_not_open_retry_circuit_on_next_failure( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_backend_idle_retirement_circuit", + "backend-idle-retirement-circuit@example.com", + ) + account = await _get_account(account_id) + upstream = _FakeBridgeUpstreamWebSocket("resp_idle_retirement_circuit") + + async def fake_select_account_with_budget(self, deadline, **kwargs): + del self, deadline, kwargs + return AccountSelection(account=account, error_message=None, error_code=None) + + async def fake_ensure_fresh_with_budget(self, target, *, force=False, timeout_seconds): + del self, force, timeout_seconds + return target + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + return upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_select_account_with_budget", fake_select_account_with_budget) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + session_id = "backend-idle-retirement-circuit-session" + prompt_cache_key = "backend-idle-retirement-circuit-thread" + headers = {"session_id": session_id} + bridge_key = _make_http_bridge_session_header_fallback_key( + headers=headers, + api_key=None, + explicit_prompt_cache_key=prompt_cache_key, + ) + assert bridge_key is not None + service = get_proxy_service_for_app(app_instance) + + # Reproduce the live ordering without waiting for production-scale + # watchdogs: an idle no-pending retirement, then one genuine pre-response + # request failure on the same hard key. Only the latter may be a strike. + idle_session = _make_dummy_bridge_session(bridge_key) + await service._retire_stale_pending_http_bridge_session( + idle_session, + detail="stream_incomplete", + response_events_seen=0, + ) + failed_request_session = _make_dummy_bridge_session(bridge_key) + failures = await service._record_http_bridge_retry_circuit_failure( + failed_request_session, + detail="missing_response_created_timeout", + ) + assert failures == 1 + assert await service._http_bridge_precreated_retry_allowed(failed_request_session) is True + + events = await _collect_sse_events( + async_client, + "/backend-api/codex/responses", + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "continue after one real timeout", + "prompt_cache_key": prompt_cache_key, + "stream": True, + }, + headers=headers, + ) + + _assert_created_text_delta_completed(events) + assert events[-1]["response"]["id"] == "resp_idle_retirement_circuit_1" + + @pytest.mark.asyncio async def test_retry_http_bridge_precreated_request_releases_pending_lock_before_reconnect(app_instance, monkeypatch): service = get_proxy_service_for_app(app_instance) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 672bc40935..fea5f4e594 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -23250,6 +23250,7 @@ async def test_http_bridge_liveness_timeout_is_neutral_not_replayed_and_forces_r session, detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, + retired_request_count=1, ) assert session.queued_request_count == 0 assert session.closed is True @@ -23400,6 +23401,7 @@ async def controlled_fail_reader( session, detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, + retired_request_count=2, ) @@ -23522,6 +23524,7 @@ def pending_sibling(request_id: str) -> proxy_service._WebSocketRequestState: session, detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, + retired_request_count=2, ) @@ -23641,7 +23644,12 @@ async def test_http_bridge_clean_close_before_response_does_not_penalize_account assert fail_pending.await_args is not None assert fail_pending.await_args.kwargs["penalize_account"] is False - retire.assert_awaited_once_with(session, detail="stream_incomplete", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="stream_incomplete", + response_events_seen=0, + retired_request_count=0, + ) @pytest.mark.asyncio @@ -23813,6 +23821,103 @@ async def test_retire_stale_pending_http_bridge_session_unregisters_aliases_and_ close.assert_awaited_once() +@pytest.mark.asyncio +async def test_http_bridge_idle_retirement_does_not_record_retry_circuit_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-idle-retire") + record_failure = AsyncMock() + close = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", close) + + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + response_events_seen=0, + ) + + record_failure.assert_not_awaited() + close.assert_awaited_once_with(session, reason="retire_stale_pending") + + +@pytest.mark.asyncio +async def test_http_bridge_eventless_pending_retirement_records_one_retry_circuit_failure( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner = _make_eventless_http_bridge_owner(request_id="req-eventless-retire") + session = _make_bridge_session( + key_value="bridge-eventless-retire", + pending_requests=deque([owner]), + queued_request_count=1, + ) + record_failure = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + await service._retire_stale_pending_http_bridge_session( + session, + detail="missing_response_created_timeout", + response_events_seen=0, + ) + + record_failure.assert_awaited_once_with(session, detail="missing_response_created_timeout") + + +@pytest.mark.asyncio +async def test_http_bridge_direct_retirement_derives_observed_response_events( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner = _make_eventless_http_bridge_owner(request_id="req-eventful-direct-retire") + owner.response_event_count = 1 + session = _make_bridge_session( + key_value="bridge-eventful-direct-retire", + pending_requests=deque([owner]), + queued_request_count=1, + ) + record_failure = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + await service._retire_stale_pending_http_bridge_session( + session, + detail="stuck_response_create_gate", + ) + + record_failure.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_reader_failure_preserves_pre_drain_request_for_retry_circuit( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner = _make_eventless_http_bridge_owner(request_id="req-reader-failure-retire") + session = _make_bridge_session( + key_value="bridge-reader-failure-retire", + pending_requests=deque([owner]), + queued_request_count=1, + ) + record_failure = AsyncMock() + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + retired = await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="upstream closed before response.completed", + penalize_account=False, + response_events_seen=0, + ) + + assert retired is True + assert not session.pending_requests + record_failure.assert_awaited_once_with(session, detail="stream_incomplete") + + @pytest.mark.asyncio async def test_http_bridge_retirement_does_not_record_midstream_retry_circuit_failure( monkeypatch: pytest.MonkeyPatch, @@ -24809,7 +24914,12 @@ async def test_http_bridge_eventless_timeout_force_retires_with_admission_waiter assert retired is True assert session.closed is True - retire.assert_awaited_once_with(session, detail="missing_response_created_timeout", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="missing_response_created_timeout", + response_events_seen=0, + retired_request_count=0, + ) fail_pending_await_args = fail_pending.await_args assert fail_pending_await_args is not None assert fail_pending_await_args.kwargs["penalize_account"] is False @@ -24833,7 +24943,12 @@ async def test_http_bridge_reader_failure_retires_without_waiters_when_notificat error_message="closed", ) - retire.assert_awaited_once_with(session, detail="stream_incomplete", response_events_seen=0) + retire.assert_awaited_once_with( + session, + detail="stream_incomplete", + response_events_seen=0, + retired_request_count=0, + ) @pytest.mark.asyncio From 6509dd0d4a577908e5940f35ba4c5ab6d66f23bc Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 12 Aug 2026 17:40:25 +0900 Subject: [PATCH 003/117] feat(reset-credits): add refresh scheduler enable toggle (#1701) * feat(reset-credits): add refresh scheduler enable toggle Expose rate_limit_reset_credits_refresh_enabled (default true) so operators can disable background reset-credit polling per replica. Scheduler start() becomes a no-op when disabled; the factory wires the setting. Updates the rate-limit-reset-credits OpenSpec delta, which previously mandated no toggle. Co-Authored-By: Claude Fable 5 * fix(tests): raise settings ratchet to 127 and regenerate settings reference Co-Authored-By: Claude Fable 5 * fix(reset-credits): warn when disabled polling starves persisted auto-redeem The refresh loop is the sole driver of automatic redemption, so rate_limit_reset_credits_refresh_enabled=false would silently disable a persisted auto_redeem_reset_credits_before_expiry opt-in. Disabled start() now reads the dashboard settings and logs a configuration-conflict warning naming both settings; the delta spec documents the precedence. Co-Authored-By: Claude Fable 5 * docs(settings): link rate-limit-reset-credits spec from the settings reference Co-Authored-By: Claude Fable 5 * fix(settings): reject new auto-redeem opt-in while reset-credit polling is disabled The startup conflict warning cannot guard a runtime PUT that enables auto_redeem_reset_credits_before_expiry after boot. The settings update now rejects a NEW opt-in with reset_credit_polling_disabled while the polling toggle is off; an already-persisted opt-in stays re-savable so full-payload dashboard saves of unrelated fields are not blocked. Route-level integration coverage for both paths; delta spec documents the contract. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/config/settings.py | 1 + .../usage/reset_credits_refresh_scheduler.py | 22 ++++ app/modules/settings/api.py | 14 +++ docs/reference/settings.md | 5 +- .../proposal.md | 19 ++++ .../specs/rate-limit-reset-credits/spec.md | 59 ++++++++++ .../add-reset-credits-refresh-toggle/tasks.md | 20 ++++ scripts/generate_settings_reference.py | 2 + tests/integration/test_settings_api.py | 43 ++++++++ ...test_rate_limit_reset_credits_scheduler.py | 104 ++++++++++++++++++ tests/unit/test_settings_reference.py | 7 +- 11 files changed, 293 insertions(+), 3 deletions(-) create mode 100644 openspec/changes/add-reset-credits-refresh-toggle/proposal.md create mode 100644 openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md create mode 100644 openspec/changes/add-reset-credits-refresh-toggle/tasks.md diff --git a/app/core/config/settings.py b/app/core/config/settings.py index d4692d7925..586c12edc2 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -287,6 +287,7 @@ class Settings(BaseSettings): usage_refresh_enabled: bool = True usage_refresh_interval_seconds: int = Field(default=60, gt=0) live_usage_ingestion_enabled: bool = True + rate_limit_reset_credits_refresh_enabled: bool = True rate_limit_reset_credits_refresh_interval_seconds: int = Field(default=60, gt=0) openai_cache_affinity_max_age_seconds: int = Field(default=1800, gt=0) warmup_model: str = "gpt-5.4-mini" diff --git a/app/core/usage/reset_credits_refresh_scheduler.py b/app/core/usage/reset_credits_refresh_scheduler.py index a107e504f2..564923d7be 100644 --- a/app/core/usage/reset_credits_refresh_scheduler.py +++ b/app/core/usage/reset_credits_refresh_scheduler.py @@ -64,16 +64,37 @@ class RateLimitResetCreditsRefreshScheduler: interval_seconds: int rng: random.Random = field(default_factory=random.Random) + enabled: bool = True _task: asyncio.Task[None] | None = None _stop: asyncio.Event = field(default_factory=asyncio.Event) _lock: asyncio.Lock = field(default_factory=asyncio.Lock) async def start(self) -> None: + if not self.enabled: + await self._warn_if_auto_redeem_conflicts() + return if self._task and not self._task.done(): return self._stop.clear() self._task = asyncio.create_task(self._run_loop()) + async def _warn_if_auto_redeem_conflicts(self) -> None: + # The refresh loop is the only driver of automatic redemption, so a + # disabled scheduler silently starves a persisted auto-redeem opt-in. + try: + async with get_background_session() as session: + dashboard_settings = await SettingsRepository(session).get_or_create() + auto_redeem_enabled = dashboard_settings.auto_redeem_reset_credits_before_expiry + except Exception: + logger.exception("Reset credits auto-redeem conflict check failed") + return + if auto_redeem_enabled: + logger.warning( + "rate_limit_reset_credits_refresh_enabled=false disables automatic reset-credit " + "redemption, but dashboard setting auto_redeem_reset_credits_before_expiry is " + "enabled; credits will expire without redemption until polling is re-enabled" + ) + async def stop(self) -> None: if not self._task: return @@ -386,4 +407,5 @@ def build_rate_limit_reset_credits_scheduler() -> RateLimitResetCreditsRefreshSc settings = get_settings() return RateLimitResetCreditsRefreshScheduler( interval_seconds=settings.rate_limit_reset_credits_refresh_interval_seconds, + enabled=settings.rate_limit_reset_credits_refresh_enabled, ) diff --git a/app/modules/settings/api.py b/app/modules/settings/api.py index 5f0cf941cc..c132267538 100644 --- a/app/modules/settings/api.py +++ b/app/modules/settings/api.py @@ -20,6 +20,7 @@ validate_dashboard_session, ) from app.core.clients.http import _build_ssl_context +from app.core.config.settings import get_settings as get_app_settings from app.core.config.settings_cache import get_settings_cache from app.core.crypto import TokenEncryptor from app.core.exceptions import DashboardBadRequestError, DashboardSettingsConflictError @@ -562,6 +563,19 @@ async def update_settings( and payload.upstream_proxy_default_pool_id is not None ): await _validate_proxy_pool_id(context, payload.upstream_proxy_default_pool_id) + if ( + payload.auto_redeem_reset_credits_before_expiry + and not current.auto_redeem_reset_credits_before_expiry + and not get_app_settings().rate_limit_reset_credits_refresh_enabled + ): + # The reset-credit refresh loop is the sole driver of automatic + # redemption; accepting the opt-in while polling is disabled would + # persist a setting that can never run. + raise DashboardBadRequestError( + "autoRedeemResetCreditsBeforeExpiry requires reset-credit polling; " + "set CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_ENABLED=true first", + code="reset_credit_polling_disabled", + ) try: legacy_threshold_provided = payload.sticky_reallocation_budget_threshold_pct is not None primary_threshold_provided = payload.sticky_reallocation_primary_budget_threshold_pct is not None diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 5f570be225..55e2789e7d 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 126 settings. Every setting is an environment +codex-lb currently exposes 127 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -148,6 +148,7 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_LIVE_USAGE_INGESTION_ENABLED` | `bool` | `True` | +| `CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_ENABLED` | `bool` | `True` | | `CODEX_LB_RATE_LIMIT_RESET_CREDITS_REFRESH_INTERVAL_SECONDS` | `int` | `60` | | `CODEX_LB_REQUEST_LOG_RETENTION_DAYS` | `int` | `0` | | `CODEX_LB_USAGE_FETCH_MAX_RETRIES` | `int` | `2` | @@ -318,4 +319,4 @@ issue [#1340](https://github.com/Soju06/codex-lb/issues/1340)): --- -*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* +*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [rate-limit-reset-credits](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* diff --git a/openspec/changes/add-reset-credits-refresh-toggle/proposal.md b/openspec/changes/add-reset-credits-refresh-toggle/proposal.md new file mode 100644 index 0000000000..bbe4130cc0 --- /dev/null +++ b/openspec/changes/add-reset-credits-refresh-toggle/proposal.md @@ -0,0 +1,19 @@ +## Why + +Reset-credit polling runs in every replica and issues one authenticated upstream `GET /wham/rate-limit-reset-credits` per eligible account per interval. Operators who do not use the reset-credit dashboard surface (or who run many replicas against large account fleets) currently have no way to shed that upstream call volume: the spec mandated that the scheduler always starts, and `rate_limit_reset_credits_refresh_interval_seconds` is constrained to positive values, so "off" is not expressible — stretching the interval still keeps periodic authenticated upstream traffic and the associated log/failure noise. + +## What Changes + +- Add setting `rate_limit_reset_credits_refresh_enabled` (default `true`) that gates background reset-credit polling. +- When disabled, the scheduler's `start()` is a no-op: no background task is created, no upstream fetches occur, and snapshot caches simply stay empty (dashboard reads already handle a missing snapshot as `null`/`0`). +- Default `true` preserves current zero-config behavior; nothing changes for existing deployments. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `rate-limit-reset-credits`: The scheduler starts with the application lifespan only when reset-credit polling is enabled, and the settings surface gains an enable/disable toggle alongside the existing interval control. diff --git a/openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md b/openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md new file mode 100644 index 0000000000..d1b7f1267c --- /dev/null +++ b/openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md @@ -0,0 +1,59 @@ +## MODIFIED Requirements + +### Requirement: Reset credits are polled per account on a fixed cadence + +The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL start with the application lifespan when reset-credit polling is enabled. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. + +#### Scenario: Default cadence polls every 60 seconds +- **WHEN** the application starts with default settings +- **THEN** each eligible account's credits are fetched from upstream at most once per 60 seconds plus the jitter bound + +#### Scenario: Every replica refreshes its local cache +- **WHEN** the application is deployed with multiple running replicas +- **THEN** each replica refreshes its own in-memory reset-credit snapshots on the configured cadence +- **AND** dashboard reads served by any replica can observe populated reset-credit data after that replica's refresh tick + +#### Scenario: Two replicas do not fetch in lockstep +- **GIVEN** two replicas start with identical configuration +- **WHEN** their refresh loops run +- **THEN** their startup delays are independent uniform draws over the full interval and each tick interval carries independent +/-10% jitter, so the replicas' tick times are not synchronized + +#### Scenario: Ineligible accounts are skipped +- **WHEN** an account is persisted as `paused`, `reauth_required`, or `deactivated` +- **THEN** the scheduler performs no upstream reset-credits fetch for that account +- **AND** the cached snapshot for that account (if any) is left untouched by the skip + +### Requirement: Reset credit polling interval is configurable + +The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL expose setting `rate_limit_reset_credits_refresh_enabled` (default `true`) to enable or disable background reset-credit polling. Because the refresh loop is the sole driver of automatic reset-credit redemption, disabling background polling SHALL also disable automatic redemption; when polling is disabled while the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is enabled, the system SHALL log a configuration-conflict warning at startup naming both settings. While polling is disabled, the dashboard settings update SHALL reject a request that newly enables `auto_redeem_reset_credits_before_expiry` with a bad-request error naming the polling toggle; an already-persisted opt-in SHALL remain readable and re-savable so unrelated settings edits are not blocked. + +#### Scenario: Operator tunes the polling interval +- **GIVEN** `rate_limit_reset_credits_refresh_interval_seconds` is set to `120` +- **WHEN** the application starts and runs +- **THEN** each eligible account's credits are fetched from upstream at most once per 120 seconds + +#### Scenario: Operator disables background polling +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **WHEN** the application starts +- **THEN** the reset-credit polling scheduler does not create a background polling task +- **AND** no upstream reset-credits fetches occur + +#### Scenario: Disabled polling conflicts with persisted auto-redeem opt-in +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `true` +- **WHEN** the application starts +- **THEN** the system logs a configuration-conflict warning naming both settings +- **AND** no automatic reset-credit redemption occurs while polling remains disabled + +#### Scenario: Auto-redeem opt-in is rejected while polling is disabled +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `false` +- **WHEN** a dashboard settings update sets `auto_redeem_reset_credits_before_expiry` to `true` +- **THEN** the update is rejected with a bad-request error naming the polling toggle +- **AND** the persisted setting remains `false` + +#### Scenario: Persisted auto-redeem does not block unrelated settings edits +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is already `true` +- **WHEN** a full settings payload that keeps the opt-in unchanged is submitted +- **THEN** the update succeeds diff --git a/openspec/changes/add-reset-credits-refresh-toggle/tasks.md b/openspec/changes/add-reset-credits-refresh-toggle/tasks.md new file mode 100644 index 0000000000..c7fa80f043 --- /dev/null +++ b/openspec/changes/add-reset-credits-refresh-toggle/tasks.md @@ -0,0 +1,20 @@ +## 1. Settings and scheduler gate + +- [x] 1.1 Add `rate_limit_reset_credits_refresh_enabled: bool = True` to `app/core/config/settings.py` next to the existing interval setting +- [x] 1.2 Add `enabled: bool = True` to `RateLimitResetCreditsRefreshScheduler` and make `start()` a no-op when disabled; wire the setting through `build_rate_limit_reset_credits_scheduler()` +- [x] 1.3 On disabled `start()`, read the persisted dashboard settings and log a configuration-conflict warning when `auto_redeem_reset_credits_before_expiry` is enabled (the refresh loop is the sole auto-redeem driver) + +## 2. Tests + +- [x] 2.1 Unit-test that `start()` creates no task when disabled and creates the loop task when enabled +- [x] 2.2 Unit-test that the factory wires `rate_limit_reset_credits_refresh_enabled` from settings +- [x] 2.3 Unit-test the disabled+auto-redeem conflict warning (warns when persisted opt-in is true, stays silent when false) +- [x] 2.4 Route-level integration tests: PUT rejecting a new auto-redeem opt-in while polling is disabled (`reset_credit_polling_disabled`), and a full PUT with an already-persisted opt-in still succeeding + +## 3.5 Settings API guard + +- [x] 3.5.1 Reject a new `auto_redeem_reset_credits_before_expiry` opt-in in `app/modules/settings/api.py` while polling is disabled; keep already-persisted opt-ins re-savable + +## 3. Spec + +- [x] 3.1 Update the `rate-limit-reset-credits` delta: scheduler starts with the lifespan when polling is enabled; settings expose the toggle with default `true` diff --git a/scripts/generate_settings_reference.py b/scripts/generate_settings_reference.py index 18ba3abdc3..90ce01ee23 100644 --- a/scripts/generate_settings_reference.py +++ b/scripts/generate_settings_reference.py @@ -262,6 +262,8 @@ def render_settings_reference() -> str: "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · " "[responses-api-compat]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · " + "[rate-limit-reset-credits]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · " "[deployment-installation]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)*", "", diff --git a/tests/integration/test_settings_api.py b/tests/integration/test_settings_api.py index 7ef4250346..d4113812ec 100644 --- a/tests/integration/test_settings_api.py +++ b/tests/integration/test_settings_api.py @@ -1228,3 +1228,46 @@ async def test_retention_override_tri_state_echo_capture_and_clear(async_client, settings = await session.get(DashboardSettings, 1) assert settings is not None assert settings.request_log_retention_days is None + + +@pytest.mark.asyncio +async def test_auto_redeem_opt_in_rejected_while_reset_credit_polling_disabled(async_client, monkeypatch): + from types import SimpleNamespace + + disabled = SimpleNamespace(rate_limit_reset_credits_refresh_enabled=False) + monkeypatch.setattr("app.modules.settings.api.get_app_settings", lambda: disabled) + + response = await async_client.get("/api/settings") + assert response.status_code == 200 + payload = response.json() + assert payload["autoRedeemResetCreditsBeforeExpiry"] is False + payload["autoRedeemResetCreditsBeforeExpiry"] = True + + response = await async_client.put("/api/settings", json=payload) + + assert response.status_code == 400 + assert response.json()["error"]["code"] == "reset_credit_polling_disabled" + + +@pytest.mark.asyncio +async def test_full_put_with_persisted_auto_redeem_allowed_while_polling_disabled(async_client, monkeypatch): + from types import SimpleNamespace + + async with SessionLocal() as session: + await session.execute( + text("UPDATE dashboard_settings SET auto_redeem_reset_credits_before_expiry = 1 WHERE id = 1") + ) + await session.commit() + + disabled = SimpleNamespace(rate_limit_reset_credits_refresh_enabled=False) + monkeypatch.setattr("app.modules.settings.api.get_app_settings", lambda: disabled) + + response = await async_client.get("/api/settings") + assert response.status_code == 200 + payload = response.json() + assert payload["autoRedeemResetCreditsBeforeExpiry"] is True + + response = await async_client.put("/api/settings", json=payload) + + assert response.status_code == 200 + assert response.json()["autoRedeemResetCreditsBeforeExpiry"] is True diff --git a/tests/unit/test_rate_limit_reset_credits_scheduler.py b/tests/unit/test_rate_limit_reset_credits_scheduler.py index 42a0433f6f..3acf640df7 100644 --- a/tests/unit/test_rate_limit_reset_credits_scheduler.py +++ b/tests/unit/test_rate_limit_reset_credits_scheduler.py @@ -1,9 +1,11 @@ from __future__ import annotations import asyncio +import logging import random from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta +from types import SimpleNamespace from typing import Any from unittest.mock import AsyncMock @@ -821,3 +823,105 @@ async def _refresh_once(self: RateLimitResetCreditsRefreshScheduler) -> None: await scheduler.stop() assert refreshed is False + + +def _patch_dashboard_settings(monkeypatch: pytest.MonkeyPatch, *, auto_redeem: bool) -> None: + class _FakeSession: + def expunge_all(self) -> None: + return None + + @asynccontextmanager + async def _fake_background_session(): + yield _FakeSession() + + monkeypatch.setattr(scheduler_module, "get_background_session", _fake_background_session) + monkeypatch.setattr( + scheduler_module, + "SettingsRepository", + lambda session: _FakeSettingsRepository(auto_redeem_reset_credits_before_expiry=auto_redeem), + ) + + +@pytest.mark.asyncio +async def test_scheduler_start_is_noop_when_disabled(monkeypatch: pytest.MonkeyPatch) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=False) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + await scheduler.start() + + assert scheduler._task is None + + +@pytest.mark.asyncio +async def test_disabled_start_warns_on_persisted_auto_redeem_conflict( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=True) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + with caplog.at_level(logging.WARNING): + await scheduler.start() + + assert scheduler._task is None + conflict_warnings = [ + record + for record in caplog.records + if record.levelno >= logging.WARNING + and "auto_redeem_reset_credits_before_expiry" in record.getMessage() + and "rate_limit_reset_credits_refresh_enabled" in record.getMessage() + ] + assert conflict_warnings + + +@pytest.mark.asyncio +async def test_disabled_start_stays_silent_without_auto_redeem_opt_in( + monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + _patch_dashboard_settings(monkeypatch, auto_redeem=False) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=False) + + with caplog.at_level(logging.WARNING): + await scheduler.start() + + assert scheduler._task is None + assert not [ + record + for record in caplog.records + if record.levelno >= logging.WARNING and "auto_redeem_reset_credits_before_expiry" in record.getMessage() + ] + + +@pytest.mark.asyncio +async def test_scheduler_start_creates_task_when_enabled(monkeypatch: pytest.MonkeyPatch) -> None: + started = asyncio.Event() + + async def _fake_run_loop(self: RateLimitResetCreditsRefreshScheduler) -> None: + started.set() + await self._stop.wait() + + monkeypatch.setattr(RateLimitResetCreditsRefreshScheduler, "_run_loop", _fake_run_loop) + scheduler = RateLimitResetCreditsRefreshScheduler(interval_seconds=60, enabled=True) + + await scheduler.start() + await asyncio.wait_for(started.wait(), timeout=1.0) + + assert scheduler._task is not None + assert not scheduler._task.done() + await scheduler.stop() + assert scheduler._task is None + + +def test_build_scheduler_wires_enabled_setting(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + scheduler_module, + "get_settings", + lambda: SimpleNamespace( + rate_limit_reset_credits_refresh_enabled=False, + rate_limit_reset_credits_refresh_interval_seconds=123, + ), + ) + + scheduler = scheduler_module.build_rate_limit_reset_credits_scheduler() + + assert scheduler.enabled is False + assert scheduler.interval_seconds == 123 diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index 328bacdc0d..c7b640d0fe 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -56,7 +56,12 @@ def _isolated_settings(**overrides: Any) -> Settings: # These remain operator-selectable because deployments differ in recovery # safety policy and available persistence/latency budgets; their conservative # defaults preserve fail-closed behavior and bound background write work. -MAX_SETTINGS_FIELDS = 126 +# 126 -> 127: rate_limit_reset_credits_refresh_enabled (reset-credit polling +# toggle, #1701). Not a hardcoded default because "off" is a deployment +# decision — operators who don't use the reset-credit surface shed the +# per-replica authenticated upstream polling; default true keeps current +# zero-config behavior and the interval setting alone cannot express "off". +MAX_SETTINGS_FIELDS = 127 def test_generated_settings_reference_matches_code() -> None: From debd7cf63c173e1e7b2982ff171bf1564c150c5a Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 12 Aug 2026 18:36:53 +0900 Subject: [PATCH 004/117] feat(telemetry): anonymous usage telemetry with informed opt-out consent (#1618) * docs(openspec): add add-anonymous-telemetry change proposal * feat(telemetry): add anonymous telemetry with informed opt-out consent Implements OpenSpec change add-anonymous-telemetry (backend): - snapshot builder over request_logs aggregates with strict field allowlist - client-family mapping table and model catalog allowlist (raw values never transmitted) - consent tri-state (env > persisted > default-active) with dialog-facing API - SHM sender (Ed25519 identity, 5s timeout, single retry, debug-only failures) - startup + 24h scheduler with undecided-consent startup notice - Alembic migration for consent state, instance id, and encrypted signing key * feat(dashboard): telemetry consent dialog and settings toggle - one-time consent dialog while consent is undecided (suppressed under env override): exact payload JSON preview, equal-weight enable/disable actions, dismiss persists nothing - settings row with telemetry toggle (env-controlled state disables the toggle with an explanatory notice) and collected-data preview dialog - i18n keys for en/ko/zh-CN, msw handlers, unit + integration tests * fix(db): rebase telemetry migration onto current alembic head * chore(telemetry): switch collection endpoint to telemetry.tokmaxxing.com * fix(telemetry): enforce outbound telemetry contracts Use typed models for every outbound body and one shared snapshot-envelope builder for consent previews and transmission. Gate snapshot work on shared leader election. The undecided-consent startup notice is leader-only so multi-replica deployments emit one operator notice instead of one per replica. * docs(telemetry): publish collection and consent contract Document every transmitted body, the on-demand preview API, leader-owned cadence, fail-honest request kinds, and the currently unspecified collector retention duration. * fix(dashboard): render transmitted envelope and harden telemetry contract - consent dialog and collected-data preview render the exact transmitted envelope (instance_id + metrics + timestamp), matching the backend's single envelope source of truth - preview is fetched lazily: base consent query never carries include_preview; the settings affordance fetches ?include_preview=true on demand only - TelemetrySnapshot declared as a full strict zod schema (every layer strictObject) so backend drift fails parsing in tests - settings loading skeleton gains a telemetry-shaped card to prevent layout shift * fix(db): re-parent telemetry migration and settings ratchet after rebase onto current main * fix(ci): scope stat monkeypatch to target db and teach browser smoke the consent dialog - the unmeasurable-db-size test monkeypatched Path.stat globally without accepting follow_symlinks, crashing the pytest runner itself outside -k telemetry runs; scope the failure to the telemetry db path and delegate everything else to the real stat - the dashboard browser smoke now exercises the first-run consent dialog as a first-class scenario: assert the transmitted envelope is rendered, keep telemetry enabled via a real PUT, then verify the dashboard underneath; /api/settings/telemetry joins the required API paths * fix(ci): type-safe stat monkeypatch and re-parent telemetry migration onto current head * fix(telemetry): exclude cancelled terminals from usage metrics; fix consent docs link - success_rate now subtracts total_cancelled: cancellations are neither errors nor successes (NON_ERROR_STATUSES), so disconnect-heavy workloads no longer inflate the transmitted rate. - top_upstream_errors restricts to actual error statuses; cancelled rows retain upstream_error_code='client_disconnected' and were displacing genuine upstream failures. - consent dialog now links the published /telemetry/ docs page (same target as the backend startup notice) instead of a nonexistent openspec path. Codex review threads on #1618; leader-election gating was already in place via run_if_leader. Co-Authored-By: Claude Fable 5 * fix(telemetry): bucket priority tier, gate guest preview, publish stable capability spec - service_tier_mix gains a priority bucket: 'fast' normalizes to 'priority' at write time, so lumping it into default hid fast-mode traffic. Backend schema, zod mirror, fixtures, and context field lists updated. - Read-only guests no longer fire the consent preview aggregation: the consent query is disabled when the session cannot persist a decision. - Synced the telemetry delta spec to openspec/specs/telemetry (spec.md + context.md) and pointed docs/telemetry.md source-of-truth links at the stable capability path instead of the change folder. Co-Authored-By: Claude Fable 5 * docs(openspec): mark telemetry T6/T8 complete Both are implemented on this head: the consent dialog (T6) ships with live payload preview and equal-prominence actions, and the delta spec is synced to the stable openspec/specs/telemetry capability (T8). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/config/settings.py | 2 + ...20260806_000000_add_anonymous_telemetry.py | 49 ++ app/db/models.py | 8 + app/main.py | 6 + app/modules/telemetry/__init__.py | 5 + app/modules/telemetry/api.py | 71 +++ app/modules/telemetry/clients.py | 61 +++ app/modules/telemetry/consent.py | 108 ++++ app/modules/telemetry/scheduler.py | 94 ++++ app/modules/telemetry/schemas.py | 152 ++++++ app/modules/telemetry/sender.py | 145 +++++ app/modules/telemetry/snapshot.py | 474 +++++++++++++++++ docs/index.md | 1 + docs/reference/settings.md | 4 +- docs/telemetry.md | 67 +++ frontend/browser-smoke/dashboard.spec.ts | 17 + frontend/src/App.tsx | 2 + .../telemetry-consent-flow.test.tsx | 50 ++ frontend/src/features/settings/api.ts | 15 + .../components/settings-page.test.tsx | 11 + .../settings/components/settings-page.tsx | 3 + .../settings/components/settings-skeleton.tsx | 23 + .../telemetry-consent-dialog.test.tsx | 189 +++++++ .../components/telemetry-consent-dialog.tsx | 87 +++ .../components/telemetry-payload-preview.tsx | 13 + .../components/telemetry-settings.test.tsx | 94 ++++ .../components/telemetry-settings.tsx | 106 ++++ .../settings/hooks/use-settings.test.ts | 47 +- .../features/settings/hooks/use-settings.ts | 46 ++ .../src/features/settings/schemas.test.ts | 90 ++++ frontend/src/features/settings/schemas.ts | 130 +++++ frontend/src/i18n/locales/en.json | 18 + frontend/src/i18n/locales/ko.json | 18 + frontend/src/i18n/locales/zh-CN.json | 18 + frontend/src/test/mocks/factories.ts | 107 +++- .../src/test/mocks/handler-coverage.test.ts | 2 + frontend/src/test/mocks/handlers.ts | 33 ++ mkdocs.yml | 1 + .../add-anonymous-telemetry/context.md | 207 +++++++ .../add-anonymous-telemetry/proposal.md | 77 +++ .../specs/telemetry/spec.md | 221 ++++++++ .../changes/add-anonymous-telemetry/tasks.md | 43 ++ openspec/specs/telemetry/context.md | 207 +++++++ openspec/specs/telemetry/spec.md | 223 ++++++++ tests/conftest.py | 7 + tests/unit/test_settings_reference.py | 6 +- tests/unit/test_telemetry_api.py | 73 +++ tests/unit/test_telemetry_consent.py | 153 ++++++ tests/unit/test_telemetry_migration.py | 61 +++ tests/unit/test_telemetry_sender.py | 175 ++++++ tests/unit/test_telemetry_snapshot.py | 503 ++++++++++++++++++ 51 files changed, 4318 insertions(+), 5 deletions(-) create mode 100644 app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py create mode 100644 app/modules/telemetry/__init__.py create mode 100644 app/modules/telemetry/api.py create mode 100644 app/modules/telemetry/clients.py create mode 100644 app/modules/telemetry/consent.py create mode 100644 app/modules/telemetry/scheduler.py create mode 100644 app/modules/telemetry/schemas.py create mode 100644 app/modules/telemetry/sender.py create mode 100644 app/modules/telemetry/snapshot.py create mode 100644 docs/telemetry.md create mode 100644 frontend/src/__integration__/telemetry-consent-flow.test.tsx create mode 100644 frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx create mode 100644 frontend/src/features/settings/components/telemetry-consent-dialog.tsx create mode 100644 frontend/src/features/settings/components/telemetry-payload-preview.tsx create mode 100644 frontend/src/features/settings/components/telemetry-settings.test.tsx create mode 100644 frontend/src/features/settings/components/telemetry-settings.tsx create mode 100644 openspec/changes/add-anonymous-telemetry/context.md create mode 100644 openspec/changes/add-anonymous-telemetry/proposal.md create mode 100644 openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md create mode 100644 openspec/changes/add-anonymous-telemetry/tasks.md create mode 100644 openspec/specs/telemetry/context.md create mode 100644 openspec/specs/telemetry/spec.md create mode 100644 tests/unit/test_telemetry_api.py create mode 100644 tests/unit/test_telemetry_consent.py create mode 100644 tests/unit/test_telemetry_migration.py create mode 100644 tests/unit/test_telemetry_sender.py create mode 100644 tests/unit/test_telemetry_snapshot.py diff --git a/app/core/config/settings.py b/app/core/config/settings.py index 586c12edc2..9b21dd0b6c 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -359,6 +359,8 @@ class Settings(BaseSettings): usage_history_retention_days: int = Field(default=0, ge=0, le=3650) quota_planner_scheduler_enabled: bool = True automations_scheduler_enabled: bool = True + telemetry_enabled: bool | None = None + telemetry_endpoint: str = "https://telemetry.tokmaxxing.com" encryption_key_file: Path = DEFAULT_ENCRYPTION_KEY_FILE # Startup cross-replica encryption-key consistency check against the shared # database sentinel: "enforce" refuses startup on mismatch, "warn" logs an diff --git a/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py b/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py new file mode 100644 index 0000000000..a769d9dfbc --- /dev/null +++ b/app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py @@ -0,0 +1,49 @@ +"""add anonymous telemetry identity and consent + +Revision ID: 20260806_000000_add_anonymous_telemetry +Revises: 20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads +Create Date: 2026-08-06 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260806_000000_add_anonymous_telemetry" +down_revision = "20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads" +branch_labels = None +depends_on = None + + +def _columns() -> set[str]: + return {column["name"] for column in sa.inspect(op.get_bind()).get_columns("dashboard_settings")} + + +def upgrade() -> None: + columns = _columns() + with op.batch_alter_table("dashboard_settings") as batch_op: + if "telemetry_consent" not in columns: + batch_op.add_column( + sa.Column( + "telemetry_consent", + sa.String(length=16), + server_default=sa.text("'undecided'"), + nullable=False, + ) + ) + if "telemetry_instance_id" not in columns: + batch_op.add_column(sa.Column("telemetry_instance_id", sa.String(length=36), nullable=True)) + if "telemetry_private_key_encrypted" not in columns: + batch_op.add_column(sa.Column("telemetry_private_key_encrypted", sa.LargeBinary(), nullable=True)) + + +def downgrade() -> None: + columns = _columns() + with op.batch_alter_table("dashboard_settings") as batch_op: + if "telemetry_private_key_encrypted" in columns: + batch_op.drop_column("telemetry_private_key_encrypted") + if "telemetry_instance_id" in columns: + batch_op.drop_column("telemetry_instance_id") + if "telemetry_consent" in columns: + batch_op.drop_column("telemetry_consent") diff --git a/app/db/models.py b/app/db/models.py index ae6d71eecd..609520e2b8 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -938,6 +938,14 @@ class DashboardSettings(Base): ) totp_secret_encrypted: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) totp_last_verified_step: Mapped[int | None] = mapped_column(Integer, nullable=True) + telemetry_consent: Mapped[str] = mapped_column( + String(16), + default="undecided", + server_default=text("'undecided'"), + nullable=False, + ) + telemetry_instance_id: Mapped[str | None] = mapped_column(String(36), nullable=True) + telemetry_private_key_encrypted: Mapped[bytes | None] = mapped_column(LargeBinary, nullable=True) http_responses_session_bridge_prompt_cache_idle_ttl_seconds: Mapped[int] = mapped_column( Integer, default=3600, diff --git a/app/main.py b/app/main.py index 0ef01d28c5..849f29ac0c 100644 --- a/app/main.py +++ b/app/main.py @@ -101,6 +101,8 @@ _abandoned_bridge_retention_seconds, build_sticky_session_cleanup_scheduler, ) +from app.modules.telemetry import api as telemetry_api +from app.modules.telemetry.scheduler import build_telemetry_scheduler from app.modules.usage import api as usage_api from app.modules.usage.additional_quota_keys import reload_additional_quota_registry from app.modules.usage.live_ingest import start_live_usage_ingestor, stop_live_usage_ingestor @@ -434,6 +436,7 @@ async def lifespan(app: FastAPI): rate_limit_reset_credits_scheduler = build_rate_limit_reset_credits_scheduler() account_usage_rollup_scheduler = build_account_usage_rollup_scheduler() data_retention_scheduler = build_data_retention_scheduler() + telemetry_scheduler = build_telemetry_scheduler() start_live_usage_ingestor() await usage_scheduler.start() await api_key_limit_reset_scheduler.start() @@ -446,6 +449,7 @@ async def lifespan(app: FastAPI): await rate_limit_reset_credits_scheduler.start() await account_usage_rollup_scheduler.start() await data_retention_scheduler.start() + await telemetry_scheduler.start() if settings.metrics_enabled and PROMETHEUS_AVAILABLE: import uvicorn @@ -667,6 +671,7 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N await rate_limit_reset_credits_scheduler.stop() await account_usage_rollup_scheduler.stop() await data_retention_scheduler.stop() + await telemetry_scheduler.stop() # Release the scheduler leader lease only after every leader-gated # scheduler has stopped so no local tick re-acquires it; followers can # then take over immediately instead of waiting out the lease TTL. @@ -765,6 +770,7 @@ def create_app() -> FastAPI: app.include_router(oauth_api.router) app.include_router(dashboard_auth_api.router) app.include_router(settings_api.router) + app.include_router(telemetry_api.router) app.include_router(firewall_api.router) app.include_router(fleet_api.router) app.include_router(sticky_sessions_api.router) diff --git a/app/modules/telemetry/__init__.py b/app/modules/telemetry/__init__.py new file mode 100644 index 0000000000..2c5dcf30e7 --- /dev/null +++ b/app/modules/telemetry/__init__.py @@ -0,0 +1,5 @@ +"""Anonymous, schema-allowlisted telemetry support.""" + +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder + +__all__ = ["TelemetrySnapshotBuilder"] diff --git a/app/modules/telemetry/api.py b/app/modules/telemetry/api.py new file mode 100644 index 0000000000..bb5ae7a7b5 --- /dev/null +++ b/app/modules/telemetry/api.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from fastapi import APIRouter, Body, Depends, Query +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.auth.dependencies import ( + require_dashboard_write_access, + set_dashboard_error_format, + validate_dashboard_session, +) +from app.db.session import get_session +from app.modules.telemetry.consent import ResolvedConsent, TelemetryConsentStore +from app.modules.telemetry.schemas import ( + TelemetryConsentResponse, + TelemetryConsentUpdate, + TelemetrySnapshotEnvelope, + build_snapshot_envelope, +) +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder + +router = APIRouter( + prefix="/api/settings", + tags=["dashboard"], + dependencies=[Depends(validate_dashboard_session), Depends(set_dashboard_error_format)], +) + + +@router.get("/telemetry", response_model=TelemetryConsentResponse) +async def get_telemetry_consent( + include_preview: bool = Query(default=False), + session: AsyncSession = Depends(get_session), +) -> TelemetryConsentResponse: + store = TelemetryConsentStore(session) + consent = await store.resolve() + return await _response( + session, + store, + consent, + include_preview=include_preview or (consent.state == "undecided" and consent.source == "default"), + ) + + +@router.put("/telemetry", response_model=TelemetryConsentResponse) +async def update_telemetry_consent( + payload: TelemetryConsentUpdate = Body(...), + _write_access=Depends(require_dashboard_write_access), + session: AsyncSession = Depends(get_session), +) -> TelemetryConsentResponse: + store = TelemetryConsentStore(session) + consent = await store.set_decision(payload.enabled) + return await _response(session, store, consent, include_preview=False) + + +async def _response( + session: AsyncSession, + store: TelemetryConsentStore, + consent: ResolvedConsent, + *, + include_preview: bool, +) -> TelemetryConsentResponse: + preview: TelemetrySnapshotEnvelope | None = None + if include_preview: + identity = await store.get_or_create_identity() + snapshot = await TelemetrySnapshotBuilder(session).build(identity.instance_id) + preview = build_snapshot_envelope(snapshot) + return TelemetryConsentResponse( + state=consent.state, + source=consent.source, + active=consent.active, + preview=preview, + ) diff --git a/app/modules/telemetry/clients.py b/app/modules/telemetry/clients.py new file mode 100644 index 0000000000..4cf29496e2 --- /dev/null +++ b/app/modules/telemetry/clients.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +from collections import defaultdict +from collections.abc import Iterable +from dataclasses import dataclass + +CLIENT_FAMILY_BY_RAW_GROUP: dict[str, str] = { + "codex_exec": "codex-cli", + "codex-tui": "codex-cli", + "Codex Desktop": "codex-desktop", + "codex_vscode": "codex-vscode", + "AsyncOpenAI": "openai-sdk-python", + "OpenAI": "openai-sdk-js", + "ai": "vercel-ai-sdk", + "ai-sdk": "vercel-ai-sdk", + "opencode": "opencode", + "Mozilla": "browser", + "curl": "script", + "undici": "script", + "node": "script", + "Python-urllib": "script", + "python-requests": "script", + "aiohttp": "script", +} + +CANONICAL_CLIENT_FAMILIES = frozenset({*CLIENT_FAMILY_BY_RAW_GROUP.values(), "other"}) + + +@dataclass(frozen=True, slots=True) +class ClientCount: + raw_group: str | None + requests: int + + +def client_family(raw_group: str | None) -> str: + return CLIENT_FAMILY_BY_RAW_GROUP.get(raw_group or "", "other") + + +def client_shares(rows: Iterable[ClientCount]) -> tuple[dict[str, float], float]: + counts: defaultdict[str, int] = defaultdict(int) + total = 0 + for row in rows: + requests = max(0, row.requests) + family = client_family(row.raw_group) + if family not in CANONICAL_CLIENT_FAMILIES: + raise ValueError(f"non-canonical telemetry client family: {family}") + counts[family] += requests + total += requests + if total == 0: + return {}, 0.0 + shares = {family: _ratio(count, total) for family, count in sorted(counts.items()) if count > 0} + return shares, shares.get("other", 0.0) + + +def catalog_model_name(model: str | None, catalog: frozenset[str]) -> str: + normalized = (model or "").strip() + return normalized if normalized in catalog else "other" + + +def _ratio(numerator: int | float, denominator: int | float) -> float: + return round(float(numerator) / float(denominator), 6) if denominator else 0.0 diff --git a/app/modules/telemetry/consent.py b/app/modules/telemetry/consent.py new file mode 100644 index 0000000000..4c3c9d2f2a --- /dev/null +++ b/app/modules/telemetry/consent.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +import base64 +from dataclasses import dataclass +from typing import Literal, cast +from uuid import uuid4 + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, PublicFormat +from sqlalchemy import or_, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.config.settings import Settings, get_settings +from app.core.crypto import TokenEncryptor +from app.db.models import DashboardSettings +from app.modules.settings.repository import SettingsRepository + +ConsentState = Literal["undecided", "enabled", "disabled"] +ConsentSource = Literal["env", "persisted", "default"] +_VALID_STATES = frozenset({"undecided", "enabled", "disabled"}) + + +@dataclass(frozen=True, slots=True) +class ResolvedConsent: + state: ConsentState + source: ConsentSource + active: bool + + +@dataclass(frozen=True, slots=True) +class TelemetryIdentity: + instance_id: str + private_key: Ed25519PrivateKey + + @property + def public_key_hex(self) -> str: + public_bytes = self.private_key.public_key().public_bytes(Encoding.Raw, PublicFormat.Raw) + return public_bytes.hex() + + +def resolve_consent(telemetry_enabled: bool | None, persisted_state: str) -> ResolvedConsent: + if telemetry_enabled is not None: + state: ConsentState = "enabled" if telemetry_enabled else "disabled" + return ResolvedConsent(state=state, source="env", active=telemetry_enabled) + if persisted_state not in _VALID_STATES: + raise ValueError(f"invalid telemetry consent state: {persisted_state}") + state = cast("ConsentState", persisted_state) + if state == "undecided": + return ResolvedConsent(state="undecided", source="default", active=True) + return ResolvedConsent(state=state, source="persisted", active=state == "enabled") + + +class TelemetryConsentStore: + def __init__( + self, + session: AsyncSession, + *, + settings: Settings | None = None, + encryptor: TokenEncryptor | None = None, + ) -> None: + self._session = session + self._settings = settings or get_settings() + self._encryptor = encryptor or TokenEncryptor() + self._repository = SettingsRepository(session) + + async def resolve(self) -> ResolvedConsent: + row = await self._repository.get_or_create() + return resolve_consent(self._settings.telemetry_enabled, row.telemetry_consent) + + async def set_decision(self, enabled: bool) -> ResolvedConsent: + row = await self._repository.get_or_create() + row.telemetry_consent = "enabled" if enabled else "disabled" + await self._repository.commit_refresh(row) + return resolve_consent(self._settings.telemetry_enabled, row.telemetry_consent) + + async def get_or_create_identity(self) -> TelemetryIdentity: + row = await self._repository.get_or_create() + if row.telemetry_instance_id is None or row.telemetry_private_key_encrypted is None: + await self._mint_identity_if_missing() + self._session.expire_all() + row = await self._repository.get_or_create() + if row.telemetry_instance_id is None or row.telemetry_private_key_encrypted is None: + raise RuntimeError("telemetry identity could not be persisted") + raw_private_key = base64.b64decode(self._encryptor.decrypt(row.telemetry_private_key_encrypted)) + private_key = Ed25519PrivateKey.from_private_bytes(raw_private_key) + return TelemetryIdentity(instance_id=row.telemetry_instance_id, private_key=private_key) + + async def _mint_identity_if_missing(self) -> None: + private_key = Ed25519PrivateKey.generate() + raw_private_key = private_key.private_bytes(Encoding.Raw, PrivateFormat.Raw, NoEncryption()) + encrypted = self._encryptor.encrypt(base64.b64encode(raw_private_key).decode("ascii")) + await self._session.execute( + update(DashboardSettings) + .where( + DashboardSettings.id == 1, + or_( + DashboardSettings.telemetry_instance_id.is_(None), + DashboardSettings.telemetry_private_key_encrypted.is_(None), + ), + ) + .values( + telemetry_instance_id=str(uuid4()), + telemetry_private_key_encrypted=encrypted, + version=DashboardSettings.version + 1, + ) + .execution_options(synchronize_session=False) + ) + await self._session.commit() diff --git a/app/modules/telemetry/scheduler.py b/app/modules/telemetry/scheduler.py new file mode 100644 index 0000000000..fd731cb61f --- /dev/null +++ b/app/modules/telemetry/scheduler.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import asyncio +import contextlib +import importlib +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from functools import partial +from typing import Protocol, TypeVar, cast + +from app.db.session import get_background_session +from app.modules.telemetry.consent import TelemetryConsentStore +from app.modules.telemetry.sender import TelemetrySender +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder + +logger = logging.getLogger(__name__) + +TELEMETRY_INTERVAL_SECONDS = 24 * 60 * 60 +TELEMETRY_FIELDS_DOCUMENTATION = "https://soju06.github.io/codex-lb/telemetry/" + +_T = TypeVar("_T") + + +class _LeaderElectionLike(Protocol): + async def run_if_leader(self, fn: Callable[[], Awaitable[_T]]) -> _T | None: ... + + +def _get_leader_election() -> _LeaderElectionLike: + module = importlib.import_module("app.core.scheduling.leader_election") + return cast(_LeaderElectionLike, module.get_leader_election()) + + +@dataclass(slots=True) +class TelemetryScheduler: + sender: TelemetrySender = field(default_factory=TelemetrySender) + interval_seconds: float = TELEMETRY_INTERVAL_SECONDS + _task: asyncio.Task[None] | None = None + _stop: asyncio.Event = field(default_factory=asyncio.Event) + _lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._run_loop(), name="anonymous-telemetry-scheduler") + + async def stop(self) -> None: + self._stop.set() + if self._task is None: + return + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + async def _run_loop(self) -> None: + first_tick = True + while not self._stop.is_set(): + await self._tick(log_undecided_notice=first_tick) + first_tick = False + try: + await asyncio.wait_for(self._stop.wait(), timeout=self.interval_seconds) + except asyncio.TimeoutError: + continue + + async def _tick(self, *, log_undecided_notice: bool = False) -> None: + await _get_leader_election().run_if_leader( + partial(self._tick_as_leader, log_undecided_notice=log_undecided_notice) + ) + + async def _tick_as_leader(self, *, log_undecided_notice: bool = False) -> None: + async with self._lock: + try: + async with get_background_session() as session: + store = TelemetryConsentStore(session) + consent = await store.resolve() + if log_undecided_notice and consent.state == "undecided" and consent.source == "default": + logger.info( + "Anonymous telemetry is active; collected fields: %s; disable with " + "CODEX_LB_TELEMETRY_ENABLED=false", + TELEMETRY_FIELDS_DOCUMENTATION, + ) + if not consent.active: + return + identity = await store.get_or_create_identity() + snapshot = await TelemetrySnapshotBuilder(session).build(identity.instance_id) + await self.sender.send_snapshot(snapshot) + except Exception as exc: + logger.debug("Anonymous telemetry scheduler tick failed", exc_info=exc) + + +def build_telemetry_scheduler() -> TelemetryScheduler: + return TelemetryScheduler() diff --git a/app/modules/telemetry/schemas.py b/app/modules/telemetry/schemas.py new file mode 100644 index 0000000000..e007dd0c3d --- /dev/null +++ b/app/modules/telemetry/schemas.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from datetime import UTC, datetime +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class TelemetryModel(BaseModel): + model_config = ConfigDict(extra="forbid") + + +class DeploymentSnapshot(TelemetryModel): + method: Literal["docker", "k8s", "pip", "bare"] + db_backend: Literal["sqlite", "postgres"] + db_size_bucket: Literal["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"] + replicas: int = Field(ge=1) + reverse_proxy: bool + + +class PlanMixSnapshot(TelemetryModel): + plus: str + pro: str + team: str + free: str + + +class AccountsSnapshot(TelemetryModel): + pool_bucket: str + plan_mix: PlanMixSnapshot + workspace_accounts: bool + routing_policy: str + limit_warmup_enabled: bool + egress_proxy_used: bool + + +class RequestKindsSnapshot(TelemetryModel): + responses: float + chat: float + images: float + unknown: float + + +class TransportMixSnapshot(TelemetryModel): + ws: float + http_bridge: float + + +class ServiceTierMixSnapshot(TelemetryModel): + default: float + flex: float + priority: float + + +class ModelUsageSnapshot(TelemetryModel): + name: str + share: float + reasoning: dict[str, float] + avg_output_tokens_bucket: str + + +class UsageSnapshot(TelemetryModel): + requests: int = Field(ge=0) + success_rate: float = Field(ge=0.0, le=1.0) + tokens_input: int = Field(ge=0) + tokens_output: int = Field(ge=0) + tokens_cached_ratio: float = Field(ge=0.0, le=1.0) + cost_usd_bucket: str + request_kinds: RequestKindsSnapshot + transport_mix: TransportMixSnapshot + service_tier_mix: ServiceTierMixSnapshot + clients: dict[str, float] + clients_other_ratio: float = Field(ge=0.0, le=1.0) + models: list[ModelUsageSnapshot] + latency_ms_p50: int = Field(ge=0) + ttft_ms_p50: int = Field(ge=0) + ttft_ms_p95: int = Field(ge=0) + rate_limit_429_ratio: float = Field(ge=0.0, le=1.0) + top_upstream_errors: list[str] = Field(max_length=5) + + +class FeaturesSnapshot(TelemetryModel): + api_firewall: bool + quota_planner: bool + sticky_sessions: bool + conversation_archive: bool + automations: bool + fleet: bool + model_sources_count: int = Field(ge=0) + api_keys_bucket: str + prometheus: bool + otel: bool + dashboard_auth: bool + reset_credits: bool + image_api_used: bool + + +class TelemetrySnapshot(TelemetryModel): + schema_version: Literal[1] = 1 + instance_id: str + version: str + python: str + os: str + arch: str + uptime_hours: int = Field(ge=0) + deploy: DeploymentSnapshot + accounts: AccountsSnapshot + usage_7d: UsageSnapshot + features: FeaturesSnapshot + + +class TelemetryRegistration(TelemetryModel): + app_name: Literal["codex-lb"] = "codex-lb" + app_version: str + deployment_mode: Literal["docker", "k8s", "pip", "bare"] + environment: str = "" + instance_id: str + os_arch: str + public_key: str + + +class TelemetryActivation(TelemetryModel): + action: Literal["activate"] = "activate" + + +class TelemetrySnapshotEnvelope(TelemetryModel): + instance_id: str + metrics: TelemetrySnapshot + timestamp: datetime + + +def build_snapshot_envelope( + snapshot: TelemetrySnapshot, + *, + timestamp: datetime | None = None, +) -> TelemetrySnapshotEnvelope: + return TelemetrySnapshotEnvelope( + instance_id=snapshot.instance_id, + metrics=snapshot, + timestamp=timestamp or datetime.now(UTC), + ) + + +class TelemetryConsentUpdate(TelemetryModel): + enabled: bool + + +class TelemetryConsentResponse(TelemetryModel): + state: Literal["undecided", "enabled", "disabled"] + source: Literal["env", "persisted", "default"] + active: bool + preview: TelemetrySnapshotEnvelope | None diff --git a/app/modules/telemetry/sender.py b/app/modules/telemetry/sender.py new file mode 100644 index 0000000000..2f7243e8a4 --- /dev/null +++ b/app/modules/telemetry/sender.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import asyncio +import json +import logging +from collections.abc import Awaitable, Callable + +import aiohttp + +from app.core.config.settings import get_settings +from app.db.session import get_background_session +from app.modules.telemetry.consent import TelemetryConsentStore, TelemetryIdentity +from app.modules.telemetry.schemas import ( + TelemetryActivation, + TelemetryModel, + TelemetryRegistration, + TelemetrySnapshot, + build_snapshot_envelope, +) + +logger = logging.getLogger(__name__) + +_TIMEOUT_SECONDS = 5.0 +_MAX_ATTEMPTS = 2 +SenderContextProvider = Callable[[], Awaitable[tuple[bool, TelemetryIdentity | None]]] + + +class TelemetryProtocolError(RuntimeError): + pass + + +class TelemetrySender: + def __init__( + self, + endpoint: str | None = None, + *, + context_provider: SenderContextProvider | None = None, + ) -> None: + self._endpoint = (endpoint or get_settings().telemetry_endpoint).rstrip("/") + self._context_provider = context_provider or _load_sender_context + self._activated_instance_id: str | None = None + + async def send_snapshot(self, snapshot: TelemetrySnapshot) -> None: + try: + active, identity = await self._context_provider() + if not active: + return + if identity is None: + raise TelemetryProtocolError("active telemetry has no identity") + if snapshot.instance_id != identity.instance_id: + raise TelemetryProtocolError("snapshot identity does not match persisted telemetry identity") + async with asyncio.timeout(_TIMEOUT_SECONDS): + timeout = aiohttp.ClientTimeout(total=_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: + await self._send_with_retry(session, snapshot, identity) + except Exception as exc: + logger.debug("Anonymous telemetry transmission failed", exc_info=exc) + + async def _send_with_retry( + self, + session: aiohttp.ClientSession, + snapshot: TelemetrySnapshot, + identity: TelemetryIdentity, + ) -> None: + last_error: Exception | None = None + for attempt in range(_MAX_ATTEMPTS): + try: + await self._transmit_once(session, snapshot, identity) + return + except Exception as exc: + last_error = exc + logger.debug("Anonymous telemetry attempt %d failed", attempt + 1, exc_info=exc) + if last_error is not None: + raise last_error + + async def _transmit_once( + self, + session: aiohttp.ClientSession, + snapshot: TelemetrySnapshot, + identity: TelemetryIdentity, + ) -> None: + if self._activated_instance_id != identity.instance_id: + registration = TelemetryRegistration( + app_version=snapshot.version, + deployment_mode=snapshot.deploy.method, + instance_id=identity.instance_id, + os_arch=f"{snapshot.os}/{snapshot.arch}", + public_key=identity.public_key_hex, + ) + await self._post(session, "/v1/register", _json_bytes(registration), accepted={200, 201}) + + activation = TelemetryActivation() + await self._post_signed(session, "/v1/activate", _json_bytes(activation), identity, accepted={200}) + self._activated_instance_id = identity.instance_id + + envelope = build_snapshot_envelope(snapshot) + await self._post_signed(session, "/v1/snapshot", _json_bytes(envelope), identity, accepted={200, 202}) + + async def _post_signed( + self, + session: aiohttp.ClientSession, + path: str, + body: bytes, + identity: TelemetryIdentity, + *, + accepted: set[int], + ) -> None: + await self._post( + session, + path, + body, + accepted=accepted, + headers={ + "X-Instance-ID": identity.instance_id, + "X-Signature": identity.private_key.sign(body).hex(), + }, + ) + + async def _post( + self, + session: aiohttp.ClientSession, + path: str, + body: bytes, + *, + accepted: set[int], + headers: dict[str, str] | None = None, + ) -> None: + request_headers = {"Content-Type": "application/json", **(headers or {})} + async with session.post(f"{self._endpoint}{path}", data=body, headers=request_headers) as response: + await response.read() + if response.status not in accepted: + raise TelemetryProtocolError(f"SHM {path} returned HTTP {response.status}") + + +async def _load_sender_context() -> tuple[bool, TelemetryIdentity | None]: + async with get_background_session() as session: + store = TelemetryConsentStore(session) + consent = await store.resolve() + if not consent.active: + return False, None + return True, await store.get_or_create_identity() + + +def _json_bytes(value: TelemetryModel) -> bytes: + return json.dumps(value.model_dump(mode="json"), separators=(",", ":"), sort_keys=True).encode("utf-8") diff --git a/app/modules/telemetry/snapshot.py b/app/modules/telemetry/snapshot.py new file mode 100644 index 0000000000..6237bd6182 --- /dev/null +++ b/app/modules/telemetry/snapshot.py @@ -0,0 +1,474 @@ +from __future__ import annotations + +import importlib.metadata +import logging +import os +import platform +import time +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import timedelta +from pathlib import Path +from typing import Literal, get_args + +from sqlalchemy import and_, case, func, select, text +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.sql.elements import ColumnElement + +from app import __version__ +from app.core.auth.dashboard_mode import DashboardAuthMode +from app.core.balancer.logic import RoutingStrategy +from app.core.config.settings import Settings, get_settings +from app.core.openai.model_registry import get_model_registry +from app.core.usage.logs import NON_ERROR_STATUSES +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + ApiFirewallAllowlist, + ApiKey, + AutomationJob, + ModelSource, + ProxyEndpoint, + QuotaPlannerSettings, + RequestLog, +) +from app.db.sqlite_utils import sqlite_db_path_from_url +from app.modules.reports.repository import ReportsRepository, _report_conditions +from app.modules.settings.repository import SettingsRepository +from app.modules.telemetry.clients import ClientCount, catalog_model_name, client_shares +from app.modules.telemetry.schemas import ( + AccountsSnapshot, + DeploymentSnapshot, + FeaturesSnapshot, + ModelUsageSnapshot, + PlanMixSnapshot, + RequestKindsSnapshot, + ServiceTierMixSnapshot, + TelemetrySnapshot, + TransportMixSnapshot, + UsageSnapshot, +) + +_PROCESS_STARTED = time.monotonic() +_MIB = 1024**2 +_GIB = 1024**3 +_REASONING_EFFORTS = frozenset({"minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) +_ROUTING_POLICIES: frozenset[str] = frozenset(get_args(RoutingStrategy)) +_SAFE_UPSTREAM_ERROR_CODES = frozenset( + { + "authentication_error", + "billing_not_active", + "context_length_exceeded", + "insufficient_quota", + "invalid_api_key", + "invalid_request_error", + "model_not_found", + "rate_limit_exceeded", + "server_error", + "server_overloaded", + "service_unavailable", + "usage_limit_reached", + "upstream_unavailable", + } +) + +logger = logging.getLogger(__name__) + +type Predicate = ColumnElement[bool] +type NullableIntegerColumn = InstrumentedAttribute[int | None] + + +def count_bucket(value: int) -> str: + if value <= 0: + return "0" + if value == 1: + return "1" + if value <= 5: + return "2-5" + if value <= 20: + return "6-20" + if value <= 100: + return "21-100" + return "100+" + + +def db_size_bucket( + size_bytes: int | None, +) -> Literal["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"]: + if size_bytes is None: + return "unknown" + if size_bytes < 100 * _MIB: + return "<100MB" + if size_bytes < _GIB: + return "100MB-1GB" + if size_bytes < 5 * _GIB: + return "1-5GB" + if size_bytes < 10 * _GIB: + return "5-10GB" + if size_bytes < 50 * _GIB: + return "10-50GB" + return "50GB+" + + +def cost_bucket(cost_usd: float) -> str: + if cost_usd < 10: + return "<10" + if cost_usd < 100: + return "10-100" + if cost_usd < 1_000: + return "100-1k" + if cost_usd < 10_000: + return "1k-10k" + if cost_usd < 50_000: + return "10k-50k" + return "50k+" + + +def output_tokens_bucket(tokens: float) -> str: + if tokens < 250: + return "<250" + if tokens < 1_000: + return "250-1k" + if tokens < 4_000: + return "1k-4k" + if tokens < 16_000: + return "4k-16k" + return "16k+" + + +def _ratio(numerator: int | float, denominator: int | float) -> float: + if not denominator: + return 0.0 + return round(min(1.0, max(0.0, float(numerator) / float(denominator))), 6) + + +@dataclass(slots=True) +class _ModelAccumulator: + requests: int = 0 + output_tokens: int = 0 + reasoning_counts: dict[str, int] = field(default_factory=lambda: defaultdict(int)) + + +class TelemetrySnapshotBuilder: + def __init__(self, session: AsyncSession, *, settings: Settings | None = None) -> None: + self._session = session + self._settings = settings or get_settings() + + async def build(self, instance_id: str) -> TelemetrySnapshot: + now = utcnow() + start = now - timedelta(days=7) + reports = ReportsRepository(self._session) + summary = await reports.aggregate_summary(start, now) + ua_rows = await reports.aggregate_by_useragent(start, now) + clients, clients_other_ratio = client_shares( + ClientCount(raw_group=row.useragent_group, requests=row.request_count) for row in ua_rows + ) + conditions = _report_conditions(start, now, None, None, None) + dashboard_settings = await SettingsRepository(self._session).get_or_create() + + account_count, workspace_accounts, plan_counts = await self._account_aggregates() + database_size = await self._database_size_bytes() + models = await self._model_usage(conditions, summary.total_requests) + request_kinds = self._request_kind_mix(summary.total_requests) + transport_mix = await self._transport_mix(conditions, summary.total_requests) + service_tier_mix = await self._service_tier_mix(conditions, summary.total_requests) + latency_p50 = await self._percentile(RequestLog.latency_ms, conditions, 0.50) + ttft_p50 = await self._percentile(RequestLog.latency_first_token_ms, conditions, 0.50) + ttft_p95 = await self._percentile(RequestLog.latency_first_token_ms, conditions, 0.95) + rate_limit_429_count = await self._count_where(conditions, RequestLog.upstream_status_code == 429) + top_errors = await self._top_upstream_errors(conditions) + feature_counts = await self._feature_counts(conditions) + + method = _deployment_method() + db_backend = "postgres" if self._session.get_bind().dialect.name == "postgresql" else "sqlite" + plan_mix = PlanMixSnapshot( + plus=count_bucket(plan_counts.get("plus", 0)), + pro=count_bucket(plan_counts.get("pro", 0)), + team=count_bucket(plan_counts.get("team", 0)), + free=count_bucket(plan_counts.get("free", 0)), + ) + return TelemetrySnapshot( + instance_id=instance_id, + version=__version__, + python=f"{platform.python_version_tuple()[0]}.{platform.python_version_tuple()[1]}", + os=platform.system().lower(), + arch=platform.machine().lower(), + uptime_hours=max(0, int((time.monotonic() - _PROCESS_STARTED) // 3600)), + deploy=DeploymentSnapshot( + method=method, + db_backend=db_backend, + db_size_bucket=db_size_bucket(database_size), + replicas=max(1, len(self._settings.http_responses_session_bridge_instance_ring)), + reverse_proxy=self._settings.firewall_trust_proxy_headers, + ), + accounts=AccountsSnapshot( + pool_bucket=count_bucket(account_count), + plan_mix=plan_mix, + workspace_accounts=workspace_accounts, + routing_policy=_canonical_routing_policy(dashboard_settings.routing_strategy), + limit_warmup_enabled=dashboard_settings.limit_warmup_enabled, + egress_proxy_used=( + dashboard_settings.upstream_proxy_routing_enabled or feature_counts.active_proxy_endpoints > 0 + ), + ), + usage_7d=UsageSnapshot( + requests=summary.total_requests, + # Cancelled terminals are neither errors nor successes + # (NON_ERROR_STATUSES); counting them as successes would + # inflate the rate on disconnect-heavy workloads. + success_rate=_ratio( + summary.total_requests - summary.total_errors - summary.total_cancelled, + summary.total_requests, + ), + tokens_input=summary.total_input_tokens, + tokens_output=summary.total_output_tokens, + tokens_cached_ratio=_ratio(summary.total_cached_tokens, summary.total_input_tokens), + cost_usd_bucket=cost_bucket(max(0.0, summary.total_cost_usd)), + request_kinds=request_kinds, + transport_mix=transport_mix, + service_tier_mix=service_tier_mix, + clients=clients, + clients_other_ratio=clients_other_ratio, + models=models, + latency_ms_p50=latency_p50, + ttft_ms_p50=ttft_p50, + ttft_ms_p95=ttft_p95, + rate_limit_429_ratio=_ratio(rate_limit_429_count, summary.total_requests), + top_upstream_errors=top_errors, + ), + features=FeaturesSnapshot( + api_firewall=feature_counts.firewall_entries > 0, + quota_planner=( + self._settings.quota_planner_scheduler_enabled and feature_counts.quota_planner_mode != "off" + ), + sticky_sessions=dashboard_settings.sticky_threads_enabled, + conversation_archive=self._settings.conversation_archive_enabled, + automations=(self._settings.automations_scheduler_enabled and feature_counts.enabled_automations > 0), + fleet=True, + model_sources_count=feature_counts.model_sources, + api_keys_bucket=count_bucket(feature_counts.api_keys), + prometheus=self._settings.metrics_enabled, + otel=self._settings.otel_enabled, + dashboard_auth=self._settings.dashboard_auth_mode != DashboardAuthMode.DISABLED, + reset_credits=( + dashboard_settings.show_reset_credit_badges + or dashboard_settings.auto_redeem_reset_credits_before_expiry + ), + image_api_used=feature_counts.image_requests > 0, + ), + ) + + async def _account_aggregates(self) -> tuple[int, bool, dict[str, int]]: + result = await self._session.execute( + select( + func.count().label("accounts"), + func.coalesce(func.sum(case((Account.workspace_id.is_not(None), 1), else_=0)), 0).label("workspace"), + ) + ) + row = result.one() + plan_result = await self._session.execute(select(Account.plan_type, func.count()).group_by(Account.plan_type)) + plan_counts: defaultdict[str, int] = defaultdict(int) + for raw_plan, raw_count in plan_result.all(): + plan_counts[_canonical_plan(raw_plan)] += int(raw_count) + return int(row.accounts), bool(row.workspace), dict(plan_counts) + + async def _model_usage(self, conditions: list[Predicate], total_requests: int) -> list[ModelUsageSnapshot]: + result = await self._session.execute( + select( + RequestLog.model, + RequestLog.reasoning_effort, + func.count().label("requests"), + func.coalesce(func.sum(RequestLog.output_tokens), 0).label("output_tokens"), + ) + .where(and_(*conditions)) + .group_by(RequestLog.model, RequestLog.reasoning_effort) + ) + catalog = frozenset(get_model_registry().get_models_with_fallback()) + grouped: defaultdict[str, _ModelAccumulator] = defaultdict(_ModelAccumulator) + for row in result.all(): + name = catalog_model_name(row.model, catalog) + accumulator = grouped[name] + count = int(row.requests) + accumulator.requests += count + accumulator.output_tokens += int(row.output_tokens) + reasoning = _canonical_reasoning(row.reasoning_effort) + accumulator.reasoning_counts[reasoning] += count + return [ + ModelUsageSnapshot( + name=name, + share=_ratio(values.requests, total_requests), + reasoning={ + reasoning: _ratio(count, values.requests) + for reasoning, count in sorted(values.reasoning_counts.items()) + }, + avg_output_tokens_bucket=output_tokens_bucket( + values.output_tokens / values.requests if values.requests else 0 + ), + ) + for name, values in sorted(grouped.items()) + ] + + def _request_kind_mix(self, total: int) -> RequestKindsSnapshot: + # ``request_logs.request_kind`` records workload classes such as + # normal/warmup/compaction, not the ingress route family. Chat, + # Responses, images, and audio can therefore be indistinguishable in + # persisted rows. Report that limitation instead of inferring a route + # from the upstream source or model name. + return RequestKindsSnapshot( + responses=0.0, + chat=0.0, + images=0.0, + unknown=1.0 if total else 0.0, + ) + + async def _transport_mix(self, conditions: list[Predicate], total: int) -> TransportMixSnapshot: + websocket = await self._count_where(conditions, RequestLog.transport == "websocket") + return TransportMixSnapshot(ws=_ratio(websocket, total), http_bridge=_ratio(total - websocket, total)) + + async def _service_tier_mix(self, conditions: list[Predicate], total: int) -> ServiceTierMixSnapshot: + # "fast" is normalized to "priority" at write time + # (_normalize_service_tier_value), so the persisted vocabulary here is + # default/flex/priority; lumping priority into default would hide fast + # mode traffic from the mix. + tier = func.coalesce(RequestLog.actual_service_tier, RequestLog.service_tier, "default") + flex = await self._count_where(conditions, tier == "flex") + priority = await self._count_where(conditions, tier == "priority") + return ServiceTierMixSnapshot( + default=_ratio(total - flex - priority, total), + flex=_ratio(flex, total), + priority=_ratio(priority, total), + ) + + async def _percentile( + self, + column: NullableIntegerColumn, + conditions: list[Predicate], + quantile: float, + ) -> int: + count_result = await self._session.execute( + select(func.count()).where(and_(*conditions, column.is_not(None), column >= 0)) + ) + count = int(count_result.scalar_one()) + if count == 0: + return 0 + rank = min(count - 1, max(0, int((count - 1) * quantile + 0.5))) + result = await self._session.execute( + select(column) + .where(and_(*conditions, column.is_not(None), column >= 0)) + .order_by(column) + .offset(rank) + .limit(1) + ) + value = result.scalar_one() + if value is None: + raise RuntimeError("percentile query returned a null value after a non-null filter") + return int(value) + + async def _count_where(self, conditions: list[Predicate], *extra_conditions: Predicate) -> int: + result = await self._session.execute(select(func.count()).where(and_(*conditions, *extra_conditions))) + return int(result.scalar_one()) + + async def _top_upstream_errors(self, conditions: list[Predicate]) -> list[str]: + # Cancelled rows keep upstream_error_code='client_disconnected', so + # filtering on the code alone would let routine disconnects displace + # genuine upstream failures; restrict to actual error statuses like + # the other error-metric surfaces. + result = await self._session.execute( + select(RequestLog.upstream_error_code, func.count().label("requests")) + .where( + and_( + *conditions, + RequestLog.upstream_error_code.is_not(None), + RequestLog.status.not_in(NON_ERROR_STATUSES), + ) + ) + .group_by(RequestLog.upstream_error_code) + ) + counts: defaultdict[str, int] = defaultdict(int) + for raw_code, count in result.all(): + code = raw_code if raw_code in _SAFE_UPSTREAM_ERROR_CODES else "other" + counts[code] += int(count) + return [code for code, _ in sorted(counts.items(), key=lambda item: (-item[1], item[0]))[:5]] + + async def _feature_counts(self, conditions: list[Predicate]) -> _FeatureCounts: + scalar_queries = ( + select(func.count()).select_from(ApiFirewallAllowlist), + select(func.count()).select_from(ApiKey), + select(func.count()).select_from(ModelSource).where(ModelSource.is_enabled.is_(True)), + select(func.count()).select_from(ProxyEndpoint).where(ProxyEndpoint.is_active.is_(True)), + select(func.count()).select_from(AutomationJob).where(AutomationJob.enabled.is_(True)), + select(func.count()).select_from(RequestLog).where(and_(*conditions, RequestLog.model.like("gpt-image-%"))), + select(QuotaPlannerSettings.mode).where(QuotaPlannerSettings.id == 1), + ) + values = [] + for query in scalar_queries: + values.append((await self._session.execute(query)).scalar_one_or_none()) + return _FeatureCounts( + firewall_entries=int(values[0] or 0), + api_keys=int(values[1] or 0), + model_sources=int(values[2] or 0), + active_proxy_endpoints=int(values[3] or 0), + enabled_automations=int(values[4] or 0), + image_requests=int(values[5] or 0), + quota_planner_mode=str(values[6] or "shadow"), + ) + + async def _database_size_bytes(self) -> int | None: + if self._session.get_bind().dialect.name == "postgresql": + result = await self._session.execute(text("SELECT pg_database_size(current_database())")) + return int(result.scalar_one()) + path = sqlite_db_path_from_url(self._settings.database_url) + if path is None: + return None + try: + return Path(path).stat().st_size + except OSError as exc: + logger.debug("Unable to measure SQLite database size path=%s", path, exc_info=exc) + return None + + +@dataclass(frozen=True, slots=True) +class _FeatureCounts: + firewall_entries: int + api_keys: int + model_sources: int + active_proxy_endpoints: int + enabled_automations: int + image_requests: int + quota_planner_mode: str + + +def _canonical_plan(raw_plan: str | None) -> str: + normalized = (raw_plan or "").strip().lower() + if normalized in {"pro", "prolite"}: + return "pro" + if normalized in {"team", "business", "enterprise", "edu", "education"}: + return "team" + if normalized == "plus": + return "plus" + return "free" + + +def _canonical_reasoning(raw_effort: str | None) -> str: + normalized = (raw_effort or "").strip().lower() + if not normalized: + return "unspecified" + return normalized if normalized in _REASONING_EFFORTS else "other" + + +def _canonical_routing_policy(raw_policy: str | None) -> str: + normalized = (raw_policy or "").strip().lower() + return normalized if normalized in _ROUTING_POLICIES else "other" + + +def _deployment_method() -> Literal["docker", "k8s", "pip", "bare"]: + if os.environ.get("KUBERNETES_SERVICE_HOST") or Path("/var/run/secrets/kubernetes.io/serviceaccount").exists(): + return "k8s" + if Path("/.dockerenv").exists() or Path("/run/.containerenv").exists(): + return "docker" + try: + importlib.metadata.distribution("codex-lb") + except importlib.metadata.PackageNotFoundError: + return "bare" + return "pip" diff --git a/docs/index.md b/docs/index.md index 668ef7f154..dbcd4f58c8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -19,6 +19,7 @@ Load balancer for ChatGPT accounts. Pool multiple accounts, track usage, manage - [Getting Started](getting-started.md) — Docker / uvx quick start, remote bootstrap token - [Client Setup](client-setup.md) — Codex CLI, OpenCode, OpenClaw, Python SDK - [Configuration](configuration.md) — the few settings that matter +- [Anonymous Telemetry](telemetry.md) — collected fields, consent, disabling, and retention - [Authentication](authentication.md) — dashboard auth modes - [Conversations](conversations.md) — dashboard view and conversation APIs - [API Keys](api-keys.md) — protecting proxy routes diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 55e2789e7d..229934236e 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 127 settings. Every setting is an environment +codex-lb currently exposes 129 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -251,6 +251,8 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | +| `CODEX_LB_TELEMETRY_ENABLED` | `bool \| None` | `None` | +| `CODEX_LB_TELEMETRY_ENDPOINT` | `str` | `'https://telemetry.tokmaxxing.com'` | | `CODEX_LB_WARMUP_MODEL` | `str` | `'gpt-5.4-mini'` | ## Removed / deprecated diff --git a/docs/telemetry.md b/docs/telemetry.md new file mode 100644 index 0000000000..b08ade6378 --- /dev/null +++ b/docs/telemetry.md @@ -0,0 +1,67 @@ +# Anonymous telemetry + +codex-lb sends an anonymous usage snapshot to the project-operated collector at +`https://telemetry.tokmaxxing.com` when the service starts and every 24 hours. In a +multi-replica deployment, only the elected leader builds and sends the snapshot. + +## What is sent + +Before the first consent decision, the dashboard shows the current JSON envelope. You can also +view it later from Settings. The signed snapshot body has three fields: + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "" +} +``` + +The versioned `metrics` schema contains only these fields: + +- `schema_version`, random `instance_id`, codex-lb `version`, Python version, OS, architecture, + and process uptime +- `deploy`: deployment method, database backend and size bucket, replica count, and whether + trusted reverse-proxy headers are enabled +- `accounts`: bucketed pool and plan counts, whether workspace accounts exist, routing policy, + limit warmup, and whether an egress proxy is used +- `usage_7d`: request/success/token aggregates, bucketed cost, request-kind and transport/service + tier shares, allowlisted client families and model names, bucketed output-token averages, + latency percentiles, rate-limit ratio, and allowlisted upstream error codes +- `features`: booleans for optional features plus bucketed API-key count and model-source count + +Registration sends `app_name`, `app_version`, `deployment_mode`, an intentionally empty +`environment`, the random `instance_id`, coarse `os_arch`, and the Ed25519 `public_key` used to +verify signed updates. Activation sends only `{"action": "activate"}`. + +The schema never includes account emails, workspace identifiers, client IP addresses, API keys, +request or response content, raw user-agent strings, per-account records, custom model names, or +free-text errors. Exact schemas and privacy constraints live in the +[telemetry OpenSpec capability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/telemetry). + +## Consent and disabling + +Telemetry uses informed opt-out consent. With no override or saved decision, it is active and +the dashboard presents a one-time dialog with the current payload. Enabling or disabling saves +the decision, and the Settings toggle can change it later. + +For a headless or deployment-level kill switch, set: + +```bash +CODEX_LB_TELEMETRY_ENABLED=false +``` + +An environment value overrides the saved dashboard setting. When telemetry resolves to +disabled, codex-lb opens no connection to the telemetry endpoint. + +## Retention and failures + +Each snapshot summarizes the previous seven days of data already present in `request_logs`. +codex-lb does not keep a separate local telemetry history and does not queue a failed send. The +collector's server-side retention duration is not currently specified; assume transmitted +snapshots remain stored until a published retention policy or explicit deletion. + +Endpoint failures use a bounded timeout, are logged only at debug level, and never interrupt +proxy traffic. + +*Source of truth: [telemetry OpenSpec capability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/telemetry)* diff --git a/frontend/browser-smoke/dashboard.spec.ts b/frontend/browser-smoke/dashboard.spec.ts index 57a55606d1..74db517532 100644 --- a/frontend/browser-smoke/dashboard.spec.ts +++ b/frontend/browser-smoke/dashboard.spec.ts @@ -9,6 +9,7 @@ const REQUIRED_API_PATHS = [ "/api/dashboard/projections", "/api/request-logs/options", "/api/request-logs", + "/api/settings/telemetry", ] as const; test("the built dashboard accepts real backend responses", async ({ page }) => { @@ -63,6 +64,22 @@ test("the built dashboard accepts real backend responses", async ({ page }) => { } DashboardProjectionsSchema.parse(await projectionsResponse.json()); + // First run against an empty database resolves telemetry consent as + // undecided/default, so the informed-consent dialog must appear before + // anything else. Exercise it as a first-class scenario: verify the exact + // transmitted envelope is rendered, then keep telemetry enabled to unblock + // the dashboard underneath. + const consentDialog = page.getByRole("dialog", { name: "Anonymous telemetry" }); + await expect(consentDialog).toBeVisible(); + await expect(consentDialog.getByText('"instance_id"').first()).toBeVisible(); + const consentDecision = page.waitForResponse( + (response) => + new URL(response.url()).pathname === "/api/settings/telemetry" && response.request().method() === "PUT", + ); + await consentDialog.getByRole("button", { name: "Keep enabled" }).click(); + expect((await consentDecision).ok()).toBe(true); + await expect(consentDialog).toBeHidden(); + await expect(page.getByRole("heading", { name: "Dashboard", exact: true })).toBeVisible(); await expect(page.getByText("No accounts connected yet", { exact: true })).toBeVisible(); await expect(page.getByText("No requests yet", { exact: true })).toBeVisible(); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 53ae46bb49..c6accac0c6 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -10,6 +10,7 @@ import { Toaster } from "@/components/ui/sonner"; import { TooltipProvider } from "@/components/ui/tooltip"; import { AuthGate } from "@/features/auth/components/auth-gate"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryConsentDialog } from "@/features/settings/components/telemetry-consent-dialog"; import { useTimeFormatStore } from "@/hooks/use-time-format"; // Route-level code splitting: only the visited page's chunk loads, instead @@ -61,6 +62,7 @@ function AppLayout() { + ); } diff --git a/frontend/src/__integration__/telemetry-consent-flow.test.tsx b/frontend/src/__integration__/telemetry-consent-flow.test.tsx new file mode 100644 index 0000000000..8ee9677c3a --- /dev/null +++ b/frontend/src/__integration__/telemetry-consent-flow.test.tsx @@ -0,0 +1,50 @@ +import { screen, waitFor } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it } from "vitest"; + +import App from "@/App"; +import { createTelemetryConsent } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +describe("telemetry consent flow integration", () => { + it("shows the one-time consent dialog on dashboard entry and persists the decision", async () => { + const user = userEvent.setup({ delay: null }); + let putBody: unknown = null; + let consent = createTelemetryConsent({ state: "undecided", source: "default", active: true }); + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(consent)), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + consent = createTelemetryConsent({ state: "disabled", source: "persisted", active: false }); + return HttpResponse.json(consent); + }), + ); + + window.history.pushState({}, "", "/dashboard"); + renderWithProviders(); + + const dialog = await screen.findByRole("dialog", { name: "Anonymous telemetry" }); + // The dialog renders the full transmitted envelope, not just the metrics. + expect(dialog).toHaveTextContent('"instance_id": "00000000-0000-4000-8000-000000000000"'); + expect(dialog).toHaveTextContent('"timestamp": "2026-08-06T00:00:00Z"'); + expect(dialog).toHaveTextContent('"schema_version": 1'); + + await user.click(screen.getByRole("button", { name: "Disable telemetry" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("does not show the consent dialog when consent is already decided", async () => { + window.history.pushState({}, "", "/dashboard"); + const { queryClient } = renderWithProviders(); + + // Default mock state is enabled/persisted. + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog", { name: "Anonymous telemetry" })).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/features/settings/api.ts b/frontend/src/features/settings/api.ts index 5c39f5d36a..e06f15f299 100644 --- a/frontend/src/features/settings/api.ts +++ b/frontend/src/features/settings/api.ts @@ -4,6 +4,8 @@ import { AccountProxyBindingSchema, DashboardSettingsSchema, SettingsUpdateRequestSchema, + TelemetryConsentSchema, + TelemetryConsentUpdateRequestSchema, UpstreamProxyAdminSchema, UpstreamProxyEndpointCreateRequestSchema, UpstreamProxyEndpointSchema, @@ -15,6 +17,7 @@ import { const SETTINGS_PATH = "/api/settings"; const UPSTREAM_PROXY_PATH = `${SETTINGS_PATH}/upstream-proxy`; +const TELEMETRY_PATH = `${SETTINGS_PATH}/telemetry`; export function getSettings() { return get(SETTINGS_PATH, DashboardSettingsSchema); @@ -27,6 +30,18 @@ export function updateSettings(payload: unknown) { }); } +export function getTelemetryConsent(options: { includePreview?: boolean } = {}) { + const path = options.includePreview ? `${TELEMETRY_PATH}?include_preview=true` : TELEMETRY_PATH; + return get(path, TelemetryConsentSchema); +} + +export function updateTelemetryConsent(payload: unknown) { + const validated = TelemetryConsentUpdateRequestSchema.parse(payload); + return put(TELEMETRY_PATH, TelemetryConsentSchema, { + body: validated, + }); +} + export function getUpstreamProxyAdmin() { return get(UPSTREAM_PROXY_PATH, UpstreamProxyAdminSchema); } diff --git a/frontend/src/features/settings/components/settings-page.test.tsx b/frontend/src/features/settings/components/settings-page.test.tsx index 66cfa4812e..dd31c2ba1a 100644 --- a/frontend/src/features/settings/components/settings-page.test.tsx +++ b/frontend/src/features/settings/components/settings-page.test.tsx @@ -19,6 +19,7 @@ const quotaPlannerSectionMock = vi.fn(); const stickySessionsSectionMock = vi.fn(); const modelSourcesSettingsMock = vi.fn(); const dataRetentionSettingsMock = vi.fn(); +const telemetrySettingsMock = vi.fn(); vi.mock("@/features/settings/hooks/use-settings", () => ({ useSettings: () => useSettingsMock(), @@ -76,6 +77,13 @@ vi.mock("@/features/settings/components/data-retention-settings", () => ({ }, })); +vi.mock("@/features/settings/components/telemetry-settings", () => ({ + TelemetrySettings: (props: unknown) => { + telemetrySettingsMock(props); + return
Telemetry Settings
; + }, +})); + vi.mock("@/features/api-keys/components/api-keys-section", () => ({ ApiKeysSection: (props: unknown) => { apiKeysSectionMock(props); @@ -162,6 +170,7 @@ describe("SettingsPage", () => { stickySessionsSectionMock.mockReset(); modelSourcesSettingsMock.mockReset(); dataRetentionSettingsMock.mockReset(); + telemetrySettingsMock.mockReset(); }); async function expandAdvancedSettings() { @@ -192,6 +201,7 @@ describe("SettingsPage", () => { expect(screen.getByText("Appearance Settings")).toBeInTheDocument(); expect(screen.getByText("Import Settings")).toBeInTheDocument(); expect(screen.getByText("API Keys Section")).toBeInTheDocument(); + expect(screen.getByText("Telemetry Settings")).toBeInTheDocument(); }); it("mounts every advanced section after one expand interaction", async () => { @@ -219,6 +229,7 @@ describe("SettingsPage", () => { expect(screen.queryByText("Session Settings")).not.toBeInTheDocument(); expect(importSettingsMock).toHaveBeenCalledWith(expect.objectContaining({ busy: true })); expect(apiKeysSectionMock).toHaveBeenCalledWith(expect.objectContaining({ disabled: true })); + expect(telemetrySettingsMock).toHaveBeenCalledWith(expect.objectContaining({ disabled: true })); await expandAdvancedSettings(); diff --git a/frontend/src/features/settings/components/settings-page.tsx b/frontend/src/features/settings/components/settings-page.tsx index 9fc5f3a291..c03efb77c3 100644 --- a/frontend/src/features/settings/components/settings-page.tsx +++ b/frontend/src/features/settings/components/settings-page.tsx @@ -20,6 +20,7 @@ import { ResetCreditSettings } from "@/features/settings/components/reset-credit import { RoutingSettings } from "@/features/settings/components/routing-settings"; import { SessionSettings } from "@/features/settings/components/session-settings"; import { SettingsSkeleton } from "@/features/settings/components/settings-skeleton"; +import { TelemetrySettings } from "@/features/settings/components/telemetry-settings"; import { UpstreamProxySettings } from "@/features/settings/components/upstream-proxy-settings"; import { StickySessionsSection } from "@/features/sticky-sessions/components/sticky-sessions-section"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; @@ -136,6 +137,8 @@ export function SettingsPage() { } /> + + + {/* Telemetry */} +
+
+
+
+ +
+ + +
+
+ +
+
+
+ + +
+ +
+
+
+ {/* Firewall */}
diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx new file mode 100644 index 0000000000..a730d1e1ec --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx @@ -0,0 +1,189 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryConsentDialog } from "@/features/settings/components/telemetry-consent-dialog"; +import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +function undecidedConsent() { + return createTelemetryConsent({ state: "undecided", source: "default", active: true }); +} + +describe("TelemetryConsentDialog", () => { + beforeEach(() => { + useAuthStore.setState({ canWrite: true }); + }); + + it("shows the exact transmitted envelope with both decision actions while undecided", async () => { + server.use(http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent()))); + + renderWithProviders(); + + const dialog = await screen.findByRole("dialog", { name: "Anonymous telemetry" }); + // The full envelope is the exact transmitted body: top-level instance_id + // and timestamp plus the snapshot under metrics. + expect( + within(dialog).getByText(/"instance_id": "00000000-0000-4000-8000-000000000000"/), + ).toBeInTheDocument(); + expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"metrics": \{/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByRole("button", { name: "Keep enabled" })).toBeInTheDocument(); + expect(within(dialog).getByRole("button", { name: "Disable telemetry" })).toBeInTheDocument(); + expect( + within(dialog).getByRole("link", { name: "Learn what is collected and why" }), + ).toBeInTheDocument(); + }); + + it("persists enabled=true when the operator keeps telemetry enabled", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "enabled", source: "persisted", active: true }), + ); + }), + ); + + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Keep enabled" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: true })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("persists enabled=false when the operator disables telemetry", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "disabled", source: "persisted", active: false }), + ); + }), + ); + + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Disable telemetry" })); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + }); + + it("closes without persisting a decision when dismissed with Escape", async () => { + const user = userEvent.setup(); + let putCalled = false; + server.use( + http.get("/api/settings/telemetry", () => HttpResponse.json(undecidedConsent())), + http.put("/api/settings/telemetry", () => { + putCalled = true; + return HttpResponse.json(undecidedConsent()); + }), + ); + + renderWithProviders(); + + await screen.findByRole("dialog"); + await user.keyboard("{Escape}"); + + await waitFor(() => expect(screen.queryByRole("dialog")).not.toBeInTheDocument()); + expect(putCalled).toBe(false); + }); + + it("stays hidden once a decision has been persisted", async () => { + // Synthetic preview keeps the preview-null gate open so this test binds + // the state === "undecided" gate alone. + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ + state: "enabled", + source: "persisted", + active: true, + preview: createTelemetrySnapshotEnvelope(), + }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden when the response carries no preview envelope", async () => { + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ state: "undecided", source: "default", active: true, preview: null }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden while the environment variable controls telemetry", async () => { + // Synthetic preview keeps the preview-null gate open so this test binds + // the source !== "env" gate alone. + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json( + createTelemetryConsent({ + state: "undecided", + source: "env", + active: false, + preview: createTelemetrySnapshotEnvelope(), + }), + ), + ), + ); + + const { queryClient } = renderWithProviders(); + + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.status).toBe("success"), + ); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); + + it("stays hidden for read-only sessions and never requests the preview aggregation", async () => { + useAuthStore.setState({ canWrite: false }); + let requested = false; + server.use( + http.get("/api/settings/telemetry", () => { + requested = true; + return HttpResponse.json(undecidedConsent()); + }), + ); + + const { queryClient } = renderWithProviders(); + + // Read-only guests can never act on the dialog, so the consent query is + // disabled entirely: no fetch fires and the query stays pending. + await waitFor(() => + expect(queryClient.getQueryState(["settings", "telemetry"])?.fetchStatus).toBe("idle"), + ); + expect(requested).toBe(false); + expect(screen.queryByRole("dialog")).not.toBeInTheDocument(); + }); +}); diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx new file mode 100644 index 0000000000..4b48fbf806 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx @@ -0,0 +1,87 @@ +import { useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { useAuthStore } from "@/features/auth/hooks/use-auth"; +import { TelemetryPayloadPreview } from "@/features/settings/components/telemetry-payload-preview"; +import { useTelemetryConsent } from "@/features/settings/hooks/use-settings"; + +// Same published page the backend startup notice points operators to +// (TELEMETRY_FIELDS_DOCUMENTATION in app/modules/telemetry/scheduler.py). +const TELEMETRY_DOCS_URL = "https://soju06.github.io/codex-lb/telemetry/"; + +export function TelemetryConsentDialog() { + const { t } = useTranslation(); + const canWrite = useAuthStore((state) => state.canWrite); + const [dismissed, setDismissed] = useState(false); + // Read-only guests can never act on the dialog, so skip the preview + // aggregation request entirely instead of fetching and discarding it. + const { telemetryConsentQuery, updateTelemetryConsentMutation } = useTelemetryConsent({ enabled: canWrite }); + + const consent = telemetryConsentQuery.data; + // The dialog exists to show the exact payload before the first send, so it + // is skipped when the backend attached no preview envelope. + const preview = consent?.preview ?? null; + const open = + canWrite && + !dismissed && + consent !== undefined && + consent.state === "undecided" && + consent.source !== "env" && + preview !== null; + + if (!open) { + return null; + } + + const busy = updateTelemetryConsentMutation.isPending; + // Dismissing without a decision (ESC, backdrop, close button) persists + // nothing; the dialog may reappear on the next dashboard entry. + const decide = (enabled: boolean) => { + updateTelemetryConsentMutation.mutate({ enabled }, { onSuccess: () => setDismissed(true) }); + }; + + return ( + setDismissed(!nextOpen)}> + + + {t("settings.telemetry.consentDialog.title")} + {t("settings.telemetry.consentDialog.description")} + +
+

+ {t("settings.telemetry.consentDialog.categories")} +

+

{t("settings.telemetry.consentDialog.payloadLabel")}

+ +

+ + {t("settings.telemetry.consentDialog.docsLink")} + +

+
+ + + + + + + ); +} diff --git a/frontend/src/features/settings/components/telemetry-payload-preview.tsx b/frontend/src/features/settings/components/telemetry-payload-preview.tsx new file mode 100644 index 0000000000..ce983755c2 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-payload-preview.tsx @@ -0,0 +1,13 @@ +import type { TelemetrySnapshotEnvelope } from "@/features/settings/schemas"; + +export type TelemetryPayloadPreviewProps = { + preview: TelemetrySnapshotEnvelope; +}; + +export function TelemetryPayloadPreview({ preview }: TelemetryPayloadPreviewProps) { + return ( + 
+      {`${JSON.stringify(preview, null, 2)}\n`}
+
+ ); +} diff --git a/frontend/src/features/settings/components/telemetry-settings.test.tsx b/frontend/src/features/settings/components/telemetry-settings.test.tsx new file mode 100644 index 0000000000..84ba1aaaa2 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-settings.test.tsx @@ -0,0 +1,94 @@ +import { screen, waitFor, within } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { HttpResponse, http } from "msw"; +import { describe, expect, it } from "vitest"; + +import { TelemetrySettings } from "@/features/settings/components/telemetry-settings"; +import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; +import { server } from "@/test/mocks/server"; +import { renderWithProviders } from "@/test/utils"; + +describe("TelemetrySettings", () => { + it("reflects the resolved state and persists a toggle change", async () => { + const user = userEvent.setup(); + let putBody: unknown = null; + server.use( + http.put("/api/settings/telemetry", async ({ request }) => { + putBody = await request.json(); + return HttpResponse.json( + createTelemetryConsent({ state: "disabled", source: "persisted", active: false }), + ); + }), + ); + + // Default mock state is enabled/persisted. + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => expect(toggle).toBeChecked()); + expect(toggle).toBeEnabled(); + + await user.click(toggle); + + await waitFor(() => expect(putBody).toEqual({ enabled: false })); + }); + + it("disables the toggle and explains the environment override", async () => { + server.use( + http.get("/api/settings/telemetry", () => + HttpResponse.json(createTelemetryConsent({ state: "disabled", source: "env", active: false })), + ), + ); + + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => + expect(screen.getByText(/CODEX_LB_TELEMETRY_ENABLED/)).toBeInTheDocument(), + ); + expect(toggle).toBeDisabled(); + expect(toggle).not.toBeChecked(); + }); + + it("keeps the toggle disabled for read-only sessions", async () => { + renderWithProviders(); + + const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); + await waitFor(() => expect(toggle).toBeChecked()); + expect(toggle).toBeDisabled(); + }); + + it("fetches the preview envelope only when the operator opens the dialog", async () => { + const user = userEvent.setup(); + const telemetryRequests: URL[] = []; + server.use( + http.get("/api/settings/telemetry", ({ request }) => { + const url = new URL(request.url); + telemetryRequests.push(url); + if (url.searchParams.get("include_preview") === "true") { + return HttpResponse.json( + createTelemetryConsent({ preview: createTelemetrySnapshotEnvelope() }), + ); + } + return HttpResponse.json(createTelemetryConsent()); + }), + ); + + renderWithProviders(); + + const viewButton = await screen.findByRole("button", { name: "View collected data" }); + await waitFor(() => expect(viewButton).toBeEnabled()); + // The always-on consent query must not carry the expensive preview flag. + expect(telemetryRequests.length).toBeGreaterThan(0); + expect(telemetryRequests.every((url) => !url.searchParams.has("include_preview"))).toBe(true); + + await user.click(viewButton); + + const dialog = await screen.findByRole("dialog", { name: "Collected telemetry data" }); + expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); + expect( + telemetryRequests.filter((url) => url.searchParams.get("include_preview") === "true"), + ).toHaveLength(1); + }); +}); diff --git a/frontend/src/features/settings/components/telemetry-settings.tsx b/frontend/src/features/settings/components/telemetry-settings.tsx new file mode 100644 index 0000000000..964dfefa29 --- /dev/null +++ b/frontend/src/features/settings/components/telemetry-settings.tsx @@ -0,0 +1,106 @@ +import { Activity } from "lucide-react"; +import { useState } from "react"; +import { useTranslation } from "react-i18next"; + +import { AlertMessage } from "@/components/alert-message"; +import { Button } from "@/components/ui/button"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogFooter, + DialogHeader, + DialogTitle, +} from "@/components/ui/dialog"; +import { Skeleton } from "@/components/ui/skeleton"; +import { Switch } from "@/components/ui/switch"; +import { TelemetryPayloadPreview } from "@/features/settings/components/telemetry-payload-preview"; +import { useTelemetryConsent, useTelemetryPreview } from "@/features/settings/hooks/use-settings"; + +export type TelemetrySettingsProps = { + disabled: boolean; +}; + +export function TelemetrySettings({ disabled }: TelemetrySettingsProps) { + const { t } = useTranslation(); + const [previewOpen, setPreviewOpen] = useState(false); + const { telemetryConsentQuery, updateTelemetryConsentMutation } = useTelemetryConsent(); + // Building the snapshot is expensive, so the preview is fetched only once + // the operator opens the dialog. + const { telemetryPreviewQuery } = useTelemetryPreview(previewOpen); + + const consent = telemetryConsentQuery.data; + const envControlled = consent?.source === "env"; + const busy = disabled || updateTelemetryConsentMutation.isPending || !consent; + const previewEnvelope = telemetryPreviewQuery.data?.preview ?? null; + + return ( +
+
+
+
+
+
+
+

{t("settings.telemetry.title")}

+

{t("settings.telemetry.description")}

+
+
+ updateTelemetryConsentMutation.mutate({ enabled: checked })} + /> +
+ + {envControlled ? ( +
+ {t("settings.telemetry.envNotice")} +
+ ) : null} + +
+
+

{t("settings.telemetry.collectedData.label")}

+

+ {t("settings.telemetry.collectedData.description")} +

+
+ +
+
+ + + {previewOpen ? ( + + + {t("settings.telemetry.previewDialog.title")} + + {t("settings.telemetry.previewDialog.description")} + + + {previewEnvelope ? ( + + ) : telemetryPreviewQuery.error ? ( + {telemetryPreviewQuery.error.message} + ) : ( + + )} + + + ) : null} + +
+ ); +} diff --git a/frontend/src/features/settings/hooks/use-settings.test.ts b/frontend/src/features/settings/hooks/use-settings.test.ts index 30ecc77e41..1bb6ae9566 100644 --- a/frontend/src/features/settings/hooks/use-settings.test.ts +++ b/frontend/src/features/settings/hooks/use-settings.test.ts @@ -3,7 +3,7 @@ import { renderHook, waitFor } from "@testing-library/react"; import { createElement, type PropsWithChildren } from "react"; import { describe, expect, it, vi } from "vitest"; -import { useSettings } from "@/features/settings/hooks/use-settings"; +import { useSettings, useTelemetryConsent, useTelemetryPreview } from "@/features/settings/hooks/use-settings"; function createTestQueryClient(): QueryClient { return new QueryClient({ @@ -55,3 +55,48 @@ describe("useSettings", () => { }); }); }); + +describe("useTelemetryConsent", () => { + it("loads consent and invalidates cache on decision", async () => { + const queryClient = createTestQueryClient(); + const invalidateSpy = vi.spyOn(queryClient, "invalidateQueries"); + + const { result } = renderHook(() => useTelemetryConsent(), { + wrapper: createWrapper(queryClient), + }); + + await waitFor(() => expect(result.current.telemetryConsentQuery.isSuccess).toBe(true)); + expect(result.current.telemetryConsentQuery.data?.state).toBe("enabled"); + expect(result.current.telemetryConsentQuery.data?.active).toBe(true); + // A persisted decision skips the expensive snapshot build entirely. + expect(result.current.telemetryConsentQuery.data?.preview).toBeNull(); + + await result.current.updateTelemetryConsentMutation.mutateAsync({ enabled: false }); + + await waitFor(() => { + expect(invalidateSpy).toHaveBeenCalledWith({ queryKey: ["settings", "telemetry"] }); + }); + }); +}); + +describe("useTelemetryPreview", () => { + it("stays idle until enabled, then loads the preview envelope", async () => { + const queryClient = createTestQueryClient(); + + const { result, rerender } = renderHook(({ enabled }) => useTelemetryPreview(enabled), { + wrapper: createWrapper(queryClient), + initialProps: { enabled: false }, + }); + + expect(result.current.telemetryPreviewQuery.isFetching).toBe(false); + expect(result.current.telemetryPreviewQuery.data).toBeUndefined(); + + rerender({ enabled: true }); + + await waitFor(() => expect(result.current.telemetryPreviewQuery.isSuccess).toBe(true)); + const preview = result.current.telemetryPreviewQuery.data?.preview; + expect(preview?.metrics.schema_version).toBe(1); + expect(preview?.instance_id).toBe("00000000-0000-4000-8000-000000000000"); + expect(preview?.timestamp).toBe("2026-08-06T00:00:00Z"); + }); +}); diff --git a/frontend/src/features/settings/hooks/use-settings.ts b/frontend/src/features/settings/hooks/use-settings.ts index 6934c68d83..6073def607 100644 --- a/frontend/src/features/settings/hooks/use-settings.ts +++ b/frontend/src/features/settings/hooks/use-settings.ts @@ -8,14 +8,17 @@ import { createUpstreamProxyEndpoint, createUpstreamProxyPool, getSettings, + getTelemetryConsent, getUpstreamProxyAdmin, putAccountProxyBinding, testUpstreamProxyEndpoint, updateSettings, + updateTelemetryConsent, } from "@/features/settings/api"; import type { SettingsUpdateRequest } from "@/features/settings/schemas"; import type { AccountProxyBindingRequest, + TelemetryConsentUpdateRequest, UpstreamProxyEndpointCreateRequest, UpstreamProxyPoolCreateRequest, UpstreamProxyPoolMemberRequest, @@ -54,6 +57,49 @@ export function useSettings() { }; } +export function useTelemetryConsent(options?: { enabled?: boolean }) { + const { t } = useTranslation(); + const queryClient = useQueryClient(); + + const { data, error, isFetching, isLoading, isPending, isSuccess, refetch } = useQuery({ + queryKey: ["settings", "telemetry"], + queryFn: () => getTelemetryConsent(), + enabled: options?.enabled ?? true, + }); + const telemetryConsentQuery = { data, error, isFetching, isLoading, isPending, isSuccess, refetch }; + + const updateTelemetryConsentMutation = useMutation({ + mutationFn: (payload: TelemetryConsentUpdateRequest) => updateTelemetryConsent(payload), + onSuccess: () => { + toast.success(t("settings.telemetry.toasts.saved")); + void queryClient.invalidateQueries({ queryKey: ["settings", "telemetry"] }); + }, + onError: (error: Error) => { + toast.error(error.message || t("settings.telemetry.toasts.saveFailed")); + }, + }); + + return { + telemetryConsentQuery, + updateTelemetryConsentMutation, + }; +} + +// On-demand snapshot preview for the settings "View collected data" dialog. +// The snapshot build is expensive, so the query stays idle until `enabled` +// flips true (the dialog opens); consent mutations invalidate it via the +// ["settings", "telemetry"] key prefix. +export function useTelemetryPreview(enabled: boolean) { + const { data, error, isFetching, isLoading, isPending, isSuccess, refetch } = useQuery({ + queryKey: ["settings", "telemetry", "preview"], + queryFn: () => getTelemetryConsent({ includePreview: true }), + enabled, + }); + return { + telemetryPreviewQuery: { data, error, isFetching, isLoading, isPending, isSuccess, refetch }, + }; +} + export function useUpstreamProxyAdmin() { const { t } = useTranslation(); const queryClient = useQueryClient(); diff --git a/frontend/src/features/settings/schemas.test.ts b/frontend/src/features/settings/schemas.test.ts index bcd95f34a2..cacef0ac89 100644 --- a/frontend/src/features/settings/schemas.test.ts +++ b/frontend/src/features/settings/schemas.test.ts @@ -3,8 +3,11 @@ import { describe, expect, it } from "vitest"; import { DashboardSettingsSchema, SettingsUpdateRequestSchema, + TelemetryConsentSchema, + TelemetrySnapshotEnvelopeSchema, UpstreamProxyAdminSchema, } from "@/features/settings/schemas"; +import { createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; describe("DashboardSettingsSchema", () => { it("parses settings payload", () => { @@ -458,6 +461,93 @@ describe("UpstreamProxyAdminSchema", () => { }); }); +describe("TelemetrySnapshotEnvelopeSchema", () => { + it("parses the exact transmitted envelope", () => { + const parsed = TelemetrySnapshotEnvelopeSchema.parse(createTelemetrySnapshotEnvelope()); + + expect(parsed.instance_id).toBe("00000000-0000-4000-8000-000000000000"); + expect(parsed.timestamp).toBe("2026-08-06T00:00:00Z"); + expect(parsed.metrics.schema_version).toBe(1); + expect(parsed.metrics.deploy.method).toBe("docker"); + expect(parsed.metrics.usage_7d.request_kinds.unknown).toBe(0); + expect(parsed.metrics.usage_7d.models[0]?.reasoning).toEqual({ high: 0.5, medium: 0.5 }); + expect(parsed.metrics.features.dashboard_auth).toBe(true); + }); + + it("rejects unknown extra fields at every object layer so backend drift fails parsing", () => { + // One path per strict object in the envelope tree; loosening any single + // layer back to a non-strict schema fails this test. + const layers: string[][] = [ + [], + ["metrics"], + ["metrics", "deploy"], + ["metrics", "accounts"], + ["metrics", "accounts", "plan_mix"], + ["metrics", "usage_7d"], + ["metrics", "usage_7d", "request_kinds"], + ["metrics", "usage_7d", "transport_mix"], + ["metrics", "usage_7d", "service_tier_mix"], + ["metrics", "usage_7d", "models", "0"], + ["metrics", "features"], + ]; + for (const path of layers) { + const envelope = structuredClone(createTelemetrySnapshotEnvelope()); + let target = envelope as unknown as Record; + for (const key of path) { + target = target[key] as Record; + } + target.drifted_field = true; + expect( + TelemetrySnapshotEnvelopeSchema.safeParse(envelope).success, + `extra field at ${path.join(".") || "envelope root"} must fail parsing`, + ).toBe(false); + } + }); + + it("rejects missing required fields so backend drift fails parsing", () => { + const missingTimestamp = structuredClone(createTelemetrySnapshotEnvelope()) as Record< + string, + unknown + >; + delete missingTimestamp.timestamp; + expect(TelemetrySnapshotEnvelopeSchema.safeParse(missingTimestamp).success).toBe(false); + + const missingNested = structuredClone(createTelemetrySnapshotEnvelope()); + delete (missingNested.metrics.usage_7d.request_kinds as Record).unknown; + expect(TelemetrySnapshotEnvelopeSchema.safeParse(missingNested).success).toBe(false); + }); +}); + +describe("TelemetryConsentSchema", () => { + it("parses consent with and without a preview envelope", () => { + const withPreview = TelemetryConsentSchema.parse({ + state: "undecided", + source: "default", + active: true, + preview: createTelemetrySnapshotEnvelope(), + }); + expect(withPreview.preview?.metrics.schema_version).toBe(1); + + const withoutPreview = TelemetryConsentSchema.parse({ + state: "enabled", + source: "persisted", + active: true, + preview: null, + }); + expect(withoutPreview.preview).toBeNull(); + }); + + it("rejects consent responses that omit the preview field", () => { + expect( + TelemetryConsentSchema.safeParse({ + state: "enabled", + source: "persisted", + active: true, + }).success, + ).toBe(false); + }); +}); + describe("retention fields", () => { it("parses effective values plus overrides, defaulting for older backends", () => { const withValues = DashboardSettingsSchema.parse({ diff --git a/frontend/src/features/settings/schemas.ts b/frontend/src/features/settings/schemas.ts index 245ccd5891..9615bec07c 100644 --- a/frontend/src/features/settings/schemas.ts +++ b/frontend/src/features/settings/schemas.ts @@ -343,6 +343,132 @@ export const UpstreamProxyAdminSchema = z.object({ bindings: z.array(AccountProxyBindingSchema), }); +export const TelemetryConsentStateSchema = z.enum(["undecided", "enabled", "disabled"]); +export const TelemetryConsentSourceSchema = z.enum(["env", "persisted", "default"]); + +// Wire-format (snake_case) mirror of app/modules/telemetry/schemas.py. Every +// object is strict so backend drift (renamed, added, or removed fields) fails +// schema parsing instead of passing silently. +const TelemetryDeploymentSnapshotSchema = z.strictObject({ + method: z.enum(["docker", "k8s", "pip", "bare"]), + db_backend: z.enum(["sqlite", "postgres"]), + db_size_bucket: z.enum(["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"]), + replicas: z.number().int().min(1), + reverse_proxy: z.boolean(), +}); + +const TelemetryPlanMixSnapshotSchema = z.strictObject({ + plus: z.string(), + pro: z.string(), + team: z.string(), + free: z.string(), +}); + +const TelemetryAccountsSnapshotSchema = z.strictObject({ + pool_bucket: z.string(), + plan_mix: TelemetryPlanMixSnapshotSchema, + workspace_accounts: z.boolean(), + routing_policy: z.string(), + limit_warmup_enabled: z.boolean(), + egress_proxy_used: z.boolean(), +}); + +const TelemetryRequestKindsSnapshotSchema = z.strictObject({ + responses: z.number(), + chat: z.number(), + images: z.number(), + unknown: z.number(), +}); + +const TelemetryTransportMixSnapshotSchema = z.strictObject({ + ws: z.number(), + http_bridge: z.number(), +}); + +const TelemetryServiceTierMixSnapshotSchema = z.strictObject({ + default: z.number(), + flex: z.number(), + priority: z.number(), +}); + +const TelemetryModelUsageSnapshotSchema = z.strictObject({ + name: z.string(), + share: z.number(), + reasoning: z.record(z.string(), z.number()), + avg_output_tokens_bucket: z.string(), +}); + +const TelemetryUsageSnapshotSchema = z.strictObject({ + requests: z.number().int().min(0), + success_rate: z.number().min(0).max(1), + tokens_input: z.number().int().min(0), + tokens_output: z.number().int().min(0), + tokens_cached_ratio: z.number().min(0).max(1), + cost_usd_bucket: z.string(), + request_kinds: TelemetryRequestKindsSnapshotSchema, + transport_mix: TelemetryTransportMixSnapshotSchema, + service_tier_mix: TelemetryServiceTierMixSnapshotSchema, + clients: z.record(z.string(), z.number()), + clients_other_ratio: z.number().min(0).max(1), + models: z.array(TelemetryModelUsageSnapshotSchema), + latency_ms_p50: z.number().int().min(0), + ttft_ms_p50: z.number().int().min(0), + ttft_ms_p95: z.number().int().min(0), + rate_limit_429_ratio: z.number().min(0).max(1), + top_upstream_errors: z.array(z.string()).max(5), +}); + +const TelemetryFeaturesSnapshotSchema = z.strictObject({ + api_firewall: z.boolean(), + quota_planner: z.boolean(), + sticky_sessions: z.boolean(), + conversation_archive: z.boolean(), + automations: z.boolean(), + fleet: z.boolean(), + model_sources_count: z.number().int().min(0), + api_keys_bucket: z.string(), + prometheus: z.boolean(), + otel: z.boolean(), + dashboard_auth: z.boolean(), + reset_credits: z.boolean(), + image_api_used: z.boolean(), +}); + +export const TelemetrySnapshotSchema = z.strictObject({ + schema_version: z.literal(1), + instance_id: z.string(), + version: z.string(), + python: z.string(), + os: z.string(), + arch: z.string(), + uptime_hours: z.number().int().min(0), + deploy: TelemetryDeploymentSnapshotSchema, + accounts: TelemetryAccountsSnapshotSchema, + usage_7d: TelemetryUsageSnapshotSchema, + features: TelemetryFeaturesSnapshotSchema, +}); + +// The exact body the instance would transmit; the consent dialog and the +// settings preview render this envelope verbatim. +export const TelemetrySnapshotEnvelopeSchema = z.strictObject({ + instance_id: z.string(), + metrics: TelemetrySnapshotSchema, + timestamp: z.iso.datetime({ offset: true }), +}); + +export const TelemetryConsentSchema = z.object({ + state: TelemetryConsentStateSchema, + source: TelemetryConsentSourceSchema, + active: z.boolean(), + // Present only when the backend built a snapshot: undecided consent with + // default source (the dialog case) or an explicit include_preview request. + preview: TelemetrySnapshotEnvelopeSchema.nullable(), +}); + +export const TelemetryConsentUpdateRequestSchema = z.object({ + enabled: z.boolean(), +}); + export type UpstreamProxyEndpoint = z.infer; export type UpstreamProxyEndpointCreateRequest = z.infer; export type UpstreamProxyEndpointTestResponse = z.infer; @@ -352,3 +478,7 @@ export type UpstreamProxyPoolMemberRequest = z.infer; export type AccountProxyBindingRequest = z.infer; export type UpstreamProxyAdmin = z.infer; +export type TelemetrySnapshot = z.infer; +export type TelemetrySnapshotEnvelope = z.infer; +export type TelemetryConsent = z.infer; +export type TelemetryConsentUpdateRequest = z.infer; diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index ad0282a21a..8f492d3039 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1242,6 +1242,24 @@ "settings.password.validation.required": "This field is required.", "settings.password.validation.minLength": "Password must be at least 8 characters.", "settings.password.validation.maxByteLength": "Password must be at most 72 bytes when encoded as UTF-8.", + "settings.telemetry.title": "Anonymous telemetry", + "settings.telemetry.description": "Share anonymous usage statistics to help improve codex-lb.", + "settings.telemetry.toggleAria": "Enable anonymous telemetry", + "settings.telemetry.envNotice": "Telemetry is controlled by the CODEX_LB_TELEMETRY_ENABLED environment variable. Unset it to manage this setting from the dashboard.", + "settings.telemetry.collectedData.label": "Collected data", + "settings.telemetry.collectedData.description": "Review the exact anonymous payload this instance would send.", + "settings.telemetry.collectedData.view": "View collected data", + "settings.telemetry.previewDialog.title": "Collected telemetry data", + "settings.telemetry.previewDialog.description": "The exact payload this instance would send. It contains no accounts, no prompts, no API keys, and no IP addresses.", + "settings.telemetry.consentDialog.title": "Anonymous telemetry", + "settings.telemetry.consentDialog.description": "codex-lb collects anonymous usage telemetry by default to help guide development. You can change this choice at any time in Settings.", + "settings.telemetry.consentDialog.categories": "Only the version, deployment shape, and aggregated usage statistics are collected — no accounts, no prompts, no API keys, and no IP addresses.", + "settings.telemetry.consentDialog.payloadLabel": "Exact payload this instance would send:", + "settings.telemetry.consentDialog.docsLink": "Learn what is collected and why", + "settings.telemetry.consentDialog.keepEnabled": "Keep enabled", + "settings.telemetry.consentDialog.disable": "Disable telemetry", + "settings.telemetry.toasts.saved": "Telemetry preference saved", + "settings.telemetry.toasts.saveFailed": "Failed to save telemetry preference", "settings.totp.title": "TOTP", "settings.totp.status.configured": "TOTP is configured.", "settings.totp.status.notConfigured": "No TOTP configured.", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 7851821adc..959b2248a4 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -1242,6 +1242,24 @@ "settings.session.lifetime.longWarning": "30일을 넘는 유지 시간은 admin session을 오래 유지합니다. 개인 laptop에서는 괜찮을 수 있지만 browser profile이나 cookie가 유출되면 영향이 커집니다.", "settings.session.lifetime.save": "유지 시간 저장", "settings.session.title": "Session", + "settings.telemetry.title": "익명 텔레메트리", + "settings.telemetry.description": "익명 사용 통계를 공유해 codex-lb 개선에 도움을 줍니다.", + "settings.telemetry.toggleAria": "익명 텔레메트리 사용", + "settings.telemetry.envNotice": "텔레메트리는 CODEX_LB_TELEMETRY_ENABLED 환경 변수로 제어되고 있습니다. 대시보드에서 이 설정을 관리하려면 해당 변수를 해제하세요.", + "settings.telemetry.collectedData.label": "수집 데이터", + "settings.telemetry.collectedData.description": "이 인스턴스가 전송할 익명 payload 원문을 확인할 수 있습니다.", + "settings.telemetry.collectedData.view": "수집 데이터 보기", + "settings.telemetry.previewDialog.title": "수집되는 텔레메트리 데이터", + "settings.telemetry.previewDialog.description": "이 인스턴스가 전송할 payload 원문입니다. 계정, 프롬프트, API 키, IP 주소는 포함되지 않습니다.", + "settings.telemetry.consentDialog.title": "익명 텔레메트리", + "settings.telemetry.consentDialog.description": "codex-lb는 개발 방향 결정에 도움이 되도록 기본적으로 익명 사용 텔레메트리를 수집합니다. 이 선택은 언제든지 Settings에서 변경할 수 있습니다.", + "settings.telemetry.consentDialog.categories": "버전, 배포 형태, 집계된 사용 통계만 수집됩니다 — 계정, 프롬프트, API 키, IP 주소는 수집되지 않습니다.", + "settings.telemetry.consentDialog.payloadLabel": "이 인스턴스가 전송할 payload 원문:", + "settings.telemetry.consentDialog.docsLink": "수집 항목과 이유 알아보기", + "settings.telemetry.consentDialog.keepEnabled": "계속 사용", + "settings.telemetry.consentDialog.disable": "텔레메트리 비활성화", + "settings.telemetry.toasts.saved": "텔레메트리 설정이 저장되었습니다", + "settings.telemetry.toasts.saveFailed": "텔레메트리 설정 저장에 실패했습니다", "settings.toasts.saved": "설정 저장됨", "settings.toasts.saveFailed": "설정 저장 실패", "settings.totp.actions.disable": "비활성화", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index a51cc0b0c5..e96ebb1a0b 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -1242,6 +1242,24 @@ "settings.password.validation.required": "此项不能为空。", "settings.password.validation.minLength": "密码至少需要 8 个字符。", "settings.password.validation.maxByteLength": "密码必须在 UTF-8 编码下最多 72 字节。", + "settings.telemetry.title": "匿名遥测", + "settings.telemetry.description": "分享匿名使用统计，帮助改进 codex-lb。", + "settings.telemetry.toggleAria": "启用匿名遥测", + "settings.telemetry.envNotice": "遥测当前由 CODEX_LB_TELEMETRY_ENABLED 环境变量控制。如需在仪表盘中管理此设置，请取消设置该变量。", + "settings.telemetry.collectedData.label": "收集的数据", + "settings.telemetry.collectedData.description": "查看此实例将发送的匿名数据的完整内容。", + "settings.telemetry.collectedData.view": "查看收集的数据", + "settings.telemetry.previewDialog.title": "收集的遥测数据", + "settings.telemetry.previewDialog.description": "此实例将发送的完整数据内容，不包含账号、提示词、API 密钥或 IP 地址。", + "settings.telemetry.consentDialog.title": "匿名遥测", + "settings.telemetry.consentDialog.description": "codex-lb 默认收集匿名使用遥测，以帮助指导开发方向。您可以随时在设置中更改此选择。", + "settings.telemetry.consentDialog.categories": "仅收集版本、部署形态和聚合使用统计 — 不含账号、提示词、API 密钥和 IP 地址。", + "settings.telemetry.consentDialog.payloadLabel": "此实例将发送的完整数据：", + "settings.telemetry.consentDialog.docsLink": "了解收集哪些数据以及原因", + "settings.telemetry.consentDialog.keepEnabled": "保持启用", + "settings.telemetry.consentDialog.disable": "禁用遥测", + "settings.telemetry.toasts.saved": "遥测偏好已保存", + "settings.telemetry.toasts.saveFailed": "保存遥测偏好失败", "settings.totp.title": "TOTP", "settings.totp.status.configured": "TOTP 已配置。", "settings.totp.status.notConfigured": "未配置 TOTP。", diff --git a/frontend/src/test/mocks/factories.ts b/frontend/src/test/mocks/factories.ts index ac3df1d3cf..d1db25b370 100644 --- a/frontend/src/test/mocks/factories.ts +++ b/frontend/src/test/mocks/factories.ts @@ -49,8 +49,18 @@ import { RequestLogSchema, RequestLogsResponseSchema, } from "@/features/dashboard/schemas"; -import type { DashboardSettings, UpstreamProxyAdmin } from "@/features/settings/schemas"; -import { DashboardSettingsSchema, UpstreamProxyAdminSchema } from "@/features/settings/schemas"; +import type { + DashboardSettings, + TelemetryConsent, + TelemetrySnapshotEnvelope, + UpstreamProxyAdmin, +} from "@/features/settings/schemas"; +import { + DashboardSettingsSchema, + TelemetryConsentSchema, + TelemetrySnapshotEnvelopeSchema, + UpstreamProxyAdminSchema, +} from "@/features/settings/schemas"; import type { QuotaPlannerDecision, QuotaPlannerForecast, @@ -82,6 +92,7 @@ export type { RequestLogsResponse, RequestLogFilterOptions, DashboardSettings, + TelemetryConsent, UpstreamProxyAdmin, OauthStartResponse, OauthStatusResponse, @@ -533,6 +544,98 @@ export function createDashboardSettings( }); } +export function createTelemetrySnapshotEnvelope(): TelemetrySnapshotEnvelope { + return TelemetrySnapshotEnvelopeSchema.parse({ + instance_id: "00000000-0000-4000-8000-000000000000", + timestamp: "2026-08-06T00:00:00Z", + metrics: { + schema_version: 1, + instance_id: "00000000-0000-4000-8000-000000000000", + version: "1.23.0", + python: "3.13", + os: "linux", + arch: "x86_64", + uptime_hours: 168, + deploy: { + method: "docker", + db_backend: "sqlite", + db_size_bucket: "<100MB", + replicas: 1, + reverse_proxy: true, + }, + accounts: { + pool_bucket: "2-5", + plan_mix: { plus: "2-5", pro: "0", team: "0", free: "0" }, + workspace_accounts: false, + routing_policy: "usage_weighted", + limit_warmup_enabled: false, + egress_proxy_used: false, + }, + usage_7d: { + requests: 1024, + success_rate: 0.99, + tokens_input: 1000000, + tokens_output: 50000, + tokens_cached_ratio: 0.8, + cost_usd_bucket: "<10", + request_kinds: { responses: 0.97, chat: 0.02, images: 0.01, unknown: 0.0 }, + transport_mix: { ws: 0.6, http_bridge: 0.4 }, + service_tier_mix: { default: 1.0, flex: 0.0, priority: 0.0 }, + clients: { "codex-cli": 0.9, other: 0.1 }, + clients_other_ratio: 0.1, + models: [ + { + name: "gpt-5.4-codex", + share: 1.0, + reasoning: { high: 0.5, medium: 0.5 }, + avg_output_tokens_bucket: "250-1k", + }, + ], + latency_ms_p50: 1200, + ttft_ms_p50: 800, + ttft_ms_p95: 3400, + rate_limit_429_ratio: 0.004, + top_upstream_errors: ["server_overloaded"], + }, + features: { + api_firewall: false, + quota_planner: false, + sticky_sessions: true, + conversation_archive: false, + automations: false, + fleet: false, + model_sources_count: 0, + api_keys_bucket: "2-5", + prometheus: false, + otel: false, + dashboard_auth: true, + reset_credits: true, + image_api_used: false, + }, + }, + }); +} + +export function createTelemetryConsent( + overrides: Partial = {}, +): TelemetryConsent { + const base = { + state: "enabled", + source: "persisted", + active: true, + ...overrides, + }; + // Mirror the backend: the base GET attaches a preview envelope only for + // the undecided/default (consent dialog) case; explicit overrides win. + const preview = + "preview" in overrides + ? overrides.preview + : base.state === "undecided" && base.source === "default" + ? createTelemetrySnapshotEnvelope() + : null; + return TelemetryConsentSchema.parse({ ...base, preview }); +} + export function createQuotaPlannerSettings( overrides: Partial = {}, ): QuotaPlannerSettings { diff --git a/frontend/src/test/mocks/handler-coverage.test.ts b/frontend/src/test/mocks/handler-coverage.test.ts index 4e5bd3702e..1efff9b8aa 100644 --- a/frontend/src/test/mocks/handler-coverage.test.ts +++ b/frontend/src/test/mocks/handler-coverage.test.ts @@ -70,6 +70,8 @@ const EXPECTED_ENDPOINTS = [ // settings "GET /api/settings", "PUT /api/settings", + "GET /api/settings/telemetry", + "PUT /api/settings/telemetry", "GET /api/settings/upstream-proxy", "POST /api/settings/upstream-proxy/endpoints", "POST /api/settings/upstream-proxy/endpoints/:endpointId/test", diff --git a/frontend/src/test/mocks/handlers.ts b/frontend/src/test/mocks/handlers.ts index 9077775ca4..2dd1e2f633 100644 --- a/frontend/src/test/mocks/handlers.ts +++ b/frontend/src/test/mocks/handlers.ts @@ -37,6 +37,8 @@ import { createQuotaPlannerSettings, createQuotaPlannerWarmupActionResponse, createRequestLogFilterOptions, + createTelemetryConsent, + createTelemetrySnapshotEnvelope, createUpstreamProxyAdmin, createRequestLogsResponse, type DashboardAuthSession, @@ -46,6 +48,7 @@ import { type QuotaPlannerForecast, type QuotaPlannerSettings, type RequestLogEntry, + type TelemetryConsent, type UpstreamProxyAdmin, } from "@/test/mocks/factories"; @@ -95,6 +98,10 @@ const AccountAliasPayloadSchema = z.object({ alias: z.string().max(255).nullable(), }); +const TelemetryConsentPayloadSchema = z.object({ + enabled: z.boolean(), +}); + const AccountRoutingPolicyPayloadSchema = z.object({ routingPolicy: z.enum(["normal", "burn_first", "preserve"]), }); @@ -248,6 +255,7 @@ type MockState = { conversationDetails: ConversationDetails[]; authSession: DashboardAuthSession; settings: DashboardSettings; + telemetryConsent: TelemetryConsent; quotaPlannerSettings: QuotaPlannerSettings; quotaPlannerDecisions: QuotaPlannerDecision[]; upstreamProxyAdmin: UpstreamProxyAdmin; @@ -340,6 +348,7 @@ function createInitialState(): MockState { ], authSession: createDashboardAuthSession(), settings: createDashboardSettings(), + telemetryConsent: createTelemetryConsent(), quotaPlannerSettings: createQuotaPlannerSettings(), quotaPlannerDecisions: [createQuotaPlannerDecision()], upstreamProxyAdmin: createUpstreamProxyAdmin(), @@ -1198,7 +1207,31 @@ export const handlers = [ return HttpResponse.json(state.settings); }), + http.get("/api/settings/telemetry", ({ request }) => { + // include_preview=true is the on-demand path: the envelope is attached + // regardless of consent state. + if (new URL(request.url).searchParams.get("include_preview") === "true") { + return HttpResponse.json({ + ...state.telemetryConsent, + preview: createTelemetrySnapshotEnvelope(), + }); + } + return HttpResponse.json(state.telemetryConsent); + }), + http.put("/api/settings/telemetry", async ({ request }) => { + const payload = await parseJsonBody(request, TelemetryConsentPayloadSchema); + if (!payload) { + return HttpResponse.json(state.telemetryConsent); + } + state.telemetryConsent = createTelemetryConsent({ + state: payload.enabled ? "enabled" : "disabled", + source: "persisted", + active: payload.enabled, + preview: null, + }); + return HttpResponse.json(state.telemetryConsent); + }), http.get("/api/settings/upstream-proxy", () => { return HttpResponse.json(state.upstreamProxyAdmin); diff --git a/mkdocs.yml b/mkdocs.yml index 01a9453f2e..62fe5737cf 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -60,6 +60,7 @@ nav: - Live Voice: live-voice.md - Conversations: conversations.md - Configuration: configuration.md + - Anonymous Telemetry: telemetry.md - Authentication: authentication.md - API Keys: api-keys.md - Routing: routing.md diff --git a/openspec/changes/add-anonymous-telemetry/context.md b/openspec/changes/add-anonymous-telemetry/context.md new file mode 100644 index 0000000000..db9e2015db --- /dev/null +++ b/openspec/changes/add-anonymous-telemetry/context.md @@ -0,0 +1,207 @@ +# Telemetry capability — context + +## Purpose + +Give the project visibility into its install base (version distribution, deployment shapes, +client ecosystem, feature usage) without collecting anything that identifies an operator, +an account, or request content. Consent model is informed opt-out: active by default, +one-time dialog with the exact payload, settings toggle, env kill switch. + +Decision record (2026-08-06, maintainer): default-on with first-run confirmation dialog for +both new and existing users; settings toggle; expanded field set over the minimal version. + +## Collection endpoint + +Self-hosted SHM (kOlapsis/shm) server operated by the maintainer at +`https://telemetry.tokmaxxing.com`. SHM provides Ed25519 instance signing, aggregate dashboards, +and public README badges (`/badge/codex-lb/instances`, `/badge/codex-lb/version`). +The SDK path is `/v1/register`, `/v1/activate`, `/v1/snapshot` (note: NOT `/api/v1/`, +which is SHM's admin namespace). codex-lb implements a small Python client (SHM ships +Go/Node SDKs only). + +## Payload schema v1 (the allowlist) + +Everything below derives from existing data (`request_logs`, settings, module registry). +No new per-request instrumentation. `*_bucket` fields use the documented bucket sets. + +Every outbound request body has an explicit Pydantic model and is covered by the wire-schema +allowlist test. The registration body sent to `/v1/register` is: + +```json +{ + "app_name": "codex-lb", + "app_version": "1.20.2", + "deployment_mode": "docker | k8s | pip | bare", + "environment": "", + "instance_id": "", + "os_arch": "linux/x86_64", + "public_key": "" +} +``` + +`app_name` identifies this project, `app_version` supports upgrade/deprecation decisions, +`deployment_mode` and `os_arch` are coarse deployment signals, `environment` is intentionally +empty, and `instance_id` plus `public_key` establish the random signing identity. Activation +sends only `{"action": "activate"}`. + +The signed `/v1/snapshot` body is an envelope. The consent preview renders this same shape via +the same constructor; its timestamp is the current preview-generation time, while an actual +send regenerates the current transmission time. + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "2026-08-06T12:00:00Z" +} +``` + +The `metrics` object is the versioned snapshot schema: + +```json +{ + "schema_version": 1, + "instance_id": "", + "version": "1.20.2", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 168, + + "deploy": { + "method": "docker | k8s | pip | bare", + "db_backend": "sqlite | postgres", + "db_size_bucket": "unknown | ", + "replicas": 3, + "reverse_proxy": true + }, + + "accounts": { + "pool_bucket": "", + "plan_mix": {"plus": "", "pro": "", "team": "", "free": ""}, + "workspace_accounts": true, + "routing_policy": "", + "limit_warmup_enabled": true, + "egress_proxy_used": false + }, + + "usage_7d": { + "requests": 203051, + "success_rate": 0.987, + "tokens_input": 18800000000, + "tokens_output": 94000000, + "tokens_cached_ratio": 0.89, + "cost_usd_bucket": "", + "request_kinds": {"responses": 0.0, "chat": 0.0, "images": 0.0, "unknown": 1.0}, + "transport_mix": {"ws": 0.6, "http_bridge": 0.4}, + "service_tier_mix": {"default": 0.90, "flex": 0.05, "priority": 0.05}, + "clients": {"codex-cli": 0.44, "openai-sdk-python": 0.3, "other": 0.02}, + "clients_other_ratio": 0.02, + "models": [ + { + "name": "gpt-5.4-codex", + "share": 0.62, + "reasoning": {"xhigh": 0.31, "high": 0.48, "medium": 0.21}, + "avg_output_tokens_bucket": "" + } + ], + "latency_ms_p50": 1200, + "ttft_ms_p50": 800, + "ttft_ms_p95": 3400, + "rate_limit_429_ratio": 0.004, + "top_upstream_errors": ["server_overloaded", "usage_limit_reached"] + }, + + "features": { + "api_firewall": true, + "quota_planner": true, + "sticky_sessions": true, + "conversation_archive": false, + "automations": false, + "fleet": false, + "model_sources_count": 2, + "api_keys_bucket": "", + "prometheus": false, + "otel": false, + "dashboard_auth": true, + "reset_credits": true, + "image_api_used": true + } +} +``` + +Field notes: + +- `top_upstream_errors`: enum `upstream_error_code` values only, top 5 by count. Free-text + `error_message` is banned by spec. +- `request_kinds`: current `request_logs` rows do not persist ingress route family. The existing + `request_kind` column is a workload class (`normal`, `warmup`, `compaction`, and similar), + while `source` identifies the upstream. Until an authoritative route-family signal exists, + rows are reported as `unknown`; source and model-name heuristics are deliberately forbidden. +- `clients`: canonical family shares from the normative mapping table in `spec.md`. Raw + `useragent_group` values never leave the instance. +- `models[].name`: official model catalog allowlist match; custom/unknown model names fold + into a single `{"name": "other"}` entry. +- Exact `requests` / token counts are transmitted raw deliberately: they power the global + aggregate counter story and cannot identify an instance. Everything correlated with spend + or org size (accounts, keys, cost, DB size) is bucketed. +- `replicas`: size of the configured HTTP bridge instance ring (multi-replica adoption signal). + +## Bucket sets + +- count buckets (accounts, api keys, plan mix): `0`, `1`, `2-5`, `6-20`, `21-100`, `100+` +- `db_size_bucket`: `unknown`, `<100MB`, `100MB-1GB`, `1-5GB`, `5-10GB`, `10-50GB`, `50GB+` +- `cost_usd_bucket` (7d): `<10`, `10-100`, `100-1k`, `1k-10k`, `10k-50k`, `50k+` +- `avg_output_tokens_bucket`: `<250`, `250-1k`, `1k-4k`, `4k-16k`, `16k+` + +## Consent resolution precedence + +`CODEX_LB_TELEMETRY_ENABLED` env (when set) > persisted decision > default +(`undecided` ⇒ active). The dialog is only shown while persisted state is `undecided` and +no env override exists. + +## Consent API and preview cost + +`GET /api/settings/telemetry` always returns `state`, `source`, `active`, and `preview`. The +default GET includes a preview envelope only for undecided/default consent, when the dialog can +appear; decided and environment-overridden responses return `preview: null` without running the +seven-day aggregate queries. Settings requests the same endpoint with +`include_preview=true` to fetch the current envelope on demand. `PUT /api/settings/telemetry` +persists the decision and returns `preview: null`. + +## Cadence and replica ownership + +The startup and 24-hour ticks run through the shared scheduler leader-election gate. Only the +leader constructs aggregates, transmits the snapshot, and logs the undecided-consent startup +notice. Followers perform none of that work, avoiding duplicate snapshots and duplicate notices. + +## Retention + +Each snapshot summarizes the previous seven days of existing local request logs. codex-lb does +not create a second local telemetry history or queue failed transmissions. The project-operated +collector is `https://telemetry.tokmaxxing.com`; its server-side retention duration is not yet +specified, so operators should assume transmitted snapshots remain stored until a published +retention policy or explicit deletion. + +## Failure modes + +- Endpoint down: bounded timeout (5s), at most one retry per interval, debug-level log, + proxy path untouched. Snapshot is rebuilt fresh next interval (no queue/backlog). +- Aggregation query cost: snapshot queries reuse the same 7-day aggregate shapes as the + dashboard reports module; they run on the leader scheduler once per tick and only on an API + request when the undecided dialog or an explicit settings preview needs them. On Postgres + instances with very large `request_logs` this is the same load class as one dashboard load. +- Clock skew / restart loops: the elected leader transmits the startup snapshot; SHM's + `/v1/activate` is idempotent (active → active refreshes last-seen). Rapid restart loops are + bounded by one snapshot per elected-leader process start; no local rate limiter in v1. + +## Example: privacy review quick check + +An instance with accounts `alice@corp.com` (workspace W1) + 12 others, a custom model source +`corp-internal-gpt`, and traffic from an internal tool `senpi/1.0`: + +- payload has `pool_bucket: "6-20"`, `workspace_accounts: true` +- `corp-internal-gpt` traffic appears as `models[].name == "other"` +- `senpi` traffic appears in `clients` under `other` and inflates `clients_other_ratio` +- the strings `alice`, `corp.com`, `W1`, `corp-internal-gpt`, `senpi` appear nowhere in the + serialized payload (schema snapshot test enforces this) diff --git a/openspec/changes/add-anonymous-telemetry/proposal.md b/openspec/changes/add-anonymous-telemetry/proposal.md new file mode 100644 index 0000000000..c68a052eaf --- /dev/null +++ b/openspec/changes/add-anonymous-telemetry/proposal.md @@ -0,0 +1,77 @@ +# Add anonymous telemetry (informed opt-out) + +## Problem + +codex-lb has grown to ~2.6k stars, ~2.2k unique cloners per 14 days, and an unknown number of +running instances. The project has zero visibility into its install base: version distribution +(how many instances still run pre-1.16 with known bugs), database backend split (SQLite vs +Postgres), transport adoption (WebSocket vs HTTP bridge), deployment shape (docker/helm/pip), +client ecosystem (codex-cli vs SDK integrations), or which optional modules are actually used. +Every roadmap and deprecation decision is currently guesswork. The only existing outbound +signal is the update check in `app/modules/runtime/service.py` (GitHub releases poll), which +proves instances phone GitHub already but gives the project nothing. + +## Solution + +Add a new `telemetry` capability: an anonymous, schema-allowlisted usage snapshot sent to a +project-operated collection endpoint (self-hosted SHM server, `https://telemetry.tokmaxxing.com`) +at startup and every 24 hours. Consent is **informed opt-out**: telemetry is active by default, +every user (new and upgrading) gets a one-time dashboard dialog showing the exact JSON payload +before deciding, a persistent settings toggle, an environment variable kill switch for headless +deployments, and a startup log notice while consent is undecided. + +All payload fields are derived from data codex-lb already stores (`request_logs`, settings, +module registry). No new per-request instrumentation is added. The payload is strictly +allowlisted: raw user-agent strings, custom model names, emails, workspace IDs, IPs, prompts, +API keys, and per-account records are never transmitted. Client statistics go through a +canonical client-family mapping table (raw UA groups like `senpi` must never leave the +instance); model statistics go through the official model catalog allowlist. + +## Why this is correct as a behavior change + +- This is a new operator-visible contract (outbound network traffic + consent flow), which is + exactly the class of change OpenSpec gates. The delta spec makes the privacy allowlist + normative and testable so it cannot regress silently. +- Default-on telemetry in a privacy-sensitive user base is defensible only if the allowlist, + the payload preview, and the kill switches are hard requirements, not implementation + details. Encoding them as MUST requirements with regression tests is the mitigation. +- No existing client or operator behavior changes: proxying, routing, and dashboards are + unaffected; telemetry failure is isolated by requirement. + +## Changes + +### Spec deltas + +- `telemetry` (new capability): payload allowlist, consent state machine, one-time dialog with + exact payload preview, env kill switch, headless notice, client-family mapping table, model + catalog allowlist, random instance identity, transmission cadence + failure isolation, + bucketed sensitive aggregates. + +### Code + +- `app/modules/telemetry/` (new module) — snapshot builder (aggregation queries over + `request_logs` + settings introspection), consent state, scheduler (startup + 24h), sender + (bounded timeout, fire-and-forget). +- `app/core/config/settings.py` — `telemetry_enabled: bool | None = None` (tri-state; env + `CODEX_LB_TELEMETRY_ENABLED` maps to it), `telemetry_endpoint` (default + `https://telemetry.tokmaxxing.com`). +- `app/db/models.py` + Alembic migration — persisted consent decision + `telemetry_instance_id` + (random UUID minted on first run). +- Dashboard (frontend) — one-time consent dialog with payload preview; Settings toggle. +- `app/main.py` — scheduler wiring + undecided-consent startup notice. + +### Tests + +- Unit: payload builder allowlist (schema snapshot test — any new field fails the test until + spec updated), client-family mapping (every observed raw group → family, unknown → `other`), + model allowlist, bucket edges, consent resolution precedence (env > persisted > default). +- Integration: consent API endpoints; disabled ⇒ zero outbound calls (socket-level assert); + telemetry endpoint unreachable ⇒ proxy path unaffected. +- Migration smoke: new columns present with correct defaults. + +## Out of scope + +- The SHM collection server deployment itself (infra task, separate from this repo). +- Public aggregate dashboard / README badges (consumes collected data; follow-up). +- Any new per-request instrumentation or Prometheus metric changes. +- Crash/error report collection (stack traces are content-adjacent; deliberately excluded). diff --git a/openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md b/openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md new file mode 100644 index 0000000000..43389e1701 --- /dev/null +++ b/openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md @@ -0,0 +1,221 @@ +# Add anonymous telemetry + +## ADDED Requirements + +### Requirement: Telemetry payload field allowlist + +The service MUST transmit only fields defined for each outbound body (registration, +activation, and snapshot envelope including its nested metrics) in this capability's +`context.md`, and MUST NOT transmit account emails, workspace identifiers, client IP +addresses, API keys, request or response content, raw user-agent strings, per-account +records, or free-text error messages in any telemetry payload. + +The snapshot metrics schema is versioned (`schema_version`). Adding a field to any transmitted +body requires a spec change to this capability; the outbound wire-schema test suite MUST fail +when registration, activation, the snapshot envelope, or nested metrics contain a field not +present in the documented schema. + +#### Scenario: Every outbound body contains only allowlisted fields + +- **WHEN** the sender serializes registration, activation, and snapshot requests +- **THEN** every top-level and nested field is present in the documented schemas, and an + outbound wire-schema regression test rejects any undeclared field in any body + +#### Scenario: Identifying data never serialized + +- **WHEN** the snapshot is built on an instance with linked accounts, API keys, and request + logs containing raw user agents and error messages +- **THEN** the serialized payload contains no email, workspace ID, IP address, API key + material, raw user-agent string, or free-text error message + +### Requirement: Consent state and default activation + +Telemetry consent MUST be a persisted tri-state (`undecided`, `enabled`, `disabled`) defaulting to `undecided`, and while consent is `undecided` the service SHALL treat telemetry as active. + +Upgrading an existing installation MUST introduce the consent state as `undecided` (existing +users get the same informed default-on treatment as new installs). + +#### Scenario: Fresh install defaults to active + +- **WHEN** codex-lb starts for the first time with no persisted consent and no environment + override +- **THEN** consent is `undecided` and telemetry snapshots are transmitted + +#### Scenario: Upgrade treats existing users as undecided + +- **WHEN** an existing installation migrates to a version with this capability +- **THEN** the migrated consent state is `undecided` and the one-time consent dialog is shown + on next dashboard entry + +### Requirement: One-time consent dialog with exact payload preview + +The dashboard MUST present a one-time consent dialog on first entry while consent is +`undecided`, and the dialog MUST display the exact snapshot envelope the instance would +transmit at that moment. Preview and sender MUST use one shared envelope constructor. The +preview timestamp MUST record preview generation time as a representative current timestamp; +the actual send MUST regenerate that value at transmission time. + +A decision (enable or disable) MUST be persisted and the dialog MUST NOT be shown again after +any decision. The dialog MUST offer disabling with no fewer clicks than enabling. + +The consent API MUST build the preview only while the undecided dialog is eligible or when an +operator explicitly requests it for the settings view. The response MUST retain the `preview` +field and set it to `null` when the preview was not requested and is not dialog-relevant. + +#### Scenario: Undecided operator sees payload preview + +- **WHEN** an operator opens the dashboard while consent is `undecided` +- **THEN** a dialog shows the live snapshot JSON with equally prominent enable and disable + actions + +#### Scenario: Decision is final until changed in settings + +- **WHEN** the operator chooses disable in the dialog +- **THEN** consent persists as `disabled`, no snapshot is transmitted afterward, and the + dialog never reappears + +#### Scenario: Decided consent status is a cheap read + +- **WHEN** the dashboard reads consent after a persisted decision without requesting a preview +- **THEN** the response contains `preview: null` and no snapshot aggregation query runs + +#### Scenario: Settings explicitly requests collected data + +- **WHEN** the settings view requests a preview for any consent state +- **THEN** the response contains a current snapshot envelope built with the same schema as the sender + +### Requirement: Settings toggle and environment kill switch + +The dashboard settings MUST expose a telemetry toggle reflecting the resolved consent state, and the environment variable `CODEX_LB_TELEMETRY_ENABLED` MUST override persisted consent when set (`false` disables all transmission, `true` enables and suppresses the consent dialog). + +#### Scenario: Headless deployment disables via environment + +- **WHEN** the service runs with `CODEX_LB_TELEMETRY_ENABLED=false` +- **THEN** no telemetry network traffic occurs regardless of persisted consent, and the + settings toggle shows telemetry as disabled by environment override + +#### Scenario: Toggle flips persisted consent + +- **WHEN** the operator disables telemetry in settings without an environment override +- **THEN** consent persists as `disabled` and transmission stops without restart + +### Requirement: Startup notice while undecided + +While consent is `undecided`, the elected leader MUST emit a single startup log line stating +that anonymous telemetry is active, where the collected-field documentation lives, and how to +disable it. Non-leader replicas MUST NOT duplicate the notice. + +#### Scenario: Headless operator is informed + +- **WHEN** the service starts with consent `undecided` +- **THEN** exactly one log line names the telemetry documentation location and the + `CODEX_LB_TELEMETRY_ENABLED=false` disable path + +### Requirement: Disabled means zero telemetry traffic + +When resolved consent is `disabled`, the service MUST NOT open any network connection to the telemetry endpoint. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle +- **THEN** no connection attempt to the telemetry endpoint is made + +### Requirement: Client family allowlist mapping + +Telemetry client statistics MUST report only canonical client-family identifiers produced by the documented mapping table, MUST map any unmatched user-agent group to `other`, and MUST NOT transmit raw user-agent group values. + +The canonical mapping table (raw `useragent_group` → family): + +| Raw group(s) | Family | +| --- | --- | +| `codex_exec`, `codex-tui` | `codex-cli` | +| `Codex Desktop` | `codex-desktop` | +| `codex_vscode` | `codex-vscode` | +| `AsyncOpenAI` | `openai-sdk-python` | +| `OpenAI` | `openai-sdk-js` | +| `ai`, `ai-sdk` | `vercel-ai-sdk` | +| `opencode` | `opencode` | +| `Mozilla` | `browser` | +| `curl`, `undici`, `node`, `Python-urllib`, `python-requests`, `aiohttp` | `script` | +| anything else | `other` | + +The payload MUST include `clients_other_ratio` so mapping coverage decay is observable +without ever transmitting the unmatched raw values. + +#### Scenario: Private tool names never leave the instance + +- **WHEN** request logs contain a user-agent group not present in the mapping table +- **THEN** its traffic is attributed to `other` and the raw group string is absent from the + payload + +#### Scenario: Codex CLI variants collapse to one family + +- **WHEN** traffic exists from both `codex_exec` and `codex-tui` +- **THEN** the payload reports a single `codex-cli` family combining both + +### Requirement: Model catalog allowlist with per-model reasoning mix + +Telemetry model statistics MUST include only model names present in the official model catalog allowlist, MUST map unmatched model names to `other`, and MUST report reasoning-effort distribution nested per model entry rather than as an instance-global aggregate. + +#### Scenario: Custom model source names are not transmitted + +- **WHEN** an operator has configured a custom model source with a private model name +- **THEN** that traffic appears under `other` and the private name is absent from the payload + +#### Scenario: Reasoning effort is model-scoped + +- **WHEN** the snapshot reports models +- **THEN** each model entry carries its own reasoning-effort share map and no global + reasoning mix field exists + +### Requirement: Fail-honest request-family attribution + +Request-family telemetry MUST be derived only from an authoritative persisted route-family +signal. Rows without such a signal MUST be attributed to `unknown`; the service MUST NOT infer +Chat, Responses, Images, or Audio families from upstream `source` or model name. + +#### Scenario: Ambiguous persisted rows remain unknown + +- **WHEN** persisted request rows identify only workload kind, upstream source, or model name +- **THEN** their request-family share is reported as `unknown` rather than a named route family + +### Requirement: Random instance identity + +The telemetry instance identifier MUST be a UUID generated randomly on first run, MUST NOT be derived from hardware, network, account, or operating-system identity, and MUST be regenerated if deleted. + +#### Scenario: Identifier carries no fingerprint + +- **WHEN** the instance identifier is created +- **THEN** it is a random UUIDv4 persisted locally, and deleting it yields a fresh unrelated + identifier on next start + +### Requirement: Transmission cadence and failure isolation + +The service SHALL transmit one snapshot at startup and one per 24-hour interval thereafter. +In a multi-replica deployment sharing a database, snapshot construction and transmission MUST +run only under the existing leader-election gate so at most one replica performs each tick. +Telemetry transmission failures MUST NOT affect proxy operation, MUST use a bounded timeout, +MUST NOT retry more than once per interval, and MUST log failures at debug level only. + +#### Scenario: Non-leader replica skips telemetry work + +- **WHEN** a telemetry tick runs in a process that does not hold the scheduler leader lease +- **THEN** that process neither builds a snapshot nor attempts a transmission + +#### Scenario: Collection endpoint outage is invisible + +- **WHEN** the telemetry endpoint is unreachable +- **THEN** proxy requests are unaffected, startup is not delayed beyond the bounded timeout, + and no warning-or-higher log noise is produced + +### Requirement: Bucketed sensitive aggregates + +Account pool size, per-plan account counts, API key count, database size, and cost aggregates MUST be transmitted as documented buckets, never as exact values. + +An unmeasurable database size MUST be reported as `unknown`, not as a plausible size bucket. + +#### Scenario: Pool size is a bucket + +- **WHEN** an instance has 13 linked accounts +- **THEN** the payload reports the `6-20` bucket and no exact account count diff --git a/openspec/changes/add-anonymous-telemetry/tasks.md b/openspec/changes/add-anonymous-telemetry/tasks.md new file mode 100644 index 0000000000..6a9d385242 --- /dev/null +++ b/openspec/changes/add-anonymous-telemetry/tasks.md @@ -0,0 +1,43 @@ +# Tasks + +## Implementation + +- [x] T1: `app/modules/telemetry/` module — snapshot builder aggregating `request_logs` + (7d window, reusing reports-module aggregate shapes), settings/module introspection, + bucket helpers, client-family mapping table, model catalog allowlist filter. +- [x] T2: Consent state — DB columns (`telemetry_consent`, `telemetry_instance_id`) + + Alembic migration on current main head; resolution precedence env > persisted > default. +- [x] T3: Settings — `telemetry_enabled: bool | None` (env `CODEX_LB_TELEMETRY_ENABLED`), + `telemetry_endpoint` default `https://telemetry.tokmaxxing.com`. +- [x] T4: Sender — SHM `/v1/register` + `/v1/activate` + `/v1/snapshot` client (Ed25519 + keypair per instance), 5s timeout, ≤1 retry/interval, debug-only failure logs. +- [x] T5: Scheduler — startup snapshot + 24h interval; undecided-consent startup notice + (single log line with docs link + disable instructions). +- [x] T6: Dashboard consent dialog — one-time while undecided, renders live payload JSON, + equal-prominence enable/disable; Settings toggle wired to consent API. +- [x] T7: Consent API endpoints (get resolved state, set decision). + +## Spec + +- [x] T8: Apply delta `specs/telemetry/spec.md` as new capability; sync payload schema into + `openspec/specs/telemetry/context.md`. + +## Validation + +- [x] T9: Unit — schema snapshot allowlist test (undeclared field ⇒ fail), client mapping + (all observed raw groups + unknown ⇒ `other`), model allowlist, bucket edges, consent + precedence. +- [x] T10: Integration — consent endpoints; disabled ⇒ zero outbound connections + (socket-level); endpoint unreachable ⇒ proxy unaffected. +- [x] T11: Migration smoke — new columns/defaults present (SQLite + Postgres). +- [x] T12: Privacy quick check from context.md reproduced as a test (identifying strings + absent from serialized payload). +- [x] T14: Review remediation — shared preview/sender envelope, typed allowlisted bodies, + preview-on-demand consent API, and wire-schema regressions. +- [x] T15: Review remediation — leader-gated telemetry ticks plus real lifespan wiring and + non-leader regression coverage. +- [x] T16: Review remediation — fail-honest request kinds, derived routing/client allowlists, + honest database-size failure reporting, and typed query expressions. +- [x] T17: Publish anonymous telemetry documentation and register it in the docs navigation. +- [x] T13: `openspec validate add-anonymous-telemetry` → valid; `make lint`; targeted + + broader pytest sweeps. diff --git a/openspec/specs/telemetry/context.md b/openspec/specs/telemetry/context.md new file mode 100644 index 0000000000..db9e2015db --- /dev/null +++ b/openspec/specs/telemetry/context.md @@ -0,0 +1,207 @@ +# Telemetry capability — context + +## Purpose + +Give the project visibility into its install base (version distribution, deployment shapes, +client ecosystem, feature usage) without collecting anything that identifies an operator, +an account, or request content. Consent model is informed opt-out: active by default, +one-time dialog with the exact payload, settings toggle, env kill switch. + +Decision record (2026-08-06, maintainer): default-on with first-run confirmation dialog for +both new and existing users; settings toggle; expanded field set over the minimal version. + +## Collection endpoint + +Self-hosted SHM (kOlapsis/shm) server operated by the maintainer at +`https://telemetry.tokmaxxing.com`. SHM provides Ed25519 instance signing, aggregate dashboards, +and public README badges (`/badge/codex-lb/instances`, `/badge/codex-lb/version`). +The SDK path is `/v1/register`, `/v1/activate`, `/v1/snapshot` (note: NOT `/api/v1/`, +which is SHM's admin namespace). codex-lb implements a small Python client (SHM ships +Go/Node SDKs only). + +## Payload schema v1 (the allowlist) + +Everything below derives from existing data (`request_logs`, settings, module registry). +No new per-request instrumentation. `*_bucket` fields use the documented bucket sets. + +Every outbound request body has an explicit Pydantic model and is covered by the wire-schema +allowlist test. The registration body sent to `/v1/register` is: + +```json +{ + "app_name": "codex-lb", + "app_version": "1.20.2", + "deployment_mode": "docker | k8s | pip | bare", + "environment": "", + "instance_id": "", + "os_arch": "linux/x86_64", + "public_key": "" +} +``` + +`app_name` identifies this project, `app_version` supports upgrade/deprecation decisions, +`deployment_mode` and `os_arch` are coarse deployment signals, `environment` is intentionally +empty, and `instance_id` plus `public_key` establish the random signing identity. Activation +sends only `{"action": "activate"}`. + +The signed `/v1/snapshot` body is an envelope. The consent preview renders this same shape via +the same constructor; its timestamp is the current preview-generation time, while an actual +send regenerates the current transmission time. + +```json +{ + "instance_id": "", + "metrics": { "": "..." }, + "timestamp": "2026-08-06T12:00:00Z" +} +``` + +The `metrics` object is the versioned snapshot schema: + +```json +{ + "schema_version": 1, + "instance_id": "", + "version": "1.20.2", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 168, + + "deploy": { + "method": "docker | k8s | pip | bare", + "db_backend": "sqlite | postgres", + "db_size_bucket": "unknown | ", + "replicas": 3, + "reverse_proxy": true + }, + + "accounts": { + "pool_bucket": "", + "plan_mix": {"plus": "", "pro": "", "team": "", "free": ""}, + "workspace_accounts": true, + "routing_policy": "", + "limit_warmup_enabled": true, + "egress_proxy_used": false + }, + + "usage_7d": { + "requests": 203051, + "success_rate": 0.987, + "tokens_input": 18800000000, + "tokens_output": 94000000, + "tokens_cached_ratio": 0.89, + "cost_usd_bucket": "", + "request_kinds": {"responses": 0.0, "chat": 0.0, "images": 0.0, "unknown": 1.0}, + "transport_mix": {"ws": 0.6, "http_bridge": 0.4}, + "service_tier_mix": {"default": 0.90, "flex": 0.05, "priority": 0.05}, + "clients": {"codex-cli": 0.44, "openai-sdk-python": 0.3, "other": 0.02}, + "clients_other_ratio": 0.02, + "models": [ + { + "name": "gpt-5.4-codex", + "share": 0.62, + "reasoning": {"xhigh": 0.31, "high": 0.48, "medium": 0.21}, + "avg_output_tokens_bucket": "" + } + ], + "latency_ms_p50": 1200, + "ttft_ms_p50": 800, + "ttft_ms_p95": 3400, + "rate_limit_429_ratio": 0.004, + "top_upstream_errors": ["server_overloaded", "usage_limit_reached"] + }, + + "features": { + "api_firewall": true, + "quota_planner": true, + "sticky_sessions": true, + "conversation_archive": false, + "automations": false, + "fleet": false, + "model_sources_count": 2, + "api_keys_bucket": "", + "prometheus": false, + "otel": false, + "dashboard_auth": true, + "reset_credits": true, + "image_api_used": true + } +} +``` + +Field notes: + +- `top_upstream_errors`: enum `upstream_error_code` values only, top 5 by count. Free-text + `error_message` is banned by spec. +- `request_kinds`: current `request_logs` rows do not persist ingress route family. The existing + `request_kind` column is a workload class (`normal`, `warmup`, `compaction`, and similar), + while `source` identifies the upstream. Until an authoritative route-family signal exists, + rows are reported as `unknown`; source and model-name heuristics are deliberately forbidden. +- `clients`: canonical family shares from the normative mapping table in `spec.md`. Raw + `useragent_group` values never leave the instance. +- `models[].name`: official model catalog allowlist match; custom/unknown model names fold + into a single `{"name": "other"}` entry. +- Exact `requests` / token counts are transmitted raw deliberately: they power the global + aggregate counter story and cannot identify an instance. Everything correlated with spend + or org size (accounts, keys, cost, DB size) is bucketed. +- `replicas`: size of the configured HTTP bridge instance ring (multi-replica adoption signal). + +## Bucket sets + +- count buckets (accounts, api keys, plan mix): `0`, `1`, `2-5`, `6-20`, `21-100`, `100+` +- `db_size_bucket`: `unknown`, `<100MB`, `100MB-1GB`, `1-5GB`, `5-10GB`, `10-50GB`, `50GB+` +- `cost_usd_bucket` (7d): `<10`, `10-100`, `100-1k`, `1k-10k`, `10k-50k`, `50k+` +- `avg_output_tokens_bucket`: `<250`, `250-1k`, `1k-4k`, `4k-16k`, `16k+` + +## Consent resolution precedence + +`CODEX_LB_TELEMETRY_ENABLED` env (when set) > persisted decision > default +(`undecided` ⇒ active). The dialog is only shown while persisted state is `undecided` and +no env override exists. + +## Consent API and preview cost + +`GET /api/settings/telemetry` always returns `state`, `source`, `active`, and `preview`. The +default GET includes a preview envelope only for undecided/default consent, when the dialog can +appear; decided and environment-overridden responses return `preview: null` without running the +seven-day aggregate queries. Settings requests the same endpoint with +`include_preview=true` to fetch the current envelope on demand. `PUT /api/settings/telemetry` +persists the decision and returns `preview: null`. + +## Cadence and replica ownership + +The startup and 24-hour ticks run through the shared scheduler leader-election gate. Only the +leader constructs aggregates, transmits the snapshot, and logs the undecided-consent startup +notice. Followers perform none of that work, avoiding duplicate snapshots and duplicate notices. + +## Retention + +Each snapshot summarizes the previous seven days of existing local request logs. codex-lb does +not create a second local telemetry history or queue failed transmissions. The project-operated +collector is `https://telemetry.tokmaxxing.com`; its server-side retention duration is not yet +specified, so operators should assume transmitted snapshots remain stored until a published +retention policy or explicit deletion. + +## Failure modes + +- Endpoint down: bounded timeout (5s), at most one retry per interval, debug-level log, + proxy path untouched. Snapshot is rebuilt fresh next interval (no queue/backlog). +- Aggregation query cost: snapshot queries reuse the same 7-day aggregate shapes as the + dashboard reports module; they run on the leader scheduler once per tick and only on an API + request when the undecided dialog or an explicit settings preview needs them. On Postgres + instances with very large `request_logs` this is the same load class as one dashboard load. +- Clock skew / restart loops: the elected leader transmits the startup snapshot; SHM's + `/v1/activate` is idempotent (active → active refreshes last-seen). Rapid restart loops are + bounded by one snapshot per elected-leader process start; no local rate limiter in v1. + +## Example: privacy review quick check + +An instance with accounts `alice@corp.com` (workspace W1) + 12 others, a custom model source +`corp-internal-gpt`, and traffic from an internal tool `senpi/1.0`: + +- payload has `pool_bucket: "6-20"`, `workspace_accounts: true` +- `corp-internal-gpt` traffic appears as `models[].name == "other"` +- `senpi` traffic appears in `clients` under `other` and inflates `clients_other_ratio` +- the strings `alice`, `corp.com`, `W1`, `corp-internal-gpt`, `senpi` appear nowhere in the + serialized payload (schema snapshot test enforces this) diff --git a/openspec/specs/telemetry/spec.md b/openspec/specs/telemetry/spec.md new file mode 100644 index 0000000000..d9e8b674bd --- /dev/null +++ b/openspec/specs/telemetry/spec.md @@ -0,0 +1,223 @@ +# telemetry Specification + +## Purpose +Anonymous install-base telemetry (version distribution, deployment shapes, client ecosystem, feature usage) under an informed opt-out consent model, with a normative privacy allowlist so identifying data can never be transmitted. See `context.md` for the exact outbound field lists. +## Requirements + +### Requirement: Telemetry payload field allowlist + +The service MUST transmit only fields defined for each outbound body (registration, +activation, and snapshot envelope including its nested metrics) in this capability's +`context.md`, and MUST NOT transmit account emails, workspace identifiers, client IP +addresses, API keys, request or response content, raw user-agent strings, per-account +records, or free-text error messages in any telemetry payload. + +The snapshot metrics schema is versioned (`schema_version`). Adding a field to any transmitted +body requires a spec change to this capability; the outbound wire-schema test suite MUST fail +when registration, activation, the snapshot envelope, or nested metrics contain a field not +present in the documented schema. + +#### Scenario: Every outbound body contains only allowlisted fields + +- **WHEN** the sender serializes registration, activation, and snapshot requests +- **THEN** every top-level and nested field is present in the documented schemas, and an + outbound wire-schema regression test rejects any undeclared field in any body + +#### Scenario: Identifying data never serialized + +- **WHEN** the snapshot is built on an instance with linked accounts, API keys, and request + logs containing raw user agents and error messages +- **THEN** the serialized payload contains no email, workspace ID, IP address, API key + material, raw user-agent string, or free-text error message + +### Requirement: Consent state and default activation + +Telemetry consent MUST be a persisted tri-state (`undecided`, `enabled`, `disabled`) defaulting to `undecided`, and while consent is `undecided` the service SHALL treat telemetry as active. + +Upgrading an existing installation MUST introduce the consent state as `undecided` (existing +users get the same informed default-on treatment as new installs). + +#### Scenario: Fresh install defaults to active + +- **WHEN** codex-lb starts for the first time with no persisted consent and no environment + override +- **THEN** consent is `undecided` and telemetry snapshots are transmitted + +#### Scenario: Upgrade treats existing users as undecided + +- **WHEN** an existing installation migrates to a version with this capability +- **THEN** the migrated consent state is `undecided` and the one-time consent dialog is shown + on next dashboard entry + +### Requirement: One-time consent dialog with exact payload preview + +The dashboard MUST present a one-time consent dialog on first entry while consent is +`undecided`, and the dialog MUST display the exact snapshot envelope the instance would +transmit at that moment. Preview and sender MUST use one shared envelope constructor. The +preview timestamp MUST record preview generation time as a representative current timestamp; +the actual send MUST regenerate that value at transmission time. + +A decision (enable or disable) MUST be persisted and the dialog MUST NOT be shown again after +any decision. The dialog MUST offer disabling with no fewer clicks than enabling. + +The consent API MUST build the preview only while the undecided dialog is eligible or when an +operator explicitly requests it for the settings view. The response MUST retain the `preview` +field and set it to `null` when the preview was not requested and is not dialog-relevant. + +#### Scenario: Undecided operator sees payload preview + +- **WHEN** an operator opens the dashboard while consent is `undecided` +- **THEN** a dialog shows the live snapshot JSON with equally prominent enable and disable + actions + +#### Scenario: Decision is final until changed in settings + +- **WHEN** the operator chooses disable in the dialog +- **THEN** consent persists as `disabled`, no snapshot is transmitted afterward, and the + dialog never reappears + +#### Scenario: Decided consent status is a cheap read + +- **WHEN** the dashboard reads consent after a persisted decision without requesting a preview +- **THEN** the response contains `preview: null` and no snapshot aggregation query runs + +#### Scenario: Settings explicitly requests collected data + +- **WHEN** the settings view requests a preview for any consent state +- **THEN** the response contains a current snapshot envelope built with the same schema as the sender + +### Requirement: Settings toggle and environment kill switch + +The dashboard settings MUST expose a telemetry toggle reflecting the resolved consent state, and the environment variable `CODEX_LB_TELEMETRY_ENABLED` MUST override persisted consent when set (`false` disables all transmission, `true` enables and suppresses the consent dialog). + +#### Scenario: Headless deployment disables via environment + +- **WHEN** the service runs with `CODEX_LB_TELEMETRY_ENABLED=false` +- **THEN** no telemetry network traffic occurs regardless of persisted consent, and the + settings toggle shows telemetry as disabled by environment override + +#### Scenario: Toggle flips persisted consent + +- **WHEN** the operator disables telemetry in settings without an environment override +- **THEN** consent persists as `disabled` and transmission stops without restart + +### Requirement: Startup notice while undecided + +While consent is `undecided`, the elected leader MUST emit a single startup log line stating +that anonymous telemetry is active, where the collected-field documentation lives, and how to +disable it. Non-leader replicas MUST NOT duplicate the notice. + +#### Scenario: Headless operator is informed + +- **WHEN** the service starts with consent `undecided` +- **THEN** exactly one log line names the telemetry documentation location and the + `CODEX_LB_TELEMETRY_ENABLED=false` disable path + +### Requirement: Disabled means zero telemetry traffic + +When resolved consent is `disabled`, the service MUST NOT open any network connection to the telemetry endpoint. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle +- **THEN** no connection attempt to the telemetry endpoint is made + +### Requirement: Client family allowlist mapping + +Telemetry client statistics MUST report only canonical client-family identifiers produced by the documented mapping table, MUST map any unmatched user-agent group to `other`, and MUST NOT transmit raw user-agent group values. + +The canonical mapping table (raw `useragent_group` → family): + +| Raw group(s) | Family | +| --- | --- | +| `codex_exec`, `codex-tui` | `codex-cli` | +| `Codex Desktop` | `codex-desktop` | +| `codex_vscode` | `codex-vscode` | +| `AsyncOpenAI` | `openai-sdk-python` | +| `OpenAI` | `openai-sdk-js` | +| `ai`, `ai-sdk` | `vercel-ai-sdk` | +| `opencode` | `opencode` | +| `Mozilla` | `browser` | +| `curl`, `undici`, `node`, `Python-urllib`, `python-requests`, `aiohttp` | `script` | +| anything else | `other` | + +The payload MUST include `clients_other_ratio` so mapping coverage decay is observable +without ever transmitting the unmatched raw values. + +#### Scenario: Private tool names never leave the instance + +- **WHEN** request logs contain a user-agent group not present in the mapping table +- **THEN** its traffic is attributed to `other` and the raw group string is absent from the + payload + +#### Scenario: Codex CLI variants collapse to one family + +- **WHEN** traffic exists from both `codex_exec` and `codex-tui` +- **THEN** the payload reports a single `codex-cli` family combining both + +### Requirement: Model catalog allowlist with per-model reasoning mix + +Telemetry model statistics MUST include only model names present in the official model catalog allowlist, MUST map unmatched model names to `other`, and MUST report reasoning-effort distribution nested per model entry rather than as an instance-global aggregate. + +#### Scenario: Custom model source names are not transmitted + +- **WHEN** an operator has configured a custom model source with a private model name +- **THEN** that traffic appears under `other` and the private name is absent from the payload + +#### Scenario: Reasoning effort is model-scoped + +- **WHEN** the snapshot reports models +- **THEN** each model entry carries its own reasoning-effort share map and no global + reasoning mix field exists + +### Requirement: Fail-honest request-family attribution + +Request-family telemetry MUST be derived only from an authoritative persisted route-family +signal. Rows without such a signal MUST be attributed to `unknown`; the service MUST NOT infer +Chat, Responses, Images, or Audio families from upstream `source` or model name. + +#### Scenario: Ambiguous persisted rows remain unknown + +- **WHEN** persisted request rows identify only workload kind, upstream source, or model name +- **THEN** their request-family share is reported as `unknown` rather than a named route family + +### Requirement: Random instance identity + +The telemetry instance identifier MUST be a UUID generated randomly on first run, MUST NOT be derived from hardware, network, account, or operating-system identity, and MUST be regenerated if deleted. + +#### Scenario: Identifier carries no fingerprint + +- **WHEN** the instance identifier is created +- **THEN** it is a random UUIDv4 persisted locally, and deleting it yields a fresh unrelated + identifier on next start + +### Requirement: Transmission cadence and failure isolation + +The service SHALL transmit one snapshot at startup and one per 24-hour interval thereafter. +In a multi-replica deployment sharing a database, snapshot construction and transmission MUST +run only under the existing leader-election gate so at most one replica performs each tick. +Telemetry transmission failures MUST NOT affect proxy operation, MUST use a bounded timeout, +MUST NOT retry more than once per interval, and MUST log failures at debug level only. + +#### Scenario: Non-leader replica skips telemetry work + +- **WHEN** a telemetry tick runs in a process that does not hold the scheduler leader lease +- **THEN** that process neither builds a snapshot nor attempts a transmission + +#### Scenario: Collection endpoint outage is invisible + +- **WHEN** the telemetry endpoint is unreachable +- **THEN** proxy requests are unaffected, startup is not delayed beyond the bounded timeout, + and no warning-or-higher log noise is produced + +### Requirement: Bucketed sensitive aggregates + +Account pool size, per-plan account counts, API key count, database size, and cost aggregates MUST be transmitted as documented buckets, never as exact values. + +An unmeasurable database size MUST be reported as `unknown`, not as a plausible size bucket. + +#### Scenario: Pool size is a bucket + +- **WHEN** an instance has 13 linked accounts +- **THEN** the payload reports the `6-20` bucket and no exact account count diff --git a/tests/conftest.py b/tests/conftest.py index a886d66e92..fd23f99058 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -163,6 +163,13 @@ def _disable_data_retention_scheduler_startup(monkeypatch): monkeypatch.setattr(main_module, "build_data_retention_scheduler", lambda: _NoopScheduler()) +@pytest.fixture(autouse=True) +def _disable_telemetry_scheduler_startup(monkeypatch): + import app.main as main_module + + monkeypatch.setattr(main_module, "build_telemetry_scheduler", lambda: _NoopScheduler()) + + @pytest.fixture(autouse=True) def _disable_leader_election_startup(monkeypatch): """Replace the ambient app-lifespan leader election with a no-op. diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index c7b640d0fe..620f94b6e0 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -61,7 +61,11 @@ def _isolated_settings(**overrides: Any) -> Settings: # decision — operators who don't use the reset-credit surface shed the # per-replica authenticated upstream polling; default true keeps current # zero-config behavior and the interval setting alone cannot express "off". -MAX_SETTINGS_FIELDS = 127 +# 127 -> 129: telemetry_enabled + telemetry_endpoint (anonymous telemetry, +# #1618). telemetry_enabled has no hardcoded default because tri-state None +# drives the informed-consent dialog; the endpoint stays settable so +# self-hosters can point at their own collector or air-gap it. +MAX_SETTINGS_FIELDS = 129 def test_generated_settings_reference_matches_code() -> None: diff --git a/tests/unit/test_telemetry_api.py b/tests/unit/test_telemetry_api.py new file mode 100644 index 0000000000..b948f8ff52 --- /dev/null +++ b/tests/unit/test_telemetry_api.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +from unittest.mock import Mock + +import pytest + +from app.core.config.settings import get_settings + +pytestmark = pytest.mark.unit + + +@pytest.mark.asyncio +async def test_consent_api_get_preview_and_put_persists_without_restart(async_client, monkeypatch) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + response = await async_client.get("/api/settings/telemetry") + assert response.status_code == 200 + initial = response.json() + assert initial["state"] == "undecided" + assert initial["source"] == "default" + assert initial["active"] is True + assert set(initial["preview"]) == {"instance_id", "metrics", "timestamp"} + assert initial["preview"]["metrics"]["schema_version"] == 1 + assert initial["preview"]["instance_id"] == initial["preview"]["metrics"]["instance_id"] + + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert response.status_code == 200 + disabled = response.json() + assert disabled["state"] == "disabled" + assert disabled["source"] == "persisted" + assert disabled["active"] is False + assert disabled["preview"] is None + + builder = Mock(side_effect=AssertionError("decided consent must not build a preview")) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySnapshotBuilder", builder) + response = await async_client.get("/api/settings/telemetry") + assert response.status_code == 200 + assert response.json()["state"] == "disabled" + assert response.json()["preview"] is None + builder.assert_not_called() + + +@pytest.mark.asyncio +async def test_consent_api_builds_decided_preview_only_when_requested(async_client, monkeypatch) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + response = await async_client.get("/api/settings/telemetry?include_preview=true") + + assert response.status_code == 200 + payload = response.json() + assert payload["state"] == "disabled" + assert payload["preview"]["instance_id"] == payload["preview"]["metrics"]["instance_id"] + + +@pytest.mark.asyncio +async def test_consent_api_env_override_wins_and_suppresses_undecided_state(async_client, monkeypatch) -> None: + monkeypatch.setenv("CODEX_LB_TELEMETRY_ENABLED", "true") + get_settings.cache_clear() + + builder = Mock(side_effect=AssertionError("environment override must not build a preview")) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySnapshotBuilder", builder) + response = await async_client.get("/api/settings/telemetry") + + assert response.status_code == 200 + payload = response.json() + assert payload["state"] == "enabled" + assert payload["source"] == "env" + assert payload["active"] is True + assert payload["preview"] is None + builder.assert_not_called() diff --git a/tests/unit/test_telemetry_consent.py b/tests/unit/test_telemetry_consent.py new file mode 100644 index 0000000000..db0a0dfe9a --- /dev/null +++ b/tests/unit/test_telemetry_consent.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import asyncio +import logging +from collections.abc import Awaitable, Callable +from unittest.mock import AsyncMock, Mock + +import pytest + +from app.core.config.settings import get_settings +from app.db.models import DashboardSettings +from app.db.session import SessionLocal +from app.modules.telemetry.consent import TelemetryConsentStore, resolve_consent +from app.modules.telemetry.scheduler import TELEMETRY_INTERVAL_SECONDS, TelemetryScheduler + +pytestmark = pytest.mark.unit + + +class _GateLeader: + def __init__(self, *, leader: bool) -> None: + self.leader = leader + self.run_if_leader_calls = 0 + + async def run_if_leader(self, fn: Callable[[], Awaitable[object]]) -> object | None: + self.run_if_leader_calls += 1 + if not self.leader: + return None + return await fn() + + +def test_consent_precedence_and_default_activation() -> None: + assert resolve_consent(False, "enabled").state == "disabled" + assert resolve_consent(False, "enabled").source == "env" + assert resolve_consent(False, "enabled").active is False + + env_enabled = resolve_consent(True, "undecided") + assert env_enabled.state == "enabled" + assert env_enabled.source == "env" + assert env_enabled.active is True + + persisted_disabled = resolve_consent(None, "disabled") + assert persisted_disabled.state == "disabled" + assert persisted_disabled.source == "persisted" + assert persisted_disabled.active is False + + undecided = resolve_consent(None, "undecided") + assert undecided.state == "undecided" + assert undecided.source == "default" + assert undecided.active is True + + +@pytest.mark.asyncio +async def test_random_uuid_v4_identity_is_persisted_and_regenerated_after_deletion(db_setup) -> None: + del db_setup + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + first = await store.get_or_create_identity() + second = await store.get_or_create_identity() + assert first.instance_id == second.instance_id + assert first.public_key_hex == second.public_key_hex + assert first.instance_id.split("-")[2].startswith("4") + + row = await session.get(DashboardSettings, 1) + assert row is not None + row.telemetry_instance_id = None + await session.commit() + session.expire_all() + + replacement = await store.get_or_create_identity() + assert replacement.instance_id != first.instance_id + assert replacement.public_key_hex != first.public_key_hex + + +@pytest.mark.asyncio +async def test_disabled_scheduler_tick_makes_zero_sender_calls(db_setup, monkeypatch) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + await store.set_decision(False) + + sender = AsyncMock() + scheduler = TelemetryScheduler(sender=sender) + await scheduler._tick() + + sender.send_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_non_leader_scheduler_tick_builds_and_transmits_nothing(monkeypatch) -> None: + import app.modules.telemetry.scheduler as scheduler_module + + leader = _GateLeader(leader=False) + builder = Mock(side_effect=AssertionError("non-leader must not construct a snapshot builder")) + monkeypatch.setattr(scheduler_module, "_get_leader_election", lambda: leader) + monkeypatch.setattr(scheduler_module, "TelemetrySnapshotBuilder", builder) + sender = AsyncMock() + + await TelemetryScheduler(sender=sender)._tick(log_undecided_notice=True) + + assert leader.run_if_leader_calls == 1 + builder.assert_not_called() + sender.send_snapshot.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_main_lifespan_constructs_starts_and_stops_telemetry_scheduler(app_instance, monkeypatch) -> None: + import app.main as main_module + + scheduler = Mock() + scheduler.start = AsyncMock() + scheduler.stop = AsyncMock() + factory = Mock(return_value=scheduler) + monkeypatch.setattr(main_module, "build_telemetry_scheduler", factory) + + async with app_instance.router.lifespan_context(app_instance): + factory.assert_called_once_with() + scheduler.start.assert_awaited_once_with() + scheduler.stop.assert_not_awaited() + + scheduler.stop.assert_awaited_once_with() + + +@pytest.mark.asyncio +async def test_scheduler_sends_startup_and_interval_snapshots_with_one_undecided_notice( + db_setup, + monkeypatch, + caplog, +) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + assert TELEMETRY_INTERVAL_SECONDS == 24 * 60 * 60 + + sender = AsyncMock() + scheduler = TelemetryScheduler(sender=sender, interval_seconds=0.01) + with caplog.at_level(logging.INFO, logger="app.modules.telemetry.scheduler"): + await scheduler.start() + for _ in range(50): + if sender.send_snapshot.await_count >= 2: + break + await asyncio.sleep(0.01) + await scheduler.stop() + + assert sender.send_snapshot.await_count >= 2 + notices = [ + record.getMessage() for record in caplog.records if "Anonymous telemetry is active" in record.getMessage() + ] + assert len(notices) == 1 + assert "https://soju06.github.io/codex-lb/telemetry/" in notices[0] + assert "CODEX_LB_TELEMETRY_ENABLED=false" in notices[0] + assert scheduler._task is None diff --git a/tests/unit/test_telemetry_migration.py b/tests/unit/test_telemetry_migration.py new file mode 100644 index 0000000000..3428c5e7ab --- /dev/null +++ b/tests/unit/test_telemetry_migration.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import pytest +from alembic import command +from anyio import to_thread +from sqlalchemy import text +from sqlalchemy.ext.asyncio import create_async_engine + +from app.db.migrate import _build_alembic_config, run_upgrade + +pytestmark = pytest.mark.unit + + +@pytest.mark.asyncio +async def test_telemetry_migration_upgrade_defaults_and_downgrade(tmp_path) -> None: + db_url = f"sqlite+aiosqlite:///{tmp_path / 'telemetry.sqlite'}" + parent = "20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads" + revision = "20260806_000000_add_anonymous_telemetry" + telemetry_columns = { + "telemetry_consent", + "telemetry_instance_id", + "telemetry_private_key_encrypted", + } + + async def columns_and_rows(engine): + async with engine.connect() as connection: + columns = {row[1] for row in await connection.execute(text("PRAGMA table_info('dashboard_settings')"))} + rows = [] + if telemetry_columns <= columns: + rows = ( + await connection.execute( + text( + "SELECT telemetry_consent, telemetry_instance_id, " + "telemetry_private_key_encrypted FROM dashboard_settings" + ) + ) + ).all() + return columns, rows + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent, bootstrap_legacy=False)) + engine = create_async_engine(db_url) + try: + columns, _ = await columns_and_rows(engine) + assert not telemetry_columns & columns + + await to_thread.run_sync(lambda: run_upgrade(db_url, revision, bootstrap_legacy=False)) + columns, rows = await columns_and_rows(engine) + assert telemetry_columns <= columns + assert rows + assert all(row == ("undecided", None, None) for row in rows) + + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent)) + columns, _ = await columns_and_rows(engine) + assert not telemetry_columns & columns + + result = await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + assert result.current_revision == revision + columns, _ = await columns_and_rows(engine) + assert telemetry_columns <= columns + finally: + await engine.dispose() diff --git a/tests/unit/test_telemetry_sender.py b/tests/unit/test_telemetry_sender.py new file mode 100644 index 0000000000..7a4553c5b2 --- /dev/null +++ b/tests/unit/test_telemetry_sender.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +import json +import logging +from unittest.mock import AsyncMock, Mock + +import pytest +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey + +from app.modules.telemetry.consent import TelemetryIdentity +from app.modules.telemetry.schemas import TelemetrySnapshot, build_snapshot_envelope +from app.modules.telemetry.sender import TelemetrySender + +pytestmark = pytest.mark.unit + + +def _snapshot() -> TelemetrySnapshot: + return TelemetrySnapshot.model_validate( + { + "instance_id": "00000000-0000-4000-8000-000000000004", + "version": "1.0.0", + "python": "3.13", + "os": "linux", + "arch": "x86_64", + "uptime_hours": 1, + "deploy": { + "method": "bare", + "db_backend": "sqlite", + "db_size_bucket": "<100MB", + "replicas": 1, + "reverse_proxy": False, + }, + "accounts": { + "pool_bucket": "0", + "plan_mix": {"plus": "0", "pro": "0", "team": "0", "free": "0"}, + "workspace_accounts": False, + "routing_policy": "capacity_weighted", + "limit_warmup_enabled": False, + "egress_proxy_used": False, + }, + "usage_7d": { + "requests": 0, + "success_rate": 0, + "tokens_input": 0, + "tokens_output": 0, + "tokens_cached_ratio": 0, + "cost_usd_bucket": "<10", + "request_kinds": {"responses": 0, "chat": 0, "images": 0, "unknown": 0}, + "transport_mix": {"ws": 0, "http_bridge": 0}, + "service_tier_mix": {"default": 0, "flex": 0, "priority": 0}, + "clients": {}, + "clients_other_ratio": 0, + "models": [], + "latency_ms_p50": 0, + "ttft_ms_p50": 0, + "ttft_ms_p95": 0, + "rate_limit_429_ratio": 0, + "top_upstream_errors": [], + }, + "features": { + "api_firewall": False, + "quota_planner": False, + "sticky_sessions": False, + "conversation_archive": False, + "automations": False, + "fleet": True, + "model_sources_count": 0, + "api_keys_bucket": "0", + "prometheus": False, + "otel": False, + "dashboard_auth": True, + "reset_credits": False, + "image_api_used": False, + }, + } + ) + + +@pytest.mark.asyncio +async def test_sender_failure_isolated_retries_once_and_logs_debug_only(caplog) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + + async def context_provider(): + return True, identity + + sender = TelemetrySender("http://127.0.0.1:1", context_provider=context_provider) + sender._transmit_once = AsyncMock(side_effect=OSError("endpoint unreachable")) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await sender.send_snapshot(snapshot) + + assert sender._transmit_once.await_count == 2 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_sender_disabled_guard_does_not_construct_http_client(monkeypatch) -> None: + async def context_provider(): + return False, None + + client_session = Mock() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", client_session) + + await TelemetrySender(context_provider=context_provider).send_snapshot(_snapshot()) + + client_session.assert_not_called() + + +@pytest.mark.asyncio +async def test_sender_uses_canonical_shm_paths_and_valid_ed25519_signature() -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + sender = TelemetrySender() + sender._post = AsyncMock() + sender._post_signed = AsyncMock() + session = Mock() + + await sender._transmit_once(session, snapshot, identity) + + register_call = sender._post.await_args + assert register_call is not None + assert register_call.args[1] == "/v1/register" + assert [call.args[1] for call in sender._post_signed.await_args_list] == ["/v1/activate", "/v1/snapshot"] + assert all("/api/v1/" not in call.args[1] for call in sender._post_signed.await_args_list) + + registration = json.loads(register_call.args[2]) + activation = json.loads(sender._post_signed.await_args_list[0].args[2]) + envelope = json.loads(sender._post_signed.await_args_list[1].args[2]) + assert set(registration) == { + "app_name", + "app_version", + "deployment_mode", + "environment", + "instance_id", + "os_arch", + "public_key", + } + assert set(activation) == {"action"} + assert set(envelope) == {"instance_id", "metrics", "timestamp"} + + signing_sender = TelemetrySender() + signing_sender._post = AsyncMock() + body = b'{"action":"activate"}' + await signing_sender._post_signed(session, "/v1/activate", body, identity, accepted={200}) + signed_call = signing_sender._post.await_args + assert signed_call is not None + headers = signed_call.kwargs["headers"] + identity.private_key.public_key().verify(bytes.fromhex(headers["X-Signature"]), body) + assert headers["X-Instance-ID"] == identity.instance_id + + +def _key_structure(value): + if isinstance(value, dict): + return {key: _key_structure(child) for key, child in value.items()} + if isinstance(value, list): + return [_key_structure(value[0])] if value else [] + return None + + +@pytest.mark.asyncio +async def test_preview_and_sender_snapshot_envelopes_have_identical_key_structure() -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + preview = build_snapshot_envelope(snapshot) + sender = TelemetrySender() + sender._post = AsyncMock() + sender._post_signed = AsyncMock() + + await sender._transmit_once(Mock(), snapshot, identity) + + sender_body = json.loads(sender._post_signed.await_args_list[-1].args[2]) + preview_body = json.loads(preview.model_dump_json()) + assert _key_structure(sender_body) == _key_structure(preview_body) diff --git a/tests/unit/test_telemetry_snapshot.py b/tests/unit/test_telemetry_snapshot.py new file mode 100644 index 0000000000..56630b989c --- /dev/null +++ b/tests/unit/test_telemetry_snapshot.py @@ -0,0 +1,503 @@ +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from pathlib import Path +from typing import get_args + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from app.core.balancer.logic import RoutingStrategy +from app.core.crypto import TokenEncryptor +from app.core.utils.time import utcnow +from app.db.models import Account, AccountStatus, ApiKey, Base, ModelSource, RequestLog +from app.modules.telemetry.clients import ( + CANONICAL_CLIENT_FAMILIES, + CLIENT_FAMILY_BY_RAW_GROUP, + ClientCount, + client_family, + client_shares, +) +from app.modules.telemetry.schemas import TelemetryActivation, TelemetryRegistration, build_snapshot_envelope +from app.modules.telemetry.snapshot import ( + _ROUTING_POLICIES, + TelemetrySnapshotBuilder, + _canonical_routing_policy, + cost_bucket, + count_bucket, + db_size_bucket, + output_tokens_bucket, +) + +pytestmark = pytest.mark.unit + + +@pytest.fixture +async def async_session() -> AsyncIterator[AsyncSession]: + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + async with factory() as session: + yield session + await engine.dispose() + + +def _request_log( + request_id: str, + *, + model: str, + useragent_group: str, + reasoning_effort: str | None = None, + output_tokens: int = 100, + account_id: str | None = None, + status: str = "success", + **values, +) -> RequestLog: + return RequestLog( + account_id=account_id, + request_id=request_id, + requested_at=utcnow(), + model=model, + status=status, + useragent_group=useragent_group, + reasoning_effort=reasoning_effort, + input_tokens=200, + output_tokens=output_tokens, + cached_input_tokens=50, + cost_usd=1.0, + latency_ms=1_000, + latency_first_token_ms=400, + transport="http", + **values, + ) + + +@pytest.mark.asyncio +async def test_snapshot_serialized_field_set_matches_documented_schema(async_session: AsyncSession) -> None: + async_session.add(_request_log("schema", model="gpt-5.4", useragent_group="codex_exec")) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000001") + payload = snapshot.model_dump() + + assert set(payload) == { + "schema_version", + "instance_id", + "version", + "python", + "os", + "arch", + "uptime_hours", + "deploy", + "accounts", + "usage_7d", + "features", + } + assert set(payload["deploy"]) == {"method", "db_backend", "db_size_bucket", "replicas", "reverse_proxy"} + assert set(payload["accounts"]) == { + "pool_bucket", + "plan_mix", + "workspace_accounts", + "routing_policy", + "limit_warmup_enabled", + "egress_proxy_used", + } + assert set(payload["accounts"]["plan_mix"]) == {"plus", "pro", "team", "free"} + assert set(payload["usage_7d"]) == { + "requests", + "success_rate", + "tokens_input", + "tokens_output", + "tokens_cached_ratio", + "cost_usd_bucket", + "request_kinds", + "transport_mix", + "service_tier_mix", + "clients", + "clients_other_ratio", + "models", + "latency_ms_p50", + "ttft_ms_p50", + "ttft_ms_p95", + "rate_limit_429_ratio", + "top_upstream_errors", + } + assert set(payload["usage_7d"]["request_kinds"]) == {"responses", "chat", "images", "unknown"} + assert set(payload["usage_7d"]["transport_mix"]) == {"ws", "http_bridge"} + assert set(payload["usage_7d"]["service_tier_mix"]) == {"default", "flex", "priority"} + assert set(payload["usage_7d"]["models"][0]) == { + "name", + "share", + "reasoning", + "avg_output_tokens_bucket", + } + assert set(payload["features"]) == { + "api_firewall", + "quota_planner", + "sticky_sessions", + "conversation_archive", + "automations", + "fleet", + "model_sources_count", + "api_keys_bucket", + "prometheus", + "otel", + "dashboard_auth", + "reset_credits", + "image_api_used", + } + + registration = TelemetryRegistration( + app_version=snapshot.version, + deployment_mode=snapshot.deploy.method, + instance_id=snapshot.instance_id, + os_arch=f"{snapshot.os}/{snapshot.arch}", + public_key="00", + ).model_dump(mode="json") + activation = TelemetryActivation().model_dump(mode="json") + envelope = build_snapshot_envelope(snapshot).model_dump(mode="json") + assert set(registration) == { + "app_name", + "app_version", + "deployment_mode", + "environment", + "instance_id", + "os_arch", + "public_key", + } + assert set(activation) == {"action"} + assert set(envelope) == {"instance_id", "metrics", "timestamp"} + + +def test_client_mapping_table_and_unknown_family_are_allowlisted() -> None: + for raw_group, expected_family in CLIENT_FAMILY_BY_RAW_GROUP.items(): + assert client_family(raw_group) == expected_family + assert client_family("senpi") == "other" + + shares, other_ratio = client_shares( + [ + ClientCount("codex_exec", 2), + ClientCount("codex-tui", 3), + ClientCount("senpi", 1), + ] + ) + assert shares == {"codex-cli": 0.833333, "other": 0.166667} + assert other_ratio == 0.166667 + assert "senpi" not in str(shares) + + +def test_client_share_emission_rejects_noncanonical_mapping(monkeypatch) -> None: + monkeypatch.setitem(CLIENT_FAMILY_BY_RAW_GROUP, "unexpected", "private-client") + assert "private-client" not in CANONICAL_CLIENT_FAMILIES + + with pytest.raises(ValueError, match="non-canonical telemetry client family"): + client_shares([ClientCount("unexpected", 1)]) + + +def test_routing_policy_allowlist_is_derived_from_balancer_declaration() -> None: + assert _ROUTING_POLICIES == frozenset(get_args(RoutingStrategy)) + for strategy in get_args(RoutingStrategy): + assert _canonical_routing_policy(strategy) == strategy + + +@pytest.mark.asyncio +async def test_model_catalog_filter_merges_custom_models_and_scopes_reasoning( + async_session: AsyncSession, +) -> None: + async_session.add_all( + [ + _request_log("official-high", model="gpt-5.4", useragent_group="OpenAI", reasoning_effort="high"), + _request_log("official-low", model="gpt-5.4", useragent_group="OpenAI", reasoning_effort="low"), + _request_log( + "private-high", + model="corp-internal-gpt", + useragent_group="senpi", + reasoning_effort="high", + output_tokens=2_000, + ), + _request_log( + "private-custom-effort", + model="another-private-model", + useragent_group="senpi", + reasoning_effort="secret-effort", + output_tokens=2_000, + ), + ] + ) + await async_session.commit() + + payload = (await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000002")).model_dump() + models = {model["name"]: model for model in payload["usage_7d"]["models"]} + + assert set(models) == {"gpt-5.4", "other"} + assert models["gpt-5.4"]["reasoning"] == {"high": 0.5, "low": 0.5} + assert models["other"]["reasoning"] == {"high": 0.5, "other": 0.5} + assert models["other"]["share"] == 0.5 + assert models["other"]["avg_output_tokens_bucket"] == "1k-4k" + assert "reasoning" not in payload["usage_7d"] + serialized = str(payload) + assert "corp-internal-gpt" not in serialized + assert "another-private-model" not in serialized + assert "secret-effort" not in serialized + + +@pytest.mark.asyncio +async def test_request_kind_mix_fails_honest_without_persisted_route_family(async_session: AsyncSession) -> None: + async_session.add_all( + [ + _request_log("subscription", model="gpt-5.4", useragent_group="codex_exec"), + _request_log( + "source-backed", + model="gpt-5.4", + useragent_group="OpenAI", + source="model_source", + ), + _request_log( + "image-shaped", + model="gpt-image-1", + useragent_group="OpenAI", + source="model_source", + ), + ] + ) + await async_session.commit() + + payload = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000005") + + assert payload.usage_7d.request_kinds.model_dump() == { + "responses": 0.0, + "chat": 0.0, + "images": 0.0, + "unknown": 1.0, + } + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (0, "0"), + (1, "1"), + (2, "2-5"), + (5, "2-5"), + (6, "6-20"), + (20, "6-20"), + (21, "21-100"), + (100, "21-100"), + (101, "100+"), + ], +) +def test_count_bucket_edges(value: int, expected: str) -> None: + assert count_bucket(value) == expected + + +def test_sensitive_aggregate_bucket_edges() -> None: + mib = 1024**2 + gib = 1024**3 + assert [ + db_size_bucket(value) for value in (None, 0, 100 * mib - 1, 100 * mib, gib, 5 * gib, 10 * gib, 50 * gib) + ] == [ + "unknown", + "<100MB", + "<100MB", + "100MB-1GB", + "1-5GB", + "5-10GB", + "10-50GB", + "50GB+", + ] + assert [cost_bucket(value) for value in (0, 9.99, 10, 99.99, 100, 999.99, 1_000, 10_000, 50_000)] == [ + "<10", + "<10", + "10-100", + "10-100", + "100-1k", + "100-1k", + "1k-10k", + "10k-50k", + "50k+", + ] + assert [output_tokens_bucket(value) for value in (0, 249, 250, 999, 1_000, 3_999, 4_000, 15_999, 16_000)] == [ + "<250", + "<250", + "250-1k", + "250-1k", + "1k-4k", + "1k-4k", + "4k-16k", + "4k-16k", + "16k+", + ] + + +@pytest.mark.asyncio +async def test_unmeasurable_database_size_is_unknown_and_logs_original_exception( + async_session: AsyncSession, + monkeypatch, + caplog, +) -> None: + error = OSError("stat denied") + target = Path("/tmp/telemetry-unmeasurable-db.sqlite3") + real_stat = Path.stat + + def fail_stat(path: Path, *, follow_symlinks: bool = True): + if path == target: + raise error + return real_stat(path, follow_symlinks=follow_symlinks) + + monkeypatch.setattr("app.modules.telemetry.snapshot.sqlite_db_path_from_url", lambda _url: str(target)) + monkeypatch.setattr(Path, "stat", fail_stat) + builder = TelemetrySnapshotBuilder(async_session) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.snapshot"): + size = await builder._database_size_bytes() + + assert size is None + assert db_size_bucket(size) == "unknown" + assert caplog.records[-1].exc_info is not None + assert caplog.records[-1].exc_info[1] is error + + +@pytest.mark.asyncio +async def test_privacy_quick_check_identifying_values_never_serialize(async_session: AsyncSession) -> None: + encryptor = TokenEncryptor() + account = Account( + id="account-private-id", + email="alice@corp.com", + workspace_id="W1", + plan_type="team", + access_token_encrypted=encryptor.encrypt("access-private"), + refresh_token_encrypted=encryptor.encrypt("refresh-private"), + id_token_encrypted=encryptor.encrypt("id-private"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + async_session.add(account) + async_session.add( + ApiKey( + id="api-key-private-id", + name="private-key-name", + key_hash="super-secret-api-key-hash", + key_prefix="sk-private", + is_active=True, + ) + ) + async_session.add( + ModelSource( + id="private-source-id", + name="private-source-name", + base_url="https://private.example.test", + api_key_encrypted=encryptor.encrypt("source-api-key"), + is_enabled=True, + ) + ) + async_session.add( + _request_log( + "privacy", + account_id=account.id, + model="corp-internal-gpt", + useragent_group="senpi", + useragent="senpi/1.0 alice@corp.com", + client_ip="192.0.2.9", + # Error status so the private code exercises the top-errors + # sanitizer; cancelled/success rows are excluded from that metric. + status="error", + error_message="free text alice W1 super-secret-api-key-hash", + upstream_error_code="private-upstream-message", + ) + ) + await async_session.commit() + + serialized = ( + await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000003") + ).model_dump_json() + + for private_value in ( + "alice", + "corp.com", + "W1", + "corp-internal-gpt", + "senpi", + "192.0.2.9", + "super-secret-api-key-hash", + "private-source-name", + "private-source-id", + "private-upstream-message", + ): + assert private_value not in serialized + assert '"pool_bucket":"1"' in serialized + assert '"workspace_accounts":true' in serialized + assert '"name":"other"' in serialized + assert '"clients":{"other":1.0}' in serialized + assert '"top_upstream_errors":["other"]' in serialized + + +@pytest.mark.asyncio +async def test_success_rate_excludes_cancelled_terminals(async_session: AsyncSession) -> None: + async_session.add(_request_log("ok", model="gpt-5.4", useragent_group="codex_exec")) + async_session.add( + _request_log( + "cancel-1", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "cancel-2", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "err", + model="gpt-5.4", + useragent_group="codex_exec", + status="error", + upstream_error_code="server_error", + ) + ) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000004") + + # 1 success out of 4 requests: cancellations are neither successes nor + # errors, so they must not inflate the numerator. + assert snapshot.usage_7d.success_rate == 0.25 + + +@pytest.mark.asyncio +async def test_top_upstream_errors_exclude_cancelled_terminals(async_session: AsyncSession) -> None: + for index in range(3): + async_session.add( + _request_log( + f"cancel-{index}", + model="gpt-5.4", + useragent_group="codex_exec", + status="cancelled", + upstream_error_code="client_disconnected", + ) + ) + async_session.add( + _request_log( + "err", + model="gpt-5.4", + useragent_group="codex_exec", + status="error", + upstream_error_code="server_error", + ) + ) + await async_session.commit() + + snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000005") + + # High-volume disconnects (status='cancelled' with a retained + # client_disconnected code) must not displace genuine upstream failures. + assert snapshot.usage_7d.top_upstream_errors == ["server_error"] From 2164b8c0656b3a8539ccdf8a4a8655a788af9ae8 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 13 Aug 2026 18:53:37 +0900 Subject: [PATCH 005/117] fix(dashboard): pin web asset MIME types against poisoned OS registries (#1709) On Windows, mimetypes merges HKCR mappings where third-party software commonly remaps .js to text/plain; FileResponse then serves every /assets/*.js as text/plain and strict browser MIME checking blanks the dashboard. Register text/javascript (+css/svg/json/woff/woff2/html) via mimetypes.add_type at app import, which overrides the merged registry table on all platforms and is a no-op where defaults are already correct. Fixes #1698 Co-authored-by: Claude Fable 5 --- app/main.py | 27 +++++++++++ .../fix-windows-asset-mime-types/proposal.md | 18 +++++++ .../specs/frontend-architecture/spec.md | 39 +++++++++++++++ .../fix-windows-asset-mime-types/tasks.md | 12 +++++ tests/integration/test_health_and_errors.py | 47 +++++++++++++++++++ 5 files changed, 143 insertions(+) create mode 100644 openspec/changes/fix-windows-asset-mime-types/proposal.md create mode 100644 openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/fix-windows-asset-mime-types/tasks.md diff --git a/app/main.py b/app/main.py index 849f29ac0c..3603e0360f 100644 --- a/app/main.py +++ b/app/main.py @@ -2,6 +2,7 @@ import asyncio import logging +import mimetypes import os import stat import sys @@ -109,6 +110,32 @@ logger = logging.getLogger(__name__) +# On Windows, ``mimetypes`` merges HKCR registry mappings where third-party +# software commonly remaps web extensions (``.js`` -> ``text/plain``), and +# browsers enforce strict MIME checking for ES module scripts, so a poisoned +# mapping renders the dashboard as a blank page (issue #1698). ``FileResponse`` +# resolves ``media_type`` through ``mimetypes.guess_type``, so pin every +# extension the built dashboard serves; ``add_type`` wins over the merged +# registry table on all platforms. +_WEB_ASSET_MIME_TYPES: dict[str, str] = { + ".js": "text/javascript", + ".mjs": "text/javascript", + ".css": "text/css", + ".svg": "image/svg+xml", + ".json": "application/json", + ".woff2": "font/woff2", + ".woff": "font/woff", + ".html": "text/html", +} + + +def _ensure_web_asset_mime_types() -> None: + for extension, mime_type in _WEB_ASSET_MIME_TYPES.items(): + mimetypes.add_type(mime_type, extension) + + +_ensure_web_asset_mime_types() + def _log_abandoned_lease_release(task: asyncio.Task[None]) -> None: if task.cancelled(): diff --git a/openspec/changes/fix-windows-asset-mime-types/proposal.md b/openspec/changes/fix-windows-asset-mime-types/proposal.md new file mode 100644 index 0000000000..f9fc5e146a --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/proposal.md @@ -0,0 +1,18 @@ +## Why + +On Windows, Python's `mimetypes` merges file-type mappings from the `HKCR` registry, where third-party software commonly remaps web extensions (`.js` → `text/plain`). Starlette's `FileResponse` resolves `media_type` through `mimetypes.guess_type`, and browsers enforce strict MIME checking for ES module scripts, so on such machines every `/assets/*.js` response ships as `text/plain` and the dashboard renders as a blank page (issue #1698). macOS/Linux use the built-in table and never hit this. + +## What Changes + +- Pin the MIME type of every extension the built dashboard serves (`.js`, `.mjs`, `.css`, `.svg`, `.json`, `.woff`, `.woff2`, `.html`) via `mimetypes.add_type` at application import, which overrides the merged registry table on all platforms. +- No behavior change on platforms whose default table is already correct — the pinned values equal the stdlib defaults. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Dashboard delivery additionally guarantees correct web MIME types independent of the host OS's `mimetypes` registry state. diff --git a/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md b/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..ef73075a74 --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/specs/frontend-architecture/spec.md @@ -0,0 +1,39 @@ +## MODIFIED Requirements + +### Requirement: Dashboard serving is compressed, cache-correct, and chart-lazy + +Dashboard API and static-asset responses MUST be served gzip-compressed when the client accepts it, while proxy paths MUST NOT pass through a compressing wrapper. Content-hashed assets under `/assets/` MUST be served with immutable year-long `Cache-Control`; `index.html` MUST remain `no-cache`. Chart vendor code MUST NOT load before first paint: it MUST live in an async-only chunk that is neither statically imported by the entry chunk nor modulepreloaded. Static assets MUST be served with their correct web MIME types (`.js`/`.mjs` as `text/javascript`, `.css` as `text/css`, `.svg` as `image/svg+xml`, `.json` as `application/json`, `.woff`/`.woff2` as `font/woff`/`font/woff2`, `.html` as `text/html`) regardless of the host operating system's `mimetypes` registry state, so strict browser MIME checking never rejects dashboard module scripts. + +#### Scenario: Assets are compressed and immutable + +- **WHEN** a browser requests a hashed asset under `/assets/` with `Accept-Encoding: gzip` +- **THEN** the response is gzip-encoded +- **AND** carries `Cache-Control: public, max-age=31536000, immutable` + +#### Scenario: index.html stays fresh across deploys + +- **WHEN** the SPA shell is requested +- **THEN** the response carries `Cache-Control: no-cache` + +#### Scenario: Proxy streaming paths are never compressed by the dashboard wrapper + +- **WHEN** a request targets a proxy path (`/backend-api/*`, `/v1/*`) +- **THEN** the dashboard gzip middleware passes it through untouched + +#### Scenario: Ranged asset requests bypass compression + +- **WHEN** an asset request carries a `Range` header +- **THEN** the response is served uncompressed with a valid 206 `Content-Range` over unencoded bytes + +#### Scenario: Chart vendor code loads lazily + +- **WHEN** the built dashboard entry page loads +- **THEN** the recharts chunk is not statically imported by the entry chunk and not modulepreloaded +- **AND** charts render correctly once their async chunk loads + +#### Scenario: Module scripts survive a poisoned OS MIME registry + +- **GIVEN** the host operating system maps `.js` to `text/plain` in its `mimetypes` sources (e.g. Windows `HKCR` registry entries) +- **WHEN** a browser requests a hashed `.js` asset under `/assets/` +- **THEN** the response `Content-Type` is `text/javascript` +- **AND** the dashboard SPA boots instead of failing strict module MIME checking diff --git a/openspec/changes/fix-windows-asset-mime-types/tasks.md b/openspec/changes/fix-windows-asset-mime-types/tasks.md new file mode 100644 index 0000000000..b0afb727da --- /dev/null +++ b/openspec/changes/fix-windows-asset-mime-types/tasks.md @@ -0,0 +1,12 @@ +## 1. Fix + +- [x] 1.1 Register `mimetypes.add_type` overrides for all dashboard asset extensions at `app/main.py` import, before any `FileResponse` is constructed + +## 2. Tests + +- [x] 2.1 Route-level regression: with a poisoned `.js -> text/plain` mapping re-registered over, `GET /assets/*.js` serves `text/javascript` (the externally failing product path from issue #1698) +- [x] 2.2 Unit: `_ensure_web_asset_mime_types()` restores every pinned extension after simulated registry poisoning + +## 3. Spec + +- [x] 3.1 Extend the `frontend-architecture` dashboard-delivery requirement with MIME-type correctness independent of the OS registry diff --git a/tests/integration/test_health_and_errors.py b/tests/integration/test_health_and_errors.py index f0b2609d54..dfa66f01b8 100644 --- a/tests/integration/test_health_and_errors.py +++ b/tests/integration/test_health_and_errors.py @@ -98,3 +98,50 @@ async def test_missing_static_asset_returns_not_found(async_client): assert response.status_code == 404 assert response.json()["detail"] == "Not Found" assert response.headers["X-App-Version"] == __version__ + + +def test_ensure_web_asset_mime_types_overrides_poisoned_registry(): + """Simulates the Windows HKCR poisoning from issue #1698. + + ``mimetypes.add_type`` mutates the global table, so this test re-runs the + startup registration after poisoning and leaves the correct mappings in + place for the rest of the suite. + """ + import mimetypes + + from app.main import _WEB_ASSET_MIME_TYPES, _ensure_web_asset_mime_types + + for extension in _WEB_ASSET_MIME_TYPES: + mimetypes.add_type("text/plain", extension) + assert mimetypes.guess_type("x.js")[0] == "text/plain" + + _ensure_web_asset_mime_types() + + for extension, expected in _WEB_ASSET_MIME_TYPES.items(): + assert mimetypes.guess_type(f"x{extension}")[0] == expected, extension + + +@pytest.mark.asyncio +async def test_assets_js_served_as_javascript_despite_poisoned_registry(async_client): + """Product-path regression for issue #1698: /assets/*.js must serve + text/javascript even when the OS mimetypes sources map .js to text/plain, + or strict browser MIME checking rejects every dashboard module script.""" + import mimetypes + + from app.main import _ensure_web_asset_mime_types + + asset_name = next( + (candidate.name for candidate in sorted((_STATIC_DIR / "assets").glob("*.js"))), + None, + ) + assert asset_name is not None, "built dashboard assets missing; run cd frontend && bun run build" + + mimetypes.add_type("text/plain", ".js") + try: + _ensure_web_asset_mime_types() + response = await async_client.get(f"/assets/{asset_name}") + finally: + _ensure_web_asset_mime_types() + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/javascript") From c3f0c568cb4dda4547f5d765951ba0748e7c923a Mon Sep 17 00:00:00 2001 From: Soju06 Date: Fri, 14 Aug 2026 11:34:24 +0900 Subject: [PATCH 006/117] docs(openspec): archive 90 landed changes and sync their specs (#1713) * docs(openspec): archive 90 landed changes and sync their specs Bulk archive of merged-and-implemented changes: 89 archived via 'openspec archive' (delta specs applied to main specs, 7 new capability specs created) plus preserve-historical-compact-side-effects archived --skip-specs (its delta is already a subset of the current main spec). 'openspec validate --specs' passes (57 items). 22 completed changes stay active: their deltas contain requirements or scenarios absent from the current main specs (spec drift from merging without sync), so blind archival would drop normative content. Tracked for per-change manual merge. Co-Authored-By: Claude Fable 5 * docs(openspec): write real Purpose sections for the seven new capability specs Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../context.md | 0 .../proposal.md | 0 .../specs/telemetry/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../after-per-account-capacity.png | Bin .../before-per-account-capacity.png | Bin .../specs/frontend-architecture/spec.md | 0 .../specs/proxy-admission-control/spec.md | 0 .../specs/proxy-runtime-observability/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/account-routing/spec.md | 0 .../specs/database-migrations/spec.md | 0 .../specs/responses-api-compat/spec.md | 0 .../specs/sticky-session-operations/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../verify-report.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/realtime-api-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/rate-limit-reset-credits/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/data-retention/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/query-caching/spec.md | 0 .../specs/upstream-proxy-routing/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/github-automation/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/account-import/spec.md | 0 .../specs/audio-transcriptions-compat/spec.md | 0 .../specs/http-ingress-limits/spec.md | 0 .../specs/images-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/account-routing/spec.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/http-ingress-limits/spec.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/query-caching/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/database-migrations/spec.md | 0 .../tasks.md | 0 .../context.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/deployment-networking/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/deployment-installation/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/deployment-networking/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/deployment-installation/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/date-display-format/spec.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/deployment-installation/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/deployment-installation/spec.md | 0 .../specs/graceful-shutdown/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/audit-logging/spec.md | 0 .../specs/fleet-summary/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/fleet-summary/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../context.md | 0 .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/usage-error-metrics/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../2026-08-13-fix-gpt-5-6-pricing}/design.md | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../2026-08-13-fix-gpt-5-6-pricing}/tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/database-backends/spec.md | 0 .../specs/deployment-installation/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/proxy-runtime-observability/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/proxy-runtime-observability/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/quota-phase-planner/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/account-routing/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../context.md | 0 .../proposal.md | 0 .../specs/database-backends/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/sticky-session-operations/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/admin-auth/spec.md | 0 .../specs/api-keys/spec.md | 0 .../specs/deployment-installation/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/database-migrations/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/runtime-portability/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/proxy-admission-control/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/release-management/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/proxy-architecture/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/upstream-proxy-routing/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../2026-08-13-self-host-mono-font}/tasks.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../context.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/query-caching/spec.md | 0 .../tasks.md | 0 .../context.md | 0 .../proposal.md | 0 .../specs/user-documentation/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/api-keys/spec.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../design.md | 0 .../proposal.md | 0 .../specs/proxy-runtime-observability/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/proxy-admission-control/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/frontend-architecture/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../design.md | 0 .../proposal.md | 0 .../specs/responses-api-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/usage-refresh-policy/spec.md | 0 .../tasks.md | 0 .../.openspec.yaml | 0 .../proposal.md | 0 .../specs/database-backends/spec.md | 0 .../specs/database-migrations/spec.md | 0 .../tasks.md | 0 openspec/specs/account-import/spec.md | 65 + openspec/specs/account-routing/spec.md | 131 +- openspec/specs/api-keys/spec.md | 152 +- .../specs/audio-transcriptions-compat/spec.md | 64 + openspec/specs/audit-logging/spec.md | 49 + openspec/specs/data-retention/spec.md | 29 + openspec/specs/database-backends/spec.md | 111 +- openspec/specs/database-migrations/spec.md | 92 ++ openspec/specs/date-display-format/spec.md | 96 ++ .../specs/deployment-installation/spec.md | 170 ++ openspec/specs/deployment-networking/spec.md | 76 + openspec/specs/fleet-summary/spec.md | 59 +- openspec/specs/frontend-architecture/spec.md | 989 +++++++++++- openspec/specs/github-automation/spec.md | 135 ++ openspec/specs/graceful-shutdown/spec.md | 233 +++ openspec/specs/http-ingress-limits/spec.md | 172 ++ openspec/specs/images-api-compat/spec.md | 68 + openspec/specs/model-catalog-compat/spec.md | 224 +++ .../specs/proxy-admission-control/spec.md | 215 +++ openspec/specs/proxy-architecture/spec.md | 73 + .../specs/proxy-runtime-observability/spec.md | 149 +- openspec/specs/query-caching/spec.md | 83 + openspec/specs/quota-phase-planner/spec.md | 14 +- .../specs/rate-limit-reset-credits/spec.md | 30 +- openspec/specs/release-management/spec.md | 73 +- openspec/specs/responses-api-compat/spec.md | 1397 ++++++++++++++++- openspec/specs/runtime-portability/spec.md | 38 + .../specs/sticky-session-operations/spec.md | 86 + openspec/specs/upstream-proxy-routing/spec.md | 64 + openspec/specs/usage-error-metrics/spec.md | 170 ++ openspec/specs/usage-refresh-policy/spec.md | 223 ++- openspec/specs/user-documentation/spec.md | 35 + 452 files changed, 5517 insertions(+), 48 deletions(-) rename openspec/changes/{active-conversations-average => archive/2026-08-13-active-conversations-average}/.openspec.yaml (100%) rename openspec/changes/{active-conversations-average => archive/2026-08-13-active-conversations-average}/proposal.md (100%) rename openspec/changes/{active-conversations-average => archive/2026-08-13-active-conversations-average}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{active-conversations-average => archive/2026-08-13-active-conversations-average}/tasks.md (100%) rename openspec/changes/{add-anonymous-telemetry => archive/2026-08-13-add-anonymous-telemetry}/context.md (100%) rename openspec/changes/{add-anonymous-telemetry => archive/2026-08-13-add-anonymous-telemetry}/proposal.md (100%) rename openspec/changes/{add-anonymous-telemetry => archive/2026-08-13-add-anonymous-telemetry}/specs/telemetry/spec.md (100%) rename openspec/changes/{add-anonymous-telemetry => archive/2026-08-13-add-anonymous-telemetry}/tasks.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/.openspec.yaml (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/design.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/proposal.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/screenshots/after-per-account-capacity.png (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/screenshots/before-per-account-capacity.png (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/specs/proxy-admission-control/spec.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/specs/proxy-runtime-observability/spec.md (100%) rename openspec/changes/{add-api-key-stream-fair-share => archive/2026-08-13-add-api-key-stream-fair-share}/tasks.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/.openspec.yaml (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/design.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/proposal.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/specs/account-routing/spec.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/specs/database-migrations/spec.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/specs/sticky-session-operations/spec.md (100%) rename openspec/changes/{add-capability-aware-routing => archive/2026-08-13-add-capability-aware-routing}/tasks.md (100%) rename openspec/changes/{add-conversation-dashboard => archive/2026-08-13-add-conversation-dashboard}/design.md (100%) rename openspec/changes/{add-conversation-dashboard => archive/2026-08-13-add-conversation-dashboard}/proposal.md (100%) rename openspec/changes/{add-conversation-dashboard => archive/2026-08-13-add-conversation-dashboard}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{add-conversation-dashboard => archive/2026-08-13-add-conversation-dashboard}/tasks.md (100%) rename openspec/changes/{add-conversation-dashboard => archive/2026-08-13-add-conversation-dashboard}/verify-report.md (100%) rename openspec/changes/{add-ko-complete-dashboard-i18n => archive/2026-08-13-add-ko-complete-dashboard-i18n}/proposal.md (100%) rename openspec/changes/{add-ko-complete-dashboard-i18n => archive/2026-08-13-add-ko-complete-dashboard-i18n}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{add-ko-complete-dashboard-i18n => archive/2026-08-13-add-ko-complete-dashboard-i18n}/tasks.md (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/.openspec.yaml (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/context.md (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/design.md (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/proposal.md (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/specs/realtime-api-compat/spec.md (100%) rename openspec/changes/{add-realtime-live-sideband => archive/2026-08-13-add-realtime-live-sideband}/tasks.md (100%) rename openspec/changes/{add-reset-credits-refresh-toggle => archive/2026-08-13-add-reset-credits-refresh-toggle}/proposal.md (100%) rename openspec/changes/{add-reset-credits-refresh-toggle => archive/2026-08-13-add-reset-credits-refresh-toggle}/specs/rate-limit-reset-credits/spec.md (100%) rename openspec/changes/{add-reset-credits-refresh-toggle => archive/2026-08-13-add-reset-credits-refresh-toggle}/tasks.md (100%) rename openspec/changes/{add-retention-zero-warning-presets => archive/2026-08-13-add-retention-zero-warning-presets}/.openspec.yaml (100%) rename openspec/changes/{add-retention-zero-warning-presets => archive/2026-08-13-add-retention-zero-warning-presets}/design.md (100%) rename openspec/changes/{add-retention-zero-warning-presets => archive/2026-08-13-add-retention-zero-warning-presets}/proposal.md (100%) rename openspec/changes/{add-retention-zero-warning-presets => archive/2026-08-13-add-retention-zero-warning-presets}/specs/data-retention/spec.md (100%) rename openspec/changes/{add-retention-zero-warning-presets => archive/2026-08-13-add-retention-zero-warning-presets}/tasks.md (100%) rename openspec/changes/{add-stale-anchor-metadata => archive/2026-08-13-add-stale-anchor-metadata}/.openspec.yaml (100%) rename openspec/changes/{add-stale-anchor-metadata => archive/2026-08-13-add-stale-anchor-metadata}/design.md (100%) rename openspec/changes/{add-stale-anchor-metadata => archive/2026-08-13-add-stale-anchor-metadata}/proposal.md (100%) rename openspec/changes/{add-stale-anchor-metadata => archive/2026-08-13-add-stale-anchor-metadata}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{add-stale-anchor-metadata => archive/2026-08-13-add-stale-anchor-metadata}/tasks.md (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/.openspec.yaml (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/design.md (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/proposal.md (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/specs/query-caching/spec.md (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/specs/upstream-proxy-routing/spec.md (100%) rename openspec/changes/{add-upstream-route-cache => archive/2026-08-13-add-upstream-route-cache}/tasks.md (100%) rename openspec/changes/{allow-developer-interleaved-fresh-resend => archive/2026-08-13-allow-developer-interleaved-fresh-resend}/.openspec.yaml (100%) rename openspec/changes/{allow-developer-interleaved-fresh-resend => archive/2026-08-13-allow-developer-interleaved-fresh-resend}/context.md (100%) rename openspec/changes/{allow-developer-interleaved-fresh-resend => archive/2026-08-13-allow-developer-interleaved-fresh-resend}/proposal.md (100%) rename openspec/changes/{allow-developer-interleaved-fresh-resend => archive/2026-08-13-allow-developer-interleaved-fresh-resend}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{allow-developer-interleaved-fresh-resend => archive/2026-08-13-allow-developer-interleaved-fresh-resend}/tasks.md (100%) rename openspec/changes/{attribute-bridge-failure-request-logs => archive/2026-08-13-attribute-bridge-failure-request-logs}/proposal.md (100%) rename openspec/changes/{attribute-bridge-failure-request-logs => archive/2026-08-13-attribute-bridge-failure-request-logs}/specs/api-keys/spec.md (100%) rename openspec/changes/{attribute-bridge-failure-request-logs => archive/2026-08-13-attribute-bridge-failure-request-logs}/tasks.md (100%) rename openspec/changes/{backoff-codex-review-usage-limits => archive/2026-08-13-backoff-codex-review-usage-limits}/proposal.md (100%) rename openspec/changes/{backoff-codex-review-usage-limits => archive/2026-08-13-backoff-codex-review-usage-limits}/specs/github-automation/spec.md (100%) rename openspec/changes/{backoff-codex-review-usage-limits => archive/2026-08-13-backoff-codex-review-usage-limits}/tasks.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/.openspec.yaml (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/design.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/proposal.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/specs/account-import/spec.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/specs/audio-transcriptions-compat/spec.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/specs/http-ingress-limits/spec.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/specs/images-api-compat/spec.md (100%) rename openspec/changes/{bound-multipart-uploads => archive/2026-08-13-bound-multipart-uploads}/tasks.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/.openspec.yaml (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/context.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/design.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/proposal.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/specs/account-routing/spec.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{bound-rate-limit-reset-metadata => archive/2026-08-13-bound-rate-limit-reset-metadata}/tasks.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/.openspec.yaml (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/context.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/design.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/proposal.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/specs/http-ingress-limits/spec.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{bound-raw-http-ingress => archive/2026-08-13-bound-raw-http-ingress}/tasks.md (100%) rename openspec/changes/{cache-request-log-count => archive/2026-08-13-cache-request-log-count}/.openspec.yaml (100%) rename openspec/changes/{cache-request-log-count => archive/2026-08-13-cache-request-log-count}/proposal.md (100%) rename openspec/changes/{cache-request-log-count => archive/2026-08-13-cache-request-log-count}/specs/query-caching/spec.md (100%) rename openspec/changes/{cache-request-log-count => archive/2026-08-13-cache-request-log-count}/tasks.md (100%) rename openspec/changes/{classify-tool-search-missing-tool-output => archive/2026-08-13-classify-tool-search-missing-tool-output}/proposal.md (100%) rename openspec/changes/{classify-tool-search-missing-tool-output => archive/2026-08-13-classify-tool-search-missing-tool-output}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{classify-tool-search-missing-tool-output => archive/2026-08-13-classify-tool-search-missing-tool-output}/tasks.md (100%) rename openspec/changes/{close-sqlite-file-handles => archive/2026-08-13-close-sqlite-file-handles}/.openspec.yaml (100%) rename openspec/changes/{close-sqlite-file-handles => archive/2026-08-13-close-sqlite-file-handles}/design.md (100%) rename openspec/changes/{close-sqlite-file-handles => archive/2026-08-13-close-sqlite-file-handles}/proposal.md (100%) rename openspec/changes/{close-sqlite-file-handles => archive/2026-08-13-close-sqlite-file-handles}/specs/database-migrations/spec.md (100%) rename openspec/changes/{close-sqlite-file-handles => archive/2026-08-13-close-sqlite-file-handles}/tasks.md (100%) rename openspec/changes/{complete-zh-cn-dashboard-i18n => archive/2026-08-13-complete-zh-cn-dashboard-i18n}/context.md (100%) rename openspec/changes/{complete-zh-cn-dashboard-i18n => archive/2026-08-13-complete-zh-cn-dashboard-i18n}/proposal.md (100%) rename openspec/changes/{complete-zh-cn-dashboard-i18n => archive/2026-08-13-complete-zh-cn-dashboard-i18n}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{complete-zh-cn-dashboard-i18n => archive/2026-08-13-complete-zh-cn-dashboard-i18n}/tasks.md (100%) rename openspec/changes/{configure-gateway-api-rules => archive/2026-08-13-configure-gateway-api-rules}/.openspec.yaml (100%) rename openspec/changes/{configure-gateway-api-rules => archive/2026-08-13-configure-gateway-api-rules}/proposal.md (100%) rename openspec/changes/{configure-gateway-api-rules => archive/2026-08-13-configure-gateway-api-rules}/specs/deployment-networking/spec.md (100%) rename openspec/changes/{configure-gateway-api-rules => archive/2026-08-13-configure-gateway-api-rules}/tasks.md (100%) rename openspec/changes/{configure-grafana-dashboard-titles => archive/2026-08-13-configure-grafana-dashboard-titles}/.openspec.yaml (100%) rename openspec/changes/{configure-grafana-dashboard-titles => archive/2026-08-13-configure-grafana-dashboard-titles}/proposal.md (100%) rename openspec/changes/{configure-grafana-dashboard-titles => archive/2026-08-13-configure-grafana-dashboard-titles}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{configure-grafana-dashboard-titles => archive/2026-08-13-configure-grafana-dashboard-titles}/tasks.md (100%) rename openspec/changes/{conversation-list-metrics => archive/2026-08-13-conversation-list-metrics}/design.md (100%) rename openspec/changes/{conversation-list-metrics => archive/2026-08-13-conversation-list-metrics}/proposal.md (100%) rename openspec/changes/{conversation-list-metrics => archive/2026-08-13-conversation-list-metrics}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{conversation-list-metrics => archive/2026-08-13-conversation-list-metrics}/tasks.md (100%) rename openspec/changes/{create-application-gateway => archive/2026-08-13-create-application-gateway}/.openspec.yaml (100%) rename openspec/changes/{create-application-gateway => archive/2026-08-13-create-application-gateway}/proposal.md (100%) rename openspec/changes/{create-application-gateway => archive/2026-08-13-create-application-gateway}/specs/deployment-networking/spec.md (100%) rename openspec/changes/{create-application-gateway => archive/2026-08-13-create-application-gateway}/tasks.md (100%) rename openspec/changes/{customize-external-secret-refs => archive/2026-08-13-customize-external-secret-refs}/proposal.md (100%) rename openspec/changes/{customize-external-secret-refs => archive/2026-08-13-customize-external-secret-refs}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{customize-external-secret-refs => archive/2026-08-13-customize-external-secret-refs}/tasks.md (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/.openspec.yaml (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/design.md (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/proposal.md (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/specs/date-display-format/spec.md (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{date-display-format-setting => archive/2026-08-13-date-display-format-setting}/tasks.md (100%) rename openspec/changes/{dedup-and-cap-response-create-dumps => archive/2026-08-13-dedup-and-cap-response-create-dumps}/.openspec.yaml (100%) rename openspec/changes/{dedup-and-cap-response-create-dumps => archive/2026-08-13-dedup-and-cap-response-create-dumps}/design.md (100%) rename openspec/changes/{dedup-and-cap-response-create-dumps => archive/2026-08-13-dedup-and-cap-response-create-dumps}/proposal.md (100%) rename openspec/changes/{dedup-and-cap-response-create-dumps => archive/2026-08-13-dedup-and-cap-response-create-dumps}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{dedup-and-cap-response-create-dumps => archive/2026-08-13-dedup-and-cap-response-create-dumps}/tasks.md (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/.openspec.yaml (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/design.md (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/proposal.md (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/specs/graceful-shutdown/spec.md (100%) rename openspec/changes/{drain-active-websocket-turns => archive/2026-08-13-drain-active-websocket-turns}/tasks.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/.openspec.yaml (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/context.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/design.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/proposal.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/specs/audit-logging/spec.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/specs/fleet-summary/spec.md (100%) rename openspec/changes/{drain-audit-fleet-tasks => archive/2026-08-13-drain-audit-fleet-tasks}/tasks.md (100%) rename openspec/changes/{durable-http-bridge-operation-recovery => archive/2026-08-13-durable-http-bridge-operation-recovery}/.openspec.yaml (100%) rename openspec/changes/{durable-http-bridge-operation-recovery => archive/2026-08-13-durable-http-bridge-operation-recovery}/context.md (100%) rename openspec/changes/{durable-http-bridge-operation-recovery => archive/2026-08-13-durable-http-bridge-operation-recovery}/proposal.md (100%) rename openspec/changes/{durable-http-bridge-operation-recovery => archive/2026-08-13-durable-http-bridge-operation-recovery}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{durable-http-bridge-operation-recovery => archive/2026-08-13-durable-http-bridge-operation-recovery}/tasks.md (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/.openspec.yaml (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/design.md (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/proposal.md (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/specs/model-catalog-compat/spec.md (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{enforced-service-tier-model-fallback => archive/2026-08-13-enforced-service-tier-model-fallback}/tasks.md (100%) rename openspec/changes/{expose-fleet-usage-refresh-timestamp => archive/2026-08-13-expose-fleet-usage-refresh-timestamp}/.openspec.yaml (100%) rename openspec/changes/{expose-fleet-usage-refresh-timestamp => archive/2026-08-13-expose-fleet-usage-refresh-timestamp}/design.md (100%) rename openspec/changes/{expose-fleet-usage-refresh-timestamp => archive/2026-08-13-expose-fleet-usage-refresh-timestamp}/proposal.md (100%) rename openspec/changes/{expose-fleet-usage-refresh-timestamp => archive/2026-08-13-expose-fleet-usage-refresh-timestamp}/specs/fleet-summary/spec.md (100%) rename openspec/changes/{expose-fleet-usage-refresh-timestamp => archive/2026-08-13-expose-fleet-usage-refresh-timestamp}/tasks.md (100%) rename openspec/changes/{extend-websocket-stream-budget => archive/2026-08-13-extend-websocket-stream-budget}/proposal.md (100%) rename openspec/changes/{extend-websocket-stream-budget => archive/2026-08-13-extend-websocket-stream-budget}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{extend-websocket-stream-budget => archive/2026-08-13-extend-websocket-stream-budget}/tasks.md (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/.openspec.yaml (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/context.md (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/design.md (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/proposal.md (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{fix-auth-guardian-detached-candidates => archive/2026-08-13-fix-auth-guardian-detached-candidates}/tasks.md (100%) rename openspec/changes/{fix-codex-catalog-required-fields => archive/2026-08-13-fix-codex-catalog-required-fields}/context.md (100%) rename openspec/changes/{fix-codex-catalog-required-fields => archive/2026-08-13-fix-codex-catalog-required-fields}/proposal.md (100%) rename openspec/changes/{fix-codex-catalog-required-fields => archive/2026-08-13-fix-codex-catalog-required-fields}/specs/model-catalog-compat/spec.md (100%) rename openspec/changes/{fix-codex-catalog-required-fields => archive/2026-08-13-fix-codex-catalog-required-fields}/tasks.md (100%) rename openspec/changes/{fix-dashboard-error-rate-cancelled => archive/2026-08-13-fix-dashboard-error-rate-cancelled}/proposal.md (100%) rename openspec/changes/{fix-dashboard-error-rate-cancelled => archive/2026-08-13-fix-dashboard-error-rate-cancelled}/specs/usage-error-metrics/spec.md (100%) rename openspec/changes/{fix-dashboard-error-rate-cancelled => archive/2026-08-13-fix-dashboard-error-rate-cancelled}/tasks.md (100%) rename openspec/changes/{fix-gpt-5-6-pricing => archive/2026-08-13-fix-gpt-5-6-pricing}/.openspec.yaml (100%) rename openspec/changes/{fix-gpt-5-6-pricing => archive/2026-08-13-fix-gpt-5-6-pricing}/design.md (100%) rename openspec/changes/{fix-gpt-5-6-pricing => archive/2026-08-13-fix-gpt-5-6-pricing}/proposal.md (100%) rename openspec/changes/{fix-gpt-5-6-pricing => archive/2026-08-13-fix-gpt-5-6-pricing}/specs/api-keys/spec.md (100%) rename openspec/changes/{fix-gpt-5-6-pricing => archive/2026-08-13-fix-gpt-5-6-pricing}/tasks.md (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/.openspec.yaml (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/design.md (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/proposal.md (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/specs/database-backends/spec.md (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{fix-postgres-pool-budget => archive/2026-08-13-fix-postgres-pool-budget}/tasks.md (100%) rename openspec/changes/{fix-promql-5xx-error-rate => archive/2026-08-13-fix-promql-5xx-error-rate}/.openspec.yaml (100%) rename openspec/changes/{fix-promql-5xx-error-rate => archive/2026-08-13-fix-promql-5xx-error-rate}/design.md (100%) rename openspec/changes/{fix-promql-5xx-error-rate => archive/2026-08-13-fix-promql-5xx-error-rate}/proposal.md (100%) rename openspec/changes/{fix-promql-5xx-error-rate => archive/2026-08-13-fix-promql-5xx-error-rate}/specs/proxy-runtime-observability/spec.md (100%) rename openspec/changes/{fix-promql-5xx-error-rate => archive/2026-08-13-fix-promql-5xx-error-rate}/tasks.md (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/.openspec.yaml (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/context.md (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/design.md (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/proposal.md (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{fix-replayed-namespaced-function-call => archive/2026-08-13-fix-replayed-namespaced-function-call}/tasks.md (100%) rename openspec/changes/{fix-reports-local-day-averages => archive/2026-08-13-fix-reports-local-day-averages}/.openspec.yaml (100%) rename openspec/changes/{fix-reports-local-day-averages => archive/2026-08-13-fix-reports-local-day-averages}/design.md (100%) rename openspec/changes/{fix-reports-local-day-averages => archive/2026-08-13-fix-reports-local-day-averages}/proposal.md (100%) rename openspec/changes/{fix-reports-local-day-averages => archive/2026-08-13-fix-reports-local-day-averages}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{fix-reports-local-day-averages => archive/2026-08-13-fix-reports-local-day-averages}/tasks.md (100%) rename openspec/changes/{fix-spark-quota-routing => archive/2026-08-13-fix-spark-quota-routing}/.openspec.yaml (100%) rename openspec/changes/{fix-spark-quota-routing => archive/2026-08-13-fix-spark-quota-routing}/design.md (100%) rename openspec/changes/{fix-spark-quota-routing => archive/2026-08-13-fix-spark-quota-routing}/proposal.md (100%) rename openspec/changes/{fix-spark-quota-routing => archive/2026-08-13-fix-spark-quota-routing}/specs/model-catalog-compat/spec.md (100%) rename openspec/changes/{fix-spark-quota-routing => archive/2026-08-13-fix-spark-quota-routing}/tasks.md (100%) rename openspec/changes/{fix-useragent-migration-unicode-whitespace => archive/2026-08-13-fix-useragent-migration-unicode-whitespace}/design.md (100%) rename openspec/changes/{fix-useragent-migration-unicode-whitespace => archive/2026-08-13-fix-useragent-migration-unicode-whitespace}/proposal.md (100%) rename openspec/changes/{fix-useragent-migration-unicode-whitespace => archive/2026-08-13-fix-useragent-migration-unicode-whitespace}/specs/proxy-runtime-observability/spec.md (100%) rename openspec/changes/{fix-useragent-migration-unicode-whitespace => archive/2026-08-13-fix-useragent-migration-unicode-whitespace}/tasks.md (100%) rename openspec/changes/{fix-warm-now-reset-utc-gate => archive/2026-08-13-fix-warm-now-reset-utc-gate}/.openspec.yaml (100%) rename openspec/changes/{fix-warm-now-reset-utc-gate => archive/2026-08-13-fix-warm-now-reset-utc-gate}/proposal.md (100%) rename openspec/changes/{fix-warm-now-reset-utc-gate => archive/2026-08-13-fix-warm-now-reset-utc-gate}/specs/quota-phase-planner/spec.md (100%) rename openspec/changes/{fix-warm-now-reset-utc-gate => archive/2026-08-13-fix-warm-now-reset-utc-gate}/tasks.md (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/.openspec.yaml (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/context.md (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/design.md (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/proposal.md (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{fix-weekly-primary-placeholder-race => archive/2026-08-13-fix-weekly-primary-placeholder-race}/tasks.md (100%) rename openspec/changes/{forward-codex-alpha-search => archive/2026-08-13-forward-codex-alpha-search}/.openspec.yaml (100%) rename openspec/changes/{forward-codex-alpha-search => archive/2026-08-13-forward-codex-alpha-search}/proposal.md (100%) rename openspec/changes/{forward-codex-alpha-search => archive/2026-08-13-forward-codex-alpha-search}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{forward-codex-alpha-search => archive/2026-08-13-forward-codex-alpha-search}/tasks.md (100%) rename openspec/changes/{header-brand-navigate-to-dashboard => archive/2026-08-13-header-brand-navigate-to-dashboard}/.openspec.yaml (100%) rename openspec/changes/{header-brand-navigate-to-dashboard => archive/2026-08-13-header-brand-navigate-to-dashboard}/proposal.md (100%) rename openspec/changes/{header-brand-navigate-to-dashboard => archive/2026-08-13-header-brand-navigate-to-dashboard}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{header-brand-navigate-to-dashboard => archive/2026-08-13-header-brand-navigate-to-dashboard}/tasks.md (100%) rename openspec/changes/{keep-request-shape-rejections-account-neutral => archive/2026-08-13-keep-request-shape-rejections-account-neutral}/proposal.md (100%) rename openspec/changes/{keep-request-shape-rejections-account-neutral => archive/2026-08-13-keep-request-shape-rejections-account-neutral}/specs/account-routing/spec.md (100%) rename openspec/changes/{keep-request-shape-rejections-account-neutral => archive/2026-08-13-keep-request-shape-rejections-account-neutral}/tasks.md (100%) rename openspec/changes/{persist-usage-snapshot-transactionally => archive/2026-08-13-persist-usage-snapshot-transactionally}/.openspec.yaml (100%) rename openspec/changes/{persist-usage-snapshot-transactionally => archive/2026-08-13-persist-usage-snapshot-transactionally}/design.md (100%) rename openspec/changes/{persist-usage-snapshot-transactionally => archive/2026-08-13-persist-usage-snapshot-transactionally}/proposal.md (100%) rename openspec/changes/{persist-usage-snapshot-transactionally => archive/2026-08-13-persist-usage-snapshot-transactionally}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{persist-usage-snapshot-transactionally => archive/2026-08-13-persist-usage-snapshot-transactionally}/tasks.md (100%) rename openspec/changes/{pin-asyncpg-session-timezone-utc => archive/2026-08-13-pin-asyncpg-session-timezone-utc}/context.md (100%) rename openspec/changes/{pin-asyncpg-session-timezone-utc => archive/2026-08-13-pin-asyncpg-session-timezone-utc}/proposal.md (100%) rename openspec/changes/{pin-asyncpg-session-timezone-utc => archive/2026-08-13-pin-asyncpg-session-timezone-utc}/specs/database-backends/spec.md (100%) rename openspec/changes/{pin-asyncpg-session-timezone-utc => archive/2026-08-13-pin-asyncpg-session-timezone-utc}/tasks.md (100%) rename openspec/changes/{preserve-dashboard-overview-on-log-failure => archive/2026-08-13-preserve-dashboard-overview-on-log-failure}/.openspec.yaml (100%) rename openspec/changes/{preserve-dashboard-overview-on-log-failure => archive/2026-08-13-preserve-dashboard-overview-on-log-failure}/design.md (100%) rename openspec/changes/{preserve-dashboard-overview-on-log-failure => archive/2026-08-13-preserve-dashboard-overview-on-log-failure}/proposal.md (100%) rename openspec/changes/{preserve-dashboard-overview-on-log-failure => archive/2026-08-13-preserve-dashboard-overview-on-log-failure}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{preserve-dashboard-overview-on-log-failure => archive/2026-08-13-preserve-dashboard-overview-on-log-failure}/tasks.md (100%) rename openspec/changes/{preserve-historical-compact-side-effects => archive/2026-08-13-preserve-historical-compact-side-effects}/proposal.md (100%) rename openspec/changes/{preserve-historical-compact-side-effects => archive/2026-08-13-preserve-historical-compact-side-effects}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{preserve-historical-compact-side-effects => archive/2026-08-13-preserve-historical-compact-side-effects}/tasks.md (100%) rename openspec/changes/{preserve-http-bridge-terminal-delivery => archive/2026-08-13-preserve-http-bridge-terminal-delivery}/.openspec.yaml (100%) rename openspec/changes/{preserve-http-bridge-terminal-delivery => archive/2026-08-13-preserve-http-bridge-terminal-delivery}/proposal.md (100%) rename openspec/changes/{preserve-http-bridge-terminal-delivery => archive/2026-08-13-preserve-http-bridge-terminal-delivery}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{preserve-http-bridge-terminal-delivery => archive/2026-08-13-preserve-http-bridge-terminal-delivery}/tasks.md (100%) rename openspec/changes/{prevent-http-bridge-model-transition-loop => archive/2026-08-13-prevent-http-bridge-model-transition-loop}/.openspec.yaml (100%) rename openspec/changes/{prevent-http-bridge-model-transition-loop => archive/2026-08-13-prevent-http-bridge-model-transition-loop}/proposal.md (100%) rename openspec/changes/{prevent-http-bridge-model-transition-loop => archive/2026-08-13-prevent-http-bridge-model-transition-loop}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{prevent-http-bridge-model-transition-loop => archive/2026-08-13-prevent-http-bridge-model-transition-loop}/tasks.md (100%) rename openspec/changes/{propagate-forwarded-compact-settlement-failure => archive/2026-08-13-propagate-forwarded-compact-settlement-failure}/.openspec.yaml (100%) rename openspec/changes/{propagate-forwarded-compact-settlement-failure => archive/2026-08-13-propagate-forwarded-compact-settlement-failure}/design.md (100%) rename openspec/changes/{propagate-forwarded-compact-settlement-failure => archive/2026-08-13-propagate-forwarded-compact-settlement-failure}/proposal.md (100%) rename openspec/changes/{propagate-forwarded-compact-settlement-failure => archive/2026-08-13-propagate-forwarded-compact-settlement-failure}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{propagate-forwarded-compact-settlement-failure => archive/2026-08-13-propagate-forwarded-compact-settlement-failure}/tasks.md (100%) rename openspec/changes/{purge-stale-bridge-sessions-on-startup => archive/2026-08-13-purge-stale-bridge-sessions-on-startup}/.openspec.yaml (100%) rename openspec/changes/{purge-stale-bridge-sessions-on-startup => archive/2026-08-13-purge-stale-bridge-sessions-on-startup}/design.md (100%) rename openspec/changes/{purge-stale-bridge-sessions-on-startup => archive/2026-08-13-purge-stale-bridge-sessions-on-startup}/proposal.md (100%) rename openspec/changes/{purge-stale-bridge-sessions-on-startup => archive/2026-08-13-purge-stale-bridge-sessions-on-startup}/specs/sticky-session-operations/spec.md (100%) rename openspec/changes/{purge-stale-bridge-sessions-on-startup => archive/2026-08-13-purge-stale-bridge-sessions-on-startup}/tasks.md (100%) rename openspec/changes/{quarantine-silent-bridge-sessions => archive/2026-08-13-quarantine-silent-bridge-sessions}/.openspec.yaml (100%) rename openspec/changes/{quarantine-silent-bridge-sessions => archive/2026-08-13-quarantine-silent-bridge-sessions}/proposal.md (100%) rename openspec/changes/{quarantine-silent-bridge-sessions => archive/2026-08-13-quarantine-silent-bridge-sessions}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{quarantine-silent-bridge-sessions => archive/2026-08-13-quarantine-silent-bridge-sessions}/tasks.md (100%) rename openspec/changes/{record-early-downstream-cancellations => archive/2026-08-13-record-early-downstream-cancellations}/.openspec.yaml (100%) rename openspec/changes/{record-early-downstream-cancellations => archive/2026-08-13-record-early-downstream-cancellations}/proposal.md (100%) rename openspec/changes/{record-early-downstream-cancellations => archive/2026-08-13-record-early-downstream-cancellations}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{record-early-downstream-cancellations => archive/2026-08-13-record-early-downstream-cancellations}/tasks.md (100%) rename openspec/changes/{recover-safe-http-bridge-continuations => archive/2026-08-13-recover-safe-http-bridge-continuations}/.openspec.yaml (100%) rename openspec/changes/{recover-safe-http-bridge-continuations => archive/2026-08-13-recover-safe-http-bridge-continuations}/proposal.md (100%) rename openspec/changes/{recover-safe-http-bridge-continuations => archive/2026-08-13-recover-safe-http-bridge-continuations}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{recover-safe-http-bridge-continuations => archive/2026-08-13-recover-safe-http-bridge-continuations}/tasks.md (100%) rename openspec/changes/{refresh-selected-account-usage => archive/2026-08-13-refresh-selected-account-usage}/.openspec.yaml (100%) rename openspec/changes/{refresh-selected-account-usage => archive/2026-08-13-refresh-selected-account-usage}/design.md (100%) rename openspec/changes/{refresh-selected-account-usage => archive/2026-08-13-refresh-selected-account-usage}/proposal.md (100%) rename openspec/changes/{refresh-selected-account-usage => archive/2026-08-13-refresh-selected-account-usage}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{refresh-selected-account-usage => archive/2026-08-13-refresh-selected-account-usage}/tasks.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/.openspec.yaml (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/design.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/proposal.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/specs/admin-auth/spec.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/specs/api-keys/spec.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/specs/deployment-installation/spec.md (100%) rename openspec/changes/{reject-conflicting-proxy-identity-headers => archive/2026-08-13-reject-conflicting-proxy-identity-headers}/tasks.md (100%) rename openspec/changes/{reject-duplicate-api-key-limit-rules => archive/2026-08-13-reject-duplicate-api-key-limit-rules}/proposal.md (100%) rename openspec/changes/{reject-duplicate-api-key-limit-rules => archive/2026-08-13-reject-duplicate-api-key-limit-rules}/specs/api-keys/spec.md (100%) rename openspec/changes/{reject-duplicate-api-key-limit-rules => archive/2026-08-13-reject-duplicate-api-key-limit-rules}/tasks.md (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/.openspec.yaml (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/context.md (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/design.md (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/proposal.md (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/specs/database-migrations/spec.md (100%) rename openspec/changes/{reject-empty-migration-db-url => archive/2026-08-13-reject-empty-migration-db-url}/tasks.md (100%) rename openspec/changes/{reject-inverted-report-date-ranges => archive/2026-08-13-reject-inverted-report-date-ranges}/.openspec.yaml (100%) rename openspec/changes/{reject-inverted-report-date-ranges => archive/2026-08-13-reject-inverted-report-date-ranges}/design.md (100%) rename openspec/changes/{reject-inverted-report-date-ranges => archive/2026-08-13-reject-inverted-report-date-ranges}/proposal.md (100%) rename openspec/changes/{reject-inverted-report-date-ranges => archive/2026-08-13-reject-inverted-report-date-ranges}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{reject-inverted-report-date-ranges => archive/2026-08-13-reject-inverted-report-date-ranges}/tasks.md (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/.openspec.yaml (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/context.md (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/design.md (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/proposal.md (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/specs/runtime-portability/spec.md (100%) rename openspec/changes/{reject-out-of-range-server-port => archive/2026-08-13-reject-out-of-range-server-port}/tasks.md (100%) rename openspec/changes/{release-idle-bridge-stream-leases => archive/2026-08-13-release-idle-bridge-stream-leases}/proposal.md (100%) rename openspec/changes/{release-idle-bridge-stream-leases => archive/2026-08-13-release-idle-bridge-stream-leases}/specs/proxy-admission-control/spec.md (100%) rename openspec/changes/{release-idle-bridge-stream-leases => archive/2026-08-13-release-idle-bridge-stream-leases}/tasks.md (100%) rename openspec/changes/{release-models-list-reservation => archive/2026-08-13-release-models-list-reservation}/proposal.md (100%) rename openspec/changes/{release-models-list-reservation => archive/2026-08-13-release-models-list-reservation}/specs/model-catalog-compat/spec.md (100%) rename openspec/changes/{release-models-list-reservation => archive/2026-08-13-release-models-list-reservation}/tasks.md (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/.openspec.yaml (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/context.md (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/design.md (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/proposal.md (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/specs/api-keys/spec.md (100%) rename openspec/changes/{release-quota-reservations-on-header-failure => archive/2026-08-13-release-quota-reservations-on-header-failure}/tasks.md (100%) rename openspec/changes/{report-pool-usage-exhaustion => archive/2026-08-13-report-pool-usage-exhaustion}/.openspec.yaml (100%) rename openspec/changes/{report-pool-usage-exhaustion => archive/2026-08-13-report-pool-usage-exhaustion}/proposal.md (100%) rename openspec/changes/{report-pool-usage-exhaustion => archive/2026-08-13-report-pool-usage-exhaustion}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{report-pool-usage-exhaustion => archive/2026-08-13-report-pool-usage-exhaustion}/tasks.md (100%) rename openspec/changes/{require-beta-soak-before-stable => archive/2026-08-13-require-beta-soak-before-stable}/proposal.md (100%) rename openspec/changes/{require-beta-soak-before-stable => archive/2026-08-13-require-beta-soak-before-stable}/specs/release-management/spec.md (100%) rename openspec/changes/{require-beta-soak-before-stable => archive/2026-08-13-require-beta-soak-before-stable}/tasks.md (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/.openspec.yaml (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/context.md (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/design.md (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/proposal.md (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/specs/proxy-architecture/spec.md (100%) rename openspec/changes/{restore-proxy-architecture-ratchets => archive/2026-08-13-restore-proxy-architecture-ratchets}/tasks.md (100%) rename openspec/changes/{retry-account-proxy-connect-failures => archive/2026-08-13-retry-account-proxy-connect-failures}/design.md (100%) rename openspec/changes/{retry-account-proxy-connect-failures => archive/2026-08-13-retry-account-proxy-connect-failures}/proposal.md (100%) rename openspec/changes/{retry-account-proxy-connect-failures => archive/2026-08-13-retry-account-proxy-connect-failures}/specs/upstream-proxy-routing/spec.md (100%) rename openspec/changes/{retry-account-proxy-connect-failures => archive/2026-08-13-retry-account-proxy-connect-failures}/tasks.md (100%) rename openspec/changes/{retry-model-capacity-errors => archive/2026-08-13-retry-model-capacity-errors}/proposal.md (100%) rename openspec/changes/{retry-model-capacity-errors => archive/2026-08-13-retry-model-capacity-errors}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{retry-model-capacity-errors => archive/2026-08-13-retry-model-capacity-errors}/tasks.md (100%) rename openspec/changes/{retry-server-is-overloaded => archive/2026-08-13-retry-server-is-overloaded}/.openspec.yaml (100%) rename openspec/changes/{retry-server-is-overloaded => archive/2026-08-13-retry-server-is-overloaded}/design.md (100%) rename openspec/changes/{retry-server-is-overloaded => archive/2026-08-13-retry-server-is-overloaded}/proposal.md (100%) rename openspec/changes/{retry-server-is-overloaded => archive/2026-08-13-retry-server-is-overloaded}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{retry-server-is-overloaded => archive/2026-08-13-retry-server-is-overloaded}/tasks.md (100%) rename openspec/changes/{retry-stale-account-model-rejection => archive/2026-08-13-retry-stale-account-model-rejection}/design.md (100%) rename openspec/changes/{retry-stale-account-model-rejection => archive/2026-08-13-retry-stale-account-model-rejection}/proposal.md (100%) rename openspec/changes/{retry-stale-account-model-rejection => archive/2026-08-13-retry-stale-account-model-rejection}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{retry-stale-account-model-rejection => archive/2026-08-13-retry-stale-account-model-rejection}/tasks.md (100%) rename openspec/changes/{self-host-mono-font => archive/2026-08-13-self-host-mono-font}/.openspec.yaml (100%) rename openspec/changes/{self-host-mono-font => archive/2026-08-13-self-host-mono-font}/proposal.md (100%) rename openspec/changes/{self-host-mono-font => archive/2026-08-13-self-host-mono-font}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{self-host-mono-font => archive/2026-08-13-self-host-mono-font}/tasks.md (100%) rename openspec/changes/{separate-dashboard-credit-metrics => archive/2026-08-13-separate-dashboard-credit-metrics}/proposal.md (100%) rename openspec/changes/{separate-dashboard-credit-metrics => archive/2026-08-13-separate-dashboard-credit-metrics}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{separate-dashboard-credit-metrics => archive/2026-08-13-separate-dashboard-credit-metrics}/tasks.md (100%) rename openspec/changes/{separate-service-and-usage-health-status => archive/2026-08-13-separate-service-and-usage-health-status}/.openspec.yaml (100%) rename openspec/changes/{separate-service-and-usage-health-status => archive/2026-08-13-separate-service-and-usage-health-status}/design.md (100%) rename openspec/changes/{separate-service-and-usage-health-status => archive/2026-08-13-separate-service-and-usage-health-status}/proposal.md (100%) rename openspec/changes/{separate-service-and-usage-health-status => archive/2026-08-13-separate-service-and-usage-health-status}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{separate-service-and-usage-health-status => archive/2026-08-13-separate-service-and-usage-health-status}/tasks.md (100%) rename openspec/changes/{sequence-public-response-failures => archive/2026-08-13-sequence-public-response-failures}/.openspec.yaml (100%) rename openspec/changes/{sequence-public-response-failures => archive/2026-08-13-sequence-public-response-failures}/proposal.md (100%) rename openspec/changes/{sequence-public-response-failures => archive/2026-08-13-sequence-public-response-failures}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{sequence-public-response-failures => archive/2026-08-13-sequence-public-response-failures}/tasks.md (100%) rename openspec/changes/{sequence-websocket-health-after-settlement => archive/2026-08-13-sequence-websocket-health-after-settlement}/.openspec.yaml (100%) rename openspec/changes/{sequence-websocket-health-after-settlement => archive/2026-08-13-sequence-websocket-health-after-settlement}/design.md (100%) rename openspec/changes/{sequence-websocket-health-after-settlement => archive/2026-08-13-sequence-websocket-health-after-settlement}/proposal.md (100%) rename openspec/changes/{sequence-websocket-health-after-settlement => archive/2026-08-13-sequence-websocket-health-after-settlement}/specs/api-keys/spec.md (100%) rename openspec/changes/{sequence-websocket-health-after-settlement => archive/2026-08-13-sequence-websocket-health-after-settlement}/tasks.md (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/.openspec.yaml (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/context.md (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/design.md (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/proposal.md (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/specs/query-caching/spec.md (100%) rename openspec/changes/{serialize-rate-limit-usage-reads => archive/2026-08-13-serialize-rate-limit-usage-reads}/tasks.md (100%) rename openspec/changes/{settings-reference-page => archive/2026-08-13-settings-reference-page}/context.md (100%) rename openspec/changes/{settings-reference-page => archive/2026-08-13-settings-reference-page}/proposal.md (100%) rename openspec/changes/{settings-reference-page => archive/2026-08-13-settings-reference-page}/specs/user-documentation/spec.md (100%) rename openspec/changes/{settings-reference-page => archive/2026-08-13-settings-reference-page}/tasks.md (100%) rename openspec/changes/{settle-aborted-terminal-bookkeeping => archive/2026-08-13-settle-aborted-terminal-bookkeeping}/.openspec.yaml (100%) rename openspec/changes/{settle-aborted-terminal-bookkeeping => archive/2026-08-13-settle-aborted-terminal-bookkeeping}/proposal.md (100%) rename openspec/changes/{settle-aborted-terminal-bookkeeping => archive/2026-08-13-settle-aborted-terminal-bookkeeping}/specs/api-keys/spec.md (100%) rename openspec/changes/{settle-aborted-terminal-bookkeeping => archive/2026-08-13-settle-aborted-terminal-bookkeeping}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{settle-aborted-terminal-bookkeeping => archive/2026-08-13-settle-aborted-terminal-bookkeeping}/tasks.md (100%) rename openspec/changes/{source-upstream-timing-metrics => archive/2026-08-13-source-upstream-timing-metrics}/design.md (100%) rename openspec/changes/{source-upstream-timing-metrics => archive/2026-08-13-source-upstream-timing-metrics}/proposal.md (100%) rename openspec/changes/{source-upstream-timing-metrics => archive/2026-08-13-source-upstream-timing-metrics}/specs/proxy-runtime-observability/spec.md (100%) rename openspec/changes/{source-upstream-timing-metrics => archive/2026-08-13-source-upstream-timing-metrics}/tasks.md (100%) rename openspec/changes/{spill-unanchored-forks-on-account-cap => archive/2026-08-13-spill-unanchored-forks-on-account-cap}/.openspec.yaml (100%) rename openspec/changes/{spill-unanchored-forks-on-account-cap => archive/2026-08-13-spill-unanchored-forks-on-account-cap}/proposal.md (100%) rename openspec/changes/{spill-unanchored-forks-on-account-cap => archive/2026-08-13-spill-unanchored-forks-on-account-cap}/specs/proxy-admission-control/spec.md (100%) rename openspec/changes/{spill-unanchored-forks-on-account-cap => archive/2026-08-13-spill-unanchored-forks-on-account-cap}/tasks.md (100%) rename openspec/changes/{split-dashboard-routes => archive/2026-08-13-split-dashboard-routes}/.openspec.yaml (100%) rename openspec/changes/{split-dashboard-routes => archive/2026-08-13-split-dashboard-routes}/proposal.md (100%) rename openspec/changes/{split-dashboard-routes => archive/2026-08-13-split-dashboard-routes}/specs/frontend-architecture/spec.md (100%) rename openspec/changes/{split-dashboard-routes => archive/2026-08-13-split-dashboard-routes}/tasks.md (100%) rename openspec/changes/{thread-goal-openapi-operation-ids => archive/2026-08-13-thread-goal-openapi-operation-ids}/.openspec.yaml (100%) rename openspec/changes/{thread-goal-openapi-operation-ids => archive/2026-08-13-thread-goal-openapi-operation-ids}/design.md (100%) rename openspec/changes/{thread-goal-openapi-operation-ids => archive/2026-08-13-thread-goal-openapi-operation-ids}/proposal.md (100%) rename openspec/changes/{thread-goal-openapi-operation-ids => archive/2026-08-13-thread-goal-openapi-operation-ids}/specs/responses-api-compat/spec.md (100%) rename openspec/changes/{thread-goal-openapi-operation-ids => archive/2026-08-13-thread-goal-openapi-operation-ids}/tasks.md (100%) rename openspec/changes/{warm-free-monthly-limit-reset => archive/2026-08-13-warm-free-monthly-limit-reset}/proposal.md (100%) rename openspec/changes/{warm-free-monthly-limit-reset => archive/2026-08-13-warm-free-monthly-limit-reset}/specs/usage-refresh-policy/spec.md (100%) rename openspec/changes/{warm-free-monthly-limit-reset => archive/2026-08-13-warm-free-monthly-limit-reset}/tasks.md (100%) rename openspec/changes/{windows-sqlite-url-encoding => archive/2026-08-13-windows-sqlite-url-encoding}/.openspec.yaml (100%) rename openspec/changes/{windows-sqlite-url-encoding => archive/2026-08-13-windows-sqlite-url-encoding}/proposal.md (100%) rename openspec/changes/{windows-sqlite-url-encoding => archive/2026-08-13-windows-sqlite-url-encoding}/specs/database-backends/spec.md (100%) rename openspec/changes/{windows-sqlite-url-encoding => archive/2026-08-13-windows-sqlite-url-encoding}/specs/database-migrations/spec.md (100%) rename openspec/changes/{windows-sqlite-url-encoding => archive/2026-08-13-windows-sqlite-url-encoding}/tasks.md (100%) create mode 100644 openspec/specs/account-import/spec.md create mode 100644 openspec/specs/audit-logging/spec.md create mode 100644 openspec/specs/date-display-format/spec.md create mode 100644 openspec/specs/graceful-shutdown/spec.md create mode 100644 openspec/specs/http-ingress-limits/spec.md create mode 100644 openspec/specs/proxy-architecture/spec.md create mode 100644 openspec/specs/usage-error-metrics/spec.md diff --git a/openspec/changes/active-conversations-average/.openspec.yaml b/openspec/changes/archive/2026-08-13-active-conversations-average/.openspec.yaml similarity index 100% rename from openspec/changes/active-conversations-average/.openspec.yaml rename to openspec/changes/archive/2026-08-13-active-conversations-average/.openspec.yaml diff --git a/openspec/changes/active-conversations-average/proposal.md b/openspec/changes/archive/2026-08-13-active-conversations-average/proposal.md similarity index 100% rename from openspec/changes/active-conversations-average/proposal.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/proposal.md diff --git a/openspec/changes/active-conversations-average/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-active-conversations-average/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/active-conversations-average/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/specs/frontend-architecture/spec.md diff --git a/openspec/changes/active-conversations-average/tasks.md b/openspec/changes/archive/2026-08-13-active-conversations-average/tasks.md similarity index 100% rename from openspec/changes/active-conversations-average/tasks.md rename to openspec/changes/archive/2026-08-13-active-conversations-average/tasks.md diff --git a/openspec/changes/add-anonymous-telemetry/context.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/context.md similarity index 100% rename from openspec/changes/add-anonymous-telemetry/context.md rename to openspec/changes/archive/2026-08-13-add-anonymous-telemetry/context.md diff --git a/openspec/changes/add-anonymous-telemetry/proposal.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/proposal.md similarity index 100% rename from openspec/changes/add-anonymous-telemetry/proposal.md rename to openspec/changes/archive/2026-08-13-add-anonymous-telemetry/proposal.md diff --git a/openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/specs/telemetry/spec.md similarity index 100% rename from openspec/changes/add-anonymous-telemetry/specs/telemetry/spec.md rename to openspec/changes/archive/2026-08-13-add-anonymous-telemetry/specs/telemetry/spec.md diff --git a/openspec/changes/add-anonymous-telemetry/tasks.md b/openspec/changes/archive/2026-08-13-add-anonymous-telemetry/tasks.md similarity index 100% rename from openspec/changes/add-anonymous-telemetry/tasks.md rename to openspec/changes/archive/2026-08-13-add-anonymous-telemetry/tasks.md diff --git a/openspec/changes/add-api-key-stream-fair-share/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/.openspec.yaml similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/.openspec.yaml diff --git a/openspec/changes/add-api-key-stream-fair-share/design.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/design.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/design.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/design.md diff --git a/openspec/changes/add-api-key-stream-fair-share/proposal.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/proposal.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/proposal.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/proposal.md diff --git a/openspec/changes/add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/after-per-account-capacity.png diff --git a/openspec/changes/add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/screenshots/before-per-account-capacity.png diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/add-api-key-stream-fair-share/tasks.md b/openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/tasks.md similarity index 100% rename from openspec/changes/add-api-key-stream-fair-share/tasks.md rename to openspec/changes/archive/2026-08-13-add-api-key-stream-fair-share/tasks.md diff --git a/openspec/changes/add-capability-aware-routing/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/.openspec.yaml similarity index 100% rename from openspec/changes/add-capability-aware-routing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/.openspec.yaml diff --git a/openspec/changes/add-capability-aware-routing/design.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/design.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/design.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/design.md diff --git a/openspec/changes/add-capability-aware-routing/proposal.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/proposal.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/proposal.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/proposal.md diff --git a/openspec/changes/add-capability-aware-routing/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/account-routing/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/database-migrations/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/responses-api-compat/spec.md diff --git a/openspec/changes/add-capability-aware-routing/specs/sticky-session-operations/spec.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/sticky-session-operations/spec.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/specs/sticky-session-operations/spec.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/specs/sticky-session-operations/spec.md diff --git a/openspec/changes/add-capability-aware-routing/tasks.md b/openspec/changes/archive/2026-08-13-add-capability-aware-routing/tasks.md similarity index 100% rename from openspec/changes/add-capability-aware-routing/tasks.md rename to openspec/changes/archive/2026-08-13-add-capability-aware-routing/tasks.md diff --git a/openspec/changes/add-conversation-dashboard/design.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/design.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/design.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/design.md diff --git a/openspec/changes/add-conversation-dashboard/proposal.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/proposal.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/proposal.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/proposal.md diff --git a/openspec/changes/add-conversation-dashboard/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-conversation-dashboard/tasks.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/tasks.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/tasks.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/tasks.md diff --git a/openspec/changes/add-conversation-dashboard/verify-report.md b/openspec/changes/archive/2026-08-13-add-conversation-dashboard/verify-report.md similarity index 100% rename from openspec/changes/add-conversation-dashboard/verify-report.md rename to openspec/changes/archive/2026-08-13-add-conversation-dashboard/verify-report.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/proposal.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/proposal.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/proposal.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/proposal.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/specs/frontend-architecture/spec.md diff --git a/openspec/changes/add-ko-complete-dashboard-i18n/tasks.md b/openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/tasks.md similarity index 100% rename from openspec/changes/add-ko-complete-dashboard-i18n/tasks.md rename to openspec/changes/archive/2026-08-13-add-ko-complete-dashboard-i18n/tasks.md diff --git a/openspec/changes/add-realtime-live-sideband/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/.openspec.yaml similarity index 100% rename from openspec/changes/add-realtime-live-sideband/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/.openspec.yaml diff --git a/openspec/changes/add-realtime-live-sideband/context.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/context.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/context.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/context.md diff --git a/openspec/changes/add-realtime-live-sideband/design.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/design.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/design.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/design.md diff --git a/openspec/changes/add-realtime-live-sideband/proposal.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/proposal.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/proposal.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/proposal.md diff --git a/openspec/changes/add-realtime-live-sideband/specs/realtime-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/specs/realtime-api-compat/spec.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/specs/realtime-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/specs/realtime-api-compat/spec.md diff --git a/openspec/changes/add-realtime-live-sideband/tasks.md b/openspec/changes/archive/2026-08-13-add-realtime-live-sideband/tasks.md similarity index 100% rename from openspec/changes/add-realtime-live-sideband/tasks.md rename to openspec/changes/archive/2026-08-13-add-realtime-live-sideband/tasks.md diff --git a/openspec/changes/add-reset-credits-refresh-toggle/proposal.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/proposal.md similarity index 100% rename from openspec/changes/add-reset-credits-refresh-toggle/proposal.md rename to openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/proposal.md diff --git a/openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md similarity index 100% rename from openspec/changes/add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md rename to openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/specs/rate-limit-reset-credits/spec.md diff --git a/openspec/changes/add-reset-credits-refresh-toggle/tasks.md b/openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/tasks.md similarity index 100% rename from openspec/changes/add-reset-credits-refresh-toggle/tasks.md rename to openspec/changes/archive/2026-08-13-add-reset-credits-refresh-toggle/tasks.md diff --git a/openspec/changes/add-retention-zero-warning-presets/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/.openspec.yaml similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/.openspec.yaml diff --git a/openspec/changes/add-retention-zero-warning-presets/design.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/design.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/design.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/design.md diff --git a/openspec/changes/add-retention-zero-warning-presets/proposal.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/proposal.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/proposal.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/proposal.md diff --git a/openspec/changes/add-retention-zero-warning-presets/specs/data-retention/spec.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/specs/data-retention/spec.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/specs/data-retention/spec.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/specs/data-retention/spec.md diff --git a/openspec/changes/add-retention-zero-warning-presets/tasks.md b/openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/tasks.md similarity index 100% rename from openspec/changes/add-retention-zero-warning-presets/tasks.md rename to openspec/changes/archive/2026-08-13-add-retention-zero-warning-presets/tasks.md diff --git a/openspec/changes/add-stale-anchor-metadata/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/.openspec.yaml similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/.openspec.yaml diff --git a/openspec/changes/add-stale-anchor-metadata/design.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/design.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/design.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/design.md diff --git a/openspec/changes/add-stale-anchor-metadata/proposal.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/proposal.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/proposal.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/proposal.md diff --git a/openspec/changes/add-stale-anchor-metadata/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/specs/responses-api-compat/spec.md diff --git a/openspec/changes/add-stale-anchor-metadata/tasks.md b/openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/tasks.md similarity index 100% rename from openspec/changes/add-stale-anchor-metadata/tasks.md rename to openspec/changes/archive/2026-08-13-add-stale-anchor-metadata/tasks.md diff --git a/openspec/changes/add-upstream-route-cache/.openspec.yaml b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/.openspec.yaml similarity index 100% rename from openspec/changes/add-upstream-route-cache/.openspec.yaml rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/.openspec.yaml diff --git a/openspec/changes/add-upstream-route-cache/design.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/design.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/design.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/design.md diff --git a/openspec/changes/add-upstream-route-cache/proposal.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/proposal.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/proposal.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/proposal.md diff --git a/openspec/changes/add-upstream-route-cache/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/query-caching/spec.md diff --git a/openspec/changes/add-upstream-route-cache/specs/upstream-proxy-routing/spec.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/upstream-proxy-routing/spec.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/specs/upstream-proxy-routing/spec.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/specs/upstream-proxy-routing/spec.md diff --git a/openspec/changes/add-upstream-route-cache/tasks.md b/openspec/changes/archive/2026-08-13-add-upstream-route-cache/tasks.md similarity index 100% rename from openspec/changes/add-upstream-route-cache/tasks.md rename to openspec/changes/archive/2026-08-13-add-upstream-route-cache/tasks.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/.openspec.yaml similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/.openspec.yaml rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/.openspec.yaml diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/context.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/context.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/context.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/context.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/proposal.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/proposal.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/proposal.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/specs/responses-api-compat/spec.md diff --git a/openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md b/openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/tasks.md similarity index 100% rename from openspec/changes/allow-developer-interleaved-fresh-resend/tasks.md rename to openspec/changes/archive/2026-08-13-allow-developer-interleaved-fresh-resend/tasks.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/proposal.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/proposal.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/proposal.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/proposal.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/specs/api-keys/spec.md diff --git a/openspec/changes/attribute-bridge-failure-request-logs/tasks.md b/openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/tasks.md similarity index 100% rename from openspec/changes/attribute-bridge-failure-request-logs/tasks.md rename to openspec/changes/archive/2026-08-13-attribute-bridge-failure-request-logs/tasks.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/proposal.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/proposal.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/proposal.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/proposal.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/specs/github-automation/spec.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/specs/github-automation/spec.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/specs/github-automation/spec.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/specs/github-automation/spec.md diff --git a/openspec/changes/backoff-codex-review-usage-limits/tasks.md b/openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/tasks.md similarity index 100% rename from openspec/changes/backoff-codex-review-usage-limits/tasks.md rename to openspec/changes/archive/2026-08-13-backoff-codex-review-usage-limits/tasks.md diff --git a/openspec/changes/bound-multipart-uploads/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/.openspec.yaml similarity index 100% rename from openspec/changes/bound-multipart-uploads/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/.openspec.yaml diff --git a/openspec/changes/bound-multipart-uploads/design.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/design.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/design.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/design.md diff --git a/openspec/changes/bound-multipart-uploads/proposal.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/proposal.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/proposal.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/proposal.md diff --git a/openspec/changes/bound-multipart-uploads/specs/account-import/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/account-import/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/account-import/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/account-import/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/audio-transcriptions-compat/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/http-ingress-limits/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/http-ingress-limits/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/http-ingress-limits/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/http-ingress-limits/spec.md diff --git a/openspec/changes/bound-multipart-uploads/specs/images-api-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/images-api-compat/spec.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/specs/images-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/specs/images-api-compat/spec.md diff --git a/openspec/changes/bound-multipart-uploads/tasks.md b/openspec/changes/archive/2026-08-13-bound-multipart-uploads/tasks.md similarity index 100% rename from openspec/changes/bound-multipart-uploads/tasks.md rename to openspec/changes/archive/2026-08-13-bound-multipart-uploads/tasks.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/.openspec.yaml similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/.openspec.yaml diff --git a/openspec/changes/bound-rate-limit-reset-metadata/context.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/context.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/context.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/context.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/design.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/design.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/design.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/design.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/proposal.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/proposal.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/proposal.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/proposal.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/account-routing/spec.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/bound-rate-limit-reset-metadata/tasks.md b/openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/tasks.md similarity index 100% rename from openspec/changes/bound-rate-limit-reset-metadata/tasks.md rename to openspec/changes/archive/2026-08-13-bound-rate-limit-reset-metadata/tasks.md diff --git a/openspec/changes/bound-raw-http-ingress/.openspec.yaml b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/.openspec.yaml similarity index 100% rename from openspec/changes/bound-raw-http-ingress/.openspec.yaml rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/.openspec.yaml diff --git a/openspec/changes/bound-raw-http-ingress/context.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/context.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/context.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/context.md diff --git a/openspec/changes/bound-raw-http-ingress/design.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/design.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/design.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/design.md diff --git a/openspec/changes/bound-raw-http-ingress/proposal.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/proposal.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/proposal.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/proposal.md diff --git a/openspec/changes/bound-raw-http-ingress/specs/http-ingress-limits/spec.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/http-ingress-limits/spec.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/specs/http-ingress-limits/spec.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/http-ingress-limits/spec.md diff --git a/openspec/changes/bound-raw-http-ingress/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/specs/responses-api-compat/spec.md diff --git a/openspec/changes/bound-raw-http-ingress/tasks.md b/openspec/changes/archive/2026-08-13-bound-raw-http-ingress/tasks.md similarity index 100% rename from openspec/changes/bound-raw-http-ingress/tasks.md rename to openspec/changes/archive/2026-08-13-bound-raw-http-ingress/tasks.md diff --git a/openspec/changes/cache-request-log-count/.openspec.yaml b/openspec/changes/archive/2026-08-13-cache-request-log-count/.openspec.yaml similarity index 100% rename from openspec/changes/cache-request-log-count/.openspec.yaml rename to openspec/changes/archive/2026-08-13-cache-request-log-count/.openspec.yaml diff --git a/openspec/changes/cache-request-log-count/proposal.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/proposal.md similarity index 100% rename from openspec/changes/cache-request-log-count/proposal.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/proposal.md diff --git a/openspec/changes/cache-request-log-count/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/cache-request-log-count/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/specs/query-caching/spec.md diff --git a/openspec/changes/cache-request-log-count/tasks.md b/openspec/changes/archive/2026-08-13-cache-request-log-count/tasks.md similarity index 100% rename from openspec/changes/cache-request-log-count/tasks.md rename to openspec/changes/archive/2026-08-13-cache-request-log-count/tasks.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/proposal.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/proposal.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/proposal.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/proposal.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/specs/responses-api-compat/spec.md diff --git a/openspec/changes/classify-tool-search-missing-tool-output/tasks.md b/openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/tasks.md similarity index 100% rename from openspec/changes/classify-tool-search-missing-tool-output/tasks.md rename to openspec/changes/archive/2026-08-13-classify-tool-search-missing-tool-output/tasks.md diff --git a/openspec/changes/close-sqlite-file-handles/.openspec.yaml b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/.openspec.yaml similarity index 100% rename from openspec/changes/close-sqlite-file-handles/.openspec.yaml rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/.openspec.yaml diff --git a/openspec/changes/close-sqlite-file-handles/design.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/design.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/design.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/design.md diff --git a/openspec/changes/close-sqlite-file-handles/proposal.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/proposal.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/proposal.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/proposal.md diff --git a/openspec/changes/close-sqlite-file-handles/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/specs/database-migrations/spec.md diff --git a/openspec/changes/close-sqlite-file-handles/tasks.md b/openspec/changes/archive/2026-08-13-close-sqlite-file-handles/tasks.md similarity index 100% rename from openspec/changes/close-sqlite-file-handles/tasks.md rename to openspec/changes/archive/2026-08-13-close-sqlite-file-handles/tasks.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/context.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/context.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/context.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/context.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/proposal.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/proposal.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/proposal.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/proposal.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/specs/frontend-architecture/spec.md diff --git a/openspec/changes/complete-zh-cn-dashboard-i18n/tasks.md b/openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/tasks.md similarity index 100% rename from openspec/changes/complete-zh-cn-dashboard-i18n/tasks.md rename to openspec/changes/archive/2026-08-13-complete-zh-cn-dashboard-i18n/tasks.md diff --git a/openspec/changes/configure-gateway-api-rules/.openspec.yaml b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/.openspec.yaml similarity index 100% rename from openspec/changes/configure-gateway-api-rules/.openspec.yaml rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/.openspec.yaml diff --git a/openspec/changes/configure-gateway-api-rules/proposal.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/proposal.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/proposal.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/proposal.md diff --git a/openspec/changes/configure-gateway-api-rules/specs/deployment-networking/spec.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/specs/deployment-networking/spec.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/specs/deployment-networking/spec.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/specs/deployment-networking/spec.md diff --git a/openspec/changes/configure-gateway-api-rules/tasks.md b/openspec/changes/archive/2026-08-13-configure-gateway-api-rules/tasks.md similarity index 100% rename from openspec/changes/configure-gateway-api-rules/tasks.md rename to openspec/changes/archive/2026-08-13-configure-gateway-api-rules/tasks.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/.openspec.yaml b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/.openspec.yaml similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/.openspec.yaml rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/.openspec.yaml diff --git a/openspec/changes/configure-grafana-dashboard-titles/proposal.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/proposal.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/proposal.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/proposal.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/specs/deployment-installation/spec.md diff --git a/openspec/changes/configure-grafana-dashboard-titles/tasks.md b/openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/tasks.md similarity index 100% rename from openspec/changes/configure-grafana-dashboard-titles/tasks.md rename to openspec/changes/archive/2026-08-13-configure-grafana-dashboard-titles/tasks.md diff --git a/openspec/changes/conversation-list-metrics/design.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/design.md similarity index 100% rename from openspec/changes/conversation-list-metrics/design.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/design.md diff --git a/openspec/changes/conversation-list-metrics/proposal.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/proposal.md similarity index 100% rename from openspec/changes/conversation-list-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/proposal.md diff --git a/openspec/changes/conversation-list-metrics/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/conversation-list-metrics/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/specs/frontend-architecture/spec.md diff --git a/openspec/changes/conversation-list-metrics/tasks.md b/openspec/changes/archive/2026-08-13-conversation-list-metrics/tasks.md similarity index 100% rename from openspec/changes/conversation-list-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-conversation-list-metrics/tasks.md diff --git a/openspec/changes/create-application-gateway/.openspec.yaml b/openspec/changes/archive/2026-08-13-create-application-gateway/.openspec.yaml similarity index 100% rename from openspec/changes/create-application-gateway/.openspec.yaml rename to openspec/changes/archive/2026-08-13-create-application-gateway/.openspec.yaml diff --git a/openspec/changes/create-application-gateway/proposal.md b/openspec/changes/archive/2026-08-13-create-application-gateway/proposal.md similarity index 100% rename from openspec/changes/create-application-gateway/proposal.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/proposal.md diff --git a/openspec/changes/create-application-gateway/specs/deployment-networking/spec.md b/openspec/changes/archive/2026-08-13-create-application-gateway/specs/deployment-networking/spec.md similarity index 100% rename from openspec/changes/create-application-gateway/specs/deployment-networking/spec.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/specs/deployment-networking/spec.md diff --git a/openspec/changes/create-application-gateway/tasks.md b/openspec/changes/archive/2026-08-13-create-application-gateway/tasks.md similarity index 100% rename from openspec/changes/create-application-gateway/tasks.md rename to openspec/changes/archive/2026-08-13-create-application-gateway/tasks.md diff --git a/openspec/changes/customize-external-secret-refs/proposal.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/proposal.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/proposal.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/proposal.md diff --git a/openspec/changes/customize-external-secret-refs/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/specs/deployment-installation/spec.md diff --git a/openspec/changes/customize-external-secret-refs/tasks.md b/openspec/changes/archive/2026-08-13-customize-external-secret-refs/tasks.md similarity index 100% rename from openspec/changes/customize-external-secret-refs/tasks.md rename to openspec/changes/archive/2026-08-13-customize-external-secret-refs/tasks.md diff --git a/openspec/changes/date-display-format-setting/.openspec.yaml b/openspec/changes/archive/2026-08-13-date-display-format-setting/.openspec.yaml similarity index 100% rename from openspec/changes/date-display-format-setting/.openspec.yaml rename to openspec/changes/archive/2026-08-13-date-display-format-setting/.openspec.yaml diff --git a/openspec/changes/date-display-format-setting/design.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/design.md similarity index 100% rename from openspec/changes/date-display-format-setting/design.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/design.md diff --git a/openspec/changes/date-display-format-setting/proposal.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/proposal.md similarity index 100% rename from openspec/changes/date-display-format-setting/proposal.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/proposal.md diff --git a/openspec/changes/date-display-format-setting/specs/date-display-format/spec.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/specs/date-display-format/spec.md similarity index 100% rename from openspec/changes/date-display-format-setting/specs/date-display-format/spec.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/specs/date-display-format/spec.md diff --git a/openspec/changes/date-display-format-setting/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/date-display-format-setting/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/specs/frontend-architecture/spec.md diff --git a/openspec/changes/date-display-format-setting/tasks.md b/openspec/changes/archive/2026-08-13-date-display-format-setting/tasks.md similarity index 100% rename from openspec/changes/date-display-format-setting/tasks.md rename to openspec/changes/archive/2026-08-13-date-display-format-setting/tasks.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/.openspec.yaml b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/.openspec.yaml similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/.openspec.yaml rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/.openspec.yaml diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/design.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/design.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/design.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/design.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/proposal.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/proposal.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/proposal.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/proposal.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/specs/deployment-installation/spec.md diff --git a/openspec/changes/dedup-and-cap-response-create-dumps/tasks.md b/openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/tasks.md similarity index 100% rename from openspec/changes/dedup-and-cap-response-create-dumps/tasks.md rename to openspec/changes/archive/2026-08-13-dedup-and-cap-response-create-dumps/tasks.md diff --git a/openspec/changes/drain-active-websocket-turns/.openspec.yaml b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/.openspec.yaml similarity index 100% rename from openspec/changes/drain-active-websocket-turns/.openspec.yaml rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/.openspec.yaml diff --git a/openspec/changes/drain-active-websocket-turns/design.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/design.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/design.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/design.md diff --git a/openspec/changes/drain-active-websocket-turns/proposal.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/proposal.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/proposal.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/proposal.md diff --git a/openspec/changes/drain-active-websocket-turns/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/deployment-installation/spec.md diff --git a/openspec/changes/drain-active-websocket-turns/specs/graceful-shutdown/spec.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/graceful-shutdown/spec.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/specs/graceful-shutdown/spec.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/specs/graceful-shutdown/spec.md diff --git a/openspec/changes/drain-active-websocket-turns/tasks.md b/openspec/changes/archive/2026-08-13-drain-active-websocket-turns/tasks.md similarity index 100% rename from openspec/changes/drain-active-websocket-turns/tasks.md rename to openspec/changes/archive/2026-08-13-drain-active-websocket-turns/tasks.md diff --git a/openspec/changes/drain-audit-fleet-tasks/.openspec.yaml b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/.openspec.yaml similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/.openspec.yaml rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/.openspec.yaml diff --git a/openspec/changes/drain-audit-fleet-tasks/context.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/context.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/context.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/context.md diff --git a/openspec/changes/drain-audit-fleet-tasks/design.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/design.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/design.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/design.md diff --git a/openspec/changes/drain-audit-fleet-tasks/proposal.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/proposal.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/proposal.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/proposal.md diff --git a/openspec/changes/drain-audit-fleet-tasks/specs/audit-logging/spec.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/audit-logging/spec.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/specs/audit-logging/spec.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/audit-logging/spec.md diff --git a/openspec/changes/drain-audit-fleet-tasks/specs/fleet-summary/spec.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/fleet-summary/spec.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/specs/fleet-summary/spec.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/specs/fleet-summary/spec.md diff --git a/openspec/changes/drain-audit-fleet-tasks/tasks.md b/openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/tasks.md similarity index 100% rename from openspec/changes/drain-audit-fleet-tasks/tasks.md rename to openspec/changes/archive/2026-08-13-drain-audit-fleet-tasks/tasks.md diff --git a/openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/.openspec.yaml similarity index 100% rename from openspec/changes/durable-http-bridge-operation-recovery/.openspec.yaml rename to openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/.openspec.yaml diff --git a/openspec/changes/durable-http-bridge-operation-recovery/context.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/context.md similarity index 100% rename from openspec/changes/durable-http-bridge-operation-recovery/context.md rename to openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/context.md diff --git a/openspec/changes/durable-http-bridge-operation-recovery/proposal.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/proposal.md similarity index 100% rename from openspec/changes/durable-http-bridge-operation-recovery/proposal.md rename to openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/proposal.md diff --git a/openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/specs/responses-api-compat/spec.md diff --git a/openspec/changes/durable-http-bridge-operation-recovery/tasks.md b/openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/tasks.md similarity index 100% rename from openspec/changes/durable-http-bridge-operation-recovery/tasks.md rename to openspec/changes/archive/2026-08-13-durable-http-bridge-operation-recovery/tasks.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/.openspec.yaml b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/.openspec.yaml similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/.openspec.yaml rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/.openspec.yaml diff --git a/openspec/changes/enforced-service-tier-model-fallback/design.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/design.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/design.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/design.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/proposal.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/proposal.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/proposal.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/proposal.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/specs/responses-api-compat/spec.md diff --git a/openspec/changes/enforced-service-tier-model-fallback/tasks.md b/openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/tasks.md similarity index 100% rename from openspec/changes/enforced-service-tier-model-fallback/tasks.md rename to openspec/changes/archive/2026-08-13-enforced-service-tier-model-fallback/tasks.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/.openspec.yaml b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/.openspec.yaml similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/.openspec.yaml rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/.openspec.yaml diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/design.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/design.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/design.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/design.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/proposal.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/proposal.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/proposal.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/proposal.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/specs/fleet-summary/spec.md diff --git a/openspec/changes/expose-fleet-usage-refresh-timestamp/tasks.md b/openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/tasks.md similarity index 100% rename from openspec/changes/expose-fleet-usage-refresh-timestamp/tasks.md rename to openspec/changes/archive/2026-08-13-expose-fleet-usage-refresh-timestamp/tasks.md diff --git a/openspec/changes/extend-websocket-stream-budget/proposal.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/proposal.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/proposal.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/proposal.md diff --git a/openspec/changes/extend-websocket-stream-budget/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/specs/responses-api-compat/spec.md diff --git a/openspec/changes/extend-websocket-stream-budget/tasks.md b/openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/tasks.md similarity index 100% rename from openspec/changes/extend-websocket-stream-budget/tasks.md rename to openspec/changes/archive/2026-08-13-extend-websocket-stream-budget/tasks.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/.openspec.yaml similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/.openspec.yaml diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/context.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/context.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/context.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/context.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/design.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/design.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/design.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/design.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/proposal.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/proposal.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/proposal.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/proposal.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/fix-auth-guardian-detached-candidates/tasks.md b/openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/tasks.md similarity index 100% rename from openspec/changes/fix-auth-guardian-detached-candidates/tasks.md rename to openspec/changes/archive/2026-08-13-fix-auth-guardian-detached-candidates/tasks.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/context.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/context.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/context.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/context.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/proposal.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/proposal.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/proposal.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/proposal.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/fix-codex-catalog-required-fields/tasks.md b/openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/tasks.md similarity index 100% rename from openspec/changes/fix-codex-catalog-required-fields/tasks.md rename to openspec/changes/archive/2026-08-13-fix-codex-catalog-required-fields/tasks.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/proposal.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/proposal.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/proposal.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/proposal.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/specs/usage-error-metrics/spec.md diff --git a/openspec/changes/fix-dashboard-error-rate-cancelled/tasks.md b/openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/tasks.md similarity index 100% rename from openspec/changes/fix-dashboard-error-rate-cancelled/tasks.md rename to openspec/changes/archive/2026-08-13-fix-dashboard-error-rate-cancelled/tasks.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/.openspec.yaml similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/.openspec.yaml diff --git a/openspec/changes/fix-gpt-5-6-pricing/design.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/design.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/design.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/design.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/proposal.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/proposal.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/proposal.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/proposal.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/specs/api-keys/spec.md diff --git a/openspec/changes/fix-gpt-5-6-pricing/tasks.md b/openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/tasks.md similarity index 100% rename from openspec/changes/fix-gpt-5-6-pricing/tasks.md rename to openspec/changes/archive/2026-08-13-fix-gpt-5-6-pricing/tasks.md diff --git a/openspec/changes/fix-postgres-pool-budget/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/.openspec.yaml similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/.openspec.yaml diff --git a/openspec/changes/fix-postgres-pool-budget/design.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/design.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/design.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/design.md diff --git a/openspec/changes/fix-postgres-pool-budget/proposal.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/proposal.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/proposal.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/proposal.md diff --git a/openspec/changes/fix-postgres-pool-budget/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/database-backends/spec.md diff --git a/openspec/changes/fix-postgres-pool-budget/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/specs/deployment-installation/spec.md diff --git a/openspec/changes/fix-postgres-pool-budget/tasks.md b/openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/tasks.md similarity index 100% rename from openspec/changes/fix-postgres-pool-budget/tasks.md rename to openspec/changes/archive/2026-08-13-fix-postgres-pool-budget/tasks.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/.openspec.yaml similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/.openspec.yaml diff --git a/openspec/changes/fix-promql-5xx-error-rate/design.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/design.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/design.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/design.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/proposal.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/proposal.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/proposal.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/proposal.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/fix-promql-5xx-error-rate/tasks.md b/openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/tasks.md similarity index 100% rename from openspec/changes/fix-promql-5xx-error-rate/tasks.md rename to openspec/changes/archive/2026-08-13-fix-promql-5xx-error-rate/tasks.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/.openspec.yaml similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/.openspec.yaml diff --git a/openspec/changes/fix-replayed-namespaced-function-call/context.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/context.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/context.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/context.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/design.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/design.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/design.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/design.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/proposal.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/proposal.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/proposal.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/proposal.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/specs/responses-api-compat/spec.md diff --git a/openspec/changes/fix-replayed-namespaced-function-call/tasks.md b/openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/tasks.md similarity index 100% rename from openspec/changes/fix-replayed-namespaced-function-call/tasks.md rename to openspec/changes/archive/2026-08-13-fix-replayed-namespaced-function-call/tasks.md diff --git a/openspec/changes/fix-reports-local-day-averages/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/.openspec.yaml similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/.openspec.yaml diff --git a/openspec/changes/fix-reports-local-day-averages/design.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/design.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/design.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/design.md diff --git a/openspec/changes/fix-reports-local-day-averages/proposal.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/proposal.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/proposal.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/proposal.md diff --git a/openspec/changes/fix-reports-local-day-averages/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/specs/frontend-architecture/spec.md diff --git a/openspec/changes/fix-reports-local-day-averages/tasks.md b/openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/tasks.md similarity index 100% rename from openspec/changes/fix-reports-local-day-averages/tasks.md rename to openspec/changes/archive/2026-08-13-fix-reports-local-day-averages/tasks.md diff --git a/openspec/changes/fix-spark-quota-routing/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/.openspec.yaml similarity index 100% rename from openspec/changes/fix-spark-quota-routing/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/.openspec.yaml diff --git a/openspec/changes/fix-spark-quota-routing/design.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/design.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/design.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/design.md diff --git a/openspec/changes/fix-spark-quota-routing/proposal.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/proposal.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/proposal.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/proposal.md diff --git a/openspec/changes/fix-spark-quota-routing/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/fix-spark-quota-routing/tasks.md b/openspec/changes/archive/2026-08-13-fix-spark-quota-routing/tasks.md similarity index 100% rename from openspec/changes/fix-spark-quota-routing/tasks.md rename to openspec/changes/archive/2026-08-13-fix-spark-quota-routing/tasks.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/design.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/design.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/design.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/design.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/proposal.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/proposal.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/proposal.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/proposal.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/fix-useragent-migration-unicode-whitespace/tasks.md b/openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/tasks.md similarity index 100% rename from openspec/changes/fix-useragent-migration-unicode-whitespace/tasks.md rename to openspec/changes/archive/2026-08-13-fix-useragent-migration-unicode-whitespace/tasks.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/.openspec.yaml similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/.openspec.yaml diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/proposal.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/proposal.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/proposal.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/proposal.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/specs/quota-phase-planner/spec.md diff --git a/openspec/changes/fix-warm-now-reset-utc-gate/tasks.md b/openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/tasks.md similarity index 100% rename from openspec/changes/fix-warm-now-reset-utc-gate/tasks.md rename to openspec/changes/archive/2026-08-13-fix-warm-now-reset-utc-gate/tasks.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/.openspec.yaml b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/.openspec.yaml similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/.openspec.yaml rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/.openspec.yaml diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/context.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/context.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/context.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/context.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/design.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/design.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/design.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/design.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/proposal.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/proposal.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/proposal.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/proposal.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/fix-weekly-primary-placeholder-race/tasks.md b/openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/tasks.md similarity index 100% rename from openspec/changes/fix-weekly-primary-placeholder-race/tasks.md rename to openspec/changes/archive/2026-08-13-fix-weekly-primary-placeholder-race/tasks.md diff --git a/openspec/changes/forward-codex-alpha-search/.openspec.yaml b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/.openspec.yaml similarity index 100% rename from openspec/changes/forward-codex-alpha-search/.openspec.yaml rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/.openspec.yaml diff --git a/openspec/changes/forward-codex-alpha-search/proposal.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/proposal.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/proposal.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/proposal.md diff --git a/openspec/changes/forward-codex-alpha-search/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/specs/responses-api-compat/spec.md diff --git a/openspec/changes/forward-codex-alpha-search/tasks.md b/openspec/changes/archive/2026-08-13-forward-codex-alpha-search/tasks.md similarity index 100% rename from openspec/changes/forward-codex-alpha-search/tasks.md rename to openspec/changes/archive/2026-08-13-forward-codex-alpha-search/tasks.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/.openspec.yaml b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/.openspec.yaml similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/.openspec.yaml rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/.openspec.yaml diff --git a/openspec/changes/header-brand-navigate-to-dashboard/proposal.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/proposal.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/proposal.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/proposal.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/specs/frontend-architecture/spec.md diff --git a/openspec/changes/header-brand-navigate-to-dashboard/tasks.md b/openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/tasks.md similarity index 100% rename from openspec/changes/header-brand-navigate-to-dashboard/tasks.md rename to openspec/changes/archive/2026-08-13-header-brand-navigate-to-dashboard/tasks.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/proposal.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/proposal.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/proposal.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/proposal.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/specs/account-routing/spec.md diff --git a/openspec/changes/keep-request-shape-rejections-account-neutral/tasks.md b/openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/tasks.md similarity index 100% rename from openspec/changes/keep-request-shape-rejections-account-neutral/tasks.md rename to openspec/changes/archive/2026-08-13-keep-request-shape-rejections-account-neutral/tasks.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/.openspec.yaml b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/.openspec.yaml similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/.openspec.yaml rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/.openspec.yaml diff --git a/openspec/changes/persist-usage-snapshot-transactionally/design.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/design.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/design.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/design.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/proposal.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/proposal.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/proposal.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/proposal.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/persist-usage-snapshot-transactionally/tasks.md b/openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/tasks.md similarity index 100% rename from openspec/changes/persist-usage-snapshot-transactionally/tasks.md rename to openspec/changes/archive/2026-08-13-persist-usage-snapshot-transactionally/tasks.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/context.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/context.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/context.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/context.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/proposal.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/proposal.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/proposal.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/proposal.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/specs/database-backends/spec.md diff --git a/openspec/changes/pin-asyncpg-session-timezone-utc/tasks.md b/openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/tasks.md similarity index 100% rename from openspec/changes/pin-asyncpg-session-timezone-utc/tasks.md rename to openspec/changes/archive/2026-08-13-pin-asyncpg-session-timezone-utc/tasks.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/.openspec.yaml similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/.openspec.yaml diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/design.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/design.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/design.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/design.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/proposal.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/proposal.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/proposal.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/specs/frontend-architecture/spec.md diff --git a/openspec/changes/preserve-dashboard-overview-on-log-failure/tasks.md b/openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/tasks.md similarity index 100% rename from openspec/changes/preserve-dashboard-overview-on-log-failure/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-dashboard-overview-on-log-failure/tasks.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/proposal.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/proposal.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/proposal.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/specs/responses-api-compat/spec.md diff --git a/openspec/changes/preserve-historical-compact-side-effects/tasks.md b/openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/tasks.md similarity index 100% rename from openspec/changes/preserve-historical-compact-side-effects/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-historical-compact-side-effects/tasks.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/.openspec.yaml similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/.openspec.yaml rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/.openspec.yaml diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/proposal.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/proposal.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/proposal.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/specs/responses-api-compat/spec.md diff --git a/openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md b/openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/tasks.md similarity index 100% rename from openspec/changes/preserve-http-bridge-terminal-delivery/tasks.md rename to openspec/changes/archive/2026-08-13-preserve-http-bridge-terminal-delivery/tasks.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/.openspec.yaml b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/.openspec.yaml similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/.openspec.yaml rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/.openspec.yaml diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/proposal.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/proposal.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/proposal.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/proposal.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/specs/responses-api-compat/spec.md diff --git a/openspec/changes/prevent-http-bridge-model-transition-loop/tasks.md b/openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/tasks.md similarity index 100% rename from openspec/changes/prevent-http-bridge-model-transition-loop/tasks.md rename to openspec/changes/archive/2026-08-13-prevent-http-bridge-model-transition-loop/tasks.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/.openspec.yaml similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/.openspec.yaml diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/design.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/design.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/design.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/design.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/proposal.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/proposal.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/proposal.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/proposal.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/propagate-forwarded-compact-settlement-failure/tasks.md b/openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/tasks.md similarity index 100% rename from openspec/changes/propagate-forwarded-compact-settlement-failure/tasks.md rename to openspec/changes/archive/2026-08-13-propagate-forwarded-compact-settlement-failure/tasks.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/.openspec.yaml b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/.openspec.yaml similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/.openspec.yaml rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/.openspec.yaml diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/design.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/design.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/design.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/design.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/proposal.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/proposal.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/proposal.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/proposal.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/specs/sticky-session-operations/spec.md diff --git a/openspec/changes/purge-stale-bridge-sessions-on-startup/tasks.md b/openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/tasks.md similarity index 100% rename from openspec/changes/purge-stale-bridge-sessions-on-startup/tasks.md rename to openspec/changes/archive/2026-08-13-purge-stale-bridge-sessions-on-startup/tasks.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/.openspec.yaml b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/.openspec.yaml similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/.openspec.yaml rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/.openspec.yaml diff --git a/openspec/changes/quarantine-silent-bridge-sessions/proposal.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/proposal.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/proposal.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/proposal.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/specs/responses-api-compat/spec.md diff --git a/openspec/changes/quarantine-silent-bridge-sessions/tasks.md b/openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/tasks.md similarity index 100% rename from openspec/changes/quarantine-silent-bridge-sessions/tasks.md rename to openspec/changes/archive/2026-08-13-quarantine-silent-bridge-sessions/tasks.md diff --git a/openspec/changes/record-early-downstream-cancellations/.openspec.yaml b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/.openspec.yaml similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/.openspec.yaml rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/.openspec.yaml diff --git a/openspec/changes/record-early-downstream-cancellations/proposal.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/proposal.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/proposal.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/proposal.md diff --git a/openspec/changes/record-early-downstream-cancellations/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/specs/responses-api-compat/spec.md diff --git a/openspec/changes/record-early-downstream-cancellations/tasks.md b/openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/tasks.md similarity index 100% rename from openspec/changes/record-early-downstream-cancellations/tasks.md rename to openspec/changes/archive/2026-08-13-record-early-downstream-cancellations/tasks.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/.openspec.yaml b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/.openspec.yaml similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/.openspec.yaml rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/.openspec.yaml diff --git a/openspec/changes/recover-safe-http-bridge-continuations/proposal.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/proposal.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/proposal.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/proposal.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/specs/responses-api-compat/spec.md diff --git a/openspec/changes/recover-safe-http-bridge-continuations/tasks.md b/openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/tasks.md similarity index 100% rename from openspec/changes/recover-safe-http-bridge-continuations/tasks.md rename to openspec/changes/archive/2026-08-13-recover-safe-http-bridge-continuations/tasks.md diff --git a/openspec/changes/refresh-selected-account-usage/.openspec.yaml b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/.openspec.yaml similarity index 100% rename from openspec/changes/refresh-selected-account-usage/.openspec.yaml rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/.openspec.yaml diff --git a/openspec/changes/refresh-selected-account-usage/design.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/design.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/design.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/design.md diff --git a/openspec/changes/refresh-selected-account-usage/proposal.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/proposal.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/proposal.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/proposal.md diff --git a/openspec/changes/refresh-selected-account-usage/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/refresh-selected-account-usage/tasks.md b/openspec/changes/archive/2026-08-13-refresh-selected-account-usage/tasks.md similarity index 100% rename from openspec/changes/refresh-selected-account-usage/tasks.md rename to openspec/changes/archive/2026-08-13-refresh-selected-account-usage/tasks.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/.openspec.yaml similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/.openspec.yaml diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/design.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/design.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/design.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/design.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/proposal.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/proposal.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/proposal.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/proposal.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/admin-auth/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/api-keys/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/specs/deployment-installation/spec.md diff --git a/openspec/changes/reject-conflicting-proxy-identity-headers/tasks.md b/openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/tasks.md similarity index 100% rename from openspec/changes/reject-conflicting-proxy-identity-headers/tasks.md rename to openspec/changes/archive/2026-08-13-reject-conflicting-proxy-identity-headers/tasks.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/proposal.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/proposal.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/proposal.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/proposal.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/specs/api-keys/spec.md diff --git a/openspec/changes/reject-duplicate-api-key-limit-rules/tasks.md b/openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/tasks.md similarity index 100% rename from openspec/changes/reject-duplicate-api-key-limit-rules/tasks.md rename to openspec/changes/archive/2026-08-13-reject-duplicate-api-key-limit-rules/tasks.md diff --git a/openspec/changes/reject-empty-migration-db-url/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/.openspec.yaml similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/.openspec.yaml diff --git a/openspec/changes/reject-empty-migration-db-url/context.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/context.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/context.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/context.md diff --git a/openspec/changes/reject-empty-migration-db-url/design.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/design.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/design.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/design.md diff --git a/openspec/changes/reject-empty-migration-db-url/proposal.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/proposal.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/proposal.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/proposal.md diff --git a/openspec/changes/reject-empty-migration-db-url/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/specs/database-migrations/spec.md diff --git a/openspec/changes/reject-empty-migration-db-url/tasks.md b/openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/tasks.md similarity index 100% rename from openspec/changes/reject-empty-migration-db-url/tasks.md rename to openspec/changes/archive/2026-08-13-reject-empty-migration-db-url/tasks.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/.openspec.yaml similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/.openspec.yaml diff --git a/openspec/changes/reject-inverted-report-date-ranges/design.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/design.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/design.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/design.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/proposal.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/proposal.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/proposal.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/proposal.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/specs/frontend-architecture/spec.md diff --git a/openspec/changes/reject-inverted-report-date-ranges/tasks.md b/openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/tasks.md similarity index 100% rename from openspec/changes/reject-inverted-report-date-ranges/tasks.md rename to openspec/changes/archive/2026-08-13-reject-inverted-report-date-ranges/tasks.md diff --git a/openspec/changes/reject-out-of-range-server-port/.openspec.yaml b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/.openspec.yaml similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/.openspec.yaml rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/.openspec.yaml diff --git a/openspec/changes/reject-out-of-range-server-port/context.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/context.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/context.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/context.md diff --git a/openspec/changes/reject-out-of-range-server-port/design.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/design.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/design.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/design.md diff --git a/openspec/changes/reject-out-of-range-server-port/proposal.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/proposal.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/proposal.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/proposal.md diff --git a/openspec/changes/reject-out-of-range-server-port/specs/runtime-portability/spec.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/specs/runtime-portability/spec.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/specs/runtime-portability/spec.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/specs/runtime-portability/spec.md diff --git a/openspec/changes/reject-out-of-range-server-port/tasks.md b/openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/tasks.md similarity index 100% rename from openspec/changes/reject-out-of-range-server-port/tasks.md rename to openspec/changes/archive/2026-08-13-reject-out-of-range-server-port/tasks.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/proposal.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/proposal.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/proposal.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/proposal.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/release-idle-bridge-stream-leases/tasks.md b/openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/tasks.md similarity index 100% rename from openspec/changes/release-idle-bridge-stream-leases/tasks.md rename to openspec/changes/archive/2026-08-13-release-idle-bridge-stream-leases/tasks.md diff --git a/openspec/changes/release-models-list-reservation/proposal.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/proposal.md similarity index 100% rename from openspec/changes/release-models-list-reservation/proposal.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/proposal.md diff --git a/openspec/changes/release-models-list-reservation/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/release-models-list-reservation/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/release-models-list-reservation/tasks.md b/openspec/changes/archive/2026-08-13-release-models-list-reservation/tasks.md similarity index 100% rename from openspec/changes/release-models-list-reservation/tasks.md rename to openspec/changes/archive/2026-08-13-release-models-list-reservation/tasks.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/.openspec.yaml b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/.openspec.yaml similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/.openspec.yaml rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/.openspec.yaml diff --git a/openspec/changes/release-quota-reservations-on-header-failure/context.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/context.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/context.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/context.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/design.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/design.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/design.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/design.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/proposal.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/proposal.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/proposal.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/proposal.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/specs/api-keys/spec.md diff --git a/openspec/changes/release-quota-reservations-on-header-failure/tasks.md b/openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/tasks.md similarity index 100% rename from openspec/changes/release-quota-reservations-on-header-failure/tasks.md rename to openspec/changes/archive/2026-08-13-release-quota-reservations-on-header-failure/tasks.md diff --git a/openspec/changes/report-pool-usage-exhaustion/.openspec.yaml b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/.openspec.yaml similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/.openspec.yaml rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/.openspec.yaml diff --git a/openspec/changes/report-pool-usage-exhaustion/proposal.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/proposal.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/proposal.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/proposal.md diff --git a/openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/specs/responses-api-compat/spec.md diff --git a/openspec/changes/report-pool-usage-exhaustion/tasks.md b/openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/tasks.md similarity index 100% rename from openspec/changes/report-pool-usage-exhaustion/tasks.md rename to openspec/changes/archive/2026-08-13-report-pool-usage-exhaustion/tasks.md diff --git a/openspec/changes/require-beta-soak-before-stable/proposal.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/proposal.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/proposal.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/proposal.md diff --git a/openspec/changes/require-beta-soak-before-stable/specs/release-management/spec.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/specs/release-management/spec.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/specs/release-management/spec.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/specs/release-management/spec.md diff --git a/openspec/changes/require-beta-soak-before-stable/tasks.md b/openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/tasks.md similarity index 100% rename from openspec/changes/require-beta-soak-before-stable/tasks.md rename to openspec/changes/archive/2026-08-13-require-beta-soak-before-stable/tasks.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/.openspec.yaml b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/.openspec.yaml similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/.openspec.yaml rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/.openspec.yaml diff --git a/openspec/changes/restore-proxy-architecture-ratchets/context.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/context.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/context.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/context.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/design.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/design.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/design.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/design.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/proposal.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/proposal.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/proposal.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/proposal.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/specs/proxy-architecture/spec.md diff --git a/openspec/changes/restore-proxy-architecture-ratchets/tasks.md b/openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/tasks.md similarity index 100% rename from openspec/changes/restore-proxy-architecture-ratchets/tasks.md rename to openspec/changes/archive/2026-08-13-restore-proxy-architecture-ratchets/tasks.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/design.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/design.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/design.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/design.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/proposal.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/proposal.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/proposal.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/proposal.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/specs/upstream-proxy-routing/spec.md diff --git a/openspec/changes/retry-account-proxy-connect-failures/tasks.md b/openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/tasks.md similarity index 100% rename from openspec/changes/retry-account-proxy-connect-failures/tasks.md rename to openspec/changes/archive/2026-08-13-retry-account-proxy-connect-failures/tasks.md diff --git a/openspec/changes/retry-model-capacity-errors/proposal.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/proposal.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/proposal.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/proposal.md diff --git a/openspec/changes/retry-model-capacity-errors/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-model-capacity-errors/tasks.md b/openspec/changes/archive/2026-08-13-retry-model-capacity-errors/tasks.md similarity index 100% rename from openspec/changes/retry-model-capacity-errors/tasks.md rename to openspec/changes/archive/2026-08-13-retry-model-capacity-errors/tasks.md diff --git a/openspec/changes/retry-server-is-overloaded/.openspec.yaml b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/.openspec.yaml similarity index 100% rename from openspec/changes/retry-server-is-overloaded/.openspec.yaml rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/.openspec.yaml diff --git a/openspec/changes/retry-server-is-overloaded/design.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/design.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/design.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/design.md diff --git a/openspec/changes/retry-server-is-overloaded/proposal.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/proposal.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/proposal.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/proposal.md diff --git a/openspec/changes/retry-server-is-overloaded/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-server-is-overloaded/tasks.md b/openspec/changes/archive/2026-08-13-retry-server-is-overloaded/tasks.md similarity index 100% rename from openspec/changes/retry-server-is-overloaded/tasks.md rename to openspec/changes/archive/2026-08-13-retry-server-is-overloaded/tasks.md diff --git a/openspec/changes/retry-stale-account-model-rejection/design.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/design.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/design.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/design.md diff --git a/openspec/changes/retry-stale-account-model-rejection/proposal.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/proposal.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/proposal.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/proposal.md diff --git a/openspec/changes/retry-stale-account-model-rejection/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/specs/responses-api-compat/spec.md diff --git a/openspec/changes/retry-stale-account-model-rejection/tasks.md b/openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/tasks.md similarity index 100% rename from openspec/changes/retry-stale-account-model-rejection/tasks.md rename to openspec/changes/archive/2026-08-13-retry-stale-account-model-rejection/tasks.md diff --git a/openspec/changes/self-host-mono-font/.openspec.yaml b/openspec/changes/archive/2026-08-13-self-host-mono-font/.openspec.yaml similarity index 100% rename from openspec/changes/self-host-mono-font/.openspec.yaml rename to openspec/changes/archive/2026-08-13-self-host-mono-font/.openspec.yaml diff --git a/openspec/changes/self-host-mono-font/proposal.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/proposal.md similarity index 100% rename from openspec/changes/self-host-mono-font/proposal.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/proposal.md diff --git a/openspec/changes/self-host-mono-font/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/self-host-mono-font/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/specs/frontend-architecture/spec.md diff --git a/openspec/changes/self-host-mono-font/tasks.md b/openspec/changes/archive/2026-08-13-self-host-mono-font/tasks.md similarity index 100% rename from openspec/changes/self-host-mono-font/tasks.md rename to openspec/changes/archive/2026-08-13-self-host-mono-font/tasks.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/proposal.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/proposal.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/proposal.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/specs/frontend-architecture/spec.md diff --git a/openspec/changes/separate-dashboard-credit-metrics/tasks.md b/openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/tasks.md similarity index 100% rename from openspec/changes/separate-dashboard-credit-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-separate-dashboard-credit-metrics/tasks.md diff --git a/openspec/changes/separate-service-and-usage-health-status/.openspec.yaml b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/.openspec.yaml similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/.openspec.yaml rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/.openspec.yaml diff --git a/openspec/changes/separate-service-and-usage-health-status/design.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/design.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/design.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/design.md diff --git a/openspec/changes/separate-service-and-usage-health-status/proposal.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/proposal.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/proposal.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/proposal.md diff --git a/openspec/changes/separate-service-and-usage-health-status/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/specs/frontend-architecture/spec.md diff --git a/openspec/changes/separate-service-and-usage-health-status/tasks.md b/openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/tasks.md similarity index 100% rename from openspec/changes/separate-service-and-usage-health-status/tasks.md rename to openspec/changes/archive/2026-08-13-separate-service-and-usage-health-status/tasks.md diff --git a/openspec/changes/sequence-public-response-failures/.openspec.yaml b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/.openspec.yaml similarity index 100% rename from openspec/changes/sequence-public-response-failures/.openspec.yaml rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/.openspec.yaml diff --git a/openspec/changes/sequence-public-response-failures/proposal.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/proposal.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/proposal.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/proposal.md diff --git a/openspec/changes/sequence-public-response-failures/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/specs/responses-api-compat/spec.md diff --git a/openspec/changes/sequence-public-response-failures/tasks.md b/openspec/changes/archive/2026-08-13-sequence-public-response-failures/tasks.md similarity index 100% rename from openspec/changes/sequence-public-response-failures/tasks.md rename to openspec/changes/archive/2026-08-13-sequence-public-response-failures/tasks.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/.openspec.yaml b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/.openspec.yaml similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/.openspec.yaml rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/.openspec.yaml diff --git a/openspec/changes/sequence-websocket-health-after-settlement/design.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/design.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/design.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/design.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/proposal.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/proposal.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/proposal.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/proposal.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/specs/api-keys/spec.md diff --git a/openspec/changes/sequence-websocket-health-after-settlement/tasks.md b/openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/tasks.md similarity index 100% rename from openspec/changes/sequence-websocket-health-after-settlement/tasks.md rename to openspec/changes/archive/2026-08-13-sequence-websocket-health-after-settlement/tasks.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/.openspec.yaml b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/.openspec.yaml similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/.openspec.yaml rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/.openspec.yaml diff --git a/openspec/changes/serialize-rate-limit-usage-reads/context.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/context.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/context.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/context.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/design.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/design.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/design.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/design.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/proposal.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/proposal.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/proposal.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/proposal.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/specs/query-caching/spec.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/specs/query-caching/spec.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/specs/query-caching/spec.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/specs/query-caching/spec.md diff --git a/openspec/changes/serialize-rate-limit-usage-reads/tasks.md b/openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/tasks.md similarity index 100% rename from openspec/changes/serialize-rate-limit-usage-reads/tasks.md rename to openspec/changes/archive/2026-08-13-serialize-rate-limit-usage-reads/tasks.md diff --git a/openspec/changes/settings-reference-page/context.md b/openspec/changes/archive/2026-08-13-settings-reference-page/context.md similarity index 100% rename from openspec/changes/settings-reference-page/context.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/context.md diff --git a/openspec/changes/settings-reference-page/proposal.md b/openspec/changes/archive/2026-08-13-settings-reference-page/proposal.md similarity index 100% rename from openspec/changes/settings-reference-page/proposal.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/proposal.md diff --git a/openspec/changes/settings-reference-page/specs/user-documentation/spec.md b/openspec/changes/archive/2026-08-13-settings-reference-page/specs/user-documentation/spec.md similarity index 100% rename from openspec/changes/settings-reference-page/specs/user-documentation/spec.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/specs/user-documentation/spec.md diff --git a/openspec/changes/settings-reference-page/tasks.md b/openspec/changes/archive/2026-08-13-settings-reference-page/tasks.md similarity index 100% rename from openspec/changes/settings-reference-page/tasks.md rename to openspec/changes/archive/2026-08-13-settings-reference-page/tasks.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/.openspec.yaml b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/.openspec.yaml similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/.openspec.yaml rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/.openspec.yaml diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/proposal.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/proposal.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/proposal.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/proposal.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/api-keys/spec.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/specs/responses-api-compat/spec.md diff --git a/openspec/changes/settle-aborted-terminal-bookkeeping/tasks.md b/openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/tasks.md similarity index 100% rename from openspec/changes/settle-aborted-terminal-bookkeeping/tasks.md rename to openspec/changes/archive/2026-08-13-settle-aborted-terminal-bookkeeping/tasks.md diff --git a/openspec/changes/source-upstream-timing-metrics/design.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/design.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/design.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/design.md diff --git a/openspec/changes/source-upstream-timing-metrics/proposal.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/proposal.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/proposal.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/proposal.md diff --git a/openspec/changes/source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/specs/proxy-runtime-observability/spec.md diff --git a/openspec/changes/source-upstream-timing-metrics/tasks.md b/openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/tasks.md similarity index 100% rename from openspec/changes/source-upstream-timing-metrics/tasks.md rename to openspec/changes/archive/2026-08-13-source-upstream-timing-metrics/tasks.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/.openspec.yaml b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/.openspec.yaml similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/.openspec.yaml rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/.openspec.yaml diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/proposal.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/proposal.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/proposal.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/proposal.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/specs/proxy-admission-control/spec.md diff --git a/openspec/changes/spill-unanchored-forks-on-account-cap/tasks.md b/openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/tasks.md similarity index 100% rename from openspec/changes/spill-unanchored-forks-on-account-cap/tasks.md rename to openspec/changes/archive/2026-08-13-spill-unanchored-forks-on-account-cap/tasks.md diff --git a/openspec/changes/split-dashboard-routes/.openspec.yaml b/openspec/changes/archive/2026-08-13-split-dashboard-routes/.openspec.yaml similarity index 100% rename from openspec/changes/split-dashboard-routes/.openspec.yaml rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/.openspec.yaml diff --git a/openspec/changes/split-dashboard-routes/proposal.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/proposal.md similarity index 100% rename from openspec/changes/split-dashboard-routes/proposal.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/proposal.md diff --git a/openspec/changes/split-dashboard-routes/specs/frontend-architecture/spec.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/specs/frontend-architecture/spec.md similarity index 100% rename from openspec/changes/split-dashboard-routes/specs/frontend-architecture/spec.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/specs/frontend-architecture/spec.md diff --git a/openspec/changes/split-dashboard-routes/tasks.md b/openspec/changes/archive/2026-08-13-split-dashboard-routes/tasks.md similarity index 100% rename from openspec/changes/split-dashboard-routes/tasks.md rename to openspec/changes/archive/2026-08-13-split-dashboard-routes/tasks.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/.openspec.yaml b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/.openspec.yaml similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/.openspec.yaml rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/.openspec.yaml diff --git a/openspec/changes/thread-goal-openapi-operation-ids/design.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/design.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/design.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/design.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/proposal.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/proposal.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/proposal.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/proposal.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/specs/responses-api-compat/spec.md diff --git a/openspec/changes/thread-goal-openapi-operation-ids/tasks.md b/openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/tasks.md similarity index 100% rename from openspec/changes/thread-goal-openapi-operation-ids/tasks.md rename to openspec/changes/archive/2026-08-13-thread-goal-openapi-operation-ids/tasks.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/proposal.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/proposal.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/proposal.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/proposal.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/specs/usage-refresh-policy/spec.md diff --git a/openspec/changes/warm-free-monthly-limit-reset/tasks.md b/openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/tasks.md similarity index 100% rename from openspec/changes/warm-free-monthly-limit-reset/tasks.md rename to openspec/changes/archive/2026-08-13-warm-free-monthly-limit-reset/tasks.md diff --git a/openspec/changes/windows-sqlite-url-encoding/.openspec.yaml b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/.openspec.yaml similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/.openspec.yaml rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/.openspec.yaml diff --git a/openspec/changes/windows-sqlite-url-encoding/proposal.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/proposal.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/proposal.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/proposal.md diff --git a/openspec/changes/windows-sqlite-url-encoding/specs/database-backends/spec.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-backends/spec.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/specs/database-backends/spec.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-backends/spec.md diff --git a/openspec/changes/windows-sqlite-url-encoding/specs/database-migrations/spec.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-migrations/spec.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/specs/database-migrations/spec.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/specs/database-migrations/spec.md diff --git a/openspec/changes/windows-sqlite-url-encoding/tasks.md b/openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/tasks.md similarity index 100% rename from openspec/changes/windows-sqlite-url-encoding/tasks.md rename to openspec/changes/archive/2026-08-13-windows-sqlite-url-encoding/tasks.md diff --git a/openspec/specs/account-import/spec.md b/openspec/specs/account-import/spec.md new file mode 100644 index 0000000000..72e8cfad7c --- /dev/null +++ b/openspec/specs/account-import/spec.md @@ -0,0 +1,65 @@ +# account-import Specification + +## Purpose +Authorization and resource bounds for importing account auth material (auth.json uploads and equivalent flows) into the pool. +## Requirements +### Requirement: Account auth imports are authorized and bounded + +`POST /api/accounts/import` MUST authenticate the dashboard session and require dashboard write access before reading any request-body bytes. It MUST accept exactly one file part named `auth_json`, no text parts, a file size no greater than 1 MiB (1,048,576 bytes), and a complete multipart body no greater than 2 MiB (2,097,152 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and the actual streamed bytes. It MUST enforce the file limit before retaining bytes beyond the limit, close every multipart spool before account persistence or import-time network work begins, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for `POST /api/accounts/import`. Its exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before dashboard authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with dashboard error `code = payload_too_large`. Missing or non-file `auth_json` input MUST retain the dashboard validation envelope, while malformed multipart syntax or additional parts MUST return a dashboard-compatible HTTP 400 without invoking account import logic. + +#### Scenario: Unauthorized import does not consume the body + +- **WHEN** a request without a valid dashboard session or write permission targets account import +- **THEN** the existing authentication or permission response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created + +#### Scenario: Valid bounded auth file imports normally + +- **WHEN** an authorized operator uploads exactly one valid `auth_json` file and both file and multipart body are within their limits +- **THEN** the existing account identity, persistence, usage-refresh, cache-invalidation, and audit behavior continues +- **AND** the multipart spool is closed before persistence or network work begins + +#### Scenario: Declared or streamed account-import body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 2 MiB or actual streamed multipart bytes cross 2 MiB +- **THEN** the service returns HTTP 413 with dashboard error `code = payload_too_large` +- **AND** it does not parse credentials, mutate an account, refresh usage, invalidate caches, or write a success audit event + +#### Scenario: Auth file exceeds its limit + +- **WHEN** the `auth_json` file part exceeds 1 MiB while the multipart body is otherwise valid +- **THEN** the service returns HTTP 413 with dashboard error `code = payload_too_large` +- **AND** bytes beyond the file limit are not retained in a multipart spool or handler buffer + +#### Scenario: Account import has an invalid multipart shape + +- **WHEN** an import omits a file-valued `auth_json` part or includes duplicate, additional file, or text parts +- **THEN** the service returns the established dashboard validation or bad-request envelope +- **AND** account import logic is not invoked + +#### Scenario: Compressed account import is rejected without prebuffering + +- **GIVEN** account import has passed dashboard session and write authorization +- **WHEN** it declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with dashboard error `code = invalid_request` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 2 MiB dedicated body limit + +#### Scenario: Generic ingress does not preempt encoded account-import authorization + +- **GIVEN** an account-import request fails dashboard session or write authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication or permission response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed + +#### Scenario: Disconnect and cancellation clean up parsing + +- **WHEN** the client disconnects or request processing is cancelled during account multipart parsing +- **THEN** every created spool is closed +- **AND** the disconnect or cancellation propagates without being converted to HTTP 413 + diff --git a/openspec/specs/account-routing/spec.md b/openspec/specs/account-routing/spec.md index 7b5f73980b..3fcca9d59b 100644 --- a/openspec/specs/account-routing/spec.md +++ b/openspec/specs/account-routing/spec.md @@ -292,26 +292,44 @@ unit. When the hint contains no recognizable unit token, the system SHALL fall back to the error-count backoff schedule. A rate-limited account SHALL NOT be re-selected before its cooldown elapses. -When the upstream rate-limit error carries no explicit reset metadata -(`resets_at`/`resets_in_seconds`), the resolved cooldown deadline SHALL be -persisted on the account row (`reset_at`) so the cooldown survives process -restarts and is visible to all replicas sharing the database: a parsed -Retry-After hint deadline SHALL be persisted rounded up to the next whole -second (persistence stores `reset_at` as an integer, so a short or fractional -hint MUST NOT truncate down to an already-elapsed deadline), and when the -cooldown comes from the error-count backoff fallback the persisted deadline -SHALL be at least `RATE_LIMITED_MIN_COOLDOWN_SECONDS` (30 seconds) in the -future. Explicit upstream reset metadata, when present, SHALL continue to be -persisted as-is. -The marking replica's in-process cooldown MAY remain shorter than the -persisted deadline so its existing fresh-usage recovery gate is unchanged. +Explicit upstream reset metadata SHALL be accepted only when it resolves to a +finite deadline strictly later than the current time and no more than +`RATE_LIMIT_RESET_MAX_HORIZON_SECONDS` (366 days) in the future. `resets_at` +SHALL be interpreted as an absolute Unix timestamp and `resets_in_seconds` +SHALL be interpreted as a relative duration. When `resets_at` is invalid but +`resets_in_seconds` is valid, the relative duration SHALL be used. An accepted +fractional deadline SHALL be rounded up to the next whole second before +persistence. A persisted integer deadline produced by that rounding MAY be +less than one second beyond the raw 366-day horizon and MUST remain valid when +selection reconstructs it. When neither field is valid, the error SHALL be +treated as carrying no explicit reset metadata. + +When the upstream rate-limit error carries no valid explicit reset metadata, +the resolved cooldown deadline SHALL be persisted on the account row +(`reset_at`) so the cooldown survives process restarts and is visible to all +replicas sharing the database: a parsed Retry-After hint deadline SHALL be +persisted rounded up to the next whole second (persistence stores `reset_at` +as an integer, so a short or fractional hint MUST NOT truncate down to an +already-elapsed deadline), and when the cooldown comes from the error-count +backoff fallback the persisted deadline SHALL be at least +`RATE_LIMITED_MIN_COOLDOWN_SECONDS` (30 seconds) in the future. The marking +replica's in-process cooldown MAY remain shorter than the persisted deadline +so its existing fresh-usage recovery gate is unchanged. + +An already-persisted `rate_limited` reset deadline beyond the same plausibility +horizon SHALL be treated as missing metadata rather than as an unexpired +cooldown. A row carrying `blocked_at` SHALL still honor the existing 30-second +minimum floor and SHALL require recent usage evidence recorded after that block +before selection-time recovery may clear it. A row without `blocked_at` SHALL +require recent available usage evidence. In both cases, every applicable +derived quota window MUST report below `100%` usage before recovery. #### Scenario: Compound minute-and-second hint sets the full cooldown - **GIVEN** an upstream 429 whose message says "try again in 6m0s" - **WHEN** the balancer records the rate limit for the account - **THEN** the account cooldown lasts 360 seconds -- **AND** the account is not re-selected until that cooldown elapses +- **AND** the account is not re-selected until its cooldown elapses #### Scenario: Minutes-only hint is honored @@ -352,6 +370,58 @@ persisted deadline so its existing fresh-usage recovery gate is unchanged. - **THEN** the persisted integer `reset_at` deadline is strictly in the future - **AND** peer replicas honor the hinted cooldown instead of reselecting the account immediately +#### Scenario: Plausible explicit reset metadata remains authoritative + +- **GIVEN** an OpenAI service 429 carrying a finite `resets_at` deadline 30 days in the future +- **WHEN** the balancer records the rate limit for the account +- **THEN** the accepted explicit deadline is persisted +- **AND** the Retry-After/backoff fallback does not replace it + +#### Scenario: Implausible explicit reset metadata uses the bounded fallback + +- **GIVEN** an OpenAI service 429 carrying `resets_at=15023672358` while the current Unix time is approximately `1784146959` +- **AND** the error carries no valid `resets_in_seconds` or parseable duration +- **WHEN** the balancer records the rate limit for the account +- **THEN** the implausible absolute deadline is rejected +- **AND** the persisted deadline uses the minimum bounded backoff instead + +#### Scenario: Valid relative metadata survives an invalid absolute value + +- **GIVEN** an OpenAI service 429 whose `resets_at` is implausibly far in the future +- **AND** whose `resets_in_seconds` is a finite positive duration within 366 days +- **WHEN** the balancer records the rate limit for the account +- **THEN** the relative duration determines the persisted deadline + +#### Scenario: Horizon-edge rounding remains stable + +- **GIVEN** valid absolute or relative reset metadata resolves exactly 366 days after a fractional current timestamp +- **WHEN** the balancer rounds and persists the deadline to a whole second +- **THEN** persisted-state reconstruction continues to accept that deadline +- **AND** does not clear the cooldown solely because rounding crossed the raw horizon by less than one second + +#### Scenario: Existing implausible deadline does not pin selection indefinitely + +- **GIVEN** a persisted `rate_limited` account whose `reset_at` is more than 366 days in the future +- **AND** whose `blocked_at` minimum floor has elapsed +- **WHEN** selection reconstructs the account from fresh available usage evidence +- **THEN** the implausible deadline is treated as missing metadata +- **AND** normal compare-and-set recovery may restore the account to `active` + +#### Scenario: Exhausted long-window quota prevents poisoned-row recovery + +- **GIVEN** a persisted `rate_limited` account whose reset deadline is implausible +- **AND** a fresh primary window reports available quota +- **AND** an applicable weekly or monthly window reports `100%` usage +- **WHEN** selection reconstructs the account +- **THEN** the account remains `rate_limited` + +#### Scenario: Implausible legacy deadline without a block marker recovers + +- **GIVEN** a persisted `rate_limited` account whose reset deadline is implausible +- **AND** the row has no `blocked_at` marker +- **WHEN** selection reconstructs the account from recent available usage in every applicable window +- **THEN** normal compare-and-set recovery may restore the account to `active` + ### Requirement: Re-authentication-required accounts are not selectable When an account credential/session is invalidated but the upstream account is not known to be disabled, the system MUST mark the account `reauth_required`. The selector MUST remove `reauth_required` accounts from every routing strategy and hard-affinity fallback until the account is re-authenticated. Operator pickers that configure single-account routing or account-scoped routing MUST only offer accounts that are not hard-blocked by paused, reauth-required, or deactivated status. @@ -641,3 +711,36 @@ Recovery admission MUST occur only after all ordinary account eligibility, coold - **WHEN** selection finalizes the stable local account-cap error - **THEN** any provisional delete or rebind decision is discarded - **AND** the existing hard-sticky owner mapping remains unchanged + +### Requirement: Trusted cyber intent narrows the existing account pool + +Account routing MUST constrain an authenticated direct Responses WebSocket +turn requiring `trusted_cyber` by passing +`require_security_work_authorized=True` to the canonical selector before the +first upstream attempt and every later retry. The selector MUST apply the +constraint only to accounts already permitted by API-key, account, model, +service-tier, ownership, health, quota, affinity, concurrency, and failover +rules. Routing MUST NOT add an account, change the configured strategy, rebind +an owner, or fall back to an ordinary account. + +#### Scenario: First attempt uses the capable pool +- **WHEN** an authenticated direct WebSocket turn establishes `trusted_cyber` +- **THEN** its first account-selection call requires a + security-work-authorized account +- **AND** no ordinary account receives an upstream attempt + +#### Scenario: Empty capable pool fails closed +- **WHEN** a required turn has no eligible security-work-authorized account +- **THEN** selection returns the existing typed + `no_security_work_authorized_accounts` error +- **AND** its advisory states that no ordinary-account fallback occurred +- **AND** an earlier reactive or account/model error cannot replace that typed + capability-routing result +- **AND** ordinary routing is not attempted + +#### Scenario: Ordinary routing is unchanged +- **WHEN** an authenticated direct WebSocket turn has neither a trusted signal + nor required lineage +- **THEN** selection receives the same scope, strategy, ownership, admission, + and retry inputs as before this change + diff --git a/openspec/specs/api-keys/spec.md b/openspec/specs/api-keys/spec.md index ac3311d678..2127be5b61 100644 --- a/openspec/specs/api-keys/spec.md +++ b/openspec/specs/api-keys/spec.md @@ -272,7 +272,13 @@ The system SHALL keep the existing lazy on-read reset strategy for API key usage ### Requirement: RequestLog API key reference -The system SHALL record the `api_key_id` in the `request_logs` table for proxy requests authenticated with an API key. The field MUST be NULL when API key auth is disabled or the request is unauthenticated. +The system SHALL record the `api_key_id` in the `request_logs` table for proxy +requests authenticated with an API key. The field MUST be NULL when API key +auth is disabled or the request is unauthenticated. This applies to error rows +as well as successes: when a shared upstream session (e.g. an HTTP-bridge +session multiplexing requests from multiple API keys) fails its pending +requests, each request's log row MUST be attributed to that request's own +authenticated key. #### Scenario: Authenticated request logged @@ -284,6 +290,16 @@ The system SHALL record the `api_key_id` in the `request_logs` table for proxy r - **WHEN** API key auth is disabled and a proxy request completes - **THEN** the `request_logs` entry has `api_key_id = NULL` +#### Scenario: Bridge failure fan-out preserves per-request key attribution + +- **GIVEN** an HTTP-bridge session holds a pending request authenticated with + API key `key-123` +- **WHEN** the session fails its pending requests (upstream close, send + failure, request timeout, or local terminal error) +- **THEN** the request's `request_logs` error entry has + `api_key_id = "key-123"` even though the session-level failure path has no + single key of its own + ### Requirement: Frontend API Key management The SPA settings page SHALL include an API Key management section with: a toggle for `apiKeyAuthEnabled`, a key list table showing prefix/name/models/limit/usage/expiry/status, a create dialog (name, model selection, assigned-account selection, usage sections multi-select, weekly limit, expiry date), and key actions (edit, delete, regenerate). On key creation, the SPA MUST display the plain key in a copy-able dialog with a warning that it will not be shown again, and the copy action MUST remain functional in secure and non-secure contexts. @@ -520,6 +536,11 @@ Usage reservation의 최종 정산(finalize 또는 release)은 요청 단위에 Reservation 생성 후 upstream API 호출에 진입하지 않고 종료되는 모든 경로에서 reservation이 release되어야 한다. `reserved` 상태로 남는 reservation이 존재하면 안 된다. 시스템은 이 동작을 SHALL 보장해야 한다. +After admission commits an owned reservation, rate-limit response-header +calculation before upstream work remains part of the early-exit cleanup window. +If that calculation fails, the system MUST attempt to release the owned +reservation exactly once before propagating the original header failure. + #### Scenario: no_accounts 즉시 종료 시 release - **WHEN** reservation 생성 후 `_stream_with_retry()`가 사용 가능한 계정 없음(`no_accounts`)으로 즉시 종료되면 @@ -536,6 +557,17 @@ Reservation 생성 후 upstream API 호출에 진입하지 않고 종료되는 - **WHEN** API key auth가 비활성이거나 reservation이 생성되지 않은 상태에서 요청이 종료되면 - **THEN** 정산 로직이 안전하게 스킵되어야 하며 에러가 발생하지 않아야 한다 (SHALL) +#### Scenario: Rate-limit header preparation fails after admission + +- **GIVEN** a limited API key has committed an owned reservation for a + streaming Responses, collected Responses, compact Responses, or audio + transcription request +- **WHEN** rate-limit response-header calculation fails before upstream work + begins +- **THEN** the reservation is released exactly once +- **AND** its reserved quota is restored +- **AND** the header failure propagates without starting upstream work + ### Requirement: Compact 경로 예외 무관 reservation cleanup `_compact_responses()` 경로에서 reservation이 존재할 때, 어떤 예외 타입이 발생하더라도 reservation이 정리되어야 한다. 특정 예외 타입에만 의존하는 cleanup은 허용되지 않는다. 시스템은 이 동작을 SHALL 보장해야 한다. @@ -1108,7 +1140,27 @@ API-key limit and usage-reporting paths used by subscription-backed requests. ### Requirement: Stream reservation settlement is detached from the response path -Settling a stream API-key reservation MUST NOT block the response/stream close, with one deliberate exception: when a keyed websocket stream terminates with an account-health error, the finalizer MUST wait for the settlement to commit before the load-balancer health write (the settlement-ordering invariant), so that error path intentionally blocks on settlement. In all other cases the settlement MUST run as a tracked background task; when it fails or is cancelled, the reservation MUST still be released by the tracking fallback, and the request's finalization path MUST NOT double-release a transferred settlement. Reservations MUST continue to count toward key limits until finalized or released, so deferred settlement can never admit usage a synchronous settlement would have rejected. +Settling a stream API-key reservation MUST NOT block the response/stream close, +with one deliberate exception: when a keyed websocket stream terminates with an +account-health error, the finalizer MUST wait for the settlement to commit +before the load-balancer health write (the settlement-ordering invariant), so +that error path intentionally blocks on settlement. If the primary settlement +fails, the finalizer MUST wait for fallback release to commit before recording +account health. If neither operation confirms settlement, the account-health +write MUST remain unapplied. Tracked persistence ownership MUST remain +registered through an ordering-sensitive fallback release, including +cancellation before the primary coroutine starts or during that release, so +graceful shutdown drains both phases. When the existing stream-retry path +deliberately defers an +account-health penalty until the same ordering-sensitive settlement, it MUST +likewise apply neither that penalty nor an immediately following terminal health +write unless settlement is confirmed, and it MUST NOT start a second settlement +for the transferred reservation. In all other cases the settlement MUST run as +a tracked background task; when it fails or is cancelled, the reservation MUST +still be released by the tracking fallback, and the request's finalization path +MUST NOT double-release a transferred settlement. Reservations MUST continue to +count toward key limits until finalized or released, so deferred settlement can +never admit usage a synchronous settlement would have rejected. #### Scenario: Response close precedes settlement completion @@ -1129,10 +1181,33 @@ Settling a stream API-key reservation MUST NOT block the response/stream close, - **WHEN** the finalizer settles the reservation - **THEN** it waits for the settlement to commit before recording the account-health error +#### Scenario: Websocket health waits for fallback settlement + +- **GIVEN** a keyed websocket stream that terminates with an account-health error +- **AND** its primary settlement fails +- **WHEN** fallback release remains in progress +- **THEN** the finalizer does not record the account-health error +- **AND** it records the error only after fallback release commits + +#### Scenario: Unconfirmed websocket settlement leaves health unapplied + +- **GIVEN** a keyed websocket stream that terminates with an account-health error +- **WHEN** both primary settlement and fallback release fail +- **THEN** the finalizer does not record the account-health error +- **AND** the upstream connection is still scheduled for reconnect and retirement + +#### Scenario: Unconfirmed retry settlement drops deferred health + +- **GIVEN** a keyed stream retry has deferred an account-health penalty until replacement selection +- **WHEN** neither primary settlement nor fallback release confirms settlement +- **THEN** the deferred penalty and any immediately following terminal health write remain unapplied +- **AND** the retry path does not start a second settlement for the transferred reservation + #### Scenario: Shutdown drains pending settlements - **WHEN** the service shuts down gracefully with settlements in flight - **THEN** shutdown waits for them up to the configured drain timeout +- **AND** a pending ordering-sensitive fallback release remains part of that drain despite cancellation before primary startup or during fallback ### Requirement: Untrusted forwarded headers do not grant unauthenticated proxy locality @@ -1255,3 +1330,76 @@ The system SHALL track `api_keys.last_used_at` through a process-local write-beh - **GIVEN** a settlement task that outlived the shutdown drain of persistence tasks - **WHEN** it records a touch after the flusher has stopped and performed its final flush - **THEN** the touch is flushed to the database immediately by the recording path rather than being lost at process exit + +### Requirement: GPT-5.6 personality pricing is recognized + +The system MUST recognize `gpt-5.6`, `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` when computing request costs. The bare `gpt-5.6` alias MUST resolve to Sol, and suffixed aliases for each personality model MUST resolve to the matching canonical pricing entry. Standard, Flex, Priority, and requests with more than 272K input tokens MUST use the published rates applicable to the model and tier. + +#### Scenario: Canonical GPT-5.6 models use personality-specific pricing + +- **WHEN** a standard-tier request completes for `gpt-5.6-sol`, `gpt-5.6-terra`, or `gpt-5.6-luna` +- **THEN** the system computes cost using that model's standard input, cached-input, and output rates + +#### Scenario: Bare GPT-5.6 alias resolves to Sol pricing + +- **WHEN** a request completes for `gpt-5.6` +- **THEN** the system resolves it to the canonical Sol pricing entry +- **AND** the system does not use the generic `gpt-5` pricing entry + +#### Scenario: Suffixed GPT-5.6 model resolves to its personality price + +- **WHEN** a request completes for a suffixed GPT-5.6 personality model ID +- **THEN** the system resolves it to the matching canonical Sol, Terra, or Luna pricing entry +- **AND** the system does not use the generic `gpt-5` pricing entry + +#### Scenario: GPT-5.6 service tiers use published tier rates + +- **WHEN** a GPT-5.6 request completes with `service_tier: "flex"` or `service_tier: "priority"` +- **THEN** the system computes cost using the published rates for that model and service tier + +#### Scenario: GPT-5.6 long-context request uses published uplift + +- **WHEN** a standard-tier or Flex GPT-5.6 request completes with more than 272K input tokens +- **THEN** the system computes cost using the published long-context input, cached-input, and output rates for that model and tier + +### Requirement: API-key limit rule identities are unique + +The system SHALL reject an API-key create or update payload when it contains +more than one limit rule with the same `(limit_type, limit_window, +model_filter)` identity. Rejection MUST use the typed API-key validation error +and MUST occur before a create request persists an API key or limit row. +The validation message MUST identify the duplicate rule identity. + +#### Scenario: Duplicate rules are rejected during creation + +- **WHEN** an administrator submits `POST /api/api-keys` with two limit rules + sharing the same type, window, and model filter +- **THEN** the API returns `400` with `invalid_api_key_payload` +- **AND** no API key or limit row is persisted + +### Requirement: Stale usage-reservation reclamation enforces a hard age ceiling + +Stale usage-reservation reclamation MUST reclaim `reserved` reservations whose +age exceeds a hard ceiling on creation time regardless of how recently their +`updated_at` was refreshed. This is the backstop for orphaned reservation +heartbeats: a leaked heartbeat task keeps touching `updated_at`, which would +otherwise exempt its reservation from the heartbeat-based staleness cutoff +forever. The ceiling MUST be far larger than any legitimate request lifetime +so it can never reclaim an in-flight reservation, and reclamation past the +ceiling MUST restore the reserved quota the same way heartbeat-based +reclamation does. + +#### Scenario: Orphaned heartbeat cannot exempt a reservation forever + +- **GIVEN** a `reserved` usage reservation created before the hard age ceiling +- **AND** a leaked heartbeat keeps refreshing its `updated_at` +- **WHEN** stale usage-reservation reclamation runs +- **THEN** the reservation is released and its reserved quota is restored + +#### Scenario: Fresh reservations are untouched by the ceiling + +- **GIVEN** a `reserved` usage reservation created within the hard age ceiling +- **AND** its `updated_at` is current +- **WHEN** stale usage-reservation reclamation runs +- **THEN** the reservation stays `reserved` + diff --git a/openspec/specs/audio-transcriptions-compat/spec.md b/openspec/specs/audio-transcriptions-compat/spec.md index 0790a5599d..133d3800e4 100644 --- a/openspec/specs/audio-transcriptions-compat/spec.md +++ b/openspec/specs/audio-transcriptions-compat/spec.md @@ -86,3 +86,67 @@ The system MUST enforce a configurable total request budget for transcription pr - **THEN** the retry uses the refreshed account metadata - **AND** the retry only proceeds if enough request budget remains for another attempt +### Requirement: Transcription multipart uploads are authorized and bounded + +`POST /backend-api/transcribe` and `POST /v1/audio/transcriptions` MUST complete their existing proxy authorization dependencies before reading multipart body bytes. Each request MUST contain exactly one file part no greater than 25,000,000 bytes, no more than 32 text fields of at most 256 KiB each, and a complete multipart body no greater than 32 MiB (33,554,432 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and actual streamed bytes. It MUST enforce file and text limits before retaining crossing bytes, close multipart spools before usage reservation, account selection, or upstream forwarding, preserve ordered text-field forwarding for configured model sources, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for both transcription operations. Their exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before proxy authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error`; a file-part failure MUST set `param = file`. Multipart syntax, count, and required-field failures MUST retain OpenAI-compatible invalid-request behavior and MUST NOT reserve usage or call upstream. + +#### Scenario: Unauthorized transcription does not consume the body + +- **WHEN** a transcription request fails the existing proxy API-key authorization +- **THEN** the authentication response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created + +#### Scenario: Bounded native transcription remains compatible + +- **WHEN** an authorized `/backend-api/transcribe` request supplies one audio file within 25,000,000 bytes, an optional bounded prompt, and a multipart body within 32 MiB +- **THEN** the service forwards the same audio bytes, filename, content type, and prompt through the existing transcription pipeline + +#### Scenario: Bounded source-model transcription preserves fields + +- **WHEN** an authorized `/v1/audio/transcriptions` request selects a configured model source and all multipart limits are satisfied +- **THEN** the service forwards the audio file and the ordered non-file form fields through the existing source pipeline + +#### Scenario: Declared or streamed transcription body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 32 MiB or actual streamed multipart bytes cross 32 MiB +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error` +- **AND** no usage reservation, account selection, or upstream request occurs + +#### Scenario: Transcription file exceeds its limit + +- **WHEN** the audio file part exceeds 25,000,000 bytes +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large`, `type = invalid_request_error`, and `param = file` +- **AND** bytes beyond the file limit are not retained in a spool or handler buffer + +#### Scenario: Transcription field resources are bounded + +- **WHEN** a request exceeds 32 text fields, one file part, or 256 KiB in any text part +- **THEN** the service rejects the request with the documented OpenAI-compatible count or byte-limit response +- **AND** it does not invoke transcription route logic + +#### Scenario: Compressed transcription is rejected without prebuffering + +- **GIVEN** the transcription request has passed proxy authorization +- **WHEN** either transcription route declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with OpenAI error `code = invalid_request_error` and `type = invalid_request_error` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 32 MiB dedicated body limit + +#### Scenario: Generic ingress does not preempt encoded transcription authorization + +- **GIVEN** a request to either transcription route fails proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed + +#### Scenario: Transcription cleanup preserves transport failures + +- **WHEN** parsing succeeds, fails a limit, encounters malformed multipart, receives a client disconnect, or is cancelled +- **THEN** every created multipart spool is closed +- **AND** disconnect and cancellation are not converted to HTTP 413 + diff --git a/openspec/specs/audit-logging/spec.md b/openspec/specs/audit-logging/spec.md new file mode 100644 index 0000000000..78b5fbcd78 --- /dev/null +++ b/openspec/specs/audit-logging/spec.md @@ -0,0 +1,49 @@ +# audit-logging Specification + +## Purpose +Ownership and shutdown-drain guarantees for asynchronous dashboard audit-log writes, so records are neither lost nor left to leaked tasks. +## Requirements +### Requirement: Asynchronous audit writes remain owned until completion + +The system MUST execute `AuditService.log_async()` writes in tracked background tasks without making the calling request wait for database persistence. Each task MUST remain strongly owned until it finishes, and success, cancellation, or failure MUST remove it from the tracked set. An unexpected task failure MUST be consumed and reported rather than becoming an unobserved task exception. + +#### Scenario: Audit logging remains fire-and-forget + +- **GIVEN** an audit-log database write is blocked +- **WHEN** application code calls `AuditService.log_async()` +- **THEN** the call returns before the database write completes +- **AND** the pending write remains tracked until completion + +#### Scenario: Failed audit task is cleaned up + +- **WHEN** an asynchronous audit task fails unexpectedly +- **THEN** the failure is reported +- **AND** the completed task is removed from the tracked set + +### Requirement: Graceful shutdown drains pending audit writes + +Immediately after the in-flight drain attempt returns, graceful shutdown MUST synchronously close asynchronous audit-task admission before any further shutdown await. An `AuditService.log_async()` call after this cutoff MUST remain non-blocking, MUST report the rejected action, and MUST NOT construct a write coroutine or task. Graceful shutdown MUST wait for audit-log tasks accepted before the cutoff for up to `shutdown_drain_timeout_seconds` before closing shared database resources. The drain MUST include tasks that complete or become visible while task-completion callbacks are running. If the deadline expires, the system MUST report each audit task that did not drain before continuing shutdown. + +#### Scenario: Late audit producer is rejected after in-flight timeout + +- **GIVEN** an HTTP handler remains alive after the in-flight drain timeout +- **AND** graceful shutdown has closed control-plane task admission +- **WHEN** the handler calls `AuditService.log_async()` +- **THEN** the call returns without waiting +- **AND** the rejected action is reported +- **AND** no audit write coroutine or task is created + +#### Scenario: Shutdown preserves a pending audit row + +- **GIVEN** an asynchronous audit write is still pending when graceful shutdown begins +- **WHEN** the write completes within the configured drain timeout +- **THEN** shutdown waits for the write +- **AND** shared database resources remain open until the write finishes + +#### Scenario: Overdue audit write is reported + +- **GIVEN** an asynchronous audit write remains pending for the full configured drain timeout +- **WHEN** graceful shutdown drains audit tasks +- **THEN** the drain reports that task as overdue +- **AND** shutdown is allowed to continue + diff --git a/openspec/specs/data-retention/spec.md b/openspec/specs/data-retention/spec.md index 4c72b6e1ee..b47c491cfa 100644 --- a/openspec/specs/data-retention/spec.md +++ b/openspec/specs/data-retention/spec.md @@ -172,3 +172,32 @@ retention MUST NOT run a pass. - **WHEN** the scheduler ticks - **THEN** no retention pass runs +### Requirement: Disabled request-log pruning is explained and presets are non-destructive + +When the effective request-log retention value is `0`, the Settings data retention card SHALL show neutral informational text that request-log pruning is disabled, logs are retained indefinitely, and storage will grow over time, and SHALL offer 30-day and 90-day request-log retention presets. The informational text MUST NOT characterize disabled pruning as unsafe or direct the operator to change it. Activating a preset MUST update only the local request-log retention form value and MUST NOT persist any setting until the operator activates the existing explicit save action. Rendering the information and presets MUST NOT change the stored override or any other retention policy. + +#### Scenario: Effective disabled state shows information and presets + +- **GIVEN** effective request-log retention is `0` +- **WHEN** an operator views the data retention card +- **THEN** the card explains neutrally that request-log pruning is disabled and + logs are retained indefinitely +- **AND** the text notes that storage will grow over time without directing the + operator to change the policy +- **AND** the card offers 30-day and 90-day request-log retention presets +- **AND** no settings update is submitted + +#### Scenario: Preset selection requires explicit save + +- **GIVEN** effective request-log retention is `0` +- **WHEN** an operator activates the 30-day or 90-day preset +- **THEN** the request-log retention form value changes to the selected number +- **AND** no settings update is submitted until the operator activates save +- **AND** usage-history retention remains unchanged + +#### Scenario: Enabled effective policy does not show disabled-state information + +- **GIVEN** effective request-log retention is greater than `0` +- **WHEN** an operator views the data retention card +- **THEN** the disabled-state information and presets are not shown + diff --git a/openspec/specs/database-backends/spec.md b/openspec/specs/database-backends/spec.md index 0d5e16e1a3..24786da2e2 100644 --- a/openspec/specs/database-backends/spec.md +++ b/openspec/specs/database-backends/spec.md @@ -198,7 +198,6 @@ advisory-lock behavior. - **WHEN** an account status transition is persisted - **THEN** the write executes inside the shared SQLite writer section - ### Requirement: Telemetry write transactions relax commit durability on PostgreSQL A write transaction is classified as a **telemetry write** when it only appends observability rows whose loss on a database-server crash changes nothing about accounting semantics: request-log inserts (`request_logs`) and usage-history appends (`usage_history`, `additional_usage_history`). API-key usage-reservation accounting is explicitly NOT telemetry (see the reservation-durability requirement below). @@ -266,3 +265,113 @@ Rationale: the "crash loses the in-flight request anyway" argument that justifie - **GIVEN** a PostgreSQL backend holding a stale usage reservation (heartbeat stopped or past the maximum age) - **WHEN** the stale-reservation release settles a batch (status flip to `released` plus its limit-counter adjustments) - **THEN** no batch transaction executes `SET LOCAL synchronous_commit = off` + +### Requirement: Asyncpg PostgreSQL sessions pin time zone to UTC + +When `database_url` resolves to a PostgreSQL backend through the asyncpg driver, the application MUST configure each SQLAlchemy async engine connection with a database session time zone of `UTC`. + +This requirement applies to the request-path `engine`, the optional background +`_background_engine`, and any app-created PostgreSQL async engine that uses the +shared PostgreSQL engine kwargs helper. + +#### Scenario: Asyncpg sessions ignore non-UTC database defaults + +- **GIVEN** `database_url` uses `postgresql+asyncpg://` +- **AND** the PostgreSQL role, database, container, or server default time zone + is not UTC +- **WHEN** the application opens a new asyncpg connection through its engine + configuration +- **THEN** `SHOW TIME ZONE` on that connection reports `UTC` +- **AND** naive UTC datetimes written by the application are interpreted as UTC + before PostgreSQL stores them in `timestamptz` columns + +#### Scenario: SQLite backends are not affected + +- **GIVEN** `database_url` resolves to a SQLite backend +- **WHEN** the application creates its async engine +- **THEN** PostgreSQL asyncpg `server_settings` are not configured +- **AND** existing SQLite PRAGMAs, busy timeout, and pooling behavior remain + unchanged + +### Requirement: PostgreSQL connection budgets include every pooled engine + +The application SHALL define its per-worker PostgreSQL connection capacity as the configured per-engine pool capacity multiplied by the declared set of independently pooled engine roles. The request-path and background-task engine creation paths MUST each use the shared role-aware PostgreSQL engine factory, and the engine-count budget MUST be derived from those declared roles. The owned server launcher MUST run the supported one worker per replica explicitly, rather than allowing `WEB_CONCURRENCY` to multiply worker processes and their pools. + +#### Scenario: One replica reaches configured pool capacity + +- **WHEN** both declared PostgreSQL engine roles in one application worker reach `database_pool_size + database_max_overflow` +- **THEN** the worker's aggregate application connection capacity is `2 * (database_pool_size + database_max_overflow)` +- **AND** both engines were created through the role-aware factory counted by that formula + +#### Scenario: WEB_CONCURRENCY cannot multiply owned-launcher pools + +- **GIVEN** `WEB_CONCURRENCY` is greater than 1 +- **WHEN** the application starts through the owned `app.cli` launcher used by Helm +- **THEN** the launcher explicitly starts one Uvicorn worker +- **AND** the replica creates only the request-path and background-task pools +- **AND** operators MUST scale supported deployments through replicas rather than custom multi-worker launchers + +#### Scenario: Test database disables pooling + +- **WHEN** `CODEX_LB_TEST_DATABASE_URL` selects `NullPool` +- **THEN** pool sizing controls and the production pooled-engine budget do not apply to that test engine + +### Requirement: SQLAlchemy-rendered Windows SQLite paths are percent-decoded before opening + +When a SQLite database URL is converted to a filesystem path for direct filesystem use (e.g. startup directory creation, startup integrity checks, migration locks, or the usage repository's read-only helper), a path that matches a recognizable SQLAlchemy-rendered Windows form — an encoded drive marker (`%3A` followed by an encoded or raw path separator) or an encoded UNC prefix (`%5C%5C`) — MUST be percent-decoded before being handed to the filesystem. SQLAlchemy's `URL.render_as_string()` percent-encodes a Windows-style default path (`C:\Users\...` -> `C%3A%5CUsers%5C...`); without decoding, the literal escaped string either fails to open with "unable to open database file" or creates a stray 0-byte database next to the current working directory, which breaks account/usage reads with `no such table`. + +Paths that do NOT match those rendered Windows forms MUST be preserved literally. Settings builds the default SQLite URL directly from the configured data directory without URL-encoding it, so a percent sequence in a POSIX or raw Windows path (e.g. `/var/lib/codex%20lb/store.db`) names a real directory and MUST NOT be rewritten by decoding. + +#### Scenario: Windows default path resolves to the real file + +- **GIVEN** the default SQLite URL on Windows (`sqlite+aiosqlite:///C:\Users\...\store.db`) +- **WHEN** `URL.render_as_string()` percent-encodes it into `sqlite:///C%3A%5CUsers%5C...%5Cstore.db` +- **AND** the path is extracted and decoded +- **THEN** `sqlite3.connect()` receives `C:\Users\...\store.db` (the real file), not the percent-escaped literal + +#### Scenario: Encoded drive with URL slash separators resolves to the real file + +- **GIVEN** a Windows SQLite URL with an encoded drive colon and normal URL path separators (`sqlite:///C%3A/Users/me/.codex-lb/store.db`) +- **WHEN** the path is extracted and decoded +- **THEN** the filesystem path is `C:/Users/me/.codex-lb/store.db`, not the literal `C%3A/Users/me/.codex-lb/store.db` + +#### Scenario: Startup uses the decoded SQLite path + +- **GIVEN** a percent-encoded SQLite file URL whose decoded parent directory differs from the percent-literal parent +- **WHEN** `init_db()` prepares the SQLite directory and runs the startup integrity check +- **THEN** the decoded parent directory is created +- **AND** the integrity check receives the decoded database path + +#### Scenario: URL normalization preserves decoded Windows path characters + +- **GIVEN** an encoded Windows SQLite URL whose decoded database path contains spaces, literal `%`, or `#` +- **WHEN** the URL is normalized for SQLAlchemy consumers +- **THEN** the returned URL contains the real decoded Windows filesystem path +- **AND** filesystem extraction from that normalized URL returns the same decoded path +- **AND** a raw Windows URL containing a literal percent sequence such as `%23` is not decoded unless it first matched a SQLAlchemy-rendered encoded Windows form + +#### Scenario: Literal percent sequences in POSIX paths are preserved + +- **GIVEN** a POSIX SQLite URL whose path contains a literal percent sequence (`sqlite+aiosqlite:////var/lib/codex%20lb/store.db`) built directly from the configured data directory +- **WHEN** the path is extracted for filesystem use or the URL is normalized +- **THEN** the filesystem path remains `/var/lib/codex%20lb/store.db` and the URL is unchanged (the sequence is not decoded to a space) + +#### Scenario: Normalized UNC paths keep fragment characters + +- **GIVEN** an encoded UNC SQLite URL whose decoded share path contains a legal `#` character (`sqlite:///%5C%5Cserver%5Cshare%23x%5Cstore.db`) +- **WHEN** the URL is normalized and the filesystem path is then extracted from the normalized URL +- **THEN** the extracted path is `\\server\share#x\store.db` +- **AND** the path is not truncated at the `#` as if it were a URL fragment separator + +#### Scenario: POSIX paths are unchanged + +- **GIVEN** a POSIX-style SQLite URL (`sqlite+aiosqlite:///var/lib/codex-lb/store.db`) +- **WHEN** the path is extracted and decoded +- **THEN** the result is identical to the input path (no `%` to decode; behavior is a no-op) + +#### Scenario: In-memory databases are not treated as file paths + +- **GIVEN** a `:memory:` SQLite URL +- **WHEN** the path is extracted +- **THEN** no filesystem path is returned and no file is created + diff --git a/openspec/specs/database-migrations/spec.md b/openspec/specs/database-migrations/spec.md index 0f72c33a1d..dd8eebf530 100644 --- a/openspec/specs/database-migrations/spec.md +++ b/openspec/specs/database-migrations/spec.md @@ -240,3 +240,95 @@ Migration state inspection SHALL classify `alembic_version` revisions that are n - **WHEN** the upgrade runs - **THEN** it fails with the ahead-specific guidance rather than a generic unsupported-revision remap error +### Requirement: Migration CLI distinguishes omitted and empty targets + +The `app.db.migrate` / `codex-lb-db` CLI SHALL use the settings-derived database URL only when `--db-url` is omitted. If `--db-url` is explicitly supplied as an exact empty string, the CLI MUST terminate with an argument error before resolving or opening a settings-derived database target. This validation MUST apply to every supported migration subcommand: `upgrade`, `current`, `check`, `wait-for-head`, `wait-for-connection`, and `stamp`. + +#### Scenario: Explicit empty target is rejected before side effects + +- **GIVEN** settings would resolve a valid database target +- **WHEN** any supported migration subcommand is invoked with `--db-url ""` +- **THEN** the CLI exits nonzero with an argument-validation error +- **AND** it does not select, connect to, create, inspect, migrate, or stamp the settings-derived target + +#### Scenario: Omitted target retains the settings fallback + +- **GIVEN** settings resolve a valid database target +- **WHEN** a supported migration subcommand is invoked without `--db-url` +- **THEN** the CLI uses the settings-derived database target + +### Requirement: Capability lineage uses one additive opaque-marker table + +The migration MUST descend from the current single Alembic head and create one +`capability_lineage_markers` table containing only an opaque SHA-256 marker +primary key plus creation and last-seen timestamps. It MUST NOT modify existing +sticky-session, account, usage, quota, request-log, or durable-bridge columns or +foreign keys, and MUST NOT backfill historical rows. + +#### Scenario: Upgrade creates an empty marker table +- **WHEN** a database at the previous head upgrades to the new head +- **THEN** the marker table exists with its primary-key uniqueness contract +- **AND** no existing application table is scanned or rewritten for backfill + +#### Scenario: Downgrade removes only the marker table +- **WHEN** the migration is downgraded to its parent revision +- **THEN** only `capability_lineage_markers` is removed +- **AND** existing application data remains unchanged + +#### Scenario: Migration graph remains single-head +- **WHEN** the repository migration graph is inspected after this change +- **THEN** it has exactly one head containing the marker-table revision + +### Requirement: SQLite maintenance releases file handles before filesystem mutation + +Synchronous SQLite maintenance operations MUST explicitly close every native +connection they open after completing or rolling back its transaction. A +pre-migration backup MUST release its source and destination connections before +retention deletes an older snapshot. Recovery with `--replace` MUST release +connections used for integrity checking, dump export, and dump import before it +renames either the source database or recovered output. Correctness MUST NOT +depend on garbage collection or interpreter object-finalization timing. + +#### Scenario: Backup retention deletes an old snapshot on Windows + +- **GIVEN** SQLite pre-migration backups have reached their retention limit +- **WHEN** a new online snapshot is complete and retention deletes the oldest + snapshot +- **THEN** every connection opened for the completed snapshot is explicitly + closed before deletion +- **AND** backup rotation succeeds on platforms that prohibit deleting an open + database file + +#### Scenario: Recovery replaces a database on Windows + +- **GIVEN** a file-backed SQLite database is recovered through the CLI with + `--replace` +- **WHEN** dump export and import complete +- **THEN** the integrity-check, source, and output connections are explicitly + closed before either database file is renamed +- **AND** the original is preserved under the corrupt-backup name while the + recovered database is moved into the original path + +### Requirement: Alembic Config escapes percent characters in the SQLAlchemy URL + +When the application builds an Alembic `Config` for migration inspection or upgrade (`_build_alembic_config`), any `%` in the SQLAlchemy URL MUST be escaped to `%%` before being stored via `set_main_option`. Alembic stores option values in a `configparser` using `BasicInterpolation`, which treats a bare `%` as interpolation syntax; a percent-encoded Windows path (`C%3A%5CUsers%5C...`) otherwise raises `ValueError: invalid interpolation syntax` during startup. `get_main_option` decodes `%%` back to `%`, so the URL handed to SQLAlchemy is unchanged. + +#### Scenario: Windows path does not crash migration inspection + +- **GIVEN** the default SQLite URL on Windows, percent-encoded by `URL.render_as_string()` into `sqlite:///C%3A%5CUsers%5C...%5Cstore.db` +- **WHEN** the Alembic `Config` is built for migration inspection +- **THEN** no `ValueError: invalid interpolation syntax` is raised +- **AND** `get_main_option("sqlalchemy.url")` returns the normalized sync URL whose path is the decoded Windows filesystem path (`sqlite:///C:\Users\...\store.db`), because `to_sync_database_url` normalizes recognizable SQLAlchemy-rendered Windows SQLite URLs before the value is stored in the Alembic `Config` + +#### Scenario: Round-trip preserves an already-encoded percent + +- **GIVEN** a path that already contains a percent-encoded `%` (rendered as `%25`) +- **WHEN** the escape turns it into `%%25` and `get_main_option` decodes it +- **THEN** the value SQLAlchemy receives decodes back to `%25`, i.e. the original URL is preserved exactly + +#### Scenario: Non-Windows URLs are unaffected + +- **GIVEN** a SQLite or PostgreSQL URL whose path contains no `%` +- **WHEN** the escape and decode round-trip is applied +- **THEN** the URL is unchanged and migration behavior is identical to before + diff --git a/openspec/specs/date-display-format/spec.md b/openspec/specs/date-display-format/spec.md new file mode 100644 index 0000000000..9e76c8b5b8 --- /dev/null +++ b/openspec/specs/date-display-format/spec.md @@ -0,0 +1,96 @@ +# date-display-format Specification + +## Purpose +Operator-selectable dashboard date/time rendering (localStorage preference, ISO 8601 contract) without disturbing chart axis formats. +## Requirements +### Requirement: Date format preference is stored in localStorage + +The system SHALL persist a date display format preference in localStorage under the key `codex-lb-date-display-format`. The valid values SHALL be `"default"` and `"iso8601"`. The default value SHALL be `"default"`. The preference SHALL apply only to read-only date/time presentation text. + +#### Scenario: No stored preference + +- **WHEN** the preference has never been saved +- **THEN** the system SHALL use `"default"` format + +#### Scenario: User selects ISO 8601 + +- **WHEN** the user selects "ISO 8601" as the date format +- **THEN** the system SHALL persist `"iso8601"` to localStorage under `codex-lb-date-display-format` +- **AND** all applicable read-only date/time presentation text SHALL use ISO 8601 formatting + +#### Scenario: User switches back to Default + +- **WHEN** the user selects "Default" as the date format +- **THEN** the system SHALL persist `"default"` to localStorage +- **AND** all applicable read-only date/time presentation text SHALL revert to locale-dependent formatting + +#### Scenario: Interactive date and time controls retain their own format + +- **GIVEN** a date or time is shown within an interactive control used to enter, edit, select, or filter a value +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** inputs, calendars, date pickers, selectors, and equivalent interactive controls SHALL retain the format provided by their component or browser +- **AND** the preference SHALL NOT change the control's value representation or interaction behavior + +#### Scenario: Verbatim API and data representations remain unchanged + +- **GIVEN** a date or timestamp appears inside a verbatim API or data representation +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** raw JSON, request and response payloads, metadata, copied values, filenames, downloads, and exports SHALL preserve their source representation +- **AND** the preference SHALL NOT rewrite those values + +#### Scenario: Read-only presentation text follows the selected format + +- **GIVEN** a date or timestamp is presented as non-interactive text in a table cell, detail field, status, or informational label +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the rendered text SHALL update immediately to the selected format + +#### Scenario: Daily report table follows the selected format + +- **GIVEN** the daily report breakdown table is mounted +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the table's Day column SHALL update immediately +- **AND** Default SHALL use locale-dependent formatting +- **AND** ISO 8601 SHALL use `YYYY-MM-DD` formatting + +#### Scenario: Quota planner decision peak follows the selected format + +- **GIVEN** a quota planner decision presents `target_peak_at` as a read-only Peak label +- **WHEN** the user switches between "Default" and "ISO 8601" +- **THEN** the Peak label SHALL update immediately to the selected format +- **AND** the underlying decision details value SHALL remain unchanged + +### Requirement: ISO 8601 format spec for date/time rendering + +When the date display format is `"iso8601"`, the `formatTimeLong` function SHALL return `{ time: "HH:MM:SS", date: "YYYY-MM-DD" }` for read-only presentation text where: +- `time` is always the clock-time portion in 24-hour format (2-digit hour, 2-digit minute, 2-digit second, colon-separated) +- `date` is always the calendar-date portion in ISO 8601 format (4-digit year, 2-digit month, 2-digit day, hyphen-separated) +The semantic meaning of these fields SHALL remain stable across date display formats. Rendered date/time surfaces that display ISO values SHALL order the `date` value before the `time` value. + +#### Scenario: ISO 8601 rendering of a UTC timestamp + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** formatting a timestamp corresponding to August 9, 2026 at 14:30:45 local time +- **THEN** `formatTimeLong` SHALL return `{ time: "14:30:45", date: "2026-08-09" }` + +#### Scenario: Default rendering unchanged + +- **GIVEN** the date display format is `"default"` +- **WHEN** formatting any timestamp +- **THEN** `formatTimeLong` SHALL return locale-dependent values as before (unchanged behavior) + +### Requirement: Chart axes are not affected by date format + +The date display format setting SHALL NOT affect Recharts x-axis tick formatting or data preparation. Charts (account trend, API trend, reports) SHALL continue to use their own x-axis formats regardless of the selected date display format. + +#### Scenario: ISO 8601 setting does not change chart tooltips + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** hovering over a point on any chart +- **THEN** the tooltip heading SHALL use `formatChartDateTime` as before (locale-dependent short month + day + time) + +#### Scenario: Reports chart x-axis unchanged + +- **GIVEN** the date display format is `"iso8601"` +- **WHEN** rendering a reports chart (tokens per day, cost per day, etc.) +- **THEN** the x-axis ticks SHALL remain `MM-DD` strings (e.g., `"08-09"`) + diff --git a/openspec/specs/deployment-installation/spec.md b/openspec/specs/deployment-installation/spec.md index b112b3f1ba..c8b57f77dd 100644 --- a/openspec/specs/deployment-installation/spec.md +++ b/openspec/specs/deployment-installation/spec.md @@ -405,3 +405,173 @@ enable switches. - **WHEN** Codex bridge requests are served - **THEN** no session prewarm is attempted and visible requests record `prewarm_status=not_applicable` + +### Requirement: Response-create dump directory is bounded without configuration + +The oversized response-create dump directory under `/debug/response-create-dumps` MUST be bounded on the base install path with no operator configuration. When the service captures an oversized `response.create` payload, it MUST NOT write a new dump if a dump for the same payload fingerprint is already stored, and after storing a dump it MUST remove the oldest stored dumps so that at most a fixed number of dump pairs remain. Each dump is a pair of a gzipped payload file and a meta file that MUST be added and removed together. Suppressing a duplicate MUST remain operator-visible in the logs, because the recurrence signal is the reason the dump path exists. + +#### Scenario: Repeated identical payloads are stored once + +- **GIVEN** an oversized `response.create` payload has already been dumped +- **WHEN** a retry of the byte-identical payload is dumped again +- **THEN** no additional dump pair is written +- **AND** the originally stored dump pair is retained +- **AND** the suppressed duplicate is logged with its payload fingerprint and the path of the existing dump + +#### Scenario: Distinct payloads are stored separately + +- **GIVEN** an oversized `response.create` payload has already been dumped +- **WHEN** a different oversized payload is dumped +- **THEN** a separate dump pair is written for it + +#### Scenario: Oldest dumps are pruned once the directory is full + +- **GIVEN** the dump directory already holds the maximum number of dump pairs +- **WHEN** a dump for a new payload is written +- **THEN** the oldest dump pairs are removed so the maximum is not exceeded +- **AND** each removed payload file has its meta file removed with it +- **AND** the newly written dump pair is retained + +#### Scenario: Dump retention needs no setting + +- **GIVEN** a default installation with no dump-related configuration +- **WHEN** oversized response-create dumps are captured over time +- **THEN** duplicate suppression and pruning apply +- **AND** no `CODEX_LB_*` setting is required to bound the directory + +### Requirement: External secret references support provider-native layouts + +When `externalSecrets.enabled=true`, the Helm chart MUST render an +`external-secrets.io/v1` ExternalSecret. The database URL and encryption key +MUST each accept an independent remote key and an optional JSON property. An +empty remote key MUST default to the release fullname, and the default +properties MUST preserve the existing `database-url` and `encryption-key` JSON +layout. Explicitly nulled remote reference overrides MUST render the default +layout instead of failing the template. + +#### Scenario: Existing JSON secret layout remains the default + +- **WHEN** external secrets mode is enabled without remote reference overrides +- **THEN** both target keys read from the remote secret named after the release +- **AND** they extract the `database-url` and `encryption-key` JSON properties +- **AND** the rendered ExternalSecret uses `external-secrets.io/v1` + +#### Scenario: Individual remote secrets need no JSON property + +- **WHEN** an operator configures separate absolute remote keys for the database URL and encryption key +- **AND** leaves both property values empty +- **THEN** each target key reads the complete value of its configured remote secret +- **AND** the rendered remote references omit `property` + +#### Scenario: Nulled overrides fall back to the default layout + +- **WHEN** an operator explicitly nulls `externalSecrets.remoteRefs` or one of its subtrees +- **THEN** rendering succeeds +- **AND** the affected target keys use the release fullname and their default JSON properties + +### Requirement: Helm PostgreSQL capacity guidance accounts for both application pools + +Helm sizing documentation and production-oriented values SHALL calculate maximum application PostgreSQL connections as `(databasePoolSize + databaseMaxOverflow) * 2 pooled engines * 1 supported worker * maxReplicas`. Values described as fitting PostgreSQL's default `max_connections=100` MUST reserve at least 20 raw server slots for PostgreSQL-reserved connections, the migration path's two-connection peak, administration, and transient non-application clients. + +#### Scenario: Default chart reaches its HPA ceiling + +- **WHEN** the default chart scales to `autoscaling.maxReplicas` +- **THEN** both application pools across all replicas require no more than 80 PostgreSQL connections +- **AND** at least 20 raw server slots remain outside the application-pool budget + +#### Scenario: Production overlay reaches its HPA ceiling + +- **WHEN** `values-prod.yaml` scales to `autoscaling.maxReplicas` +- **THEN** both application pools across all replicas require no more than 80 PostgreSQL connections +- **AND** at least 20 raw server slots remain available for PostgreSQL reservations, migrations, administration, and transient non-application clients + +### Requirement: Helm Grafana dashboard titles are configurable + +The Helm chart MUST allow operators to override the titles of packaged Grafana +dashboards by JSON filename. The default values MUST preserve the packaged +dashboard titles. + +#### Scenario: Operator uses concise titles in a folder hierarchy + +- **GIVEN** Grafana dashboard provisioning is enabled +- **AND** title overrides map `codex-lb.json` to `Overview` and + `ttft-breakdown.json` to `TTFT Breakdown` +- **WHEN** the chart renders the Grafana dashboard ConfigMap +- **THEN** each dashboard JSON document contains its configured title +- **AND** dashboard UIDs and all panel definitions remain unchanged + +#### Scenario: Default titles remain compatible + +- **GIVEN** Grafana dashboard provisioning is enabled +- **AND** the operator does not customize dashboard titles +- **WHEN** the chart renders the Grafana dashboard ConfigMap +- **THEN** the overview title remains `codex-lb` +- **AND** the TTFT title remains `codex-lb TTFT Breakdown` +- **AND** each ConfigMap value remains byte-identical to the chart's raw-file rendering + +### Requirement: Helm preStop shares the application drain deadline + +The Helm lifecycle hook MUST start local drain and poll its strict status. The configured routing dwell and application deadline MUST be measured from Python preStop-helper start. The hook MUST convey its helper-anchored absolute monotonic drain deadline to the loopback drain-start endpoint; that deadline-bearing request MUST commit the one-way process barrier. The application MUST reject non-finite values, clamp the supplied deadline so it cannot exceed the configured application timeout measured from receipt, and return the effective committed absolute deadline. The hook MUST validate that response and use the earlier of its local and returned deadlines. Local drain-start request latency or an earlier process deadline MUST therefore consume that single absolute budget rather than create another period. The hook MUST exit once the dwell has elapsed with `draining=true` and `in_flight=0`, or when the effective application drain deadline is exhausted. It MUST NOT add a second fixed drain period. A start, status, or status-schema failure MUST end the hook promptly so kubelet can deliver SIGTERM as the fallback, without rolling back a barrier already accepted by the application. Kubernetes termination grace MUST be documented as beginning before helper launch, with exec/Python launch latency consuming the hard grace but not restarting or shortening the helper-anchored application budget. + +#### Scenario: Routing dwell completes with no in-flight work + +- **WHEN** the Python preStop helper starts the routing dwell and status reports zero in-flight work +- **THEN** the hook waits through the routing dwell measured from helper start +- **AND** the loopback drain-start request establishes the helper-start-anchored application deadline +- **AND** local drain-start request latency does not restart that dwell +- **AND** exits without waiting through the rest of the drain timeout + +#### Scenario: Drain-start request cannot extend the deadline + +- **WHEN** the loopback drain-start request reaches the application after helper start +- **THEN** the application uses no deadline later than the hook's supplied absolute deadline +- **AND** clamps that value to no later than its configured timeout from receipt +- **AND** commits the process barrier and returns the effective deadline +- **AND** the hook bounds all later polling by that returned deadline +- **AND** rejects a non-finite supplied deadline + +#### Scenario: Work remains after routing dwell + +- **WHEN** routing dwell has elapsed and status still reports positive `in_flight` +- **THEN** the hook continues polling until `in_flight=0` or the shared deadline + +#### Scenario: Drain start or status fails + +- **WHEN** the local drain start request, status request, or status schema fails +- **THEN** preStop exits promptly with failure +- **AND** it does not blindly sleep through another timeout + +#### Scenario: Helm timing values are unsafe + +- **WHEN** `config.shutdownDrainTimeoutSeconds` is shorter than `preStopSleepSeconds` +- **OR** `terminationGracePeriodSeconds` is shorter than `config.shutdownDrainTimeoutSeconds + 32` +- **THEN** chart rendering fails with a helpful timing-contract error + +#### Scenario: Operator reads shutdown documentation + +- **WHEN** an operator inspects Helm shutdown tuning +- **THEN** documentation states that preStop and SIGTERM share one application deadline +- **AND** distinguishes the earlier Kubernetes hard-grace start from the Python helper's application-deadline start +- **AND** uses the nested `config.shutdownDrainTimeoutSeconds` values key +- **AND** warns that an old or custom `terminationGracePeriodSeconds` from a values file, `--set`, or `--reuse-values` below `config.shutdownDrainTimeoutSeconds + 32` makes Helm rendering fail before resources are applied +- **AND** states that the minimum is the configured drain timeout plus 32 seconds, is 62 seconds at the default 30-second drain timeout, and that the chart default is 65 seconds +- **AND** directs the operator to remove the override or raise it to at least the computed minimum before installing or upgrading +- **AND** states that omitting the key under `--reuse-values` retains the stored low value, so that path must set at least the computed minimum explicitly, while adopting the chart default requires an intentional non-reuse or `--reset-values` upgrade with the key absent + +### Requirement: Shipped launch paths use the pre-connection drain server + +Every shipped or documented launch path for the main application MUST delegate to the project CLI so direct SIGTERM commits the application drain barrier before Uvicorn closes connections. Development Compose MUST preserve source-watch behavior without replacing the project server with Uvicorn's reload supervisor. + +#### Scenario: Development Compose watches application source + +- **WHEN** the development Compose service is started with watch enabled +- **THEN** it launches the main application through `python -m app.cli` +- **AND** an application source sync restarts that service +- **AND** it does not launch direct Uvicorn reload + +#### Scenario: Operator follows a shipped local command + +- **WHEN** an operator follows a repository-documented command for the main application +- **THEN** that command delegates to `app.cli` +- **AND** direct SIGTERM reaches the pre-connection drain server + diff --git a/openspec/specs/deployment-networking/spec.md b/openspec/specs/deployment-networking/spec.md index 3d40f98c32..65eb698207 100644 --- a/openspec/specs/deployment-networking/spec.md +++ b/openspec/specs/deployment-networking/spec.md @@ -98,3 +98,79 @@ The default responses-ingress sticky mechanism MUST NOT rely on `nginx.ingress.k - **WHEN** the operator sets a non-empty `ingress.responses.nginx.configurationSnippet` - **THEN** the `configuration-snippet` annotation renders with the configured content + +### Requirement: Helm Gateway API routes support rule-level matches and filters + +The Helm chart MUST allow operators to configure an ordered list of HTTPRoute +rules containing Gateway API `matches` and `filters`. The chart MUST attach the +codex-lb Service backend to every configured rule. The feature MUST be optional +and preserve the existing backend-only catch-all rule when no rules are set. + +#### Scenario: Paths use different Gateway filters + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.rules` contains an unfiltered API rule matching `/v1`, + `/backend-api/codex`, `/backend-api/wham`, `/backend-api/transcribe`, + `/backend-api/files`, and `/api/codex`, followed by a filtered `/` catch-all + rule +- **WHEN** the chart renders its HTTPRoute +- **THEN** both rules retain their configured matches in order +- **AND** only the catch-all rule contains the configured filter +- **AND** both rules target the chart-managed codex-lb Service and port +- **AND** WHAM identity discovery, file-upload, and Codex usage/reset-credit + paths retain their own caller-authentication contracts instead of traversing + the dashboard filter + +#### Scenario: Empty rule configuration preserves the default route + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.rules` is empty +- **WHEN** the chart renders its HTTPRoute +- **THEN** it contains one backend-only rule targeting the chart-managed + codex-lb Service and port + +### Requirement: Helm chart can create an application-specific Gateway + +The Helm chart MUST allow operators to render a Gateway API `Gateway` +dedicated to the release in the release namespace instead of attaching to a +pre-existing shared Gateway. The mode MUST be optional and default off, +preserving the existing `gatewayApi.parentRefs` attachment. When enabled, the +chart MUST require an operator-supplied GatewayClass name, MUST default the +Gateway to a single HTTP listener on port 80 while honoring operator-defined +listeners verbatim, and MUST attach the chart-managed HTTPRoute to the +chart-managed Gateway while ignoring `gatewayApi.parentRefs`. + +#### Scenario: Chart-managed Gateway with default listener + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.gateway.create=true` with a GatewayClass name +- **WHEN** the chart renders its Gateway API resources +- **THEN** a Gateway named after the release renders in the release namespace + with the configured GatewayClass and one HTTP listener on port 80 +- **AND** the HTTPRoute's only parent reference is the chart-managed Gateway + +#### Scenario: Operator-defined listeners + +- **GIVEN** `gatewayApi.gateway.create=true` with a GatewayClass name +- **AND** `gatewayApi.gateway.listeners` contains an HTTPS listener with TLS + configuration +- **WHEN** the chart renders the Gateway +- **THEN** the configured listeners replace the default HTTP listener verbatim + +#### Scenario: Missing GatewayClass name fails rendering + +- **GIVEN** `gatewayApi.gateway.create=true` +- **AND** `gatewayApi.gateway.gatewayClassName` is empty +- **WHEN** the chart renders +- **THEN** rendering fails with an error naming + `gatewayApi.gateway.gatewayClassName` + +#### Scenario: Default configuration keeps existing Gateway attachment + +- **GIVEN** `gatewayApi.enabled=true` +- **AND** `gatewayApi.gateway.create` is unset +- **WHEN** the chart renders its Gateway API resources +- **THEN** no Gateway resource renders +- **AND** the HTTPRoute attaches to the operator-supplied + `gatewayApi.parentRefs` + diff --git a/openspec/specs/fleet-summary/spec.md b/openspec/specs/fleet-summary/spec.md index d4f7ef2955..5689576ce4 100644 --- a/openspec/specs/fleet-summary/spec.md +++ b/openspec/specs/fleet-summary/spec.md @@ -72,7 +72,13 @@ sticky-session account distribution. ### Requirement: Fleet summary requires API key authentication -The system SHALL expose `GET /api/fleet/summary` for trusted local fleet consumers. The route MUST require a valid Bearer API key even when global proxy API-key authentication is disabled. For callers allowed to view upstream usage, each account SHALL expose `lastRefreshAt` as OAuth token freshness and `usageRefreshedAt` as quota-snapshot freshness. `usageRefreshedAt` MUST equal the newest `recorded_at` value among the persisted usage samples used to build that account summary, or `null` when no such sample exists. +The system SHALL expose `GET /api/fleet/summary` for trusted local fleet +consumers. The route MUST require a valid Bearer API key even when global proxy +API-key authentication is disabled. For callers allowed to view upstream +usage, each account SHALL expose `lastRefreshAt` as OAuth token freshness and +`usageRefreshedAt` as quota-snapshot freshness. `usageRefreshedAt` MUST equal +the newest `recorded_at` value among the persisted usage samples used to build +that account summary, or `null` when no such sample exists. #### Scenario: Missing fleet summary key is rejected @@ -84,13 +90,15 @@ The system SHALL expose `GET /api/fleet/summary` for trusted local fleet consume - **WHEN** a client calls `GET /api/fleet/summary` with a valid Bearer API key - **THEN** the response includes `accounts[]` -- **AND** each account includes `accountId`, `displayName`, `email`, `status`, `planType`, `primary`, `secondary`, `lastRefreshAt`, and `usageRefreshedAt` +- **AND** each account includes `accountId`, `displayName`, `email`, `status`, + `planType`, `primary`, `secondary`, `lastRefreshAt`, and `usageRefreshedAt` - **AND** each window includes `remainingPercent`, `resetAt`, and `windowMinutes` #### Scenario: Usage refresh advances independently of OAuth refresh - **GIVEN** an account has an existing quota snapshot and OAuth refresh time -- **WHEN** force probe or fleet refresh persists a newer usage sample without refreshing OAuth credentials +- **WHEN** force probe or fleet refresh persists a newer usage sample without + refreshing OAuth credentials - **THEN** `usageRefreshedAt` advances to the newer usage sample time - **AND** `lastRefreshAt` remains unchanged @@ -132,3 +140,48 @@ The route MUST preserve existing usage-refresh rules for disabled refresh, fresh - **WHEN** a valid client calls `POST /api/fleet/refresh` - **THEN** active accounts are eligible for the refresh attempt - **AND** paused, reauth-required, and deactivated accounts are not attempted + +### Requirement: Fleet refreshes participate in graceful shutdown + +The system MUST strongly own every accepted `POST /api/fleet/refresh` task from creation until its dedicated session has finished and closed, regardless of whether its caller remains attached. Task creation and registry insertion MUST occur synchronously before the route first awaits the task. Graceful shutdown MUST wait for all such tracked refreshes for up to `shutdown_drain_timeout_seconds` before stopping usage-refresh singleflight work or closing shared HTTP and database resources. If the deadline expires, the system MUST report each fleet refresh that did not drain before continuing shutdown. + +#### Scenario: Caller cancellation does not orphan fleet refresh work + +- **GIVEN** a fleet refresh is running in its dedicated session +- **WHEN** the requesting client disconnects or its request task is cancelled +- **THEN** the refresh continues independently of the cancelled caller +- **AND** it remains tracked until its session exits + +#### Scenario: Shutdown begins before caller cancellation + +- **GIVEN** a fleet refresh was accepted and its caller remains attached +- **WHEN** the in-flight drain times out and graceful shutdown starts draining fleet tasks +- **THEN** the refresh is already present in the fleet task registry +- **AND** cancelling the caller afterward does not remove the refresh from shutdown ownership + +#### Scenario: Shutdown waits for a detached fleet refresh + +- **GIVEN** a cancelled-request fleet refresh is still pending when graceful shutdown begins +- **WHEN** the refresh completes within the configured drain timeout +- **THEN** shutdown waits for the refresh +- **AND** usage singleflight, shared HTTP clients, and database engines remain available until it finishes + +#### Scenario: Overdue fleet refresh is reported + +- **GIVEN** a detached fleet refresh remains pending for the full configured drain timeout +- **WHEN** graceful shutdown drains fleet tasks +- **THEN** the drain reports that task as overdue +- **AND** shutdown is allowed to continue + +### Requirement: Post-cutoff fleet refreshes are rejected before resource work + +Immediately after the in-flight drain attempt returns, graceful shutdown MUST synchronously close fleet task admission before any further shutdown await. A `POST /api/fleet/refresh` request that reaches its producer after this cutoff MUST return the dashboard `503 service_unavailable` error envelope and MUST NOT create a refresh coroutine, task, background session, or other refresh resource work. + +#### Scenario: Late fleet producer receives service unavailable + +- **GIVEN** graceful shutdown has closed control-plane task admission +- **WHEN** an authenticated caller requests `POST /api/fleet/refresh` +- **THEN** the response status is 503 +- **AND** the dashboard error code is `service_unavailable` +- **AND** no fleet refresh task or background session starts + diff --git a/openspec/specs/frontend-architecture/spec.md b/openspec/specs/frontend-architecture/spec.md index 0da0717919..87176c2129 100644 --- a/openspec/specs/frontend-architecture/spec.md +++ b/openspec/specs/frontend-architecture/spec.md @@ -929,30 +929,52 @@ selected value using the settings API field `preferEarlierResetWindow`. ### Requirement: Dashboard account cards show live credit state -Account summary responses SHALL expose the latest upstream credit metadata for -each account as nullable `creditsHas`, `creditsUnlimited`, and `creditsBalance` -fields. The dashboard account schema SHALL accept those fields. +Account summary responses SHALL expose nullable upstream purchased-credit metadata as `creditsHas`, `creditsUnlimited`, and `creditsBalance`, alongside calculated remaining subscription credits for each available quota window. Dashboard card and list views MUST present calculated subscription quota and purchased credits as separate labeled metrics and MUST NOT use one as a fallback replacement for the other. -The dashboard account card SHALL render a compact Credits row. If -`creditsUnlimited` is true, the value SHALL be `Unlimited`. Otherwise, when a -numeric credit balance is available it SHALL render that balance. If no credit -balance is available, the card MAY fall back to the account's remaining weekly -or primary credit value, and SHALL render `-` when no credit value is known. +When `creditsUnlimited` is true, the purchased-credit metric SHALL render `Unlimited`. Otherwise, it SHALL render the numeric `creditsBalance` when available and `-` when unavailable. The subscription metric SHALL select remaining credits with the following precedence: monthly credits for monthly-only accounts; secondary credits for weekly-only accounts; otherwise secondary credits when available, falling back to primary credits. It SHALL render `-` when the selected value is unavailable. + +The compact list SHALL sort subscription and purchased credits independently. A persisted legacy `credits` sort preference SHALL migrate to the purchased-credit sort so existing operator preferences remain valid after upgrade. + +#### Scenario: Zero purchased balance does not hide subscription quota + +- **WHEN** an account summary has `creditsBalance = 0.0` +- **AND** `remainingCreditsSecondary = 35910.0` +- **THEN** the dashboard shows subscription quota `35910.00` +- **AND** separately shows purchased credits `0.00` + +#### Scenario: Unlimited applies only to purchased credits + +- **WHEN** an account summary has `creditsUnlimited = true` +- **THEN** the purchased-credit metric shows `Unlimited` +- **AND** the subscription metric still shows its own remaining quota value or `-` + +#### Scenario: Missing metrics render independently + +- **WHEN** an account summary has no purchased credit balance and no calculated remaining subscription credits +- **THEN** both separately labeled metrics show `-` #### Scenario: Unlimited credits render explicitly - **WHEN** an account summary has `creditsUnlimited = true` -- **THEN** the dashboard account card shows `Credits: Unlimited` +- **THEN** the dashboard account card shows purchased credits as `Unlimited` +- **AND** the subscription quota remains independently visible #### Scenario: Positive credit balance renders on the card - **WHEN** an account summary includes `creditsBalance = 1.5` -- **THEN** the dashboard account card shows that numeric credit balance +- **THEN** the dashboard account card shows purchased credits as `1.50` +- **AND** does not replace the subscription quota value #### Scenario: Missing credit data renders a placeholder -- **WHEN** an account summary has no credit balance and no remaining credit fallback -- **THEN** the dashboard account card shows `Credits: -` +- **WHEN** an account summary has no purchased credit balance and no remaining subscription credit value +- **THEN** the dashboard account card shows `-` for both separately labeled metrics + +#### Scenario: Legacy credit sort remains valid + +- **WHEN** local dashboard preferences contain the legacy `credits` sort key +- **THEN** the dashboard migrates it to the purchased-credit sort key +- **AND** persists the migrated preference ### Requirement: Dashboard settings must expose upstream proxy routing controls The settings dashboard MUST allow operators to inspect upstream proxy routing state, enable or disable routing, choose the default proxy pool, create proxy endpoints, create proxy pools, and add endpoints to pools. @@ -2383,3 +2405,946 @@ unchanged. - **GIVEN** the dashboard principal has role `admin` - **WHEN** the dashboard view selector opens - **THEN** it exposes both Request Logs and Conversations + +### Requirement: Dashboard routes are code-split + +Each dashboard route's page component MUST load lazily so the entry chunk excludes the code of pages the operator has not visited; the built entry chunk MUST NOT statically import or modulepreload page chunks. + +#### Scenario: Entry chunk excludes unvisited pages + +- **WHEN** the dashboard entry page loads +- **THEN** only the visited route's page chunk is fetched +- **AND** the built entry chunk neither statically imports nor modulepreloads the other pages' chunks + +### Requirement: Dashboard assets are fully self-hosted + +The dashboard MUST NOT load fonts or other render-blocking resources from external origins; all font assets ship with the build and declare `font-display: swap`. + +#### Scenario: No external origins in the built shell + +- **WHEN** the dashboard shell is built +- **THEN** `index.html` and the emitted assets reference no external font or stylesheet origins + +#### Scenario: First paint proceeds without network egress + +- **GIVEN** a deployment without outbound internet access +- **WHEN** an operator loads the dashboard +- **THEN** first paint is not blocked on any external request and monospace text renders via the bundled font or the system fallback + +### Requirement: Dashboard supports Korean runtime locale + +The dashboard SHALL support Korean (`ko`) as a runtime locale in addition to +English (`en`) and Simplified Chinese (`zh-CN`). Korean language detection SHALL +select `ko` for browser language tags whose base language is `ko`, and the +language switcher SHALL let users choose Korean without reloading the page. + +#### Scenario: First visit with a Korean browser + +- **WHEN** a user opens the dashboard for the first time with `navigator.language = "ko-KR"` and no persisted preference +- **THEN** the dashboard renders the translated in-scope surface in Korean +- **AND** `localStorage` contains `codex-lb-language=ko` +- **AND** `document.documentElement.lang` is set to `ko` + +#### Scenario: User toggles Korean + +- **WHEN** the user activates the language switcher and selects Korean +- **THEN** the dashboard re-renders translated strings in Korean without a full page reload +- **AND** the selected language persists across reloads + +### Requirement: Dashboard feature surfaces render in the active locale + +Dashboard feature surfaces SHALL render user-visible copy through the active +i18n locale, including page headings, section headings, empty states, table +headings, filter labels, button labels, accessible labels, dialog titles, +dialog descriptions, validation messages, and client-side toast fallback copy. +This requirement applies to Accounts, Dashboard, API Keys, APIs, Reports, +Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, Settings +subsections, and shared dashboard components. + +The dashboard MAY keep protocol names, product names, model/API terminology, +quota window abbreviations, and compact operational abbreviations in English +when the English form is the clearest operator-facing label. + +#### Scenario: Korean feature page rendering + +- **WHEN** a user selects `ko` +- **AND** opens Accounts, Dashboard, API Keys, APIs, Reports, Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, or Settings subsections +- **THEN** user-visible labels, headings, empty states, dialog copy, accessible labels, and client-side toast fallback copy render in Korean +- **AND** technical terms such as `API Key`, `Model`, `TOTP`, `OAuth`, `TTFT`, `TPS`, and `Fast Mode` MAY remain English where appropriate + +#### Scenario: Simplified Chinese feature page rendering + +- **WHEN** a user selects `zh-CN` +- **AND** opens a dashboard feature page beyond the original auth/header/settings coverage +- **THEN** newly migrated user-visible strings render in Simplified Chinese +- **AND** the page does not fall back to English because a locale key is missing + +#### Scenario: Locale bundles stay in sync + +- **WHEN** the frontend locale bundles are compared +- **THEN** `en`, `zh-CN`, and `ko` expose the same translation keys + +### Requirement: Reports endpoint rejects inverted date ranges before repository work + +After applying defaults for any omitted date bound, the Reports service MUST reject a `start_date` later than `end_date` before converting report boundaries or awaiting any repository operation. `GET /api/reports` MUST map that domain failure to HTTP 400 with the exact dashboard envelope `{"error":{"code":"invalid_report_date_range","message":"start_date must be on or before end_date"}}`. Valid one-day ranges and valid inclusive ranges of 730 calendar days MUST remain accepted. + +#### Scenario: Explicit inverted Reports range is rejected + +- **WHEN** an authenticated operator requests `GET /api/reports` with `start_date` later than `end_date` +- **THEN** the endpoint returns HTTP 400 +- **AND** the response body is exactly `{"error":{"code":"invalid_report_date_range","message":"start_date must be on or before end_date"}}` +- **AND** the Reports repository receives no call + +#### Scenario: Defaulted end date makes the range inverted + +- **WHEN** an authenticated operator requests `GET /api/reports` with an explicit `start_date` later than the defaulted current `end_date` +- **THEN** the endpoint returns the same `invalid_report_date_range` HTTP 400 before repository work + +#### Scenario: Boundary-valid Reports ranges remain accepted + +- **WHEN** an authenticated operator requests a one-day range whose `start_date` equals `end_date` +- **THEN** the endpoint accepts the request and reports data for that day +- **WHEN** an authenticated operator requests an inclusive range of exactly 730 calendar days +- **THEN** the endpoint accepts the request under the existing range limit + +### Requirement: Reports date controls prevent, explain, and recover from inverted ranges + +The `/reports` start-date input MUST use the earlier of the browser-local current day and a present end date as its native `max`, and the end-date input MUST use a present start date as its native `min` while retaining the browser-local current day as its `max`. If both values are present and the start date is later than the end date, both controls MUST expose `aria-invalid`, both MUST reference the same localized inline corrective message through an accessible description, and neither the filtered Reports query nor the relaxed Reports filter-catalog query MAY send a request. Correcting either bound so the range is ordered MUST clear the invalid state and resume each distinct Reports query with the corrected bounds. + +#### Scenario: Reciprocal native bounds prevent routine inverted selection + +- **GIVEN** `/reports` has a selected start date and end date +- **THEN** the start-date control's `max` is the earlier of the end date and the browser-local current day +- **AND** the end-date control's `min` is the start date +- **AND** the end-date control's `max` remains the browser-local current day + +#### Scenario: Bypassed inverted input is accessible and sends no Reports request + +- **WHEN** typed, restored, or programmatically supplied Reports dates have a start date later than the end date +- **THEN** both date controls expose `aria-invalid` +- **AND** both controls reference one visible localized message that tells the operator to place the start date on or before the end date +- **AND** no `GET /api/reports` request is sent for either Reports query + +#### Scenario: Retry while inverted only retries Accounts + +- **GIVEN** `/reports` has an inverted date range and loading account options failed +- **WHEN** the operator activates the page-level Retry action +- **THEN** the Accounts query sends a retry request +- **AND** neither Reports query sends a request + +#### Scenario: Correcting either invalid bound resumes Reports queries + +- **GIVEN** `/reports` has an inverted date range and both Reports queries are disabled +- **WHEN** the operator corrects either date bound so the start date is on or before the end date +- **THEN** both controls clear the invalid state and accessible description +- **AND** the corrective message is removed +- **AND** each distinct Reports query sends one request using the corrected ordered bounds + +### Requirement: Dashboard overview and request-log listing fail independently + +The Dashboard SHALL gate overview-backed statistics, quota, projections, and account controls only on dashboard overview availability. The Request Logs section SHALL own the initial loading, terminal error, and ready states of its listing query without hiding healthy overview-backed content. + +When the initial request-log listing reaches a terminal error, the Request Logs section MUST remain visible, MUST render the listing error inside that section, MUST announce that error through an alert semantic local to the section, and MUST expose a keyboard-operable, accessibly named Retry action. Activating Retry MUST refetch only the request-log listing query and MUST NOT refetch or hide healthy overview-backed content. + +#### Scenario: Initial request-log failure preserves healthy overview + +- **GIVEN** dashboard overview, projections, and request-log filter options load successfully +- **WHEN** the initial request-log listing reaches a terminal error +- **THEN** overview statistics, quota, and account content remain rendered +- **AND** the page-wide Dashboard loading skeleton is not rendered +- **AND** the Request Logs section contains and announces the listing error and exposes a Retry action + +#### Scenario: Request-log retry recovers independently + +- **GIVEN** healthy overview-backed content is rendered and the initial request-log listing has failed +- **WHEN** the listing endpoint recovers and the operator activates Retry +- **THEN** only the request-log listing query is refetched +- **AND** healthy overview-backed content remains visible throughout recovery +- **AND** the recovered request-log rows render in the Request Logs section + +#### Scenario: Request logs load inside their section + +- **GIVEN** dashboard overview data is available +- **WHEN** the initial request-log listing is still pending +- **THEN** overview-backed content is rendered +- **AND** the Request Logs section renders its own loading state +- **AND** the page-wide Dashboard loading skeleton is not rendered + +#### Scenario: Initial overview loading keeps the existing page skeleton + +- **WHEN** the dashboard overview is not yet available +- **THEN** the Dashboard renders its existing page-wide loading skeleton +- **AND** it does not render overview-backed content prematurely + +### Requirement: App header brand links to dashboard + +The app header brand area SHALL render a `` wrapping the +logo and "Codex LB" text so that clicking the brand navigates back to the +dashboard home page. The link SHALL preserve the existing visual layout (logo +size, gradient background, text styling) and SHALL include keyboard +focus-visible ring styling matching the project's existing interactive-element +conventions. + +#### Scenario: Brand click navigates to dashboard + +- **WHEN** an operator clicks the header brand area (logo or "Codex LB" text) +- **THEN** the SPA navigates to `/dashboard` + +#### Scenario: Brand link is keyboard-accessible + +- **WHEN** an operator tabs to the header brand +- **THEN** the brand area receives a visible focus ring +- **AND** pressing Enter navigates to `/dashboard` + +#### Scenario: Brand link preserves visual appearance + +- **WHEN** the header renders +- **THEN** the logo and "Codex LB" text appear visually identical to the prior + non-interactive `
` layout + +### Requirement: Dashboard metrics expose conversation-bearing requests + +The dashboard overview response MUST expose +`summary.metrics.conversationRequests` as the count of non-warmup request-log +rows in the selected timeframe whose trimmed `conversation_id` is nonblank. +The existing `requests` field MUST continue counting all non-warmup rows, and +the existing `conversations` field MUST continue counting distinct nonblank +conversation IDs in that timeframe. + +#### Scenario: Requests without conversation IDs are excluded from the new count + +- **GIVEN** a timeframe contains four requests with nonblank conversation IDs + and two requests with null or whitespace-only IDs +- **WHEN** the dashboard overview is requested +- **THEN** `conversationRequests` is `4` +- **AND** `requests` includes all six non-warmup requests + +### Requirement: Dashboard conversation card shows the filtered average + +The dashboard conversation card MUST be labeled `Active Conversations` with +the selected timeframe, and its secondary metadata MUST show `Avg req/conv` +followed by `conversationRequests / conversations`, formatted to one decimal +place. When `conversations` is zero, the metadata MUST show an em dash instead +of dividing by zero. + +#### Scenario: Average uses only conversation-bearing requests + +- **GIVEN** `conversationRequests` is `5` and `conversations` is `2` +- **WHEN** the dashboard card is rendered +- **THEN** its metadata shows `Avg req/conv 2.5` + +#### Scenario: Average is safe when no conversations exist + +- **GIVEN** `conversationRequests` is `4` and `conversations` is `0` +- **WHEN** the dashboard card is rendered +- **THEN** its metadata shows `Avg req/conv —` + +### Requirement: Dashboard and report labels identify active conversations + +The dashboard and report conversation summary cards MUST use the localized +equivalent of `Active Conversations`; their numeric values and ordering MUST +remain unchanged. The report card MUST NOT gain the dashboard average. + +#### Scenario: Report uses the active-conversation label + +- **WHEN** the report summary cards render +- **THEN** the conversation card label is `Active Conversations` in English +- **AND** its numeric value remains the existing distinct conversation total +- **AND** no `Avg req/conv` metadata is rendered on the report card + +### Requirement: Simplified Chinese locale bundle covers all dashboard keys + +The `zh-CN` locale bundle SHALL provide an entry for every user-visible i18n +key present in the `en` bundle, so no dashboard surface falls back to English +because of a missing key. Values MAY keep protocol names, product names, +model/API terminology, and compact operational abbreviations in English when +the English form is the clearest operator-facing label. + +#### Scenario: zh-CN rendering without English fallback + +- **WHEN** a user selects `zh-CN` +- **AND** opens Accounts, Dashboard, API Keys, APIs, Reports, Automations, Firewall, Model Sources, Quota Planner, Sticky Sessions, Upstream Proxy, or Settings subsections +- **THEN** user-visible labels, headings, empty states, dialog copy, accessible labels, and client-side toast fallback copy render through the `zh-CN` bundle +- **AND** no string falls back to English because of a missing locale key +- **AND** technical terms such as `API Key`, `Model`, `OAuth`, `TOTP`, `Credits`, and `Quota` MAY remain English where appropriate + +### Requirement: zh-CN terminology stays consistent across feature surfaces + +Translated `zh-CN` strings SHALL reuse established dashboard terminology for +repeated concepts, and labels that sit inside a label group whose siblings are +already translated SHALL render in Simplified Chinese as well. + +#### Scenario: Consistent wording for repeated concepts + +- **WHEN** a concept already has an established `zh-CN` translation on one surface (e.g. 账户消耗预测 in the settings appearance section) +- **THEN** other surfaces referencing the same concept reuse that wording instead of introducing a synonym + +#### Scenario: Mixed-label groups render fully in Chinese + +- **WHEN** a filter group or table header contains several labels and some already render in Simplified Chinese (e.g. 状态, 类型) +- **THEN** the remaining labels in that group render in Simplified Chinese (e.g. 触发方式) instead of falling back to English + +### Requirement: Dashboard numeric units stay locale-independent + +Dashboard quantities that use compact formatting SHALL use `K`, `M`, and `B` +suffixes regardless of the selected interface locale so requests, tokens, +balances, pool totals, projections, and configured thresholds remain directly +comparable. Dashboard USD values SHALL use the `$` prefix across locales. + +#### Scenario: Simplified Chinese compact quantity display + +- **WHEN** a user selects `zh-CN` +- **AND** views compact request, token, or credit quantities +- **THEN** 10,200 renders as `10.2K` +- **AND** 1,500,000 renders as `1.5M` +- **AND** 1,500,000,000 renders as `1.5B` +- **AND** 12 USD renders as `$12.00` + +### Requirement: Reports per-day averages use the inclusive local calendar window + +`GET /api/reports` MUST calculate `summary.avgCostPerDay` and +`summary.avgRequestsPerDay` by dividing the current report totals by exactly +`(end_date - start_date).days + 1`. The divisor MUST represent the selected +inclusive local calendar-date window and MUST NOT be derived from the +UTC-converted filter boundaries. + +#### Scenario: Offset-to-zero transition keeps a two-day divisor + +- **WHEN** an operator requests `2026-02-15` through `2026-02-16` in + `Africa/Casablanca` and the report totals are 60 cost units and 30 requests +- **THEN** `avgCostPerDay` is `30` +- **AND** `avgRequestsPerDay` is `15` + +#### Scenario: Offset-from-zero transition keeps a two-day divisor + +- **WHEN** an operator requests `2026-03-22` through `2026-03-23` in + `Africa/Casablanca` and the report totals are 60 cost units and 30 requests +- **THEN** `avgCostPerDay` is `30` +- **AND** `avgRequestsPerDay` is `15` + +### Requirement: Dashboard status separates service readiness from usage synchronization + +The fixed dashboard status bar MUST render independent `Service ready` and +`Usage synced` signals. `Service ready` MUST use the existing `/health/ready` +response and MUST treat a failed request or a non-`ok` status as not ready. +`Usage synced` MUST remain derived only from the dashboard overview +`lastSyncAt` value and MUST be fresh only while that timestamp is less than 60 +seconds old. The service-readiness signal MUST NOT use upstream account or +provider health. The dashboard layout MUST reserve at least the status bar's +rendered height so wrapped status rows do not cover page content. + +#### Scenario: Ready service with stale usage + +- **WHEN** `/health/ready` returns `status: "ok"` +- **AND** `lastSyncAt` is absent or at least 60 seconds old +- **THEN** the status bar shows the service as ready +- **AND** independently shows usage as stale + +#### Scenario: Unready service with fresh usage + +- **WHEN** `/health/ready` fails or returns a non-`ok` status +- **AND** `lastSyncAt` is less than 60 seconds old +- **THEN** the status bar shows the service as not ready +- **AND** independently shows usage as synced + +#### Scenario: Readiness is still being checked + +- **WHEN** the initial `/health/ready` request has not completed +- **THEN** the service-readiness signal shows a checking state +- **AND** the usage synchronization signal remains independently derived from + `lastSyncAt` + +#### Scenario: Status signals wrap onto additional rows + +- **WHEN** the fixed status bar grows because its signals wrap +- **THEN** the dashboard updates its reserved bottom space to the rendered + status-bar height +- **AND** the fixed status bar does not cover page content + +### Requirement: Dashboard conversation listing + +The authenticated dashboard MUST expose `GET /api/conversations`. The list +endpoint MUST accept `limit`, `offset`, `search`, `since`, and `timeframe` query +parameters. The server-authoritative `timeframe` parameter MUST accept `1d`, +`7d`, or `30d`; when it is supplied, the server MUST derive the activity window +from the shared dashboard timeframe configuration and the client MUST NOT +substitute a browser-clock-generated `since` value. `timeframe` and `since` MUST +not be supplied together. When `since` is omitted, the server MUST apply a +rolling 30-day lower bound; +explicitly older `since` values MUST be capped at that same bound, and incoming +timezone-aware datetimes MUST be normalized to naive UTC before querying. It +MUST aggregate eligible `request_logs` rows by the raw, non-empty +`conversation_id` column, excluding rows whose request kind is `warmup` or +`limit_warmup`, and rows with `deleted_at IS NOT NULL`. Production request-log +writes MUST normalize ASCII padding and blank conversation IDs before storage; +conversation list, facet, and detail queries MUST use raw-column +`conversation_id` predicates and grouping rather than function-wrapped +expressions. + +Search MUST be case-insensitive and match the normalized conversation ID or any +eligible row's user-agent family. Search MUST select whole conversations first: +after a conversation matches, aggregation MUST include all eligible rows in that +conversation, including rows whose user-agent family or ID did not match the +search text. The endpoint MUST derive aggregates from `request_logs` only. + +When `since` is provided, a conversation MUST be selected when at least one +eligible row has `requested_at >= since`. A conversation MAY have eligible rows +before `since` and MUST still be included when it has activity in the window. +The grouped summary MUST aggregate all eligible rows for every selected +conversation, so `firstRequest`, `lastRequest`, `requestCount`, token totals, +cached-token totals, and cost MUST NOT be clipped to the window. Membership MUST +be implemented as an in-window aggregate condition and MUST NOT use a global +pre-window ID set or a pre-window anti-join. + +After page membership is selected, the account, API-key, and model facet +queries for the returned page MUST use the same full eligible-row scope as the +summary, restricted only by the selected page's raw `conversation_id` values. +The facet queries MUST NOT add a `requested_at >= since` restriction after +membership selection; facet representatives and remaining counts MUST include +eligible history before `since` and MUST remain consistent with the full-history +summary aggregates. + +The response MUST contain `conversations`, `total`, and `hasMore` pagination +fields. Each row in `conversations` MUST contain exactly these fields and no +response summary object: + +- `conversationId`: the normalized, non-empty conversation identity. +- `firstRequest`: the earliest `requested_at` among all eligible rows in the + conversation. +- `lastRequest`: the latest `requested_at` among all eligible rows in the + conversation. +- `requestCount`: the number of eligible rows in the conversation. +- `representativeAccount` and `remainingAccountCount`. +- `apiKeyId` and `apiKeyName`. +- `representativeModel` and `remainingModelCount`. +- `totalTokens`. +- `cachedInputTokens`. +- `totalCostUsd`. + +The camelCase names above are the external Dashboard API JSON contract. Python +schema, service, and repository identifiers MAY remain snake_case internally; +internal names MUST NOT be emitted as alternate response fields. + +`totalTokens` MUST equal total input tokens plus total output tokens, with +`reasoning_tokens` used for a row when `output_tokens` is null. +`cachedInputTokens` MUST use the existing per-row clamp: null remains null; +otherwise the cached value is clamped to `[0, input_tokens]` when input tokens +are present. At aggregate level, null per-row values MUST NOT be converted to +zero; when every eligible row has a null cached value, `cachedInputTokens` MUST +be null, and otherwise it MUST equal the sum of the known clamped values. + +Representative account values MUST use `request_count DESC, +latest_requested_at DESC, lexical account ASC`. List model values MUST be +grouped by distinct model, combining all reasoning efforts for that model, and +the representative model MUST use `request_count DESC, latest_requested_at DESC, +model lexical ASC`. Null account values MUST be excluded from account +candidates; if no non-null account exists, `representativeAccount` MUST be null +and `remainingAccountCount` MUST be 0. The list MUST NOT split model values by +`reasoning_effort`; `(model, reasoning_effort)` grouping MUST be used only for +conversation details. + +Nullable and multiple-key conversations MUST be handled deterministically. Null +API-key values MUST not be candidates; if no non-null key exists, both API-key +fields MUST be null. When multiple distinct non-null keys exist, `apiKeyId` MUST be selected by +`request_count DESC, latest_requested_at DESC, lexical API-key ID ASC`, and +`apiKeyName` MUST be the corresponding existing dashboard-safe display name. +`apiKeyName` MUST never expose a secret, hash, or plaintext key material. + +The list order MUST be stable: `lastRequest DESC`, then normalized +`conversationId ASC`. Pagination MUST be applied after this ordering. + +#### Scenario: Pagination uses the stable list order + +- **GIVEN** matching conversations have different latest request times and a + tie exists on `lastRequest` +- **WHEN** the client calls `GET /api/conversations?limit=10&offset=20` +- **THEN** rows are ordered by `lastRequest DESC` and ties by normalized + `conversationId ASC` +- **AND** the response starts at the 21st row in that order and reports the + matching total and whether another page exists + +#### Scenario: Blank IDs, warmups, and soft-deleted rows are excluded + +- **GIVEN** request logs include null IDs, whitespace-only IDs, `warmup` rows, + `limit_warmup` rows, soft-deleted rows, and eligible rows with non-empty IDs +- **WHEN** the client calls `GET /api/conversations` +- **THEN** only rows whose request kind is neither `warmup` nor `limit_warmup`, + which are non-soft-deleted and have non-empty normalized IDs, contribute to + returned conversations + +#### Scenario: Search selects whole conversations + +- **GIVEN** one eligible conversation contains a matching user-agent family on + one row and non-matching user-agent/ID values on other rows +- **WHEN** the client calls `GET /api/conversations?search=opencode` +- **THEN** that conversation is selected +- **AND** all eligible rows in that conversation contribute to its counts, + tokens, cached tokens, and cost +- **AND** rows from conversations with no matching ID or user-agent family are + not returned + +#### Scenario: List search is case-insensitive over normalized IDs and user-agent families + +- **GIVEN** an eligible conversation has a normalized ID and user-agent family + whose letters differ in case from the search text +- **WHEN** the client calls `GET /api/conversations?search=OPENCODE` +- **THEN** the conversation is selected when either the normalized ID or any + eligible row's user-agent family matches case-insensitively + +#### Scenario: Since filter selects conversations active in the window + +- **GIVEN** conversation `conv-old` has its earliest eligible row at `t-10d` + and a later row at `t-1d`, and conversation `conv-new` has its earliest + eligible row at `t-1d` +- **WHEN** the client calls `GET /api/conversations?since=` +- **THEN** both `conv-new` and `conv-old` are returned +- **AND** `conv-old` is included because it has a row inside the window even + though its first message predates the window +- **AND** both conversations' summaries aggregate every eligible row, not only + rows at or after `since` + +#### Scenario: Since membership and facets share the full conversation scope + +- **GIVEN** a selected conversation has eligible account, API-key, and model + values both before and after the `since` boundary +- **WHEN** the client calls `GET /api/conversations?since=` +- **THEN** `firstRequest`, `lastRequest`, `requestCount`, and summary totals + include all eligible rows for the conversation +- **AND** account, API-key, and model facet counts and representatives include + all eligible rows in the selected conversation, including rows before `since` + +#### Scenario: Since filter composes with search and pagination + +- **GIVEN** two conversations have activity inside the `since` window and only + one matches the search text +- **WHEN** the client calls `GET /api/conversations?since=&search=opencode` +- **THEN** only the matching conversation is returned +- **AND** the response total and hasMore reflect the since-and-search filtered + set + +#### Scenario: List model representatives ignore reasoning effort + +- **GIVEN** a conversation has requests for the same model with multiple + reasoning-effort values and requests for another model +- **WHEN** the client calls `GET /api/conversations` +- **THEN** the list groups the same model's requests into one model value +- **AND** the representative model is ordered by request count descending, + latest request descending, and model lexical ascending +- **AND** the remaining model count counts distinct models, not model/effort + combinations + +#### Scenario: API-key representation is safe and deterministic + +- **GIVEN** a conversation has null API-key rows and multiple non-null API-key + values with tied counts +- **WHEN** the client calls `GET /api/conversations` +- **THEN** null values do not become the representative +- **AND** the non-null representative is selected by count, latest request, and + lexical API-key ID +- **AND** the response contains only the corresponding dashboard-safe name and + never secret, hash, or plaintext key material + +### Requirement: Dashboard conversation activity uses the list eligibility scope + +The dashboard overview conversation metrics and per-bucket conversation trend +MUST use the same eligible `request_logs` row scope as the conversation list: +non-empty conversation IDs, request kinds other than `warmup` and +`limit_warmup`, and `deleted_at IS NULL`. This scope MUST apply to both the +distinct conversation count and conversation request count in the overview +summary and to each conversation trend bucket. + +#### Scenario: Soft-deleted-only conversations are absent from dashboard activity + +- **GIVEN** the selected timeframe contains an eligible conversation and a + second conversation whose only rows are soft-deleted +- **WHEN** the client requests the conversation list and dashboard overview for + that timeframe +- **THEN** the list total and summary conversation count include only the + eligible conversation +- **AND** the summary conversation request count and conversation trend contain + no contribution from the soft-deleted-only conversation + +### Requirement: Conversation listing total is served from a short-TTL cache + +The grouped `total` returned by `GET /api/conversations` is display-only +pagination metadata that tolerates short staleness, and the dashboard polls the +endpoint every 30 seconds. Recomputing the grouped count over the full eligible +`request_logs` history on every poll risks the same dashboard-induced +database contention this repository has previously optimized away. + +The conversation listing total MUST be served from the same short-TTL +per-filter-signature cache as the request-log listing total (fixed 30 s TTL +application constant; bounded LRU-ish eviction; per-instance). The cache +signature MUST include every dimension that changes the grouped count: `search` +and the semantic window identity MUST be included, using +`("timeframe", timeframe)` for server-authoritative timeframe requests and +`("since", effective_since)` for legacy `since` requests. `limit` and `offset` +MUST be excluded from the signature because the total is page-independent. Two +requests with different search text or window identities MUST NOT reuse one +another's cached total. + +#### Scenario: Repeated polls reuse the cached conversation total + +- **GIVEN** the conversation listing has computed a total for a given + `search` and semantic window signature +- **WHEN** the dashboard polls the same endpoint within the TTL with the same + signature +- **THEN** the grouped count MUST NOT be recomputed +- **AND** the response total MUST equal the previously computed value + +#### Scenario: Different window signatures isolate cached conversation totals + +- **GIVEN** two listing requests differ only by their timeframe or legacy + `since` window +- **WHEN** their totals are served through the cache +- **THEN** each request MUST use its own cache entry and grouped total + +#### Scenario: Search participates in the conversation total cache signature + +- **GIVEN** two listing requests differ only by the `search` text +- **WHEN** their totals are served through the cache +- **THEN** each request MUST use its own cache entry and grouped total + +### Requirement: Conversation details + +The authenticated dashboard MUST expose +`GET /api/conversations/{conversation_id}`. Detail aggregation MUST use the same +eligible-row scope as listing: normalized non-empty IDs, rows whose request kind +is neither `warmup` nor `limit_warmup`, and `deleted_at IS NULL`. + +For a matching conversation, the detail response MUST expose the conversation ID, +`start` (earliest `requested_at`), `latest` (latest `requested_at`), +`accountCount` (distinct non-null accounts), `totalElapsedTime`, and +`dominantUseragentGroup`. `totalElapsedTime` MUST be +`SUM(COALESCE(latency_ms, 0))` over all eligible rows, never the wall-clock span. +`dominantUseragentGroup` MUST use +`request_count DESC, latest_requested_at DESC, lexical ASC`. + +The response MUST include one model/effort row per distinct +`(model, reasoning_effort)` combination. Each row MUST contain exactly: +`modelEffort`, `reqs`, `totalElapsedTime`, `totalInputTokens`, +`cachedInputTokens`, `totalOutputTokens`, and `totalCostUsd`. The row +elapsed time MUST use `SUM(COALESCE(latency_ms, 0))` for that combination; +output tokens MUST use the reasoning-token fallback; cached input MUST use the +existing per-row clamp. No error-count or other column may be returned. + +The API MUST order model/effort rows by `reqs DESC`, latest request DESC, and +lexical key ASC. It MUST NOT accept a sort query parameter. Client-side sorting +MUST operate only on returned rows. + +An encoded blank path such as `GET /api/conversations/%20` MUST return the +project-standard 404 response. An unknown non-empty conversation ID MUST also +return the project-standard 404 response. The detail route MUST accept any +normalized non-empty stored conversation ID, including IDs containing `/`, when +the client percent-encodes the opaque ID as one path value. + +#### Scenario: Details preserve cumulative elapsed time + +- **GIVEN** a conversation has known latencies across multiple accounts and + model/effort combinations +- **WHEN** the client calls `GET /api/conversations/conv-a` +- **THEN** conversation `totalElapsedTime` is the sum of + `COALESCE(latency_ms, 0)` across eligible rows +- **AND** each model/effort row uses the same cumulative sum over its matching + rows rather than the start/latest wall-clock span + +#### Scenario: Details exclude warmups and soft-deleted rows + +- **GIVEN** a conversation contains normal, `warmup`, `limit_warmup`, and + soft-deleted request logs +- **WHEN** the client calls `GET /api/conversations/conv-a` +- **THEN** the summary and every model/effort row include only rows whose request + kind is neither `warmup` nor `limit_warmup` and which are non-soft-deleted + +#### Scenario: Blank and unknown detail IDs use standard not-found behavior + +- **WHEN** the client calls `GET /api/conversations/%20` or requests an unknown + non-empty ID +- **THEN** the API returns the standard 404 error envelope + +#### Scenario: Slash-containing detail IDs remain addressable + +- **GIVEN** an eligible conversation has the normalized ID `workspace/thread-1` +- **WHEN** the client calls `GET /api/conversations/workspace%2Fthread-1` +- **THEN** the API returns that conversation's details with + `conversationId` equal to `workspace/thread-1` + +### Requirement: Dashboard conversation view + +The dashboard MUST render Request Logs by default. The original uppercase +section-title typography MUST be retained, and the title itself MUST be the +single accessible Radix-style selector trigger with `ChevronDown` for Request +Logs and Conversations. A separate selector MUST NOT render to the title's +right. Selecting Conversations MUST persist `view=conversations` in the URL; +selecting Request Logs MUST return to the existing request-log view. + +The dashboard MUST retain separate URL-backed query state for Request Logs and +Conversations, including each view's applicable filters and pagination. +Switching views MUST NOT reinterpret, overwrite, or clear the inactive view's +query state, and returning to a view MUST restore its retained state. + +The Conversations view MUST NOT render a free-text filter input above the list. +The view MUST render a day-range selector with exactly three options — `1d`, +`7d`, and `30d` — placed at the top-right of the dashboard page alongside the +refresh action and shown only while the Conversations view is active. The +selector MUST default to `7d`. The selected value MUST be persisted in the URL +as `conversationTimeframe`, MUST drive the list endpoint's `timeframe` query +parameter using the same symbolic key (the server derives the effective window), +and MUST NOT generate a browser-clock-derived `since` parameter. It MUST reset +pagination to offset 0 on change. The selector's values and default +MUST mirror the dashboard overview timeframe selector, with no unbounded +"all" option. The view MUST use the list endpoint's +established loading, error, empty, and pagination behavior. +While Conversations is active, the dashboard overview query that supplies the +statistics cards MUST use the active `conversationTimeframe`, including on the +initial render when that value is restored from the URL. The independently +retained `overviewTimeframe` MUST continue to drive the overview query when +Request Logs is active. + +The conversation list MUST render exactly these columns in order: Last request, +Conversation, Accounts, API key, Models, Tokens, Cost, and Details. Last request +MUST use the request-log Time column's two-line time/date presentation. Accounts +MUST resolve the representative account ID through the dashboard account +summaries and display `displayName`, then email, then the ID as a final fallback. +Accounts and models MUST render remaining values as a smaller muted `+ N more` +secondary line. Tokens MUST show total tokens with cached input tokens on a +subordinate line. +When dashboard privacy blur is enabled, an account label resolved from an email +fallback MUST render with the established `privacy-blur` class; display-name +and account-ID fallback labels MUST remain unblurred. +The API-key column MUST use `apiKeyName` only. Details MUST use the existing +Details button treatment. + +The details dialog MUST render row one as conversation ID, start, and latest; +row two as account count, total elapsed time, and dominant user-agent family; +and a model/effort table with exactly these displayed columns, in order: Model +(effort), Reqs, Total elapsed, Total input (with total cache as a +subordinate/parenthetical value), Total output, and Total cost. Total cache MUST +not be a separate displayed column. The table MUST default to Reqs descending +and MUST support client-side sorting for every displayed column without adding a +sort query parameter. +The displayed conversation ID MUST NOT provide a copy action. + +The detail dialog MUST use the established dashboard loading state while the +detail API is pending. Unknown or malformed conversation IDs, including a +standard detail API 404, MUST use the standard dashboard error display and retry +behavior. Nullable optional aggregate values MUST render the established +em-dash or other dashboard fallback value without breaking the row or dialog. +An empty conversation list MUST render the established dashboard empty state. +When an empty conversation list is returned for a nonzero pagination offset, the +Conversations view MUST retain its pagination controls so the operator can +navigate back to the first or previous page. The initial empty state at offset +zero MUST NOT render pagination controls. + +#### Scenario: Request Logs is the default and selector switches views + +- **WHEN** an operator opens the dashboard +- **THEN** Request Logs is visible and active by default +- **WHEN** the operator selects Conversations +- **THEN** the Conversations list renders and the URL contains + `view=conversations` + +#### Scenario: Request Logs and Conversations retain independent URL query state + +- **GIVEN** Request Logs has active filters and pagination and Conversations has + different active filters and pagination retained in the URL +- **WHEN** the operator switches between the two views +- **THEN** each view restores its own filters and pagination +- **AND** switching views does not reinterpret, overwrite, or clear the other + view's query state + +#### Scenario: Conversations has no free-text filter and renders the day selector + +- **WHEN** the operator opens the Conversations view +- **THEN** no free-text filter input is rendered above the list +- **AND** a day-range selector with exactly `1d`, `7d`, and `30d` options is + rendered at the top-right of the dashboard page alongside the refresh action +- **AND** the selector defaults to `7d` and no unbounded "all" option is offered +- **AND** the list renders the specified reordered columns and two-line request + time presentation +- **AND** representative account IDs resolve to display name, then email, then ID +- **AND** smaller muted `+ N more` account/model secondary lines and cached + tokens as a subordinate line are rendered + +#### Scenario: Conversation day selector persists in the URL and drives timeframe + +- **WHEN** the operator changes the Conversations day selector from `7d` to `30d` +- **THEN** the URL gains `conversationTimeframe=30d` (or drops the param when the + default `7d` is selected) +- **AND** the list endpoint is called with `timeframe=30d` +- **AND** the list endpoint does not receive a browser-clock-derived `since` +- **AND** pagination resets to offset 0 + +#### Scenario: Conversation timeframe drives active dashboard statistics + +- **GIVEN** the URL restores `conversationTimeframe=30d` while + `overviewTimeframe` is absent or has a different value +- **WHEN** the operator opens the Conversations view +- **THEN** the statistics-card overview query uses the `30d` timeframe +- **AND** the conversation list uses `timeframe=30d` +- **AND** the independently retained overview timeframe remains unchanged + for the Request Logs view + +#### Scenario: Conversation day selector state is independent per view + +- **GIVEN** the Conversations day selector is set to `30d` +- **WHEN** the operator switches to Request Logs and back to Conversations +- **THEN** the Conversations view restores its retained `30d` selector state +- **AND** the Request Logs view state is unaffected + +#### Scenario: Conversation account privacy blur applies only to email fallback + +- **GIVEN** dashboard privacy blur is enabled and account labels resolve using + display name, email fallback, and account-ID fallback values +- **WHEN** the Conversations list renders +- **THEN** only the email-fallback label has the established `privacy-blur` class +- **AND** the display-name and account-ID fallback labels remain unblurred + +#### Scenario: The original-styled title is the only view selector + +- **WHEN** the list section renders +- **THEN** its uppercase title typography is retained +- **AND** activating the title opens the Request Logs/Conversations selector +- **AND** no separate selector is rendered to the title's right + +#### Scenario: Conversation details use established loading and retry states + +- **WHEN** the detail API is loading for a selected conversation +- **THEN** the dialog uses the established dashboard loading state +- **WHEN** the detail API returns an unknown or malformed-ID error +- **THEN** the dialog uses the standard dashboard error display with retry + +#### Scenario: Nullable detail aggregates use dashboard fallbacks + +- **GIVEN** a successful detail response contains nullable optional aggregate + values +- **WHEN** the operator opens the details dialog +- **THEN** each nullable value renders the established em-dash or dashboard + fallback without breaking the row or dialog + +#### Scenario: Empty conversation results use the existing empty state + +- **GIVEN** the conversation list response contains no rows +- **WHEN** the operator opens the Conversations view +- **THEN** the existing dashboard empty state is rendered + +#### Scenario: Empty later conversation pages retain pagination controls + +- **GIVEN** the operator is on a nonzero Conversations page and the list + response contains no rows +- **WHEN** the Conversations view renders the response +- **THEN** the existing dashboard empty state is rendered +- **AND** pagination controls remain visible +- **AND** the first-page and previous-page controls provide a path back to + earlier results + +#### Scenario: Details dialog has the approved layout and sorting + +- **WHEN** the operator opens a conversation's Details dialog +- **THEN** row one contains conversation ID/start/latest +- **AND** conversation ID has no copy action +- **AND** row two contains account count/total elapsed/dominant user-agent +- **AND** the table displays exactly Model (effort), Reqs, Total elapsed, Total + input (with total cache as a subordinate/parenthetical value), Total output, + and Total cost +- **AND** the table initially sorts by Reqs descending +- **AND** activating any displayed table column header reorders only the returned + rows client-side + +### Requirement: Conversation list exposes grouped request metrics + +`GET /api/conversations` SHALL include `requestCount` and `firstRequest` for +every conversation. `requestCount` SHALL count all eligible request-log rows +in that conversation, and `firstRequest` SHALL be the earliest eligible +`requested_at`. Existing `lastRequest` SHALL remain the latest eligible +`requested_at`. + +#### Scenario: A conversation aggregates request metrics + +- **GIVEN** one conversation has eligible requests at 10:00, 10:07, and + 12:15 +- **WHEN** the conversation list is requested +- **THEN** its `requestCount` is `3` +- **AND** its `firstRequest` is the 10:00 timestamp +- **AND** its `lastRequest` is the 12:15 timestamp +- **AND** warmup, limit-warmup, deleted, blank-ID, and otherwise ineligible + rows do not affect those values + +### Requirement: Conversation list renders metrics and readable duration + +The dashboard SHALL render columns in this order: Last request, Lasted, +Conversation, Accounts, API key, Models, Requests, Tokens, Cost, Details. +The Lasted value SHALL use `lastRequest - firstRequest`, displaying `0s` for +zero duration, seconds for durations under one minute, `xm ys` for durations +under one hour, `xh ym` for durations under one day, and `xd yh` for durations +of at least one day. The conversation-ID cell SHALL be top-aligned. + +#### Scenario: Duration uses two units and preserves zero + +- **WHEN** a row spans 2 hours and 15 minutes +- **THEN** Lasted displays `2h 15m` +- **WHEN** a row spans 2 days and 3 hours +- **THEN** Lasted displays `2d 3h` +- **WHEN** firstRequest equals lastRequest +- **THEN** Lasted displays `0s` + +### Requirement: Fair-share congestion threshold is configurable from routing settings + +The dashboard routing settings MUST expose the API-key fair-share congestion threshold as a numeric field adjacent to the per-account capacity limits, accepting integers from 0 to 100 where 0 disables the gate, with null-inherits-environment semantics matching the per-account capacity overrides. Values outside 0-100 MUST be rejected by both the client-side validation and the settings API. The field's label, description, and validation copy MUST be localized in the en, ko, and zh-CN locale bundles. + +#### Scenario: Threshold round-trips through the settings API + +- **GIVEN** an operator sets the threshold to 80 in routing settings +- **WHEN** the settings are saved and reloaded +- **THEN** the field shows 80 and the settings API reports 80 as the effective value + +#### Scenario: Migrated null row inherits the environment default + +- **GIVEN** a deployment whose dashboard settings row predates the field (a migrated NULL column) +- **AND** an environment-configured threshold +- **WHEN** the effective settings are read +- **THEN** the effective value inherits the environment setting + +#### Scenario: Out-of-range values are rejected + +- **GIVEN** an operator enters 101 or a negative number +- **WHEN** they attempt to save +- **THEN** the client blocks the save and the settings API rejects the value if submitted directly + +#### Scenario: Copy is localized in all three locales + +- **GIVEN** the dashboard language is set to en, ko, or zh-CN +- **WHEN** routing settings render +- **THEN** the threshold label and description display in the selected locale + +### Requirement: Appearance settings include date format toggle + +The Appearance settings section SHALL include a "Date format" toggle row with two options: "Default" and "ISO 8601". The toggle SHALL be placed between the Time format and Account rows settings. Selecting an option SHALL immediately apply the new format to applicable read-only date/time presentation text across the dashboard. + +#### Scenario: Default date format is selected initially + +- **WHEN** a user opens the Appearance settings section with no prior date format preference +- **THEN** the "Default" option SHALL be selected (aria-pressed true) +- **AND** applicable read-only date/time presentation text SHALL render using locale-dependent formatting + +#### Scenario: Switching to ISO 8601 + +- **WHEN** the user clicks the "ISO 8601" option in the Date format row +- **THEN** the "ISO 8601" option SHALL be selected +- **AND** request log and conversation table cells SHALL display date on the top line in `YYYY-MM-DD` format and time on the bottom line in `HH:MM:SS` format +- **AND** the preference persists across page reloads + +### Requirement: Accounts and API trend chart x-axis uses MM-DD format + +The x-axis tick format of the Account Trend and API Trend charts SHALL be `MM-DD` (month and day extracted from the ISO timestamp data key), matching the reports chart convention. This format SHALL be locale-independent. + +#### Scenario: Account trend chart x-axis ticks + +- **WHEN** the Account Trend chart renders with timestamp data +- **THEN** the x-axis tick labels SHALL be in `MM-DD` format (e.g., `"08-09"`) + +#### Scenario: API trend chart x-axis ticks + +- **WHEN** the API Trend chart renders with timestamp data +- **THEN** the x-axis tick labels SHALL be in `MM-DD` format (e.g., `"08-09"`) + diff --git a/openspec/specs/github-automation/spec.md b/openspec/specs/github-automation/spec.md index ddec209a68..2883d6309a 100644 --- a/openspec/specs/github-automation/spec.md +++ b/openspec/specs/github-automation/spec.md @@ -333,3 +333,138 @@ labels, so the override MUST NOT apply there: a change that would leave - **WHEN** a budget is exceeded - **THEN** no pull-request label set is resolved and the check fails regardless of any label on the originating pull request + +### Requirement: Codex review trigger usage-limit backoff + +The Codex label synchronization script MUST NOT post a new `@codex review` comment while the comment sender's latest Codex response within the configured backoff window is a usage-limit reply. A usage-limit reply is a Codex response whose body starts (after optional leading whitespace) with the quota envelope "You have reached your Codex usage limits"; Codex reviews that merely discuss usage limits MUST NOT latch the backoff. Usage-limit evidence MUST be attributed to the sender whose request comment preceded the reply, and a newer normal Codex response for that same sender MUST lift the backoff; a clean THUMBS_UP reaction by a Codex reviewer on the sender's request comment counts as a normal response. Backoff state MUST be shared across all repositories processed in one run, so a usage limit observed in one repository suppresses the remaining review requests in the run; classified timelines from repositories without their own triggers MUST still contribute evidence. Before posting review requests in a repository, the script MUST also gather the repository's recent issue comments (within the backoff window, grouped per issue) as evidence, so quota replies on pull requests outside the current selection — including single `--pr` runs and closed pull requests — still latch the backoff; a failure to gather this evidence degrades to the classified-timeline evidence with a warning. When no quota evidence exists for the sender, the script MUST post the first `@codex review`, wait briefly, reread that pull request's timeline, and suppress the remaining review requests in the run if that probe observed a usage-limit reply; probing MUST stop once a normal Codex response has been observed, and MUST NOT run when the review request was not actually posted (for example after a tolerated write denial). Apply-loop status lines and error reports MUST reference the pull request of the decision being applied. + +The script MUST resolve the sender identity in a way that works with GitHub App installation tokens (which cannot call `GET /user`): it prefers the app slug exported by the workflow (`GH_APP_SLUG`, yielding `[bot]`) and falls back to `GET /user` for PAT-backed runs. Once the run has switched to the fallback token, the app slug no longer describes the active identity: sender resolution MUST ignore it, and review triggers MUST be suppressed with a warning because posted comments would no longer be authored by the resolved sender. The review-request POST itself MUST NOT be silently retried under the fallback token after a rate-limit response: the fallback activates for subsequent calls, but the identity-sensitive comment fails instead of posting under the wrong author. If the sender cannot be resolved, only the review-trigger path is disabled (with a warning per affected decision); label synchronization and workflow-run approvals MUST proceed. + +#### Scenario: Recent usage-limit reply latches the backoff + +- **GIVEN** the sender's `@codex review` comment was answered by a Codex usage-limit reply within the backoff window +- **AND** the sender has no newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post and surfaces a write warning naming the usage-limit evidence + +#### Scenario: Reviews that merely discuss usage limits do not latch + +- **GIVEN** a Codex review whose body discusses usage limits but does not start with the quota envelope +- **WHEN** the script classifies Codex responses for the backoff +- **THEN** the response is treated as a normal Codex response, not a usage-limit reply + +#### Scenario: Newer normal response lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** the same sender has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Newer clean reaction lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** a Codex reviewer later reacted with THUMBS_UP to the sender's `@codex review` comment +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Senders are attributed independently + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** only a different account has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it still skips the `@codex review` post for the sender + +#### Scenario: Backoff persists across repositories in one run + +- **GIVEN** a run selecting multiple repositories +- **AND** the sender's usage limit was observed while processing an earlier repository +- **WHEN** the script would trigger a missing Codex review in a later repository +- **THEN** it skips the `@codex review` post there as well + +#### Scenario: Evidence from a non-triggering repository still counts + +- **GIVEN** an earlier repository whose classified timelines contain the sender's usage-limit reply but whose decisions need no review trigger +- **WHEN** a later repository in the same run would trigger a missing Codex review +- **THEN** the earlier repository's evidence latches the backoff and the post is skipped + +#### Scenario: Quota evidence outside the selected pull requests still counts + +- **GIVEN** a single `--pr` run where the sender's usage-limit reply lives on a different (possibly closed) pull request of the repository +- **WHEN** the script would trigger a missing Codex review +- **THEN** the repository's recent issue comments provide the evidence and the post is skipped + +#### Scenario: Probe requires an actual post + +- **GIVEN** the review-request comment was not posted because the write was denied and tolerated +- **WHEN** the script would otherwise probe for a quota reply +- **THEN** it neither waits nor rereads the pull request timeline for that decision + +#### Scenario: No-data probe latches off remaining triggers + +- **GIVEN** no Codex quota evidence exists for the sender in the classified timelines +- **WHEN** the script posts the first `@codex review` of the run +- **THEN** it waits the configured probe interval, rereads that pull request's timeline, and skips the remaining review requests if the probe observed a usage-limit reply + +#### Scenario: Probing stops after a normal response + +- **GIVEN** a normal Codex response for the sender has already been observed +- **WHEN** the script posts further `@codex review` comments in the run +- **THEN** it does not wait or reread pull request timelines for those posts + +#### Scenario: Installation tokens resolve the sender from the app slug + +- **GIVEN** the run authenticates with a GitHub App installation token and the workflow exports the app slug +- **WHEN** the script resolves the `@codex review` sender +- **THEN** it derives `[bot]` without calling `GET /user` + +#### Scenario: Sender resolution failure only disables review triggers + +- **GIVEN** the sender cannot be resolved from either the app slug or `GET /user` +- **WHEN** the script applies decisions +- **THEN** label synchronization proceeds and each suppressed review trigger surfaces a warning naming the unresolved sender + +#### Scenario: Fallback token activation suppresses review triggers + +- **GIVEN** the run has switched to `GH_FALLBACK_TOKEN` after rate-limit exhaustion +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post with a warning, because the comment author would no longer match the resolved sender + +#### Scenario: The review-request POST is not retried under the fallback identity + +- **GIVEN** the review-request comment POST itself hits the primary token's rate limit +- **WHEN** the fallback token activates +- **THEN** the POST fails instead of being silently retried under the fallback identity, while later calls use the fallback token + +#### Scenario: Apply status is attributed to the applied pull request + +- **WHEN** the script applies decisions for multiple pull requests in one run +- **THEN** each status line and error report references the pull request of the decision being applied + +### Requirement: Apply-time reclassification + +Before performing writes for a classified decision (label changes, legacy label removal, workflow-run approvals, or review triggers), the Codex label synchronization script MUST reclassify the pull request and act on the fresh evidence only. If the head SHA no longer matches the SHA the decision was classified against, the decision MUST be skipped with a warning. If the head is unchanged but the evidence changed (checks, reviews, mergeability), the writes MUST follow the fresh decision, and a review trigger MUST only fire when both the original and the fresh classification want it. The freshly read timeline MUST feed the shared usage-limit backoff so quota replies that arrived after bulk classification suppress the remaining review requests. Reclassification read failures MUST honor `--tolerate-read-errors` (log and skip the decision without failing the run). Decisions without pending writes need not be reclassified. + +#### Scenario: Stale decision is skipped after a head move + +- **GIVEN** a pull request whose head changed between classification and apply +- **WHEN** the script reaches that decision in the apply loop +- **THEN** it skips all writes for the decision and warns that the head moved + +#### Scenario: Same-head evidence changes are applied fresh + +- **GIVEN** a pull request whose head is unchanged but where Codex raised a new finding after classification +- **WHEN** the script reaches that decision in the apply loop +- **THEN** the writes reflect the fresh classification instead of the superseded one + +#### Scenario: Fresh quota evidence suppresses later triggers + +- **GIVEN** a quota reply that arrived between bulk classification and apply-time reclassification of one pull request +- **WHEN** later decisions in the run would trigger missing Codex reviews +- **THEN** the reclassified timeline has latched the backoff and those posts are skipped + +#### Scenario: Reclassification honors tolerant reads + +- **GIVEN** a run with `--tolerate-read-errors` +- **WHEN** apply-time reclassification of one pull request fails with a GitHub read error +- **THEN** the decision is logged and skipped without failing the run + diff --git a/openspec/specs/graceful-shutdown/spec.md b/openspec/specs/graceful-shutdown/spec.md new file mode 100644 index 0000000000..852e4d333d --- /dev/null +++ b/openspec/specs/graceful-shutdown/spec.md @@ -0,0 +1,233 @@ +# graceful-shutdown Specification + +## Purpose +Ordered process drain: a pre-connection barrier, WebSocket admission closure, and finalization of active turns so shutdown never strands in-flight work or settlement. +## Requirements +### Requirement: Process shutdown establishes a pre-connection drain barrier + +The project-owned Uvicorn server MUST commit graceful drain before Uvicorn closes HTTP or WebSocket connections. A deadline-bearing preStop request or the first shutdown transition MUST establish one monotonic application drain deadline; later SIGTERM, shutdown, or lifespan transitions MUST reuse that deadline without extending it. A delayed preStop request MAY tighten a signal-committed deadline only to preserve the earlier absolute deadline carried by that request. A headerless operator drain MUST remain reversible. Once process shutdown is committed, operator actions MUST NOT reopen admission or erase its deadline. + +#### Scenario: Direct SIGTERM precedes connection shutdown + +- **WHEN** the process receives SIGTERM with an admitted Responses turn active and no prior preStop request +- **THEN** drain admission closes before Uvicorn invokes connection shutdown +- **AND** Uvicorn waits for the turn within the remaining application deadline + +#### Scenario: preStop is followed by SIGTERM + +- **WHEN** preStop starts drain and SIGTERM arrives later +- **THEN** SIGTERM reuses the original absolute deadline +- **AND** local preStop request latency has already consumed that deadline +- **AND** it does not start a second drain period + +#### Scenario: SIGTERM overtakes a deadline-bearing preStop request + +- **WHEN** preStop anchors an absolute deadline and SIGTERM commits a later deadline before the local start request is handled +- **THEN** the accepted preStop deadline tightens the committed process deadline +- **AND** every later shutdown stage uses that earlier absolute value +- **AND** an in-flight drain wait that already started re-reads and adopts that earlier deadline + +#### Scenario: SIGTERM interleaves with deadline initialization + +- **WHEN** a drain-start invocation observes no prior drain and a synchronous shutdown signal commits between its later state operations +- **THEN** both invocations publish monotonic deadline candidates +- **AND** every drain stage uses the earlier candidate +- **AND** neither stale continuation can extend the effective deadline + +#### Scenario: Drain stop races committed shutdown + +- **WHEN** an operator drain stop interleaves with process shutdown commitment +- **THEN** committed WebSocket and HTTP admission remains closed +- **AND** the committed deadline remains available to every later drain stage + +#### Scenario: Signal commit precedes matching lifespan startup + +- **WHEN** a server start prepares shutdown state and SIGTERM commits the barrier before its application lifespan begins +- **THEN** matching lifespan startup preserves the committed barrier and deadline +- **AND** it does not reset process shutdown state + +#### Scenario: Completed embedded lifespan is followed by a new start + +- **WHEN** an embedded application lifespan has completed and another embedded lifespan starts in the same process +- **THEN** the new lifecycle starts with admission open and no inherited committed deadline + +### Requirement: Graceful drain closes WebSocket admission + +Once graceful drain begins, the application MUST reject every new external WebSocket connection before invoking the route handler. A Responses WebSocket scope admitted before the barrier MUST remain tracked until its handler exits. Other WebSocket protocols MUST receive the same late-admission rejection but MUST NOT hold the Responses in-flight counter for their full connection lifetime. + +#### Scenario: New WebSocket arrives during drain + +- **WHEN** a new WebSocket connection scope arrives after drain has begun +- **THEN** the application rejects the connection without invoking its route handler +- **AND** the rejected connection does not increase the in-flight count + +#### Scenario: WebSocket crosses the drain barrier + +- **WHEN** a Responses WebSocket scope is admitted immediately before drain begins +- **THEN** it remains in the in-flight count until its route handler exits +- **AND** shutdown waits for that scope within the configured drain timeout + +#### Scenario: Realtime or Live connection predates drain + +- **WHEN** a non-Responses WebSocket scope is admitted before drain +- **THEN** it does not hold the Responses in-flight counter +- **AND** normal Uvicorn connection shutdown remains its lifecycle bound + +### Requirement: Graceful drain preserves active WebSocket turn finalization + +An admitted Responses WebSocket connection MUST stop accepting new `response.create` turns after drain begins. An idle connection MUST close promptly, while a connection with an already registered active turn MUST remain open until that turn reaches terminal downstream delivery, request-log persistence ownership, and API-key settlement ownership or the shared drain deadline expires. + +#### Scenario: Idle admitted WebSocket observes drain + +- **WHEN** drain begins while an admitted Responses WebSocket has no active turn +- **THEN** the server closes that connection promptly +- **AND** the connection no longer holds the shutdown drain + +#### Scenario: Active turn completes during drain + +- **WHEN** drain begins while an admitted Responses WebSocket turn is active +- **THEN** the turn continues through terminal downstream delivery, request logging, and API-key settlement +- **AND** the WebSocket closes after the active turn is finalized + +#### Scenario: Existing connection submits a new turn during drain + +- **WHEN** an admitted Responses WebSocket submits a new `response.create` after drain begins +- **THEN** the server rejects that turn locally +- **AND** the turn is not registered or sent upstream + +#### Scenario: Upstream clean close races a new turn + +- **WHEN** an upstream transport-end frame is received while a new turn is blocked in admission or account ownership +- **THEN** the connection is synchronously marked for reconnect before that attribution wait +- **AND** the reader does not fail or replay-mutate the still-unsent turn +- **AND** the sender checks the latch after all admission and account awaits immediately before send +- **AND** the turn is sent exactly once on a fresh upstream socket +- **AND** the retired account-local create lease is released and the fresh account re-acquires its own lease +- **AND** no post-send replay budget is consumed + +#### Scenario: Generic send failure races a reader-owned clean-close replay + +- **WHEN** a generic upstream send failure occurs after the reader has classified the same sent turn as clean-close replayable +- **THEN** the sender marks the connection for reconnect before cancelling and awaiting the reader +- **AND** harvests the reader's published replay owner before retiring its control state +- **AND** releases the retired account-local create lease before a replacement account acquires its own lease +- **AND** sends the turn exactly once on the replacement socket +- **AND** produces exactly one terminal event, one terminal request log, and one API-key settlement + +#### Scenario: Typed transport send failure races a reader claim + +- **WHEN** a typed transport send failure occurs after the reader has claimed the same sent turn +- **THEN** the sender does not replay the ambiguously delivered turn +- **AND** transfers the reader claim to one registered finalization task before awaiting it +- **AND** produces exactly one `response.failed` with the typed transport error +- **AND** releases or settles the API-key reservation and persists the terminal request log exactly once + +### Requirement: Terminal WebSocket work has explicit cancellation-safe ownership + +After an upstream message is received, processing and downstream delivery MUST be owned by a registered task before terminal handling can remove its request state from the pending queue. Reader or scope cancellation MUST NOT orphan that task. The reader MUST wait for owned terminal work only within the remaining shared application deadline; when no application drain is active, normal scope cancellation MUST instead use the existing bounded task-cancellation timeout. Shutdown persistence drain MUST observe both terminal-message and transport-end child tasks plus any request-log or settlement follow-up work they create. + +#### Scenario: Cancellation lands after terminal state leaves pending + +- **WHEN** a terminal event removes its request state from the pending queue and the upstream reader is then cancelled before settlement or downstream delivery completes +- **THEN** the reader waits for the owned terminal task within the remaining shared application deadline before propagating cancellation +- **AND** actual usage is settled exactly once +- **AND** exactly one terminal event and one terminal request log are produced + +#### Scenario: Cancellation lands before a terminal event + +- **WHEN** a Responses scope is cancelled while its request state remains pending or staged for transparent replay +- **THEN** its API-key reservation is released exactly once +- **AND** exactly one cancelled request log is produced against the last upstream account that owned the turn + +#### Scenario: Cancellation lands after a pending batch is claimed + +- **WHEN** terminal cleanup removes pending request states from the shared queue +- **THEN** it atomically transfers them to a registered, shielded finalization task before releasing the queue lock +- **AND** caller cancellation waits only within the remaining shared deadline without cancelling that sole child owner +- **AND** persistence drain continues to observe the child +- **AND** each request releases its turn admission, account-local create lease, API-key reservation, and create gate and persists its terminal log exactly once + +#### Scenario: Pending account-health failure preserves settlement ordering + +- **GIVEN** a keyed WebSocket turn is claimed by terminal cleanup +- **WHEN** the failure would record an account-health penalty +- **THEN** every claimed API-key reservation release commits before the account-health write +- **AND** every claimed terminal request log is handed to tracked persistence before the account-health write +- **AND** a failed or indeterminate reservation release prevents that account-health write + +#### Scenario: Upstream terminal event preserves account-health ordering + +- **GIVEN** a keyed WebSocket turn receives its upstream terminal event +- **WHEN** finalization would write account health +- **THEN** API-key settlement commits and the terminal request log is handed to tracked persistence before that health write +- **AND** a failed or indeterminate settlement or request-log handoff prevents the health write +- **AND** an account-health write failure does not prevent terminal downstream delivery + +#### Scenario: Reader cancellation occurs outside process drain + +- **WHEN** a reader is cancelled while its owned terminal task ignores cancellation and no application drain deadline exists +- **THEN** the reader waits only for the existing bounded task-cancellation timeout +- **AND** the owned task remains registered for eventual result consumption + +#### Scenario: Active turn exceeds the shared deadline + +- **WHEN** terminal processing remains blocked past the application drain deadline +- **THEN** Uvicorn proceeds with bounded connection and task shutdown +- **AND** the process does not start another application drain timeout + +#### Scenario: Cancelled reader requires transport close + +- **WHEN** scope cleanup cancels an upstream reader whose receive operation waits for transport close before propagating cancellation +- **THEN** cleanup first transfers its local replay and request-state owners to one registered finalization task +- **AND** requests reader cancellation exactly once +- **AND** closes the upstream transport before awaiting the already-cancelled reader +- **AND** bounds that await, lease release, and remaining terminal cleanup by the shared deadline +- **AND** expiry of the caller's wait does not cancel that sole cleanup owner +- **AND** produces exactly one terminal request log and one reservation settlement or release + +#### Scenario: Connection lease release fails during scope cancellation + +- **WHEN** scope cancellation owns a pending turn and releasing the upstream connection lease fails +- **THEN** request finalization completes before connection-lease release is attempted +- **AND** turn admission, account-local create lease, API-key reservation, create gate, and terminal request log are finalized exactly once +- **AND** the lease failure is reported without replacing the original scope cancellation + +### Requirement: Owned launchers preserve shutdown semantics + +The project CLI MUST use the pre-connection drain server with exactly one worker per process while preserving Uvicorn's startup-failure exit status and clean KeyboardInterrupt behavior. Every supported server launch path shipped or documented by the project MUST delegate to that owned CLI rather than invoking raw FastAPI or Uvicorn startup. Development Compose source synchronization MUST restart the owned server instead of relying on Uvicorn's incompatible reload launcher. Ambient `WEB_CONCURRENCY` MUST NOT create an unsupported multiprocess launch. The project MUST declare a Uvicorn version whose launcher API includes `Config.load_app()`. An embedded metrics server MUST NOT replace the main server's process signal handlers. During shutdown, after the shared application drain deadline, the owned server MUST stop awaiting Uvicorn connection and lifespan cleanup after 25 seconds. If that bound expires, it MUST terminate with the most recently captured shutdown signal, or SIGTERM when shutdown was programmatic, rather than return a cancellation-resistant cleanup task to asyncio runner teardown. + +#### Scenario: Lifespan startup fails + +- **WHEN** Uvicorn does not reach its started state +- **THEN** the project CLI exits with Uvicorn's startup-failure status + +#### Scenario: Metrics endpoint is enabled + +- **WHEN** the main process starts the embedded metrics server +- **THEN** only the main application server owns SIGTERM and SIGINT handlers + +#### Scenario: Launcher dependency is resolved + +- **WHEN** the project runtime dependencies are resolved +- **THEN** Uvicorn versions older than 0.47.0 are rejected + +#### Scenario: Ambient worker count is greater than one + +- **WHEN** `WEB_CONCURRENCY` requests multiple workers +- **THEN** the owned launcher still starts exactly one worker for the instance + +#### Scenario: Operator follows a shipped or documented launch path + +- **WHEN** the server is started through a project Compose file or documented local command +- **THEN** that path delegates to the owned pre-connection drain launcher +- **AND** direct SIGTERM commits the barrier before Uvicorn closes connections +- **AND** development source synchronization restarts that owned launcher + +#### Scenario: Lifespan cleanup blocks after application drain + +- **WHEN** Uvicorn connection or lifespan cleanup remains blocked after the shared drain phase +- **THEN** the owned launcher cancels and stops waiting after 25 seconds +- **AND** terminates with the most recently captured signal, or SIGTERM when no signal initiated shutdown +- **AND** does not leave cancellation-resistant cleanup registered for unbounded asyncio runner teardown +- **AND** Helm termination grace reserves two seconds for failed preStop start plus 30 seconds after the application deadline, leaving five seconds after the cleanup bound for process exit before SIGKILL + diff --git a/openspec/specs/http-ingress-limits/spec.md b/openspec/specs/http-ingress-limits/spec.md new file mode 100644 index 0000000000..5798af560c --- /dev/null +++ b/openspec/specs/http-ingress-limits/spec.md @@ -0,0 +1,172 @@ +# http-ingress-limits Specification + +## Purpose +Incremental, budget-reusing bounds on raw HTTP request ingress, including encoded bodies before and after decompression and exact route-owned multipart exceptions. +## Requirements +### Requirement: Raw HTTP request ingress is bounded incrementally + +The service MUST enforce the applicable request-body budget against actual raw bytes received for each guarded HTTP request. It MUST reject the request before exposing a chunk that would make the cumulative raw body exceed the budget, and it MUST NOT prebuffer the complete body solely to enforce this limit. + +#### Scenario: Declared oversized body is rejected before downstream parsing + +- **WHEN** a guarded HTTP request declares a valid `Content-Length` greater than its applicable budget +- **THEN** the service returns HTTP 413 without invoking downstream request-body parsing + +#### Scenario: Chunked body crosses the budget + +- **WHEN** a guarded HTTP request has no usable `Content-Length` and its received chunks cumulatively exceed the applicable budget +- **THEN** the service returns HTTP 413 +- **AND** the chunk that crosses the budget is not exposed to downstream body parsing + +#### Scenario: Exact-boundary body is accepted by the ingress guard + +- **WHEN** a guarded HTTP request's actual raw body size equals its applicable budget +- **THEN** the raw ingress guard allows the complete body to continue downstream + +#### Scenario: Client disconnect remains a disconnect + +- **WHEN** the ASGI server reports `http.disconnect` while a guarded body is being received +- **THEN** the ingress guard propagates the disconnect without converting it into an HTTP 413 response + +### Requirement: HTTP ingress reuses existing budgets + +The service MUST use `max_decompressed_body_bytes` as the general raw and decompressed HTTP request-body budget. When an owning route capability defines a larger budget from an existing route-specific setting, the ingress guard MUST use that route budget. The HTTP ingress guard MUST NOT add another setting or change existing defaults. + +Route-specific budget and error-envelope selection MUST use the application-relative route path after removing any matching ASGI `root_path` prefix. + +The generic guard MUST apply to requests solely because they declare `multipart/form-data`; the client-declared media type MUST NOT grant an exemption. An owning route capability MAY define an exact method/path-scoped authorization-before-read contract and dedicated bounded multipart parser. Only unencoded multipart requests to that exact operation, or requests marked by its outer content-encoding gate, MAY bypass generic admission. The gate MUST identify its operation independently of the declared media type, remove the encoding and mark the scope as handled without consuming the body, and the exception MUST NOT apply to any other operation. + +#### Scenario: Another HTTP path uses the general budget + +- **WHEN** a guarded request targets any other HTTP path +- **THEN** its raw and decompressed HTTP ingress budget is `max_decompressed_body_bytes` + +#### Scenario: Route-owned unencoded multipart uses dedicated admission + +- **GIVEN** an exact operation has a capability-defined authorization-before-read contract and dedicated bounded multipart parser +- **WHEN** an unencoded request to that operation declares media type `multipart/form-data` +- **THEN** the generic raw whole-body guard does not preempt operation authorization or its dedicated parser limit + +#### Scenario: Unrelated unencoded multipart remains guarded + +- **WHEN** an unencoded request outside a route-owned multipart operation declares media type `multipart/form-data` +- **THEN** the service applies the generic raw-body budget +- **AND** the declared media type alone does not bypass admission + +#### Scenario: Encoded multipart remains guarded + +- **WHEN** a `multipart/form-data` request outside a route-owned multipart exception carries a `Content-Encoding` header +- **THEN** the service applies both the raw and decompressed budget checks + +#### Scenario: Route-owned multipart admission can preserve authorization precedence + +- **GIVEN** an exact operation has a capability-defined outer content-encoding gate, authorization-before-read contract, and dedicated bounded multipart parser +- **WHEN** an encoded request targets that operation, regardless of its declared media type +- **THEN** the generic raw and decompressed-body guards do not preempt operation authorization or its dedicated parser limit +- **AND** encoded multipart requests to all other operations remain guarded + +#### Scenario: Mounted Responses route keeps its route-specific policy + +- **GIVEN** the service is mounted under a non-empty ASGI `root_path` +- **WHEN** the request scope path includes that prefix and targets `/v1/responses` relative to the application +- **THEN** the service applies the Responses-specific ingress budget +- **AND** any ingress failure uses the OpenAI-compatible error envelope + +### Requirement: Encoded HTTP bodies are bounded before and after decompression + +For request bodies using `gzip`, `deflate`, `zstd`, `identity`, or supported stacked `Content-Encoding` values that remain under generic ingress admission, the service MUST enforce the applicable budget independently against the encoded raw body and every intermediate and final decoded representation. The service MUST remove stacked encodings in reverse header/application order. Unsupported encodings or malformed compressed bodies under generic admission MUST fail with HTTP 400. Exact route-owned exceptions MUST instead follow their owning capability's authorization and encoded-body rejection contract. + +#### Scenario: Encoded raw body exceeds the budget + +- **WHEN** a generic-guarded encoded request's raw bytes exceed the applicable budget before decompression +- **THEN** the service returns HTTP 413 before attempting to hold an unbounded encoded body + +#### Scenario: Expanded body exceeds the budget + +- **WHEN** a generic-guarded encoded request is within the raw budget but expands beyond the applicable decompressed budget +- **THEN** the service returns HTTP 413 + +#### Scenario: Supported stacked encoding remains compatible + +- **WHEN** a generic-guarded request uses a valid supported stack of `gzip`, `deflate`, `zstd`, or `identity` encodings and both representations fit the budget +- **THEN** the service decodes the body in reverse header/application order, caps every intermediate representation, and continues request handling + +#### Scenario: Invalid compression is rejected + +- **WHEN** a generic-guarded request uses an unsupported content encoding or carries malformed compressed bytes +- **THEN** the service returns HTTP 400 without invoking route logic + +### Requirement: HTTP ingress failures use the path-family error envelope + +Ingress failures on `/v1/*`, `/backend-api/*`, `/api/codex/*`, and `/internal/bridge/*` MUST use an OpenAI-compatible error envelope with `type = invalid_request_error`. Equivalent paths MUST be classified after the existing outer path canonicalization. Other ingress paths MUST retain the dashboard-compatible error envelope. Oversized requests MUST use `code = payload_too_large`; malformed or unsupported compression MUST use `code = invalid_request_error` on OpenAI paths and `code = invalid_request` on other paths. + +#### Scenario: OpenAI path rejects an oversized body + +- **WHEN** a raw or decompressed request body on an OpenAI-compatible proxy path exceeds its budget +- **THEN** the service returns HTTP 413 +- **AND** the response has OpenAI error `code = payload_too_large` and `type = invalid_request_error` + +#### Scenario: OpenAI path rejects invalid compression + +- **WHEN** a request on an OpenAI-compatible proxy path uses unsupported or malformed compression +- **THEN** the service returns HTTP 400 +- **AND** the response has OpenAI error `code = invalid_request_error` and `type = invalid_request_error` + +#### Scenario: Dashboard settings path rejects an oversized body + +- **WHEN** a raw or decompressed request body on `/api/settings` exceeds its budget +- **THEN** the service returns HTTP 413 +- **AND** the response has dashboard error `code = payload_too_large` + +#### Scenario: Dashboard settings path rejects invalid compression + +- **WHEN** a request on `/api/settings` uses unsupported or malformed compression +- **THEN** the service returns HTTP 400 +- **AND** the response has dashboard error `code = invalid_request` + +#### Scenario: Duplicated Codex alias is classified after canonicalization + +- **WHEN** an ingress failure targets `/backend-api/codex/v1/responses/` +- **THEN** the service applies the same Responses budget and OpenAI-compatible envelope as `/backend-api/codex/responses/` + +### Requirement: Ingress admission preserves endpoint authorization + +The HTTP ingress guard MUST NOT authenticate callers or replace, bypass, or relocate existing dashboard, proxy API-key, ChatGPT-identity, or internal-bridge authorization. Requests that reach dependency resolution MUST continue through the endpoint's existing authorization path. Existing FastAPI parsing order remains unchanged, so ingress rejection or syntactically invalid typed bodies can fail before router-level authorization. + +#### Scenario: Admitted unauthenticated request still reaches proxy authorization + +- **WHEN** a syntactically valid under-limit request without required credentials targets an API-key-protected proxy route +- **THEN** the ingress guard allows normal routing to continue +- **AND** the existing proxy authorization rejects the request with its established authentication response + +#### Scenario: Declared oversized generic-guarded request fails before authorization + +- **WHEN** a guarded request outside a route-owned admission exception declares a body larger than its ingress budget +- **THEN** the service returns the deterministic ingress 413 without invoking router-level authorization + +### Requirement: Multipart ingress exceptions are exact and route-owned + +The generic raw HTTP body guard MUST exempt an unencoded `multipart/form-data` request only when the request is `POST` to `/api/accounts/import`, `/backend-api/transcribe`, `/v1/audio/transcriptions`, or `/v1/images/edits`, including their application-relative trailing-slash and mounted equivalents. Each exempt operation MUST apply its capability-defined authorization-before-read contract and dedicated bounded multipart parser. + +For the same exact operations, an outer content-encoding gate MUST remove `Content-Encoding` and mark the copied request scope as route-owned without reading the body. The generic raw and decompression guards MUST honor that internal marker regardless of the declared media type so `identity` reaches dedicated multipart admission and non-identity encoding reaches the post-authorization rejection path. + +The client-declared multipart media type MUST NOT exempt any other method or path. Every unrelated unencoded or encoded multipart request MUST remain under generic raw admission, and encoded requests MUST also retain generic decompressed-body admission. + +#### Scenario: Exact unencoded multipart operation uses its dedicated parser + +- **WHEN** an unencoded multipart request targets one of the four exact route-owned `POST` operations +- **THEN** generic admission does not preempt operation authorization +- **AND** the operation's dedicated multipart body limit remains authoritative + +#### Scenario: Exact encoded operation preserves authorization precedence + +- **WHEN** a request with `Content-Encoding` targets one of the four exact route-owned `POST` operations and declares either multipart or another media type +- **THEN** the outer gate marks the request without consuming its body +- **AND** generic raw or decompressed-body admission does not preempt operation authorization or its encoded-body contract + +#### Scenario: Unrelated multipart media type grants no exemption + +- **WHEN** an unencoded or encoded request outside the four exact route-owned `POST` operations declares `multipart/form-data` +- **THEN** the generic raw-body budget remains enforced +- **AND** an oversized declared body is rejected before downstream parsing + diff --git a/openspec/specs/images-api-compat/spec.md b/openspec/specs/images-api-compat/spec.md index a9e242af5c..2506e09c6b 100644 --- a/openspec/specs/images-api-compat/spec.md +++ b/openspec/specs/images-api-compat/spec.md @@ -154,3 +154,71 @@ surface. (log and metrics) with the same `generations`/`edits` route label as the `/v1` counterpart, exactly once +### Requirement: Image edit multipart uploads are authorized and bounded + +`POST /v1/images/edits` MUST complete its existing proxy authorization dependencies before reading multipart body bytes. It MUST accept at most 16 source-image file parts across `image` and `image[]`, at most one `mask`, no unknown file-part names, no more than 32 text fields of at most 256 KiB each, every individual file smaller than 50,000,000 bytes, fewer than 50,000,000 bytes across all source images and the mask, and a complete multipart body no greater than 64 MiB (67,108,864 bytes). + +The service MUST enforce the body limit against both a usable declared `Content-Length` and actual streamed bytes. It MUST enforce file, aggregate-binary, and text limits before retaining crossing bytes, close multipart spools before usage reservation, account selection, base64 conversion, or internal Responses forwarding, and add no new runtime setting. + +This route-owned policy MUST take precedence over the generic raw HTTP body budget for `POST /v1/images/edits`. Its exact-path content-encoding gate MUST run outside the generic raw and decompression guards regardless of the declared media type. Requests handled by that gate, and unencoded requests declared as multipart, MUST NOT be rejected by the generic guards before proxy authorization or the dedicated parser applies this capability's body limit. An unencoded request that does not declare multipart remains under generic admission and MAY be rejected there before authorization. This exception MUST NOT change generic ingress behavior for any other operation. + +Byte-limit failures MUST return HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error`; a known file-part failure MUST set `param = image` or `param = mask`. Multipart syntax, count, and required-field failures MUST retain OpenAI-compatible invalid-request behavior. Every parser rejection MUST emit exactly one bounded image-route observation with HTTP status and `outcome = invalid_request`. + +#### Scenario: Unauthorized image edit does not consume the body + +- **WHEN** an image-edit request fails the existing proxy API-key authorization +- **THEN** the authentication response is returned before the ASGI request body is consumed +- **AND** no multipart temporary file is created +- **AND** exactly one auth-error route observation is recorded without parsing the multipart body, using bounded pre-parse labels + +#### Scenario: Bounded image edit remains compatible + +- **WHEN** an authorized image-edit request supplies at least one source image, an optional mask, required text fields, and all parts are within their limits +- **THEN** the service preserves the ordered `image` and `image[]` bytes, content types, mask, and validated form fields through the existing image-edit pipeline + +#### Scenario: Source image count combines canonical and bracketed keys + +- **WHEN** the combined number of `image` and `image[]` file parts exceeds 16, the request contains more than one `mask`, or an unknown file-part name is present +- **THEN** the service returns an OpenAI-compatible HTTP 400 invalid-request response +- **AND** no image bytes are base64-encoded or forwarded internally + +#### Scenario: Declared or streamed image-edit body exceeds its limit + +- **WHEN** a usable `Content-Length` exceeds 64 MiB or actual streamed multipart bytes cross 64 MiB +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large` and `type = invalid_request_error` +- **AND** no usage reservation, account selection, base64 conversion, or internal Responses request occurs + +#### Scenario: Image binary limit is exceeded + +- **WHEN** one source image or mask reaches 50,000,000 bytes, or their combined binary bytes reach 50,000,000 +- **THEN** the service returns HTTP 413 with OpenAI error `code = payload_too_large`, `type = invalid_request_error`, and the applicable `image` or `mask` parameter +- **AND** bytes beyond the applicable limit are not retained in a spool or handler buffer + +#### Scenario: Image text-field resources are bounded + +- **WHEN** an image-edit request exceeds 32 text fields or 256 KiB in any text part +- **THEN** the service rejects the request with the documented OpenAI-compatible count or byte-limit response +- **AND** it records one invalid-request route observation without invoking image-edit route logic + +#### Scenario: Compressed image edit is rejected without prebuffering + +- **GIVEN** image edit has passed proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` +- **THEN** the service returns HTTP 400 with OpenAI error `code = invalid_request_error` and `type = invalid_request_error` before reading the request body +- **AND** a no-op `identity` encoding is handled as an ordinary multipart request governed by the 64 MiB dedicated body limit +- **AND** exactly one invalid-request route observation is recorded without parsing the multipart body + +#### Scenario: Generic ingress does not preempt encoded image-edit authorization + +- **GIVEN** an image-edit request fails proxy authorization +- **WHEN** it declares a non-identity `Content-Encoding` and a `Content-Length` greater than the generic raw HTTP budget +- **THEN** the existing authentication response is returned instead of a generic HTTP 413 or encoded-body HTTP 400 +- **AND** the request body is not consumed +- **AND** exactly one auth-error route observation is recorded + +#### Scenario: Image-edit cleanup preserves transport failures + +- **WHEN** parsing succeeds, fails a limit, encounters malformed multipart, receives a client disconnect, or is cancelled +- **THEN** every created multipart spool is closed +- **AND** disconnect and cancellation are not converted to HTTP 413 + diff --git a/openspec/specs/model-catalog-compat/spec.md b/openspec/specs/model-catalog-compat/spec.md index fd742b4043..8cbc2cc3ba 100644 --- a/openspec/specs/model-catalog-compat/spec.md +++ b/openspec/specs/model-catalog-compat/spec.md @@ -375,6 +375,22 @@ capability. Requests that omit a tier or use the omit-equivalent `auto` or `default` tiers MUST use model-only account filtering, including when reusing an HTTP bridge session. +A service tier imposed by an API key's enforced service tier is not an explicit +request for that tier. When the requested tier originates from API key +enforcement and the model's catalog does not advertise that tier at all, the +system MUST remove the tier from the account-routed request, MUST select +accounts and reuse HTTP bridge sessions using model-only filtering, MUST +reserve, settle, and log API-key usage at the effective default tier, and MUST +omit the unsupported tier from the upstream request. This account-catalog +fallback MUST NOT alter a request selected for an external model source, and an +unknown or account-catalog-absent model MUST retain the enforced tier. When the model's catalog +does advertise the tier, account-level tier filtering MUST continue to apply +regardless of the tier's origin. A tier supplied explicitly by the client MUST +continue to filter accounts even when it equals the enforced value or uses an +equivalent alias and the model does not advertise it. When an +unavailable service tier is what excluded every account, the selection error +MUST name that tier. + #### Scenario: Same-plan accounts expose different models - **GIVEN** two active accounts share a plan @@ -390,6 +406,38 @@ an HTTP bridge session. - **WHEN** a request explicitly asks for priority - **THEN** selection considers only the account that advertised priority +#### Scenario: Enforced tier does not exclude a model that never advertises it + +- **GIVEN** an active account advertises a model at its default tier +- **AND** the model's catalog advertises no `priority` service tier +- **AND** an API key sets `enforced_service_tier` to `priority` +- **WHEN** account selection is requested for that model +- **THEN** the enforced tier is removed from the account-routed request +- **AND** API-key accounting, bridge compatibility, and upstream forwarding use the effective default tier +- **AND** the advertising account is selected + +#### Scenario: Account-catalog fallback does not alter a model source + +- **GIVEN** an API key enforces the `priority` service tier +- **AND** the selected model is routed through an external model source +- **WHEN** the subscription-account catalog does not advertise `priority` for that model +- **THEN** the source-routed request retains `priority` + +#### Scenario: Explicitly requested unadvertised tier is still rejected + +- **GIVEN** an active account advertises a model at its default tier +- **AND** the model's catalog advertises no `priority` service tier +- **WHEN** a client explicitly requests that model with `priority` or an equivalent `fast` alias +- **THEN** no account is selected + +#### Scenario: Unavailable advertised tier names the tier in the error + +- **GIVEN** a model's catalog advertises the `priority` service tier +- **AND** no active account carries `priority` for that model +- **WHEN** account selection is requested for that model with `priority` +- **THEN** no account is selected +- **AND** the selection error names the `priority` service tier + ### Requirement: Unknown account catalogs degrade without false exclusion The system MUST distinguish an account catalog that successfully omitted a @@ -833,3 +881,179 @@ At startup every replica SHALL load the persisted model-registry snapshot into i - **WHEN** the replica loses leadership and its next reconcile runs (poller callback or refresh-tick backstop) - **THEN** it drops the unpublished catalog, reverts to the bootstrap floor, and invalidates its account-selection cache +### Requirement: Every Codex-native catalog entry is wire-parseable + +Every model entry returned by `GET /backend-api/codex/models` or the equivalent +`GET /v1/models?client_version=` route MUST include the non-defaulted +Codex wire fields `truncation_policy` and `experimental_supported_tools`, even +when the entry comes from hidden retained bootstrap metadata or a persisted +legacy registry snapshot. When either field is absent from stored raw metadata, +the mapper MUST provide a conservative model-compatible default. Wire-valid +values provided by a live upstream catalog or model source MUST remain +authoritative and MUST NOT be overwritten by the compatibility defaults. When +`experimental_supported_tools` is not a list, the mapper MUST emit an empty +list. When it contains non-string members, the mapper MUST omit those members +rather than failing the complete catalog. A wire-valid `truncation_policy` MUST +use the `bytes` or `tokens` mode and a JSON integer representable by Codex's +signed 64-bit `limit` field. When an explicit policy does not satisfy that wire +shape, the mapper MUST emit the same conservative model-compatible policy used +when the field is absent. + +#### Scenario: Hidden bootstrap metadata cannot invalidate the live catalog + +- **GIVEN** a successful live refresh omits an older bundled model +- **AND** codex-lb retains that model as hidden metadata whose raw payload lacks + required Codex wire fields +- **WHEN** a Codex client requests the native model catalog +- **THEN** the hidden entry includes a valid `truncation_policy` +- **AND** it includes `experimental_supported_tools` as a list +- **AND** the complete catalog can be deserialized instead of falling back to + bundled client metadata + +#### Scenario: Explicit valid upstream compatibility values win + +- **GIVEN** a live catalog or model source provides `truncation_policy` or + `experimental_supported_tools` +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it preserves those explicit values unchanged + +#### Scenario: Invalid source tool members cannot fail the catalog + +- **GIVEN** a model source provides `experimental_supported_tools` with both + string and non-string members +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it retains the string tool names +- **AND** it omits non-string members instead of returning a server error + +#### Scenario: Non-list source tool metadata cannot fail the catalog + +- **GIVEN** a model source provides a non-list value for + `experimental_supported_tools` +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it emits an empty list instead of returning a server error + +#### Scenario: Malformed source truncation policy cannot fail the catalog + +- **GIVEN** a model source provides an invalid `truncation_policy`, such as a + null, non-object, incomplete object, unknown mode, non-integer limit, or + out-of-range limit +- **WHEN** codex-lb renders the Codex-native catalog entry +- **THEN** it emits the conservative model-compatible truncation policy +- **AND** it does not return a server error + +#### Scenario: Client-version alias has the same complete contract + +- **WHEN** Codex requests `GET /v1/models` with a non-empty `client_version` +- **THEN** every returned `models` entry satisfies the same required-field + contract as `GET /backend-api/codex/models` + +### Requirement: Fresh additional-quota evidence can establish account support + +For a model canonically mapped to a separately metered additional quota, account selection MUST allow fresh account-specific additional-quota telemetry to establish model support when an authoritative general per-account model catalog omits that model. The system MUST continue to enforce registry plan and service-tier restrictions and MUST apply the existing additional-quota freshness, exhaustion, account-health, cooldown, capacity, security, and routing gates before selecting an account. When such a selected account is bound to an HTTP bridge session, every existing-session reuse entry point, including direct key lookup, previous-response alias fallback, and in-flight creation waiters, MUST enforce exact normalized model, canonical quota key, and normalized effective service-tier compatibility before returning the session. For a genuinely catalog-omitted account, reuse MUST re-evaluate current registry plan and requested service-tier plan eligibility without synchronously re-reading quota telemetry. This behavior MUST NOT apply to unknown models or to an unrelated additional-limit key supplied independently of the requested model. + +#### Scenario: Fresh Spark quota overrides general account-catalog omission + +- **GIVEN** an authoritative general account catalog omits `gpt-5.3-codex-spark` for a plan-compatible active account +- **AND** that account has fresh, non-exhausted `codex_spark` quota telemetry +- **WHEN** account selection is requested for `gpt-5.3-codex-spark` +- **THEN** the general account-catalog omission does not remove that account from consideration +- **AND** the account proceeds through the remaining additional-quota and routing gates + +#### Scenario: Quota-admitted bridge session remains reusable + +- **GIVEN** an account omitted from the authoritative general account catalog was selected for `gpt-5.3-codex-spark` using fresh, non-exhausted `codex_spark` telemetry +- **AND** an HTTP bridge session records that selection's normalized model, canonical quota key, and effective service tier +- **WHEN** a later turn requests the same normalized model, canonical quota mapping, and effective service tier +- **THEN** the existing bridge session remains reusable +- **AND** the synchronous reuse check does not re-read quota telemetry + +#### Scenario: Bridge admission provenance is narrowly bound + +- **GIVEN** an HTTP bridge session carries quota-backed catalog-omission provenance +- **WHEN** a later request reaches that session through direct key lookup, previous-response alias fallback, or an in-flight creation waiter with a different normalized model, canonical quota key, or effective service tier +- **THEN** that provenance does not bypass the normal catalog and service-tier checks +- **AND** a catalog-supported account rejected by the requested account-level service-tier index remains rejected + +#### Scenario: Reuse rechecks current plan-tier eligibility for a catalog omission + +- **GIVEN** an HTTP bridge session carries exact quota-backed catalog-omission provenance for a requested service tier +- **AND** the registry's current requested service-tier plan restrictions exclude the session account's current plan +- **WHEN** a later request reaches that session through any reuse entry point +- **THEN** the existing session is not returned under the recorded provenance +- **AND** the current request follows a request-scope fork or fail-closed path without synchronously re-reading quota telemetry or mutating the existing live session + +#### Scenario: Incompatible request preserves another request's live bridge state + +- **GIVEN** a live or in-flight HTTP bridge session is compatible with its creator request +- **AND** another direct, previous-response-alias, turn-state-alias, or in-flight-waiter request has mismatched quota-backed admission provenance or current plan-tier eligibility +- **WHEN** bridge request compatibility rejects that second request +- **THEN** an unanchored request uses an independent collision-resistant request-scope session, or an anchored request alone fails closed +- **AND** the creator's session remains registered, open, and unscheduled for close with its request model, service tier, and transport unchanged +- **AND** live previous-response and turn-state aliases remain unchanged so a subsequent compatible request can resolve and reuse the owner +- **AND** an alias mapping is removed only when its target is missing, closed, or inactive + +#### Scenario: Forwarded prompt-cache mismatch forks on the receiving owner + +- **GIVEN** two bridge replicas agree that a prompt-cache key belongs to one canonical owner +- **AND** that owner has an open quota-admitted Spark session whose effective service tier is incompatible with a priority request already forwarded to the owner +- **AND** the priority request's collision-resistant `internal_request_parallel` fork key rendezvous-hashes to the other replica +- **WHEN** compatibility rejects either the registered session or a session returned to an in-flight creation waiter +- **THEN** the receiving canonical owner creates and owns the request-local mismatch fork without forwarding again +- **AND** both requests can complete on independent transports while the creator session remains open and registered +- **AND** normal rendezvous ownership remains unchanged for canonical prompt-cache, session, turn-state, previous-response, and unforwarded fork keys + +#### Scenario: Catalog-supported account-level service-tier exclusion remains authoritative + +- **GIVEN** an authoritative general per-account catalog includes a mapped separately metered model for two plan-compatible accounts +- **AND** the authoritative requested service-tier account index includes only one of those accounts +- **AND** both accounts have fresh, non-exhausted additional-quota telemetry for the model +- **WHEN** account selection requests that model and service tier +- **THEN** the account absent from the requested service-tier account index is not selected +- **AND** quota evidence does not reclassify that catalog-supported account as model-catalog-omitted + +#### Scenario: Plan incompatibility remains authoritative + +- **GIVEN** a requested separately metered model is mapped to an additional quota +- **AND** an account's plan is excluded by the model registry's plan or requested service-tier restrictions +- **WHEN** account selection evaluates that account +- **THEN** the account is not selected even if additional-quota telemetry exists + +#### Scenario: Missing or stale quota evidence fails closed + +- **GIVEN** the general account catalog omits a mapped separately metered model +- **AND** no plan-compatible account has fresh additional-quota telemetry for that model +- **WHEN** account selection is requested +- **THEN** selection fails with the existing additional-quota data-unavailable behavior +- **AND** the system does not route based only on bootstrap metadata +- **AND** no quota-backed HTTP bridge session is admitted from that failed selection + +#### Scenario: Explicit unrelated quota cannot bypass model support + +- **GIVEN** a caller supplies an additional-limit key that is not the requested model's canonical quota mapping +- **WHEN** the general per-account catalog excludes an account for that model +- **THEN** the supplied quota key does not override the account-catalog exclusion + +### Requirement: Model catalog reservations are released on every exit path + +The model catalog builders for `GET /v1/models` and `GET /backend-api/codex/models` SHALL release the API-key usage reservation after acquisition on normal return, exception, or cancellation. The builders MUST preserve the existing reservation amount and successful response shape. + +#### Scenario: OpenAI-compatible catalog lookup fails + +- **WHEN** `_list_enabled_source_catalog_models` raises after reservation + acquisition while serving `GET /v1/models` +- **THEN** the reservation row is released +- **AND** its reserved usage is no longer charged to the key + +#### Scenario: Codex-native catalog lookup fails + +- **WHEN** `_list_enabled_source_catalog_models` raises after reservation + acquisition while serving `GET /backend-api/codex/models` +- **THEN** the reservation row is released +- **AND** its reserved usage is no longer charged to the key + +#### Scenario: Catalog request is cancelled + +- **WHEN** either model catalog builder is cancelled after reservation + acquisition +- **THEN** the reservation is released before cancellation propagates + diff --git a/openspec/specs/proxy-admission-control/spec.md b/openspec/specs/proxy-admission-control/spec.md index cc47fa7cac..45192cad8b 100644 --- a/openspec/specs/proxy-admission-control/spec.md +++ b/openspec/specs/proxy-admission-control/spec.md @@ -50,6 +50,8 @@ For `/v1/responses`, `/backend-api/codex/responses`, and compact Responses traff When an account is at either cap, new soft-affinity work MUST prefer another eligible account before returning local overload. A bare process-session mapping MAY supply soft locality only while the request is self-contained, pre-visible, and has no required owner. Account-cap spillover MUST be decided during account selection and MUST NOT switch an account after a request enters shared transport, replay, or durable bridge ownership. Hard-continuity work MUST remain on its required owner and MAY fail closed when that owner is saturated. Hard Codex ownership rows MUST bypass soft sticky fallback/reallocation so pressure cannot delete or rewrite them. +An unanchored parallel fork bridge session whose payload is self-contained (no `previous_response_id`, no `conversation`, and no input file references) and whose current request context has no turn-state owner or anchored forwarding provenance carries no continuity ownership. When its preferred account is rejected by a local account cap (`account_stream_cap` or `account_response_create_cap`) during session creation, the proxy MUST drop the preferred-account hint exactly once for that request and retry account selection among eligible accounts before entering the recoverable account-capacity wait. Requests that carry any continuity owner signal MUST NOT spill and MUST keep the existing preferred-owner behavior, even when durable alias lookup resolves to an `internal_unanchored_parallel` canonical key. + #### Scenario: Soft work avoids saturated account - **GIVEN** account A is at its account response-create cap @@ -79,6 +81,27 @@ When an account is at either cap, new soft-affinity work MUST prefer another eli - **THEN** the request follows the existing hard bridge-capacity behavior - **AND** account-cap spillover does not publish a replacement bridge under the same canonical identity +#### Scenario: Unanchored parallel fork spills off a capped preferred account + +- **GIVEN** an unanchored parallel fork bridge session creation whose payload carries no previous response, conversation, or input file reference +- **AND** its preferred account is rejected with `account_stream_cap` +- **AND** another eligible account is below its stream cap +- **WHEN** session creation retries selection after dropping the preferred-account hint +- **THEN** the fork session is created on the eligible account instead of waiting on the capped account + +#### Scenario: Owner-bearing fork payloads do not spill + +- **GIVEN** a parallel fork bridge session creation whose payload carries a `previous_response_id` +- **WHEN** its preferred account is rejected with a local account cap +- **THEN** the preferred-account hint is kept and the existing preferred-owner behavior applies + +#### Scenario: Turn-state aliases do not spill through an unanchored canonical key + +- **GIVEN** a request carries a turn-state alias whose durable row resolves to an `internal_unanchored_parallel` canonical key +- **AND** that row has a latest turn state but no latest response ID +- **WHEN** its owner account is rejected with a local account cap +- **THEN** the preferred-account hint is kept and the request does not spill to another account + ### Requirement: Local overload reasons are stable and distinguishable Local Responses overload failures MUST expose stable low-cardinality reason fields in logs and metrics so operators can distinguish `bridge_queue_full`, `response_create_gate_timeout`, `hard_affinity_saturated`, `previous_response_owner_unavailable`, `global_admission_timeout`, `capacity_exhausted_active_sessions`, `account_response_create_cap`, and `account_stream_cap`. These local reasons MUST NOT be reported as upstream rate limits. @@ -458,3 +481,195 @@ Per-account concurrency caps are partitioned per bridge-ring replica and are cor - **WHEN** the process loads its settings at startup - **THEN** startup fails with a settings validation error naming `CODEX_LB_WORKERS_PER_INSTANCE` - **AND** the error states multi-worker-per-instance is not supported and directs the operator to run one worker per pod/container and scale via replicas + +### Requirement: Stream leases reflect in-flight turns, not session lifetime + +An HTTP bridge session's per-account stream lease MUST be held only while the session has in-flight work. When a session's last in-flight turn detaches — no queued requests, no admission waiters, and no pending requests — the session MUST release its account stream lease while remaining alive for reuse, so a warm idle upstream WebSocket does not occupy a per-account stream slot for its idle TTL. Cancellation MUST NOT interrupt that idle lease settlement after the lease is detached from the session. A turn admitted to a session holding no lease MUST reacquire one under normal cap admission before it is counted into the session queue, and a denied reacquisition MUST fail with the standard HTTP 429 `account_stream_cap` envelope so the recoverable capacity wait and client retry semantics apply unchanged. Reacquisition MUST carry the turn's usage-budget token estimate into the lease, matching initial bridge selection and reconnect, so capacity-weighted routing pressure continues to see turns running on reused warm sessions. The stream recovery reserve MUST NOT be consulted at reacquisition, consistent with the reserve being a selection-time reserve. Session close MUST keep its existing lease settlement; a session that already released while idle has nothing further to settle. + +The lease remains per-session, matching the pre-existing lease lifecycle: a session MUST hold at most one stream lease at a time, and turns queued on a session that already holds a lease MUST NOT acquire additional leases — queued turns multiplex over the session's single upstream stream, which is what the per-account stream cap bounds. If the session closes while a reacquisition is in flight, the freshly acquired lease MUST be released back rather than installed on the closed session, and the turn MUST fail with the standard closed-bridge error envelope. Cancellation MUST NOT interrupt release of that detached lease. A submit MUST be registered as in-flight work (admission waiter) atomically with its lease reacquisition, so a completed turn's finalizer running concurrently cannot observe the session as idle and release the reacquired lease before the new turn is counted into the session queue. Any failure after waiter registration and before queue admission MUST remove that waiter and settle an otherwise-idle lease. Reconnect and reacquisition MUST serialize changes to the session lease so a reconnect lease cannot be overwritten and leaked by a concurrent reacquisition. Cancellation MUST NOT interrupt settlement of a lease detached during reconnect replacement. If prewarm fails after the upstream reader closes the session and defers retirement for that admission waiter, removing the final waiter MUST retire the closed session and release its stream lease. Prewarm cancellation MUST NOT interrupt removal of the admission waiter or settlement of an otherwise-idle stream lease. + +#### Scenario: Finished turn returns the account's stream slot + +- **GIVEN** a bridge session whose only in-flight turn completes +- **WHEN** the turn's stream finalizes and detaches +- **THEN** the session releases its account stream lease +- **AND** the session remains alive for reuse within its idle TTL + +#### Scenario: Idle sessions do not starve new admissions + +- **GIVEN** an account at its stream cap where some leases belong to idle sessions +- **WHEN** those sessions' turns complete +- **THEN** the freed slots admit new work immediately +- **AND** the freed slots are not held until the idle sessions' TTL expiry + +#### Scenario: Next turn on an idle session passes cap admission + +- **GIVEN** an idle bridge session that released its stream lease +- **WHEN** a new turn is admitted to that session +- **THEN** the session reacquires a stream lease before the turn is counted into the session queue + +#### Scenario: Reacquisition denial uses the standard cap envelope + +- **GIVEN** an idle bridge session whose account is at its stream cap +- **WHEN** a new turn's lease reacquisition is denied +- **THEN** the turn fails with HTTP 429 and `error.code = "account_stream_cap"` +- **AND** the recoverable account-capacity wait applies to the retry + +#### Scenario: Close racing reacquisition does not leak the slot + +- **GIVEN** an idle bridge session whose stream lease reacquisition is awaiting cap admission +- **WHEN** the session is closed or evicted before the acquisition completes +- **THEN** the freshly acquired lease is released back to the account +- **AND** the turn fails with the standard closed-bridge error envelope + +#### Scenario: Cancellation during close-race settlement does not leak the slot + +- **GIVEN** a session closes while reacquisition is awaiting cap admission +- **AND** the submit is cancelled while the freshly acquired lease is being returned +- **WHEN** lease settlement completes +- **THEN** cancellation propagates only after the lease is released + +#### Scenario: Stale finalizer cannot release a lease reacquired for a new turn + +- **GIVEN** a warm session whose new turn has reacquired a stream lease but is not yet counted into the session queue +- **WHEN** a previous turn's finalizer runs its idle-release check concurrently +- **THEN** the session is not considered idle +- **AND** the reacquired lease is retained for the new turn + +#### Scenario: Failed queue admission removes its waiter + +- **GIVEN** a submit has registered an admission waiter before queue admission +- **WHEN** its final lease check fails +- **THEN** the admission waiter is removed +- **AND** an otherwise-idle session releases its stream lease + +#### Scenario: Reconnect racing reacquisition retains one lease + +- **GIVEN** a reconnect and idle-session lease reacquisition overlap +- **WHEN** both acquire a stream lease before either operation completes +- **THEN** the session retains exactly one of those leases +- **AND** the losing lease is released immediately + +#### Scenario: Queued turns share the session's single stream slot + +- **GIVEN** a bridge session that holds a stream lease for an active turn +- **WHEN** additional turns are admitted to the session queue +- **THEN** no additional stream leases are acquired +- **AND** the session continues to hold exactly one stream lease + +#### Scenario: Prewarm failure retires a closed session after its waiter leaves + +- **GIVEN** a new turn has reacquired a stream lease and registered an admission waiter +- **AND** the upstream reader closes the session during prewarm and defers retirement for that waiter +- **WHEN** prewarm fails and the final admission waiter is removed +- **THEN** the closed session is retired +- **AND** its stream lease is released + +#### Scenario: Prewarm cancellation completes lease cleanup + +- **GIVEN** a new turn has reacquired a stream lease and registered an admission waiter +- **WHEN** the downstream task is cancelled during prewarm +- **THEN** cleanup removes the admission waiter before propagating cancellation +- **AND** an otherwise-idle session releases its stream lease + +#### Scenario: Grouped terminal errors release an abandoned session's lease + +- **GIVEN** a bridge session whose only pending turns are detached follow-ups (no downstream consumers remain) +- **WHEN** a grouped terminal error (for example `previous_response_not_found`) settles all of them together +- **THEN** the session releases its account stream lease +- **AND** the freed slot admits new work without waiting for session close or idle TTL expiry + +#### Scenario: Busy sessions keep their lease + +- **GIVEN** a bridge session with another turn still queued or pending +- **WHEN** one of its turns detaches +- **THEN** the session's stream lease is retained + +### Requirement: Stream admission applies congestion-aware per-API-key fair share + +When `proxy_api_key_fair_share_congestion_threshold_pct` is greater than zero, stream-lease selection MUST evaluate a per-API-key fair-share gate over the selection's candidate account set before admitting a stream. Pool capacity MUST be computed as the candidate-account count multiplied by each account's effective stream slots (`max(1, stream_limit - stream_reserve_slots)`), pool in-flight as the sum of the candidate accounts' in-flight stream leases, and both compared with integer arithmetic: the pool is congested if and only if `pool_inflight * 100 >= pool_capacity * threshold_pct`. When the pool is not congested the gate MUST admit unconditionally. When the pool is congested the gate MUST admit a key only if the key's in-flight stream count on the candidate accounts plus one does not exceed `max(2, pool_capacity // active_keys)`, where `active_keys` is the number of API keys holding at least one in-flight stream lease on the candidate accounts with the requester counted exactly once. The gate MUST NOT apply when the configured threshold is zero, when the request carries no API key, when the selection is for a reattach stage, when the lease kind is not stream, or when the effective stream limit is nonpositive; keyless streams MUST still count toward pool in-flight. The gate MUST NOT read the database and MUST evaluate under the same runtime lock that guards lease counters. + +#### Scenario: Disabled threshold changes no admission outcome + +- **GIVEN** `proxy_api_key_fair_share_congestion_threshold_pct` is 0 (the default) +- **WHEN** any mix of API keys saturates the pool's stream slots +- **THEN** every selection outcome is identical to the behavior before this change + +#### Scenario: Uncongested pool admits an already-heavy key + +- **GIVEN** a threshold of 80 and pool utilization below 80% +- **AND** one key already holds more streams than `pool_capacity // active_keys` +- **WHEN** that key requests another stream +- **THEN** the request is admitted + +#### Scenario: Congested pool denies a key at or above its fair share + +- **GIVEN** a threshold of 80 and pool utilization at or above 80% +- **AND** a key holding at least `max(2, pool_capacity // active_keys)` in-flight streams +- **WHEN** that key requests another stream +- **THEN** selection returns the stable reason `api_key_stream_fair_share` and no lease is acquired + +#### Scenario: Minimum guarantee admits light keys under congestion + +- **GIVEN** a congested pool dominated by another key's streams +- **WHEN** a key holding fewer than two in-flight streams requests a stream +- **THEN** the fair-share gate admits it + +#### Scenario: Requester is counted exactly once in the divisor + +- **GIVEN** a congested pool where the requester already holds in-flight streams +- **WHEN** the fair share is computed +- **THEN** `active_keys` counts the requester once and does not change whether the requester is currently active or newly arriving + +#### Scenario: Keyless requests bypass the gate but consume capacity + +- **GIVEN** a congested pool +- **WHEN** a request without an API key selects an account +- **THEN** the fair-share gate does not deny it +- **AND** its in-flight stream counts toward pool in-flight for keyed requesters + +#### Scenario: Reattach-stage selection bypasses the gate + +- **GIVEN** a congested pool and a heavy key at its fair share +- **WHEN** that key's reattach-stage selection resumes an existing in-flight response +- **THEN** the fair-share gate does not deny it + +### Requirement: Fair-share denials reuse local capacity-wait semantics + +A fair-share denial MUST surface the stable local-overload reason `api_key_stream_fair_share` and MUST inherit the existing account-capacity handling: the transport layer parks the request with `waiting_for_account_capacity` keepalives and retries selection within the request budget, and a request that exhausts its budget while denied MUST receive HTTP 429 with `error.type` `rate_limit_error` and a `Retry-After` header rather than a 503. The denial message MUST state the key's in-flight count, the fair share, the pool in-flight and capacity, and the active-key count without naming other API keys. + +#### Scenario: Denied request parks and admits after the pool decongests + +- **GIVEN** a heavy key denied by the fair-share gate +- **WHEN** enough streams release for the key to fall under its fair share or the pool to fall below the threshold +- **THEN** a subsequent parked retry admits the request without client intervention + +#### Scenario: Budget exhaustion surfaces 429 with fair-share numbers + +- **GIVEN** a request that remains fair-share denied until its budget is exhausted +- **WHEN** the terminal error is rendered +- **THEN** the status is 429 with `error.type` `rate_limit_error` and a `Retry-After` header +- **AND** the message includes the key in-flight count, fair share, pool in-flight, pool capacity, and active-key count + +### Requirement: Per-API-key stream accounting follows the lease lifecycle + +Every stream lease acquired through account selection MUST record the requesting API key, and the per-account per-key in-flight map MUST be maintained under the runtime lock across acquire, explicit release, and stale reclaim, with map entries removed when a key's count reaches zero and removed together with pruned account runtime state. Account-scoped keys MUST be measured against their scoped candidate accounts only. On the sticky selection path the gate decision MUST be re-validated in the commit lock section before the lease is acquired, so concurrent selections for one key cannot overshoot the share between the filter and commit sections; the unbound path MUST evaluate the gate and acquire the lease in a single lock section. + +#### Scenario: Release and stale reclaim decrement the owning key + +- **GIVEN** a key holding in-flight stream leases +- **WHEN** a lease is released explicitly or reclaimed as stale +- **THEN** that key's in-flight count decreases accordingly and its map entry is removed at zero + +#### Scenario: Scoped key is measured against its scoped pool + +- **GIVEN** a key restricted to a subset of accounts via account assignment scope +- **WHEN** the fair-share gate evaluates its request +- **THEN** pool capacity, pool in-flight, and the key's in-flight count are computed over the scoped candidate accounts only + +#### Scenario: Concurrent sticky selections cannot overshoot the share + +- **GIVEN** a congested pool and one key one stream below its fair share +- **WHEN** two sticky selections for that key pass the filter-phase gate concurrently +- **THEN** at most one acquires a lease and the other is denied at the commit re-check + diff --git a/openspec/specs/proxy-architecture/spec.md b/openspec/specs/proxy-architecture/spec.md new file mode 100644 index 0000000000..e0f0168804 --- /dev/null +++ b/openspec/specs/proxy-architecture/spec.md @@ -0,0 +1,73 @@ +# proxy-architecture Specification + +## Purpose +Structural fitness gates for the proxy: ProxyService stays a stable façade and internal decomposition (selection orchestration, bridge mixins) cannot drift behavior or re-grow god-modules. +## Requirements +### Requirement: Proxy architecture fitness gates are enforced + +The repository SHALL enforce the accepted proxy architecture thresholds during +the required lint gate. `app/modules/proxy/service.py` SHALL contain no more +than 2,600 lines, `app/modules/proxy/load_balancer.py` SHALL contain no more +than 3,021 lines, and `LoadBalancer.select_account()` SHALL span no more than +527 lines. Implementations SHALL restore or lower these ratchets rather than +increase, bypass, or remove them to make CI pass. + +#### Scenario: Multiple ratchets are violated + +- **WHEN** more than one independent proxy architecture threshold or boundary is violated +- **THEN** one architecture-check run reports every independently evaluable violation in deterministic order +- **AND** the check exits non-zero + +#### Scenario: All architecture gates pass + +- **WHEN** every proxy architecture threshold and boundary is satisfied +- **THEN** the architecture check exits zero +- **AND** it reports that the proxy architecture checks passed + +### Requirement: ProxyService remains a stable façade + +`app.modules.proxy.service.ProxyService` and the required compatibility exports +SHALL remain available to existing consumers. Behavior extracted from +`ProxyService` or `service.py` SHALL be owned by focused private modules under +`app/modules/proxy/_service/`. +Compatibility shims SHALL remain re-export-only and private service domains +SHALL comply with the repository's explicit cross-domain dependency policy. + +#### Scenario: Existing consumers import the proxy façade + +- **WHEN** an existing caller imports `ProxyService` or a required compatibility export from `app.modules.proxy.service` +- **THEN** the import resolves to behavior compatible with the pre-change façade +- **AND** no caller migration is required + +### Requirement: Account selection orchestration is decomposed without behavior drift + +`LoadBalancer.select_account()` SHALL remain the public account-selection entry +point and SHALL delegate cohesive sticky-key retry orchestration and policy to a +private, protocol-typed load-balancer implementation unit. The decomposition +MUST preserve account scope, continuity ownership, security authorization, +exclusions, routing policy, quota and health filtering, concurrency caps, +affinity, stale-state retries, lease cleanup, persistence, result metadata, and +error-code behavior. + +#### Scenario: Selection succeeds with or without stickiness + +- **WHEN** a request is eligible for account selection with either a sticky key or no sticky key +- **THEN** the selected account, lease, persisted runtime state, and result metadata match the pre-change behavior for the same inputs + +#### Scenario: Ownership or capacity prevents selection + +- **WHEN** continuity ownership is ambiguous or conflicting, a hard-affinity owner is unavailable, or account caps are exhausted +- **THEN** selection returns the same fail-closed outcome, error code, and mapping-preservation behavior as before the decomposition + +#### Scenario: Persistence or cancellation interrupts selection + +- **WHEN** persistence fails, a selected row becomes stale, or the selection task is cancelled +- **THEN** acquired leases are released exactly once +- **AND** retries and final errors follow the existing bounded behavior + +#### Scenario: Non-sticky selection observes a cache-generation change + +- **WHEN** non-sticky selection acquires a lease and the selection-input cache generation changes during persistence +- **THEN** the acquired lease is released exactly once +- **AND** non-sticky selection reloads its inputs and retries within the existing bound + diff --git a/openspec/specs/proxy-runtime-observability/spec.md b/openspec/specs/proxy-runtime-observability/spec.md index 4dab9abd80..d715e5cca3 100644 --- a/openspec/specs/proxy-runtime-observability/spec.md +++ b/openspec/specs/proxy-runtime-observability/spec.md @@ -184,13 +184,22 @@ analytics. - **AND** request-log metadata stores `upstream_status_code = null` ### Requirement: Request logs persist prompt-client user-agent metadata -The proxy MUST persist prompt-client user-agent metadata on `request_logs` for both HTTP and WebSocket Responses traffic. Each persisted row MUST store the full inbound `User-Agent` header value when present and a derived `useragent_group` value extracted from the first product token. When the inbound header is missing or blank after trimming, both persisted values MUST be `null`. +The proxy MUST persist prompt-client user-agent metadata on `request_logs` for both HTTP and WebSocket Responses traffic. Each persisted row MUST store the full inbound `User-Agent` header value when present and a derived `useragent_group` value. When the inbound header contains `/`, `useragent_group` MUST be the complete sequence of characters before its first `/`; when it contains no `/`, the existing group extraction behavior MUST remain unchanged. When the inbound header is missing or blank after trimming, both persisted values MUST be `null`. + +#### Scenario: Historical request-log user-agent families are backfilled without normalization +- **WHEN** the user-agent family migration processes historical `request_logs` rows +- **THEN** rows whose `useragent` is non-null and contains `/` MUST have `useragent_group` set to the exact unprocessed full prefix before the first `/` +- **AND** rows whose `useragent` is `null` or contains no `/` MUST remain unchanged #### Scenario: HTTP request log stores user-agent metadata - **WHEN** an HTTP or HTTP/SSE proxy request includes `User-Agent: opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14` - **THEN** the persisted `request_logs` row stores `useragent = "opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14"` - **AND** the persisted row stores `useragent_group = "opencode"` +#### Scenario: Multi-word product family retains its full prefix +- **WHEN** an HTTP or HTTP/SSE proxy request includes `User-Agent: Codex Desktop/0.142.4` +- **THEN** the persisted `request_logs` row stores `useragent_group = "Codex Desktop"` + #### Scenario: WebSocket request log stores user-agent metadata - **WHEN** a proxied WebSocket Responses session is opened with `User-Agent: opencode/1.15.13 ai-sdk/provider-utils/4.0.23 runtime/bun/1.3.14` - **THEN** the persisted `request_logs` row for that request stores the full header in `useragent` @@ -638,3 +647,141 @@ The service MUST expose a Prometheus gauge named `codex_lb_cap_partition_replica - **WHEN** a partition refresh observes and adopts two active members - **THEN** `codex_lb_cap_partition_replicas` reports 2 - **AND** an info-level log records the rebalance from count 1 to count 2 with the replica's rank + +### Requirement: Source-routed requests report upstream-measured generation timings + +The proxy MUST record upstream-reported generation timing on the request log +for source-routed chat/responses/audio-transcription requests when the +OpenAI-compatible source's response body includes a `metrics` object with +`time_to_first_token_ms` and `generation_time_ms`. The proxy MUST set +`latency_first_token_ms` to the reported time-to-first-token and `latency_ms` +to the sum of time-to-first-token and generation time, using the same +request-log fields subscription-backed requests already populate. Sources +that do not return a `metrics` object MUST leave both fields `null`, and +negative or non-numeric values MUST be rejected rather than recorded. +Non-finite numeric values (`NaN`, positive infinity, or negative infinity) +MUST also be rejected rather than failing or interrupting the proxied request. + +#### Scenario: Source metrics populate TTFT and total latency + +- **GIVEN** an OpenAI-compatible source's chat completion response includes + `metrics: {time_to_first_token_ms: 108.83, generation_time_ms: 162.98}` +- **WHEN** the request is logged +- **THEN** the request log's `latency_first_token_ms` is `109` +- **AND** the request log's `latency_ms` is `272` + +#### Scenario: Streamed responses capture metrics from the final frame + +- **GIVEN** a source-routed streaming chat completion whose final SSE frame + carries both `usage` and `metrics` +- **WHEN** the stream completes successfully +- **THEN** the request log records the same `latency_first_token_ms` / + `latency_ms` derived from that frame's `metrics` + +#### Scenario: Missing metrics leaves latency fields null + +- **GIVEN** an OpenAI-compatible source's response includes no `metrics` object +- **WHEN** the request is logged +- **THEN** `latency_ms` and `latency_first_token_ms` remain `null`, unchanged + from prior behavior + +#### Scenario: Dashboard retains generation-only throughput semantics + +- **GIVEN** a source response reports `time_to_first_token_ms: 108.83`, + `generation_time_ms: 162.98`, and `9` output tokens +- **WHEN** the existing dashboard computes tokens per second as output tokens + divided by `latency_ms - latency_first_token_ms` +- **THEN** it reports approximately `55.2` generation tokens per second +- **AND** it does not substitute an upstream `tokens_per_second` value that may + include TTFT + +#### Scenario: Non-finite metrics are ignored safely + +- **GIVEN** a source response contains `NaN` or infinity in either timing field +- **WHEN** the proxy parses the optional metrics +- **THEN** both timing values remain unset +- **AND** the otherwise successful proxied request is not interrupted + +### Requirement: Shipped high-error-rate alert uses aggregate request share + +The shipped `CodexLBHighErrorRate` alert MUST calculate, independently for each +namespace and job, the sum of five-minute 5xx request rates divided by the sum +of all five-minute request rates. Method, path, status, instance, replica, and +other non-scope labels MUST be aggregated before division. The alert MUST +compare the aggregate ratio to 0.05 and MUST require it to remain above that +threshold for five minutes. + +#### Scenario: Mixed success and error series produce their aggregate share + +- **GIVEN** one namespace and job have positive 2xx and 5xx request rates +- **WHEN** the high-error-rate alert expression is evaluated +- **THEN** the ratio equals the sum of 5xx request rates divided by the sum of + all request rates +- **AND** the ratio is not 1 unless all requests in that group are 5xx + +#### Scenario: Alert groups remain isolated + +- **GIVEN** request series exist for more than one namespace or job +- **WHEN** the high-error-rate alert expression is evaluated +- **THEN** each namespace and job pair has an independent aggregate ratio +- **AND** traffic from one pair is not included in another pair + +#### Scenario: Threshold and duration apply to the aggregate ratio + +- **GIVEN** one namespace and job have an aggregate 5xx share above 0.05 +- **WHEN** that aggregate share remains above 0.05 for five minutes +- **THEN** `CodexLBHighErrorRate` fires for that namespace and job pair + +### Requirement: Bundled Grafana 5xx stat uses selected aggregate request share + +The bundled Grafana `Error Rate (5xx)` stat MUST apply the selected namespace +and job filters to both operands, aggregate all remaining request-series labels +before division, and display the resulting 5xx share as one value. When the +selected total request rate is positive but no matching 5xx series exists, the +stat MUST display 0%. + +#### Scenario: Selected mixed traffic produces one aggregate value + +- **GIVEN** the selected namespace and job have positive 2xx and 5xx request + rates across one or more request or replica label combinations +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** it displays the sum of selected 5xx request rates divided by the sum + of all selected request rates + +#### Scenario: Dashboard selection filters both operands + +- **GIVEN** request series exist inside and outside the selected namespace and + job +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** both the 5xx numerator and total denominator exclude traffic outside + the selected namespace and job + +#### Scenario: Success-only traffic displays zero + +- **GIVEN** the selected scope has a positive successful-request rate +- **AND** no matching 5xx series exists +- **WHEN** the Grafana error-rate stat is evaluated +- **THEN** the stat displays 0% + +### Requirement: Stream pool congestion is observable + +When Prometheus support is available the service MUST expose a gauge named `codex_lb_stream_pool_capacity` whose value equals the fair-share gate's most recently computed candidate pool capacity and a gauge named `codex_lb_stream_pool_inflight` whose value equals the corresponding pool in-flight stream count, and a counter named `codex_lb_api_key_fair_share_rejections_total` incremented once per fair-share denial. The gauges and the counter MUST NOT carry API-key, account, or request labels. Each fair-share denial MUST log at warning level with the requesting `api_key_id`, the key's in-flight count, the computed fair share, the pool in-flight and capacity, and the active-key count, and MUST NOT include other keys' identifiers, instance secrets, or request payload content. All fair-share metrics MUST degrade to no-ops when the Prometheus client is absent. + +#### Scenario: Pool gauges are exported during gate evaluation + +- **GIVEN** the fair-share gate is enabled and evaluates a stream selection +- **WHEN** metrics are scraped +- **THEN** `codex_lb_stream_pool_capacity` and `codex_lb_stream_pool_inflight` report the evaluated pool values without per-key or per-account labels + +#### Scenario: Denials are counted without key cardinality + +- **GIVEN** repeated fair-share denials for multiple keys +- **WHEN** metrics are scraped +- **THEN** `codex_lb_api_key_fair_share_rejections_total` reflects the total denial count with no per-key label + +#### Scenario: Denial log carries the diagnostic numbers + +- **GIVEN** a fair-share denial +- **WHEN** the warning is logged +- **THEN** it includes the requester's `api_key_id`, key in-flight count, fair share, pool in-flight, pool capacity, and active-key count and no other key's identifier + diff --git a/openspec/specs/query-caching/spec.md b/openspec/specs/query-caching/spec.md index 4f6abf3ce3..0d4c827ecd 100644 --- a/openspec/specs/query-caching/spec.md +++ b/openspec/specs/query-caching/spec.md @@ -767,3 +767,86 @@ Non-PostgreSQL backends MUST NOT be affected (no visibility map). - **GIVEN** a PostgreSQL deployment where the identical autovacuum settings were already applied manually (the reference deployment's hotfix) - **WHEN** the autovacuum tuning revision is applied - **THEN** the migration MUST complete without error and leave the same settings in place + +### Requirement: Request-log listing totals are cached per filter signature + +The request-log listing MUST NOT execute an exact `COUNT(*)` over the filtered set on every page request; the total MUST be reused from a per-filter-signature cache within a fixed 30-second TTL (an application constant per the `reduce-settings-surface-phase-2` change — not an operator tunable). Cached totals are display-only: page contents themselves MUST remain exact and newest-first. + +#### Scenario: Repeated pages reuse the cached total + +- **GIVEN** two listing requests with the same filters but different offsets within the TTL +- **WHEN** both pages are served +- **THEN** the filtered set is counted once and both responses report the same total + +#### Scenario: Distinct filter signatures count independently + +- **WHEN** a listing request arrives with different filters +- **THEN** its total comes from its own count, not another signature's cache entry + +#### Scenario: Expired entries are recounted + +- **GIVEN** a cached total whose 30-second TTL has elapsed +- **WHEN** a listing request with the same filter signature arrives +- **THEN** an exact count is executed and the cache entry is refreshed + +### Requirement: Upstream-route resolution is invalidation-driven with a TTL backstop + +Proxy hot-path upstream-route resolution MUST be served from a per-account cache of resolver outcomes. Admin mutations of any resolver input (account proxy bindings, proxy pool membership, upstream-proxy dashboard settings, account deletion cascading a binding away) MUST invalidate the cache on the mutating replica before the mutating response returns and durably bump a cache-invalidation namespace so peer replicas converge within one poll interval. If the durable bump write fails (the bump primitive is non-raising), the implementation MUST enqueue the coalesced retry so peers still converge on the first poll cycle after the write path recovers. The cache TTL MUST default to 60 seconds as a backstop for out-of-band database edits, and a TTL of 0 MUST disable caching entirely. + +#### Scenario: Repeat turns skip route re-resolution + +- **GIVEN** an account whose route resolved less than the TTL ago with no intervening route-input mutation +- **WHEN** another proxy request uses that account +- **THEN** the route MUST be served from the cache without opening a database session + +#### Scenario: Binding change invalidates before the response returns + +- **GIVEN** a cached route outcome for an account +- **WHEN** an operator upserts that account's proxy binding +- **THEN** the mutating replica's cache MUST be cleared before the HTTP response returns +- **AND** the `upstream_route` namespace MUST be durably bumped so peers clear their caches via the poller + +#### Scenario: Pool membership change invalidates + +- **GIVEN** a cached route outcome resolved from a pool +- **WHEN** an operator adds a member to any proxy pool +- **THEN** the local cache MUST be cleared and the `upstream_route` namespace durably bumped before the response returns + +#### Scenario: Account deletion invalidates + +- **GIVEN** a cached route outcome for an account +- **WHEN** an operator deletes the account (cascading its proxy binding away) +- **THEN** the local cache MUST be cleared and the `upstream_route` namespace durably bumped before the response returns + +#### Scenario: Peer replicas converge through the poller + +- **GIVEN** a cached route outcome on a replica that did not perform the mutation +- **WHEN** the `upstream_route` or `settings` namespace version advances +- **THEN** that replica's cache-invalidation poller MUST clear its route cache within one poll interval + +#### Scenario: Upstream settings change invalidates + +- **GIVEN** a cached route outcome +- **WHEN** an operator changes `upstream_proxy_routing_enabled` or `upstream_proxy_default_pool_id` +- **THEN** the mutating replica's route cache MUST be cleared and the `upstream_route` namespace durably bumped (with the coalesced retry on write failure) before the response returns +- **AND** peers MUST also clear theirs via the durable `settings` namespace bump + +### Requirement: Aggregated rate-limit reads never run concurrently on a shared session + +Proxy rate-limit header and usage-payload construction MUST NOT execute +multiple statements concurrently on one `AsyncSession`. Repository objects +exposed by the same `ProxyRepositories` context SHALL be treated as sharing that +single-session ownership constraint. + +#### Scenario: Rate-limit header reads execute sequentially + +- **WHEN** the proxy constructs upstream-quota rate-limit headers from primary, secondary, monthly, and credit usage rows +- **THEN** each database read MUST complete before the next read starts on the shared session +- **AND** the returned header names and values remain unchanged for equivalent rows + +#### Scenario: Codex usage payload reads execute sequentially + +- **WHEN** the proxy constructs the aggregate `/api/codex/usage` payload for a request that does not resolve to a codex-lb API key, using usage windows, credits, and additional limits +- **THEN** each database read MUST complete before the next read starts on the shared session +- **AND** the returned payload remains schema- and value-compatible for equivalent rows + diff --git a/openspec/specs/quota-phase-planner/spec.md b/openspec/specs/quota-phase-planner/spec.md index bdcbedb80a..494df4c204 100644 --- a/openspec/specs/quota-phase-planner/spec.md +++ b/openspec/specs/quota-phase-planner/spec.md @@ -65,7 +65,9 @@ produce an `observed`, `known`, or `high` confidence warmup-effect observation. The quota planner SHALL expose authenticated dashboard APIs and UI controls for settings, forecast, decisions, warm-now, and cancellation. Settings changes and scheduler decisions MUST remain auditable, and decision responses SHOULD expose -parsed decision details when stored audit JSON is available. +parsed decision details when stored audit JSON is available. Warm-now reset +eligibility gates MUST compare persisted quota reset epochs against the current +UTC instant regardless of the server process timezone. #### Scenario: Operators can inspect planner decisions @@ -80,6 +82,16 @@ parsed decision details when stored audit JSON is available. execution - **AND** it records a skipped, failed, or executed decision outcome +#### Scenario: Warm-now reset gate is timezone-independent + +- **GIVEN** a short-window usage reset epoch is already due in UTC +- **AND** the server process local timezone is UTC+ +- **WHEN** a dashboard user requests a manual warm-now probe for that account +- **THEN** the reset gate MUST NOT skip with `account_window_already_active` + because of process-local timestamp conversion +- **AND** the warm-now request remains eligible for execution when the other + server-side gates allow it + ### Requirement: Quota planner decisions persist naive UTC instants The quota phase planner SHALL normalize timezone-aware datetimes to naive UTC diff --git a/openspec/specs/rate-limit-reset-credits/spec.md b/openspec/specs/rate-limit-reset-credits/spec.md index e2642244ae..511b9beb60 100644 --- a/openspec/specs/rate-limit-reset-credits/spec.md +++ b/openspec/specs/rate-limit-reset-credits/spec.md @@ -5,7 +5,7 @@ TBD - created by archiving change add-rate-limit-reset-credits. Update Purpose a ## Requirements ### Requirement: Reset credits are polled per account on a fixed cadence -The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL always start with the application lifespan. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. +The system SHALL poll upstream `GET /wham/rate-limit-reset-credits` for each eligible account on a configurable cadence that defaults to 60 seconds, using that account's stored OAuth bearer token and `chatgpt-account-id`. The scheduler SHALL start with the application lifespan when reset-credit polling is enabled. Because snapshots are kept in process-local memory, every running replica SHALL refresh its own snapshot cache instead of relying on leader election, and the scheduler SHALL NOT be leader-gated while snapshots remain process-local. Each replica SHALL apply a randomized startup delay of up to one full interval and randomized per-tick jitter of +/-10% so replica ticks are desynchronized. The aggregate upstream fetch rate scales with the number of running replicas; `rate_limit_reset_credits_refresh_interval_seconds` is the operator control for total upstream load. The poll SHALL skip any account that is paused, requires reauthentication, deactivated, or lacks a usable `chatgpt-account-id`. #### Scenario: Default cadence polls every 60 seconds - **WHEN** the application starts with default settings @@ -145,13 +145,39 @@ The reset-credits refresh scheduler SHALL NOT transition any account's persisted ### Requirement: Reset credit polling interval is configurable -The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL NOT expose a separate enable/disable toggle for reset-credit polling. +The system SHALL expose setting `rate_limit_reset_credits_refresh_interval_seconds` (default `60`) to control the polling cadence. The system SHALL expose setting `rate_limit_reset_credits_refresh_enabled` (default `true`) to enable or disable background reset-credit polling. Because the refresh loop is the sole driver of automatic reset-credit redemption, disabling background polling SHALL also disable automatic redemption; when polling is disabled while the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is enabled, the system SHALL log a configuration-conflict warning at startup naming both settings. While polling is disabled, the dashboard settings update SHALL reject a request that newly enables `auto_redeem_reset_credits_before_expiry` with a bad-request error naming the polling toggle; an already-persisted opt-in SHALL remain readable and re-savable so unrelated settings edits are not blocked. #### Scenario: Operator tunes the polling interval - **GIVEN** `rate_limit_reset_credits_refresh_interval_seconds` is set to `120` - **WHEN** the application starts and runs - **THEN** each eligible account's credits are fetched from upstream at most once per 120 seconds +#### Scenario: Operator disables background polling +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **WHEN** the application starts +- **THEN** the reset-credit polling scheduler does not create a background polling task +- **AND** no upstream reset-credits fetches occur + +#### Scenario: Disabled polling conflicts with persisted auto-redeem opt-in +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `true` +- **WHEN** the application starts +- **THEN** the system logs a configuration-conflict warning naming both settings +- **AND** no automatic reset-credit redemption occurs while polling remains disabled + +#### Scenario: Auto-redeem opt-in is rejected while polling is disabled +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is `false` +- **WHEN** a dashboard settings update sets `auto_redeem_reset_credits_before_expiry` to `true` +- **THEN** the update is rejected with a bad-request error naming the polling toggle +- **AND** the persisted setting remains `false` + +#### Scenario: Persisted auto-redeem does not block unrelated settings edits +- **GIVEN** `rate_limit_reset_credits_refresh_enabled` is set to `false` +- **AND** the persisted dashboard setting `auto_redeem_reset_credits_before_expiry` is already `true` +- **WHEN** a full settings payload that keeps the opt-in unchanged is submitted +- **THEN** the update succeeds + ### Requirement: Reset credit redemption is serialized and idempotent across replicas Per-account redemption serialization MUST hold across all replicas and processes sharing one database. On PostgreSQL the system SHALL use `pg_advisory_xact_lock` keyed by the account id on the caller's session. On SQLite the system SHALL acquire a durable claim row via a single atomic conditional upsert (`INSERT ... ON CONFLICT(account_id) DO UPDATE ... WHERE expires_at < now`) with a 30-second lease, a bounded retry loop that surfaces a client-facing conflict on timeout, release on completion, and takeover of expired claims. While the redeem section runs, the claim holder SHALL renew its lease on a heartbeat cadence shorter than the lease (10 seconds) so a redemption that legitimately outlives one lease (e.g. slow upstream fetch/consume) is NOT taken over by a concurrent process; lease expiry without renewal remains the crash-recovery path. A claim-acquisition timeout SHALL surface in the caller surface's native error envelope: the dashboard error envelope on the dashboard consume endpoint and the `/v1/*` OpenAI error envelope (HTTP 409) on `POST /v1/reset-credit`. The system SHALL persist the `(account_id, redeem_request_id) -> credit_id` mapping in the shared database, committed inside the serialized section BEFORE the upstream consume call; a retry carrying the same `redeem_request_id`, served by ANY replica, MUST resolve to the originally selected `credit_id` and MUST NOT consume a different credit. Ledger rows SHALL be retained at least 24 hours (including after a failed consume, so a retry retargets the same credit) and purged opportunistically afterwards. Expired rows for an account SHALL be purged BEFORE a new pin is inserted, so that reusing a `redeem_request_id` after its prior row has aged past the 24h TTL durably re-pins the new attempt to its newly selected `credit_id` instead of silently discarding the new pin because an `ON CONFLICT DO NOTHING` insert collided with the soon-purged expired row. The pin lookup SHALL apply the same 24h TTL on read: a ledger row whose `created_at` is older than the TTL MUST be treated as absent (not returned as a durable pin) so a reused `redeem_request_id` is re-selected against the fresh fetch and re-pinned rather than forwarded for the stale expired `credit_id`; the read TTL and the purge TTL SHALL be the same duration. Both the dashboard consume endpoint and `POST /v1/reset-credit` SHALL redeem inside this cross-replica serialized section. diff --git a/openspec/specs/release-management/spec.md b/openspec/specs/release-management/spec.md index b56cf6f7f8..801a73e143 100644 --- a/openspec/specs/release-management/spec.md +++ b/openspec/specs/release-management/spec.md @@ -88,7 +88,32 @@ The release publishing workflow SHALL accept both stable tags (`vX.Y.Z`) and pre ### Requirement: Stable release promotion remains release-please owned -A beta-tested release train SHALL be promoted by merging the normal release-please stable release PR for the corresponding base version. Stable promotion SHALL rebuild PyPI, Docker, Helm, and GitHub Release artifacts with the stable version instead of retagging prerelease artifacts. +A beta-tested release train SHALL be promoted by merging the normal +release-please stable release PR for the corresponding base version. Stable +promotion SHALL rebuild PyPI, Docker, Helm, and GitHub Release artifacts with +the stable version instead of retagging prerelease artifacts. + +Before the stable release PR for `X.Y.Z` is merged, every change in the +release candidate SHALL either be covered by a `vX.Y.Z-beta.N` prerelease +that has been published and deployed to at least one production-scale +environment for a soak of at least 48 hours without new regressions +attributable to the release train, or fall under the safe-delta exception +below. The unsoaked delta — every change not covered by such a soaked +prerelease, whether because no prerelease of the train completed a soak or +because the change landed after the last soaked prerelease — qualifies for +the exception only when it consists solely of documentation, CI, or +release-tooling changes, an urgent security or outage hotfix, or a +combination of these; otherwise the train SHALL soak (again) as a new +prerelease before stable promotion. When promotion relies on the exception, +the exception and its reason SHALL be recorded on the stable release PR +before merge. + +When the release train contains Alembic revisions, the maintainer SHALL review +the revisions between the previous stable tag and the release candidate +directly for data-backfill migrations and SHALL estimate their startup impact +against a production-scale dataset before merging the stable release PR. +Generated changelog titles SHALL NOT be treated as sufficient evidence that +the train contains no data backfills. #### Scenario: beta train is promoted to stable @@ -99,6 +124,52 @@ A beta-tested release train SHALL be promoted by merging the normal release-plea - **AND** the release publishing workflow publishes stable artifacts for `1.19.0` - **AND** stable Docker aliases `latest`, `1`, and `1.19` are updated only by the stable release +#### Scenario: stable promotion waits for the beta soak + +- **GIVEN** `v1.20.0-beta.1` was published 12 hours ago and is deployed on a + production-scale environment +- **WHEN** a maintainer considers merging the stable release PR for `1.20.0` +- **THEN** promotion waits until the beta has soaked for at least 48 hours + without new regressions attributable to the release train + +#### Scenario: stable promotion without a soaked beta records an exception + +- **GIVEN** no `v1.21.1-beta.N` prerelease has completed a 48-hour soak +- **AND** the entire delta since `v1.21.0` consists of an urgent security + hotfix and CI changes only +- **WHEN** a maintainer merges the stable release PR for `1.21.1` with the + exception and its reason recorded on the PR +- **THEN** the promotion is compliant with this requirement + +#### Scenario: unrelated unsoaked changes cannot ride a hotfix exception + +- **GIVEN** no `v1.22.0-beta.N` prerelease has completed a 48-hour soak +- **AND** the delta since the previous stable release contains an urgent + hotfix alongside unrelated feature or migration changes +- **WHEN** a maintainer considers promoting `1.22.0` directly to stable +- **THEN** the hotfix exception does not apply to the train +- **AND** the train either soaks as a beta or the hotfix is released + separately + +#### Scenario: changes landing after the soaked beta restart the soak + +- **GIVEN** `v1.23.0-beta.1` completed a 48-hour production-scale soak +- **AND** a feature or migration change lands on `main` afterwards, before + the stable release PR for `1.23.0` is merged +- **WHEN** a maintainer considers promoting `1.23.0` to stable +- **THEN** the post-beta change is part of the unsoaked delta +- **AND** because a feature or migration change is not exception-eligible, + promotion requires a new soaked prerelease covering it + +#### Scenario: data backfills are identified from Alembic revisions + +- **GIVEN** the release train adds revisions under `app/db/alembic/versions` + since the previous stable tag +- **WHEN** the maintainer prepares to merge the stable release PR +- **THEN** they review those revisions directly for data-backfill operations +- **AND** changelog titles alone are not treated as evidence that no backfill + is present + ### Requirement: Stable release promotions guard every release-managed version field Stable release promotion pull requests SHALL fail CI unless every release-managed version field agrees on the stable version and every field that previously held the prior release train version advances together. The guarded fields SHALL include `pyproject.toml`, `app/__init__.py`, `frontend/package.json`, both Helm chart version fields, and the editable `codex-lb` entry in `uv.lock`. diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index f210efbdaa..4f19aa8e26 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -266,13 +266,28 @@ and durable circuit state. - **AND** the failure is logged and exposed through retry-circuit observability ### Requirement: Long Codex websocket turns tolerate extended upstream silence -The default compact request budget MUST be at least 180 seconds, and the default upstream stream idle timeout MUST be at least 600 seconds, so long-running Codex turns can survive expensive compaction or tool execution without a local proxy watchdog ending the turn prematurely. +The default compact request budget MUST be at least 180 seconds, and the default upstream stream idle timeout MUST be at least 600 seconds, so long-running Codex turns can survive expensive compaction or tool execution without a local proxy watchdog ending the turn prematurely. Responses streams over both HTTP and WebSocket transports MUST use `http_responses_stream_request_budget_seconds` when it is configured; they MUST fall back to `proxy_request_budget_seconds` only when no stream-specific budget is available. #### Scenario: compact and stream watchdog defaults leave room for long turns - **WHEN** the service starts with default configuration - **THEN** `compact_request_budget_seconds` is at least 180 seconds - **AND** `stream_idle_timeout_seconds` is at least 600 seconds +#### Scenario: WebSocket Responses stream uses the stream-specific request budget +- **GIVEN** `proxy_request_budget_seconds = 600` +- **AND** `http_responses_stream_request_budget_seconds = 7200` +- **WHEN** a native WebSocket Responses stream computes its request deadline +- **THEN** the stream budget is 7200 seconds +- **AND** the generic 600 second proxy request budget does not terminate the turn + +#### Scenario: WebSocket reconnect keeps the stream-specific deadline +- **GIVEN** `proxy_request_budget_seconds = 600` +- **AND** `http_responses_stream_request_budget_seconds = 7200` +- **AND** a native WebSocket Responses request needs to reconnect after more than 600 seconds but less than 7200 seconds +- **WHEN** the reconnect performs account selection and opens its replacement upstream WebSocket +- **THEN** both operations remain bounded by the original 7200-second stream deadline +- **AND** the reconnect does not fail solely because the generic 600-second budget elapsed + ### Requirement: Responses upstream websocket liveness is bounded The proxy MUST configure direct and routed upstream Responses WebSocket transports with finite ping/pong liveness detection derived from `proxy_downstream_websocket_idle_timeout_seconds`. When an established Responses WebSocket is terminated because its transport did not receive the required pong, the adapter MUST classify the failure as `upstream_websocket_liveness_timeout`. Direct WebSocket and HTTP bridge relay owners MUST treat that failure as account neutral, MUST NOT transparently replay a pending request whose delivery is ambiguous, MUST finalize its pending request ownership exactly once, and MUST retire the affected upstream socket so a later client retry opens a fresh connection. An HTTP bridge reader MUST suppress its own pending-deque settlement only when a concurrent submitter explicitly claimed liveness-settlement ownership under the session lifecycle lock; `session.closed` alone MUST NOT suppress settlement. @@ -664,14 +679,42 @@ authoritative actual tier even when it differs from the requested tier. ### Requirement: API key service tier enforcement applies to upstream Responses requests -When an API key carries an enforced service tier, the proxy MUST override any incoming Responses request service tier with that enforced value before forwarding upstream. The legacy alias `fast` MUST be treated as `priority`. +When an API key carries an enforced service tier, the proxy MUST override any +incoming Responses request service tier with that enforced value before route +selection. The omit-equivalent client values `auto` and `default` MUST count as +an omitted tier when tracking whether the enforced value supplied the request's +tier. The legacy alias `fast` MUST be treated as `priority`. + +For a subscription-account route, when an authoritative account catalog says +the selected model never advertises the enforced tier, the proxy MUST remove +that tier from the effective request before account selection and upstream +forwarding. The resulting effective tier MUST survive internal owner +forwarding unchanged. This fallback MUST NOT remove an explicit non-default +client tier, MUST NOT alter a request routed through an external model source, +and MUST NOT apply when the account catalog has no authoritative answer for the +model. #### Scenario: Enforced service tier overrides the request payload +- **GIVEN** the selected account model advertises the `priority` service tier - **WHEN** an API key is configured with `enforcedServiceTier: "priority"` - **AND** an incoming Responses request asks for `service_tier: "default"` - **THEN** the forwarded upstream payload uses `service_tier: "priority"` +#### Scenario: Omit-equivalent request permits account-catalog fallback + +- **GIVEN** an account model authoritatively advertises no `priority` service tier +- **WHEN** an API key is configured with `enforcedServiceTier: "priority"` +- **AND** an incoming Responses request omits `service_tier` or supplies `auto` or `default` +- **THEN** the account-routed upstream payload omits `service_tier` +- **AND** an internal owner forward preserves that effective omission + +#### Scenario: Explicit non-default tier is not downgraded + +- **GIVEN** an account model authoritatively advertises no `priority` service tier +- **WHEN** a client explicitly requests `service_tier: "priority"` or the equivalent `fast` alias +- **THEN** API-key enforcement does not make the tier eligible for account-catalog fallback + #### Scenario: Fast alias is applied as priority - **WHEN** an API key is configured with `enforcedServiceTier: "fast"` @@ -1960,8 +2003,9 @@ terminal `response.failed` SSE event with error code `stream_incomplete`, record the request-log row as an upstream `stream_incomplete` error, and apply the normal transient upstream account-health signal. If the downstream client cancels or disconnects before a terminal event, the proxy MUST record the -request-log row as a downstream `client_disconnected` error and MUST NOT -penalize the upstream account. +request-log row with status `cancelled`, downstream error code +`client_disconnected`, and downstream failure metadata, and MUST NOT penalize +the upstream account. #### Scenario: Raw stream upstream EOF is not successful @@ -1980,7 +2024,7 @@ penalize the upstream account. - **GIVEN** a raw HTTP streaming Responses request has not observed a terminal SSE event - **WHEN** the downstream client cancels or disconnects from the stream -- **THEN** the request log stores status `error`, error code +- **THEN** the request log stores status `cancelled`, error code `client_disconnected`, and downstream failure metadata - **AND** the selected account is not penalized for the client-side close @@ -3542,6 +3586,7 @@ and object key order. - **WHEN** two requests differ only in tool array order or tool object key order - **THEN** their tools affinity/observability hash is identical + ### Requirement: Streaming events are parsed once and re-serialized only when modified Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Event framing, payload contents, dedupe/rewrite semantics, and error normalization MUST be unchanged. @@ -3674,3 +3719,1345 @@ exist. The default threshold MUST be no greater than seven failures. - **WHEN** the dead-owner classifier evaluates lease liveness against the application's naive-UTC clock - **THEN** both timestamps MUST be normalized to naive UTC before comparison - **AND** the anchored-lookup path MUST NOT raise on mixed-awareness datetimes + +### Requirement: HTTP bridge model-transition isolation is single-pass + +When an HTTP bridge request cannot reuse the session selected by its incoming affinity because that session uses an incompatible model, the service MUST preserve the resulting internal model-parallel key until bridge creation or reuse completes. It MUST NOT reapply the original session-header or turn-state fallback to the same request after selecting that fork. + +#### Scenario: Fresh turn state falls back to a session on another model + +- **GIVEN** a request carries a fresh generated turn-state header and a session header whose active bridge uses an incompatible model +- **WHEN** lookup isolates the request with an internal model-parallel key +- **THEN** lookup emits at most one model-transition fork for that request scope +- **AND** bridge creation continues under the internal key without closing or reusing the incompatible session + +#### Scenario: Follow-up fallback has no previous-response lookup + +- **GIVEN** a request carries a fresh generated turn-state header, a `previous_response_id` without a local or durable lookup, and a session header whose active bridge uses an incompatible model +- **WHEN** lookup isolates the request with an internal model-parallel key +- **THEN** the session-header fallback remains an anchored continuation for the rest of that lookup/create operation +- **AND** bridge creation continues under the internal key without a `continuity_lost` error + +#### Scenario: Full cache preserves the incompatible parent + +- **GIVEN** the HTTP bridge cache is at its session limit and a model transition isolates a session-header fallback into a child key +- **WHEN** creation needs to evict an idle session +- **THEN** the incompatible session-header parent MUST NOT be selected for that eviction +- **AND** ordinary LRU eviction remains eligible for other idle sessions + +#### Scenario: In-flight parent completes before model isolation + +- **GIVEN** a request waits for an in-flight session-header parent whose completed bridge uses an incompatible model +- **WHEN** the request isolates itself with an internal model-parallel key after that wait +- **THEN** the completed parent MUST receive the same capacity-eviction protection as an immediately available parent + +#### Scenario: Compatible session fallback remains reusable + +- **GIVEN** a request carries a fresh generated turn-state header and a session header whose active bridge uses a compatible model +- **WHEN** lookup applies the session-header fallback +- **THEN** the compatible bridge remains eligible for normal reuse + +### Requirement: Standalone Codex web search is forwarded faithfully + +The proxy SHALL expose `POST /backend-api/codex/alpha/search` through the same +proxy-authenticated Codex control-request path used by other unary Codex control +endpoints. The proxy MUST preserve the inbound request body and query parameters, +MUST apply the existing API-key scope, account selection, token refresh, session +affinity, failover, and upstream-route policies, and MUST forward the request to +the upstream `POST /codex/alpha/search` path. Successful downstream responses +MUST preserve the upstream status and body and MUST include only response +headers allowed by the existing Codex control-response policy. Final non-2xx +responses MUST preserve their status while using the existing Codex control +OpenAI error-envelope normalization. The proxy MUST NOT parse, normalize, or +invent a local schema for successful search requests or responses. + +#### Scenario: authenticated standalone search reaches the upstream Codex path + +- **GIVEN** a valid proxy API key and at least one eligible ChatGPT account +- **WHEN** Codex sends `POST /backend-api/codex/alpha/search` with a JSON body and + query parameters +- **THEN** the proxy forwards the unchanged body and query parameters to + `POST /codex/alpha/search` using the selected account credentials +- **AND** the downstream client receives the upstream status and body + +#### Scenario: unsafe upstream response headers are not exposed + +- **WHEN** the upstream search response includes both allowlisted metadata and + a response header outside the Codex control-response allowlist +- **THEN** the proxy returns the allowlisted metadata +- **AND** it omits the non-allowlisted response header + +#### Scenario: final upstream search failures use the control error contract + +- **WHEN** upstream search failure handling finishes with a non-2xx response +- **THEN** the proxy preserves the final HTTP status +- **AND** it returns the failure through the existing OpenAI error envelope +- **AND** existing account refresh, health, and failover handling remains active + +#### Scenario: unsupported methods do not enter search forwarding + +- **WHEN** a client sends a non-POST request to + `/backend-api/codex/alpha/search` +- **THEN** the request does not enter the upstream search forwarding path + +### Requirement: Pre-acceptance account-model rejections fail over safely + +When upstream rejects a Responses request with `invalid_request_error` and the exact message `The '' model is not supported when using Codex with a ChatGPT account.` before accepting the response, the proxy MUST classify the failure internally as `account_model_unsupported`. The quoted model MUST match +the requested model. For native WebSocket, HTTP responses bridge, and raw +HTTP/SSE transports, the proxy MUST make at most one transparent attempt on a +different account that advertises the same model, provided the request can move +without violating continuation or uploaded-file ownership. The proxy MUST +exclude the rejecting account only for that request and MUST NOT record an +account-health penalty for this rejection. + +The proxy MUST NOT replay after any response id recognized in an upstream payload, +including a `response.failed` payload that carries `response.id` even when +`response.created` was not observed or an `error` payload with top-level +`response_id`, a nonterminal `response.*` +event, downstream sequence/output, another pending request on the shared +socket, or an earlier replay. If no compatible replacement is available, or +the request is account-bound, the proxy MUST preserve the original upstream +400 error instead of replacing it with `no_accounts`, `stream_incomplete`, or +another proxy-generated failure. + +#### Scenario: stale model route retries another advertising account + +- **GIVEN** two accounts advertise the requested model in the current routing snapshot +- **AND** upstream rejects the first account with the exact account/model unsupported envelope before `response.created` +- **WHEN** the request has no hard account or uploaded-file binding +- **THEN** the proxy excludes the first account for this request and retries once on the second account +- **AND** it forwards only the replacement attempt's response events downstream +- **AND** it does not penalize the first account's global health + +#### Scenario: no replacement preserves the upstream rejection + +- **GIVEN** upstream rejects a pre-acceptance request with the exact account/model unsupported envelope +- **AND** no other compatible account is available +- **WHEN** transparent failover cannot select a replacement +- **THEN** the client receives the original HTTP 400 `invalid_request_error` +- **AND** the error is not rewritten to `no_accounts`, `stream_incomplete`, or HTTP 502 + +#### Scenario: selected replacement failure is surfaced + +- **GIVEN** upstream rejects a pre-acceptance request with the exact account/model unsupported envelope +- **AND** the proxy selects a different compatible replacement account +- **WHEN** that replacement attempt fails before acceptance +- **THEN** the client receives the replacement attempt's failure +- **AND** the skipped account's original HTTP 400 is not used as a fallback +- **AND** the proxy does not select a third account after a retryable replacement + refresh, transport, or server failure + +#### Scenario: failed bridge replacement retires without restoring rejected metadata + +- **GIVEN** an HTTP responses bridge reconnect has selected and installed a + replacement account after an account/model rejection +- **WHEN** replacement response-create lease acquisition or request send fails +- **THEN** the proxy forwards the replacement failure and retires that bridge + session after draining the rejected request +- **AND** it does not restore the rejected account's turn state or headers onto + the replacement socket + +#### Scenario: accepted or visible request is never replayed + +- **WHEN** the account/model unsupported envelope arrives after a response id, a nonterminal response event, downstream sequence/output, or an earlier replay +- **THEN** the proxy does not transparently replay the request on another account + +#### Scenario: account-bound request is never migrated + +- **WHEN** a rejected request depends on an account-scoped uploaded file or an owner-bound continuation without a verified self-contained fresh replay body +- **THEN** the proxy does not move the request to another account +- **AND** it preserves the original upstream rejection + +### Requirement: Model-capacity messages are retryable transient failures + +When upstream returns a temporary model-capacity failure whose message says that the selected model is at capacity, the proxy MUST treat the failure as retryable transient even if the upstream error code or HTTP status would otherwise look non-retryable. + +#### Scenario: Selected model capacity with invalid request code is retryable + +- **WHEN** upstream returns an error envelope with `error.message = "Selected model is at capacity. Please try a different model."` +- **AND** the normalized error code is `invalid_request_error` +- **AND** the HTTP status is `400` +- **THEN** `classify_upstream_failure` returns `failure_class = "retryable_transient"` +- **AND** pre-visible streaming/websocket paths are eligible to retry or fail over instead of surfacing a terminal client error. + +#### Scenario: Serialized selected-model capacity event surfaces without replay + +- **WHEN** a streaming Responses request receives a first upstream `response.failed` or `error` event whose message says the selected model is at capacity +- **AND** no downstream-visible output has been emitted +- **THEN** the proxy MUST surface that terminal event without transparently re-POSTing the request +- **AND** the absence of an upstream response id MUST NOT by itself prove the POST was safe to replay. + +#### Scenario: Post-connect body-read disconnect is not replayed as capacity retry + +- **WHEN** a streaming Responses request fails while reading the upstream stream body after the upstream request has been dispatched +- **AND** the failure is an `aiohttp` client error, timeout, EOF, or other transport/body-read close without typed pre-dispatch provenance +- **THEN** the proxy MUST surface the stream failure to the downstream client +- **AND** the proxy MUST NOT transparently re-POST the request as a model-capacity retry. + +#### Scenario: Websocket connect failure retries before request dispatch + +- **WHEN** an upstream websocket handshake raises a typed connector failure or connect timeout before the `response.create` frame is sent +- **THEN** the proxy MUST preserve typed pre-dispatch provenance and MAY retry or fail over before any downstream-visible output +- **AND** a websocket transport selection MUST NOT turn that failure into a terminal serialized SSE event. + +#### Scenario: Direct HTTP TLS verification failure is not retried + +- **WHEN** a direct HTTP stream raises a certificate or TLS connector failure before request dispatch +- **THEN** the proxy MUST surface the TLS failure without transparently retrying or failing over +- **AND** pre-dispatch provenance MUST NOT classify the non-transient TLS failure as retryable. + +#### Scenario: Quota and rate-limit codes retain their stronger classification + +- **WHEN** upstream returns a quota or rate-limit error code +- **THEN** the proxy MUST keep classifying it as quota or rate-limit before applying message-based model-capacity detection. + +#### Scenario: Post-refresh transient exhaustion preserves every health signal + +- **WHEN** one or more accounts each exhaust multiple same-account post-refresh transient retries before the request succeeds or terminates +- **THEN** the proxy MUST settle API-key usage before recording any deferred account-health failure +- **AND** each exhausted account MUST receive exactly one classified health failure plus one additional failure for every remaining exhausted retry +- **AND** selecting or exhausting a later account MUST NOT replace, lose, or duplicate an earlier account's deferred failures. + +#### Scenario: Classified quota failures still use the model-capacity replay wait + +- **WHEN** a replayable pre-created HTTP bridge request receives the selected-model capacity message with a quota or + rate-limit error code +- **THEN** the proxy MUST preserve that quota or rate-limit classification for account health handling +- **AND** the proxy MUST still apply the model-capacity wait before replaying the request. + +### Requirement: HTTP bridge model-capacity retry waits preserve stream contracts + +The proxy MUST wait before replaying a pre-created HTTP bridge request with a selected-model capacity failure only +when the failure happened before any downstream-visible response event and the request is still replayable as a fresh +request. + +#### Scenario: Public propagated-error streams do not receive pre-retry keepalives + +- **WHEN** a `/v1/responses`-compatible HTTP bridge stream is configured to propagate startup HTTP errors +- **AND** upstream returns a selected-model capacity error before `response.created` +- **THEN** the proxy MUST NOT emit `codex.keepalive` or account-capacity wait events before the retry completes. + +#### Scenario: Replay waits remain bounded by the original bridge deadline + +- **WHEN** the selected-model capacity error arrives near or after the original bridge request deadline +- **THEN** the proxy MUST NOT start a fresh upstream replay after that deadline is exhausted. + +#### Scenario: Only fresh replayable bridge requests wait + +- **WHEN** the selected-model capacity error belongs to an anchored request that cannot be replayed without + `previous_response_id` +- **THEN** the proxy MUST forward the terminal error promptly without sleeping for the model-capacity retry delay. + +#### Scenario: Retry-safe injected anchors still wait + +- **WHEN** the proxy injected `previous_response_id` and retained a fresh request body that is safe to replay without + that anchor +- **AND** upstream returns a selected-model capacity error before visible output +- **THEN** the proxy MUST apply the model-capacity wait before stripping the injected anchor and replaying the fresh + request. + +#### Scenario: Remote-owner relay preserves the hidden startup wait + +- **WHEN** an origin replica forwards a bridge request to its remote owner +- **THEN** the origin MUST keep its startup probe pending until the owner relay returns response headers or a terminal + startup error +- **AND** a selected-model capacity wait on the owner MUST NOT cause the origin to commit HTTP 200 before that wait + completes. + +#### Scenario: Waiting keeps the retry tied to the pending request + +- **WHEN** the proxy waits before replaying a selected-model capacity failure +- **THEN** the request MUST remain reserved in the bridge pending queue while it waits +- **AND** the proxy MUST retain the session response-create gate so a younger request cannot enter while the sole + upstream reader is sleeping +- **AND** the proxy MUST release account-level and shared response-create capacity during the wait +- **AND** the proxy MUST reacquire both capacity leases before sending the replay +- **AND** the proxy MUST skip the replay if that queued request detaches before the wait completes. + +### Requirement: WebSocket stale-anchor failures include diagnostic metadata +When a direct Responses WebSocket request fails closed because upstream rejects `previous_response_id` with `previous_response_not_found`, the service MUST emit stale-anchor diagnostic metadata in operator logs and request-log failure metadata. The metadata MUST distinguish `previous_response_source` (`client_supplied`, `proxy_injected`, or `unknown`), whether a fresh no-anchor replay body was available, owner lookup outcome/source, whether the matched previous response belongs to the same Codex session when known, and the previous-response age in seconds when known. The metadata MUST NOT expose raw `previous_response_id` values or request payload content. + +#### Scenario: client-supplied stale anchor is classifiable +- **GIVEN** a direct WebSocket request arrives with a client-supplied `previous_response_id` +- **AND** upstream rejects that anchor with `previous_response_not_found` +- **THEN** the continuity failure log and request-log failure metadata identify `previous_response_source=client_supplied` +- **AND** they include owner lookup and replay-availability metadata without raw response ids + +#### Scenario: proxy-injected stale anchor is classifiable +- **GIVEN** codex-lb injects a session-continuity `previous_response_id` into a direct WebSocket request +- **AND** upstream rejects that anchor with `previous_response_not_found` +- **THEN** the continuity failure log and request-log failure metadata identify `previous_response_source=proxy_injected` +- **AND** they state whether a retry-safe fresh no-anchor replay body was available +- **AND** owner lookup, age, and same-session fields remain explicit as `unknown` when unavailable rather than being omitted + +#### Scenario: stale anchor owner hit records age and session relationship +- **GIVEN** owner lookup finds a previous response row for the rejected anchor +- **WHEN** the direct WebSocket request fails closed with `previous_response_not_found` +- **THEN** the stale-anchor diagnostics include the owner lookup source +- **AND** include previous-response age seconds and same-session status when those values can be derived + +#### Scenario: account-only cache hits do not guess owner session metadata +- **GIVEN** owner resolution hits a request cache entry that retains the account id but not the matched request-log row +- **WHEN** the direct WebSocket request fails closed with `previous_response_not_found` +- **THEN** the stale-anchor diagnostics identify the owner lookup source as the request cache +- **AND** leave previous-response age and same-session status unknown rather than inferring them from the current request scope + +### Requirement: Responses HTTP ingress uses the expanded bounded budget + +HTTP requests to `/v1/responses` and `/backend-api/codex/responses`, including trailing-slash variants, MUST use the larger of `max_decompressed_body_bytes` and `max_decompressed_responses_body_bytes` as both the raw-body and decompressed-body ingress budget. The Responses-specific default MUST remain 128 MiB. + +The trailing-slash variants MUST be hidden aliases of the canonical HTTP handlers rather than redirects, so streamed bodies receive the same admission, authorization, and route behavior. + +If either representation exceeds that budget, the service MUST stop before route logic or upstream forwarding and return HTTP 413 with an OpenAI-compatible error envelope carrying `error.code = payload_too_large` and `error.type = invalid_request_error`. + +This transport-ingress 413 applies before parsing and is distinct from the existing application-level oversized-`response.create` guard. A request that fits the 128 MiB transport budget but still exceeds the upstream websocket budget after historical slimming MUST retain the existing HTTP 400 `payload_too_large` behavior and `param = input`. + +#### Scenario: Larger Responses request fits both ingress checks + +- **WHEN** a Responses HTTP request is larger than the general budget but no larger than the Responses budget in either raw or decompressed form +- **THEN** the ingress guards allow the request to continue to Responses route handling + +#### Scenario: Trailing-slash Responses request is admitted without redirect + +- **WHEN** a client sends a chunked HTTP request to `/v1/responses/` or `/backend-api/codex/responses/` +- **THEN** the service applies the same ingress budget and handler as the corresponding canonical path +- **AND** it does not return a trailing-slash redirect before consuming the guarded body + +#### Scenario: Responses raw body exceeds its budget + +- **WHEN** a Responses HTTP request's raw body exceeds the Responses budget +- **THEN** the service returns HTTP 413 with `error.code = payload_too_large` and `error.type = invalid_request_error` +- **AND** the service does not invoke Responses route logic or forward the request upstream + +#### Scenario: Responses expanded body exceeds its budget + +- **WHEN** an encoded Responses HTTP request fits the raw budget but expands beyond the Responses budget +- **THEN** the service returns HTTP 413 with `error.code = payload_too_large` and `error.type = invalid_request_error` +- **AND** the service does not invoke Responses route logic or forward the request upstream + +#### Scenario: Post-slimming application rejection remains 400 + +- **WHEN** a Responses HTTP request fits the raw and decompressed transport-ingress budget +- **AND** its serialized `response.create` still exceeds the upstream websocket budget after historical slimming +- **THEN** the existing application-level guard returns HTTP 400 with `error.code = payload_too_large`, `error.type = invalid_request_error`, and `error.param = input` + +### Requirement: Thread-goal OpenAPI operations have unique stable identifiers +The generated OpenAPI document MUST assign a unique `operationId` to every documented HTTP operation. The GET and POST operations at `/backend-api/codex/thread/goal/get` MUST remain available through the same runtime behavior and MUST expose the deterministic identifiers `thread_goal_get_backend_api_codex_thread_goal_get_get` and `thread_goal_get_backend_api_codex_thread_goal_get_post`, respectively. Correcting this schema metadata MUST NOT change either method's authentication, dependency, request forwarding, upstream operation, response status, or response payload behavior. + +#### Scenario: Full OpenAPI schema has unique operation identifiers +- **WHEN** an unauthenticated client requests `GET /openapi.json` +- **THEN** every documented HTTP operation has an `operationId` +- **AND** no two documented HTTP operations share an `operationId` + +#### Scenario: Thread-goal methods publish deterministic identifiers +- **WHEN** an unauthenticated client inspects `/openapi.json` +- **THEN** `GET /backend-api/codex/thread/goal/get` has `operationId` `thread_goal_get_backend_api_codex_thread_goal_get_get` +- **AND** `POST /backend-api/codex/thread/goal/get` has `operationId` `thread_goal_get_backend_api_codex_thread_goal_get_post` + +#### Scenario: Thread-goal runtime forwarding remains compatible +- **WHEN** a client invokes either GET or POST `/backend-api/codex/thread/goal/get` with valid existing dependencies +- **THEN** the request is forwarded through the existing thread-goal handler using the original request method +- **AND** the upstream operation, response status, and response payload remain unchanged + +### Requirement: Public synthetic Responses failures carry numeric sequences + +Public streaming `POST /v1/responses` MUST emit every terminal +`response.failed` with a finite integer `sequence_number` so +strict OpenAI SDK Responses parsers recognize the terminal failure. If the +upstream or proxy-generated event omits a finite integer sequence, the public +normalizer MUST assign the next sequence after all finite integer sequences it +has observed in the same downstream stream. If it also synthesizes a leading +`response.created` from that failure, the created event MUST consume the next +sequence and the failure MUST use the following sequence so both events have +distinct values. Otherwise, if no finite integer sequence has been observed, +failure numbering MUST begin at zero. + +The public normalizer MUST preserve an existing finite integer +`sequence_number` and advance its next-sequence watermark accordingly. This +repair MUST NOT change Codex-private backend stream shapes. + +#### Scenario: Bridge failure after reasoning remains parseable + +- **GIVEN** public `/v1/responses` has emitted sequenced reasoning events +- **WHEN** the upstream bridge closes before a terminal response +- **THEN** the downstream terminal `response.failed` carries the next numeric + `sequence_number` +- **AND** a strict OpenAI SDK parser recognizes it as a terminal failure + +#### Scenario: Leading failure follows synthesized created sequence + +- **GIVEN** public `/v1/responses` has not emitted a finite integer sequence +- **WHEN** an unsequenced leading `response.failed` requires a synthesized + `response.created` +- **THEN** the created event carries `sequence_number = 0` +- **AND** the terminal failure carries `sequence_number = 1` + +#### Scenario: Failure after an unsequenced created event starts at zero + +- **GIVEN** public `/v1/responses` has emitted an unsequenced + `response.created` and no finite integer sequence +- **WHEN** the proxy synthesizes a terminal `response.failed` +- **THEN** the terminal event carries `sequence_number = 0` + +#### Scenario: Valid upstream failure sequence remains unchanged + +- **GIVEN** an upstream terminal `response.failed` carries a finite integer + `sequence_number` +- **WHEN** the public normalizer forwards the event +- **THEN** it preserves that sequence number unchanged +- **AND** if it must synthesize a leading `response.created`, that event uses + the immediately preceding integer sequence + +#### Scenario: Backend Codex stream shape remains unchanged + +- **GIVEN** a Codex-private backend Responses stream carries an unsequenced + terminal failure +- **WHEN** the stream is served without the public OpenAI SDK contract +- **THEN** the proxy does not add a public compatibility sequence + +### Requirement: Direct WebSocket capability intent is trusted and private + +A direct Responses WebSocket MUST recognize the exact internal marker +`X-Codex-LB-Required-Capability: trusted_cyber` only after successful existing +proxy API-key authentication. It MUST accept one marker from either the +handshake headers or the current `response.create.client_metadata`. Duplicate, +conflicting, non-string, unknown, malformed, or unauthenticated signals MUST +fail before account selection. Raw duplicate JSON keys or duplicate +`client_metadata` containers MUST NOT collapse into an ordinary request. The +marker MUST be rejected on every downstream frame type other than +`response.create`. + +The proxy MUST remove the capability header and the exact consumed metadata +key before upstream dispatch, request archival, diagnostics, and logging. +Unrelated client metadata MUST remain unchanged. + +#### Scenario: Per-frame intent routes before upstream open +- **WHEN** an authenticated frame carries the exact metadata marker on a + downstream socket opened without the header +- **THEN** the proxy establishes REQUIRED before opening or reusing an upstream + socket + +#### Scenario: Ambiguous or untrusted signal fails closed +- **WHEN** a signal is duplicated, malformed, unknown, or lacks an authenticated + proxy API-key principal +- **THEN** the proxy returns a typed error before account or model-source + dispatch + +#### Scenario: Duplicate JSON cannot erase intent +- **WHEN** raw JSON repeats the capability key or repeats `client_metadata` + around a capability marker +- **THEN** the proxy returns the typed unsupported-capability error before + selection + +#### Scenario: Capability metadata on another frame is rejected +- **WHEN** a downstream frame other than `response.create` contains the + capability metadata key +- **THEN** the proxy returns a typed error without forwarding or archiving that + frame upstream +- **AND** malformed JSON text is rejected rather than passed through an already + open upstream socket +- **AND** binary downstream frames are rejected before parsing, archiving, or + upstream forwarding + +#### Scenario: Internal metadata is not forwarded or archived +- **WHEN** a valid capability-bearing frame is dispatched and archived +- **THEN** neither capability carrier appears in upstream headers, upstream + payload, archive payload, diagnostics, or logs + +### Requirement: A late capability cannot reuse an ordinary upstream socket + +A later REQUIRED frame MUST NOT reuse an upstream socket selected for an +ordinary request on the same downstream WebSocket. An idle ordinary socket +MUST be retired before capable +reselection. If another frame is still pending, the proxy MUST fail closed +rather than change the account requirement beneath in-flight work. The socket's +selection contract, not whether its account happened to have the capability +grant, MUST determine whether it was selected as ordinary. Before reusing a +REQUIRED-selected socket, the proxy MUST revalidate the pinned account and its +current capability grant through the canonical selector. + +#### Scenario: Idle ordinary socket is replaced +- **WHEN** an idle downstream session previously selected an ordinary account + and a later frame establishes REQUIRED +- **THEN** the ordinary upstream is retired before the frame is sent +- **AND** the replacement selection requires a security-work-authorized account + +#### Scenario: Pending ordinary work blocks a requirement change +- **WHEN** ordinary work is still pending and a later frame establishes REQUIRED +- **THEN** the later frame fails before upstream send +- **AND** the pending frame's account and request state are not rewritten + +#### Scenario: Revoked capability grant prevents socket reuse +- **WHEN** a socket was selected for REQUIRED but its pinned account's grant is + no longer valid at canonical revalidation +- **THEN** the stale socket does not receive the next REQUIRED frame +- **AND** an idle socket is retired before constrained reselection + +#### Scenario: Revalidation uncertainty fails closed +- **WHEN** canonical account revalidation cannot complete for a REQUIRED socket +- **THEN** the frame receives a typed capability-routing-unavailable error +- **AND** its reservation is settled without forwarding the frame upstream + +### Requirement: Proof-gated recovery attempts are durably fenced + +When an HTTP bridge request has a verified, account-neutral, unanchored full +resend body, the proxy MUST record that request fingerprint in the durable +recovery journal before dispatching it upstream. The record MUST be owned by +the current durable session owner epoch and MUST start in `unknown` state. +Requests without that replay-safety proof MUST NOT create a recovery-journal +record. + +#### Scenario: Safe resend is journaled before dispatch + +- **GIVEN** a request has a verified full-resend body that is safe to replay + without `previous_response_id` +- **WHEN** the proxy admits the request for upstream dispatch +- **THEN** the durable journal contains one `unknown` record for its session + and request fingerprint before `response.create` is sent + +#### Scenario: Suppressed request is not journaled + +- **GIVEN** a hard session retry circuit is cooling down +- **WHEN** the request is rejected before upstream dispatch +- **THEN** no recovery-journal record is created or refreshed + +### Requirement: Durable replay is limited to ambiguous transport outcomes + +The proxy MUST consume an `unknown` recovery-journal record for a fresh +account-neutral replay only after an ambiguous transport outcome, represented +by `stream_incomplete`, `stream_idle_timeout`, or +`upstream_request_timeout`, and only before any response event or downstream +output. Explicit deterministic `response.failed` errors MUST settle normally +and MUST NOT trigger a cross-account replay or consume the recovery fence. + +#### Scenario: Transport ambiguity permits one replay + +- **GIVEN** an `unknown` proof-gated journal record exists +- **AND** the upstream closes or times out before any response event +- **WHEN** the bridge handles the ambiguous transport failure +- **THEN** the record is atomically claimed and the request is replayed once + on a fresh account-neutral upstream session + +#### Scenario: Deterministic failure is not replayed + +- **GIVEN** an `unknown` proof-gated journal record exists +- **AND** upstream emits an explicit pre-output `response.failed` such as an + invalid request or quota rejection +- **WHEN** the bridge handles that terminal event +- **THEN** it forwards the terminal failure +- **AND** it leaves the journal available for settlement without replaying on + another account + +### Requirement: Recovery journal settlement is owner-fenced and idempotent + +After a replayed request reaches `response.completed`, the proxy MUST mark its +journal record `replayed` only through the current durable owner epoch and +MUST retain the downstream response id when available. Repeated settlement, +stale owners, and concurrent claim attempts MUST NOT produce a second replay. +The migration MUST be on the current Alembic head and startup schema checks +MUST require the journal table. + +#### Scenario: Completed replay settles once + +- **GIVEN** a replayed request completes successfully +- **WHEN** the completion event is processed +- **THEN** the matching journal record becomes `replayed` +- **AND** a later retry cannot claim it again + +#### Scenario: Stale owner cannot settle or replay + +- **GIVEN** a journal record belongs to a newer durable owner epoch +- **WHEN** an old replica attempts settlement or replay +- **THEN** the operation is rejected without changing the record state + +### Requirement: Claimed HTTP bridge completed queues remain deliverable + +When HTTP bridge processing of `response.completed` removes a request from +pending ownership, it MUST retain the request's downstream event queue for the +remainder of that completed operation. Later asynchronous bookkeeping or +request detachment MUST NOT revoke that claimed queue before the completed +operation's selected terminal event and end-of-stream marker are enqueued. If +fail-closed bookkeeping replaces the upstream completion with a terminal +failure, that selected failure event is the terminal event governed by this +requirement. + +While the claimed completed-delivery operation remains active, ordinary stream +idle accounting MUST NOT replace the upstream completion with a synthetic idle +failure, and the stream MUST continue emitting its existing liveness frames. +The completed-queue claim and the terminal idle-timeout decision MUST be +serialized under the bridge pending lock. If completed processing wins that +serialization and claims a live queue, the timeout MUST be suppressed. If the +terminal event and end-of-stream marker are already queued when a concurrent +timeout finishes awaited recovery work, the completed claim MUST remain +authoritative until the stream consumes that queued delivery. If the +terminal idle timeout wins while no completed delivery is active, it MUST +revoke the request's mutable event queue before releasing the pending lock so a +later completed event cannot claim an orphaned queue. + +The first idle-timeout suppression for one completed-delivery operation MUST +emit one bounded diagnostic containing the request ID, downstream response ID, +and elapsed seconds. Further liveness intervals for that same operation MUST +NOT repeat the diagnostic. + +When that operation returns, raises, or is cancelled before delivery, idle +timeout behavior MUST resume. + +If detachment removes the request from pending ownership first, existing +client-disconnect and drain behavior MUST remain unchanged. + +#### Scenario: Completed processing claims the request before detachment + +- **GIVEN** an HTTP bridge stream is waiting on its request event queue +- **AND** an upstream `response.completed` event removes that request from pending ownership +- **WHEN** request detachment overlaps later completed-event bookkeeping +- **THEN** the stream receives the terminal event selected for downstream delivery exactly once +- **AND** the stream receives its end-of-stream marker + +#### Scenario: Completed bookkeeping exceeds the idle window + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** later completed bookkeeping exceeds the configured stream idle window +- **THEN** the stream continues emitting liveness frames +- **AND** it does not emit a synthetic idle failure while that operation remains active +- **AND** it logs the suppression once with request, response, and elapsed-time context + +#### Scenario: Terminal idle timeout wins before completed processing + +- **GIVEN** an HTTP bridge stream has exhausted its configured idle window +- **AND** no completed-delivery operation has claimed its queue +- **WHEN** the stream acquires the bridge pending lock before a concurrent completed event +- **THEN** it revokes the mutable event queue while still holding that lock +- **AND** it emits the existing synthetic idle failure +- **AND** later completed processing does not deliver to the revoked queue + +#### Scenario: Completed delivery finishes during timeout recovery + +- **GIVEN** an HTTP bridge timeout path is awaiting pre-response recovery work +- **AND** completed processing claims the live queue and enqueues its terminal event and end-of-stream marker +- **WHEN** completed processing returns before the timeout path rechecks ownership +- **THEN** the completed claim remains authoritative +- **AND** the stream consumes the queued completion without emitting a synthetic idle failure + +#### Scenario: Completed bookkeeping aborts + +- **GIVEN** completed-event processing has claimed a live request queue +- **WHEN** that completed-delivery operation exits without enqueueing its terminal event +- **THEN** idle timeout suppression ends +- **AND** the existing idle-timeout failure behavior resumes + +#### Scenario: Detachment claims the request first + +- **GIVEN** an HTTP bridge request is still pending +- **WHEN** detachment removes downstream queue ownership before completed-event matching +- **THEN** existing client-disconnect and upstream-drain behavior is preserved +- **AND** no completed event is delivered to another request + +### Requirement: Replayed tool-call namespace metadata is local-only on upstream input + +For standard and compact Responses requests, the proxy MUST omit `namespace` from every replayed `input` item whose `type` is `function_call`, `custom_tool_call`, or `apply_patch_call` before forwarding the request upstream. The proxy MUST preserve all other fields on that item, MUST retain the original namespace metadata for local call-identity and replay-deduplication processing, and MUST NOT alter client-provided top-level tool entries as part of this normalization. + +#### Scenario: Standard Responses replay omits tool-call namespaces upstream + +- **WHEN** a standard Responses request replays `function_call` and `custom_tool_call` input items with `namespace` +- **THEN** the upstream payload omits only those items' `namespace` +- **AND** preserves their remaining call fields +- **AND** the local request input retains the namespace metadata + +#### Scenario: Compact Responses replay omits tool-call namespace upstream + +- **WHEN** `/v1/responses/compact` replays a recognized tool-call input item with a namespace +- **THEN** its upstream payload omits the input item's `namespace` +- **AND** preserves the remaining tool-call fields + +#### Scenario: WebSocket response.create omits tool-call namespaces upstream + +- **WHEN** a Responses WebSocket request replays namespaced `function_call` and `custom_tool_call` input items +- **THEN** the upstream `response.create` frame omits only those items' `namespace` +- **AND** preserves their remaining call fields + +#### Scenario: Configured Responses model source omits tool-call namespaces upstream + +- **WHEN** `/v1/responses` routes a replayed namespaced tool call to a configured OpenAI-compatible Responses model source +- **THEN** the source payload omits only the call item's `namespace` +- **AND** preserves source-compatible request fields that the Codex upstream path does not support + +#### Scenario: Account-neutral replay classification retains namespace identity + +- **WHEN** an HTTP bridge evaluates a namespaced tool-call history for cross-account replay safety +- **THEN** the classifier input retains the namespace metadata +- **AND** the request fails closed rather than becoming account-neutral because of wire normalization + +#### Scenario: Malformed replay item type does not fail serialization + +- **WHEN** a permissively parsed input item has a non-string `type` and a `namespace` +- **THEN** outbound serialization does not raise an internal type error +- **AND** does not treat the item as a recognized replayed tool call + +#### Scenario: Top-level namespace tool remains byte-preserved + +- **WHEN** the client includes a top-level tool entry whose `type` is `namespace` +- **THEN** standard Responses serialization forwards that tool entry byte-identically + +### Requirement: Responses-Lite replay proof tolerates only verified developer interleaving + +When a fresh durable HTTP bridge classifies a client-unanchored Responses-Lite +full resend whose `additional_tools` bundle preserves developer messages inline, +the replay proof MUST tolerate a developer message only in the historical and +fresh positions defined below. Every other developer position or shape MUST +remain fail-closed. + +A tolerated fresh developer message MUST have `type` omitted or equal to `message`, +MUST have role `developer`, MUST have no non-empty response-owned ID or phase, +MUST have no status or a `completed` status, MUST contain exact account-neutral +metadata with one nonblank `turn_id`, MUST contain exactly one self-contained +`input_text` content part, and MUST contain no unknown or account-scoped fields. +Explicit null or malformed item types MUST fail closed. + +Classification MUST retain response-owned developer-message ID evidence until +these checks have completed, even when other response-owned IDs are projected +out. It MUST retain developer-role items before applying projection rules that +normally omit their declared item type, so a malformed developer item cannot +disappear before validation. A canonical Lite-prefix developer instruction MAY +appear immediately after the `additional_tools` bundle when it passes the same +account-neutral item checks as historical interleaving and has no response-owned +ID. A developer message in the stored prefix outside that canonical position or +the verified pending-call/matching-output interleave MUST fail closed. Non-Lite +`input` or `messages` forms whose instruction-role messages are normalized into +top-level `instructions` remain outside this requirement. + +#### Scenario: Canonical Responses-Lite prefix remains transparent + +- **GIVEN** a fingerprint-verified stored prefix begins with an `additional_tools` bundle +- **AND** a valid account-neutral developer instruction appears immediately after that bundle +- **WHEN** exact manifest or retained-output replay proof validates the stored prefix +- **THEN** the canonical developer instruction is transparent +- **AND** the original full input remains eligible for account-neutral replay + +#### Scenario: Verified historical Responses-Lite developer message is transparent + +- **GIVEN** a Responses-Lite input contains an `additional_tools` bundle +- **AND** its fingerprint-verified stored prefix contains a supported direct call +- **AND** a valid developer message appears before that call's matching output +- **AND** the fresh suffix exactly settles the durable pending-tool manifest +- **WHEN** the HTTP bridge opens a replacement session on the durable owner +- **THEN** it sends the original full input without injecting `previous_response_id` +- **AND** it sends the request once + +#### Scenario: Other historical messages remain fail-closed + +- **GIVEN** a supported direct call is pending in the verified stored prefix +- **WHEN** a user, assistant, system, malformed developer, or response-owned message appears before its output +- **THEN** exact manifest proof fails + +#### Scenario: Other stored developer positions remain fail-closed + +- **GIVEN** a fingerprint-verified stored prefix has no pending direct call +- **WHEN** a developer message appears outside the canonical adjacent Lite-prefix position +- **OR** the adjacent message has a response-owned ID +- **THEN** exact manifest and retained-output proofs fail + +#### Scenario: Projection-omitted developer type remains visible to validation + +- **GIVEN** a developer-role item declares a type normally omitted by replay projection +- **WHEN** account-neutral replay classification projects the full resend +- **THEN** the malformed developer item remains visible to replay proof +- **AND** replay classification fails closed + +#### Scenario: Historical output remains mandatory + +- **GIVEN** a valid developer message follows a supported historical call +- **WHEN** the matching output is missing or has another call ID or type +- **THEN** exact manifest proof fails + +#### Scenario: Historical developer interleaving is bounded to one call and one message + +- **GIVEN** a fingerprint-verified stored prefix opens a pending direct-call window +- **WHEN** that window holds more than one outstanding call at any point before the developer message +- **OR** a further call opens in that window after it has consumed a developer message +- **OR** a second developer message appears while the same window is still open +- **THEN** exact manifest proof fails +- **AND** a later window that holds exactly one outstanding call may still interleave one developer message + +#### Scenario: Fresh developer suffix bounds are measured on the projected input + +- **GIVEN** account-neutral replay classification projects the full resend +- **WHEN** the projection omits reasoning or completed bookkeeping items from the fresh suffix +- **THEN** the fresh developer suffix and terminality bounds are evaluated on the projected positions +- **AND** the accepted width is limited to shapes whose projected suffix satisfies those bounds + +#### Scenario: Bounded fresh custom-tool developer interleave is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a fresh suffix +- **AND** the durable pending-tool manifest contains exactly one `custom_tool_call` +- **WHEN** the entire suffix is exactly that custom call, one valid developer message, and its matching custom-tool output +- **THEN** exact manifest proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` + +#### Scenario: Other fresh tool-loop developer positions remain fail-closed + +- **GIVEN** a durable pending-tool manifest +- **WHEN** a fresh developer message is used with a function or apply-patch call, appears in a parallel batch, is duplicated, lacks exact metadata, contains malformed or account-scoped content, or has leading or trailing suffix items +- **THEN** exact manifest proof fails + +#### Scenario: Bounded retained-output developer follow-up is transparent + +- **GIVEN** the fingerprint-verified stored prefix is followed by a completed assistant `final_answer` +- **AND** exactly one explicit user message follows that retained output +- **WHEN** one valid developer message is the terminal suffix item +- **THEN** retained-output proof passes +- **AND** the original full input is sent once without injecting `previous_response_id` + +#### Scenario: Unproven retained-output developer follow-up remains fail-closed + +- **GIVEN** a retained-output full resend +- **WHEN** the latest assistant output is not `final_answer`, the developer message is not terminal, the fresh input is raw or contains multiple user items, the developer metadata or content is not account-neutral, or the stored prefix contains historical developer interleaving +- **THEN** retained-output proof fails + +### Requirement: Aborted terminal bookkeeping settles claimed reservations exactly once + +The HTTP bridge MUST settle a request's API-key reservation exactly once even +when terminal-event bookkeeping aborts after removing the request from pending +ownership; that bookkeeping continuation exclusively owns the settlement. If +the continuation raises or is cancelled before finalization transfers that +settlement, the abort path MUST settle every request it still owns: the +reservation heartbeat MUST be cancelled, the +reservation MUST be released, and the downstream waiter SHOULD be unblocked +with an end-of-stream marker instead of waiting for its idle timeout. The +abort settlement MUST run to completion under cancellation (shielded), MUST +apply to the grouped previous-response error path's not-yet-finalized +remainder, and MUST NOT settle requests that a retry branch restored to +pending ownership. Settlement MUST remain idempotent so an abort overlapping +an already-transferred finalization cannot double-account usage. + +If the abort settlement itself fails, the claim MUST be marked abandoned and +request detachment MUST be allowed to reclaim that settlement even though the +request is no longer in pending ownership. Detachment MUST NOT settle a live +claim whose bookkeeping continuation is still running. + +#### Scenario: Completed bookkeeping raises after the pending pop + +- **GIVEN** an upstream `response.completed` event has removed a request with an API-key reservation from pending ownership +- **WHEN** later completed bookkeeping raises before finalization +- **THEN** the reservation heartbeat task finishes +- **AND** the API-key reservation is released exactly once +- **AND** no reservation heartbeat touch runs afterward + +#### Scenario: Completed bookkeeping is cancelled after the pending pop + +- **GIVEN** an upstream `response.completed` event has removed a request with an API-key reservation from pending ownership +- **WHEN** the bookkeeping continuation is cancelled before finalization +- **THEN** the shielded abort settlement still cancels the heartbeat and releases the reservation +- **AND** the cancellation is re-raised after settlement + +#### Scenario: Grouped previous-response finalization aborts mid-loop + +- **GIVEN** a grouped previous-response error has removed multiple requests from pending ownership +- **WHEN** finalization aborts after settling only a prefix of those requests +- **THEN** every not-yet-finalized request in the group has its heartbeat cancelled and its reservation released + +#### Scenario: Detachment reclaims an abandoned claim + +- **GIVEN** terminal bookkeeping claimed a request out of pending ownership, aborted, and its abort settlement failed +- **WHEN** the downstream stream detaches that request +- **THEN** detachment cancels the heartbeat and releases the reservation even though the request is not in pending ownership + +#### Scenario: Detachment leaves a live claim to its owner + +- **GIVEN** terminal bookkeeping has claimed a request out of pending ownership and is still running +- **WHEN** the downstream stream detaches that request +- **THEN** detachment does not release the reservation out from under the in-flight finalization + +### Requirement: Pool usage exhaustion is reported as a usage-limit error + +The proxy MUST report pool-wide Responses usage exhaustion as a usage-limit +error. When every account eligible for a Responses request is exhausted by known +usage windows, the proxy MUST reject the request with HTTP `429` and an +OpenAI-style error envelope whose `error.code` and `error.type` are both +`usage_limit_reached`. If account selection has an authoritative upstream reset +timestamp for the exhausted pool, the response envelope MUST include that +timestamp as `error.resets_at`; the proxy MUST NOT expose the capped +human-facing retry hint or a synthesized fallback as `error.resets_at`. The +proxy MUST NOT collapse this condition into generic `no_accounts`, +`server_error`, or HTTP `503` semantics. Exhaustion classification MUST be +based on structured account state after the same eligibility filtering as +ordinary selection, and MUST NOT reclassify local capacity or overload codes +(account caps, admission gates, fair-share throttles) as usage exhaustion. + +#### Scenario: Public Responses request exhausts the eligible usage pool + +- **WHEN** account selection for a public `/v1/responses` or + `/backend-api/codex/responses` request finds only usage-exhausted eligible + accounts +- **THEN** the response status is HTTP `429` +- **AND** the response body has `error.code = "usage_limit_reached"` +- **AND** the response body has `error.type = "usage_limit_reached"` +- **AND** any selected pool reset timestamp is surfaced as `error.resets_at` + +#### Scenario: Streaming selection failure preserves usage-limit semantics + +- **WHEN** a streaming Responses request cannot select an account because every + eligible account is usage-exhausted before downstream-visible output +- **THEN** the terminal error event uses `usage_limit_reached` +- **AND** clients do not receive a generic no-account/server-unavailable error + +#### Scenario: Usage-limit selection failures are terminal, not waitable + +- **WHEN** account selection fails with `usage_limit_reached` on a streaming, + HTTP-bridge, or WebSocket Responses path +- **THEN** the proxy reports the structured usage-limit failure immediately +- **AND** it does not enter an account-capacity recovery wait for the + remaining request budget before reporting it + +#### Scenario: Local capacity codes keep their rate-limit contract + +- **WHEN** account selection fails with a local capacity or overload code such + as `account_stream_cap` or `account_response_create_cap` +- **THEN** the response keeps HTTP `429` with `error.type = "rate_limit_error"` + and the stable local error code +- **AND** the response is not reported as `usage_limit_reached` + +#### Scenario: Unusable non-exhausted pools keep existing semantics + +- **WHEN** every account is paused, deactivated, or requires re-authentication + and no eligible account is exhausted by a known usage window +- **THEN** the pre-existing `no_accounts` failure semantics are preserved + +#### Scenario: Owner-scoped exhaustion preserves continuity semantics + +- **WHEN** a request is pinned to a previous-response or file owner account and + only that owner is usage-exhausted while the wider eligible pool is usable +- **THEN** the proxy keeps the existing continuity-owner failure semantics +- **AND** it does not report pool-wide `usage_limit_reached` + +### Requirement: Silent HTTP bridge sessions are quarantined from re-attach and reuse + +When an HTTP bridge session proves silent/wedged, the proxy MUST quarantine its session key for a bounded window so later requests stop attaching to it. A session proves silent/wedged when either (a) a pending request being failed or retired carried a proxy-injected `previous_response_id`, had sent `response.create`, observed upstream response events, and never had `response.created` assigned, or (b) the session key hits two consecutive eventless `missing_response_created_timeout` retires. This holds for every path that fails or retires the request — partial stale-holder cleanup, the reader-failure funnel, and direct all-stale session retirement alike. The quarantine MUST be evaluated only when a request is already being failed or its session retired — never against a live owned turn — so a stream whose `response.created` was observed (including deferred-reasoning streams with long event gaps) MUST NOT be quarantined, and mere event silence during an owned live turn MUST NOT trigger quarantine by itself. + +While a session key is quarantined: an existing session under that key MUST NOT be selected for reuse (a new request detaches it and proceeds on a fresh session), and for durable-anchor selection a quarantined session that is still open MUST count as absent, exactly as if it were already gone. The quarantine registry verdict is authoritative for the key: any session under the key while the quarantine window is active — including a freshly created replacement whose own completion has not yet cleared the quarantine — is equally excluded from reuse and equally absent for anchor selection. A fresh reattach whose incoming payload already looks like a full conversation resend MUST NOT receive a proxy-injected durable anchor through any injection point — the fresh-reattach injection, session-state hydration of the durable anchor, or the session-level injection — so the dispatch goes upstream genuinely unanchored with the client's own untrimmed payload. A payload that does not look like a full resend (a genuine delta-only continuation) MUST still receive the durable anchor, because it has no other way to convey prior conversation state. + +Quarantine state MUST be bounded and self-recovering: it is in-memory and session-scoped, expires by TTL (a live session that outlives its quarantine window MUST become reusable again), is cleared when a response completes on the same session key, and MUST NOT write account health or alter account selection. + +#### Scenario: Reattach streams events but response.created is never assigned (#1534) + +- **GIVEN** a durable HTTP bridge session with a stored anchor whose fresh reattach injected a proxy-owned `previous_response_id` +- **AND** the reattached upstream stream delivers response events but `response.created` is never assigned +- **WHEN** the stream fails or the session is retired with that request still pending +- **THEN** the request fails terminally as before +- **AND** the session key is quarantined with reason `reattach_missing_response_created` + +#### Scenario: All-stale direct retirement still quarantines the key + +- **GIVEN** a wedged reattach (proxy-injected `previous_response_id`, `response.create` sent, response events observed, `response.created` never assigned) that is the ONLY stale pending request on its session +- **WHEN** the stuck-gate watchdog retires the session directly instead of failing the stale holder individually +- **THEN** the session key is quarantined with reason `reattach_missing_response_created` +- **AND** the next request takes the fresh no-anchor path instead of rebuilding the identical anchored reattach + +#### Scenario: Next request after the wedge completes on the fresh path + +- **GIVEN** a session key quarantined after a reattach that streamed events without `response.created` +- **WHEN** a later request arrives for the same key with a full-conversation-resend payload and no client `previous_response_id` +- **THEN** the proxy does not inject the durable anchor for that request +- **AND** the request is sent upstream unanchored with the client's own full payload +- **AND** the request can complete normally instead of rebuilding the identical wedged reattach + +#### Scenario: Suppressed anchor does not come back through session state + +- **GIVEN** a quarantined session key and a full-conversation-resend payload whose stored durable prefix is trimmable but whose fresh suffix does not retain the prior output +- **WHEN** the fresh-reattach durable-anchor injection is skipped because of the quarantine +- **THEN** the durable anchor is not rehydrated into the fresh session's completed-response state +- **AND** the session-level injection does not re-add the same anchor or trim the stored prefix +- **AND** the dispatch goes upstream genuinely unanchored with the client's untrimmed payload +- **AND** the suppression applies even when the fresh-reattach injection was already ineligible for other reasons (for example a conversation-scoped payload, a live alias session, or an active-owner forward that falls back to a local rebind) + +#### Scenario: Quarantined session is excluded from reuse selection + +- **GIVEN** a session marked quarantined that is still live or retained for admission handoff +- **WHEN** a new request looks up that session key +- **THEN** the session is not considered reusable +- **AND** the request proceeds on a fresh session instead +- **AND** a replacement session created under the same still-quarantined key is likewise not reusable until a completion or the TTL clears the quarantine + +#### Scenario: Repeated eventless timeouts quarantine the key + +- **GIVEN** a session key whose pending request already retired once with the eventless `missing_response_created_timeout` +- **WHEN** a subsequent attach on the same key retires with the same eventless timeout before any response completes on the key +- **THEN** the session key is quarantined with reason `repeated_eventless_timeout` +- **AND** the first timeout alone does not quarantine the key + +#### Scenario: Deferred-reasoning live turn is never quarantined + +- **GIVEN** an owned live turn whose `response.created` was observed and whose events flow with long gaps (deferred reasoning) +- **WHEN** its stream later fails or its session is retired +- **THEN** the session key is not quarantined +- **AND** later requests keep the existing reuse and anchor-injection behavior + +#### Scenario: Delta-only payloads keep their anchor while quarantined + +- **GIVEN** a quarantined session key — including one whose quarantined session is still open with other active requests +- **WHEN** a later request arrives whose payload does not look like a full conversation resend +- **THEN** the still-open quarantined session counts as absent for durable-anchor selection +- **AND** the durable anchor is still injected for that request, preserving the client's only way to convey prior context + +#### Scenario: Quarantine is bounded and self-clearing + +- **GIVEN** a quarantined session key +- **WHEN** a response completes on that session key, or the quarantine TTL elapses +- **THEN** the quarantine (and its eventless strike counter) is cleared +- **AND** a session that survived the quarantine window is reusable again instead of staying rejected forever +- **AND** no durable row, janitor work, or account-health write was involved at any point + +### Requirement: Scoped operation identity + +The system MUST include the normalized API-key scope in every durable HTTP +bridge operation fingerprint and MUST apply that scope to fingerprint and +completed-operation lookups. + +#### Scenario: Equal requests from different keys remain isolated + +- **WHEN** two API keys submit the same logical request +- **THEN** each key receives an independent durable operation identity + +### Requirement: Recoverable startup takeover + +Startup cleanup MUST retain sessions that own submitted, acknowledged, or +unknown operations and MUST detach ownership before a replacement instance +takes over. + +#### Scenario: Restart preserves an in-flight operation + +- **WHEN** an instance restarts while an operation is nonterminal +- **THEN** cleanup detaches the old owner without deleting the operation spool + +### Requirement: Fresh retry transcript + +When an explicit failed operation is rebound, the system MUST atomically remove +the prior operation events and reset event-byte/spool state before accepting new +events. + +#### Scenario: Failed retry cannot replay stale failure output + +- **WHEN** a failed operation is retried and later completes +- **THEN** replay contains only the new attempt's events + +### Requirement: Proof-gated sibling anchoring + +The system MUST advance a continuation to a completed sibling response only +when the sibling has the same parent and logical request fingerprint in the +same API-key scope. + +#### Scenario: Distinct sibling input keeps its requested parent + +- **WHEN** a request reuses a parent with a different fingerprint +- **THEN** the service does not silently anchor it to another child response + +### Requirement: Single migration head + +The Alembic graph MUST converge the durable operation revisions with the current +release head and MUST expose one canonical head after upgrade. + +#### Scenario: Upgrade resolves one head + +- **WHEN** migrations are upgraded to the release tip +- **THEN** Alembic reports one canonical head + +### Requirement: Conservative spool defaults + +New operation rows MUST start with an incomplete event spool on SQLite and +PostgreSQL. A transcript MUST become replayable only after terminal event drain +and explicit finalization. + +#### Scenario: Nonterminal spool is not replayable + +- **WHEN** an operation has events but no finalized terminal event +- **THEN** recovery does not replay its transcript as complete + +### Requirement: Retain completed recovery transcripts + +Startup ownership cleanup MUST retain sessions with operation transcripts that +remain inside the configured operation retention window, including completed +operations, and MUST let normal spool retention remove the operation rows. + +#### Scenario: Recent completed transcript survives takeover + +- **WHEN** startup cleanup sees a recent completed transcript +- **THEN** it retains the session until normal retention expires it + +### Requirement: Continuous transcript retention + +Operation transcript cleanup MUST run periodically in a leader-gated scheduler +and MUST drain all eligible batches during each pass. Disabling the existing +sticky-session mapping cleanup switch MUST NOT disable operation transcript +retention; that switch MAY skip sticky mapping maintenance while durable +operation retention continues. + +#### Scenario: Retention drains all eligible batches + +- **WHEN** more rows are eligible than one deletion batch +- **THEN** one scheduler pass removes every eligible batch + +#### Scenario: Sticky cleanup toggle does not disable transcript retention + +- **WHEN** sticky-session cleanup is disabled and the durable bridge schema is + available +- **THEN** the leader-gated scheduler still drains expired operation transcript + rows while skipping sticky mapping cleanup + +### Requirement: Fresh indefinite-recovery spool + +Before dispatching a server-owned retry for a nonterminal operation, the system +MUST atomically clear any partial event spool under the durable owner fence. + +#### Scenario: Retry starts with a clean transcript + +- **WHEN** an anchored retry is dispatched after partial persistence +- **THEN** old events and byte counts are cleared before new output is accepted + +### Requirement: Ordered deferred reasoning persistence + +Deferred reasoning events released before a visible event MUST be persisted in +the same order in which they are delivered downstream, before the visible +event is persisted. + +#### Scenario: Deferred events preserve downstream order + +- **WHEN** buffered reasoning is released before visible output +- **THEN** the durable spool stores the reasoning blocks before that output + +### Requirement: Per-operation disconnect classification + +When a shared bridge websocket closes, each pending operation MUST be +classified from that operation's own observed response-event count. Activity +from a sibling request MUST NOT make an eventless operation safely retryable. + +#### Scenario: Sibling output does not acknowledge an eventless request + +- **WHEN** one pending request emitted output and another emitted none +- **THEN** the two operations receive different disconnect classifications + +### Requirement: Abandoned operation retention + +Operation retention MUST expire stale submitted and acknowledged rows in +addition to terminal and ambiguous rows, so a crashed or abandoned operation +cannot retain raw request data indefinitely. + +#### Scenario: Stale abandoned request is purged + +- **WHEN** a submitted operation exceeds retention age +- **THEN** its request data and event spool are removed + +### Requirement: Acknowledged alias persistence failure + +If upstream has acknowledged a response but local continuity-alias persistence +fails, the downstream error MUST NOT transition the durable operation to a +retryable failed state. The operation MUST remain acknowledged/ambiguous so an +identical retry cannot dispatch a duplicate upstream turn. + +#### Scenario: Alias write failure remains fail-closed + +- **WHEN** an acknowledged response cannot publish its continuity alias +- **THEN** the operation remains non-retryable and the client receives a terminal error + +### Requirement: Cross-session nonterminal handoff + +When a scoped operation fingerprint is found under a different durable +session, a nonterminal operation MUST be atomically rebound to the currently +owned session before its event spool is reset or a recovery attempt is sent. +Completed replayable operations MUST remain attached to their original session. +The handoff MUST be refused while the prior session has an unexpired owner +lease, preventing concurrent owners from dispatching the same turn. + +#### Scenario: Active prior owner fences handoff + +- **WHEN** a duplicate request finds a nonterminal operation under another session +- **AND** that session still has an unexpired owner lease +- **THEN** the operation remains with the prior session and no concurrent retry is dispatched + +#### Scenario: Expired prior owner permits handoff + +- **WHEN** the prior session lease is absent or expired +- **THEN** the operation can be atomically rebound before recovery + +### Requirement: Fenced one-shot recovery dispatch + +The durable recovery journal MUST persist a one-shot replay budget for every +recovery-safe request. The budget MUST be consumed atomically when a replay is +claimed for dispatch, and a caller that proves the replay never reached the +upstream send boundary MUST restore that claim under the same session owner +fence. A replacement session MUST retain or transfer a fenced origin owner +until the claim is rolled back or settled; selecting a replacement or failing +preflight MUST NOT permanently consume an unsent replay. + +#### Scenario: Concurrent reconnects consume one replay + +- **WHEN** concurrent reconnects observe the same ambiguous operation +- **THEN** exactly one owner atomically claims the persisted replay budget and + other reconnects fail closed without dispatching a duplicate + +#### Scenario: Pre-dispatch replacement failure restores the budget + +- **WHEN** a replay claim is made but replacement admission or preflight fails + before the exact upstream frame is sent +- **THEN** the claim returns to the available state and the fenced origin + owner is released only after that rollback succeeds + +#### Scenario: Successful replacement settles the origin journal + +- **WHEN** a replacement session dispatches the claimed replay and receives a + terminal response event +- **THEN** settlement uses the retained origin owner fence before releasing it + and the replay budget cannot be claimed again + +### Requirement: Lease-aware operation retention + +Retention MUST NOT delete stale submitted or acknowledged operations while +their session is actively owned with an unexpired lease. The owner/lease +predicate MUST be rechecked in the deletion transaction. + +#### Scenario: Active lease protects stale operation + +- **WHEN** a stale operation belongs to a session with a live lease +- **THEN** retention leaves it intact + +### Requirement: Anchored indefinite recovery gate + +The server-indefinite recovery loop MUST be installed only for an eventless +anchored continuation with a durable parent operation. Fresh first-turn +requests and streams that already emitted downstream response events MUST +terminate normally rather than being resent indefinitely. + +#### Scenario: Fresh request is not held indefinitely + +- **WHEN** a first-turn request loses its upstream connection +- **THEN** the proxy returns its normal error path without an indefinite loop + +### Requirement: Retry reservation terminalization + +If reacquiring API-key usage limits for a recovery attempt fails, the proxy +MUST settle the prior reservation and emit a terminal `response.failed` SSE +event instead of aborting the already-started stream. + +#### Scenario: Quota failure produces terminal SSE + +- **WHEN** a recovery retry cannot reacquire its usage reservation +- **THEN** the client receives `response.failed` and the prior reservation is settled + +#### Scenario: Unexpected admission failure produces terminal SSE + +- **WHEN** recovery admission raises an unexpected infrastructure error before + a replacement stream starts +- **THEN** the client receives `response.failed` and the prior reservation is + settled instead of receiving a truncated stream + +### Requirement: Failure spool/state ordering + +For an explicit deterministic failure, the proxy MUST persist the terminal SSE +block before exposing the durable operation as failed. The event append and +failed-state transition MUST use the same owner fence and transaction when the +durable repository supports it. + +#### Scenario: Concurrent retry cannot reset an unspooled failure + +- **WHEN** a response failure is being settled while an identical reconnect is + admitted +- **THEN** the reconnect observes the terminal operation fence and cannot reset + or mix the previous failure into a new transcript + +### Requirement: Partial disconnect acknowledgement + +When a bridge disconnects after an operation has emitted any response event but +before a terminal event, the durable operation MUST remain acknowledged or +ambiguous. It MUST NOT be classified as retryable failed solely because the +disconnect was non-terminal. + +#### Scenario: Partial output is never resent as a fresh turn + +- **WHEN** the upstream closes after `response.created` but before completion +- **THEN** the operation remains non-retryable + +### Requirement: Retry output stops indefinite recovery + +An indefinite recovery attempt MUST stop retrying once that attempt emits any +downstream response event, even if the attempt later fails with a retryable +transport error. + +#### Scenario: Retry output prevents a second attempt + +- **WHEN** a retry emits a data event and then times out +- **THEN** the server stops the indefinite loop instead of appending another response + +### Requirement: Preserve repeated event occurrences + +The durable event spool MUST preserve repeated identical SSE blocks as distinct +ordered occurrences. Event identity MUST include its operation-local sequence +position rather than content alone. + +#### Scenario: Identical deltas replay twice + +- **WHEN** two consecutive SSE blocks have identical text +- **THEN** both occurrences are present in the replay transcript + +### Requirement: Stop event persistence during shutdown + +Proxy shutdown MUST close the HTTP bridge event batcher and cancel its +background flusher before the process exits. + +#### Scenario: Shutdown cancels the flusher + +- **WHEN** the proxy service begins shutdown after queueing an event +- **THEN** the batcher's background task is cancelled and awaited + +### Requirement: Classify response.incomplete as terminal + +An anchored `response.incomplete` event MUST transition the durable operation to +an explicit terminal state and finalize its transcript so it is not left in an +unknown in-flight state. + +#### Scenario: Incomplete response is replayable as terminal + +- **WHEN** upstream emits `response.incomplete` +- **THEN** the operation is terminalized and its drained transcript is eligible for replay + +### Requirement: Settle reservations before timeout health + +When an eventless timeout retires a keyed bridge, the proxy MUST settle all +pending request reservations before recording the account timeout health signal. +If settlement fails, the health signal MUST NOT claim that cleanup completed. + +#### Scenario: Failed reservation release does not poison health state + +- **WHEN** the timeout cleanup cannot release a pending reservation +- **THEN** the account timeout signal is not recorded before that failure is surfaced + +### Requirement: Replay finalized incomplete operations + +A finalized `incomplete` operation transcript MUST be replayed for an identical +request and MUST NOT be reset or treated as an unknown in-flight operation. + +#### Scenario: Reconnect receives stored incomplete transcript + +- **WHEN** an identical request finds a finalized incomplete operation +- **THEN** the stored terminal transcript is delivered without a new upstream dispatch + +### Requirement: Validate final response.create size + +After adding durable operation metadata, the proxy MUST revalidate the exact +serialized `response.create` frame against the upstream size limit before +sending it. + +#### Scenario: Metadata cannot create an oversized frame + +- **WHEN** operation metadata makes the final frame exceed the configured limit +- **THEN** the request is rejected or slimmed before any upstream send + +### Requirement: Fence same-session active operations + +Server-indefinite recovery MUST NOT reset or redispatch a nonterminal operation +when another pending request in the same durable session still references that +operation. Submitted and acknowledged operations MUST remain fail-closed; +only an inactive `unknown` operation may enter a fresh recovery attempt. + +#### Scenario: Active same-session operation is not duplicated + +- **WHEN** a duplicate request finds a submitted operation still referenced by another pending request +- **THEN** the proxy refuses a second dispatch and preserves the existing spool + diff --git a/openspec/specs/runtime-portability/spec.md b/openspec/specs/runtime-portability/spec.md index 957a866c8d..bd790b4d94 100644 --- a/openspec/specs/runtime-portability/spec.md +++ b/openspec/specs/runtime-portability/spec.md @@ -53,3 +53,41 @@ The `codex-lb` CLI SHALL provide a `codex-sessions retag` subcommand that rewrit - **THEN** the command uses that path as the Codex data directory - **AND** otherwise it falls back to `CODEX_HOME`, `/codex-home` in containers, a discoverable WSL Windows profile Codex directory, or `~/.codex` +### Requirement: Server CLI validates the main listener port before startup + +The `codex-lb` server CLI SHALL accept integer main-listener ports in the inclusive range `0..65535` when supplied through `--port` or `PORT`, and an explicit `--port` SHALL continue to take precedence over `PORT`. The CLI SHALL reject non-integer values and integers outside that range before loading Uvicorn, importing or starting the ASGI application, running its lifespan or migrations, or creating runtime data. A rejection MUST identify `--port/PORT`, state the supported range, and include the invalid value. + +#### Scenario: Out-of-range command-line port is rejected before startup + +- **WHEN** an operator supplies `--port` with an integer below `0` or above `65535` +- **THEN** the CLI exits with an error that identifies `--port/PORT`, the invalid value, and the supported range `0..65535` +- **AND** Uvicorn is not loaded +- **AND** the ASGI lifespan, migrations, and runtime data creation do not run + +#### Scenario: Out-of-range environment port is rejected before startup + +- **WHEN** `PORT` contains an integer below `0` or above `65535` +- **AND** no `--port` flag is supplied +- **THEN** the CLI exits with an error that identifies `--port/PORT`, the invalid value, and the supported range `0..65535` +- **AND** Uvicorn is not loaded +- **AND** the ASGI lifespan, migrations, and runtime data creation do not run + +#### Scenario: Non-integer listener port is rejected before startup + +- **WHEN** the selected `--port` or `PORT` value is not an integer +- **THEN** the CLI exits with an error that identifies `--port/PORT` and the invalid value +- **AND** Uvicorn is not loaded + +#### Scenario: Inclusive listener-port boundaries are forwarded + +- **WHEN** the selected `--port` or `PORT` value is `0` or `65535` +- **THEN** the CLI forwards the same integer to Uvicorn +- **AND** port `0` retains Uvicorn's ephemeral-listener behavior + +#### Scenario: Command-line port retains precedence over the environment + +- **WHEN** `PORT` contains any value +- **AND** the operator supplies an in-range `--port` value +- **THEN** the CLI validates and forwards the flag value +- **AND** the environment value does not replace it + diff --git a/openspec/specs/sticky-session-operations/spec.md b/openspec/specs/sticky-session-operations/spec.md index 8091fe5fb4..0c2e146df1 100644 --- a/openspec/specs/sticky-session-operations/spec.md +++ b/openspec/specs/sticky-session-operations/spec.md @@ -576,3 +576,89 @@ The background cleanup loop MUST delete ACTIVE and DRAINING `http_bridge_session - **GIVEN** the prompt-cache bridge idle TTL exceeds the prompt-cache affinity max age - **WHEN** the cleanup loop runs against an ACTIVE row whose lease expired but whose `last_seen_at` is within the prompt-cache bridge idle TTL - **THEN** the row and its aliases are preserved so a local reuse keeps its durable ownership and continuity anchors + +### Requirement: Restart removes stale owned bridge state + +On startup, the system MUST remove ordinary persisted HTTP bridge session rows owned by the configured bridge instance from the previous process. A recent server-namespaced account-neutral recovery row MUST instead be changed to ownerless DRAINING with an expired lease while preserving its aliases and original activity timestamp. The cleanup MUST remove ownerless ACTIVE/DRAINING rows with expired leases once their activity predates the abandoned-row retention cutoff. Deleted rows MUST lose their associated durable bridge aliases. The cleanup MUST NOT remove sticky-session mappings or rows owned by other bridge instances. + +#### Scenario: First request after restart starts without stale bridge state + +- **GIVEN** the previous process left ordinary durable HTTP bridge rows owned by the configured instance +- **WHEN** the next process completes startup +- **THEN** those durable bridge rows and their aliases MUST be removed before accepting requests +- **AND** the first request MUST create fresh bridge state instead of reusing the previous process's bridge row +- **AND** sticky-session mappings MUST remain available + +#### Scenario: Recent verified recovery proof survives restart only until retention + +- **GIVEN** the previous process left a recent server-namespaced account-neutral recovery row with task-specific aliases +- **WHEN** the next process completes startup +- **THEN** the row MUST become ownerless DRAINING with an expired lease +- **AND** its task-specific aliases and original activity timestamp MUST remain unchanged +- **AND** a later startup or abandoned-row cleanup MUST remove the row and aliases after the activity timestamp passes the retention cutoff + +#### Scenario: Ownerless stale rows with expired leases are removed + +- **GIVEN** durable HTTP bridge rows exist with no owner instance, expired leases, and activity older than the abandoned-row retention cutoff +- **WHEN** the process completes startup +- **THEN** those rows and their aliases MUST be removed +- **AND** rows owned by other instances MUST NOT be removed + +#### Scenario: Sticky-session mappings are preserved + +- **GIVEN** sticky-session mappings exist for the account +- **WHEN** the process completes startup and purges stale bridge rows +- **THEN** sticky-session mappings MUST remain available for account affinity + +### Requirement: Trusted capability requirements are monotonic across lineage + +Before dispatch, the proxy MUST persist an authenticated `trusted_cyber` +requirement as API-key-scoped, domain-separated opaque hashes for every known +session, accepted or synthesized turn-state, previous-response, and Codex task +lineage alias. Marker writes MUST be monotonic and MUST NOT store raw lineage +or account identifiers. + +A later authenticated request MUST restore REQUIRED before account selection +when any presented alias matches under the same API-key scope. It MUST persist +that requirement onto newly generated aliases. A marker under one API key MUST +NOT establish REQUIRED under another key. Read or write uncertainty MUST fail +before ordinary dispatch. + +When `response.created` first reveals a response ID for a durably REQUIRED +request, the proxy MUST persist the upstream and downstream-visible response +aliases before forwarding that created event. If this propagation fails, the +proxy MUST NOT expose the unpersisted response ID, replay the accepted request, +or penalize the upstream account. + +#### Scenario: No-echo reconnect remains required +- **WHEN** a capability-bearing direct WebSocket turn persists an accepted + session identity and a proxy-synthesized turn state +- **AND** a new connection presents the same session identity without the + capability marker or generated turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Echoed synthesized turn state remains required +- **WHEN** the reconnect instead echoes the accepted synthesized turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Response-only reconnect remains required +- **WHEN** a capability-bearing turn exposes a response ID only after upstream + acceptance +- **AND** a new connection presents only that `previous_response_id` under the + same API key, without a matching session or turn state +- **THEN** REQUIRED is restored before its first account selection + +#### Scenario: Requirement survives a fresh service instance +- **WHEN** a new repository and proxy service instance reads an alias marked by + an earlier instance +- **THEN** the alias still restores REQUIRED + +#### Scenario: API-key scope is isolated +- **WHEN** API key B presents the same visible lineage identifier previously + marked under API key A +- **THEN** key A's marker does not establish REQUIRED for key B + +#### Scenario: Persistence uncertainty cannot downgrade +- **WHEN** required lineage cannot be read or established durably +- **THEN** the request fails before ordinary account selection or dispatch + diff --git a/openspec/specs/upstream-proxy-routing/spec.md b/openspec/specs/upstream-proxy-routing/spec.md index ba2abac2f6..75ec4b972e 100644 --- a/openspec/specs/upstream-proxy-routing/spec.md +++ b/openspec/specs/upstream-proxy-routing/spec.md @@ -95,3 +95,67 @@ failure. - **GIVEN** a routed WebSocket context manager enters successfully - **WHEN** the client returns the opened WebSocket and its context to the caller - **THEN** the caller can exit the returned context using the existing ownership contract + +### Requirement: Cached route resolution preserves fail-closed semantics + +Any cache in front of upstream-route resolution MUST store the resolver's outcome verbatim — a resolved route, a permitted direct-egress `None`, or a fail-closed error with its reason. A cache hit MUST reproduce that outcome exactly: it MUST NOT convert a fail-closed outcome or a routed outcome into direct egress, and it MUST NOT substitute a different pool or endpoint than the resolver chose. Cache staleness MUST be bounded by invalidation on admin mutations (same-replica: before the mutating response returns; peers: within one cache-invalidation poll interval) with a TTL backstop for out-of-band edits. + +#### Scenario: Cached fail-closed outcome keeps failing closed + +- **GIVEN** an account-bound pool with no active usable endpoint whose fail-closed resolution outcome is cached +- **WHEN** further upstream operations are attempted for that account +- **THEN** each operation MUST fail before opening an upstream network connection with the same fail-closed reason +- **AND** it MUST NOT use the default pool, environment proxy, or direct egress + +#### Scenario: New binding takes effect without a direct-egress window on the mutating replica + +- **GIVEN** an account whose cached resolution outcome is direct-egress `None` +- **WHEN** an operator saves an active proxy binding for that account +- **THEN** the mutating replica's cached outcome MUST be invalidated before the binding response returns, so subsequent requests on that replica resolve the bound pool + +### Requirement: Confirmed account-proxy connection failures fail over safely + +When an account-routed transport reports that it could not connect to the selected proxy endpoint and proves that the upstream request was not dispatched, the service MUST classify the failure with sanitized structured pre-dispatch provenance. For a route with another usable endpoint in the same proxy pool, the client MUST try that endpoint before moving accounts, including for a non-idempotent request. If the pool cannot connect, movable Responses requests MUST exclude the failed account and retry another eligible account within the existing request budget and attempt limits. + +This behavior MUST cover raw HTTP/SSE, native Responses WebSocket, and the HTTP responses bridge. Before recording transient account backoff, the service MUST release response-create and stream leases held for the failed account. A request-scoped API-key reservation MUST remain singular across an internal pre-dispatch failover, MUST settle or release at the terminal request outcome before the account-health write, and MUST NOT be reacquired solely for the internal failover. If neither settlement nor fallback release can be confirmed, the service MUST leave the health write unapplied. HTTP-bridge startup cleanup MUST release only an unowned current request lifecycle, and each reservation lifecycle MUST drain only its own health writes after confirmed settlement or release. The confirmed failure MUST place the account at the existing bounded transient error-backoff floor, but MUST NOT pause, deactivate, rate-limit, or quota-penalize it. + +The service MUST NOT replay a request when dispatch is unknown or when the request depends on hard previous-response, turn-state, uploaded-file, single-account, or other required account ownership. If no eligible replacement account exists, the service MUST preserve the original sanitized upstream-unavailable failure instead of replacing it with a generated `no_accounts` error. + +#### Scenario: POST uses a healthy endpoint from the same proxy pool + +- **GIVEN** a non-idempotent Responses POST is routed through a proxy pool with two endpoints +- **AND** connecting to the first endpoint fails before request dispatch +- **WHEN** the second endpoint is reachable +- **THEN** the service sends the request through the second endpoint +- **AND** it does not move the request to another account + +#### Scenario: movable request retries another account + +- **GIVEN** two eligible accounts and the first account's complete proxy route refuses connections before dispatch +- **WHEN** a fresh Responses request has no hard account ownership +- **THEN** the service releases the first account's response-create and stream leases +- **AND** it settles or releases any request-scoped API-key reservation before the account-health write +- **AND** it records bounded transient backoff for the first account +- **AND** it excludes the first account and completes through the second account +- **AND** no failure event from the first attempt is forwarded downstream + +#### Scenario: hard account ownership fails closed + +- **GIVEN** a Responses request depends on a previous-response owner or an account-scoped uploaded file +- **AND** the required account's proxy refuses the connection before dispatch +- **WHEN** another account is otherwise eligible +- **THEN** the service does not send the request to the other account +- **AND** it returns the sanitized upstream-unavailable failure for the required account + +#### Scenario: ambiguous transport failure is not replayed + +- **WHEN** a POST transport failure cannot prove that request dispatch was impossible +- **THEN** the service does not use that failure as authorization to retry another proxy endpoint or account + +#### Scenario: empty replacement pool preserves the original failure + +- **GIVEN** a movable request has a confirmed pre-dispatch proxy connection failure +- **AND** no other eligible account can be selected +- **THEN** the client receives the original sanitized upstream-unavailable failure +- **AND** the failure is not replaced with `no_accounts` + diff --git a/openspec/specs/usage-error-metrics/spec.md b/openspec/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..484b300d87 --- /dev/null +++ b/openspec/specs/usage-error-metrics/spec.md @@ -0,0 +1,170 @@ +# usage-error-metrics Specification + +## Purpose +Error-rate accounting that counts only genuinely-failed terminals: cancelled client disconnects fold into a separate cancelled_count in live metrics and hourly rollups, with historical rows kept compatible. +## Requirements +### Requirement: Error metrics count only genuinely-failed terminals + +Every materialization of a request-log error count or error rate — the usage +summary metrics, the dashboard overview activity metrics and per-bucket +error-rate trend inputs, the reports daily and summary aggregates, and the +fleet pressure metrics — MUST classify a request-log row as an error only +when `status NOT IN ('success', 'cancelled')`. Rows with `status = +'cancelled'` (normal client-side disconnect terminals, e.g. +`error_code=client_disconnected`) MUST NOT be counted in any error numerator. +Error-rate denominators MUST remain the total request count of the window. +Every request-log producer MUST record a downstream client disconnect as +`status='cancelled'`; in particular the model-source streaming path MUST NOT +record a mid-stream client disconnect as `status='error'`. + +#### Scenario: Cancelled rows do not inflate the dashboard error rate + +- **GIVEN** a window containing 1 successful, 2 cancelled + (`client_disconnected`), and 1 error (`upstream_500`) request-log rows +- **WHEN** the dashboard overview activity metrics are computed +- **THEN** the error count is `1` and the error rate is `0.25` +- **AND** the request total remains `4` + +#### Scenario: Reports and fleet windows exclude cancelled rows from errors + +- **GIVEN** the same window of rows +- **WHEN** the reports summary/daily aggregates and the fleet pressure + metrics are computed +- **THEN** each reports `error_count` / `total_errors` and each fleet + `error_count` equals `1` + +#### Scenario: Model-source stream disconnects land as cancelled + +- **GIVEN** a streamed model-source request whose downstream client + disconnects mid-stream +- **WHEN** the request log is written +- **THEN** the row has `status='cancelled'` and + `error_code='client_disconnected'` +- **AND** the window's error count excludes it, its cancelled count includes + it, and `top_error` does not report `client_disconnected` + +### Requirement: Hourly rollups fold a cancelled_count measure + +The `request_usage_hourly_rollups` table MUST carry a `cancelled_count` +measure (non-null, server default 0), introduced by an additive Alembic +migration whose parent is the current single migration head and whose +downgrade drops only the new column. The hourly fold MUST populate +`cancelled_count` as `sum(status = 'cancelled')` and MUST fold `error_count` +as `sum(status NOT IN ('success', 'cancelled'))`. Account lifecycle mirrors +MUST move `cancelled_count` with the other measures. + +#### Scenario: Fold splits error and cancelled measures + +- **GIVEN** one hour of raw rows with 1 success, 2 cancelled, and 1 error + sharing the same dimensions +- **WHEN** the hourly fold pass folds that hour +- **THEN** the folded bucket has `request_count=4`, `error_count=1`, and + `cancelled_count=2` + +### Requirement: Historical hourly rollup rows keep the old error fold + +Hourly rollup rows folded before the `cancelled_count` measure existed MUST +NOT be backfilled or re-split: their `error_count` keeps the legacy +`sum(status != 'success')` fold and their `cancelled_count` reads 0 via the +column's server default. Error-rate trends over such buckets exhibit a +disclosed step change at deploy. + +#### Scenario: Pre-existing rollup rows are readable unchanged + +- **GIVEN** a rollup row folded before the migration +- **WHEN** the dashboard reads it after upgrading +- **THEN** the read succeeds with `cancelled_count=0` and the row's stored + `error_count` unchanged + +### Requirement: New code repairs the rolling-upgrade fold window + +Because the migration runs before old replicas drain, a legacy replica may +fold post-migration hours with the old error fold and advance the shared +watermark — by up to its full per-pass slice budget, so no fixed trailing +window can bound the damage. The migration MUST persist the legacy-suspect +range start on the fold-state row (`upgrade_repair_from`): existing rows are +stamped with their migration-time `hourly_folded_through`, and the column's +epoch server default covers a state row bootstrapped by an old replica after +the migration (its entire backfill is legacy-suspect); new code's own +bootstrap MUST write the marker as NULL, and NULL MUST only ever be written +by new code, meaning no legacy-suspect range is outstanding. + +While the marker is set, the hourly fold pass MUST refold +`[upgrade_repair_from, hourly_folded_through)` from raw request logs in +bounded slice-sized chunks, persisting progress by advancing the marker with +each chunk's commit and setting it to NULL only once the range is covered — +a crash resumes instead of restarting, and a pass-bounded incomplete repair +continues on later passes. With the marker NULL, the first fold pass of each +new-code process MUST still refold the trailing repair window below the +watermark (a span that comfortably exceeds any rolling-upgrade duration) as +defense against a legacy replica regaining fold leadership after the marker +was cleared. + +Both paths MUST be idempotent (converging DELETE-then-INSERT recomputation), +MUST run under the existing fold leader gate and fold-state row lock, MUST +NOT move the watermark, and MUST NOT touch folded buckets below the +surviving-raw clamp (whole hours fully covered by surviving raw rows; +retention-pruned history is irrecoverable and keeps the disclosed legacy +fold). This is a targeted repair of the rollout window only — not a +historical backfill. + +#### Scenario: A legacy-folded post-migration bucket is repaired + +- **GIVEN** a bucket inside the repair window whose rollup rows carry the + legacy fold (cancelled rows in `error_count`, `cancelled_count=0`, + `client_disconnected` in the error satellite) while its raw rows survive +- **WHEN** the new code runs its first hourly fold pass +- **THEN** the bucket is recomputed with `error_count` excluding cancelled + rows, `cancelled_count` populated, and the `client_disconnected` satellite + rows removed + +#### Scenario: A multi-slice legacy advance is fully repaired via the marker + +- **GIVEN** `upgrade_repair_from` set below legacy-folded buckets spanning + more than one fold slice (a legacy leader advanced several slices in one + pass) +- **WHEN** the new code runs its hourly fold passes +- **THEN** every bucket in `[upgrade_repair_from, watermark)` with surviving + raw rows is recomputed and the marker ends NULL + +#### Scenario: Buckets below the surviving-raw clamp are preserved + +- **GIVEN** a folded bucket inside the repair span whose raw rows were + already pruned by retention +- **WHEN** the repair runs +- **THEN** that bucket's rollup rows are left untouched + +### Requirement: Top error excludes cancelled terminals + +`top_error` computations MUST NOT derive from cancelled rows: raw request-log +scans MUST filter `status NOT IN ('success', 'cancelled')`, the error +satellite fold MUST apply the same status filter going forward, and reads of +historical error-satellite rows (folded under the legacy filter) MUST exclude +the `client_disconnected` error code. + +#### Scenario: client_disconnected no longer dominates top error + +- **GIVEN** a window with 200 cancelled rows (`client_disconnected`) and 3 + error rows (`upstream_500`) +- **WHEN** `top_error` is computed for the dashboard or fleet windows +- **THEN** the result is `upstream_500` + +### Requirement: Cancelled counts surface alongside error counts + +Metric surfaces that expose an error count MUST also expose the window's +cancelled count as an additive field: the dashboard overview metrics +(`cancelledCount`), the usage summary metrics (`cancelled7d`), the reports +daily rows (`cancelled_count`) and summary (`total_cancelled`), and the fleet +pressure metrics (`cancelledCount`). The dashboard overview cancelled total +MUST be sourced from the demand quarter rollup (status grain) for the folded +segment plus the raw tail, so it stays accurate across history already folded +without the hourly `cancelled_count` measure. + +#### Scenario: Dashboard overview reports the status breakdown + +- **GIVEN** a window containing 1 successful, 2 cancelled, and 1 error rows + that are partially folded into the rollups +- **WHEN** the dashboard overview metrics are computed +- **THEN** the metrics expose `requests=4`, `errorCount=1`, and + `cancelledCount=2` + diff --git a/openspec/specs/usage-refresh-policy/spec.md b/openspec/specs/usage-refresh-policy/spec.md index 5e44ca14d6..bc87e533bc 100644 --- a/openspec/specs/usage-refresh-policy/spec.md +++ b/openspec/specs/usage-refresh-policy/spec.md @@ -505,10 +505,9 @@ The system SHALL NOT infer weekly secondary semantics solely because a primary-s ### Requirement: Background usage refresh is staggered across accounts -Background usage refresh MUST distribute account refresh attempts across the -configured usage refresh interval instead of refreshing every eligible account -in one burst. Each scheduler slice MUST attempt at most one eligible account. -Over a full cycle, all eligible accounts SHOULD be considered once. +Background usage refresh MUST distribute account refresh attempts across the configured usage refresh interval instead of refreshing every eligible account in one burst. Each scheduler slice MUST attempt at most one eligible account. Over a full cycle, all eligible accounts SHOULD be considered once. + +Each slice MUST select its account before reading usage history and MUST scope its latest-usage lookups, updater input, warm-up candidate evaluation, and recoverable-status evaluation to that selected account. The scheduler MAY retain the full eligible account roster only to choose the deterministic rotation and calculate staggered warm-up phases; that roster MUST NOT cause usage-history reads, upstream refresh attempts, warm-up sends, or status mutations for an unrelated account in the slice. A selected-account refresh failure MUST NOT trigger same-slice fallback to another account. Database sessions used to load scheduler state MUST close before upstream network I/O begins, and concurrent follow-up work MUST NOT share an `AsyncSession`. #### Scenario: Scheduler refreshes one account per slice @@ -516,8 +515,7 @@ Over a full cycle, all eligible accounts SHOULD be considered once. - **WHEN** the scheduler runs consecutive refresh slices - **THEN** the first slice attempts one account - **AND** the second slice attempts the other account -- **AND** cache invalidation for usage-derived routing state runs at the cycle - boundary +- **AND** cache invalidation for usage-derived routing state runs at the cycle boundary #### Scenario: Unrefreshable accounts are skipped by scheduler rotation @@ -527,6 +525,38 @@ Over a full cycle, all eligible accounts SHOULD be considered once. - **WHEN** the scheduler builds the refresh rotation - **THEN** only the active account is considered +#### Scenario: Selected slot scopes usage history and follow-up work + +- **GIVEN** two eligible accounts have stored primary, secondary, and monthly usage +- **AND** the first account is selected for the current scheduler slice +- **WHEN** the scheduler reads before/after usage and evaluates warm-up and recoverable status +- **THEN** every usage-history lookup is filtered to the first account +- **AND** only the first account is passed to usage refresh, warm-up candidate evaluation, and recoverable-status evaluation +- **AND** the second account cannot be mutated or contacted during that slice + +#### Scenario: Warm-up phase cohort does not widen evaluation scope + +- **GIVEN** multiple warm-up-enabled accounts participate in staggered-idle phase calculation +- **AND** one account is selected for the current usage-refresh slice +- **WHEN** refreshed usage is evaluated for warm-up +- **THEN** the phase calculation retains the eligible fleet cohort +- **AND** only the selected account can create a warm-up attempt or send warm-up traffic + +#### Scenario: Selected-account failure does not fail over within the slice + +- **GIVEN** two accounts are eligible for scheduler rotation +- **AND** the first account is selected +- **WHEN** that account's usage refresh fails +- **THEN** the scheduler does not attempt the second account in the same slice +- **AND** the second account remains eligible for its normal later slice + +#### Scenario: Scheduler session closes before selected-account network work + +- **GIVEN** the scheduler loaded the account roster and selected account usage +- **WHEN** the selected account's upstream refresh starts +- **THEN** the scheduler read session is already closed +- **AND** any concurrent warm-up follow-up owns an independent database session + ### Requirement: Usage refresh trusts recognized paid-plan transitions without workspace identity Usage refresh MUST persist a stored account's `plan_type` change when @@ -1328,3 +1358,184 @@ A non-2xx upstream response or the network-failure sentinel MUST NOT count as a - **WHEN** the older successful probe attempts to settle - **THEN** settlement is rejected as stale - **AND** the newer transient error state and reset success streak remain intact + +### Requirement: Implausible persisted rate-limit deadlines do not block recovery + +Background usage refresh MUST treat a persisted `rate_limited` reset deadline +as invalid when it is non-finite, elapsed, or beyond +`RATE_LIMIT_RESET_MAX_HORIZON_SECONDS` (366 days) plus the less-than-one-second +whole-second persistence tolerance. An invalid deadline MUST NOT be treated as +an unexpired explicit cooldown. When the account carries `blocked_at`, recovery +MUST still honor the existing 30-second minimum floor and MUST still require +the existing fresh available quota evidence recorded after the block. Without +`blocked_at`, recent available evidence SHALL suffice. Every applicable quota +window MUST report below `100%` usage before recovery. + +#### Scenario: Scheduler recovers an implausible persisted cooldown + +- **WHEN** an account is persisted as `rate_limited` with a reset deadline more than 366 days in the future +- **AND** its persisted `blocked_at` minimum floor has elapsed +- **AND** a later background usage refresh writes fresh available quota evidence +- **THEN** the scheduler treats the reset deadline as invalid +- **AND** marks the account `active` +- **AND** clears persisted `reset_at` and `blocked_at` + +#### Scenario: Scheduler preserves a plausible unexpired cooldown + +- **GIVEN** an account is persisted as `rate_limited` with a finite reset deadline within 366 days +- **AND** that deadline has not elapsed +- **WHEN** a later background usage refresh writes fresh available quota evidence +- **THEN** the scheduler leaves the account `rate_limited` + +#### Scenario: Scheduler recovers an implausible legacy deadline without a block marker + +- **GIVEN** an account is persisted as `rate_limited` with an implausible reset deadline and no `blocked_at` +- **WHEN** a later background usage refresh writes recent available quota evidence for every applicable window +- **THEN** the scheduler marks the account `active` +- **AND** clears persisted `reset_at` + +### Requirement: Weekly-primary remap tiebreak is data-aware within a fetch + +The weekly-primary to secondary remap tiebreak (`should_use_weekly_primary` / `normalize_weekly_only_rows`) MUST be data-aware within a single refresh fetch and MUST NOT let a sub-second `recorded_at` difference between same-fetch rows decide the winner. + +A row carries real quota metadata when it has a positive `window_minutes` AND a non-null `reset_at`; a row that lacks both is a no-data placeholder. For the data-aware tiebreak, a no-data placeholder MUST be classified as the absence of a measurement and MUST NOT be treated as a measurement of zero usage merely because its stored `used_percent` is zero — a timestamped placeholder must not beat an untimestamped real row, and a same-fetch real row must not be displaced by a placeholder. When two competing rows are from the same fetch (their `recorded_at` values differ by at most `SIBLING_FETCH_MARGIN_SECONDS`, 5.0 seconds, or one/both timestamps are unavailable), a weekly `primary` row that carries real quota metadata MUST be selected over a competing `secondary` row that is a no-data placeholder, and a real `secondary` row MUST be selected over a no-data `primary` placeholder. (Rendering a newer no-data placeholder that wins a cross-fetch comparison as an explicit "unavailable" window is out of scope for this change; the cross-fetch winner is rendered per existing placeholder rules.) + +When both rows carry `recorded_at` and their difference is strictly greater than `SIBLING_FETCH_MARGIN_SECONDS`, the rows are from genuinely different fetches and the newer row MUST win — a later fetch is more authoritative about what upstream currently reports. This preserves the pre-fix cross-fetch behavior so a stale real weekly primary cannot freeze the weekly value over a fresh placeholder from a later fetch. + +This tiebreak MUST be shared by every consumer of `should_use_weekly_primary`, including account-summary remap, dashboard overview and projection aggregation, and per-bucket account usage trend remap, so the weekly quota is reported consistently across all surfaces. + +#### Scenario: Same-fetch real weekly primary beats a no-data secondary placeholder + +- **GIVEN** an account whose latest `primary` usage row reports a weekly window (`window_minutes == 10080`) with a non-null `reset_at` and `used_percent` below 100 +- **AND** the latest `secondary` usage row is a no-data placeholder (`window_minutes` falsy or null, `reset_at` null, `used_percent` 0.0, no credit metadata) +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) of each other in the same refresh cycle +- **WHEN** the system derives the effective secondary (weekly) usage window for account summaries, dashboard overview/projection aggregation, or account usage trends +- **THEN** the weekly `primary` row is selected as the source of weekly usage +- **AND** the reported weekly remaining percent equals `100 - primary.used_percent` +- **AND** the reported value does not jump to 100% remaining + +#### Scenario: Real secondary beats a no-data primary placeholder in the same fetch + +- **GIVEN** an account whose latest `secondary` usage row carries real quota metadata (positive `window_minutes` and a non-null `reset_at`) +- **AND** the latest `primary` usage row is a no-data placeholder +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` of each other +- **WHEN** the system derives the effective secondary usage window +- **THEN** the real `secondary` row is selected as the source of weekly usage +- **AND** the reported weekly remaining percent reflects that row's `used_percent` + +#### Scenario: Genuinely newer row from a later fetch wins regardless of metadata + +- **GIVEN** an account whose latest `primary` usage row reports a weekly window with real quota metadata but was written in an earlier fetch +- **AND** a later fetch wrote a competing `secondary` row whose `recorded_at` is more than `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) after the primary row +- **WHEN** the system derives the effective secondary usage window +- **THEN** the newer row from the later fetch is selected +- **AND** the stale real weekly primary does not freeze the weekly value indefinitely + +#### Scenario: Two real same-fetch weekly rows resolve by reset-at precedence + +- **GIVEN** an account whose latest `primary` and `secondary` usage rows both carry real quota metadata +- **AND** the two rows were recorded within `SIBLING_FETCH_MARGIN_SECONDS` (5.0 seconds) of each other in the same refresh cycle +- **WHEN** the system derives the effective secondary usage window across repeated refresh cycles +- **THEN** the selected row is determined by reset-at precedence and the stable weekly-primary default +- **AND** the selection does not flip between the two rows on a sub-second `recorded_at` difference + +### Requirement: Standard usage refresh snapshots persist atomically + +For one account's successful upstream usage response, the system MUST persist every available normalized standard usage window (`primary`, `secondary`, and any applicable `monthly` window) in one database transaction. All standard rows from that response MUST use the same capture timestamp. If any standard row cannot be persisted or the transaction cannot commit, the system MUST roll back the transaction so none of that response's standard rows becomes visible, and a caller-owned database session MUST remain open and reusable. This atomic unit applies to standard `usage_history` rows; additional per-model usage history and independent live-ingest writes retain their existing persistence contracts. + +#### Scenario: Multi-window response commits as one snapshot + +- **WHEN** a successful account usage response contains multiple normalized standard windows +- **THEN** the system persists all of those standard rows in one transaction with one shared capture timestamp + +#### Scenario: Later row failure leaves no partial snapshot + +- **WHEN** persistence fails after at least one standard row from an account response has been staged +- **THEN** the system rolls back the transaction and no standard row from that response is visible + +#### Scenario: Caller retains its session after rollback + +- **WHEN** a caller-owned session is used for a standard usage snapshot and the snapshot transaction fails +- **THEN** the repository leaves that session open and reusable after rolling back the failed transaction + +### Requirement: Owner-forwarded compact settlement failures fail closed + +An HTTP-bridge owner MUST treat any persistence exception while finalizing or +releasing a forwarded compact API-key usage reservation as a failed settlement, +and MUST NOT swallow it. +The owner MUST log the persistence failure, MUST attempt to release the +reservation through a fresh repository context, and MUST surface a `502` +`usage_settlement_failed` server error regardless of whether that fail-safe +release succeeds. The settlement failure MUST carry trusted internal provenance +that is checked before compact upstream retry, failover, and account-health error +handling, so the compact request is not sent upstream again and the selected +account is not penalized for a local persistence failure. When the reservation +is still `reserved` when the fail-safe release begins and that release succeeds, +the reservation's final status MUST be `released`. This behavior MUST NOT add or +alter stale-reservation cleanup or WebSocket health handling. + +#### Scenario: Forwarded compact finalization fails after upstream success + +- **GIVEN** a signed owner-forwarded compact request whose API-key reservation is `reserved` +- **AND** the upstream compact succeeds but usage finalization raises a persistence exception +- **WHEN** the owner handles the settlement failure +- **THEN** the owner attempts a fail-safe reservation release through a fresh repository context +- **AND** the request returns `502` with error code `usage_settlement_failed` +- **AND** the upstream compact is called exactly once and no account-health error is recorded +- **AND** when the reservation is still `reserved` at fail-safe release and that release succeeds, its status is `released` + +### Requirement: Reset-confirmed warm-up follows the plan-applicable long window + +When reset-confirmed limit warm-up evaluates an account's selected long quota +window, the system MUST use the monthly usage row when that account's plan has +monthly quota capacity and MUST otherwise use the secondary usage row. The +persisted warm-up attempt MUST retain the canonical window name from the +selected usage row. + +#### Scenario: Free monthly reset triggers one monthly warm-up + +- **GIVEN** limit warm-up is enabled globally and for a free-plan account +- **AND** long-window warm-up is selected +- **AND** the account's previous monthly usage sample was exhausted +- **WHEN** background usage refresh records a newer monthly sample with + available quota and a later `reset_at` +- **THEN** the system sends at most one warm-up request for that + account/monthly/reset tuple +- **AND** the durable warm-up attempt records `window="monthly"` + +#### Scenario: Paid plans retain secondary long-window warm-up + +- **GIVEN** an account plan has no monthly quota capacity +- **AND** primary and secondary usage samples are available +- **WHEN** background usage refresh evaluates long-window warm-up +- **THEN** the system uses the secondary usage row +- **AND** it does not substitute an unrelated monthly row + +#### Scenario: First monthly sample is not treated as a reset + +- **GIVEN** a free-plan account has no previous monthly usage sample +- **AND** its latest secondary sample is exhausted +- **WHEN** background usage refresh records the account's first monthly sample +- **THEN** the system does not compare the secondary and monthly `reset_at` + values as one window +- **AND** it does not send a reset-confirmed warm-up for that transition + +#### Scenario: Scheduler scopes monthly snapshots to the selected account + +- **GIVEN** multiple accounts are eligible for background usage refresh +- **AND** one account is selected for the current scheduler slice +- **WHEN** the scheduler loads before and after usage for warm-up evaluation +- **THEN** monthly lookups are filtered to the selected account +- **AND** monthly usage from another account cannot create a warm-up attempt + +### Requirement: Auth Guardian candidate handoff survives session closure + +Auth Guardian MUST preserve stable account identities while its candidate-query session is active and MUST execute selected refresh work after that session closes without reading unloaded or expired state from detached persistence objects. Each selected account MUST still be re-read in the separately owned refresh session before eligibility is confirmed and credentials are refreshed. + +#### Scenario: Stale candidate crosses the query-session boundary + +- **GIVEN** a stale eligible account is selected during an Auth Guardian pass +- **WHEN** the candidate-query session closes before per-account refresh work begins +- **THEN** Auth Guardian refreshes the selected account without a detached-instance failure +- **AND** the refresh worker re-reads the account in its own session before refreshing it + diff --git a/openspec/specs/user-documentation/spec.md b/openspec/specs/user-documentation/spec.md index b83a6f1645..237348d06e 100644 --- a/openspec/specs/user-documentation/spec.md +++ b/openspec/specs/user-documentation/spec.md @@ -59,3 +59,38 @@ OpenSpec remains the normative source of truth. Every docs page that documents s - **THEN** it contains the commented line `# CODEX_LB_LEADER_ELECTION_ENABLED=false` - **AND** no active (uncommented) assignment disables leader election +### Requirement: Generated settings reference stays in sync with the code + +The documentation site SHALL include a settings reference page +(`docs/reference/settings.md`) generated from `Settings.model_fields` by +`scripts/generate_settings_reference.py`. The page SHALL list, for every +setting, the `CODEX_LB_`-prefixed environment variable name, its type, and +its default (environment-derived defaults rendered symbolically), grouped by +functional area; it SHALL document the bare `PORT` special case and SHALL +list the removed (`_REMOVED_SETTINGS`) and deprecated env names sourced from +the code. The generated page SHALL be checked into the repository so the +strict docs build stays hermetic, SHALL carry a header identifying it as +generated, and SHALL link the owning OpenSpec capability. CI unit tests MUST +fail when the checked-in page differs from regenerated output, when the +settings surface exceeds its ratchet (115 fields; lower-only without a +simplicity-budget decision), or when an uncommented `.env.example` assignment +differs from the code default. + +#### Scenario: Settings change without regeneration fails CI + +- **GIVEN** a change to `Settings` fields in `app/core/config/settings.py` +- **WHEN** the unit test suite runs without regenerating `docs/reference/settings.md` +- **THEN** the regenerate-and-diff test fails until the page is regenerated and committed + +#### Scenario: Reference page is reachable and generated + +- **WHEN** a reader opens the published settings reference page +- **THEN** it is in the site navigation and linked from the Configuration page +- **AND** it identifies itself as generated from `scripts/generate_settings_reference.py` +- **AND** it links the owning OpenSpec capability + +#### Scenario: Settings surface growth trips the ratchet + +- **WHEN** the number of `Settings` fields exceeds the ratchet value +- **THEN** the ratchet unit test fails, forcing a simplicity-budget discussion before the surface grows + From e439043d4ad2869ca992d3d9bbb632b814406013 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Fri, 14 Aug 2026 09:49:53 +0200 Subject: [PATCH 007/117] fix(dashboard): distinguish first-run empty states from filter mismatch (#1729) * fix(dashboard): distinguish first-run empty states from filter mismatch Empty Accounts, APIs, and request logs told first-run operators to adjust filters. Reports drew a zero line when there was no daily data. /firewall landed on Settings with Advanced collapsed. Use first-run copy when the source list is empty, keep filter-empty copy when something is hidden, add a dashboard CTA to /accounts, show no-data on empty Reports line charts, and expand Advanced for /firewall. * fix(dashboard): satisfy eslint on Advanced settings deeplink Move the URL helper out of the component module and remount the collapsed group from the route instead of setting open state in an effect. * fix(dashboard): keep nonempty request-log totals out of first-run empty copy A deep-linked later page can have requests=[] while total>0. That is not a first-run fleet; treat a positive total as existing logs when choosing empty copy. * test(dashboard): align API first-run integration expectation * fix(dashboard): stabilize empty-state transitions and firewall scroll * fix(dashboard): preserve firewall deeplink through async layout * fix(dashboard): settle firewall deeplink scroll * fix(dashboard): stabilize async navigation transitions --- frontend/src/App.tsx | 2 +- .../__integration__/apis-page-flow.test.tsx | 2 +- .../__integration__/firewall-flow.test.tsx | 6 +- frontend/src/components/empty-state.tsx | 5 +- .../accounts/components/account-list.test.tsx | 16 +++ .../accounts/components/account-list.tsx | 8 +- .../apis/components/api-list.test.tsx | 36 +++++ .../src/features/apis/components/api-list.tsx | 8 +- .../components/account-cards.test.tsx | 11 ++ .../dashboard/components/account-cards.tsx | 7 + .../components/account-list.test.tsx | 11 ++ .../dashboard/components/account-list.tsx | 6 + .../components/dashboard-page.test.tsx | 10 +- .../dashboard/components/dashboard-page.tsx | 3 +- .../components/recent-requests-table.test.tsx | 35 ++++- .../components/recent-requests-table.tsx | 15 +- .../dashboard/hooks/use-request-logs.test.ts | 130 +++++++++++++++++- .../dashboard/hooks/use-request-logs.ts | 26 +++- .../firewall/components/firewall-section.tsx | 2 +- .../components/cost-per-day-chart.test.tsx | 11 +- .../reports/components/cost-per-day-chart.tsx | 8 +- .../reports/components/queue-wait-chart.tsx | 8 +- .../reports/components/report-chart-card.tsx | 32 +++++ .../components/time-to-first-token-chart.tsx | 8 +- .../components/tokens-per-day-chart.tsx | 8 +- .../components/tokens-per-second-chart.tsx | 8 +- .../settings/advanced-settings-deeplink.ts | 7 + .../advanced-settings-group.test.tsx | 88 ++++++++++++ .../components/advanced-settings-group.tsx | 55 +++++++- .../components/settings-page.test.tsx | 32 ++++- .../settings/components/settings-page.tsx | 18 ++- frontend/src/i18n/locales/en.json | 12 +- frontend/src/i18n/locales/ko.json | 12 +- frontend/src/i18n/locales/zh-CN.json | 12 +- .../.openspec.yaml | 2 + .../context.md | 36 +++++ .../design.md | 59 ++++++++ .../proposal.md | 35 +++++ .../specs/frontend-architecture/spec.md | 108 +++++++++++++++ .../dashboard-first-run-empty-states/tasks.md | 19 +++ .../specs/frontend-architecture/context.md | 7 +- 41 files changed, 861 insertions(+), 63 deletions(-) create mode 100644 frontend/src/features/apis/components/api-list.test.tsx create mode 100644 frontend/src/features/reports/components/report-chart-card.tsx create mode 100644 frontend/src/features/settings/advanced-settings-deeplink.ts create mode 100644 frontend/src/features/settings/components/advanced-settings-group.test.tsx create mode 100644 openspec/changes/dashboard-first-run-empty-states/.openspec.yaml create mode 100644 openspec/changes/dashboard-first-run-empty-states/context.md create mode 100644 openspec/changes/dashboard-first-run-empty-states/design.md create mode 100644 openspec/changes/dashboard-first-run-empty-states/proposal.md create mode 100644 openspec/changes/dashboard-first-run-empty-states/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/dashboard-first-run-empty-states/tasks.md diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index c6accac0c6..fbc03db24b 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -81,7 +81,7 @@ export default function App() { } /> } /> } /> - } /> + } /> diff --git a/frontend/src/__integration__/apis-page-flow.test.tsx b/frontend/src/__integration__/apis-page-flow.test.tsx index 40ea6f0335..6152004180 100644 --- a/frontend/src/__integration__/apis-page-flow.test.tsx +++ b/frontend/src/__integration__/apis-page-flow.test.tsx @@ -119,7 +119,7 @@ describe("apis page integration", () => { renderWithProviders(); expect(await screen.findByRole("heading", { name: "APIs" })).toBeInTheDocument(); - expect(await screen.findByText("No matching API keys")).toBeInTheDocument(); + expect(await screen.findByText("No API keys yet")).toBeInTheDocument(); expect(screen.getByText("Select an API key")).toBeInTheDocument(); }); diff --git a/frontend/src/__integration__/firewall-flow.test.tsx b/frontend/src/__integration__/firewall-flow.test.tsx index 999226688e..6aa5dd73ad 100644 --- a/frontend/src/__integration__/firewall-flow.test.tsx +++ b/frontend/src/__integration__/firewall-flow.test.tsx @@ -55,6 +55,7 @@ describe("firewall flow integration", () => { // Scope queries to the firewall section const firewallSection = firewallHeading.closest("section")!; + expect(firewallSection).toHaveClass("scroll-mt-16"); const fw = within(firewallSection); await user.type(fw.getByPlaceholderText("127.0.0.1 or 2001:db8::1"), "127.0.0.1"); @@ -78,6 +79,9 @@ describe("firewall flow integration", () => { expect(await screen.findByRole("heading", { name: "Settings" })).toBeInTheDocument(); expect(window.location.pathname).toBe("/settings"); - expect(await screen.findByRole("button", { name: "Show advanced settings" })).toBeInTheDocument(); + expect(window.location.search).toBe("?advanced=1"); + expect(window.location.hash).toBe("#firewall"); + expect(await screen.findByRole("heading", { name: "Firewall" })).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "Hide advanced settings" })).toBeInTheDocument(); }); }); diff --git a/frontend/src/components/empty-state.tsx b/frontend/src/components/empty-state.tsx index 332c61ebfd..2fbf97f5ca 100644 --- a/frontend/src/components/empty-state.tsx +++ b/frontend/src/components/empty-state.tsx @@ -1,12 +1,14 @@ import type { LucideIcon } from "lucide-react"; +import type { ReactNode } from "react"; export type EmptyStateProps = { icon: LucideIcon; title: string; description?: string; + action?: ReactNode; }; -export function EmptyState({ icon: Icon, title, description }: EmptyStateProps) { +export function EmptyState({ icon: Icon, title, description, action }: EmptyStateProps) { return (
@@ -16,6 +18,7 @@ export function EmptyState({ icon: Icon, title, description }: EmptyStateProps)

{title}

{description ?

{description}

: null}
+ {action ?
{action}
: null}
); } diff --git a/frontend/src/features/accounts/components/account-list.test.tsx b/frontend/src/features/accounts/components/account-list.test.tsx index d6c9483d86..1a4bf2507a 100644 --- a/frontend/src/features/accounts/components/account-list.test.tsx +++ b/frontend/src/features/accounts/components/account-list.test.tsx @@ -432,6 +432,22 @@ describe("AccountList", () => { expect(screen.getByText("No matching accounts")).toBeInTheDocument(); }); + it("shows first-run empty copy when no accounts exist", () => { + render( + {}} + onOpenImport={() => {}} + onOpenOauth={() => {}} + />, + ); + + expect(screen.getByText("No accounts yet")).toBeInTheDocument(); + expect(screen.getByText("Add an account to start routing.")).toBeInTheDocument(); + expect(screen.queryByText("Adjust filters")).not.toBeInTheDocument(); + }); + it("keeps the add account action outside the scrollable account list", () => { render( {filtered.length === 0 ? (
-

{t("accounts.list.noMatches")}

-

{t("accounts.list.adjustFilters")}

+

+ {accounts.length === 0 ? t("accounts.list.emptyTitle") : t("accounts.list.noMatches")} +

+

+ {accounts.length === 0 ? t("accounts.list.emptyDescription") : t("accounts.list.adjustFilters")} +

) : ( filtered.map((account) => ( diff --git a/frontend/src/features/apis/components/api-list.test.tsx b/frontend/src/features/apis/components/api-list.test.tsx new file mode 100644 index 0000000000..b95ca36da5 --- /dev/null +++ b/frontend/src/features/apis/components/api-list.test.tsx @@ -0,0 +1,36 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { describe, expect, it } from "vitest"; + +import { ApiList } from "@/features/apis/components/api-list"; +import { createApiKey } from "@/test/mocks/factories"; + +describe("ApiList", () => { + it("shows first-run empty copy when no API keys exist", () => { + render( + {}} onOpenCreate={() => {}} />, + ); + + expect(screen.getByText("No API keys yet")).toBeInTheDocument(); + expect(screen.getByText("Create an API key to authenticate clients.")).toBeInTheDocument(); + expect(screen.queryByText("Adjust filters")).not.toBeInTheDocument(); + }); + + it("shows filter-empty copy when keys exist but none match", async () => { + const user = userEvent.setup(); + + render( + {}} + onOpenCreate={() => {}} + />, + ); + + await user.type(screen.getByPlaceholderText("Search API keys..."), "not-found"); + + expect(screen.getByText("No matching API keys")).toBeInTheDocument(); + expect(screen.getByText("Adjust filters")).toBeInTheDocument(); + }); +}); diff --git a/frontend/src/features/apis/components/api-list.tsx b/frontend/src/features/apis/components/api-list.tsx index 3731bc2437..3a293f399e 100644 --- a/frontend/src/features/apis/components/api-list.tsx +++ b/frontend/src/features/apis/components/api-list.tsx @@ -93,8 +93,12 @@ export function ApiList({ apiKeys, selectedKeyId, onSelect, onOpenCreate }: ApiL
{filtered.length === 0 ? (
-

{t("apis.list.noMatches")}

-

{t("accounts.list.adjustFilters")}

+

+ {apiKeys.length === 0 ? t("apis.list.emptyTitle") : t("apis.list.noMatches")} +

+

+ {apiKeys.length === 0 ? t("apis.list.emptyDescription") : t("accounts.list.adjustFilters")} +

) : ( filtered.map((apiKey) => ( diff --git a/frontend/src/features/dashboard/components/account-cards.test.tsx b/frontend/src/features/dashboard/components/account-cards.test.tsx index 50be15a461..d95e0d685b 100644 --- a/frontend/src/features/dashboard/components/account-cards.test.tsx +++ b/frontend/src/features/dashboard/components/account-cards.test.tsx @@ -1,4 +1,5 @@ import { render, screen } from "@testing-library/react"; +import { MemoryRouter } from "react-router-dom"; import { describe, expect, it, vi } from "vitest"; import { AccountCards } from "@/features/dashboard/components/account-cards"; @@ -87,4 +88,14 @@ describe("AccountCards", () => { expect(screen.queryByText((_content, el) => el?.tagName === "P" && !!el.textContent?.match(/dup@example\.com .* ID d48f0bfc\.\.\.12b5d5/))).not.toBeInTheDocument(); expect(screen.getByText((_content, el) => el?.tagName === "P" && !!el.textContent?.match(/dup@example\.com .* ID 7f9de2ad\.\.\.a95cee/))).toBeInTheDocument(); }); + + it("links the empty-account state to the Accounts page", () => { + render( + + + , + ); + + expect(screen.getByRole("link", { name: "Add accounts" })).toHaveAttribute("href", "/accounts"); + }); }); diff --git a/frontend/src/features/dashboard/components/account-cards.tsx b/frontend/src/features/dashboard/components/account-cards.tsx index 802f7fac32..a645a28e5a 100644 --- a/frontend/src/features/dashboard/components/account-cards.tsx +++ b/frontend/src/features/dashboard/components/account-cards.tsx @@ -1,7 +1,9 @@ import { Users } from "lucide-react"; import { useTranslation } from "react-i18next"; +import { Link } from "react-router-dom"; import { EmptyState } from "@/components/empty-state"; +import { Button } from "@/components/ui/button"; import { AccountCard, type AccountCardProps } from "@/features/dashboard/components/account-card"; import type { AccountSummary } from "@/features/dashboard/schemas"; @@ -25,6 +27,11 @@ export function AccountCards({ accounts, readOnly = false, onAction }: AccountCa icon={Users} title={t("dashboard.accounts.emptyTitle")} description={t("dashboard.accounts.emptyDescription")} + action={ + + } /> ); } diff --git a/frontend/src/features/dashboard/components/account-list.test.tsx b/frontend/src/features/dashboard/components/account-list.test.tsx index c5dfa2f23f..616d3b377f 100644 --- a/frontend/src/features/dashboard/components/account-list.test.tsx +++ b/frontend/src/features/dashboard/components/account-list.test.tsx @@ -1,5 +1,6 @@ import { act, render, screen, within } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; +import { MemoryRouter } from "react-router-dom"; import { afterEach, describe, expect, it, vi } from "vitest"; import { AccountList } from "@/features/dashboard/components/account-list"; @@ -349,4 +350,14 @@ describe("AccountList", () => { const resetButton = screen.getByRole("button", { name: "Redeem reset credit for Many Reset Account" }); expect(within(resetButton).getByText("99+")).toBeInTheDocument(); }); + + it("links the empty-account state to the Accounts page", () => { + render( + + + , + ); + + expect(screen.getByRole("link", { name: "Add accounts" })).toHaveAttribute("href", "/accounts"); + }); }); diff --git a/frontend/src/features/dashboard/components/account-list.tsx b/frontend/src/features/dashboard/components/account-list.tsx index dbffa14b1f..88a840d67c 100644 --- a/frontend/src/features/dashboard/components/account-list.tsx +++ b/frontend/src/features/dashboard/components/account-list.tsx @@ -1,6 +1,7 @@ import { ArrowDown, ArrowUp, ArrowUpDown, Clock, ExternalLink, List, Play, RotateCcw, Zap } from "lucide-react"; import { useMemo, useState } from "react"; import { useTranslation } from "react-i18next"; +import { Link } from "react-router-dom"; import { EmptyState } from "@/components/empty-state"; import { StatusBadge } from "@/components/status-badge"; @@ -304,6 +305,11 @@ export function AccountList({ icon={List} title={t("dashboard.accountList.emptyTitle")} description={t("dashboard.accountList.emptyDescription")} + action={ + + } /> ); } diff --git a/frontend/src/features/dashboard/components/dashboard-page.test.tsx b/frontend/src/features/dashboard/components/dashboard-page.test.tsx index 97e50f1ea3..7890243d39 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.test.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.test.tsx @@ -31,9 +31,13 @@ vi.mock("@/features/dashboard/hooks/use-dashboard", () => ({ useDashboardProjections: vi.fn(), })); -vi.mock("@/features/dashboard/hooks/use-request-logs", () => ({ - useRequestLogs: vi.fn(), -})); +vi.mock("@/features/dashboard/hooks/use-request-logs", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + useRequestLogs: vi.fn(), + }; +}); vi.mock("@/features/dashboard/hooks/use-conversations", () => ({ useConversations: vi.fn(), diff --git a/frontend/src/features/dashboard/components/dashboard-page.tsx b/frontend/src/features/dashboard/components/dashboard-page.tsx index 54160071a0..6f28be006f 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.tsx @@ -93,7 +93,7 @@ export function DashboardPage() { enabled: isAdmin && dashboardView === "conversations", }); const { conversationsQuery } = conversationsState; - const { filters, logsQuery, optionsQuery, updateFilters } = useRequestLogs({ + const { filters, emptyStateFiltersApplied, logsQuery, optionsQuery, updateFilters } = useRequestLogs({ enabled: dashboardView === "request-logs", }); const { resumeMutation, limitWarmupMutation } = useAccountMutations(); @@ -505,6 +505,7 @@ export function DashboardPage() { limit={filters.limit} offset={filters.offset} hasMore={logPage?.hasMore ?? false} + filtersApplied={emptyStateFiltersApplied} onLimitChange={(limit) => updateFilters({ limit, offset: 0 })} onOffsetChange={(offset) => updateFilters({ offset })} onConversationClick={handleConversationClick} diff --git a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx index f9f29d8d81..911a38d482 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx @@ -281,8 +281,41 @@ describe("RecentRequestsTable", () => { expect(within(row as HTMLElement).queryByText("250.0")).not.toBeInTheDocument(); }); - it("renders empty state", () => { + it("renders first-run empty copy when no filters are applied", () => { render(); + expect(screen.getByText("No requests yet")).toBeInTheDocument(); + expect( + screen.getByText("Requests will appear here after clients start using the proxy."), + ).toBeInTheDocument(); + expect(screen.queryByText("No request logs match the current filters.")).not.toBeInTheDocument(); + }); + + it("renders filter-empty copy when a later page has no rows but logs exist", () => { + render( + , + ); + expect(screen.getByText("No matching requests")).toBeInTheDocument(); + expect(screen.getByText("No request logs match the current filters.")).toBeInTheDocument(); + expect(screen.queryByText("No requests yet")).not.toBeInTheDocument(); + }); + + it("renders filter-empty copy when filters are applied", () => { + render( + , + ); + expect(screen.getByText("No matching requests")).toBeInTheDocument(); expect(screen.getByText("No request logs match the current filters.")).toBeInTheDocument(); }); diff --git a/frontend/src/features/dashboard/components/recent-requests-table.tsx b/frontend/src/features/dashboard/components/recent-requests-table.tsx index 53be393677..a161d673e5 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.tsx @@ -82,6 +82,7 @@ export type RecentRequestsTableProps = { limit: number; offset: number; hasMore: boolean; + filtersApplied?: boolean; onLimitChange: (limit: number) => void; onOffsetChange: (offset: number) => void; onConversationClick?: (conversationId: string) => void; @@ -170,6 +171,7 @@ export function RecentRequestsTable({ limit, offset, hasMore, + filtersApplied = false, onLimitChange, onOffsetChange, onConversationClick, @@ -202,11 +204,20 @@ export function RecentRequestsTable({ }, [accounts]); if (requests.length === 0) { + const emptyFromExistingLogs = filtersApplied || total > 0; return ( ); } diff --git a/frontend/src/features/dashboard/hooks/use-request-logs.test.ts b/frontend/src/features/dashboard/hooks/use-request-logs.test.ts index 0eea93bcb0..fcae731258 100644 --- a/frontend/src/features/dashboard/hooks/use-request-logs.test.ts +++ b/frontend/src/features/dashboard/hooks/use-request-logs.test.ts @@ -2,10 +2,15 @@ import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; import { act, renderHook, waitFor } from "@testing-library/react"; import { HttpResponse, http } from "msw"; import { createElement, type PropsWithChildren, useEffect } from "react"; -import { MemoryRouter, useLocation } from "react-router-dom"; +import { + MemoryRouter, + useLocation, + useNavigate, + type NavigateFunction, +} from "react-router-dom"; import { describe, expect, it } from "vitest"; -import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; +import { requestLogFiltersApplied, useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; import { server } from "@/test/mocks/server"; function createTestQueryClient(): QueryClient { @@ -19,13 +24,24 @@ function createTestQueryClient(): QueryClient { }); } -function LocationSpy({ onChange }: { onChange?: (search: string) => void }) { +function LocationSpy({ + onChange, + onNavigateReady, +}: { + onChange?: (search: string) => void; + onNavigateReady?: (navigate: NavigateFunction) => void; +}) { const routeLocation = useLocation(); + const navigate = useNavigate(); useEffect(() => { onChange?.(routeLocation.search); }, [routeLocation.search, onChange]); + useEffect(() => { + onNavigateReady?.(navigate); + }, [navigate, onNavigateReady]); + return null; } @@ -33,6 +49,7 @@ function createWrapper( queryClient: QueryClient, initialEntry = "/dashboard", onLocationChange?: (search: string) => void, + onNavigateReady?: (navigate: NavigateFunction) => void, ) { return function Wrapper({ children }: PropsWithChildren) { return createElement( @@ -41,7 +58,7 @@ function createWrapper( createElement( MemoryRouter, { initialEntries: [initialEntry] }, - createElement(LocationSpy, { onChange: onLocationChange }), + createElement(LocationSpy, { onChange: onLocationChange, onNavigateReady }), children, ), ); @@ -105,6 +122,87 @@ describe("useRequestLogs", () => { expect(locationSearch).toContain("search=quota"); }); + it("keeps filtered-empty semantics while clearing a filter", async () => { + let releaseUnfiltered: (() => void) | undefined; + const unfilteredResponse = new Promise((resolve) => { + releaseUnfiltered = resolve; + }); + server.use( + http.get("/api/request-logs", async ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get("search")) { + return HttpResponse.json({ requests: [], total: 0, hasMore: false }); + } + await unfilteredResponse; + return HttpResponse.json({ requests: [], total: 1, hasMore: false }); + }), + ); + + const queryClient = createTestQueryClient(); + const wrapper = createWrapper(queryClient, "/dashboard?search=missing"); + const { result } = renderHook(() => useRequestLogs(), { wrapper }); + + await waitFor(() => expect(result.current.logsQuery.isSuccess).toBe(true)); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + act(() => { + result.current.updateFilters({ search: "", offset: 0 }); + }); + + await waitFor(() => expect(result.current.filters.search).toBe("")); + expect(result.current.logsQuery.isPlaceholderData).toBe(true); + expect(result.current.logsQuery.data?.total).toBe(0); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + releaseUnfiltered?.(); + await waitFor(() => expect(result.current.logsQuery.data?.total).toBe(1)); + expect(result.current.emptyStateFiltersApplied).toBe(false); + }); + + it("keeps filtered-empty semantics when route navigation clears filters", async () => { + let releaseUnfiltered: (() => void) | undefined; + let navigate: NavigateFunction | undefined; + const unfilteredResponse = new Promise((resolve) => { + releaseUnfiltered = resolve; + }); + server.use( + http.get("/api/request-logs", async ({ request }) => { + const url = new URL(request.url); + if (url.searchParams.get("search")) { + return HttpResponse.json({ requests: [], total: 0, hasMore: false }); + } + await unfilteredResponse; + return HttpResponse.json({ requests: [], total: 1, hasMore: false }); + }), + ); + + const queryClient = createTestQueryClient(); + const wrapper = createWrapper( + queryClient, + "/dashboard?search=missing", + undefined, + (routerNavigate) => { + navigate = routerNavigate; + }, + ); + const { result } = renderHook(() => useRequestLogs(), { wrapper }); + + await waitFor(() => expect(result.current.logsQuery.isSuccess).toBe(true)); + await waitFor(() => expect(navigate).toBeDefined()); + + act(() => { + navigate?.("/dashboard"); + }); + + await waitFor(() => expect(result.current.filters.search).toBe("")); + expect(result.current.logsQuery.isPlaceholderData).toBe(true); + expect(result.current.emptyStateFiltersApplied).toBe(true); + + releaseUnfiltered?.(); + await waitFor(() => expect(result.current.logsQuery.data?.total).toBe(1)); + expect(result.current.emptyStateFiltersApplied).toBe(false); + }); + it("supports pagination updates with total/hasMore response", async () => { const queryClient = createTestQueryClient(); const wrapper = createWrapper(queryClient, "/dashboard?limit=1&offset=0"); @@ -433,3 +531,27 @@ describe("useRequestLogs", () => { expect(apiParams.some((p) => p === rawId)).toBe(true); }); }); + +describe("requestLogFiltersApplied", () => { + const defaults = { + search: "", + timeframe: "all" as const, + accountIds: [], + apiKeyIds: [], + modelOptions: [], + statuses: [], + conversationId: null, + limit: 25, + offset: 0, + }; + + it("is false for default request-log filters", () => { + expect(requestLogFiltersApplied(defaults)).toBe(false); + }); + + it("is true when a narrowing filter is set", () => { + expect(requestLogFiltersApplied({ ...defaults, timeframe: "24h" })).toBe(true); + expect(requestLogFiltersApplied({ ...defaults, search: "rate" })).toBe(true); + expect(requestLogFiltersApplied({ ...defaults, conversationId: "conv-1" })).toBe(true); + }); +}); diff --git a/frontend/src/features/dashboard/hooks/use-request-logs.ts b/frontend/src/features/dashboard/hooks/use-request-logs.ts index 0b54552aec..c479fae6a8 100644 --- a/frontend/src/features/dashboard/hooks/use-request-logs.ts +++ b/frontend/src/features/dashboard/hooks/use-request-logs.ts @@ -22,6 +22,18 @@ const DEFAULT_FILTER_STATE: FilterState = { offset: 0, }; +export function requestLogFiltersApplied(filters: FilterState): boolean { + return ( + filters.search.trim() !== "" || + filters.timeframe !== DEFAULT_FILTER_STATE.timeframe || + filters.accountIds.length > 0 || + filters.apiKeyIds.length > 0 || + filters.modelOptions.length > 0 || + filters.statuses.length > 0 || + Boolean(filters.conversationId) + ); +} + const REQUEST_LOG_PARAM_KEYS = [ "search", "timeframe", @@ -114,6 +126,7 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { const [searchParams, setSearchParams] = useSearchParams(); const filters = useMemo(() => parseFilterState(searchParams), [searchParams]); + const filtersApplied = requestLogFiltersApplied(filters); const since = useMemo(() => timeframeToSinceIso(filters.timeframe), [filters.timeframe]); const listFilters = useMemo( () => ({ @@ -140,28 +153,36 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { ); const { - data: logsData, + data: logsResult, error: logsError, isFetching: logsIsFetching, isLoading: logsIsLoading, isPending: logsIsPending, + isPlaceholderData: logsIsPlaceholderData, isSuccess: logsIsSuccess, refetch: refetchLogs, } = useQuery({ queryKey: ["dashboard", "request-logs", listFilters], - queryFn: () => getRequestLogs(listFilters), + queryFn: async () => ({ + page: await getRequestLogs(listFilters), + filtersApplied, + }), enabled, refetchInterval: 30_000, refetchIntervalInBackground: false, refetchOnWindowFocus: true, placeholderData: keepPreviousData, }); + const logsData = logsResult?.page; + const emptyStateFiltersApplied = + filtersApplied || (logsIsPlaceholderData && Boolean(logsResult?.filtersApplied)); const logsQuery = { data: logsData, error: logsError, isFetching: logsIsFetching, isLoading: logsIsLoading, isPending: logsIsPending, + isPlaceholderData: logsIsPlaceholderData, isSuccess: logsIsSuccess, refetch: refetchLogs, }; @@ -204,6 +225,7 @@ export function useRequestLogs(options: UseRequestLogsOptions = {}) { filters, listFilters, facetFilters, + emptyStateFiltersApplied, logsQuery, optionsQuery, updateFilters, diff --git a/frontend/src/features/firewall/components/firewall-section.tsx b/frontend/src/features/firewall/components/firewall-section.tsx index 0ac64b5cf2..d47ddb7fd0 100644 --- a/frontend/src/features/firewall/components/firewall-section.tsx +++ b/frontend/src/features/firewall/components/firewall-section.tsx @@ -60,7 +60,7 @@ export function FirewallSection({ disabled = false }: FirewallSectionProps) { }; return ( -
+
{ if (!open) setSelectedRequest(null); }}> - + {t("dashboard.requestDetails.title")} {t("dashboard.requestDetails.description")} -
+
+ {selectedRequest?.upstreamProxyRouteMode || + selectedRequest?.upstreamProxyPoolId || + selectedRequest?.upstreamProxyEndpointId || + selectedRequest?.upstreamProxyFallbackUsed != null || + selectedRequest?.upstreamProxyFailClosedReason ? ( +
+ {selectedRequest.upstreamProxyRouteMode ? ( + + ) : null} + {selectedRequest.upstreamProxyPoolId ? ( + + ) : null} + {selectedRequest.upstreamProxyEndpointId ? ( + + ) : null} + {selectedRequest.upstreamProxyFallbackUsed != null ? ( + + ) : null} + {selectedRequest.upstreamProxyFailClosedReason ? ( + + ) : null} +
+ ) : null} {isAdmin ? ( { connectionRequestKind: "prewarm", model: "gpt-5.1", transport: "websocket", + upstreamProxyRouteMode: "account_bound", + upstreamProxyPoolId: "pool-1", + upstreamProxyEndpointId: "endpoint-1", + upstreamProxyFallbackUsed: true, + upstreamProxyFailClosedReason: null, useragent: "Mozilla/5.0", useragentGroup: "Mozilla", clientIp: "203.0.113.7", @@ -239,6 +244,11 @@ describe("RequestLogsResponseSchema", () => { expect(parsed.requests[0]?.connectionRequestKind).toBe("prewarm"); expect(parsed.requests[0]?.planType).toBe("plus"); expect(parsed.requests[0]?.transport).toBe("websocket"); + expect(parsed.requests[0]?.upstreamProxyRouteMode).toBe("account_bound"); + expect(parsed.requests[0]?.upstreamProxyPoolId).toBe("pool-1"); + expect(parsed.requests[0]?.upstreamProxyEndpointId).toBe("endpoint-1"); + expect(parsed.requests[0]?.upstreamProxyFallbackUsed).toBe(true); + expect(parsed.requests[0]?.upstreamProxyFailClosedReason).toBeNull(); expect(parsed.requests[0]?.useragent).toBe("Mozilla/5.0"); expect(parsed.requests[0]?.useragentGroup).toBe("Mozilla"); expect(parsed.requests[0]?.clientIp).toBe("203.0.113.7"); diff --git a/frontend/src/features/dashboard/schemas.ts b/frontend/src/features/dashboard/schemas.ts index be44af3e93..7530cd4fae 100644 --- a/frontend/src/features/dashboard/schemas.ts +++ b/frontend/src/features/dashboard/schemas.ts @@ -184,6 +184,11 @@ export const RequestLogSchema = z.object({ modelSourceKind: z.string().nullable().optional(), transport: z.string().nullable().optional().default(null), upstreamTransport: z.string().nullable().optional(), + upstreamProxyRouteMode: z.string().nullable().optional(), + upstreamProxyPoolId: z.string().nullable().optional(), + upstreamProxyEndpointId: z.string().nullable().optional(), + upstreamProxyFallbackUsed: z.boolean().nullable().optional(), + upstreamProxyFailClosedReason: z.string().nullable().optional(), useragent: z.string().nullable().optional().default(null), useragentGroup: z.string().nullable().optional().default(null), clientIp: z.string().nullable().optional().default(null), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index e30e06de9d..7d3c983df5 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -683,6 +683,13 @@ "dashboard.requestDetails.queue": "Queue", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "Request kind", + "dashboard.requestDetails.routeEndpoint": "Proxy endpoint", + "dashboard.requestDetails.routeFailClosedReason": "Fail-closed reason", + "dashboard.requestDetails.routeFallback": "Same-pool fallback", + "dashboard.requestDetails.routeFallbackNotUsed": "Not used", + "dashboard.requestDetails.routeFallbackUsed": "Used", + "dashboard.requestDetails.routeMode": "Route mode", + "dashboard.requestDetails.routePool": "Proxy pool", "dashboard.requestDetails.title": "Request Details", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "Account", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 5465090067..04c91ebe4b 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -683,6 +683,13 @@ "dashboard.requestDetails.queue": "Queue", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "요청 종류", + "dashboard.requestDetails.routeEndpoint": "프록시 엔드포인트", + "dashboard.requestDetails.routeFailClosedReason": "실패 종료 사유", + "dashboard.requestDetails.routeFallback": "동일 풀 폴백", + "dashboard.requestDetails.routeFallbackNotUsed": "사용되지 않음", + "dashboard.requestDetails.routeFallbackUsed": "사용됨", + "dashboard.requestDetails.routeMode": "라우팅 모드", + "dashboard.requestDetails.routePool": "프록시 풀", "dashboard.requestDetails.title": "요청 상세", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "Account", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index fbff0c88a7..2049f4a403 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -683,6 +683,13 @@ "dashboard.requestDetails.queue": "队列", "dashboard.requestDetails.requestId": "请求 ID", "dashboard.requestDetails.requestKind": "请求类型", + "dashboard.requestDetails.routeEndpoint": "代理端点", + "dashboard.requestDetails.routeFailClosedReason": "失败关闭原因", + "dashboard.requestDetails.routeFallback": "同池回退", + "dashboard.requestDetails.routeFallbackNotUsed": "未使用", + "dashboard.requestDetails.routeFallbackUsed": "已使用", + "dashboard.requestDetails.routeMode": "路由模式", + "dashboard.requestDetails.routePool": "代理池", "dashboard.requestDetails.title": "请求详情", "dashboard.requestDetails.userAgent": "User Agent", "dashboard.requests.columns.account": "账户", diff --git a/openspec/changes/surface-upstream-route-metadata/design.md b/openspec/changes/surface-upstream-route-metadata/design.md new file mode 100644 index 0000000000..176af320a5 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/design.md @@ -0,0 +1,54 @@ +## Context + +The proxy writer and database model already preserve five credential-safe +route diagnostics. The request-log read model stops at a narrower Pydantic +schema, and the frontend has a second Zod boundary before the existing details +dialog. Operators need the same persisted values across both boundaries. + +## Goals / Non-Goals + +**Goals:** + +- Carry the five existing route fields through the API and frontend unchanged. +- Show populated values in the existing request details dialog. +- Keep the presentation compact and omit absent metadata. + +**Non-Goals:** + +- Change route selection, persistence, or database schema. +- Expose proxy URLs, usernames, passwords, or headers. +- Add table columns, filters, settings, or navigation. + +## Decisions + +- Extend the existing `RequestLogEntry` and `RequestLogSchema` instead of + adding a second endpoint or nested routing object. This matches the flat + persisted model and existing request-log response. +- Render metadata only in the details dialog. Routing diagnostics are useful + during investigation but too sparse for permanent table columns. +- Reuse the details grid's existing label/value rows and localized strings. + Boolean fallback state is rendered as localized yes/no text. + +Alternative considered: expose only the fail-closed reason. That would leave +successful fallback and endpoint selection unobservable, so all five +credential-safe fields move together. + +## Risks / Trade-offs + +- [Risk] Internal identifiers add visual noise → Render only non-null values in + the opt-in details dialog. +- [Risk] Future fields accidentally expose secrets → Explicitly whitelist the + five existing credential-safe columns rather than serializing route objects. +- [Risk] Mixed-version deployments omit fields → Keep every added field + nullable and optional in the frontend parser. + +## Migration Plan + +The API change is additive and reads existing columns, so no migration or +backfill is needed. Older frontends ignore the new keys; newer frontends accept +responses from older backends. Rollback removes the response/UI fields without +changing stored data. + +## Open Questions + +None. diff --git a/openspec/changes/surface-upstream-route-metadata/proposal.md b/openspec/changes/surface-upstream-route-metadata/proposal.md new file mode 100644 index 0000000000..9835eb039e --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/proposal.md @@ -0,0 +1,30 @@ +## Why + +Request logs persist the upstream route mode, pool, endpoint, fallback use, +and fail-closed reason, but the request-log API read model drops every field. +The dashboard therefore cannot explain which configured route served or +blocked a request even though the diagnostic data already exists. + +## What Changes + +- Preserve credential-safe upstream routing metadata through the request-log + API response and frontend parser. +- Present the metadata in the existing request details dialog. +- Add API and frontend contract regressions for the five fields. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `upstream-proxy-routing`: Persisted route metadata is exposed through the + operator request-log surface without proxy credentials. + +## Impact + +Request-log response schemas/mapping, dashboard parsing and request details, +focused API/frontend tests, and localized labels. Database persistence and +proxy routing behavior do not change. diff --git a/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md b/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md new file mode 100644 index 0000000000..3ddb7b7bc8 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/specs/upstream-proxy-routing/spec.md @@ -0,0 +1,25 @@ +## MODIFIED Requirements + +### Requirement: Route metadata must be persisted for migrated upstream calls + +Request logs for migrated upstream calls MUST record route mode, proxy pool +id, proxy endpoint id, same-pool fallback use, and fail-closed reason where +applicable. The request-log API and dashboard request details MUST expose +those credential-safe values to operators and MUST NOT expose proxy +credentials. + +#### Scenario: Fail-closed route is diagnosable from request details + +- **GIVEN** route resolution fails closed before network open +- **AND** the request log records the route mode and fail-closed reason +- **WHEN** an operator opens that request in the dashboard +- **THEN** the request details show the recorded route mode and fail-closed + reason +- **AND** no proxy credentials are included + +#### Scenario: Successful routed request exposes its selected route + +- **GIVEN** a request log records a proxy pool id, proxy endpoint id, and + same-pool fallback use +- **WHEN** the request-log API returns that row +- **THEN** all three values are present unchanged diff --git a/openspec/changes/surface-upstream-route-metadata/tasks.md b/openspec/changes/surface-upstream-route-metadata/tasks.md new file mode 100644 index 0000000000..bd7a8f9994 --- /dev/null +++ b/openspec/changes/surface-upstream-route-metadata/tasks.md @@ -0,0 +1,18 @@ +## 1. API contract + +- [x] 1.1 Add a failing request-log API regression for all five route fields +- [x] 1.2 Add the route fields to `RequestLogEntry` and its mapper + +## 2. Dashboard contract + +- [x] 2.1 Add a failing frontend schema regression for all five route fields +- [x] 2.2 Add the route fields to the request-log Zod schema +- [x] 2.3 Show credential-safe route metadata in request details with + localized labels + +## 3. Validation + +- [x] 3.1 Run focused request-log API and dashboard schema/component tests +- [x] 3.2 Run Python diagnostics, frontend typecheck, and frontend build +- [x] 3.3 Validate OpenSpec +- [x] 3.4 Browser-verify request details with deterministic route metadata diff --git a/tests/integration/test_request_logs_api.py b/tests/integration/test_request_logs_api.py index a949247293..beb6427829 100644 --- a/tests/integration/test_request_logs_api.py +++ b/tests/integration/test_request_logs_api.py @@ -136,6 +136,57 @@ async def test_request_logs_api_returns_recent(async_client, db_setup): assert older["connectionRequestKind"] is None +@pytest.mark.asyncio +async def test_request_logs_api_returns_upstream_proxy_route_metadata(async_client, db_setup): + del db_setup + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + now = utcnow() + await logs_repo.add_log( + account_id=None, + request_id="req_route_success", + model="gpt-5.1", + input_tokens=10, + output_tokens=20, + latency_ms=100, + status="success", + error_code=None, + requested_at=now - timedelta(seconds=1), + upstream_proxy_route_mode="account_bound", + upstream_proxy_pool_id="pool_route", + upstream_proxy_endpoint_id="endpoint_route", + upstream_proxy_fallback_used=True, + ) + await logs_repo.add_log( + account_id=None, + request_id="req_route_fail_closed", + model="gpt-5.1", + input_tokens=None, + output_tokens=None, + latency_ms=0, + status="error", + error_code="upstream_proxy_unavailable", + requested_at=now, + upstream_proxy_route_mode="account_bound", + upstream_proxy_pool_id="pool_route", + upstream_proxy_fail_closed_reason="no_healthy_endpoint", + ) + + response = await async_client.get("/api/request-logs?limit=2") + assert response.status_code == 200 + fail_closed, success = response.json()["requests"] + assert fail_closed["upstreamProxyRouteMode"] == "account_bound" + assert fail_closed["upstreamProxyPoolId"] == "pool_route" + assert fail_closed["upstreamProxyEndpointId"] is None + assert fail_closed["upstreamProxyFallbackUsed"] is None + assert fail_closed["upstreamProxyFailClosedReason"] == "no_healthy_endpoint" + assert success["upstreamProxyRouteMode"] == "account_bound" + assert success["upstreamProxyPoolId"] == "pool_route" + assert success["upstreamProxyEndpointId"] == "endpoint_route" + assert success["upstreamProxyFallbackUsed"] is True + assert success["upstreamProxyFailClosedReason"] is None + + @pytest.mark.asyncio async def test_request_logs_api_returns_model_source_metadata(async_client, db_setup): del db_setup From 8488bc462a46be07ae70f805ff6bf351f0ba4d97 Mon Sep 17 00:00:00 2001 From: DuyBui Date: Sun, 16 Aug 2026 14:43:34 +0700 Subject: [PATCH 028/117] fix(models): correct GPT-5.6 context windows (#1691) * fix(models): correct GPT-5.6 context windows * fix(models): correct GPT-5.6 context windows * docs: add @kidclone3 as a contributor * docs(models): cite corrected GPT-5.6 catalog * fix(models): address GPT-5.6 catalog review * docs(openspec): exclude persisted snapshots from gpt-5.6 bootstrap scenarios --------- Co-authored-by: Darafei Praliaskouski --- .all-contributorsrc | 10 ++++ README.md | 4 ++ app/core/openai/model_registry.py | 10 ++-- docs/client-setup.md | 20 +++---- .../fix-gpt56-context-window/proposal.md | 23 ++++++++ .../specs/model-catalog-compat/spec.md | 56 +++++++++++++++++++ .../changes/fix-gpt56-context-window/tasks.md | 23 ++++++++ tests/integration/test_v1_models.py | 9 +-- tests/unit/test_model_registry.py | 11 ++-- 9 files changed, 142 insertions(+), 24 deletions(-) create mode 100644 openspec/changes/fix-gpt56-context-window/proposal.md create mode 100644 openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md create mode 100644 openspec/changes/fix-gpt56-context-window/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index 1a7e2eb7bb..02b873a9de 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1222,6 +1222,16 @@ "contributions": [ "code" ] + }, + { + "login": "kidclone3", + "name": "DuyBui", + "avatar_url": "https://avatars.githubusercontent.com/u/54184969?v=4", + "profile": "https://github.com/kidclone3", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 5fa37d9099..e7ba0594a1 100644 --- a/README.md +++ b/README.md @@ -282,6 +282,10 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e rknightion
rknightion
💻 ⚠️ glopyglerky
glopyglerky
💻 ⚠️ Ahmad Maulana Iqbal
Ahmad Maulana Iqbal
💻 ⚠️ + Dvredin
Dvredin
💻 + + + DuyBui
DuyBui
💻 ⚠️ diff --git a/app/core/openai/model_registry.py b/app/core/openai/model_registry.py index eaea3c16db..2cb5728731 100644 --- a/app/core/openai/model_registry.py +++ b/app/core/openai/model_registry.py @@ -142,7 +142,7 @@ class ModelRegistryExport: ) # GPT-5.6 ships to four additional plan tiers upstream -# (codex-rs/models-manager/models.json at rust-v0.144.1). +# (codex-rs/models-manager/models.json at rust-v0.145.0). _BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS = frozenset( { *_BOOTSTRAP_AVAILABLE_IN_PLANS, @@ -219,7 +219,7 @@ def _gpt56_raw( availability_nux: dict[str, JsonValue] | None = None, ) -> dict[str, JsonValue]: """Raw catalog fields for the GPT-5.6 family, mirroring the upstream - bundled catalog (codex-rs/models-manager/models.json at rust-v0.144.1) + bundled catalog (codex-rs/models-manager/models.json at rust-v0.145.0) field-for-field. The ~16.5 KB ``base_instructions`` string and the personality-templated ``model_messages`` object are deliberately not bundled; the live upstream registry supplies them on the first refresh. @@ -270,7 +270,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_ULTRA, - context_window=372_000, + context_window=272_000, default_reasoning_level="low", priority=1, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, @@ -283,7 +283,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_ULTRA, - context_window=372_000, + context_window=272_000, default_reasoning_level="medium", priority=2, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, @@ -296,7 +296,7 @@ def _gpt56_raw( prefer_websockets=True, minimal_client_version="0.144.0", reasoning_levels=_REASONING_LEVELS_MAX, - context_window=372_000, + context_window=272_000, default_reasoning_level="medium", priority=3, available_in_plans=_BOOTSTRAP_GPT56_AVAILABLE_IN_PLANS, diff --git a/docs/client-setup.md b/docs/client-setup.md index 3a96f7b6b4..eb479336df 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -4,7 +4,7 @@ Point any OpenAI-compatible client at codex-lb. If [API key auth](api-keys.md) i Model availability is discovered from the upstream Codex model catalog and can vary by account plan, workspace, rollout, and upstream deprecation state. Prefer the live `GET /v1/models` or `GET /backend-api/codex/models` response over a copied static table when configuring clients or API-key model allowlists. -The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all 372k context. `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. +The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all 272k context. `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. | Client | Endpoint | Config | |--------|----------|--------| @@ -222,19 +222,19 @@ jq 'del(.openai)' ~/.local/share/opencode/auth.json > auth.json.tmp && mv auth.j "name": "GPT-5.6-Sol", "reasoning": true, "options": { "reasoningEffort": "xhigh", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.6-terra": { "name": "GPT-5.6-Terra", "reasoning": true, "options": { "reasoningEffort": "high", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.6-luna": { "name": "GPT-5.6-Luna", "reasoning": true, "options": { "reasoningEffort": "medium", "reasoningSummary": "detailed" }, - "limit": { "context": 372000, "output": 65536 } + "limit": { "context": 272000, "output": 65536 } }, "gpt-5.5": { "name": "GPT-5.5", @@ -283,8 +283,8 @@ opencode { "id": "gpt-5.6-sol", "name": "gpt-5.6-sol (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false @@ -292,8 +292,8 @@ opencode { "id": "gpt-5.6-terra", "name": "gpt-5.6-terra (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false @@ -301,8 +301,8 @@ opencode { "id": "gpt-5.6-luna", "name": "gpt-5.6-luna (codex-lb)", - "contextWindow": 372000, - "contextTokens": 372000, + "contextWindow": 272000, + "contextTokens": 272000, "maxTokens": 4096, "input": ["text"], "reasoning": false diff --git a/openspec/changes/fix-gpt56-context-window/proposal.md b/openspec/changes/fix-gpt56-context-window/proposal.md new file mode 100644 index 0000000000..bd006a45c3 --- /dev/null +++ b/openspec/changes/fix-gpt56-context-window/proposal.md @@ -0,0 +1,23 @@ +## Why + +The GPT-5.6 bootstrap catalog was originally pinned to Codex +`rust-v0.144.1`, whose entries reported a 372,000-token context window. The +upstream bundled catalog corrected Sol, Terra, and Luna to 272,000 tokens in +`rust-v0.145.0`. codex-lb must advertise the corrected upstream input budget in +its bootstrap catalog and normative compatibility contract so startup/offline +clients do not overfill requests before the live registry refreshes. + +## What Changes + +- Re-pin GPT-5.6 bootstrap catalog provenance from Codex `rust-v0.144.1` to + `rust-v0.145.0`. +- Require `context_window` and `max_context_window` of 272,000 for Sol, Terra, + and Luna. +- Update regression-test evidence comments to cite the reproducible upstream + bundled catalog release instead of untracked live-fetch artifacts. + +## Impact + +- No schema, route, or database migration change. +- Before a live registry refresh, bootstrap `/v1/models` and `/backend-api/codex/models` change the GPT-5.6 advertised context budget from 372,000 to 272,000 tokens. +- Affects `model-catalog-compat` documentation, client setup examples, and GPT-5.6 bootstrap regression coverage. diff --git a/openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..ea158329db --- /dev/null +++ b/openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md @@ -0,0 +1,56 @@ +## MODIFIED Requirements + +### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog + +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves. In particular each +entry MUST carry: `context_window` and `max_context_window` of `272000`; +`minimal_client_version` `"0.144.0"`; `tool_mode` `"code_mode_only"`; +`use_responses_lite` `true`; `apply_patch_tool_type` `"freeform"`; +`web_search_tool_type` `"text_and_image"`; `supports_image_detail_original` +`true`; `truncation_policy` `{ "mode": "tokens", "limit": 10000 }`; +`comp_hash` `"3000"`; `reasoning_summary_format` `"experimental"`; +`default_reasoning_summary` `"none"`; `include_skills_usage_instructions` +`false`; `experimental_supported_tools` `[]` (a field the Codex client's +deserializer requires); `supports_search_tool` `true`; `additional_speed_tiers` +`["fast"]`; the `priority`/`Fast` service tier entry; `shell_type` +`"shell_command"`; `prefer_websockets` `true`; and the 21-plan +`available_in_plans` list upstream advertises (including `edu_plus`, +`edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST +be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and +reasoning-level descriptions MUST be the verbatim upstream strings. + +The ~16.5 KB upstream `base_instructions` prompt and the personality-templated +`model_messages` object are deliberately NOT bundled in the bootstrap catalog; +the first successful live registry refresh supplies them. This is the only +sanctioned divergence from the upstream GPT-5.6 entries. + +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` and `max_context_window=272000` + +#### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` +- **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna + +#### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` +- **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` +- **AND** only `gpt-5.6-sol` carries a non-null `availability_nux` message diff --git a/openspec/changes/fix-gpt56-context-window/tasks.md b/openspec/changes/fix-gpt56-context-window/tasks.md new file mode 100644 index 0000000000..59c94076f0 --- /dev/null +++ b/openspec/changes/fix-gpt56-context-window/tasks.md @@ -0,0 +1,23 @@ +## 1. Regression coverage + +- [x] 1.1 Update GPT-5.6 bootstrap catalog test evidence to cite + `codex-rs/models-manager/models.json` at Codex `rust-v0.145.0`. +- [x] 1.2 Run the focused bootstrap metadata tests and verify every GPT-5.6 + entry reports `context_window` and `max_context_window` of 272,000. +- [x] 1.3 Assert the top-level `context_window` and raw + `max_context_window` are 272,000 for every GPT-5.6 bootstrap entry. + +## 2. Specification + +- [x] 2.1 Add a `model-catalog-compat` delta that re-pins the GPT-5.6 bootstrap + catalog source to Codex `rust-v0.145.0` and requires both context-window + fields to be 272,000. +- [x] 2.2 State that operator context-window overrides take precedence over + the default bootstrap budget. + +- [x] 2.3 Correct the documented OpenCode and OpenClaw GPT-5.6 budgets to + 272,000 tokens. +## 3. Validation + +- [x] 3.1 Validate the OpenSpec change and the complete specification set. +- [x] 3.2 Run the focused bootstrap metadata tests after review follow-up. diff --git a/tests/integration/test_v1_models.py b/tests/integration/test_v1_models.py index dee695dd43..7d268f8cb1 100644 --- a/tests/integration/test_v1_models.py +++ b/tests/integration/test_v1_models.py @@ -281,7 +281,7 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien sol = entries["gpt-5.6-sol"] assert sol["display_name"] == "GPT-5.6-Sol" - assert sol["context_window"] == 372_000 + assert sol["context_window"] == 272_000 assert sol["default_reasoning_level"] == "low" assert {level["effort"] for level in sol["supported_reasoning_levels"]} == { "low", @@ -314,10 +314,11 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien "max", } - # Upstream-exact GPT-5.6 metadata as served on the Codex catalog wire - # (codex-rs/models-manager/models.json at rust-v0.144.1). + # Reproducible upstream catalog evidence: + # codex-rs/models-manager/models.json at rust-v0.145.0. for gpt56 in (sol, terra, luna): assert gpt56["minimal_client_version"] == "0.144.0" + assert gpt56["context_window"] == 272_000 assert gpt56["tool_mode"] == "code_mode_only" assert gpt56["use_responses_lite"] is True assert gpt56["apply_patch_tool_type"] == "freeform" @@ -327,7 +328,7 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien assert gpt56["reasoning_summary_format"] == "experimental" assert gpt56["comp_hash"] == "3000" assert gpt56["experimental_supported_tools"] == [] - assert gpt56["max_context_window"] == 372_000 + assert gpt56["max_context_window"] == 272_000 assert gpt56["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] diff --git a/tests/unit/test_model_registry.py b/tests/unit/test_model_registry.py index 6514132b86..2cc88b077e 100644 --- a/tests/unit/test_model_registry.py +++ b/tests/unit/test_model_registry.py @@ -27,7 +27,7 @@ } # The 21-plan list upstream advertises for GPT-5.6 -# (codex-rs/models-manager/models.json at rust-v0.144.1). +# (codex-rs/models-manager/models.json at rust-v0.145.0). EXPECTED_GPT56_MODEL_PLANS = { "business", "edu", @@ -242,7 +242,7 @@ def test_bootstrap_models_include_representative_upstream_metadata(): sol = models["gpt-5.6-sol"] assert sol.display_name == "GPT-5.6-Sol" - assert sol.context_window == 372_000 + assert sol.context_window == 272_000 assert sol.default_reasoning_level == "low" assert [level.effort for level in sol.supported_reasoning_levels] == [ "low", @@ -271,10 +271,11 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert luna.default_reasoning_level == "medium" assert [level.effort for level in luna.supported_reasoning_levels] == ["low", "medium", "high", "xhigh", "max"] - # Upstream-exact GPT-5.6 raw metadata (codex-rs/models-manager/models.json - # at rust-v0.144.1). + # Reproducible upstream catalog evidence: + # codex-rs/models-manager/models.json at rust-v0.145.0. for gpt56 in (sol, terra, luna): assert gpt56.minimal_client_version == "0.144.0" + assert gpt56.context_window == 272_000 assert gpt56.raw["tool_mode"] == "code_mode_only" assert gpt56.raw["use_responses_lite"] is True assert gpt56.raw["apply_patch_tool_type"] == "freeform" @@ -287,7 +288,7 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert gpt56.raw["include_skills_usage_instructions"] is False assert gpt56.raw["experimental_supported_tools"] == [] assert gpt56.raw["supports_search_tool"] is True - assert gpt56.raw["max_context_window"] == 372_000 + assert gpt56.raw["max_context_window"] == 272_000 assert gpt56.raw["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] From 4ace71e8044e5180905f7a53f2d00d010cd43233 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 16 Aug 2026 11:43:38 +0400 Subject: [PATCH 029/117] fix(auth): guard the refresh singleflight negative cache by successor ownership (#1652) * fix(auth): guard refresh singleflight successor settlement * test(auth): cover refresh successor settlement race * test(auth): exercise delayed refresh completion callback --- app/modules/accounts/auth_manager.py | 9 ++- .../proposal.md | 14 ++++ .../specs/usage-refresh-policy/spec.md | 25 ++++++ .../tasks.md | 3 + tests/unit/test_auth_manager.py | 81 +++++++++++++++++++ 5 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md create mode 100644 openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md create mode 100644 openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md diff --git a/app/modules/accounts/auth_manager.py b/app/modules/accounts/auth_manager.py index 483ce72fc2..0c4339930c 100644 --- a/app/modules/accounts/auth_manager.py +++ b/app/modules/accounts/auth_manager.py @@ -205,8 +205,13 @@ async def _complete(self, key: _RefreshSingleflightKey, task: asyncio.Task[Accou try: async with self._lock: current = self._inflight.get(key) - if current is task: - self._inflight.pop(key, None) + if current is not task: + # A successor owns settlement for this key; consume the + # stale task's result without touching its cache state. + if not task.cancelled(): + task.exception() + return + self._inflight.pop(key, None) if task.cancelled(): self._recent_failures.pop(key, None) return diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md b/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md new file mode 100644 index 0000000000..1fd011930d --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/proposal.md @@ -0,0 +1,14 @@ +# Fix refresh singleflight successor settlement + +The refresh singleflight negative cache must not publish a failed attempt's +error after a successor refresh has replaced that attempt for the same key. +This keeps callers arriving during the successor refresh joined to the live +operation instead of serving stale failure state. + +## Scope + +- Guard negative-cache writes and clears with the same current-task check that + guards inflight removal. +- Add the successor-race regression coverage already exercised by the F8 + bughunt probe. +- Do not change downstream account-status failure handling. diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md b/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..9589909b7d --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/specs/usage-refresh-policy/spec.md @@ -0,0 +1,25 @@ +# usage-refresh-policy Delta + +## ADDED Requirements + +### Requirement: Refresh singleflight settlement cannot poison a successor + +When a refresh task completes, it MUST mutate the inflight entry and +refresh-failure cache only if it is still the current inflight task for that +singleflight key. A completion from an older attempt MUST NOT publish or clear +negative-cache state belonging to a successor refresh. + +#### Scenario: Failed attempt is followed by a live successor + +- **GIVEN** a refresh task fails for a key +- **AND** a successor task for the same key is installed before the failed + task's completion settlement runs +- **WHEN** another caller arrives while the successor is still in flight +- **THEN** the caller joins the successor task +- **AND** the failed attempt's error is not served from the negative cache + +#### Scenario: Existing failure settlement has no successor + +- **GIVEN** a refresh task fails and remains the current inflight task +- **WHEN** its completion settlement runs +- **THEN** the configured negative-cache cooldown behavior is preserved diff --git a/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md b/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md new file mode 100644 index 0000000000..a1e5412413 --- /dev/null +++ b/openspec/changes/fix-refresh-singleflight-successor-guard/tasks.md @@ -0,0 +1,3 @@ +- [x] Guard refresh singleflight settlement cache mutations by task ownership. +- [x] Verify the independent-caller successor-race regression. +- [x] Run the account refresh unit suites and OpenSpec validation. diff --git a/tests/unit/test_auth_manager.py b/tests/unit/test_auth_manager.py index a1967c6d2a..1b9da7aa1c 100644 --- a/tests/unit/test_auth_manager.py +++ b/tests/unit/test_auth_manager.py @@ -593,6 +593,87 @@ async def _fake_refresh(_: str, **_kwargs: object) -> TokenRefreshResult: assert refresh_calls == 1 +@pytest.mark.asyncio +async def test_ensure_fresh_old_failure_cannot_replace_successor(monkeypatch): + """A delayed failed completion must not evict a newer refresh task.""" + encryptor = TokenEncryptor() + stale_refresh = utcnow().replace(year=utcnow().year - 1) + account = Account( + id="acc_sf_successor", + email="user@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access-old"), + refresh_token_encrypted=encryptor.encrypt("refresh-old"), + id_token_encrypted=encryptor.encrypt("id-old"), + last_refresh=stale_refresh, + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + refreshed_payload = {column.name: getattr(account, column.name) for column in Account.__table__.columns} + refreshed_payload.update( + access_token_encrypted=encryptor.encrypt("access-new"), + refresh_token_encrypted=encryptor.encrypt("refresh-new"), + ) + refreshed = Account(**refreshed_payload) + repo = _DummyRepo() + manager = AuthManager(cast(AccountsRepositoryPort, repo)) + monkeypatch.setattr(manager, "_ensure_chatgpt_account_id", lambda value: _identity(value)) + + mode = {"calls": 0} + successor_started = asyncio.Event() + release_successor = asyncio.Event() + + async def fake_run(_account): + mode["calls"] += 1 + if mode["calls"] == 1: + raise RefreshError("invalid_grant", "old refresh failed", False) + successor_started.set() + await release_successor.wait() + return refreshed + + monkeypatch.setattr(manager, "_run_refresh", fake_run) + singleflight = auth_manager_module._REFRESH_SINGLEFLIGHT + old_completion_started = asyncio.Event() + old_completion_finished = asyncio.Event() + release_old_completion = asyncio.Event() + original_complete = singleflight._complete + + async def hold_old_completion(key, task): + old_completion_started.set() + await release_old_completion.wait() + await original_complete(key, task) + old_completion_finished.set() + + monkeypatch.setattr(singleflight, "_complete", hold_old_completion) + + with pytest.raises(RefreshError, match="old refresh failed"): + await manager.ensure_fresh(account, force=True) + await old_completion_started.wait() + + successor = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await successor_started.wait() + joined_successor = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await asyncio.sleep(0) + assert not joined_successor.done() + assert mode["calls"] == 2 + + # The failed task's callback settles after the successor is installed. + release_old_completion.set() + await old_completion_finished.wait() + late_caller = asyncio.create_task(manager.ensure_fresh(account, force=True)) + await asyncio.sleep(0) + assert not late_caller.done() + release_successor.set() + assert await successor is refreshed + assert await joined_successor is refreshed + assert await late_caller is refreshed + assert mode["calls"] == 2 + + +async def _identity(value): + return value + + @pytest.mark.asyncio async def test_ensure_fresh_singleflights_refresh_admission_for_same_account(monkeypatch): started = asyncio.Event() From a60ef9a07000975a6502b2df115da80a703bacd6 Mon Sep 17 00:00:00 2001 From: Ahmad Maulana Iqbal <78488507+iqbalmaulana03@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:43:42 +0700 Subject: [PATCH 030/117] test(proxy): add property tests for response payload invariants (#1699) * test(proxy): add property tests for response payload invariants * test(proxy): harden hypothesis property test setup --- .gitignore | 1 + pyproject.toml | 1 + tests/unit/hypothesis_strategies.py | 30 ++++ tests/unit/test_openai_requests.py | 217 ++++++++++++++++++++++++++++ tests/unit/test_proxy_utils.py | 128 ++++++++++++++++ tests/unit/test_sse.py | 56 +++++++ uv.lock | 57 ++++++++ 7 files changed, 490 insertions(+) create mode 100644 tests/unit/hypothesis_strategies.py diff --git a/.gitignore b/.gitignore index 92b567da04..f1489e0a3e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ poc/ __pycache__/ *.py[cod] .pytest_cache/ +.hypothesis/ .mypy_cache/ .ruff_cache/ diff --git a/pyproject.toml b/pyproject.toml index 4e1e36ac0f..5057644eed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -86,6 +86,7 @@ dev = [ "openai>=2.16.0", "pytest-xdist>=3.8.0", "pytest-cov>=7.1.0", + "hypothesis>=6.165.3", ] docs = [ "mkdocs-material>=9.6", diff --git a/tests/unit/hypothesis_strategies.py b/tests/unit/hypothesis_strategies.py new file mode 100644 index 0000000000..ab843a9a34 --- /dev/null +++ b/tests/unit/hypothesis_strategies.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +from hypothesis import strategies as st + +json_scalars = st.one_of( + st.none(), + st.booleans(), + st.integers(min_value=-10_000, max_value=10_000), + st.floats(allow_nan=False, allow_infinity=False, width=32), + st.text(max_size=80), +) + +json_values = st.recursive( + json_scalars, + lambda children: st.one_of( + st.lists(children, max_size=5), + st.dictionaries(st.text(max_size=20), children, max_size=5), + ), + max_leaves=20, +) + +json_directive_types = json_values.filter(lambda value: value is not None and value != "message") + +json_objects = st.dictionaries( + st.text(max_size=20), + json_values, + max_size=8, +) + +json_arrays = st.lists(json_values, max_size=8) diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 8a883ad4be..4780625cc4 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -1,24 +1,34 @@ from __future__ import annotations import json +import re +from copy import deepcopy from typing import Mapping, cast import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from pydantic import ValidationError from app.core.openai.exceptions import ClientPayloadError from app.core.openai.requests import ( _ESTIMATED_CHARS_PER_TOKEN, _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS, + _UNSUPPORTED_UPSTREAM_FIELDS, ResponsesCompactRequest, ResponsesRequest, + _estimated_json_array_item_tokens, _estimated_json_tokens, _input_image_file_reference, + _sanitize_input_items, + _strip_unsupported_fields, + _trim_compact_input_for_upstream, extract_input_file_ids, extract_input_image_file_references, ) from app.core.openai.v1_requests import V1ResponsesCompactRequest, V1ResponsesRequest from app.core.types import JsonValue +from tests.unit.hypothesis_strategies import json_arrays, json_directive_types, json_objects, json_values def test_responses_requires_instructions(): @@ -115,6 +125,56 @@ def test_known_unsupported_upstream_fields_are_stripped(): assert dumped["custom_field"] == "kept" +@given(json_arrays) +@settings(deadline=None) +def test_sanitize_input_items_is_idempotent_for_json(input_items): + original = deepcopy(input_items) + try: + sanitized = _sanitize_input_items(input_items) + except ValueError: + # Tool items without a usable call ID are deliberately rejected. + return + + assert input_items == original + assert _sanitize_input_items(deepcopy(sanitized)) == sanitized + + +@given( + role=st.sampled_from(["system", "developer"]), + item_type=json_directive_types, + extra=json_objects, +) +@settings(deadline=None) +def test_sanitize_input_items_preserves_typed_directives(role, item_type, extra): + directive = dict(extra) + directive.update({"role": role, "type": item_type}) + + assert _sanitize_input_items([directive]) == [directive] + + +@given(payload=json_objects.map(lambda value: {key: item for key, item in value.items() if key != "input"})) +@settings(deadline=None) +def test_strip_unsupported_fields_is_idempotent(payload): + payload = cast(dict[str, JsonValue], payload) + first = _strip_unsupported_fields(deepcopy(payload)) + second = _strip_unsupported_fields(deepcopy(first)) + + assert second == first + assert _UNSUPPORTED_UPSTREAM_FIELDS.isdisjoint(first) + + +@given(namespace=json_values) +@settings(deadline=None) +def test_strip_unsupported_fields_namespace_flag_controls_replayed_calls(namespace): + payload = cast(dict[str, JsonValue], {"input": [{"type": "function_call", "namespace": namespace}]}) + + preserved = _strip_unsupported_fields(deepcopy(payload), strip_replayed_tool_call_namespaces=False) + stripped = _strip_unsupported_fields(deepcopy(payload)) + + assert preserved["input"] == payload["input"] + assert stripped["input"] == [{"type": "function_call"}] + + def test_responses_preserves_service_tier(): payload = { "model": "gpt-5.1", @@ -1163,6 +1223,163 @@ def test_compact_many_small_items_include_array_wire_framing_in_budget(): assert wire_bytes <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS * _ESTIMATED_CHARS_PER_TOKEN +@given(input_items=json_arrays) +@settings(max_examples=30, deadline=None) +def test_compact_trim_leaves_budget_fitting_json_unchanged(input_items): + if _estimated_json_tokens(input_items) > _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS: + return + + payload = cast(dict[str, JsonValue], {"input": deepcopy(input_items)}) + original = deepcopy(payload["input"]) + + _trim_compact_input_for_upstream(payload) + + assert payload["input"] == original + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_budget_order_and_is_stable(size): + input_items = [ + {"id": "head", "role": "user", "content": "head"}, + {"id": "middle", "role": "assistant", "content": "x" * size}, + {"id": "latest", "role": "user", "content": "latest"}, + ] + payload = cast(dict[str, JsonValue], {"input": input_items}) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], deepcopy(payload["input"])) + + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + retained_ids = [item["id"] for item in trimmed_input if isinstance(item, dict) and isinstance(item.get("id"), str)] + assert retained_ids == ["head", "latest"] + + _trim_compact_input_for_upstream(payload) + assert payload["input"] == trimmed_input + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_marker_accounts_for_omitted_middle_item(size): + input_items = [ + {"id": "head", "role": "user", "content": "head"}, + {"id": "middle", "role": "assistant", "content": "x" * size}, + {"id": "latest", "role": "user", "content": "latest"}, + ] + payload = cast(dict[str, JsonValue], {"input": input_items}) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + marker = next( + item + for item in trimmed_input + if isinstance(item, dict) and "[compact trim] Omitted " in str(item.get("content")) + ) + marker_text = str(marker["content"]) + match = re.search(r"Omitted (\d+) input items \(~(\d+) estimated tokens\)", marker_text) + + assert match is not None + assert match.groups() == ("1", str(_estimated_json_array_item_tokens(cast(JsonValue, input_items[1])))) + + +@given( + pair=st.sampled_from( + [ + ("function_call", "function_call_output"), + ("custom_tool_call", "custom_tool_call_output"), + ("apply_patch_call", "apply_patch_call_output"), + ] + ), + filler_size=st.integers(min_value=300_000, max_value=400_000), +) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_generated_tool_pairs(pair, filler_size): + call_type, output_type = pair + call = { + "type": call_type, + "name": "exec_command", + "call_id": "call-generated", + "arguments" if call_type == "function_call" else "input": "{}", + } + if call_type == "apply_patch_call": + call = { + "type": call_type, + "call_id": "call-generated", + "operation": {"patch": "noop"}, + } + output = {"type": output_type, "call_id": "call-generated", "output": "result"} + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "assistant", "content": "x" * filler_size}, + call, + output, + ] + }, + ) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + assert call in trimmed_input + assert output in trimmed_input + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + + +@given(anchor=st.sampled_from(["goal", "plan"]), filler_size=st.integers(350_000, 450_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_keeps_generated_state_anchor(anchor, filler_size): + anchor_text = ( + 'continue the goal' + if anchor == "goal" + else "# Plan Mode" + ) + anchor_item = { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": anchor_text}], + } + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "user", "content": "head"}, + {"role": "assistant", "content": "x" * filler_size}, + anchor_item, + {"role": "user", "content": "latest"}, + ] + }, + ) + + _trim_compact_input_for_upstream(payload) + trimmed_input = cast(list[JsonValue], payload["input"]) + + assert anchor_item in trimmed_input + assert _estimated_json_tokens(trimmed_input) <= _MAX_COMPACT_UPSTREAM_ESTIMATED_TOKENS + + +@given(size=st.integers(min_value=400_000, max_value=500_000)) +@settings(max_examples=8, deadline=None) +def test_compact_trim_rejects_generated_oversized_latest_item(size): + payload = cast( + dict[str, JsonValue], + { + "input": [ + {"role": "assistant", "content": "head"}, + {"role": "user", "content": "x" * size}, + ] + }, + ) + + with pytest.raises(ClientPayloadError) as raised: + _trim_compact_input_for_upstream(payload) + + assert raised.value.param == "input" + assert raised.value.code == "responses_compact_input_too_large" + + def test_compact_trims_oversized_input_by_estimated_tokens_with_head_tail_and_marker(): input_items = [ {"role": "user", "content": "initial goal and instructions"}, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 43a1f8c887..cb9718d7f4 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -25,6 +25,8 @@ from aiohttp.client_exceptions import ClientConnectorCertificateError from aiohttp.client_reqrep import ConnectionKey, RequestInfo from fastapi import WebSocket +from hypothesis import given, settings +from hypothesis import strategies as st from starlette.requests import Request from starlette.responses import StreamingResponse from websockets.exceptions import ConnectionClosedError @@ -100,6 +102,7 @@ from app.modules.request_logs.repository import PreviousResponseOwnerRecord, RequestLogsRepository from app.modules.usage.repository import AdditionalUsageRepository, UsageRepository from tests.unit._proxy_test_helpers import runtime_basic_auth_url +from tests.unit.hypothesis_strategies import json_objects, json_values pytestmark = pytest.mark.unit @@ -22767,6 +22770,131 @@ def test_slim_response_create_ignores_malformed_unhashable_item_type(): ] +def _non_user_item(value: JsonValue) -> JsonValue: + if not isinstance(value, dict): + return value + item = dict(value) + if item.get("role") == "user": + item["role"] = "assistant" + return item + + +@given( + extra=json_objects, + historical=st.lists(json_values.map(_non_user_item), max_size=5), + recent=st.lists(json_values.map(_non_user_item), max_size=5), +) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_preserves_recent_suffix_and_top_level_fields(extra, historical, recent): + recent_user = {"role": "user", "content": "latest request"} + payload = cast( + dict[str, JsonValue], + { + **{key: value for key, value in extra.items() if key != "input"}, + "input": [*historical, recent_user, *recent], + }, + ) + original = deepcopy(payload) + original_input = cast(list[JsonValue], original["input"]) + + slimmed_payload, _ = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + assert payload == original + assert {key: value for key, value in slimmed_payload.items() if key != "input"} == { + key: value for key, value in original.items() if key != "input" + } + slimmed_input = cast(list[JsonValue], slimmed_payload["input"]) + preserve_from = len(historical) + assert slimmed_input[preserve_from:] == original_input[preserve_from:] + assert json.dumps(slimmed_input[preserve_from:], ensure_ascii=True, sort_keys=True) == json.dumps( + original_input[preserve_from:], ensure_ascii=True, sort_keys=True + ) + + +@given( + cases=st.lists( + st.sampled_from(["top_image", "content_image", "tool_image", "file_image", "plain"]), + min_size=1, + max_size=5, + ) +) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_counts_historical_image_replacements_and_is_idempotent(cases): + historical: list[JsonValue] = [] + expected_images = 0 + inline_url = "data:image/png;base64,AAAA" + for index, case in enumerate(cases): + if case == "top_image": + historical.append({"type": "input_image", "image_url": inline_url, "id": f"image-{index}"}) + expected_images += 1 + elif case == "content_image": + historical.append( + { + "role": "assistant", + "content": [{"type": "input_image", "image_url": inline_url, "id": f"image-{index}"}], + } + ) + expected_images += 1 + elif case == "tool_image": + historical.append( + { + "type": "function_call_output", + "call_id": f"call-{index}", + "output": [{"type": "input_image", "image_url": inline_url}], + } + ) + expected_images += 1 + elif case == "file_image": + historical.append({"type": "input_image", "image_url": "file-id", "id": f"file-{index}"}) + else: + historical.append({"role": "assistant", "content": f"ordinary-{index}"}) + + latest = {"role": "user", "content": "latest"} + payload = cast(dict[str, JsonValue], {"input": [*historical, latest]}) + original = deepcopy(payload) + + slimmed_payload, summary = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + assert payload == original + if expected_images: + assert summary is not None + assert summary["historical_images_slimmed"] == expected_images + else: + assert summary is None + assert cast(list[JsonValue], slimmed_payload["input"])[-1] == latest + + second_payload, second_summary = proxy_service._slim_response_create_payload_for_upstream( + slimmed_payload, max_bytes=256 + ) + assert second_payload == slimmed_payload + assert second_summary is None + + +@given(sizes=st.lists(st.integers(min_value=0, max_value=34 * 1024), min_size=1, max_size=5)) +@settings(max_examples=30, deadline=None) +def test_slim_response_create_counts_oversized_historical_tool_outputs(sizes): + historical = [ + { + "type": "custom_tool_call_output", + "call_id": f"call-{index}", + "output": "x" * size, + } + for index, size in enumerate(sizes) + ] + latest = {"role": "user", "content": "latest"} + payload = cast(dict[str, JsonValue], {"input": [*historical, latest]}) + + slimmed_payload, summary = proxy_service._slim_response_create_payload_for_upstream(payload, max_bytes=256) + + expected_count = sum(size > 32 * 1024 for size in sizes) + if expected_count: + assert summary is not None + assert summary["historical_tool_outputs_slimmed"] == expected_count + else: + assert summary is None + assert cast(list[JsonValue], slimmed_payload["input"])[-1] == latest + + def test_websocket_receive_timeout_prefers_idle_timeout_when_budget_allows(monkeypatch): monkeypatch.setattr(proxy_service.time, "monotonic", lambda: 100.0) diff --git a/tests/unit/test_sse.py b/tests/unit/test_sse.py index 69a1e97a53..44983fbdc6 100644 --- a/tests/unit/test_sse.py +++ b/tests/unit/test_sse.py @@ -6,16 +6,20 @@ from typing import Any, cast import pytest +from hypothesis import given, settings +from hypothesis import strategies as st from app.core.openai.parsing import parse_sse_event from app.core.utils.sse import ( CODEX_KEEPALIVE_FRAME, SSE_KEEPALIVE_FRAME, extract_sse_data, + format_sse_data, format_sse_event, inject_sse_keepalives, parse_sse_data_json, ) +from tests.unit.hypothesis_strategies import json_objects, json_values pytestmark = pytest.mark.unit @@ -26,6 +30,58 @@ def test_format_sse_event_serializes_payload(): assert result == 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_1"}}\n\n' +@given(payload=json_objects) +@settings(max_examples=40, deadline=None) +def test_format_sse_event_round_trips_arbitrary_json_objects(payload): + assert parse_sse_data_json(format_sse_event(payload)) == payload + + +@given(payload=json_objects) +@settings(max_examples=40, deadline=None) +def test_format_sse_data_round_trips_arbitrary_json_objects(payload): + assert parse_sse_data_json(format_sse_data(payload)) == payload + + +@given( + boundary=st.sampled_from(["\r", "\n", "\r\n"]), + key=st.text(max_size=40), + value=st.integers(), +) +@settings(max_examples=30, deadline=None) +def test_sse_line_boundaries_are_equivalent_in_multiline_data(boundary, key, value): + encoded_key = json.dumps(key, ensure_ascii=True) + block = f"data: {{{encoded_key}:" + boundary + f"data: {value}}}" + boundary * 2 + + assert parse_sse_data_json(block) == {key: value} + + +@given(text=st.text(max_size=80)) +@settings(max_examples=30, deadline=None) +def test_sse_unicode_line_separators_remain_data(text): + payload = {"value": f"before{text}\u2028middle\u2029after"} + block = "data: " + json.dumps(payload, ensure_ascii=False) + "\n\n" + + assert parse_sse_data_json(block) == payload + + +@given( + boundary=st.sampled_from(["\r", "\n", "\r\n"]), + first=st.text(alphabet=st.characters(blacklist_categories=("C", "Z")), min_size=1, max_size=40), + second=st.text(alphabet=st.characters(blacklist_categories=("C", "Z")), min_size=1, max_size=40), +) +@settings(max_examples=30, deadline=None) +def test_sse_multiline_data_ignores_comments_and_joins_with_newline(boundary, first, second): + block = f": comment{boundary}data: {first}{boundary}event: ignored{boundary}data: {second}{boundary}{boundary}" + + assert extract_sse_data(block) == f"{first}\n{second}" + + +@given(value=st.one_of(st.none(), st.booleans(), st.integers(), st.lists(json_values, max_size=4))) +@settings(max_examples=30, deadline=None) +def test_parse_sse_data_json_rejects_non_object_json(value): + assert parse_sse_data_json("data: " + json.dumps(value) + "\n\n") is None + + async def _agen(items: list[str]) -> AsyncIterator[str]: for item in items: yield item diff --git a/uv.lock b/uv.lock index 902c566c5f..60b221b908 100644 --- a/uv.lock +++ b/uv.lock @@ -534,6 +534,7 @@ tracing = [ [package.dev-dependencies] dev = [ { name = "httpx" }, + { name = "hypothesis" }, { name = "openai" }, { name = "pre-commit" }, { name = "pytest" }, @@ -590,6 +591,7 @@ provides-extras = ["metrics", "tracing"] [package.metadata.requires-dev] dev = [ { name = "httpx", specifier = ">=0.28.1" }, + { name = "hypothesis", specifier = ">=6.165.3" }, { name = "openai", specifier = ">=2.16.0" }, { name = "pre-commit", specifier = ">=4.5.1" }, { name = "pytest", specifier = ">=9.0.2" }, @@ -1163,6 +1165,52 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "hypothesis" +version = "6.165.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/7a/7a277ac07776191be594f74f6425649d529e4876f7d3ff1ee96d393ffdbc/hypothesis-6.165.3.tar.gz", hash = "sha256:687c5abb1a9c11478577c2cf18685c0eb82150d278477d3e14da290a1ef2a098", size = 502263, upload-time = "2026-08-11T01:23:09.1Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/c7/18152acad5f85f91554b2030000319b952a54151509953651ec40f37d50d/hypothesis-6.165.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:56af539c811b11ab5475704c300b8f0b46cc6dd0edc267e02a16487e803c77f8", size = 781671, upload-time = "2026-08-11T01:22:09.176Z" }, + { url = "https://files.pythonhosted.org/packages/d3/77/4293ea8a7fdb713956a8bf460b9070115df69f8216a900507633f9cdb225/hypothesis-6.165.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f40c10cfdb1ea2cd75e5d4e6e0cfdcb6198ab8406e8922666480e6dc11eea341", size = 777291, upload-time = "2026-08-11T01:22:15.991Z" }, + { url = "https://files.pythonhosted.org/packages/02/fa/fa2071a6afaefc082dc7a033f41ae61436caf442d5973ba8ca9c29a69460/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a0854b1de4577f7e1beb1d681360285b5d678b65a809787ff4eab5b8b25efca", size = 1106490, upload-time = "2026-08-11T01:22:07.858Z" }, + { url = "https://files.pythonhosted.org/packages/ba/86/de724b7f9cd10e3be4efa21770457172e549d7576b1d8e29d6177eef5e47/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:360991cda8e488924905af48949033b90d4877ac97b9ad5d826d4d0f5a4b8cfb", size = 1135054, upload-time = "2026-08-11T01:22:29.499Z" }, + { url = "https://files.pythonhosted.org/packages/12/6a/96721cf447bd3c64b5e6843dde4444b20f3ddd901ad366cc73d0e7314bf5/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf502000f4a8ef4c9ab9493ca3b4fe17ae3033c18a8e2a31cdd69515dc7d97be", size = 1155997, upload-time = "2026-08-11T01:21:48.496Z" }, + { url = "https://files.pythonhosted.org/packages/0f/ae/a793cce6497f233b155f97684bf7d0e424c25613dd87b8af8a4e87820232/hypothesis-6.165.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:b9fcf47ad18f87f7c15bd36289bd45708bbfd250129d73bf554653e2f9afc931", size = 1111326, upload-time = "2026-08-11T01:22:21.9Z" }, + { url = "https://files.pythonhosted.org/packages/28/8d/dc3cdfd55843d038effa2458a9c9bd73002218a8c0fd58c2c0ab7fa328db/hypothesis-6.165.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb05529cbcab5a317d03d7bb0e90d382f79ef1643e3568577916d0e24bfe70b", size = 1148079, upload-time = "2026-08-11T01:21:51.071Z" }, + { url = "https://files.pythonhosted.org/packages/db/4b/2f62924ac41f3d3482b29ded4c213f27ff4a103e56e84eeb528d4900cac7/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:57eae10a64340cd621a78eae9cb0459bd68ea99fbaa933c4f00e34d5087b6376", size = 1281862, upload-time = "2026-08-11T01:21:39.274Z" }, + { url = "https://files.pythonhosted.org/packages/7e/d4/01c78b7b7348b6e8cef9b999109dfb93b14c7e1e38bc22170129f8b17181/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:19df0f2239052e9a870634a1d9bcdff95e2a2ab508573e5dd5c3d1ca545f5b3c", size = 1408437, upload-time = "2026-08-11T01:22:13.243Z" }, + { url = "https://files.pythonhosted.org/packages/35/76/e940b5a5aaf75bcd4784f1f3f9bf2b9a642a706bc0a9639077ca84f1325f/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:9781a8026adff4b4516404cf0e5f2cadcb471318c2882a264e1c57c4c092266f", size = 1281168, upload-time = "2026-08-11T01:21:58.964Z" }, + { url = "https://files.pythonhosted.org/packages/fc/84/b153e81a614f45e0902e3b9e8a8b079e64214c50abb6fbe9acc62ccf686d/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1dd7e05f88e3e108a5e4f5f71a3eaf205559e8951e3c1f1ffd04cea82ed3b731", size = 1323263, upload-time = "2026-08-11T01:21:52.374Z" }, + { url = "https://files.pythonhosted.org/packages/fb/5f/e5144d9e91ab7260650cb1ee032ca23208d49ebb1334845baf6407c1a9d9/hypothesis-6.165.3-cp310-abi3-win32.whl", hash = "sha256:d1389bda38cb222acc109aef5b31643ce799a39a76294a50ad8b84e32f92d76d", size = 667499, upload-time = "2026-08-11T01:21:47.36Z" }, + { url = "https://files.pythonhosted.org/packages/a9/18/f008b6f1f1c293d51c2776f8815d95bccb777dcf87df2a0ab56b273b47dc/hypothesis-6.165.3-cp310-abi3-win_amd64.whl", hash = "sha256:10cda6988ca4b1da389548b6fdd71af236b588a601fc1757e56eb8988e4240d8", size = 673643, upload-time = "2026-08-11T01:22:46.975Z" }, + { url = "https://files.pythonhosted.org/packages/03/3e/95cba31dbe775b99a4548cdae192e1ad15cee7b64fbdfe6cd4c9d00031b4/hypothesis-6.165.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:447f139d6dd70a5d8b178ef507463fb0430ace9ce42e3b2351d2803a391fe774", size = 783183, upload-time = "2026-08-11T01:22:45.286Z" }, + { url = "https://files.pythonhosted.org/packages/56/0e/51bf125cdf7855b69097b8f59c73ef3cf5f4e3d68a16e808d2d1f08a1ff1/hypothesis-6.165.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6152c718606f1705e673c6b30a6ebd3ff08d340da85291dd3c432c73b28a9b3a", size = 774820, upload-time = "2026-08-11T01:22:24.825Z" }, + { url = "https://files.pythonhosted.org/packages/0a/69/b954f742b97441a5c49f8f8704826ee0637a6cce3a7d06ce85fbefc54ac5/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27fe7826ad83ccc2e8062f0fab43b137bab34cef1a149a926b34a7b8382ee22c", size = 1105186, upload-time = "2026-08-11T01:21:41.733Z" }, + { url = "https://files.pythonhosted.org/packages/6e/8a/33e41d9cc1be7661e0b4129c225a93c3f12544714300aadb95ae7eedf894/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1ff92876a324f7b9cdb92cedf103e380b7a12aa7df55ebcb16dd0f495a879e8", size = 1155215, upload-time = "2026-08-11T01:22:06.604Z" }, + { url = "https://files.pythonhosted.org/packages/ee/53/ba09526c9100ace5752908ac7251d2dc3960ce7e0e97a31152aaa26c33ee/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53f1564c97d27fc109f212404d49cd71d7789777dbe0685628ffe9838df56240", size = 1279245, upload-time = "2026-08-11T01:21:56.182Z" }, + { url = "https://files.pythonhosted.org/packages/c1/93/fc637d355791a65364a3409ff06ade7ba5d3fc6f1d07a729781dec315fa0/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:788a9b0a7aae719a2b71a1c2f07e51deb1d0fe990164a9090c686833ed4bfbad", size = 1322370, upload-time = "2026-08-11T01:22:48.492Z" }, + { url = "https://files.pythonhosted.org/packages/b6/8d/826053ba0263143fed2b0e8af009dc868d8a932e7246e191deb8ca7ce8ff/hypothesis-6.165.3-cp313-cp313-win_amd64.whl", hash = "sha256:37830f0795abfdf738d2a5b6f829a73f3ab498de45a2e61b0bf3bd38d8c9ddb9", size = 670804, upload-time = "2026-08-11T01:22:00.103Z" }, + { url = "https://files.pythonhosted.org/packages/e7/27/3230f8de3d853b2b547731916ae1d1026bd197cd3f2d35dafc0b445da46b/hypothesis-6.165.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:38826441dbf528cc156388d0a05526086a12da3e1348353d3fa14de03e57c4b2", size = 783286, upload-time = "2026-08-11T01:22:40.816Z" }, + { url = "https://files.pythonhosted.org/packages/6c/28/9f9ca830d376c50babe55c616f6d99eea886c6ebcd8b512dcd5d56f9e40c/hypothesis-6.165.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:87490115edd34a246a4ba8b1144cbdf571438c46c406ece05caf65908667c9a9", size = 774963, upload-time = "2026-08-11T01:23:05.409Z" }, + { url = "https://files.pythonhosted.org/packages/33/3c/3c81f08ec1edce160da509c5785d78c0e25a7913899c4b9ff724bfd01420/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f51f4346cfa26bca68c68f7bbbd2b1812208bc9f572187c95ecab080ed402153", size = 1105730, upload-time = "2026-08-11T01:22:42.311Z" }, + { url = "https://files.pythonhosted.org/packages/bf/b5/f6f81b9aec9999ec63920d168617cab67a038be05487eff3410ccd072bfe/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4da89eb4b36b3260ff714d2ecc3274b9bd599fd96687d2d9ed53d5e1a801a7a7", size = 1155383, upload-time = "2026-08-11T01:21:57.58Z" }, + { url = "https://files.pythonhosted.org/packages/dd/27/7f3a8c6101675bf95c80cd8c9173d65892ca0b7b640551156dc4537fab1f/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eb6d31c14d7bdfe03e501d88ee296c149a74cc93e3d01c76ea335e64ee5f33ec", size = 1279606, upload-time = "2026-08-11T01:22:37.56Z" }, + { url = "https://files.pythonhosted.org/packages/d9/16/0c23e06a24e421e532f62a95021fae34f685f3a194c081c6991b4ab202b3/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0863e1a9258bc103abe616fa9471cfa66a1535ea404dd8a0bf360e0a29502397", size = 1322697, upload-time = "2026-08-11T01:22:27.857Z" }, + { url = "https://files.pythonhosted.org/packages/dc/56/8356dadf45e5c635b46aa2b57fa74f3210250a8e38b860b6b75f50ed0b42/hypothesis-6.165.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:53c56155f2cfbb45ec97fef9ea3b8453b4a34c48c3c5cacee16f97dd2a037994", size = 614859, upload-time = "2026-08-11T01:22:01.304Z" }, + { url = "https://files.pythonhosted.org/packages/e3/79/124d4faf235219acd685c359760a5cb3995609bc50ce465e54c3249841ee/hypothesis-6.165.3-cp314-cp314-win_amd64.whl", hash = "sha256:c48f41e950b5e602e2fdf8f92dcc8ac7bf715a003bf822afb7c9d5cbc41bc344", size = 670600, upload-time = "2026-08-11T01:22:10.356Z" }, + { url = "https://files.pythonhosted.org/packages/01/7a/41ac5e68d9ce079d1b76d4c54126354df61b948c3d519d1289aca877eedc/hypothesis-6.165.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9563d3040178fb1f522665bcec6458cc0d21ab77d7c637058a8be4ea8c01d236", size = 781746, upload-time = "2026-08-11T01:22:34.472Z" }, + { url = "https://files.pythonhosted.org/packages/5d/fb/7ecc21aae63a83dbc8036f9a0544c6b3d798db566b97b5202bdf8e770f80/hypothesis-6.165.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1fe1783543b43ba9808c016950e5e84b3804dc3365ba77c37c427b5896a558a1", size = 773382, upload-time = "2026-08-11T01:22:55.279Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f2/9cc2a4768f9a483b12e307ba585f5eb9c7f5500bd16ff82ddbf62a9a1b88/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea34806a4df4e8305a096dcf8e53cdd903c96c1e0d2dd5b001d2283f639c3f1", size = 1103911, upload-time = "2026-08-11T01:23:00.286Z" }, + { url = "https://files.pythonhosted.org/packages/54/9e/b551a494f84976ee5bb9374c197ccc126dea2ec6f22098d5f70705237473/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d188454b95ce46ba991e3c52161255d76af25170ad28591f6b30b045e501216e", size = 1154060, upload-time = "2026-08-11T01:22:20.413Z" }, + { url = "https://files.pythonhosted.org/packages/69/37/8e22a236f1f1e599525549a34672fb0523109f571486fe209b12a84a942e/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1c47b15ce97a9b1346bc7d7013c5f215380f78ae01c0f73a1638bd8b98bdd76", size = 1277631, upload-time = "2026-08-11T01:22:30.965Z" }, + { url = "https://files.pythonhosted.org/packages/13/0f/feb33bfc23853b4ba6360ff5e34235cd8bea0d7dd1eb21e17491f581c4e2/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:996077ef7a3bb332b6638f698ddf7555c82784b58dde80eeba3f07c0a322b40f", size = 1321326, upload-time = "2026-08-11T01:21:45.089Z" }, + { url = "https://files.pythonhosted.org/packages/9f/3b/ad56b56540a0719f493edec0dd442ebb21272147d2482ef505d19760a6d3/hypothesis-6.165.3-cp314-cp314t-win_amd64.whl", hash = "sha256:57a8273bdafe3f450afe66999fd130d4935d775eaf4ef63fcac0bee8015fc512", size = 670613, upload-time = "2026-08-11T01:22:05.294Z" }, +] + [[package]] name = "identify" version = "2.6.19" @@ -2424,6 +2472,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + [[package]] name = "sqlalchemy" version = "2.0.51" From 5780a27f8c77f033ece2d144cb25ff89fb9db679 Mon Sep 17 00:00:00 2001 From: Choi138 <84369321+choi138@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:43:45 +0900 Subject: [PATCH 031/117] fix(http-bridge): dedupe retry circuit failures per send (#1743) * fix(http-bridge): dedupe retry circuit failures per send * fix(http-bridge): preserve retry failure races * fix(http-bridge): retain retry attempt through recovery * fix(http-bridge): make retry settlement race-safe --------- Co-authored-by: Darafei Praliaskouski --- .../proxy/_service/http_bridge/helpers.py | 48 + .../_service/http_bridge/request_submit.py | 75 +- .../_service/http_bridge/retry_circuit.py | 192 +++- .../proxy/_service/http_bridge/streaming.py | 7 +- .../_service/http_bridge/upstream_events.py | 60 +- app/modules/proxy/_service/support.py | 42 + app/modules/proxy/service.py | 26 +- .../design.md | 128 +++ .../proposal.md | 71 ++ .../specs/responses-api-compat/spec.md | 95 ++ .../tasks.md | 28 + tests/unit/test_proxy_http_bridge.py | 857 +++++++++++++++++- tests/unit/test_proxy_utils.py | 1 + 13 files changed, 1562 insertions(+), 68 deletions(-) create mode 100644 openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md create mode 100644 openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md create mode 100644 openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 090a4cfc7b..4c3cdf75ca 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -125,6 +125,8 @@ _REQUEST_TRANSPORT_HTTP, _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 _http_bridge_session_supports_service_tier, + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, _HTTPBridgeSessionKey, _WebSocketRequestState, @@ -814,6 +816,52 @@ def _http_bridge_eventless_precreated_deadline( ) +def _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + request_states: Sequence[_WebSocketRequestState], +) -> _HTTPBridgeRetryCircuitAttemptSelection: + eligible_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + recorded_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + settled_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + attempt_seen = False + for request_state in request_states: + attempt = getattr(request_state, "response_create_attempt", None) + if attempt is None: + continue + attempt_seen = True + if attempt.retry_circuit_failure_recorded: + recorded_attempts.append(attempt) + continue + if attempt.disarmed or attempt.response_observed: + settled_attempts.append(attempt) + continue + if ( + request_state.transport != _REQUEST_TRANSPORT_HTTP + or request_state.skip_request_log + or request_state.response_id is not None + or request_state.latency_response_created_ms is not None + or request_state.response_event_count != 0 + or request_state.downstream_visible + ): + continue + eligible_attempts.append(attempt) + + for kind, attempts in ( + ("eligible", eligible_attempts), + ("recorded", recorded_attempts), + ("settled", settled_attempts), + ): + unique_attempts: list[_HTTPBridgeResponseCreateAttempt] = [] + for attempt in attempts: + if not any(candidate is attempt for candidate in unique_attempts): + unique_attempts.append(attempt) + if unique_attempts: + return _HTTPBridgeRetryCircuitAttemptSelection( + kind=kind, + attempts=tuple(unique_attempts), + ) + return _HTTPBridgeRetryCircuitAttemptSelection(kind="ineligible" if attempt_seen else "absent") + + def _http_bridge_session_has_admission_waiter(session: object | None) -> bool: """Keep a closed bridge registered while an unsent request owns its handoff.""" return session is not None and bool(getattr(session, "admission_waiter_count", 0)) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 7662fee5d8..10f536a6ce 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -7,7 +7,7 @@ import random from collections import deque from collections.abc import Callable -from dataclasses import replace +from dataclasses import dataclass, replace from typing import Any, Literal, Mapping, cast from uuid import uuid4 @@ -85,6 +85,7 @@ _http_bridge_prewarm_enabled, _http_bridge_request_budget_seconds, _http_bridge_request_counts_against_queue, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _log_http_bridge_event, _record_continuity_fail_closed, _record_http_bridge_prewarm_outcome, @@ -146,6 +147,8 @@ _clear_websocket_request_error_overrides, _copy_websocket_route_metadata_from_session, _event_type_from_payload, + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, _request_log_client_fields, _websocket_request_can_replay_before_visible_output, @@ -228,6 +231,16 @@ ) +@dataclass(frozen=True, slots=True) +class _HTTPBridgeStaleGateSnapshot: + pending_states: list[_WebSocketRequestState] + queued_count: int + threshold_seconds: float + stale_request_states: list[_WebSocketRequestState] + should_retire: bool + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection + + def _http_bridge_client_full_history_recovery_enabled(request_state: _WebSocketRequestState) -> bool: """Return whether an ambiguous send failure may ask the client to replay.""" settings = _service_get_settings() @@ -357,6 +370,9 @@ async def _send_http_bridge_request_text_with_archive_id( on_send_started() token = set_request_id(request_state.archive_request_id) try: + request_state.response_create_attempt_count += 1 + attempt = _HTTPBridgeResponseCreateAttempt(ordinal=request_state.response_create_attempt_count) + request_state.response_create_attempt = attempt request_state.response_create_sent_at = _service_time().monotonic() session.upstream_reader_wakeup.set() try: @@ -365,7 +381,9 @@ async def _send_http_bridge_request_text_with_archive_id( # A failed or cancelled send is settled by its caller. Disarm the # owner watchdog before lifecycle ownership is released so the # reader cannot race that cleanup and settle the request twice. - request_state.response_create_sent_at = None + attempt.disarmed = True + if request_state.response_create_attempt is attempt: + request_state.response_create_sent_at = None session.upstream_reader_wakeup.set() raise finally: @@ -2624,7 +2642,15 @@ async def _fail_stale_http_bridge_pending_requests( request_states: list[_WebSocketRequestState], *, detail: str, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, ) -> None: + if retry_circuit_attempt_selection is None: + # Capture the physical sends before waiting for pending ownership. + # A concurrent recovery may replace request_state.response_create_attempt + # while this task is suspended on pending_lock. + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + request_states + ) stale_requests: deque[_WebSocketRequestState] = deque() response_events_seen = 0 async with session.pending_lock: @@ -2651,7 +2677,11 @@ async def _fail_stale_http_bridge_pending_requests( # even when the session itself survives with other active requests. _record_http_bridge_quarantine_wedged_pending(self, session, stale_requests) if response_events_seen == 0: - await self._record_http_bridge_retry_circuit_failure(session, detail=detail) + await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail=detail, + selection=retry_circuit_attempt_selection, + ) await self._fail_pending_websocket_requests( account=session.account, account_id_value=session.account.id, @@ -2694,6 +2724,37 @@ def _classify_http_bridge_stale_gate_holders( return stale_states, False return [], bool(stale_states) + async def _snapshot_http_bridge_stale_gate_state( + self: Any, + session: "_HTTPBridgeSession", + *, + now: float, + ) -> _HTTPBridgeStaleGateSnapshot: + threshold_seconds = float( + getattr(_service_get_settings(), "http_responses_session_bridge_stuck_gate_retire_after_seconds", 300.0) + ) + async with session.pending_lock: + pending_states = list(session.pending_requests) + stale_request_states, should_retire = self._classify_http_bridge_stale_gate_holders( + pending_states, + now=now, + threshold_seconds=threshold_seconds, + session_closed=session.closed, + ) + retry_circuit_request_states = ( + stale_request_states if stale_request_states else (pending_states if should_retire else ()) + ) + return _HTTPBridgeStaleGateSnapshot( + pending_states=pending_states, + queued_count=session.queued_request_count, + threshold_seconds=threshold_seconds, + stale_request_states=stale_request_states, + should_retire=should_retire, + retry_circuit_attempt_selection=( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests(retry_circuit_request_states) + ), + ) + async def _retire_http_bridge_after_drain_if_ready(self: Any, session: "_HTTPBridgeSession") -> bool: if not (session.upstream_control.reconnect_requested and session.upstream_control.retire_after_drain): return False @@ -2724,6 +2785,7 @@ async def _retire_stale_pending_http_bridge_session( retry_circuit_detail: str | None = None, response_events_seen: int | None = None, retired_request_count: int | None = None, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, ) -> None: async with session.pending_lock: retired_request_states = list(session.pending_requests) @@ -2755,6 +2817,10 @@ async def _retire_stale_pending_http_bridge_session( ), default=0, ) + if retry_circuit_attempt_selection is None: + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + retired_request_states + ) # Direct retirement (for example the all-stale stuck-gate path, where # the wedged reattach is the only pending request) cancels the reader # and fails the pendings without passing the partial-cleanup hook or @@ -2773,9 +2839,10 @@ async def _retire_stale_pending_http_bridge_session( # that handoff, genuine pre-response failures disappear from circuit # accounting while idle closes and request failures look identical. if retired_request_count > 0 and response_events_seen == 0: - await self._record_http_bridge_retry_circuit_failure( + await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, detail=retry_circuit_detail or detail, + selection=retry_circuit_attempt_selection, ) session.closed = True async with self._http_bridge_lock: diff --git a/app/modules/proxy/_service/http_bridge/retry_circuit.py b/app/modules/proxy/_service/http_bridge/retry_circuit.py index 891bf49dca..f152a0a7af 100644 --- a/app/modules/proxy/_service/http_bridge/retry_circuit.py +++ b/app/modules/proxy/_service/http_bridge/retry_circuit.py @@ -9,7 +9,11 @@ from app.core.metrics.prometheus import PROMETHEUS_AVAILABLE, http_bridge_retry_circuit_total from app.modules.proxy._service.observability import _hash_identifier -from app.modules.proxy._service.support import _HTTPBridgeSession +from app.modules.proxy._service.support import ( + _HTTPBridgeResponseCreateAttempt, + _HTTPBridgeRetryCircuitAttemptSelection, + _HTTPBridgeSession, +) from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS logger = logging.getLogger(__name__) @@ -57,7 +61,95 @@ def _initialize_http_bridge_retry_circuit(service: Any, reset_transient_cache: A service._http_bridge_retry_circuit_lock = anyio.Lock() +def _record_http_bridge_retry_circuit_duplicate_suppressed( + session: _HTTPBridgeSession, + *, + attempt: _HTTPBridgeResponseCreateAttempt, + consecutive_failures: int, + detail: str, +) -> None: + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome="duplicate_suppressed").inc() + logger.info( + "http_bridge_retry_circuit event=duplicate_suppressed bridge_kind=%s bridge_key=%s " + "failures=%s detail=%s attempt=%s", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + consecutive_failures, + detail, + attempt.ordinal, + ) + + class _HTTPBridgeRetryCircuitMixin: + async def _http_bridge_retry_circuit_current_count(self: Any, session: _HTTPBridgeSession) -> int: + async with self._http_bridge_retry_circuit_lock: + current_state = self._http_bridge_retry_circuits.get(session.key) + return current_state.consecutive_failures if current_state is not None else 0 + + async def _await_http_bridge_retry_circuit_attempt_settlement( + self: Any, + session: _HTTPBridgeSession, + *, + attempt: _HTTPBridgeResponseCreateAttempt, + detail: str, + ) -> int: + settled = attempt.retry_circuit_failure_settled + if settled is not None: + await settled.wait() + consecutive_failures = await self._http_bridge_retry_circuit_current_count(session) + _record_http_bridge_retry_circuit_duplicate_suppressed( + session, + attempt=attempt, + consecutive_failures=consecutive_failures, + detail=detail, + ) + return consecutive_failures + + async def _record_http_bridge_retry_circuit_failure_for_attempt_selection( + self: Any, + session: _HTTPBridgeSession, + *, + detail: str, + selection: _HTTPBridgeRetryCircuitAttemptSelection, + ) -> int | None: + attempt = selection.attempt + if attempt is not None: + return await self._record_http_bridge_retry_circuit_failure( + session, + detail=detail, + attempt=attempt, + ) + if selection.kind == "absent": + return await self._record_http_bridge_retry_circuit_failure(session, detail=detail) + if selection.kind == "recorded": + for recorded_attempt in selection.attempts: + settled = recorded_attempt.retry_circuit_failure_settled + if settled is not None: + await settled.wait() + consecutive_failures = await self._http_bridge_retry_circuit_current_count(session) + for recorded_attempt in selection.attempts: + _record_http_bridge_retry_circuit_duplicate_suppressed( + session, + attempt=recorded_attempt, + consecutive_failures=consecutive_failures, + detail=detail, + ) + return consecutive_failures + + outcome = "ambiguous_suppressed" if selection.ambiguous else "ineligible_suppressed" + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome=outcome).inc() + logger.info( + "http_bridge_retry_circuit event=%s bridge_kind=%s bridge_key=%s detail=%s candidate_attempts=%s", + outcome, + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + detail, + len(selection.attempts), + ) + return None + def _prune_http_bridge_retry_circuit_state(self: Any, now: float) -> None: expiry = now - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS for key, state in list(self._http_bridge_retry_circuits.items()): @@ -354,51 +446,85 @@ async def _record_http_bridge_retry_circuit_failure( session: _HTTPBridgeSession, *, detail: str, + attempt: _HTTPBridgeResponseCreateAttempt | None = None, ) -> int | None: detail = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail) if session.key.strength != "hard" or detail not in _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS: return None + scoped_attempt = attempt + if scoped_attempt is not None: + if scoped_attempt.retry_circuit_failure_recorded: + return await self._await_http_bridge_retry_circuit_attempt_settlement( + session, + attempt=scoped_attempt, + detail=detail, + ) + if scoped_attempt.disarmed or scoped_attempt.response_observed: + return None + await self._load_http_bridge_retry_circuit(session) threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD) base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS) max_backoff = max(base_backoff, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS) clean_close_max_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS) now = time.monotonic() + duplicate_attempt: _HTTPBridgeResponseCreateAttempt | None = None + state: _HTTPBridgeRetryCircuitState | None = None async with self._http_bridge_retry_circuit_lock: - state = self._http_bridge_retry_circuits.setdefault( - session.key, - _HTTPBridgeRetryCircuitState(last_touched_monotonic=now), - ) - state.last_touched_monotonic = now - state.last_failure_monotonic = now - state.half_open_until = 0.0 - state.consecutive_failures += 1 - state.last_detail = detail - if state.consecutive_failures >= threshold: - backoff = min( - max_backoff, - base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)), - ) - if detail == "clean_close": - backoff = min(backoff, clean_close_max_backoff) - state.cooldown_until = max(state.cooldown_until, now + backoff) - if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: - http_bridge_retry_circuit_total.labels(outcome="opened").inc() - logger.warning( - "http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s " - "failures=%s cooldown_seconds=%.1f detail=%s", - session.key.affinity_kind, - _hash_identifier(session.key.affinity_key), - state.consecutive_failures, - backoff, - detail, + if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_recorded: + duplicate_attempt = scoped_attempt + elif scoped_attempt is not None and (scoped_attempt.disarmed or scoped_attempt.response_observed): + return None + else: + state = self._http_bridge_retry_circuits.setdefault( + session.key, + _HTTPBridgeRetryCircuitState(last_touched_monotonic=now), ) - await self._persist_http_bridge_retry_circuit(session, state) - async with self._http_bridge_retry_circuit_lock: - if self._http_bridge_retry_circuits.get(session.key) is state: - self._http_bridge_retry_circuit_loaded_keys.add(session.key) - return state.consecutive_failures + state.last_touched_monotonic = now + state.last_failure_monotonic = now + state.half_open_until = 0.0 + if scoped_attempt is not None: + scoped_attempt.retry_circuit_failure_recorded = True + scoped_attempt.retry_circuit_failure_settled = anyio.Event() + state.consecutive_failures += 1 + state.last_detail = detail + if state.consecutive_failures >= threshold: + backoff = min( + max_backoff, + base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)), + ) + if detail == "clean_close": + backoff = min(backoff, clean_close_max_backoff) + state.cooldown_until = max(state.cooldown_until, now + backoff) + if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None: + http_bridge_retry_circuit_total.labels(outcome="opened").inc() + logger.warning( + "http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s " + "failures=%s cooldown_seconds=%.1f detail=%s", + session.key.affinity_kind, + _hash_identifier(session.key.affinity_key), + state.consecutive_failures, + backoff, + detail, + ) + if duplicate_attempt is not None: + return await self._await_http_bridge_retry_circuit_attempt_settlement( + session, + attempt=duplicate_attempt, + detail=detail, + ) + assert state is not None + try: + await self._persist_http_bridge_retry_circuit(session, state) + async with self._http_bridge_retry_circuit_lock: + if self._http_bridge_retry_circuits.get(session.key) is state: + self._http_bridge_retry_circuit_loaded_keys.add(session.key) + consecutive_failures = state.consecutive_failures + return consecutive_failures + finally: + if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_settled is not None: + scoped_attempt.retry_circuit_failure_settled.set() async def _clear_http_bridge_retry_circuit(self: Any, session: _HTTPBridgeSession) -> None: if session.key.strength != "hard": diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index c7cb0ace1a..a4cbc518fd 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -86,6 +86,7 @@ _http_bridge_request_needs_unanchored_handoff, _http_bridge_request_stage, _http_bridge_requires_cluster_registration, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _http_bridge_runtime_config, _http_bridge_should_attempt_local_bootstrap_rebind, _http_bridge_should_attempt_local_previous_response_recovery, @@ -4014,6 +4015,9 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: if not completed_delivery_in_progress: keepalive_count += 1 if not completed_delivery_in_progress and keepalive_count >= max_keepalive_count: + timed_out_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests((request_state,)) + ) if not response_started: retried = False if not circuit_keepalive_waiting: @@ -4204,9 +4208,10 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: if keepalive_event is not None: yield keepalive_event continue - await self._record_http_bridge_retry_circuit_failure( + await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, detail="stream_idle_timeout", + selection=timed_out_retry_circuit_attempt_selection, ) if PROMETHEUS_AVAILABLE and stream_idle_timeout_total is not None: stream_idle_timeout_total.labels(surface="http_bridge").inc() diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 431aaea01a..3d8ba2d0fb 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -59,6 +59,7 @@ _http_bridge_eventless_precreated_deadline, _http_bridge_request_budget_seconds, _http_bridge_request_counts_against_queue, + _http_bridge_retry_circuit_attempt_selection_for_pending_requests, _log_http_bridge_event, _normalize_http_bridge_error_event, _record_http_bridge_stuck_retire, @@ -136,7 +137,9 @@ _clear_websocket_request_error_overrides, _event_type_from_payload, _HTTPBridgeCompletedDeliveryScope, + _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, + _mark_response_create_attempt_observed, _pop_websocket_deferred_reasoning_downstream_texts, _record_response_event, _signal_propagated_capacity_startup_ready, @@ -210,7 +213,6 @@ 120.0, ) _HTTP_BRIDGE_RECOVERY_SETTLEMENT_LEASE_REFRESH_INTERVAL_SECONDS = 10.0 - # A single missing response.created is not proof that an account is bad: the # upstream may have accepted the request while the transport was silent. Only # repeated failures on separate bridge retirements are allowed to influence @@ -910,6 +912,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( upstream_close_code: int | None = None, response_events_seen: int | None = None, transport_classification: str | None = None, + retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, ) -> bool: session.closed = True async with session.pending_lock: @@ -924,6 +927,13 @@ async def _fail_http_bridge_reader_and_maybe_retire( default=0, ) pending_request_states = list(session.pending_requests) + if retry_circuit_attempt_selection is None: + retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + pending_request_states + ) + retry_circuit_attempt_kwargs = { + "retry_circuit_attempt_selection": retry_circuit_attempt_selection, + } # The #1534 wedge shape: a reattached stream that streamed response # events whose ``response.created`` was never assigned. The eventless # watchdog and the durable-anchor clear both key on @@ -1041,9 +1051,10 @@ async def _fail_http_bridge_reader_and_maybe_retire( None, ) if failed_pending_count > 0 and retry_circuit_detail is not None: - consecutive_failures = await self._record_http_bridge_retry_circuit_failure( + consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, detail=retry_circuit_detail, + selection=retry_circuit_attempt_selection, ) poison_after_deferred_failures = bool( retry_circuit_detail == "stream_idle_timeout" @@ -1059,6 +1070,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( session, detail="repeated_zero_event_idle_timeout", response_events_seen=observed_response_events, + **retry_circuit_attempt_kwargs, ) force_retire = True else: @@ -1091,6 +1103,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( retry_circuit_detail="clean_close", response_events_seen=observed_response_events, retired_request_count=failed_pending_count, + **retry_circuit_attempt_kwargs, ) else: await self._retire_stale_pending_http_bridge_session( @@ -1104,6 +1117,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( # strike. The deferred/poison branch records its own # strike above and intentionally does not pass it. retired_request_count=failed_pending_count, + **retry_circuit_attempt_kwargs, ) return force_retire or session.admission_waiter_count == 0 @@ -1115,8 +1129,10 @@ async def _relay_http_bridge_upstream_messages( relay_upstream = session.upstream receive_task: asyncio.Task[UpstreamWebSocketMessage] | None = None wakeup_task: asyncio.Task[bool] | None = None + reader_failure_retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None try: while True: + reader_failure_retry_circuit_attempt_selection = None # Clear before taking the deadline snapshot. A send before the # clear is represented by its timestamp; a send after it leaves # the event set and wakes the persistent receive wait below. @@ -1204,6 +1220,12 @@ async def _relay_http_bridge_upstream_messages( ] if not expired_request_states: continue + expired_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + expired_request_states + ) + ) + reader_failure_retry_circuit_attempt_selection = expired_retry_circuit_attempt_selection pending_count = len(session.pending_requests) # A delta-only request has no other way to # convey prior context once its anchor is @@ -1261,6 +1283,7 @@ async def _relay_http_bridge_upstream_messages( penalize_account=False, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, + retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection, ) break # A successfully cancelled receive cannot deliver @@ -1306,9 +1329,17 @@ async def _relay_http_bridge_upstream_messages( penalize_account=False, retire_detail=_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, force_retire=True, + retry_circuit_attempt_selection=expired_retry_circuit_attempt_selection, ) break + async with session.pending_lock: + retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) + reader_failure_retry_circuit_attempt_selection = retry_circuit_attempt_selection if receive_task is not None: receive_cancelled = await _cancel_http_bridge_reader_child( receive_task, @@ -1326,6 +1357,7 @@ async def _relay_http_bridge_upstream_messages( session, error_code=receive_timeout.error_code, error_message=receive_timeout.error_message, + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) break @@ -1349,6 +1381,11 @@ async def _relay_http_bridge_upstream_messages( (request_state.response_event_count for request_state in session.pending_requests), default=0, ) + reader_failure_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) _archive_http_bridge_upstream_message(session, message, archive_request_state) session.last_upstream_close_generation += 1 session.last_upstream_close_code = message.close_code @@ -1389,6 +1426,7 @@ async def _relay_http_bridge_upstream_messages( if close_classification is not None else "websocket_transport_error" ), + retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection, penalize_account=( not account_neutral and not (message.kind == "close" and close_classification == "clean") ), @@ -1405,6 +1443,17 @@ async def _relay_http_bridge_upstream_messages( except asyncio.CancelledError: raise except Exception as exc: + if reader_failure_retry_circuit_attempt_selection is None: + # A receive/processing exception can jump here before the + # ordinary timeout or close branches publish their snapshot. + # Capture before waiting for lifecycle ownership so a + # concurrent recovery cannot replace the failed physical send. + async with session.pending_lock: + reader_failure_retry_circuit_attempt_selection = ( + _http_bridge_retry_circuit_attempt_selection_for_pending_requests( + tuple(session.pending_requests) + ) + ) logger.warning( "HTTP bridge upstream reader crashed account_id=%s bridge_kind=%s", session.account.id, @@ -1429,6 +1478,7 @@ async def _relay_http_bridge_upstream_messages( else "HTTP bridge upstream reader crashed before response.completed" ), penalize_account=not account_neutral, + retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection, # Preserve ordinary crash handoff behavior, but never hand # a heartbeat-expired socket to an admission waiter. **({"force_retire": True} if error_code == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE else {}), @@ -1623,6 +1673,12 @@ async def _process_parsed_http_bridge_upstream_event( pending_request_count = len(session.pending_requests) if matched_request_state is not None: + # The deferred reasoning prelude intentionally skips ordinary + # response-event accounting below, but it still proves that the + # physical response.create received an upstream response. Publish + # that attempt transition before any later recovery await can + # classify the send as eventless. + _mark_response_create_attempt_observed(matched_request_state, event_type) now = _service_time().monotonic() if matched_request_state.latency_first_upstream_event_ms is None: matched_request_state.latency_first_upstream_event_ms = int( diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 6dc428cc3c..80db95953f 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -818,6 +818,34 @@ class _DeferredAccountBackoffTracker: current_lifecycle: _DeferredAccountBackoffLifecycle | None = None +@dataclass(eq=False, slots=True) +class _HTTPBridgeResponseCreateAttempt: + ordinal: int + disarmed: bool = False + response_observed: bool = False + retry_circuit_failure_recorded: bool = False + retry_circuit_failure_settled: anyio.Event | None = None + + +@dataclass(frozen=True, slots=True) +class _HTTPBridgeRetryCircuitAttemptSelection: + kind: Literal["absent", "eligible", "recorded", "settled", "ineligible"] + attempts: tuple[_HTTPBridgeResponseCreateAttempt, ...] = () + + def __post_init__(self) -> None: + carries_attempts = self.kind in {"eligible", "recorded", "settled"} + if carries_attempts != bool(self.attempts): + raise ValueError(f"invalid retry-circuit attempt selection: {self.kind}") + + @property + def attempt(self) -> _HTTPBridgeResponseCreateAttempt | None: + return self.attempts[0] if len(self.attempts) == 1 else None + + @property + def ambiguous(self) -> bool: + return len(self.attempts) > 1 + + @dataclass class _WebSocketRequestState: request_id: str @@ -840,6 +868,8 @@ class _WebSocketRequestState: # send. Retries replace this value so admission wait and prior attempts do # not age a fresh send into the eventless owner deadline. response_create_sent_at: float | None = None + response_create_attempt_count: int = 0 + response_create_attempt: _HTTPBridgeResponseCreateAttempt | None = None bridge_queue_wait_started_at: float | None = None # Monotonic deadline of the original bridge request budget. Retry and # recovery paths re-prepare request states with a fresh started_at, so @@ -1355,9 +1385,21 @@ def _clear_websocket_deferred_reasoning_downstream_texts(request_state: _WebSock request_state.deferred_reasoning_downstream_texts = [] +def _mark_response_create_attempt_observed( + request_state: _WebSocketRequestState | None, + event_type: str | None, +) -> None: + if request_state is None or event_type is None or not event_type.startswith("response."): + return + attempt = request_state.response_create_attempt + if attempt is not None: + attempt.response_observed = True + + def _record_response_event(request_state: _WebSocketRequestState | None, event_type: str | None) -> None: if request_state is None or event_type is None or not event_type.startswith("response."): return + _mark_response_create_attempt_observed(request_state, event_type) request_state.last_upstream_activity_at = time.monotonic() if event_type in {"response.failed", "response.incomplete"}: return diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 18db15a76f..90f7c6b129 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -1302,25 +1302,19 @@ async def _acquire_request_state_response_create_admission( pending_request_ages_seconds: list[float] | None = None should_retire_stuck_session = False stale_pending_requests_to_fail: list[_WebSocketRequestState] = [] + retry_circuit_attempt_selection = None if bridge_session is not None: now = time.monotonic() - async with bridge_session.pending_lock: - pending_states = list(bridge_session.pending_requests) - pending_count = len(pending_states) - queued_count = bridge_session.queued_request_count + stale_gate_snapshot = await self._snapshot_http_bridge_stale_gate_state(bridge_session, now=now) + pending_states = stale_gate_snapshot.pending_states + pending_count = len(pending_states) + queued_count = stale_gate_snapshot.queued_count + threshold_seconds = stale_gate_snapshot.threshold_seconds + stale_pending_requests_to_fail = stale_gate_snapshot.stale_request_states + should_retire_stuck_session = stale_gate_snapshot.should_retire + retry_circuit_attempt_selection = stale_gate_snapshot.retry_circuit_attempt_selection pending_request_ids = [state.request_log_id or state.request_id for state in pending_states] pending_request_ages_seconds = [max(0.0, now - state.started_at) for state in pending_states] - threshold_seconds = float( - getattr(get_settings(), "http_responses_session_bridge_stuck_gate_retire_after_seconds", 300.0) - ) - stale_pending_requests_to_fail, should_retire_stuck_session = ( - self._classify_http_bridge_stale_gate_holders( - pending_states, - now=now, - threshold_seconds=threshold_seconds, - session_closed=bridge_session.closed, - ) - ) if not should_retire_stuck_session and any( max(0.0, now - state.started_at) >= threshold_seconds for state in pending_states ): @@ -1368,6 +1362,7 @@ async def _acquire_request_state_response_create_admission( bridge_session, stale_pending_requests_to_fail, detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) elif bridge_session is not None and should_retire_stuck_session: _record_http_bridge_stuck_retire( @@ -1377,6 +1372,7 @@ async def _acquire_request_state_response_create_admission( await self._retire_stale_pending_http_bridge_session( bridge_session, detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=retry_circuit_attempt_selection, ) raise _http_bridge_startup_wait_timeout_error( "http_bridge_response_create_gate", diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md new file mode 100644 index 0000000000..ccb42df761 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/design.md @@ -0,0 +1,128 @@ +# Design: attempt-scoped retry-circuit recording + +## Context + +All HTTP bridge upstream sends pass through +`_send_http_bridge_request_text_with_archive_id`, but retry-circuit failures can +be reported by four paths: partial stale cleanup, direct stale retirement, the +upstream reader failure funnel, and downstream stream-idle handling. These +paths may run concurrently and may await recovery or settlement before they +record the failure. + +The durable retry-circuit upsert intentionally merges separate writes as +separate observations. It cannot infer that two writes came from the same +physical send, and adding a durable attempt key would expand the schema and the +rolling-upgrade contract unnecessarily. + +## Decisions + +### Keep identity process-local and object-scoped + +Each upstream send creates a new attempt object stored on its request state. +The object carries a diagnostic ordinal plus `disarmed`, `response_observed`, +and `retry_circuit_failure_recorded` state. It also carries a settlement signal, +but deliberately does not cache a historical failure count. Observers capture +the object itself, not merely the request's current ordinal. + +An older observer therefore retains the identity of the send it classified even +if a retry replaces the request state's current attempt. The old object remains +alive only while an observer references it, so no unbounded generation set is +needed. + +### Preserve the existing failure eligibility contract + +Creating an attempt does not itself record a failure. A send exception or +cancellation disarms it using the same cleanup boundary that clears +`response_create_sent_at`. A matched `response.*` event marks it observed before +the reader performs another await. An observer that has not already recorded +the attempt must not record it after either condition wins. + +If the failure was already recorded, later duplicate observers wait for its +durable merge to settle and then read the live circuit state without another +increment. This means a later independent attempt is reflected in the returned +count, while a successful response that cleared the circuit is reported as +zero. It preserves the reader's existing threshold-dependent durable-anchor +handling without allowing a cached count from an older send to reopen or poison +state that has since changed. + +### Distinguish absent attribution from ambiguous attribution + +Failure funnels pass an explicit selection result rather than overloading +`None`. `absent` means the legacy path has no attempt object and may use the +unscoped recorder. `eligible`, `recorded`, and `settled` retain the exact object +identities, including multiple candidates. `ineligible` means an attempt was +present but lifecycle evidence makes it unsafe to charge. + +A single candidate is handled with the normal attempt-scoped recorder. Multiple +eligible or settled candidates are deliberately suppressed rather than falling +back to an unscoped strike, because an unscoped increment cannot identify which +physical send it represents and can double-count a later observer. Multiple +already-recorded candidates wait for settlement and return the live circuit +count without incrementing. + +### Claim under the existing retry-circuit lock + +The attempt marker and `consecutive_failures` increment are changed in the same +critical section guarded by `_http_bridge_retry_circuit_lock`. Durable I/O stays +outside that lock. Duplicate calls may both perform the existing durable load, +but only the first claim persists a failure. + +No new lock is introduced. Failure paths release `pending_lock` before entering +the recorder, and no retry-circuit path acquires `pending_lock` or +`lifecycle_lock` while holding the retry-circuit lock. + +### Capture before ownership and recovery awaits + +The response-create gate classifies stale owners and snapshots their attempts +while holding `pending_lock`. Shared cleanup also snapshots before it waits to +acquire that lock, so callers that do not provide a locked snapshot still retain +the pre-wait identity. The downstream timeout and reader watchdog likewise +capture before calling retry, reconnect, receive cancellation, or settlement +helpers. Reading the request state's current attempt afterward could attribute +an old timeout to a newer retry or suppress the old failure incorrectly. + +### Mark lifecycle observation before deferred delivery + +Some reasoning prelude events are intentionally deferred and therefore do not +increment the ordinary response-event counter. They still prove that upstream +accepted and began answering the physical `response.create`. The matched +attempt is marked observed immediately, before deferred-delivery branching or +any later await, while the existing event-count and downstream-visibility +semantics remain unchanged. + +### Keep replica behavior unchanged + +The active owner alone holds the upstream WebSocket and its request state, so +duplicate local observers share one attempt object. Owner forwarding does not +create another upstream send on the forwarding replica. A replay after owner +handoff is a new send and is intentionally a new strike. Existing durable +conflict merging continues to combine genuinely independent replica failures. + +## Failure Modes + +- If durable lookup or persistence fails, the first claim remains in local + circuit state as it does today; a duplicate observer must not retry the write + because the durable upsert would interpret it as a second failure. +- If a response event wins before the first claim, the attempt is not counted. + If a failure claim wins first, a later response cannot turn a duplicate + observer into another strike. +- If a successful terminal response clears the circuit before a delayed + duplicate observer resumes, the retained attempt marker prevents the old + observer from recreating the cleared failure and the live count returned to + it is zero. +- If a cleanup snapshot contains multiple eligible sends, it records none at + that ambiguous boundary. Later observers that retain an exact send identity + can still claim each genuine failure independently. +- If response accounting is deferred for a reasoning prelude, the attempt's + observed marker still wins against an eventless timeout without making the + deferred event visible or incrementing its ordinary event count. + +## Example + +Attempt A is sent and remains eventless. The downstream stream watchdog and the +reader watchdog both capture A. The downstream task claims A first, records +failure count 1, and persists once. The reader later sees that A is already +recorded, waits for settlement, reads the live count, and does not persist. If +recovery sends attempt B and B also fails before that reader resumes, the reader +returns the current count 2 without adding a third strike. If a successful +response clears the state instead, it returns 0. diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md new file mode 100644 index 0000000000..9f4dc9cd0c --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/proposal.md @@ -0,0 +1,71 @@ +# Deduplicate HTTP bridge retry-circuit failures by send attempt + +## Summary + +An eventless HTTP bridge `response.create` can be observed by both the upstream +reader watchdog and the downstream stream-idle watchdog. Each observer currently +persists an independent retry-circuit failure even though both are reporting the +same upstream send. With the default threshold of two, one silent send can +therefore open the circuit and make a later request fail locally with HTTP 503. + +Track the individual process-local upstream send attempt and let all failure +observers claim that attempt through the existing retry-circuit lock. The first +eligible observer records and persists the failure; later observers of the same +attempt reuse the resulting count without incrementing or persisting again. + +## Why + +The retry circuit is intended to protect a hard-affinity key after repeated +failures. A single upstream send observed through two local timeout paths is one +failure, not two. Durable conflict merging deliberately treats independent +persistence calls as independent failures, so deduplication must happen before +the durable write. + +A time-window or session-key dedupe would hide legitimate retries. A single +"last generation" marker is also insufficient because an observer for an older +send may resume after a newer send has started. A stable object per send keeps +old and new attempts distinct without adding a durable identifier or an +unbounded process-level set. + +## What Changes + +- Create a process-local attempt object immediately before every HTTP bridge + upstream `response.create` send. +- Disarm that attempt when the send fails or is cancelled, and mark it observed + when a matching upstream response lifecycle event wins the race. +- Capture the attempt at the moment a watchdog classifies a timeout, before any + pending-ownership, recovery, reconnect, or cleanup await can install a newer + attempt. +- Pass an explicit absent/eligible/recorded/settled/ineligible selection through + all retry-circuit failure funnels so ambiguous attribution cannot become an + unscoped strike. +- Atomically claim the attempt and increment the circuit under the existing + retry-circuit lock; only the first claim performs durable persistence. +- Let duplicate observers wait for settlement and then read the live circuit + count instead of caching a historical count on the attempt. +- Mark matched response lifecycle events on the attempt even when reasoning + prelude delivery and ordinary event accounting are deferred. +- Emit low-cardinality observability when a duplicate observer is suppressed. + +## Impact + +- One eventless send contributes at most one consecutive retry-circuit failure. +- A separately dispatched retry or replay remains a distinct eligible failure + and can open the circuit as the second strike. +- A delayed observer sees later independent failures or a successful clear in + the current circuit state without adding another strike. +- Ambiguous multi-pending cleanup fails safe by undercounting at that boundary, + never by creating an unattributed failure that could double-count a send. +- Existing circuit thresholds, cooldowns, error envelopes, account-health + handling, continuity guards, and durable conflict merging remain unchanged. +- No schema migration, runtime setting, or operator action is required. + +## Non-Goals + +- Determining or eliminating the upstream cause of an eventless send. +- Changing the eventless timeout, stream-idle timeout, retry threshold, or + cooldown durations. +- Adding cross-replica send-attempt identifiers. Only the active bridge owner + owns the upstream socket; a send after owner handoff is a new physical attempt. +- Changing replay eligibility, account selection, or continuity fail-closed + behavior. diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..956988ce85 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/specs/responses-api-compat/spec.md @@ -0,0 +1,95 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: HTTP bridge retry circuits count each upstream send attempt at most once + +For an HTTP Responses bridge request, multiple local failure observers that +classify the same upstream `response.create` send attempt MUST contribute at +most one consecutive retry-circuit failure and at most one durable failure +persistence operation. A separately dispatched retry or replay MUST be treated +as a new send attempt and MAY contribute the next eligible failure under the +existing retry-circuit policy. + +The proxy MUST capture the attempt being classified before awaiting recovery, +reconnection, settlement, or pending-request ownership that can dispatch a +newer attempt. When stale ownership is classified while holding the pending +lock, the classified request set and its attempt selection MUST come from that +same snapshot. A send attempt that is disarmed by send-failure or cancellation +cleanup, or that observes a matching upstream response lifecycle event before +its first failure claim, MUST NOT add a retry-circuit failure. A matched +response lifecycle event MUST mark its attempt observed even when downstream +delivery or ordinary response-event accounting is intentionally deferred. + +The proxy MUST distinguish a failure path with no attempt identity from one +whose attempt identity is present but ineligible or ambiguous. Only the former +MAY preserve legacy unscoped recording. An ineligible attempt or multiple +eligible candidates MUST NOT fall back to an unscoped failure. Duplicate +observers MUST wait for the first claim's settlement and then use the current +circuit count; they MUST NOT expose a cached historical count after a later +failure or successful clear. Deduplication MUST NOT change existing failure +classes, thresholds, cooldowns, continuity guards, or cross-replica conflict +merging. + +#### Scenario: reader and downstream watchdogs observe one eventless send + +- **GIVEN** one hard-affinity HTTP bridge `response.create` send remains eventless +- **AND** the upstream reader watchdog and downstream stream-idle watchdog both classify that send +- **WHEN** both observers report the retry-circuit failure +- **THEN** the circuit's consecutive failure count increases by exactly one +- **AND** the failure is durably persisted exactly once +- **AND** the default two-failure circuit does not open from that send alone + +#### Scenario: a separately dispatched retry is a second failure + +- **GIVEN** one send attempt has already contributed one retry-circuit failure +- **WHEN** a later retry or replay dispatches a new `response.create` and that attempt also fails eligibility checks +- **THEN** the new attempt contributes a second failure +- **AND** the existing threshold and cooldown behavior may open the circuit + +#### Scenario: a delayed old observer cannot count a newer attempt + +- **GIVEN** an observer captured attempt A before recovery dispatched attempt B +- **AND** attempt A has already contributed its failure +- **WHEN** the delayed observer resumes after attempt B is current +- **THEN** it does not increment or persist another failure for attempt A +- **AND** it does not mark attempt B as recorded +- **AND** it observes the current circuit count, including attempt B's independent failure + +#### Scenario: an upstream response wins the timeout race + +- **GIVEN** a watchdog is evaluating an eventless send attempt +- **WHEN** a matching upstream response lifecycle event is observed before the attempt's first failure claim +- **THEN** that attempt does not contribute a retry-circuit failure + +#### Scenario: a deferred reasoning prelude wins the timeout race + +- **GIVEN** a matched reasoning lifecycle event is held for deferred downstream delivery +- **AND** ordinary response-event accounting remains zero for that prelude +- **WHEN** an eventless failure observer evaluates the same send attempt +- **THEN** the attempt is already marked as response-observed +- **AND** it does not contribute or persist a retry-circuit failure +- **AND** deferred-delivery and downstream-visibility behavior remain unchanged + +#### Scenario: multiple pending attempts are ambiguous at a shared failure boundary + +- **GIVEN** a shared cleanup boundary contains multiple distinct eligible send attempts +- **WHEN** the boundary cannot attribute its failure to exactly one physical send +- **THEN** it does not fall back to an unscoped retry-circuit failure +- **AND** it does not mark any candidate attempt as recorded +- **AND** a later observer with an exact attempt identity can still record each genuine failure independently + +#### Scenario: pending-lock wait cannot replace the classified attempt + +- **GIVEN** stale cleanup captures attempt A for a request before acquiring pending ownership +- **AND** recovery installs attempt B while cleanup is waiting for the pending lock +- **WHEN** cleanup later records the classified failure +- **THEN** it retains attempt A's identity +- **AND** it does not mark attempt B as recorded + +#### Scenario: a cleared circuit is not recreated by a delayed duplicate + +- **GIVEN** a send attempt contributed a failure and a later successful terminal response cleared the circuit +- **WHEN** another observer of the old send attempt resumes +- **THEN** the old observer does not recreate or persist the cleared failure +- **AND** it receives the current circuit count of zero diff --git a/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md new file mode 100644 index 0000000000..7d842dcc64 --- /dev/null +++ b/openspec/changes/dedupe-http-bridge-retry-circuit-attempts/tasks.md @@ -0,0 +1,28 @@ +## 1. Specification + +- [x] 1.1 Add the attempt-scoped retry-circuit requirement and race scenarios. +- [x] 1.2 Validate the change in strict mode. + +## 2. Implementation + +- [x] 2.1 Add the process-local HTTP bridge send-attempt state and lifecycle transitions. +- [x] 2.2 Capture and thread the classified attempt through all retry-circuit failure paths. +- [x] 2.3 Claim the attempt atomically with the circuit increment and suppress duplicate persistence. +- [x] 2.4 Add low-cardinality duplicate-suppression observability without new settings or schema. +- [x] 2.5 Represent absent, ineligible, and ambiguous attempt attribution explicitly; never use ambiguous `None` as an unscoped fallback. +- [x] 2.6 Read live circuit state after duplicate settlement and mark deferred response lifecycle events observed without changing delivery accounting. + +## 3. Coverage + +- [x] 3.1 Add a full HTTP bridge regression where reader and downstream watchdogs observe one send. +- [x] 3.2 Prove a new send is a new strike and a delayed observer of an old send is not. +- [x] 3.3 Cover response-wins, send-failure/cancellation, successful reset, and multiple-pending races. +- [x] 3.4 Preserve clean-close, continuity, owner-handoff, and durable conflict-merge coverage. +- [x] 3.5 Cover pending-lock attempt replacement, ambiguous selection suppression, deferred reasoning observation, and live-count changes after later failures or clear. + +## 4. Verification + +- [x] 4.1 Run focused HTTP bridge and durable retry-circuit tests. +- [x] 4.2 Run Ruff, formatting checks, and the full unit suite. +- [x] 4.3 Validate the OpenSpec change strictly and validate all main specs. +- [x] 4.4 Review the final diff for lock ordering, persistence cardinality, scope creep, and rollback compatibility. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index bc361b6f56..e581ee7087 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -938,7 +938,15 @@ def test_http_bridge_eventless_precreated_deadline_survives_reasoning_prelude_wi async def test_process_http_bridge_upstream_text_anchors_deferred_reasoning_prelude_without_created() -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) request_state = _make_eventless_http_bridge_owner(sent_at=time.monotonic() - 2.0) + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt session = _make_bridge_session(pending_requests=deque([request_state]), queued_request_count=1) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) await service._process_http_bridge_upstream_text( session, @@ -955,6 +963,7 @@ async def test_process_http_bridge_upstream_text_anchors_deferred_reasoning_prel assert request_state.response_id is None assert request_state.downstream_visible is False assert request_state.upstream_model_output_seen is True + assert attempt.response_observed is True assert request_state.last_upstream_activity_at is not None sent_at = request_state.response_create_sent_at assert sent_at is not None @@ -968,6 +977,22 @@ async def test_process_http_bridge_upstream_text_anchors_deferred_reasoning_prel == request_state.last_upstream_activity_at + http_bridge_helpers_module._HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (request_state,) + ) + assert selection.kind == "settled" + assert selection.attempt is attempt + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() @pytest.mark.asyncio @@ -998,6 +1023,7 @@ async def send_text(_text: str) -> None: request_state, "first", ) + first_attempt = request_state.response_create_attempt session.upstream_reader_wakeup.clear() await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( session, @@ -1007,6 +1033,11 @@ async def send_text(_text: str) -> None: assert seen_sent_ats == [100.0, 200.0] assert request_state.response_create_sent_at == 200.0 + assert first_attempt is not None + assert first_attempt.ordinal == 1 + assert request_state.response_create_attempt is not first_attempt + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.ordinal == 2 assert session.upstream_reader_wakeup.is_set() is True @@ -1042,6 +1073,8 @@ async def send_text(_text: str) -> None: ) assert request_state.response_create_sent_at is None + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.disarmed is True assert session.upstream_reader_wakeup.is_set() is True assert ( http_bridge_helpers_module._http_bridge_eventless_precreated_deadline( @@ -2501,7 +2534,9 @@ async def fake_retire( retire_session: proxy_service._HTTPBridgeSession, *, detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" retire_calls.append(detail) retire_session.closed = True @@ -2608,7 +2643,9 @@ async def fake_retire( retire_session: proxy_service._HTTPBridgeSession, *, detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" retire_calls.append(detail) retire_session.closed = True @@ -2697,7 +2734,9 @@ async def fake_retire( retire_session: proxy_service._HTTPBridgeSession, *, detail: str, + retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, ) -> None: + assert retry_circuit_attempt_selection.kind == "absent" retire_calls.append(detail) retire_session.closed = True @@ -24326,6 +24365,119 @@ async def close(self) -> None: session=session, ) assert "http_bridge_event event=missing_response_created_timeout" in caplog.text + if leading_telemetry: + assert session.key not in service._http_bridge_retry_circuits + else: + assert service._http_bridge_retry_circuits[session.key].consecutive_failures == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_stream_and_reader_count_one_eventless_send_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + upstream = _SilentEventlessUpstream() + session = _make_bridge_session(key_value="eventless-observer-race") + session.upstream = cast(UpstreamWebSocket, upstream) + service._http_bridge_sessions[session.key] = session + settings = _make_app_settings( + sse_keepalive_interval_seconds=0.02, + stream_idle_timeout_seconds=0.3, + http_responses_session_bridge_request_budget_seconds=1.0, + http_responses_session_bridge_stuck_gate_retire_after_seconds=0.005, + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(http_bridge_streaming_module, "_stream_keepalive_max_count", lambda: 1) + monkeypatch.setattr(proxy_service, "_HTTP_BRIDGE_STARTUP_KEEPALIVE_GRACE_SECONDS", 0.001) + monkeypatch.setattr(service, "_handle_stream_error", AsyncMock()) + monkeypatch.setattr(service, "_write_request_log", AsyncMock()) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = cast( + Any, + SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ), + ) + + async def submit( + target_session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + ) -> None: + del queue_limit + gate = target_session.response_create_gate + await gate.acquire() + request_state.response_create_gate = gate + request_state.response_create_gate_acquired = True + request_state.awaiting_response_created = True + request_state.request_text = text_data + async with target_session.pending_lock: + target_session.pending_requests.append(request_state) + target_session.queued_request_count = 1 + await http_bridge_request_submit_module._send_http_bridge_request_text_with_archive_id( + target_session, + request_state, + text_data, + ) + + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + request_state = _make_eventless_http_bridge_owner(request_id="req-eventless-observer-race", sent_at=0.0) + request_state.started_at = time.monotonic() + request_state.response_create_sent_at = None + request_state.response_create_gate = None + request_state.response_create_gate_acquired = False + reader_retry_started = asyncio.Event() + release_reader_retry = asyncio.Event() + + async def retry_precreated( + _session: proxy_service._HTTPBridgeSession, + *, + restart_reader: bool = False, + ) -> bool: + if restart_reader: + await asyncio.wait_for(reader_retry_started.wait(), timeout=2.0) + return False + request_state.response_create_sent_at = None + reader_retry_started.set() + await asyncio.wait_for(release_reader_retry.wait(), timeout=2.0) + return False + + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + + reader_task = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) + await asyncio.wait_for(upstream.first_receive_started.wait(), timeout=0.5) + stream = service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create","model":"gpt-5.6-sol","input":"hello"}', + queue_limit=8, + propagate_http_errors=False, + downstream_turn_state=None, + ) + try: + terminal_task = asyncio.create_task(anext(stream)) + await asyncio.wait_for(reader_retry_started.wait(), timeout=1.0) + terminal = await asyncio.wait_for(terminal_task, timeout=1.0) + assert '"code":"stream_idle_timeout"' in terminal + release_reader_retry.set() + await asyncio.wait_for(reader_task, timeout=1.0) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert state.consecutive_failures == 1 + assert state.cooldown_until == 0.0 + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 1 + finally: + release_reader_retry.set() + if not reader_task.done(): + reader_task.cancel() + with pytest.raises(asyncio.CancelledError): + await reader_task + await stream.aclose() @pytest.mark.asyncio @@ -25100,6 +25252,7 @@ async def test_http_bridge_liveness_timeout_is_neutral_not_replayed_and_forces_r detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, retired_request_count=1, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) assert session.queued_request_count == 0 assert session.closed is True @@ -25246,12 +25399,17 @@ async def controlled_fail_reader( assert session.queued_request_count == 0 assert session.closed is True assert session.liveness_settlement_owner == "send" - retire.assert_awaited_once_with( - session, - detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, - response_events_seen=0, - retired_request_count=2, - ) + retire.assert_awaited_once() + retire_call = retire.await_args + assert retire_call is not None + assert retire_call.args == (session,) + assert retire_call.kwargs["detail"] == UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE + assert retire_call.kwargs["response_events_seen"] == 0 + assert retire_call.kwargs["retired_request_count"] == 2 + assert request_state.response_create_attempt is not None + retry_circuit_attempt_selection = retire_call.kwargs["retry_circuit_attempt_selection"] + assert retry_circuit_attempt_selection.attempt is request_state.response_create_attempt + assert request_state.response_create_attempt.disarmed is True @pytest.mark.asyncio @@ -25374,6 +25532,7 @@ def pending_sibling(request_id: str) -> proxy_service._WebSocketRequestState: detail=UPSTREAM_WEBSOCKET_LIVENESS_TIMEOUT_CODE, response_events_seen=0, retired_request_count=2, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) @@ -25393,6 +25552,10 @@ async def test_http_bridge_retry_send_network_failure_is_neutral_and_not_replaye request_text='{"type":"response.create","model":"gpt-5.4","input":"hello"}', transport="http", ) + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt_count = 1 + request_state.response_create_attempt = first_attempt + request_state.response_create_sent_at = time.monotonic() session = _make_bridge_session( key_value="bridge-retry-send-network", pending_requests=deque([request_state]), @@ -25431,15 +25594,22 @@ async def fail_reader( await service._relay_http_bridge_upstream_messages(session) - assert failure_calls == [ - { - "error_code": "proxy_network_unavailable", - "error_message": "Codex upstream websocket send failed: OSError", - "penalize_account": False, - } - ] + assert len(failure_calls) == 1 + failure_call = failure_calls[0] + assert failure_call["error_code"] == "proxy_network_unavailable" + assert failure_call["error_message"] == "Codex upstream websocket send failed: OSError" + assert failure_call["penalize_account"] is False + retry_circuit_attempt_selection = failure_call["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is first_attempt assert request_state.replay_count == 1 retry_send.assert_awaited_once() + assert request_state.response_create_attempt is not first_attempt + assert request_state.response_create_attempt is not None + assert request_state.response_create_attempt.disarmed is True assert session.closed is True @@ -25498,9 +25668,142 @@ async def test_http_bridge_clean_close_before_response_does_not_penalize_account detail="stream_incomplete", response_events_seen=0, retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) +@pytest.mark.asyncio +async def test_http_bridge_clean_close_retry_failure_preserves_pre_recovery_attempt( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-clean-close-retry-attempt") + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="bridge-clean-close-retry-attempt", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1000)), + close=AsyncMock(), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + + async def retry_precreated(target_session: Any, **_kwargs: object) -> bool: + assert target_session is session + request_state.response_create_attempt = replacement_attempt + request_state.response_create_sent_at = None + return False + + failure_calls: list[dict[str, object]] = [] + + async def fail_reader(target_session: Any, **kwargs: object) -> bool: + assert target_session is session + failure_calls.append(dict(kwargs)) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", retry_precreated) + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + await service._relay_http_bridge_upstream_messages(session) + + assert len(failure_calls) == 1 + retry_circuit_attempt_selection = failure_calls[0]["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is original_attempt + + +@pytest.mark.asyncio +async def test_http_bridge_reader_exception_captures_attempt_before_lifecycle_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-reader-exception-attempt") + request_state.started_at = time.monotonic() + request_state.response_create_sent_at = None + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="bridge-reader-exception-attempt", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="text", + text='{"type":"response.created","response":{"id":"resp_reader_exception"}}', + ) + ), + close=AsyncMock(), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr( + service, + "_process_http_bridge_upstream_text", + AsyncMock(side_effect=RuntimeError("processing failed")), + ) + attempt_captured = asyncio.Event() + original_selector = ( + http_bridge_upstream_events_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests + ) + + def capture_attempt_before_lifecycle_wait( + request_states: tuple[proxy_service._WebSocketRequestState, ...], + ) -> proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection: + selection = original_selector(request_states) + attempt_captured.set() + return selection + + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_http_bridge_retry_circuit_attempt_selection_for_pending_requests", + capture_attempt_before_lifecycle_wait, + ) + failure_calls: list[dict[str, object]] = [] + + async def fail_reader( + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> bool: + assert target_session is session + failure_calls.append(dict(kwargs)) + target_session.closed = True + return True + + monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", fail_reader) + + async with session.lifecycle_lock: + relay_task = asyncio.create_task(service._relay_http_bridge_upstream_messages(session)) + await asyncio.wait_for(attempt_captured.wait(), timeout=1.0) + request_state.response_create_attempt = replacement_attempt + + await asyncio.wait_for(relay_task, timeout=1.0) + + assert len(failure_calls) == 1 + retry_circuit_attempt_selection = failure_calls[0]["retry_circuit_attempt_selection"] + assert isinstance( + retry_circuit_attempt_selection, + proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + ) + assert retry_circuit_attempt_selection.attempt is original_attempt + assert replacement_attempt.retry_circuit_failure_recorded is False + + @pytest.mark.asyncio @pytest.mark.parametrize("routed", [False, True], ids=["direct-close", "routed-receive-error"]) async def test_http_bridge_reader_maps_ordinary_websocket_receive_failure_to_stream_incomplete( @@ -26261,6 +26564,470 @@ async def test_http_bridge_retry_circuit_counts_stream_idle_timeout() -> None: assert cast(Any, service)._http_bridge_retry_circuits[hard_session.key].consecutive_failures == 1 +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_claims_one_attempt_once_across_concurrent_observers() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-concurrent") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + both_lookups_started = asyncio.Event() + lookup_count = 0 + + async def lookup_retry_circuit(**_kwargs: object) -> None: + nonlocal lookup_count + lookup_count += 1 + if lookup_count == 2: + both_lookups_started.set() + await asyncio.wait_for(both_lookups_started.wait(), timeout=0.5) + + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(side_effect=lookup_retry_circuit), + persist_retry_circuit=persist_retry_circuit, + ) + + results = await asyncio.gather( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ), + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ), + ) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert results == [1, 1] + assert state.consecutive_failures == 1 + assert state.cooldown_until == 0.0 + assert attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 1 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_duplicate_waits_for_persisted_merge( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-persist-merge") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + persist_started = asyncio.Event() + allow_persist = asyncio.Event() + duplicate_wait_started = asyncio.Event() + + async def persist_retry_circuit(**_kwargs: object) -> SimpleNamespace: + persist_started.set() + await asyncio.wait_for(allow_persist.wait(), timeout=0.5) + return SimpleNamespace( + consecutive_failures=2, + cooldown_until_epoch=time.time() + 60.0, + last_detail="stream_idle_timeout", + updated_at_epoch=time.time(), + ) + + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(side_effect=persist_retry_circuit), + ) + await_attempt_settlement = service._await_http_bridge_retry_circuit_attempt_settlement + + async def track_attempt_settlement( + target_session: proxy_service._HTTPBridgeSession, + *, + attempt: proxy_support_module._HTTPBridgeResponseCreateAttempt, + detail: str, + ) -> int: + duplicate_wait_started.set() + return await await_attempt_settlement( + target_session, + attempt=attempt, + detail=detail, + ) + + monkeypatch.setattr( + service, + "_await_http_bridge_retry_circuit_attempt_settlement", + track_attempt_settlement, + ) + + first_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(persist_started.wait(), timeout=0.5) + duplicate_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(duplicate_wait_started.wait(), timeout=0.5) + assert duplicate_task.done() is False + + allow_persist.set() + + assert await asyncio.wait_for(first_task, timeout=0.5) == 2 + assert await asyncio.wait_for(duplicate_task, timeout=0.5) == 2 + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert state.consecutive_failures == 2 + assert attempt.retry_circuit_failure_settled is not None + assert attempt.retry_circuit_failure_settled.is_set() is True + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_duplicate_does_not_retry_failed_persistence( + caplog: pytest.LogCaptureFixture, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-persist-failure") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + persist_retry_circuit = AsyncMock(side_effect=RuntimeError("durable write unavailable")) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + + with caplog.at_level(logging.WARNING): + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + duplicate_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + + assert (first_count, duplicate_count) == (1, 1) + assert cast(Any, service)._http_bridge_retry_circuits[session.key].consecutive_failures == 1 + assert persist_retry_circuit.await_count == 1 + assert "Failed to persist HTTP bridge retry circuit" in caplog.text + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_counts_distinct_send_attempts_and_not_delayed_old_observer() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-generations") + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + second_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + delayed_first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + + state = cast(Any, service)._http_bridge_retry_circuits[session.key] + assert (first_count, second_count, delayed_first_count) == (1, 2, 2) + assert state.consecutive_failures == 2 + assert state.cooldown_until > time.monotonic() + assert first_attempt.retry_circuit_failure_recorded is True + assert second_attempt.retry_circuit_failure_recorded is True + assert persist_retry_circuit.await_count == 2 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_does_not_claim_attempt_when_response_wins_lookup_race() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-response-race") + request_state = _make_eventless_http_bridge_owner() + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt + lookup_started = asyncio.Event() + release_lookup = asyncio.Event() + + async def lookup_retry_circuit(**_kwargs: object) -> None: + lookup_started.set() + await asyncio.wait_for(release_lookup.wait(), timeout=0.5) + + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(side_effect=lookup_retry_circuit), + persist_retry_circuit=persist_retry_circuit, + ) + record_task = asyncio.create_task( + service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + ) + await asyncio.wait_for(lookup_started.wait(), timeout=0.5) + + proxy_support_module._record_response_event(request_state, "response.created") + release_lookup.set() + + assert await asyncio.wait_for(record_task, timeout=0.5) is None + assert attempt.response_observed is True + assert attempt.retry_circuit_failure_recorded is False + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ignores_disarmed_send_attempt() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-disarmed") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1, disarmed=True) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + + result = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + + assert result is None + assert attempt.retry_circuit_failure_recorded is False + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_delayed_duplicate_does_not_recreate_cleared_state() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-reset-race") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + == 1 + ) + await service._clear_http_bridge_retry_circuit(session) + lookup_count_after_clear = lookup_retry_circuit.await_count + + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=attempt, + ) + == 0 + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert lookup_retry_circuit.await_count == lookup_count_after_clear + assert persist_retry_circuit.await_count == 1 + + +def test_http_bridge_retry_circuit_attempt_selection_prefers_one_eventless_owner() -> None: + eventless = _make_eventless_http_bridge_owner(request_id="req-attempt-eventless") + eventless_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + eventless.response_create_attempt = eventless_attempt + responded = _make_eventless_http_bridge_owner(request_id="req-attempt-responded") + responded_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt( + ordinal=1, + response_observed=True, + ) + responded.response_create_attempt = responded_attempt + responded.response_event_count = 1 + responded.response_id = "resp-attempt-responded" + + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (responded, eventless) + ) + assert selection.kind == "eligible" + assert selection.attempt is eventless_attempt + + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (responded,) + ) + assert selection.kind == "settled" + assert selection.attempt is responded_attempt + + reconnecting = _make_eventless_http_bridge_owner(request_id="req-attempt-reconnecting") + reconnecting_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + reconnecting.response_create_attempt = reconnecting_attempt + reconnecting.response_create_sent_at = None + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (reconnecting,) + ) + assert selection.kind == "eligible" + assert selection.attempt is reconnecting_attempt + + other_eventless = _make_eventless_http_bridge_owner(request_id="req-attempt-other-eventless") + other_eventless_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + other_eventless.response_create_attempt = other_eventless_attempt + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (eventless, other_eventless) + ) + assert selection.kind == "eligible" + assert selection.ambiguous is True + assert selection.attempt is None + assert selection.attempts == (eventless_attempt, other_eventless_attempt) + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ambiguous_attempts_never_fall_back_to_unscoped_failure() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-ambiguous") + first_request = _make_eventless_http_bridge_owner(request_id="req-attempt-ambiguous-first") + second_request = _make_eventless_http_bridge_owner(request_id="req-attempt-ambiguous-second") + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + first_request.response_create_attempt = first_attempt + second_request.response_create_attempt = second_attempt + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (first_request, second_request) + ) + + assert selection.kind == "eligible" + assert selection.ambiguous is True + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert first_attempt.retry_circuit_failure_recorded is False + assert second_attempt.retry_circuit_failure_recorded is False + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + first_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + second_count = await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + + assert (first_count, second_count) == (1, 2) + assert persist_retry_circuit.await_count == 2 + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_ineligible_attempt_never_falls_back_to_unscoped_failure() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-ineligible") + request_state = _make_eventless_http_bridge_owner(request_id="req-attempt-ineligible") + attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + request_state.response_create_attempt = attempt + request_state.response_event_count = 1 + lookup_retry_circuit = AsyncMock(return_value=None) + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=lookup_retry_circuit, + persist_retry_circuit=persist_retry_circuit, + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (request_state,) + ) + + assert selection.kind == "ineligible" + assert selection.attempt is None + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + is None + ) + assert session.key not in cast(Any, service)._http_bridge_retry_circuits + assert attempt.retry_circuit_failure_recorded is False + lookup_retry_circuit.assert_not_awaited() + persist_retry_circuit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_retry_circuit_multiple_recorded_attempts_report_live_count_without_increment() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-attempt-recorded-selection") + first_request = _make_eventless_http_bridge_owner(request_id="req-attempt-recorded-first") + second_request = _make_eventless_http_bridge_owner(request_id="req-attempt-recorded-second") + first_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + second_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + first_request.response_create_attempt = first_attempt + second_request.response_create_attempt = second_attempt + persist_retry_circuit = AsyncMock(return_value=None) + service._durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=persist_retry_circuit, + ) + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=first_attempt, + ) + == 1 + ) + assert ( + await service._record_http_bridge_retry_circuit_failure( + session, + detail="stream_idle_timeout", + attempt=second_attempt, + ) + == 2 + ) + selection = http_bridge_helpers_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests( + (first_request, second_request) + ) + + assert selection.kind == "recorded" + assert selection.ambiguous is True + assert ( + await service._record_http_bridge_retry_circuit_failure_for_attempt_selection( + session, + detail="stream_idle_timeout", + selection=selection, + ) + == 2 + ) + assert cast(Any, service)._http_bridge_retry_circuits[session.key].consecutive_failures == 2 + assert persist_retry_circuit.await_count == 2 + + @pytest.mark.parametrize( ("error_code", "expected_ambiguous"), [ @@ -26722,6 +27489,7 @@ async def test_http_bridge_repeated_zero_event_idle_timeouts_poison_anchor_with_ session, detail="repeated_zero_event_idle_timeout", response_events_seen=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) @@ -26806,6 +27574,7 @@ async def test_http_bridge_eventless_timeout_force_retires_with_admission_waiter detail="missing_response_created_timeout", response_events_seen=0, retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) fail_pending_await_args = fail_pending.await_args assert fail_pending_await_args is not None @@ -26835,6 +27604,7 @@ async def test_http_bridge_reader_failure_retires_without_waiters_when_notificat detail="stream_incomplete", response_events_seen=0, retired_request_count=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) @@ -28357,6 +29127,67 @@ async def test_fail_stale_http_bridge_pending_requests_quarantines_wedged_gate_h record_failure.assert_not_awaited() +@pytest.mark.asyncio +async def test_fail_stale_http_bridge_pending_requests_captures_attempt_before_pending_lock_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = _make_eventless_http_bridge_owner(request_id="req-stale-attempt-snapshot") + original_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=1) + replacement_attempt = proxy_support_module._HTTPBridgeResponseCreateAttempt(ordinal=2) + request_state.response_create_attempt = original_attempt + session = _make_bridge_session( + key_value="stale-attempt-snapshot", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + attempt_captured = asyncio.Event() + original_selector = ( + http_bridge_request_submit_module._http_bridge_retry_circuit_attempt_selection_for_pending_requests + ) + + def capture_attempt_before_lock( + request_states: list[proxy_service._WebSocketRequestState], + ) -> proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection: + selection = original_selector(request_states) + attempt_captured.set() + return selection + + monkeypatch.setattr( + http_bridge_request_submit_module, + "_http_bridge_retry_circuit_attempt_selection_for_pending_requests", + capture_attempt_before_lock, + ) + record_failure = AsyncMock(return_value=1) + monkeypatch.setattr( + service, + "_record_http_bridge_retry_circuit_failure_for_attempt_selection", + record_failure, + ) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) + + async with session.pending_lock: + fail_task = asyncio.create_task( + service._fail_stale_http_bridge_pending_requests( + session, + [request_state], + detail="response_create_gate_timeout_stuck_pending", + ) + ) + await asyncio.wait_for(attempt_captured.wait(), timeout=0.5) + request_state.response_create_attempt = replacement_attempt + + await asyncio.wait_for(fail_task, timeout=0.5) + + record_failure.assert_awaited_once() + record_failure_call = record_failure.await_args + assert record_failure_call is not None + selection = record_failure_call.kwargs["selection"] + assert selection.kind == "eligible" + assert selection.attempt is original_attempt + assert replacement_attempt.retry_circuit_failure_recorded is False + + @pytest.mark.asyncio async def test_http_bridge_missing_created_timeout_records_eventless_quarantine_strike( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index cb9718d7f4..434eb12a20 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -37930,6 +37930,7 @@ async def test_response_create_admission_stuck_gate_retire_ignores_draining_pend bridge_session, [stale_gate_holder], detail="response_create_gate_timeout_stuck_pending", + retry_circuit_attempt_selection=proxy_support._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), ) From 17ae866e2f6fd3d2daa1f1c4a0b2a8d0a5d2f25b Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:44:58 +0200 Subject: [PATCH 032/117] fix(proxy): abandon unavailable owner on thread-scoped goal restart (#1764) * fix(proxy): abandon unavailable owner on thread-scoped goal restart Current Codex sends thread-id with the process session, so affinity classified the restart as thread_header and never retired the raw legacy owner. Grant the one-shot abandonment flag when a process session is present, allow retirement CAS from thread_header requests, and consult the raw row as session_header interpretation so later thread turns stay on the replacement. * test(proxy): avoid optional sticky-map subscript in thread restart test ty rejects subscripting account_ids_by_key because the stub field is optional. Compare the whole mapping like the existing session-header test. * fix(proxy): keep thread-only raw owners and cover route restart Codex review: a hardcoded session_header lookup hid thread-only raw rows after a process-session tombstone, and the restart path lacked /backend-api/codex/responses coverage for session-id plus thread-id. Look up the raw key with the source that actually wrote it, and add the route-level restart plus follow-up continuity test. * test(proxy): accept legacy continuity source in control selection mock * fix(proxy): use legacy continuity source on HTTP-bridge rebind Security-authorized replacement still looked up the raw process-session row with thread_header, so a session_header tombstone resurrected the retired owner as a continuity conflict. Lookup the raw key with legacy_continuity_source instead. --- .../proxy/_load_balancer/sticky_selection.py | 2 +- app/modules/proxy/_service/codex_control.py | 3 + .../_service/http_bridge/request_submit.py | 2 +- app/modules/proxy/_service/websocket/mixin.py | 1 + app/modules/proxy/affinity.py | 22 ++- app/modules/proxy/load_balancer.py | 7 +- app/modules/proxy/service.py | 4 + .../.openspec.yaml | 2 + .../context.md | 24 +++ .../design.md | 56 ++++++ .../proposal.md | 43 +++++ .../specs/sticky-session-operations/spec.md | 43 +++++ .../tasks.md | 23 +++ .../sticky-session-operations/context.md | 2 +- .../specs/sticky-session-operations/spec.md | 28 +++ .../integration/test_proxy_sticky_sessions.py | 160 +++++++++++++++++- tests/unit/test_load_balancer_concurrency.py | 50 ++++++ tests/unit/test_proxy_utils.py | 132 +++++++++++++++ 18 files changed, 595 insertions(+), 9 deletions(-) create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/context.md create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/design.md create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/proposal.md create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md create mode 100644 openspec/changes/goal-restart-thread-header-abandonment/tasks.md diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 981d65403f..1dc29cbf71 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -511,7 +511,7 @@ def _direct_error( abandon_unavailable_legacy_owner and hard_sticky and sticky_existing_is_legacy - and sticky_source == "session_header" + and sticky_source in {"session_header", "thread_header"} and legacy_sticky_key is not None and isinstance(sticky_existing_account_id, str) and legacy_owner_in_effective_policy_scope diff --git a/app/modules/proxy/_service/codex_control.py b/app/modules/proxy/_service/codex_control.py index 242f2bbd30..5c3fad98a2 100644 --- a/app/modules/proxy/_service/codex_control.py +++ b/app/modules/proxy/_service/codex_control.py @@ -229,6 +229,7 @@ async def _select_codex_control_account_without_budget( reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, sticky_seed_key=affinity.seed_selection_key, sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, @@ -398,6 +399,7 @@ async def _select_control_failover(excluded_account_ids: set[str]) -> AccountSel reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, sticky_seed_key=affinity.seed_selection_key, sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, @@ -492,6 +494,7 @@ async def _select_control_failover(excluded_account_ids: set[str]) -> AccountSel reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, sticky_seed_key=affinity.seed_selection_key, sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 10f536a6ce..9258d60bf3 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -3666,7 +3666,7 @@ async def _claim_http_bridge_replacement_before_swap( # remains durable hard ownership. kind=StickySessionKind.CODEX_SESSION, max_age_seconds=None, - continuity_source=owner_rebind_affinity.codex_session_source, + continuity_source=(owner_rebind_affinity.legacy_continuity_source or "session_header"), ) if legacy_owner_id is not None and legacy_owner_id != account_id: raise ProxyResponseError( diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index ea866d4bf8..0e1894e559 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -3476,6 +3476,7 @@ async def _select_websocket_connect_account( reallocate_sticky=reallocate_sticky, sticky_source=request_state.affinity_policy.codex_session_source, legacy_sticky_key=request_state.affinity_policy.legacy_selection_key, + legacy_continuity_source=request_state.affinity_policy.legacy_continuity_source, sticky_seed_key=request_state.affinity_policy.seed_selection_key, sticky_seed_kind=request_state.affinity_policy.seed_selection_kind, spill_bare_session_on_account_cap=request_state.affinity_policy.spill_on_account_cap, diff --git a/app/modules/proxy/affinity.py b/app/modules/proxy/affinity.py index 4dd41a114d..a8e3e35761 100644 --- a/app/modules/proxy/affinity.py +++ b/app/modules/proxy/affinity.py @@ -43,6 +43,7 @@ class _AffinitySelectionKwargs(TypedDict): reallocate_sticky: bool sticky_source: _CodexSessionSource | None legacy_sticky_key: str | None + legacy_continuity_source: _CodexSessionSource | None sticky_seed_key: str | None sticky_seed_kind: StickySessionKind | None spill_bare_session_on_account_cap: bool @@ -69,6 +70,10 @@ class _AffinityPolicy: # compatibility lookup explicit instead of trying to reconstruct it from # the new opaque thread key. legacy_codex_session_key: str | None = None + # Interpretation used when consulting that raw key. Process-session text + # is session_header even on a thread-scoped request; a thread-only raw + # key stays thread_header so a session_header tombstone cannot hide it. + legacy_continuity_source: _CodexSessionSource | None = None # A previously unseen thread should inherit the healthy process preference # once, then persist its own bounded row. This is never ownership: a # missing process default may be initialized once by insert-if-absent, but @@ -109,6 +114,9 @@ def selection_kwargs(self) -> _AffinitySelectionKwargs: "reallocate_sticky": self.reallocate_sticky, "sticky_source": self.codex_session_source, "legacy_sticky_key": self.legacy_selection_key, + "legacy_continuity_source": ( + None if self.legacy_selection_key is None else (self.legacy_continuity_source or "session_header") + ), "sticky_seed_key": self.seed_selection_key, "sticky_seed_kind": self.seed_selection_kind, "spill_bare_session_on_account_cap": self.spill_on_account_cap, @@ -433,6 +441,7 @@ def _thread_codex_session_affinity( max_age_seconds=max_age_seconds, codex_session_source="thread_header", legacy_codex_session_key=legacy_key, + legacy_continuity_source=("session_header" if identity.process_session is not None else "thread_header"), seed_selection_key=( _codex_session_selection_key(identity.process_session) if identity.process_session is not None else None ), @@ -735,10 +744,15 @@ def _sticky_key_for_responses_request( else: policy = _AffinityPolicy() if ( - # Only typed process-session provenance can represent the legacy row - # this escape hatch targets. An explicit turn-state header stays hard - # even when a client includes the same goal marker. - policy.codex_session_source == "session_header" + # The raw row this escape hatch retires is the process-session key. + # Current Codex also sends thread-id, so locality source is often + # thread_header; that must not hide the process-session exception. + # An explicit turn-state header stays hard even with the same marker. + policy.codex_session_source in {"session_header", "thread_header"} + and ( + policy.codex_session_source == "session_header" + or _codex_backend_identity(headers).process_session is not None + ) and _request_allows_unavailable_legacy_owner_abandonment(payload) ): policy = replace(policy, abandon_unavailable_legacy_owner=True) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index 0bc5717dd5..3a6dd88e2f 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -532,6 +532,7 @@ async def select_account( reallocate_sticky: bool = False, sticky_source: _CodexSessionSource | None = None, legacy_sticky_key: str | None = None, + legacy_continuity_source: _CodexSessionSource | None = None, sticky_seed_key: str | None = None, sticky_seed_kind: StickySessionKind | None = None, spill_bare_session_on_account_cap: bool = False, @@ -731,7 +732,11 @@ async def load_selection_inputs() -> _SelectionInputs: # Raw rows may be historical turn-state ownership. The # bounded thread TTL must never age out that hard evidence. max_age_seconds=None, - continuity_source=sticky_source, + # Process-session raw text is session_header even when + # request locality is thread_header. Thread-only raw keys + # keep thread_header so a session_header tombstone cannot + # hide a distinct thread owner. + continuity_source=legacy_continuity_source or "session_header", ) legacy_existing_account_id = legacy_owner_lookup.account_id abandoned_account_id = legacy_owner_lookup.abandoned_account_id diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 90f7c6b129..196a69964f 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -1084,6 +1084,7 @@ async def _select_goal_failover(excluded_account_ids: set[str]) -> AccountSelect reallocate_sticky=affinity.reallocate_sticky, sticky_source=affinity.codex_session_source, legacy_sticky_key=affinity.legacy_selection_key, + legacy_continuity_source=affinity.legacy_continuity_source, sticky_seed_key=affinity.seed_selection_key, sticky_seed_kind=affinity.seed_selection_kind, sticky_max_age_seconds=affinity.max_age_seconds, @@ -1698,6 +1699,7 @@ async def _select_account_with_budget( reallocate_sticky: bool = False, sticky_source: _CodexSessionSource | None = None, legacy_sticky_key: str | None = None, + legacy_continuity_source: _CodexSessionSource | None = None, sticky_seed_key: str | None = None, sticky_seed_kind: StickySessionKind | None = None, spill_bare_session_on_account_cap: bool = False, @@ -1858,6 +1860,7 @@ def log_account_id(account_id: str | None) -> str | None: sticky_max_age_seconds=preferred_sticky_inputs[3], sticky_source=preferred_sticky_inputs[4], legacy_sticky_key=preferred_sticky_inputs[5], + legacy_continuity_source=legacy_continuity_source, # Exact ownership chooses the account; a first-ever thread # still seeds atomically without overwriting a process default. sticky_seed_key=sticky_seed_key, @@ -1920,6 +1923,7 @@ def log_account_id(account_id: str | None) -> str | None: reallocate_sticky=reallocate_sticky, sticky_source=sticky_source, legacy_sticky_key=legacy_sticky_key, + legacy_continuity_source=legacy_continuity_source, sticky_seed_key=sticky_seed_key, sticky_seed_kind=sticky_seed_kind, spill_bare_session_on_account_cap=_AffinityPolicy.cap_spillover_allowed( diff --git a/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml b/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/goal-restart-thread-header-abandonment/context.md b/openspec/changes/goal-restart-thread-header-abandonment/context.md new file mode 100644 index 0000000000..ba6c52edab --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/context.md @@ -0,0 +1,24 @@ +## Purpose + +Close the `#1703` × `#1680` composition hole: current Codex always +sends `thread-id`, so the merged goal-restart recovery never fires. + +## Decision + +Abandonment stays a `session_header` *interpretation* of the raw +process-session key. Request locality may be `thread_header`. Explicit +`turn_state` is unchanged. + +## Failure modes + +- Incremental or file-pinned restarts must still fail closed on the + required owner. +- After retirement, a later thread-id turn must not revive the raw + row as hard ownership. + +## Example + +Process session `sid` maps to quota-exceeded account A. Codex resends +an account-neutral goal body with `session-id: sid` and +`thread-id: t1`. Selection retires `sid` for `session_header`, routes +to B, and later `t1` turns stay on B. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/design.md b/openspec/changes/goal-restart-thread-header-abandonment/design.md new file mode 100644 index 0000000000..3f2988efc6 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/design.md @@ -0,0 +1,56 @@ +## Context + +`#1679` / `#1680` added a proof-gated exception that retires an +unavailable raw `codex_session` owner for `session_header` +interpretation. `#1703` then made `thread-id` the winning locality +source for current Codex. The two compose incorrectly: the flag and +CAS both require `sticky_source == "session_header"`, which current +Codex never is. + +The raw compatibility row is the process-session key. Looking it up +with `continuity_source=thread_header` treats a `session_header` +tombstone as a live hard owner, so even a successful session-only +restart is undone by the next thread-id turn. + +## Goals / Non-Goals + +**Goals:** + +- Account-neutral goal restart with `session-id` + `thread-id` retires + the unavailable raw owner for process-session interpretation and + routes to a replacement. +- Later same-thread turns without a new hard owner stay on that + replacement. +- Explicit `turn_state` of the same text stays hard-bound. + +**Non-Goals:** + +- Changing file-pin, previous-response, conversation, or tool-state + fail-closed ownership. +- Making `thread_header` an abandonment scope on the raw row. +- Dashboard, settings, or schema changes. + +## Decisions + +- Grant `abandon_unavailable_legacy_owner` for `thread_header` only + when a process session is also present. Thread-only clients have no + process-session raw row to retire. +- Allow retirement CAS when request source is `thread_header`. The + write remains `abandonment_scope=session_header`. +- Load the raw `legacy_sticky_key` with `continuity_source=session_header`. + That lookup is process-session interpretation, not thread identity. + +**Alternative considered:** keep CAS gated on request source and only +set the flag. Rejected because the CAS would still not run. + +**Alternative considered:** abandon the raw row for every source. +Rejected because colliding explicit `turn_state` must stay hard. + +## Risks / Trade-offs + +- [Risk] A thread-header request could retire a raw row that was + written as turn-state with equal text. → Mitigation: CAS still + writes `session_header` scope only; turn-state lookup of that text + keeps the stored owner. +- [Risk] Existing tests only exercise `session_id` without `thread-id`. + → Mitigation: add the missing header combination next to those tests. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/proposal.md b/openspec/changes/goal-restart-thread-header-abandonment/proposal.md new file mode 100644 index 0000000000..c4a953aae3 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/proposal.md @@ -0,0 +1,43 @@ +## Why + +Current Codex sends both a shared process `session-id` and a distinct +`thread-id` on a self-contained goal restart. Affinity classifies that +request as `thread_header`, so the one-shot +`abandon_unavailable_legacy_owner` flag never sets and retirement CAS +never runs. The restart stays fail-closed on the unavailable legacy +owner even though the payload is account-neutral. + +## What Changes + +- Grant goal-restart abandonment when a thread-scoped request still + carries a process session, not only when locality source is + `session_header`. +- Let retirement CAS retire the raw process-session row for + `session_header` interpretation from that thread-scoped request. +- Consult the raw process-session row as `session_header` + interpretation so a scoped tombstone hides it from later thread-id + turns. Explicit `turn_state` of the same text stays hard. +- Keep incremental, file-pinned, conversation-bound, and unresolved + tool-state requests fail-closed. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `sticky-session-operations`: Current Codex `thread-id` on a + self-contained goal restart MUST still abandon the unavailable raw + process-session owner for `session_header` interpretation and keep + later same-thread continuity on the replacement. + +## Impact + +- `app/modules/proxy/affinity.py` restart-capability gate. +- `app/modules/proxy/_load_balancer/sticky_selection.py` retirement CAS + source check. +- `app/modules/proxy/load_balancer.py` raw-row lookup source. +- Focused affinity and sticky-selection tests. +- No API, schema, setting, dashboard, or wire-format change. diff --git a/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md b/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..655f1d7553 --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/specs/sticky-session-operations/spec.md @@ -0,0 +1,43 @@ +## ADDED Requirements + +### Requirement: Thread-scoped current Codex restarts still abandon a raw process-session owner + +A self-contained Codex goal-continuation restart that also carries a distinct `thread-id` MUST still be eligible for the existing process-session abandonment exception. The request's thread-scoped locality source MUST NOT prevent the one-shot abandonment capability or the compare-and-set retirement of the raw process-session row. + +The retirement write MUST remain scoped to `session_header` +interpretation of that raw key. An explicit `turn_state` lookup of the +same text MUST stay hard-bound to the stored account. After a +successful retirement, later same-thread turns that have no new hard +owner MUST keep continuity on the replacement account and MUST NOT +treat the `session_header`-abandoned raw row as live hard ownership. + +Ordinary incremental, file-pinned, conversation-bound, and unresolved +tool-state requests MUST remain fail-closed on their required owner. + +#### Scenario: Goal restart with process session and thread-id abandons the unavailable raw owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **AND** the request also carries a distinct `thread-id` +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent same-thread continuity remains on account B + +#### Scenario: Thread-id on a goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a later request carries the same text as a process-session header plus a distinct `thread-id` +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Account-dependent thread-scoped restart stays fail-closed + +- **GIVEN** a process-session identifier has a raw legacy mapping to unavailable account A +- **AND** the request carries a distinct `thread-id` +- **AND** the body has a previous response, conversation, file pin, or unresolved tool state +- **WHEN** the request is selected +- **THEN** the request fails closed on account A +- **AND** the raw mapping is neither deleted nor rebound diff --git a/openspec/changes/goal-restart-thread-header-abandonment/tasks.md b/openspec/changes/goal-restart-thread-header-abandonment/tasks.md new file mode 100644 index 0000000000..788859600e --- /dev/null +++ b/openspec/changes/goal-restart-thread-header-abandonment/tasks.md @@ -0,0 +1,23 @@ +## 1. Implementation + +- [x] 1.1 Grant `abandon_unavailable_legacy_owner` for `thread_header` + when a process session is present and the payload is + account-neutral. +- [x] 1.2 Allow retirement CAS when request source is `thread_header`. + Keep the write scoped to `session_header`. +- [x] 1.3 Load the raw `legacy_sticky_key` as `session_header` + interpretation so a scoped tombstone hides it from later + thread-id turns. + +## 2. Regression coverage + +- [x] 2.1 Assert session-id + thread-id goal restart sets the + abandonment flag; turn-state and account-dependent payloads do + not. +- [x] 2.2 Assert sticky selection retires the raw owner and selects a + replacement when source is `thread_header`. + +## 3. Validation + +- [x] 3.1 Run the focused affinity and sticky-selection tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/specs/sticky-session-operations/context.md b/openspec/specs/sticky-session-operations/context.md index a85e9c0220..3dcc5d12ad 100644 --- a/openspec/specs/sticky-session-operations/context.md +++ b/openspec/specs/sticky-session-operations/context.md @@ -12,7 +12,7 @@ See `openspec/specs/sticky-session-operations/spec.md` for normative requirement - Bare process-session headers use a header-inaccessible, source-separated storage key and are soft only for self-contained pre-visible work. - Account-cap spillover is request-local: it selects an alternate without deleting or rebinding the process-session row. - Raw and legacy Codex rows remain hard during rolling upgrades because they may represent explicit turn-state ownership. -- A raw legacy Codex owner can be abandoned only for an explicit goal-continuation restart whose canonical upstream payload passes the account-neutral fresh-replay proof, and only while that owner has a persisted unavailable status. Canonicalization keeps accepted compatibility fields and transport envelopes from changing classification. The compare-and-set marker is scoped to `session_header`, so an explicit turn-state lookup with colliding raw text retains the stored owner; a concurrent rebind or owner recovery still wins. The scoped marker deliberately leaves the historical global-tombstone timestamp empty, so replicas that do not understand scope continue to fail closed on the retained owner. +- A raw legacy Codex owner can be abandoned only for an explicit goal-continuation restart whose canonical upstream payload passes the account-neutral fresh-replay proof, and only while that owner has a persisted unavailable status. Canonicalization keeps accepted compatibility fields and transport envelopes from changing classification. The compare-and-set marker is scoped to `session_header`, so an explicit turn-state lookup with colliding raw text retains the stored owner; a concurrent rebind or owner recovery still wins. The scoped marker deliberately leaves the historical global-tombstone timestamp empty, so replicas that do not understand scope continue to fail closed on the retained owner. Current Codex also sends `thread-id`; that locality source does not block the process-session exception. The raw compatibility lookup stays a `session_header` interpretation so a scoped tombstone cannot revive the retired owner on later thread-id turns. - Restart mutation authority is the authenticated account-assignment and security-policy scope before model and service-tier eligibility. Model filtering constrains only replacement selection. - Goal-restart retirement is an account-selection capability. An existing HTTP bridge cannot consume the request first through local reuse, durable-owner promotion, or forwarding. The retired owner is excluded from stale account snapshots for the remainder of the request, including when another selector wrote the scoped marker and this selector discovers it after losing the compare-and-set. - Canonical bridge replacement preserves request-owned pre-submit admission on the detached predecessor, but that predecessor cannot publish new continuity aliases under the replacement's key. Every detached generation remains lifecycle-owned and capacity-counted until resource closure ends, including an idle predecessor already marked closed for admission. diff --git a/openspec/specs/sticky-session-operations/spec.md b/openspec/specs/sticky-session-operations/spec.md index babf08b41a..3472080f29 100644 --- a/openspec/specs/sticky-session-operations/spec.md +++ b/openspec/specs/sticky-session-operations/spec.md @@ -66,6 +66,34 @@ A later security-authorized bridge replacement that revalidates a raw legacy row - **THEN** the process-session restart may select account B - **AND** an explicit turn-state lookup of the same text remains hard-bound to account A +#### Scenario: Goal restart with process session and thread-id abandons the unavailable raw owner + +- **GIVEN** a process-session identifier has a raw legacy `codex_session` mapping to account A +- **AND** account A is paused, rate-limited, or quota-exceeded +- **AND** account B is eligible +- **AND** the request also carries a distinct `thread-id` +- **WHEN** Codex sends the recognized goal-continuation marker with an account-neutral self-contained full resend and no other continuity dependency +- **THEN** the proxy marks the still-current raw mapping to account A abandoned only for process-session interpretation +- **AND** it routes the restarted turn to account B +- **AND** subsequent same-thread continuity remains on account B + +#### Scenario: Thread-id on a goal restart cannot erase colliding explicit turn-state ownership + +- **GIVEN** a raw legacy `codex_session` row was written as explicit turn-state ownership for account A +- **AND** a later request carries the same text as a process-session header plus a distinct `thread-id` +- **WHEN** a marked self-contained goal restart abandons that text for process-session interpretation +- **THEN** the restart may select account B +- **AND** an explicit turn-state lookup of the same text remains hard-bound to account A + +#### Scenario: Account-dependent thread-scoped restart stays fail-closed + +- **GIVEN** a process-session identifier has a raw legacy mapping to unavailable account A +- **AND** the request carries a distinct `thread-id` +- **AND** the body has a previous response, conversation, file pin, or unresolved tool state +- **WHEN** the request is selected +- **THEN** the request fails closed on account A +- **AND** the raw mapping is neither deleted nor rebound + #### Scenario: Source-qualified retirement fails closed on an older replica - **GIVEN** a current replica marks a raw account A mapping abandoned only for `session_header` interpretation diff --git a/tests/integration/test_proxy_sticky_sessions.py b/tests/integration/test_proxy_sticky_sessions.py index 5ad05217e2..0302976c05 100644 --- a/tests/integration/test_proxy_sticky_sessions.py +++ b/tests/integration/test_proxy_sticky_sessions.py @@ -23,7 +23,7 @@ _REALTIME_CALL_AFFINITY_MAX_AGE_SECONDS, realtime_call_affinity_key, ) -from app.modules.proxy.affinity import _codex_session_selection_key +from app.modules.proxy.affinity import _codex_backend_identity, _codex_session_selection_key from app.modules.usage.repository import UsageRepository pytestmark = pytest.mark.integration @@ -401,6 +401,164 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert legacy_replica_owner == owner_id +@pytest.mark.asyncio +async def test_codex_goal_restart_with_thread_id_retires_unavailable_legacy_owner_and_stays_on_replacement( + async_client, + monkeypatch, +): + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + _install_proxy_settings_cache(monkeypatch, sticky_threads_enabled=False) + owner_id = await _import_account( + async_client, + "acc_goal_restart_thread_owner", + "goal-restart-thread-owner@example.com", + ) + replacement_id = await _import_account( + async_client, + "acc_goal_restart_thread_replacement", + "goal-restart-thread-replacement@example.com", + ) + raw_session = "goal-restart-thread-session" + thread_id = "goal-restart-thread" + headers = {"session_id": raw_session, "thread-id": thread_id} + thread_key = _codex_backend_identity(headers).thread_selection_key + assert thread_key is not None + + now_epoch = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + async with SessionLocal() as session: + usage_repo = UsageRepository(session) + await usage_repo.add_entry( + account_id=owner_id, + used_percent=10.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await usage_repo.add_entry( + account_id=replacement_id, + used_percent=20.0, + window="primary", + reset_at=now_epoch + 3600, + window_minutes=300, + ) + await StickySessionsRepository(session).upsert( + raw_session, + owner_id, + kind=StickySessionKind.CODEX_SESSION, + ) + + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, base_url, raise_for_status, kwargs + seen.append(account_id) + yield f'data: {{"type":"response.completed","response":{{"id":"resp_goal_thread_{len(seen)}"}}}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + restart_payload = { + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": ('\nContinue working toward the active thread goal.'), + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + } + + healthy_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert healthy_response.status_code == 200 + assert seen == ["acc_goal_restart_thread_owner"] + + async with SessionLocal() as session: + await session.execute(update(Account).where(Account.id == owner_id).values(status=AccountStatus.QUOTA_EXCEEDED)) + await session.commit() + + restart_response = await async_client.post( + "/backend-api/codex/responses", + json=restart_payload, + headers=headers, + ) + assert restart_response.status_code == 200 + assert seen == ["acc_goal_restart_thread_owner", "acc_goal_restart_thread_replacement"] + + follow_up_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers=headers, + ) + assert follow_up_response.status_code == 200 + assert seen == [ + "acc_goal_restart_thread_owner", + "acc_goal_restart_thread_replacement", + "acc_goal_restart_thread_replacement", + ] + + turn_state_response = await async_client.post( + "/backend-api/codex/responses", + json={"model": "gpt-5.1", "instructions": "continue", "input": [], "stream": True}, + headers={"x-codex-turn-state": raw_session}, + ) + assert turn_state_response.status_code == 502 + assert turn_state_response.json()["error"]["code"] == "turn_state_owner_unavailable" + assert seen == [ + "acc_goal_restart_thread_owner", + "acc_goal_restart_thread_replacement", + "acc_goal_restart_thread_replacement", + ] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + raw_row = ( + await session.execute( + select(StickySession).where( + StickySession.key == raw_session, + StickySession.kind == StickySessionKind.CODEX_SESSION, + ) + ) + ).scalar_one() + thread_row = ( + await session.execute( + select(StickySession).where( + StickySession.key == thread_key, + StickySession.kind == StickySessionKind.PROMPT_CACHE, + ) + ) + ).scalar_one() + session_header_lookup = await repo.get_account_id_and_abandonment( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="session_header", + ) + thread_legacy_lookup = await repo.get_account_id_and_abandonment( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="thread_header", + ) + turn_state_owner = await repo.get_account_id( + raw_session, + kind=StickySessionKind.CODEX_SESSION, + continuity_source="turn_state", + ) + assert raw_row.account_id == owner_id + assert raw_row.continuity_abandonment_scope == "session_header" + assert thread_row.account_id == replacement_id + assert session_header_lookup.account_id is None + assert session_header_lookup.continuity_abandoned is True + assert thread_legacy_lookup.account_id == owner_id + assert turn_state_owner == owner_id + + @pytest.mark.asyncio async def test_codex_goal_restart_cas_miss_reloads_concurrently_rebound_raw_owner( async_client, diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index dcb1bfab87..685f22fae1 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -3126,6 +3126,56 @@ async def test_goal_restart_does_not_repin_retired_owner_from_stale_selection_sn await balancer.release_account_lease(selected.lease) +@pytest.mark.asyncio +async def test_goal_restart_with_thread_header_retires_unavailable_legacy_owner() -> None: + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + stale_owner = _make_account("goal-restart-thread-header-owner") + replacement = _make_account("goal-restart-thread-header-replacement") + raw_session = "goal-restart-thread-header-session" + thread_key = _codex_backend_identity( + {"session-id": raw_session, "thread-id": "goal-restart-thread"} + ).thread_selection_key + assert thread_key is not None + sticky_repo = _RetiringStaleOwnerStickySessionsRepository( + raw_key=raw_session, + owner_account_id=stale_owner.id, + ) + balancer = LoadBalancer( + lambda: _repo_factory( + _StubAccountsRepository([stale_owner, replacement]), + _StubUsageRepository( + { + stale_owner.id: _usage_row(311, stale_owner.id, window="primary", reset_at=now_epoch + 300), + replacement.id: _usage_row(312, replacement.id, window="primary", reset_at=now_epoch + 300), + }, + {}, + ), + sticky_repo, + ) + ) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + sticky_max_age_seconds=300, + legacy_sticky_key=raw_session, + abandon_unavailable_legacy_owner=True, + routing_strategy="single_account", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == replacement.id + assert sticky_repo.tombstones == [(raw_session, stale_owner.id)] + assert sticky_repo.account_ids_by_key == { + raw_session: stale_owner.id, + thread_key: replacement.id, + } + assert all(account_id != stale_owner.id for _, account_id, _ in sticky_repo.upserts) + await balancer.release_account_lease(selected.lease) + + @pytest.mark.asyncio async def test_goal_restart_cas_loser_does_not_repin_concurrently_retired_owner() -> None: now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 434eb12a20..3611c1a17e 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -5754,6 +5754,7 @@ async def test_select_codex_control_account_without_budget_uses_balancer(monkeyp reallocate_sticky=False, sticky_source=None, legacy_sticky_key=None, + legacy_continuity_source=None, sticky_seed_key=None, sticky_seed_kind=None, sticky_max_age_seconds=123, @@ -10509,8 +10510,35 @@ def test_goal_restart_affinity_can_abandon_only_legacy_session_owner(): sticky_threads_enabled=False, ) + thread_policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={ + "session_id": "goal-restart-session", + "thread-id": "goal-restart-thread", + }, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + thread_only_policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={"thread-id": "goal-restart-thread"}, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + assert policy.codex_session_source == "session_header" assert policy.abandon_unavailable_legacy_owner is True + assert thread_policy.codex_session_source == "thread_header" + assert thread_policy.abandon_unavailable_legacy_owner is True + assert thread_policy.legacy_selection_key == "goal-restart-session" + assert thread_policy.legacy_continuity_source == "session_header" + assert thread_only_policy.codex_session_source == "thread_header" + assert thread_only_policy.abandon_unavailable_legacy_owner is False + assert thread_only_policy.legacy_continuity_source == "thread_header" assert turn_state_policy.codex_session_source == "turn_state" assert turn_state_policy.abandon_unavailable_legacy_owner is False @@ -10574,6 +10602,25 @@ def test_goal_restart_affinity_preserves_owner_for_account_dependent_payloads( assert policy.abandon_unavailable_legacy_owner is False +def test_goal_restart_affinity_preserves_owner_for_account_dependent_thread_payloads(): + payload = _goal_restart_payload(previous_response_id="resp_owner") + + policy = proxy_service._sticky_key_for_responses_request( + payload, + headers={ + "session_id": "goal-restart-session", + "thread-id": "goal-restart-thread", + }, + codex_session_affinity=True, + openai_cache_affinity=False, + openai_cache_affinity_max_age_seconds=300, + sticky_threads_enabled=False, + ) + + assert policy.codex_session_source == "thread_header" + assert policy.abandon_unavailable_legacy_owner is False + + def test_full_resend_without_goal_marker_cannot_abandon_legacy_owner(): payload = ResponsesRequest.model_validate( { @@ -18068,6 +18115,91 @@ async def __aexit__(self, exc_type, exc, tb) -> bool: assert session.closed is False +@pytest.mark.asyncio +async def test_http_bridge_replacement_uses_legacy_continuity_source_for_raw_row() -> None: + rejected_account = _make_account("acc_bridge_thread_restart_owner") + authorized_account = _make_account("acc_bridge_thread_restart_replacement") + sticky_sessions = AsyncMock() + seen_sources: list[str | None] = [] + + async def legacy_owner_for_source( + _key: str, + *, + kind: StickySessionKind, + max_age_seconds: int | None = None, + continuity_source: str | None = None, + ) -> str | None: + del kind, max_age_seconds + seen_sources.append(continuity_source) + return rejected_account.id if continuity_source == "thread_header" else None + + sticky_sessions.get_account_id.side_effect = legacy_owner_for_source + + class _TrackingRepoContext: + def __init__(self) -> None: + self._repos = ProxyRepositories( + accounts=cast(AccountsRepository, AsyncMock()), + usage=cast(UsageRepository, AsyncMock()), + request_logs=cast(RequestLogsRepository, _RequestLogsRecorder()), + sticky_sessions=cast(StickySessionsRepository, sticky_sessions), + api_keys=cast(ApiKeysRepository, AsyncMock()), + additional_usage=cast(AdditionalUsageRepository, AsyncMock()), + ) + + async def __aenter__(self) -> ProxyRepositories: + return self._repos + + async def __aexit__(self, exc_type, exc, tb) -> bool: + return False + + service = proxy_service.ProxyService(_TrackingRepoContext) + replacement_upstream = AsyncMock() + affinity = proxy_service._AffinityPolicy( + key="thread-restart-rebind", + kind=StickySessionKind.PROMPT_CACHE, + codex_session_source="thread_header", + legacy_codex_session_key="process-restart-rebind", + legacy_continuity_source="session_header", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("thread_header", "thread-restart-rebind", None), + headers={"session_id": "process-restart-rebind", "thread-id": "thread-restart-rebind"}, + affinity=affinity, + request_model="gpt-5.1", + account=rejected_account, + upstream=AsyncMock(), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=1.0, + idle_ttl_seconds=300.0, + durable_session_id="durable-thread-restart-rebind", + durable_owner_epoch=2, + ) + durable_claim = AsyncMock() + service._claim_durable_http_bridge_session = durable_claim + + await service._claim_http_bridge_replacement_before_swap( + session, + account_id=authorized_account.id, + upstream=replacement_upstream, + release_selected_account_lease=AsyncMock(), + owner_rebind_affinity=affinity, + ) + + assert seen_sources == ["session_header"] + sticky_sessions.get_account_id.assert_awaited_once_with( + "process-restart-rebind", + kind=StickySessionKind.CODEX_SESSION, + max_age_seconds=None, + continuity_source="session_header", + ) + durable_claim.assert_awaited_once() + replacement_upstream.close.assert_not_awaited() + + @pytest.mark.asyncio async def test_http_bridge_security_retry_restores_codex_affinity_and_turn_aliases_on_failure( monkeypatch: pytest.MonkeyPatch, From 34ef7b262ecabc597be0c7dd73978dc24973ee82 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:45:25 +0200 Subject: [PATCH 033/117] fix(proxy): do not rewrite thread locality for a file-pin owner (#1765) * fix(proxy): do not rewrite thread locality for a file-pin owner A live input_file pin is hard ownership. After thread-scoped affinity the current Codex soft row is thread_header / PROMPT_CACHE, but the preferred-owner bypass still only nulled session_header keys. Bypass the writable thread key too so a file-pinned turn cannot upsert an existing thread mapping onto the upload account. * fix(proxy): seed process preference after unbound file-pin selection Nulling the thread sticky key sent required-owner selection down the unbound path, which never persisted insert-if-absent process preference. Later unpinned siblings then lost the exact owner. Persist the missing process seed after a successful unbound required owner selection, without writing the thread row. * test(proxy): cover file-pin thread locality on Codex responses Prove a file-pinned /backend-api/codex/responses turn keeps the existing thread row, seeds process preference, and leaves later unpinned thread and sibling requests on the documented owners. --- app/modules/proxy/affinity.py | 11 +- app/modules/proxy/load_balancer.py | 21 ++++ .../.openspec.yaml | 2 + .../context.md | 14 +++ .../design.md | 42 +++++++ .../proposal.md | 36 ++++++ .../specs/sticky-session-operations/spec.md | 21 ++++ .../tasks.md | 19 +++ .../sticky-session-operations/context.md | 2 +- .../specs/sticky-session-operations/spec.md | 8 ++ tests/integration/test_proxy_files.py | 113 +++++++++++++++++- tests/unit/test_load_balancer_concurrency.py | 106 +++++++++++++++- tests/unit/test_proxy_utils.py | 5 +- 13 files changed, 390 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md create mode 100644 openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md diff --git a/app/modules/proxy/affinity.py b/app/modules/proxy/affinity.py index a8e3e35761..03fe030d33 100644 --- a/app/modules/proxy/affinity.py +++ b/app/modules/proxy/affinity.py @@ -150,7 +150,7 @@ def preferred_owner_sticky_inputs( _CodexSessionSource | None, str | None, ]: - if sticky_source != "session_header": + if sticky_source not in {"session_header", "thread_header"}: return ( sticky_key, sticky_kind, @@ -159,10 +159,11 @@ def preferred_owner_sticky_inputs( sticky_source, legacy_sticky_key, ) - # A resolved response/file/bridge owner bypasses the new soft row, but - # the raw compatibility row still has to be checked for conflicting - # legacy hard ownership. Selection receives no writable sticky key, so - # a raw miss cannot manufacture or rebind a mapping. The caller also + # A resolved response/file/bridge owner bypasses the current-Codex + # soft row (process-session or thread PROMPT_CACHE). The raw + # compatibility row still has to be checked for conflicting legacy + # hard ownership. Selection receives no writable sticky key, so a + # raw miss cannot manufacture or rebind a mapping. The caller also # deliberately omits any broader process seed in this exact-owner path. return None, StickySessionKind.CODEX_SESSION, False, sticky_max_age_seconds, sticky_source, legacy_sticky_key diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index 3a6dd88e2f..2624fd4cea 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -822,6 +822,27 @@ async def load_selection_inputs() -> _SelectionInputs: error_message=error_message, error_code=selection_error_code, ) + if ( + selected_snapshot is not None + and selected_lease is not None + and sticky_seed_key is not None + and sticky_seed_kind is not None + and sticky_seed_account_id is None + ): + # Required-owner selection bypasses the thread row, but a + # first-ever process preference still has to land so later + # unpinned siblings inherit that exact owner. + try: + async with self._repo_factory() as repos: + await repos.sticky_sessions.insert_if_absent( + sticky_seed_key, + selected_snapshot.id, + sticky_seed_kind, + ) + except BaseException: + await self.release_account_lease(selected_lease) + selected_lease = None + raise else: sticky_outcome = await run_sticky_selection_path( self, diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml b/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md new file mode 100644 index 0000000000..65772f6eba --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/context.md @@ -0,0 +1,14 @@ +## Purpose + +Stop a live file pin from rewriting current-Codex thread locality. + +## Decision + +The required owner bypasses the thread PROMPT_CACHE row the same way +it already bypasses the process-session soft row. + +## Example + +Upload pins `file_xyz` to account A. Thread `t1` is already mapped to +account B. A Responses turn that references `file_xyz` goes to A; the +`t1` row remains B. The next unpinned `t1` turn still uses B. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md new file mode 100644 index 0000000000..08dff38f7f --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/design.md @@ -0,0 +1,42 @@ +## Context + +`#1521` made file pins durable hard ownership and bypassed the +process-session soft row. `#1703` then made `thread_header` / +PROMPT_CACHE the current Codex soft mapping. The bypass was not +updated, so a required file owner still enters sticky persist and +upserts thread B→A. + +## Goals / Non-Goals + +**Goals:** + +- File-pinned routing stays on the pin account. +- An existing thread PROMPT_CACHE row is not rewritten. +- Process-session seed remains insert-if-absent. + +**Non-Goals:** + +- Changing 1011 file-pin reconnect. +- Weakening file-pin fail-closed or hard-owner conflict checks. +- Dashboard or settings changes. + +## Decisions + +- Null the writable sticky key for both `session_header` and + `thread_header` in `preferred_owner_sticky_inputs`. Selection then + takes the unbound required-owner path. +- Keep `legacy_sticky_key` so a conflicting raw process-session owner + still fail-closes. +- Leave `sticky_seed_key` to the caller so a missing process + preference can still initialize without writing the thread row. + +**Alternative considered:** persist the thread row onto the file +owner so later unpinned turns stay there. Rejected: the file pin is +hard only for this turn; thread locality is a separate soft mapping. + +## Risks / Trade-offs + +- [Risk] A later unpinned turn on the same thread stays on the + pre-file account and cannot see the upload. → Mitigation: that is + the existing unpinned-file compatibility path; the pin still binds + any turn that references the file. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md new file mode 100644 index 0000000000..80167449d5 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/proposal.md @@ -0,0 +1,36 @@ +## Why + +A live `input_file.file_id` pin is hard ownership. After thread-scoped +affinity, current Codex locality is the `thread_header` PROMPT_CACHE +row, but `preferred_owner_sticky_inputs` only bypasses +`session_header`. A file-pinned Responses turn therefore rewrites the +thread mapping to the upload account, so later unpinned turns follow +the file owner. + +## What Changes + +- Treat `thread_header` as the current-Codex soft row that a resolved + file/response/bridge owner must bypass. +- Keep consulting the raw process-session compatibility row for hard + conflicts. +- Keep process-session seed insert-if-absent. Do not write or rebind + the thread row on the required-owner path. +- Keep explicit `turn_state` as hard ownership. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `sticky-session-operations`: A resolved file-pin owner MUST be + selected without consulting or rewriting the thread-scoped soft + mapping. + +## Impact + +- `app/modules/proxy/affinity.py` preferred-owner sticky inputs. +- Focused selection tests. +- No API, schema, setting, dashboard, or wire-format change. diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..3ec60a18b6 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/specs/sticky-session-operations/spec.md @@ -0,0 +1,21 @@ +## ADDED Requirements + +### Requirement: File-pin required owner does not rewrite thread locality + +A resolved live `input_file.file_id` pin MUST be selected as the required owner without consulting or rewriting the current-Codex thread-scoped soft mapping. The process-session compatibility row MAY still be consulted as independent hard ownership. If that raw row conflicts with the pin account, the request MUST fail closed. A missing process-session preference MAY still initialize insert-if-absent. + +#### Scenario: File-pinned request owner overrides thread locality + +- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the thread mapping is neither consulted as an owner nor rewritten + +#### Scenario: File pin still conflicts with a raw process-session owner + +- **GIVEN** a raw process-session `codex_session` row points to account A +- **AND** a live file pin points to account B +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` before upstream dispatch +- **AND** neither the raw row nor the thread row is rewritten diff --git a/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md b/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md new file mode 100644 index 0000000000..c016d94535 --- /dev/null +++ b/openspec/changes/file-pin-does-not-rewrite-thread-locality/tasks.md @@ -0,0 +1,19 @@ +## 1. Implementation + +- [x] 1.1 Bypass the writable `thread_header` sticky key in + `preferred_owner_sticky_inputs` the same way as `session_header`. + +## 2. Regression coverage + +- [x] 2.1 Assert preferred-owner selection nulls the thread sticky key + and keeps the process seed / raw legacy key. +- [x] 2.2 Assert an existing thread row is not upserted when a file + pin is the required owner. +- [x] 2.3 Cover the same file-pin plus existing-thread case through + `/backend-api/codex/responses`, including the later unpinned + thread turn and process-seed sibling. + +## 3. Validation + +- [x] 3.1 Run the focused selection tests. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/openspec/specs/sticky-session-operations/context.md b/openspec/specs/sticky-session-operations/context.md index 3dcc5d12ad..545d76556b 100644 --- a/openspec/specs/sticky-session-operations/context.md +++ b/openspec/specs/sticky-session-operations/context.md @@ -18,7 +18,7 @@ See `openspec/specs/sticky-session-operations/spec.md` for normative requirement - Canonical bridge replacement preserves request-owned pre-submit admission on the detached predecessor, but that predecessor cannot publish new continuity aliases under the replacement's key. Every detached generation remains lifecycle-owned and capacity-counted until resource closure ends, including an idle predecessor already marked closed for admission. - Drain status counts unsettled pending or queued work after detachment closes a generation for admission. If an idle predecessor alone fills the cap, the verified restart owns its bounded close synchronously and rechecks capacity before opening a replacement. - Resource close is single-flight across reader retirement, account invalidation, and shutdown. Capacity is released only after resource finalization, not after detachment or a bounded-close timeout. Close finalization defers caller cancellation until owned resources are released, while shutdown starts all snapshotted closes before propagating cancellation and retains failed generations for a later close pass. Durable claims are fenced per websocket generation as well as per replica, so a replacement for a row still owned by the same configured replica advances the owner epoch before serving work even when model-transition isolation no longer uses that row for routing. Security-authorized rebind keeps typed continuity provenance so a source-qualified session-header tombstone cannot reappear as an untyped hard owner. -- Durable file pins, responses, conversations, live/durable bridges, replay, and reattach sources are independent hard evidence; conflicting evidence fails closed instead of using source precedence. Opaque file IDs with no live durable pin remain unpinned for compatibility with uploads that occurred outside the current process. +- Durable file pins, responses, conversations, live/durable bridges, replay, and reattach sources are independent hard evidence; conflicting evidence fails closed instead of using source precedence. Opaque file IDs with no live durable pin remain unpinned for compatibility with uploads that occurred outside the current process. A resolved file-pin owner bypasses the current-Codex thread PROMPT_CACHE row the same way it bypasses process-session locality, so an upload does not rebind later unpinned turns on that thread. - Dashboard prompt-cache TTL is persisted in settings so operators can adjust it without restart. - Background cleanup removes stale prompt-cache rows proactively, while manual delete and purge endpoints provide operator override. diff --git a/openspec/specs/sticky-session-operations/spec.md b/openspec/specs/sticky-session-operations/spec.md index 3472080f29..718e1b31c4 100644 --- a/openspec/specs/sticky-session-operations/spec.md +++ b/openspec/specs/sticky-session-operations/spec.md @@ -360,6 +360,14 @@ A nonblank `conversation` without a dedicated resolved owner MUST proceed only w - **THEN** account B is treated as the required owner - **AND** the process-session mapping is neither consulted as an owner nor rewritten +#### Scenario: File-pinned request owner overrides thread locality + +- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the thread mapping is neither consulted as an owner nor rewritten + #### Scenario: Conflicting hard owners fail closed - **GIVEN** a turn state, previous response, bridge, or input file resolves to account A diff --git a/tests/integration/test_proxy_files.py b/tests/integration/test_proxy_files.py index fc9cb926eb..45068df607 100644 --- a/tests/integration/test_proxy_files.py +++ b/tests/integration/test_proxy_files.py @@ -24,9 +24,11 @@ from app.core.auth.refresh import RefreshError from app.core.clients.files import FileProxyError from app.core.clients.proxy import ProxyResponseError -from app.db.models import FileAccountPin +from app.db.models import FileAccountPin, StickySessionKind from app.db.session import SessionLocal +from app.modules.proxy.affinity import _codex_backend_identity, _codex_session_selection_key from app.modules.proxy.file_pin_repository import FileAccountPinRepository +from app.modules.proxy.sticky_repository import StickySessionsRepository pytestmark = pytest.mark.integration @@ -53,11 +55,12 @@ def _make_auth_json(account_id: str, email: str) -> dict: } -async def _import_account(async_client, account_id: str, email: str) -> None: +async def _import_account(async_client, account_id: str, email: str) -> str: auth_json = _make_auth_json(account_id, email) files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} response = await async_client.post("/api/accounts/import", files=files) assert response.status_code == 200 + return response.json()["accountId"] @pytest.mark.asyncio @@ -865,6 +868,112 @@ async def fake_stream( assert resolved is not None +@pytest.mark.asyncio +async def test_backend_responses_file_pin_does_not_rewrite_existing_thread_row( + async_client, + monkeypatch, +): + from app.dependencies import get_proxy_service_for_app + + thread_owner_chatgpt_id = "acc_file_pin_thread_owner" + file_owner_chatgpt_id = "acc_file_pin_file_owner" + thread_owner_id = await _import_account( + async_client, + thread_owner_chatgpt_id, + "file-pin-thread-owner@example.com", + ) + file_owner_id = await _import_account( + async_client, + file_owner_chatgpt_id, + "file-pin-file-owner@example.com", + ) + process_session = "file-pin-process" + thread_headers = {"session-id": process_session, "thread-id": "file-pin-thread"} + sibling_headers = {"session-id": process_session, "thread-id": "file-pin-sibling"} + thread_key = _codex_backend_identity(thread_headers).thread_selection_key + sibling_key = _codex_backend_identity(sibling_headers).thread_selection_key + process_key = _codex_session_selection_key(process_session) + assert thread_key is not None + assert sibling_key is not None + + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert( + thread_key, + thread_owner_id, + kind=StickySessionKind.PROMPT_CACHE, + ) + + service = get_proxy_service_for_app(async_client._transport.app) + await service._pin_file_account("file_thread_locality", file_owner_id) + seen: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, **kwargs): + del payload, headers, access_token, kwargs + seen.append(account_id) + yield 'data: {"type":"response.completed","response":{"id":"resp_file_pin_thread"}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + pinned_response = await async_client.post( + "/backend-api/codex/responses", + headers=thread_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Read the file."}, + {"type": "input_file", "file_id": "file_thread_locality"}, + ], + } + ], + "stream": True, + }, + ) + assert pinned_response.status_code == 200 + assert seen == [file_owner_chatgpt_id] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + assert await repo.get_account_id(thread_key, kind=StickySessionKind.PROMPT_CACHE) == thread_owner_id + assert await repo.get_account_id(process_key, kind=StickySessionKind.CODEX_SESSION) == file_owner_id + assert await repo.get_account_id(sibling_key, kind=StickySessionKind.PROMPT_CACHE) is None + + unpinned_response = await async_client.post( + "/backend-api/codex/responses", + headers=thread_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": "Continue without the file.", + "stream": True, + }, + ) + assert unpinned_response.status_code == 200 + assert seen == [file_owner_chatgpt_id, thread_owner_chatgpt_id] + + sibling_response = await async_client.post( + "/backend-api/codex/responses", + headers=sibling_headers, + json={ + "model": "gpt-5.2", + "instructions": "You are a helpful assistant.", + "input": "Sibling thread without a file.", + "stream": True, + }, + ) + assert sibling_response.status_code == 200 + assert seen == [file_owner_chatgpt_id, thread_owner_chatgpt_id, file_owner_chatgpt_id] + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + assert await repo.get_account_id(thread_key, kind=StickySessionKind.PROMPT_CACHE) == thread_owner_id + assert await repo.get_account_id(process_key, kind=StickySessionKind.CODEX_SESSION) == file_owner_id + assert await repo.get_account_id(sibling_key, kind=StickySessionKind.PROMPT_CACHE) == file_owner_id + + @pytest.mark.asyncio async def test_derived_prompt_cache_key_does_not_block_file_id_pin(async_client): """Regression: a ``prompt_cache_key`` that the proxy itself derived diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index 685f22fae1..5bf62dcbc7 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -29,7 +29,11 @@ from app.core.crypto import TokenEncryptor from app.db.models import Account, AccountStatus, StickySessionKind, UsageHistory from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.proxy.affinity import _codex_backend_identity, _codex_session_selection_key +from app.modules.proxy.affinity import ( + _AffinityPolicy, + _codex_backend_identity, + _codex_session_selection_key, +) from app.modules.proxy.cap_partitioning import CapPartition from app.modules.proxy.load_balancer import LoadBalancer, RuntimeState, effective_account_concurrency_caps from app.modules.proxy.repo_bundle import ProxyRepositories @@ -3023,6 +3027,106 @@ async def test_first_codex_thread_initializes_process_preference_once_for_later_ ] +@pytest.mark.asyncio +async def test_required_file_owner_does_not_rewrite_existing_thread_row() -> None: + balancer, thread_owner, file_owner, sticky_repo = _make_cap_spillover_balancer("file-pin-thread") + assert file_owner is not None + process_session = "file-pin-process" + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-thread"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: thread_owner.id} + preferred = _AffinityPolicy.preferred_owner_sticky_inputs( + thread_key, + StickySessionKind.PROMPT_CACHE, + False, + 300, + "thread_header", + process_session, + ) + + selected = await balancer.select_account( + sticky_key=preferred[0], + sticky_kind=preferred[1], + reallocate_sticky=preferred[2], + sticky_max_age_seconds=preferred[3], + sticky_source=preferred[4], + legacy_sticky_key=preferred[5], + required_account_id=file_owner.id, + required_account_is_ownership_constraint=True, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + + assert selected.account is not None + assert selected.account.id == file_owner.id + assert sticky_repo.account_ids_by_key == {thread_key: thread_owner.id} + assert sticky_repo.upserts == [] + await balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_required_file_owner_seeds_process_preference_for_later_sibling() -> None: + balancer, thread_owner, file_owner, sticky_repo = _make_cap_spillover_balancer("file-pin-seed") + assert file_owner is not None + process_session = "file-pin-seed-process" + process_key = _codex_session_selection_key(process_session) + first_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-first"} + ).thread_selection_key + sibling_thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "file-pin-sibling"} + ).thread_selection_key + assert first_thread_key is not None + assert sibling_thread_key is not None + sticky_repo.account_ids_by_key = {} + preferred = _AffinityPolicy.preferred_owner_sticky_inputs( + first_thread_key, + StickySessionKind.PROMPT_CACHE, + False, + 300, + "thread_header", + process_session, + ) + + first = await balancer.select_account( + sticky_key=preferred[0], + sticky_kind=preferred[1], + reallocate_sticky=preferred[2], + sticky_max_age_seconds=preferred[3], + sticky_source=preferred[4], + legacy_sticky_key=preferred[5], + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + required_account_id=file_owner.id, + required_account_is_ownership_constraint=True, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert first.account is not None + assert first.account.id == file_owner.id + assert first_thread_key not in (sticky_repo.account_ids_by_key or {}) + assert sticky_repo.account_ids_by_key == {process_key: file_owner.id} + + sibling = await balancer.select_account( + sticky_key=sibling_thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert sibling.account is not None + assert sibling.account.id == file_owner.id + assert sticky_repo.account_ids_by_key[process_key] == file_owner.id + await balancer.release_account_lease(first.lease) + await balancer.release_account_lease(sibling.lease) + + @pytest.mark.asyncio async def test_legacy_raw_process_owner_wins_over_thread_locality() -> None: balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-legacy-owner") diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 3611c1a17e..6561ee9582 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -38391,7 +38391,10 @@ async def test_select_account_with_budget_keeps_thread_seed_for_first_exact_owne select_account.assert_awaited_once() assert select_account.await_args is not None assert select_account.await_args.kwargs["required_account_id"] == owner.id - assert select_account.await_args.kwargs["sticky_key"] == thread_policy.selection_key + assert select_account.await_args.kwargs["sticky_key"] is None + assert select_account.await_args.kwargs["sticky_kind"] == proxy_service.StickySessionKind.CODEX_SESSION + assert select_account.await_args.kwargs["sticky_source"] == "thread_header" + assert select_account.await_args.kwargs["legacy_sticky_key"] == "process-first-exact-owner" assert select_account.await_args.kwargs["sticky_seed_key"] == process_key assert select_account.await_args.kwargs["sticky_seed_kind"] == proxy_service.StickySessionKind.CODEX_SESSION From 3f66c288a230d6eb73c39b397c558427c21e167f Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:45:50 +0200 Subject: [PATCH 034/117] fix(usage): settle live snapshots after account consolidation (#1773) * fix(usage): settle live snapshots after account consolidation * test(http-bridge): include upstream identity in session fixture * fix(accounts): serialize upstream identity membership * fix(accounts): close identity lock review findings * fix(usage): relock current snapshot owner identity --- app/core/clients/proxy.py | 6 +- app/db/account_identity_lock.py | 56 ++ app/modules/accounts/repository.py | 174 ++++- .../_service/http_bridge/upstream_events.py | 1 + app/modules/usage/live_ingest.py | 88 +-- app/modules/usage/repository.py | 179 +++++- .../design.md | 177 +++++ .../proposal.md | 47 ++ .../specs/account-identity/spec.md | 23 + .../specs/live-usage-ingestion/spec.md | 77 +++ .../tasks.md | 72 +++ tests/integration/test_live_usage_ingest.py | 602 +++++++++++++++++- tests/integration/test_repositories.py | 34 + tests/unit/test_accounts_repository_locks.py | 215 ++++++- tests/unit/test_live_snapshot_owner_relock.py | 132 ++++ tests/unit/test_live_usage_ingest.py | 5 +- tests/unit/test_proxy_http_bridge.py | 21 +- tests/unit/test_usage_snapshot_repository.py | 40 ++ 18 files changed, 1821 insertions(+), 128 deletions(-) create mode 100644 app/db/account_identity_lock.py create mode 100644 openspec/changes/settle-live-usage-after-account-consolidation/design.md create mode 100644 openspec/changes/settle-live-usage-after-account-consolidation/proposal.md create mode 100644 openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md create mode 100644 openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md create mode 100644 openspec/changes/settle-live-usage-after-account-consolidation/tasks.md create mode 100644 tests/unit/test_live_snapshot_owner_relock.py diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index 5fb1b15d67..fe50a4085e 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -2690,7 +2690,7 @@ async def stream_responses( publish_live_usage( parse_rate_limit_event_text(event_block), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) yield event_block @@ -2861,7 +2861,7 @@ async def _stream_via_http_attempt( publish_live_usage( parse_rate_limit_headers(getattr(raw_resp, "headers", None)), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) if resp.status >= 400: if raise_for_status: @@ -2955,7 +2955,7 @@ async def _stream_via_http_attempt( publish_live_usage( parse_rate_limit_headers(getattr(resp, "headers", None)), account_id=codex_lb_account_id, - chatgpt_account_id=None if codex_lb_account_id else account_id, + chatgpt_account_id=account_id, ) if resp.status >= 400: if raise_for_status: diff --git a/app/db/account_identity_lock.py b/app/db/account_identity_lock.py new file mode 100644 index 0000000000..a8fd57aedc --- /dev/null +++ b/app/db/account_identity_lock.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from collections.abc import Collection +from hashlib import sha256 + +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +_POSTGRES_ACCOUNT_IDENTITY_LOCK_TIMEOUT_MS = 30_000 + + +def advisory_lock_key(scope: str, value: str) -> int: + digest = sha256(f"{scope}:{value}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], byteorder="big", signed=True) + + +def account_identity_lock_key(chatgpt_account_id: str) -> int: + """Return the existing PostgreSQL lock namespace for one upstream identity.""" + return advisory_lock_key("account-id", f"chatgpt:{chatgpt_account_id}") + + +async def lock_postgresql_account_identities( + session: AsyncSession, + chatgpt_account_ids: Collection[str | None], +) -> tuple[int, ...]: + """Lock upstream identity membership in canonical transaction-scoped order.""" + bind = session.get_bind() + if bind is None or bind.dialect.name != "postgresql": + return () + lock_keys = tuple( + sorted( + { + account_identity_lock_key(chatgpt_account_id) + for chatgpt_account_id in chatgpt_account_ids + if chatgpt_account_id + } + ) + ) + try: + if lock_keys: + # Match the database layer's existing 30-second contention budget. + # Transaction-local scope bounds every subsequent lock in this unit + # of work and PostgreSQL restores it at transaction end. + await session.execute( + text("SELECT set_config('lock_timeout', :timeout, true)"), + {"timeout": f"{_POSTGRES_ACCOUNT_IDENTITY_LOCK_TIMEOUT_MS}ms"}, + ) + for lock_key in lock_keys: + await session.execute( + text("SELECT pg_advisory_xact_lock(:lock_key)"), + {"lock_key": lock_key}, + ) + except BaseException: + await session.rollback() + raise + return lock_keys diff --git a/app/modules/accounts/repository.py b/app/modules/accounts/repository.py index c56039e80a..3d1b62fd24 100644 --- a/app/modules/accounts/repository.py +++ b/app/modules/accounts/repository.py @@ -1,6 +1,5 @@ from __future__ import annotations -import hashlib import json import uuid from dataclasses import dataclass @@ -15,6 +14,7 @@ from app.core.upstream_proxy.cache import get_upstream_route_cache from app.core.utils.time import utcnow +from app.db.account_identity_lock import advisory_lock_key, lock_postgresql_account_identities from app.db.models import ( Account, AccountLimitWarmup, @@ -74,6 +74,10 @@ def __init__(self, email: str) -> None: ) +class AccountIdentityRelockError(RuntimeError): + """Raised after identity membership changes across both bounded lock attempts.""" + + class AccountsRepository: def __init__(self, session: AsyncSession) -> None: self._session = session @@ -197,6 +201,7 @@ async def _upsert_unlocked( *, merge_by_email: bool | None = None, merge_by_chatgpt_identity: bool = False, + _identity_lock_attempt: int = 0, ) -> Account: dialect_name = self._dialect_name() sqlite_lock_acquired = False @@ -212,27 +217,34 @@ async def _upsert_unlocked( # exclusive on this dialect. await self._acquire_sqlite_merge_lock() elif dialect_name == "postgresql": - # Identity-keyed advisory lock must always be acquired when - # identity reconciliation is in play, regardless of - # merge_by_email. Two concurrent reauths for the same - # upstream chatgpt_account_id but different email claims - # (e.g. user changed email upstream) would otherwise take - # different email-scoped locks, both miss the canonical-row - # lookup below, and both INSERT a duplicate row for the - # same identity. - # - # Ordering identity-first, then email, gives a stable - # acquisition order across all callers so two concurrent - # reauths that overlap on either key serialize without - # deadlock. - identity_locked = False - if merge_by_chatgpt_identity and account.chatgpt_account_id: - await self._acquire_postgresql_identity_lock(f"chatgpt:{account.chatgpt_account_id}") - identity_locked = True + # Upstream identity membership always serializes before email + # locks and account row locks. This applies to ordinary imports as + # well as explicit identity reconciliation because either can add, + # replace, or remove a membership used by live-usage fallback. + locked_identities = await self._lock_postgresql_upsert_identity_candidates( + account, + include_email=bool(merge_by_email), + ) if merge_by_email: await self._acquire_postgresql_merge_lock(account.email) - elif not identity_locked: + elif not locked_identities: await self._acquire_postgresql_identity_lock(account.id) + if not await self._postgresql_upsert_identity_candidates_are_locked( + account, + include_email=bool(merge_by_email), + locked_identities=locked_identities, + ): + await self._session.rollback() + if _identity_lock_attempt >= 1: + raise AccountIdentityRelockError( + "Account identity candidates changed during PostgreSQL upsert locking" + ) + return await self._upsert_unlocked( + account, + merge_by_email=merge_by_email, + merge_by_chatgpt_identity=merge_by_chatgpt_identity, + _identity_lock_attempt=_identity_lock_attempt + 1, + ) # Identity-aware reconciliation runs before the deterministic-id # check so that a deactivated row whose refresh token was revoked @@ -302,7 +314,13 @@ async def upsert_reauthorized(self, account: Account) -> Account: async def replace_reauthorized(self, account_id: str, account: Account) -> Account | None: """Replace credentials on the exact local row selected for reauthentication.""" async with sqlite_writer_section(): - existing = await self._session.get(Account, account_id) + if self._dialect_name() == "postgresql": + existing = await self._lock_postgresql_account_identity_membership( + account_id, + account.chatgpt_account_id, + ) + else: + existing = await self._session.get(Account, account_id) if existing is None: return None await self._apply_account_replacement(existing, account) @@ -344,6 +362,7 @@ async def _upsert_account_slot_unlocked( *, preserve_unknown_workspace_duplicates: bool | None = None, preserve_identity_slots: bool = False, + _identity_lock_attempt: int = 0, ) -> Account: if preserve_unknown_workspace_duplicates is None: preserve_unknown_workspace_duplicates = not await self._merge_by_email_enabled() @@ -351,6 +370,10 @@ async def _upsert_account_slot_unlocked( if dialect_name == "sqlite": await self._acquire_sqlite_merge_lock() elif dialect_name == "postgresql": + locked_identities = await self._lock_postgresql_upsert_identity_candidates( + account, + include_email=True, + ) for lock_key in sorted( _slot_lock_keys( account, @@ -358,6 +381,22 @@ async def _upsert_account_slot_unlocked( ) ): await self._acquire_postgresql_identity_lock(lock_key) + if not await self._postgresql_upsert_identity_candidates_are_locked( + account, + include_email=True, + locked_identities=locked_identities, + ): + await self._session.rollback() + if _identity_lock_attempt >= 1: + raise AccountIdentityRelockError( + "Account identity candidates changed during PostgreSQL slot locking" + ) + return await self._upsert_account_slot_unlocked( + account, + preserve_unknown_workspace_duplicates=preserve_unknown_workspace_duplicates, + preserve_identity_slots=preserve_identity_slots, + _identity_lock_attempt=_identity_lock_attempt + 1, + ) existing = await self._account_by_slot_identity(account) if existing: @@ -779,6 +818,10 @@ async def update_routing_policy(self, account_id: str, routing_policy: str) -> b async def delete(self, account_id: str, *, delete_history: bool = False) -> bool: async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + # Identity membership precedes the fold-state lock so live + # settlement and deletion cannot form an identity/fold cycle. + await self._lock_postgresql_account_identity_membership(account_id, None) # Serialize against fold passes before touching the account's # request logs: without the fold-state lock an in-flight hourly # slice could aggregate the pre-delete attribution but commit @@ -843,6 +886,8 @@ async def rotate_tokens( material at all). """ async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + await self._lock_postgresql_account_identity_membership(account_id, chatgpt_account_id) values: dict[str, bytes | datetime | str] = { "access_token_encrypted": access_token_encrypted, "refresh_token_encrypted": refresh_token_encrypted, @@ -901,6 +946,8 @@ async def update_account_metadata( no-op existence check. """ async with sqlite_writer_section(): + if self._dialect_name() == "postgresql": + await self._lock_postgresql_account_identity_membership(account_id, chatgpt_account_id) values: dict[str, str | datetime] = {} if plan_type is not None: values["plan_type"] = plan_type @@ -1047,6 +1094,75 @@ async def _account_by_slot_identity(self, account: Account) -> Account | None: return matched return None + async def _lock_postgresql_account_identity_membership( + self, + account_id: str, + incoming_chatgpt_account_id: str | None, + *, + second_attempt: bool = False, + ) -> Account | None: + """Lock one row's old/new upstream memberships before mutating it.""" + observed_identity = await self._session.scalar( + select(Account.chatgpt_account_id).where(Account.id == account_id) + ) + await lock_postgresql_account_identities( + self._session, + (observed_identity, incoming_chatgpt_account_id), + ) + locked_account = await self._session.scalar( + select(Account) + .where(Account.id == account_id) + # PostgreSQL FOR NO KEY UPDATE stabilizes identity membership but + # remains compatible with the KEY SHARE lock taken by concurrent + # rollup FK inserts. Deletion upgrades only after the fold lock. + .with_for_update(key_share=True) + .execution_options(populate_existing=True) + ) + locked_identity = locked_account.chatgpt_account_id if locked_account is not None else None + if locked_identity == observed_identity: + return locked_account + await self._session.rollback() + if second_attempt: + raise AccountIdentityRelockError("Account identity changed during PostgreSQL membership lock acquisition") + return await self._lock_postgresql_account_identity_membership( + account_id, + incoming_chatgpt_account_id, + second_attempt=True, + ) + + async def _lock_postgresql_upsert_identity_candidates( + self, + account: Account, + *, + include_email: bool, + ) -> frozenset[str]: + predicates = _upsert_identity_candidate_predicates(account, include_email=include_email) + observed = ( + (await self._session.execute(select(Account.chatgpt_account_id).where(or_(*predicates)))).scalars().all() + ) + identities = frozenset(identity for identity in (*observed, account.chatgpt_account_id) if identity) + await lock_postgresql_account_identities(self._session, identities) + return identities + + async def _postgresql_upsert_identity_candidates_are_locked( + self, + account: Account, + *, + include_email: bool, + locked_identities: frozenset[str], + ) -> bool: + predicates = _upsert_identity_candidate_predicates(account, include_email=include_email) + current = ( + ( + await self._session.execute( + select(Account.chatgpt_account_id).where(or_(*predicates)).with_for_update(key_share=True) + ) + ) + .scalars() + .all() + ) + return all(identity is None or identity in locked_identities for identity in current) + def _dialect_name(self) -> str: return self._session.get_bind().dialect.name @@ -1062,14 +1178,14 @@ async def _acquire_sqlite_merge_lock(self) -> None: await self._session.execute(text("UPDATE accounts SET id = id WHERE 1 = 0")) async def _acquire_postgresql_merge_lock(self, email: str) -> None: - lock_key = _advisory_lock_key("merge-email", email) + lock_key = advisory_lock_key("merge-email", email) await self._session.execute( text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key}, ) async def _acquire_postgresql_identity_lock(self, account_id: str) -> None: - lock_key = _advisory_lock_key("account-id", account_id) + lock_key = advisory_lock_key("account-id", account_id) await self._session.execute( text("SELECT pg_advisory_xact_lock(:lock_key)"), {"lock_key": lock_key}, @@ -1125,6 +1241,15 @@ def _slot_lock_keys(account: Account, *, preserve_unknown_workspace_duplicates: return (f"slot-local:{account.id}",) +def _upsert_identity_candidate_predicates(account: Account, *, include_email: bool) -> list[Any]: + predicates = [Account.id == account.id] + if account.chatgpt_account_id: + predicates.append(Account.chatgpt_account_id == account.chatgpt_account_id) + if include_email and account.email: + predicates.append(Account.email == account.email) + return predicates + + def _same_unknown_workspace_identity(existing: Account, incoming: Account) -> bool: return ( _workspace_slot_key(existing) is None @@ -1176,8 +1301,3 @@ def _can_reuse_email_fallback(existing: Account, incoming: Account) -> bool: or not existing.chatgpt_account_id or existing.chatgpt_account_id == incoming.chatgpt_account_id ) - - -def _advisory_lock_key(scope: str, value: str) -> int: - digest = hashlib.sha256(f"{scope}:{value}".encode("utf-8")).digest() - return int.from_bytes(digest[:8], byteorder="big", signed=True) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 3d8ba2d0fb..351b3fb54f 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -1369,6 +1369,7 @@ async def _relay_http_bridge_upstream_messages( publish_live_usage( parse_rate_limit_event_text(message.text), account_id=session.account.id, + chatgpt_account_id=session.account.chatgpt_account_id, ) await self._process_http_bridge_upstream_text(session, message.text) if await self._retire_http_bridge_after_drain_if_ready(session): diff --git a/app/modules/usage/live_ingest.py b/app/modules/usage/live_ingest.py index d26c2e7e2c..b36a5ea525 100644 --- a/app/modules/usage/live_ingest.py +++ b/app/modules/usage/live_ingest.py @@ -5,22 +5,17 @@ import time from dataclasses import dataclass -from sqlalchemy import select - from app.core import usage as usage_core from app.core.config.settings import get_settings from app.core.usage.live_hub import register_live_usage_publisher from app.core.usage.live_snapshots import LiveRateLimitSnapshot, LiveUsageWindow -from app.db.models import Account from app.db.session import get_background_session from app.modules.proxy.account_cache import get_account_selection_cache from app.modules.proxy.rate_limit_cache import get_rate_limit_headers_cache -from app.modules.usage.repository import UsageRepository +from app.modules.usage.repository import UsageRepository, UsageWindowWrite logger = logging.getLogger(__name__) -_RESOLUTION_TTL_SECONDS = 300.0 - # Write-coalescing tuning (fixed; issue #1340 / PRINCIPLES.md P2). The # ingestor keeps both as constructor fields so tests can exercise queue # overflow and coalescing with small values. @@ -68,7 +63,6 @@ def __init__( self._queue: asyncio.Queue[_QueuedSnapshot] = asyncio.Queue(maxsize=max(1, queue_size)) self._write_min_interval_seconds = write_min_interval_seconds self._last_write: dict[str, tuple[tuple[object, ...], float]] = {} - self._resolution_cache: dict[str, tuple[str | None, float]] = {} self._consumer: asyncio.Task[None] | None = None self._dropped = 0 self._last_cache_invalidation = 0.0 @@ -141,14 +135,6 @@ async def _run(self) -> None: ) async def _ingest(self, item: _QueuedSnapshot) -> None: - account_id = item.account_id - if account_id is None: - account_id = await self._resolve_account_id(item.chatgpt_account_id) - if account_id is None: - return - if self._should_skip(account_id, item.snapshot): - return - snapshot = item.snapshot primary = snapshot.primary secondary = snapshot.secondary @@ -162,51 +148,56 @@ async def _ingest(self, item: _QueuedSnapshot) -> None: and primary.window_minutes == usage_core.DEFAULT_WINDOW_MINUTES_MONTHLY ): monthly, primary = primary, None - async with get_background_session() as session: - repo = UsageRepository(session) - if primary is not None: - await repo.add_entry( - account_id=account_id, - used_percent=float(primary.used_percent), - input_tokens=None, - output_tokens=None, + windows: list[UsageWindowWrite] = [] + if primary is not None: + windows.append( + UsageWindowWrite( window="primary", + used_percent=float(primary.used_percent), reset_at=primary.reset_at, window_minutes=primary.window_minutes, credits_has=snapshot.credits_has, credits_unlimited=snapshot.credits_unlimited, credits_balance=snapshot.credits_balance, ) - if secondary is not None: - # Mirror the poller: credits normally ride the primary row. - # A secondary-only snapshot (e.g. the short window is not - # being reported) must still carry the fresh credit state. - secondary_carries_credits = primary is None - await repo.add_entry( - account_id=account_id, - used_percent=float(secondary.used_percent), - input_tokens=None, - output_tokens=None, + ) + if secondary is not None: + # Mirror the poller: credits normally ride the primary row. A + # secondary-only snapshot must still carry fresh credit state. + secondary_carries_credits = primary is None + windows.append( + UsageWindowWrite( window="secondary", + used_percent=float(secondary.used_percent), reset_at=secondary.reset_at, window_minutes=secondary.window_minutes, credits_has=snapshot.credits_has if secondary_carries_credits else None, credits_unlimited=snapshot.credits_unlimited if secondary_carries_credits else None, credits_balance=snapshot.credits_balance if secondary_carries_credits else None, ) - if monthly is not None: - await repo.add_entry( - account_id=account_id, - used_percent=float(monthly.used_percent), - input_tokens=None, - output_tokens=None, + ) + if monthly is not None: + windows.append( + UsageWindowWrite( window="monthly", + used_percent=float(monthly.used_percent), reset_at=monthly.reset_at, window_minutes=monthly.window_minutes, credits_has=snapshot.credits_has, credits_unlimited=snapshot.credits_unlimited, credits_balance=snapshot.credits_balance, ) + ) + + async with get_background_session() as session: + account_id = await UsageRepository(session).settle_live_account_snapshot( + account_id=item.account_id, + chatgpt_account_id=item.chatgpt_account_id, + windows=windows, + should_skip=lambda resolved: self._should_skip(resolved, snapshot), + ) + if account_id is None: + return self._last_write[account_id] = (_fingerprint(snapshot), time.monotonic()) await self._invalidate_caches_throttled() @@ -236,25 +227,6 @@ async def _invalidate_caches_now(self) -> None: # values before the TTL expires. await get_rate_limit_headers_cache().invalidate() - async def _resolve_account_id(self, chatgpt_account_id: str | None) -> str | None: - if not chatgpt_account_id: - return None - cached = self._resolution_cache.get(chatgpt_account_id) - now = time.monotonic() - if cached is not None and now - cached[1] < _RESOLUTION_TTL_SECONDS: - return cached[0] - async with get_background_session() as session: - rows = ( - (await session.execute(select(Account.id).where(Account.chatgpt_account_id == chatgpt_account_id))) - .scalars() - .all() - ) - # Ambiguous identities (multiple workspace slots) are dropped rather - # than guessed; the poller stays authoritative for them. - resolved = rows[0] if len(rows) == 1 else None - self._resolution_cache[chatgpt_account_id] = (resolved, now) - return resolved - _ingestor: LiveUsageIngestor | None = None diff --git a/app/modules/usage/repository.py b/app/modules/usage/repository.py index 3626bd34f5..2965c3efdd 100644 --- a/app/modules/usage/repository.py +++ b/app/modules/usage/repository.py @@ -7,16 +7,17 @@ from datetime import datetime from hashlib import sha256 from threading import RLock -from typing import Any, cast +from typing import Any, Callable, cast from anyio import to_thread -from sqlalchemy import Integer, and_, delete, func, literal_column, or_, select, true, tuple_ +from sqlalchemy import Integer, and_, delete, func, literal_column, or_, select, text, true, tuple_ from sqlalchemy import cast as sqlalchemy_cast from sqlalchemy.ext.asyncio import AsyncSession from app.core.config.settings import get_settings from app.core.usage.types import UsageAggregateRow, UsageTrendBucket from app.core.utils.time import utcnow +from app.db.account_identity_lock import lock_postgresql_account_identities from app.db.models import Account, AdditionalUsageHistory, UsageHistory from app.db.session import relax_commit_durability, sqlite_writer_section from app.db.sqlite_utils import sqlite_db_path_from_url @@ -50,6 +51,35 @@ class UsageWindowWrite: credits_balance: float | None = None +class LiveSnapshotOwnerIdentityRelockError(RuntimeError): + """The selected live-snapshot owner's identity changed twice.""" + + +def _account_snapshot_entries( + account_id: str, + windows: Collection[UsageWindowWrite], + *, + recorded_at: datetime | None = None, +) -> list[UsageHistory]: + captured_at = recorded_at or utcnow() + return [ + UsageHistory( + account_id=account_id, + used_percent=window.used_percent, + input_tokens=None, + output_tokens=None, + window=window.window, + reset_at=window.reset_at, + window_minutes=window.window_minutes, + credits_has=window.credits_has, + credits_unlimited=window.credits_unlimited, + credits_balance=window.credits_balance, + recorded_at=captured_at, + ) + for window in windows + ] + + @dataclass(frozen=True, slots=True) class _BulkHistoryCacheMetadata: row_count: int @@ -635,23 +665,7 @@ async def add_account_snapshot( """Persist one account's standard usage windows atomically.""" if not windows: return [] - captured_at = recorded_at or utcnow() - entries = [ - UsageHistory( - account_id=account_id, - used_percent=window.used_percent, - input_tokens=None, - output_tokens=None, - window=window.window, - reset_at=window.reset_at, - window_minutes=window.window_minutes, - credits_has=window.credits_has, - credits_unlimited=window.credits_unlimited, - credits_balance=window.credits_balance, - recorded_at=captured_at, - ) - for window in windows - ] + entries = _account_snapshot_entries(account_id, windows, recorded_at=recorded_at) try: async with sqlite_writer_section(): # Telemetry write: this transaction only appends usage-history @@ -664,6 +678,133 @@ async def add_account_snapshot( raise return entries + async def _resolve_postgresql_live_snapshot_owner( + self, + account_id: str | None, + chatgpt_account_id: str | None, + ) -> str | None: + locked_identities = (chatgpt_account_id,) + fallback_identity = chatgpt_account_id + relocked = False + + while True: + await lock_postgresql_account_identities(self._session, locked_identities) + locked_identity_values = frozenset(identity for identity in locked_identities if identity) + identity_to_relock: str | None = None + + if account_id is not None: + # Read before taking the row lock so MVCC preserves the + # current recovery identity even when its writer has already + # deleted the local row but not committed yet. + observed = ( + await self._session.execute( + select(Account.id, Account.chatgpt_account_id).where(Account.id == account_id) + ) + ).one_or_none() + if observed is not None: + observed_identity = observed.chatgpt_account_id + if observed_identity and observed_identity not in locked_identity_values: + identity_to_relock = observed_identity + else: + locked = ( + await self._session.execute( + select(Account.id, Account.chatgpt_account_id) + .where(Account.id == account_id) + .with_for_update(key_share=True) + ) + ).one_or_none() + if locked is not None: + if locked.chatgpt_account_id and locked.chatgpt_account_id not in locked_identity_values: + identity_to_relock = locked.chatgpt_account_id + else: + return locked.id + + if identity_to_relock is not None: + if relocked: + raise LiveSnapshotOwnerIdentityRelockError( + "Live snapshot owner identity changed during PostgreSQL relock" + ) + # Release the first lock before adding another identity; the + # shared helper can then reacquire the full set in canonical + # order without inverting an account writer's lock order. + await self._session.rollback() + fallback_identity = identity_to_relock + locked_identities = (chatgpt_account_id, identity_to_relock) + relocked = True + continue + + if fallback_identity: + upstream_stmt = ( + select(Account.id) + .where(Account.chatgpt_account_id == fallback_identity) + .with_for_update(key_share=True) + ) + matches = list((await self._session.execute(upstream_stmt)).scalars().all()) + if len(matches) == 1: + return matches[0] + return None + + async def settle_live_account_snapshot( + self, + *, + account_id: str | None, + chatgpt_account_id: str | None, + windows: Collection[UsageWindowWrite], + should_skip: Callable[[str], bool], + ) -> str | None: + """Resolve a live snapshot owner and atomically persist its windows.""" + if not windows: + return None + + try: + async with sqlite_writer_section(): + bind = self._session.get_bind() + dialect_name = bind.dialect.name if bind is not None else "sqlite" + if dialect_name == "sqlite": + # Acquire SQLite's database-wide writer slot before owner + # lookup. Consolidation then commits before this lookup or + # waits until the snapshot commit, so the chosen FK owner + # cannot disappear between SELECT and INSERT. + await self._session.execute(text("BEGIN IMMEDIATE")) + resolved_account_id = None + if account_id is not None: + resolved_account_id = await self._session.scalar( + select(Account.id).where(Account.id == account_id) + ) + if resolved_account_id is None and chatgpt_account_id: + matches = list( + ( + await self._session.execute( + select(Account.id).where(Account.chatgpt_account_id == chatgpt_account_id) + ) + ) + .scalars() + .all() + ) + if len(matches) == 1: + resolved_account_id = matches[0] + else: + resolved_account_id = await self._resolve_postgresql_live_snapshot_owner( + account_id, + chatgpt_account_id, + ) + + if resolved_account_id is None or should_skip(resolved_account_id): + await self._session.rollback() + return None + + entries = _account_snapshot_entries(resolved_account_id, windows) + # Telemetry write: this transaction only locks the owner and + # appends usage-history rows, so it may skip synchronous WAL + # flush just like add_account_snapshot(). + await relax_commit_durability(self._session) + self._session.add_all(entries) + await self._session.commit() + except BaseException: + await self._session.rollback() + raise + return resolved_account_id + async def aggregate_since( self, since: datetime, diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/design.md b/openspec/changes/settle-live-usage-after-account-consolidation/design.md new file mode 100644 index 0000000000..ad42f7de6c --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/design.md @@ -0,0 +1,177 @@ +## Context + +Live usage publication and account reconciliation run in different ownership +domains. The proxy captures a snapshot and enqueues it without waiting; the +single background consumer later opens its own database session. Meanwhile, +identity-aware account upsert can select canonical account `C`, reparent the +persisted children of duplicate `D`, and delete `D` in one transaction. + +The loss sequence is therefore deterministic: publication records only `D`; +consolidation commits `D -> C`; `_ingest` attempts to append with stale `D`; +the account foreign key rejects the write; and the serving-safe consumer logs +and drops it. Existing history reparenting cannot cover a row that did not +exist when consolidation ran. + +The relevant identity constraint is equally important: an upstream ChatGPT +account id can be shared by distinct real-email slots. Upstream identity is a +safe fallback only when it resolves to exactly one surviving local row. + +## Goals / Non-Goals + +**Goals:** + +- Preserve every already-captured live snapshot across same-slot duplicate + consolidation when one canonical owner survives. +- Keep publication non-blocking and persistence in the background consumer. +- Prefer a valid captured local owner; use upstream identity only to recover a + stale or absent local owner and only when the result is unique. +- Persist each accepted snapshot once under one owner, with all represented + windows committed atomically. +- Prove the stale-local, valid-local, and upstream-only paths without sleeps or + timing-dependent scheduling. + +**Non-Goals:** + +- Changing duplicate-account selection, shared-workspace slot preservation, or + canonical-account choice. +- Guessing between multiple local rows that share an upstream identity. +- Retrying arbitrary ingestion failures or changing the queue's drop-oldest, + throttling, or serving-path isolation behavior. +- Adding a schema migration, configuration flag, or API response field. + +## Decisions + +### D1: Queue an ownership envelope containing local and upstream identities + +Every proxy tap point that knows a local serving account and its upstream +ChatGPT account id will publish both. The queued item remains an in-memory typed +value containing `account_id`, `chatgpt_account_id`, and the snapshot; no +database or wire schema is introduced. Upstream-only callers continue to leave +the local id absent. + +Capturing the upstream id at publication time is necessary because `D` cannot +be queried after consolidation deletes it. Looking up the upstream id only +after detecting stale `D` would already have lost the recovery key. + +### D2: Select and protect the persistence owner at consume time + +Ingestion will settle ownership in this order: + +1. If the captured local id still identifies an account, select it even when + the upstream identity is absent, shared, or points at another candidate. +2. If the local id is absent or no longer exists, resolve the captured upstream + id against current account rows and select it only when exactly one row + survives. +3. If neither rule selects an owner, do not guess; retain the current logged, + serving-safe drop behavior. + +Owner selection and the atomic append of all represented usage windows belong +to one serialized write operation. SQLite acquires `BEGIN IMMEDIATE` before +lookup and keeps its database-wide writer serialization through commit. +PostgreSQL first acquires the existing transaction-scoped advisory-lock +namespace keyed by the captured upstream identity before owner lookup. It then +reads the local owner's current identity without a row lock. When that current +non-null identity is not covered, settlement rolls back to release the initial +lock, reacquires the captured/current identities through the shared canonical +sort, and reselects the owner. This rollback is required: acquiring the current +identity while retaining the captured lock could invert the account-writer lock +order. If reconciliation wins the current-identity lock and deletes the local +row, settlement uses the last observed current identity as the unique fallback. +The reselected owner is held `FOR NO KEY UPDATE` through the append. That row +lock blocks deletion and key-changing writes without blocking the `KEY SHARE` +lock taken by concurrent foreign-key inserts. One relock is allowed; a second +identity change raises a typed terminal error, and null identities add no lock +key. + +Every PostgreSQL writer that can add, replace, move, consolidate, or delete an +`Account.chatgpt_account_id` membership acquires that same upstream lock and +holds it through commit. Old and incoming non-null identities are converted to +the stable advisory keys and acquired in canonical sorted order before any +email/slot advisory locks, account row locks, fold-state lock, or writes. A +local-id writer first reads the current identity without a row lock, acquires +the sorted old/new identity locks, and then row-locks and re-reads the account; +a changed observation rolls back and repeats that lock acquisition at most +once. Upsert candidate changes use the same bounded rollback/restart before any +mutation. Membership re-reads use PostgreSQL `FOR NO KEY UPDATE`, which +stabilizes identity changes while remaining compatible with the `KEY SHARE` +locks taken by concurrent fold rollup foreign-key inserts; deletion upgrades +its lock only after acquiring the fold-state lock. The shared helper applies a +transaction-local 30-second PostgreSQL lock timeout before advisory acquisition, +so request and background transactions propagate lock contention instead of +waiting indefinitely; it performs no polling or retry. + +This ordering gives both legal interleavings the same outcome: a snapshot +committed before consolidation is included when history is reparented, while a +snapshot whose current-identity reconciliation wins first relocks and writes +directly to `C` after the local duplicate disappears. +The per-account fingerprint is evaluated against the selected current owner, +and the successful-write marker is updated only after the atomic append. One +queued item therefore cannot write once to stale `D` and again to `C`. + +### D3: Preserve account-slot ambiguity and consolidation policy + +The fallback reuses the existing unique-upstream resolution rule. Distinct +real-email slots sharing one ChatGPT workspace remain distinct and ambiguous; +the change does not merge them or choose one. Duplicate reconciliation keeps +its current email/workspace candidate filters and canonical selection. It only +runs when the incoming upstream identity is non-null, and its duplicate query +requires `Account.chatgpt_account_id == incoming_identity`; an identity-less +local row therefore cannot be selected or deleted as an identity-reconciliation +duplicate. It only needs to leave the canonical row's existing upstream +identity intact, which it already does. + +This choice rejects two alternatives: always preferring upstream identity +could cross account slots even while the serving local row is valid, and +changing consolidation to force uniqueness would violate the established +shared-workspace account-slot contract. + +### D4: Deterministic regression and authenticated surface QA + +The deterministic transaction regression captures a queued item for `D` with +the shared upstream identity and coordinates independent PostgreSQL sessions at +exact lock and commit events, with no sleep, polling delay, or retry. Database +assertions prove one row per represented window under `C`, no row under `D`, +and no duplicate snapshot in both transaction orderings. A composition test +also drives the real proxied SSE publication tap through the live hub and +background consumer after consolidation, awaiting the exact settlement event +with a bounded timeout. Separate controls prove that an existing local id wins +and that an upstream-only item still resolves uniquely. + +Manual QA will use an isolated database and authenticated backend, execute a +literal `curl -i` request to `GET /api/accounts`, and verify HTTP 200, one +canonical `C`, no `D`, and the injected primary and secondary usage values. +The database diff will independently show one canonical snapshot and no +duplicate-owned history. All QA processes, credentials, database files, ports, +and temporary artifacts will be removed after capture. + +## Risks / Trade-offs + +- **Shared upstream id remains ambiguous.** A stale item can still be dropped + when multiple real-email slots survive. This is deliberate: preserving slot + ownership is safer than attributing usage to the wrong account. +- **Captured upstream identity can be absent.** Publication preserves the valid + local id together with the nullable upstream field, so valid-local settlement + still succeeds. Identity reconciliation cannot delete that identity-less row: + reconciliation requires a non-null incoming identity and selects duplicates + by equality to it. If the local row is already stale, no upstream fallback can + be recovered; genuinely upstream-less callers retain that serving-safe drop. +- **Settlement races consolidation.** A selected-row lock protects a snapshot + when settlement wins the row, but a current-identity consolidator can win + first and delete the local owner while settlement holds only the stale + captured-identity lock. SQLite writer serialization and PostgreSQL's bounded + rollback/relock close both transaction orderings without acquiring locks out + of canonical order. +- **Atomic append changes failure granularity.** If one represented window + cannot be stored, none of that snapshot's windows commit. This is preferable + to a partial snapshot and supports exactly-once settlement. + +## Migration Plan + +Ship publication and ingestion changes atomically. There is no schema or data +migration and no backfill: only snapshots captured after deployment carry both +identities. Rollback reverts the code; existing in-memory queued items disappear +with process shutdown exactly as they do today. + +## Open Questions + +None. diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md b/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md new file mode 100644 index 0000000000..b53edb3ec6 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/proposal.md @@ -0,0 +1,47 @@ +## Why + +A live usage snapshot can be captured for duplicate local account `D` and wait +in the fire-and-forget queue while account reconciliation consolidates `D` into +canonical account `C`. Reconciliation reparents existing history and deletes +`D`, but the delayed ingestor still trusts the captured local id. Its +usage-history insert then violates the account foreign key and the serving-safe +consumer drops the already-captured snapshot. The invariant for this change is: +**an already-captured live snapshot survives duplicate-account consolidation.** + +## What Changes + +- Queue both the serving local account id and its upstream ChatGPT account id + when both identities are available at proxy publication time. +- Settle ownership at ingestion time: prefer a still-valid local account; + otherwise resolve the captured upstream identity only when it identifies one + surviving canonical account. +- Preserve the upstream-only publication path and the existing ambiguity rule + for shared-workspace identities. +- Persist one accepted snapshot atomically under the selected owner so a stale + `D` produces exactly one primary/secondary snapshot under `C` and no history + under `D`. +- Add deterministic, no-sleep regression coverage and authenticated + `/api/accounts` QA for the externally visible canonical result. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `live-usage-ingestion`: retain both ownership identities and settle queued + snapshots against the current account rows before persistence. +- `account-identity`: keep duplicate consolidation's canonical identity usable + for delayed ownership settlement without changing which accounts consolidate. + +## Impact + +- Affected code: live-usage publication call sites and hub contract, + `app/modules/usage/live_ingest.py`, and the existing atomic usage-snapshot + persistence path. +- Affected tests: focused live-ingestion integration coverage for stale-local, + valid-local, and upstream-only ownership paths. +- No database schema migration, new setting, API schema change, or account + consolidation policy change. diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md b/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md new file mode 100644 index 0000000000..4cbaa1fd9e --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/specs/account-identity/spec.md @@ -0,0 +1,23 @@ +## ADDED Requirements + +### Requirement: Duplicate consolidation preserves a recoverable canonical identity + +Identity reconciliation MUST preserve the upstream ChatGPT account id on the canonical row, reparent existing account-owned usage history to that row, and remove selected duplicate rows when it consolidates duplicate local accounts under the existing email and workspace-slot policy. Reconciliation MUST NOT consolidate distinct real-email account slots solely to make an upstream identity unique. On PostgreSQL, every account insertion, replacement, token/metadata identity update, duplicate consolidation, and deletion that changes upstream-identity membership MUST acquire the same transaction-scoped upstream-identity advisory lock as live-usage settlement before row or fold-state locks and hold it through commit. An old-to-new membership move MUST acquire both stable identity lock keys in canonical sorted order. + +#### Scenario: Same-slot duplicate leaves one upstream-resolvable canonical row + +- **GIVEN** canonical account `C` and duplicate account `D` are selected for consolidation by the existing identity policy +- **AND** both rows carry the same upstream ChatGPT account id +- **WHEN** reconciliation consolidates `D` into `C` +- **THEN** `C` remains with that upstream ChatGPT account id +- **AND** existing usage history formerly owned by `D` is owned by `C` +- **AND** `D` no longer exists +- **AND** the upstream ChatGPT account id resolves uniquely to `C` + +#### Scenario: Shared-workspace sibling slots remain distinct + +- **GIVEN** two current accounts have different real email addresses +- **AND** they share the same upstream ChatGPT account id +- **WHEN** identity reconciliation evaluates the accounts +- **THEN** it preserves both local account slots +- **AND** it does not consolidate either account solely to make upstream resolution unique diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md b/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md new file mode 100644 index 0000000000..0b1d75ccb9 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/specs/live-usage-ingestion/spec.md @@ -0,0 +1,77 @@ +## ADDED Requirements + +### Requirement: Captured live snapshots survive account consolidation + +The proxy MUST enqueue both the serving local account id and the upstream +ChatGPT account id when both are available. At consumption, live usage +ingestion MUST prefer the captured local id when it still identifies an +account. If that local id is absent or no longer exists, ingestion MUST use the +captured upstream id only when it resolves to exactly one current local account. +For a selected owner, ingestion MUST atomically persist no more than one history +row for each window represented by the queued snapshot. On PostgreSQL, +ingestion MUST acquire a transaction-scoped advisory lock keyed by the captured +upstream identity before either owner lookup and hold it through snapshot +commit. If the selected local owner's current non-null upstream identity is not +already locked, ingestion MUST roll back the initial transaction, reacquire the +captured and current identity locks in canonical sorted order, and reselect and +revalidate the owner before persistence. If that local owner was consolidated +while the current-identity lock was acquired, ingestion MUST use the last +observed current identity only when it resolves to exactly one surviving local +account. Ingestion MUST perform at most one such relock and MUST raise a typed +error if the selected owner's identity changes again; a null current identity +MUST NOT create an advisory-lock key. Every account writer that can change +membership in an upstream identity MUST acquire the same lock before row locks +or mutation and hold it through commit. Writers moving membership between two +non-null upstream identities MUST acquire both stable lock keys in canonical +sorted order. + +#### Scenario: Stale duplicate settles under the unique canonical account + +- **GIVEN** a primary/secondary live snapshot was queued for duplicate account `D` +- **AND** the queued item contains `D` and the upstream identity shared with canonical account `C` +- **AND** duplicate reconciliation reparents existing history to `C` and deletes `D` +- **WHEN** the queued snapshot is consumed +- **THEN** exactly one primary row and one secondary row are persisted under `C` +- **AND** no usage-history row is persisted under `D` +- **AND** the persisted values equal the captured snapshot + +#### Scenario: A valid local owner takes precedence + +- **GIVEN** a queued snapshot contains a local account id that still exists +- **AND** it also contains an upstream identity usable for fallback +- **WHEN** the queued snapshot is consumed +- **THEN** the snapshot is persisted under the captured local account +- **AND** ingestion does not substitute another account selected by the upstream identity + +#### Scenario: A selected owner's current identity is revalidated + +- **GIVEN** a queued snapshot contains local account `A` and captured identity `X` +- **AND** `A` currently belongs to identity `Y` +- **WHEN** settlement overlaps reconciliation of `A` into a canonical `Y` owner +- **THEN** settlement releases its initial `X` lock before acquiring the canonical sorted lock set for `X` and `Y` +- **AND** settlement reselects and revalidates the owner under that full lock set +- **AND** exactly one row per represented window survives under the canonical `Y` owner +- **AND** a second selected-owner identity change raises a typed terminal error without persisting the snapshot + +#### Scenario: Upstream-only publication still resolves + +- **GIVEN** a queued snapshot has no local account id +- **AND** its upstream identity resolves to exactly one current local account +- **WHEN** the queued snapshot is consumed +- **THEN** the snapshot is persisted once under that local account + +#### Scenario: Consolidation cannot delete a snapshot inserted after reparenting + +- **GIVEN** PostgreSQL settlement has selected duplicate `D` for a captured upstream identity +- **AND** reconciliation would reparent `D` history to `C` and then delete `D` +- **WHEN** settlement and reconciliation overlap across independent sessions +- **THEN** their shared transaction-scoped upstream-identity lock serializes the complete membership change +- **AND** the snapshot is either committed under `D` before reparenting or directly under `C` after reconciliation +- **AND** exactly one row per represented window survives under `C` + +#### Scenario: Ambiguous fallback does not guess an owner + +- **GIVEN** the captured local account id is absent or no longer exists +- **AND** the captured upstream identity matches multiple current local accounts +- **WHEN** the queued snapshot is consumed +- **THEN** no usage-history row is persisted for that snapshot diff --git a/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md b/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md new file mode 100644 index 0000000000..aa25b1fca3 --- /dev/null +++ b/openspec/changes/settle-live-usage-after-account-consolidation/tasks.md @@ -0,0 +1,72 @@ +## 1. Deterministic regression coverage + +- [x] 1.1 Add a no-sleep integration test that queues a primary/secondary live + snapshot for duplicate `D` with local and upstream identities, completes + same-slot reconciliation into canonical `C`, then directly consumes the + captured item. +- [x] 1.2 Assert exactly one persisted row for each represented window under + `C`, no usage row under `D`, no duplicate snapshot, and preservation of the + injected usage/reset/credits values. +- [x] 1.3 Add controls proving a still-valid local id is preferred even when an + upstream fallback exists, and an upstream-only queued item still resolves to + its unique local account. +- [x] 1.4 Capture the focused failing-first command and RED output before any + production edit; do not use sleeps, polling delays, retries, or a background + consumer timing race. +- [x] 1.5 Add a deterministic two-session PostgreSQL regression that pauses at + exact transaction events and proves consolidation cannot reparent before a + snapshot append and then cascade-delete that append. +- [x] 1.6 Add both legal PostgreSQL interleavings for queued identity `X` when + the selected local owner currently belongs to `Y`, including the causal RED + where `Y` reconciliation wins the owner row and deletes it before lookup. + +## 2. Publication ownership envelope + +- [x] 2.1 Retain both local account id and upstream ChatGPT account id in the + typed live-usage hub/queue contract. +- [x] 2.2 Update every local-account HTTP/SSE and WebSocket publication tap + point to supply the upstream identity when available, preserving the existing + upstream-only path and no-op hub behavior. + +## 3. Consume-time settlement + +- [x] 3.1 Resolve the persistence owner in the background ingestion session: + prefer an existing local row; if it is stale or absent, accept only one + current row matching the captured upstream identity. +- [x] 3.2 Protect owner resolution through persistence and write all represented + windows atomically so the item settles once under one account on SQLite and + PostgreSQL; use one shared transaction-scoped upstream-identity lock across + settlement, ordinary/slot upserts, replacement, rotation, metadata update, + consolidation, and deletion before row/fold locks. +- [x] 3.3 Keep ambiguous/missing ownership serving-safe and logged; do not alter + account consolidation policy, queue overflow, throttling, or retry behavior. +- [x] 3.4 Add no Alembic revision, model column, setting, or API schema change. +- [x] 3.5 Roll back before bounded relock of the canonical captured/current + identity set, reselect and revalidate ownership, and raise a typed terminal + error on a second identity change without fabricating a null lock key. + +## 4. Automated verification + +- [x] 4.1 Run the focused live-ingestion integration selection once to GREEN, + proving stale-local consolidation, valid-local preference, and upstream-only + resolution. +- [x] 4.2 Run diagnostics on every changed Python file and the affected backend + lint/type/test gates on both supported database paths where registered. +- [x] 4.3 Run `openspec validate settle-live-usage-after-account-consolidation --strict`. +- [x] 4.4 Run the deterministic PostgreSQL race repeatedly plus lock-routing, + identity, live-ingest, snapshot, and HTTP publication regressions after the + shared lock implementation is complete. +- [x] 4.5 Run the selected-owner identity race in both transaction orders and + the focused no-relock, one-relock, terminal-change, rollback, sorted-lock, + and null-identity unit coverage. + +## 5. Authenticated QA and cleanup + +- [x] 5.1 Start an isolated QA database/backend, reproduce `D -> C` settlement, + and execute authenticated `curl -i GET /api/accounts` with the QA bearer key. +- [x] 5.2 Capture HTTP 200 evidence showing exactly one canonical `C`, no `D`, + and the injected primary and secondary usage values; capture an independent + database diff showing one canonical row per represented window and no + duplicate-owned row. +- [x] 5.3 Stop and remove every QA process, listener, credential, database file, + and temporary artifact; record the cleanup receipt. diff --git a/tests/integration/test_live_usage_ingest.py b/tests/integration/test_live_usage_ingest.py index db079ee890..dd4d4d66c7 100644 --- a/tests/integration/test_live_usage_ingest.py +++ b/tests/integration/test_live_usage_ingest.py @@ -1,17 +1,27 @@ from __future__ import annotations import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any, cast import pytest +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.sql.dml import Delete +from app.core.clients import proxy as core_proxy from app.core.crypto import TokenEncryptor +from app.core.openai.requests import ResponsesRequest from app.core.usage import live_hub from app.core.usage.live_snapshots import LiveRateLimitSnapshot, LiveUsageWindow from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.db.session import SessionLocal +from app.modules.accounts import repository as accounts_repository_module from app.modules.accounts.repository import AccountsRepository from app.modules.usage import live_ingest +from app.modules.usage import repository as usage_repository_module from app.modules.usage.repository import UsageRepository pytestmark = pytest.mark.integration @@ -43,6 +53,21 @@ def _snapshot() -> LiveRateLimitSnapshot: ) +async def _usage_rows_for(*account_ids: str) -> list[UsageHistory]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(UsageHistory) + .where(UsageHistory.account_id.in_(account_ids)) + .order_by(UsageHistory.account_id, UsageHistory.window, UsageHistory.id) + ) + ) + .scalars() + .all() + ) + + async def _wait_for_rows(account_id: str, *, timeout: float = 5.0) -> tuple[UsageHistory | None, UsageHistory | None]: deadline = asyncio.get_event_loop().time() + timeout while True: @@ -234,22 +259,585 @@ async def test_live_ingestor_normalizes_monthly_only_snapshots(db_setup) -> None @pytest.mark.asyncio -async def test_live_ingestor_resolves_chatgpt_account_id(db_setup) -> None: +async def test_live_ingestor_settles_snapshot_after_duplicate_account_consolidation(db_setup) -> None: del db_setup + canonical_id = "acc_live_consolidated" + duplicate_id = "acc_live_consolidated__copy" + upstream_id = "workspace-live-consolidated" + email = "live-consolidated@example.com" + async with SessionLocal() as session: - await AccountsRepository(session).upsert( - _make_account("acc_live_resolved", "live-resolved@example.com", chatgpt_account_id="workspace-live-1") + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(duplicate_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, ) + snapshot = _snapshot() ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) - ingestor.start() + ingestor.publish( + snapshot, + account_id=duplicate_id, + chatgpt_account_id=upstream_id, + ) + queued = ingestor._queue.get_nowait() + + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_live_consolidated_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + assert saved.id == canonical_id + assert await session.get(Account, duplicate_id) is None + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(canonical_id, duplicate_id) + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + primary_rows = [row for row in rows if row.window == "primary"] + secondary_rows = [row for row in rows if row.window == "secondary"] + assert len(primary_rows) == 1 + assert len(secondary_rows) == 1 + + primary = primary_rows[0] + secondary = secondary_rows[0] + assert snapshot.primary is not None + assert snapshot.secondary is not None + assert primary.used_percent == pytest.approx(snapshot.primary.used_percent) + assert primary.window_minutes == snapshot.primary.window_minutes + assert primary.reset_at == snapshot.primary.reset_at + assert primary.credits_has == snapshot.credits_has + assert primary.credits_unlimited == snapshot.credits_unlimited + assert primary.credits_balance == pytest.approx(snapshot.credits_balance) + assert secondary.used_percent == pytest.approx(snapshot.secondary.used_percent) + assert secondary.window_minutes == snapshot.secondary.window_minutes + assert secondary.reset_at == snapshot.secondary.reset_at + + +@pytest.mark.asyncio +async def test_sse_publication_tap_settles_queued_duplicate_snapshot_under_canonical_account( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + canonical_id = "acc_live_sse_canonical" + duplicate_id = "acc_live_sse_duplicate" + upstream_id = "workspace-live-sse" + email = "live-sse@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(_make_account(canonical_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + await repo.upsert(_make_account(duplicate_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + + rate_limit_event = ( + 'data: {"type":"codex.rate_limits","rate_limits":' + '{"primary":{"used_percent":33,"window_minutes":300,"reset_at":1700000300},' + '"secondary":{"used_percent":44,"window_minutes":10080,"reset_at":1700604800}}}\n\n' + ) + + @asynccontextmanager + async def _fake_http_session(_session): + yield cast(Any, object()) + + async def _fake_upstream_stream(**kwargs): + assert kwargs["account_id"] == upstream_id + assert kwargs["codex_lb_account_id"] == duplicate_id + yield rate_limit_event + + monkeypatch.setattr(core_proxy, "lease_http_session", _fake_http_session) + monkeypatch.setattr(core_proxy, "_stream_responses_with_session", _fake_upstream_stream) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingest_completed = asyncio.Event() + ingest_snapshot = ingestor._ingest + + async def _observed_ingest(item: live_ingest._QueuedSnapshot) -> None: + await ingest_snapshot(item) + ingest_completed.set() + + monkeypatch.setattr(ingestor, "_ingest", _observed_ingest) + live_hub.register_live_usage_publisher(ingestor.publish) try: - ingestor.publish(_snapshot(), chatgpt_account_id="workspace-live-1") - primary, secondary = await _wait_for_rows("acc_live_resolved") + events = [ + event + async for event in core_proxy.stream_responses( + ResponsesRequest(model="gpt-5.1", instructions="", input="hello", stream=True), + {}, + "access-token", + upstream_id, + session=cast(Any, object()), + codex_lb_account_id=duplicate_id, + ) + ] + assert events == [rate_limit_event] + + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_live_sse_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + assert saved.id == canonical_id + + ingestor.start() + await asyncio.wait_for(ingest_completed.wait(), timeout=5.0) finally: await ingestor.stop() + live_hub.register_live_usage_publisher(None) - assert primary is not None and secondary is not None + rows = await _usage_rows_for(canonical_id, duplicate_id) + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + assert {row.used_percent for row in rows} == {33.0, 44.0} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_serializes_identity_membership_through_snapshot_commit( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + canonical_id = "acc_live_pg_canonical" + duplicate_id = "acc_live_pg_duplicate" + upstream_id = "workspace-live-pg-consolidated" + email = "live-pg-consolidated@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(duplicate_id, email, chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish( + _snapshot(), + account_id=duplicate_id, + chatgpt_account_id=upstream_id, + ) + queued = ingestor._queue.get_nowait() + + settlement_commit_started = asyncio.Event() + release_settlement_commit = asyncio.Event() + writer_lock_attempted = asyncio.Event() + release_writer_delete = asyncio.Event() + settlement_lock_keys: list[int] = [] + writer_lock_keys: list[int] = [] + settlement_session = SessionLocal() + writer_session = SessionLocal() + settlement_task: asyncio.Task[None] | None = None + writer_task: asyncio.Task[Account] | None = None + + def _lock_key(args: tuple[Any, ...], kwargs: dict[str, Any]) -> int: + parameters = args[0] if args else kwargs.get("params") + assert isinstance(parameters, dict) + lock_key = parameters["lock_key"] + assert isinstance(lock_key, int) + return lock_key + + settlement_execute = settlement_session.execute + + async def _settlement_execute(statement: Any, *args: Any, **kwargs: Any): + if "pg_advisory_xact_lock" in str(statement): + settlement_lock_keys.append(_lock_key(args, kwargs)) + return await settlement_execute(statement, *args, **kwargs) + + settlement_commit = settlement_session.commit + + async def _settlement_commit() -> None: + settlement_commit_started.set() + await asyncio.wait_for(release_settlement_commit.wait(), timeout=5.0) + await settlement_commit() + + writer_execute = writer_session.execute + + async def _writer_execute(statement: Any, *args: Any, **kwargs: Any): + if "pg_advisory_xact_lock" in str(statement): + writer_lock_keys.append(_lock_key(args, kwargs)) + writer_lock_attempted.set() + if isinstance(statement, Delete) and statement.table.name == Account.__tablename__: + await asyncio.wait_for(release_writer_delete.wait(), timeout=5.0) + return await writer_execute(statement, *args, **kwargs) + + monkeypatch.setattr(settlement_session, "execute", _settlement_execute) + monkeypatch.setattr(settlement_session, "commit", _settlement_commit) + monkeypatch.setattr(writer_session, "execute", _writer_execute) + + @asynccontextmanager + async def _settlement_session() -> AsyncIterator[AsyncSession]: + yield settlement_session + + monkeypatch.setattr(live_ingest, "get_background_session", _settlement_session) + + try: + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_commit_started.wait(), timeout=5.0) + assert settlement_lock_keys, "settlement must take the upstream identity lock" + + writer_task = asyncio.create_task( + AccountsRepository(writer_session).upsert( + _make_account("acc_live_pg_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(writer_lock_attempted.wait(), timeout=5.0) + + assert writer_lock_keys[0] == settlement_lock_keys[0] + release_settlement_commit.set() + await asyncio.wait_for(settlement_task, timeout=5.0) + release_writer_delete.set() + saved = await asyncio.wait_for(writer_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_settlement_commit.set() + release_writer_delete.set() + for task in (settlement_task, writer_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (settlement_task, writer_task) if task is not None), + return_exceptions=True, + ) + await settlement_session.rollback() + await writer_session.rollback() + await settlement_session.close() + await writer_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_waits_for_identity_consolidation_commit( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + canonical_id = "acc_live_pg_writer_first_canonical" + duplicate_id = "acc_live_pg_writer_first_duplicate" + upstream_id = "workspace-live-pg-writer-first" + email = "live-pg-writer-first@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(_make_account(canonical_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + await repo.upsert(_make_account(duplicate_id, email, chatgpt_account_id=upstream_id), merge_by_email=False) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(_snapshot(), account_id=duplicate_id, chatgpt_account_id=upstream_id) + queued = ingestor._queue.get_nowait() + + writer_commit_started = asyncio.Event() + release_writer_commit = asyncio.Event() + settlement_lock_attempted = asyncio.Event() + writer_session = SessionLocal() + writer_commit = writer_session.commit + real_settlement_lock = usage_repository_module.lock_postgresql_account_identities + writer_task: asyncio.Task[Account] | None = None + settlement_task: asyncio.Task[None] | None = None + + async def _writer_commit() -> None: + writer_commit_started.set() + await asyncio.wait_for(release_writer_commit.wait(), timeout=5.0) + await writer_commit() + + async def _observed_settlement_lock(session: AsyncSession, identities): + settlement_lock_attempted.set() + return await real_settlement_lock(session, identities) + + monkeypatch.setattr(writer_session, "commit", _writer_commit) + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", _observed_settlement_lock) + + try: + writer_task = asyncio.create_task( + AccountsRepository(writer_session).upsert( + _make_account("acc_live_pg_writer_first_reauth", email, chatgpt_account_id=upstream_id), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(writer_commit_started.wait(), timeout=5.0) + + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_lock_attempted.wait(), timeout=5.0) + assert not settlement_task.done() + + release_writer_commit.set() + saved = await asyncio.wait_for(writer_task, timeout=5.0) + await asyncio.wait_for(settlement_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_writer_commit.set() + for task in (writer_task, settlement_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (writer_task, settlement_task) if task is not None), + return_exceptions=True, + ) + await writer_session.rollback() + await writer_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_postgresql_live_ingest_recovers_when_current_identity_reconciliation_wins_owner_lock( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL selected-owner lock regression") + + canonical_id = "acc_live_pg_current_identity_canonical" + selected_id = "acc_live_pg_current_identity_selected" + queued_identity = "workspace-live-pg-current-before" + current_identity = "workspace-live-pg-current-after" + email = "live-pg-current-identity@example.com" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(canonical_id, email, chatgpt_account_id=current_identity), + merge_by_email=False, + ) + selected = await repo.upsert( + _make_account(selected_id, email, chatgpt_account_id=queued_identity), + merge_by_email=False, + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish( + _snapshot(), + account_id=selected_id, + chatgpt_account_id=queued_identity, + ) + queued = ingestor._queue.get_nowait() + + async with SessionLocal() as session: + moved = await AccountsRepository(session).rotate_tokens( + selected.id, + selected.access_token_encrypted, + selected.refresh_token_encrypted, + selected.id_token_encrypted, + utcnow(), + expected_refresh_token_encrypted=selected.refresh_token_encrypted, + chatgpt_account_id=current_identity, + ) + assert moved is True + + reconciliation_commit_started = asyncio.Event() + release_reconciliation_commit = asyncio.Event() + settlement_local_lookup_started = asyncio.Event() + settlement_session = SessionLocal() + reconciliation_session = SessionLocal() + settlement_task: asyncio.Task[None] | None = None + reconciliation_task: asyncio.Task[Account] | None = None + settlement_execute = settlement_session.execute + reconciliation_commit = reconciliation_session.commit + + async def _settlement_execute(statement: Any, *args: Any, **kwargs: Any): + sql = str(statement) + if sql.startswith("SELECT accounts.id, accounts.chatgpt_account_id") and "WHERE accounts.id =" in sql: + settlement_local_lookup_started.set() + return await settlement_execute(statement, *args, **kwargs) + + async def _reconciliation_commit() -> None: + reconciliation_commit_started.set() + await asyncio.wait_for(release_reconciliation_commit.wait(), timeout=5.0) + await reconciliation_commit() + + monkeypatch.setattr(settlement_session, "execute", _settlement_execute) + monkeypatch.setattr(reconciliation_session, "commit", _reconciliation_commit) + + @asynccontextmanager + async def _settlement_session() -> AsyncIterator[AsyncSession]: + yield settlement_session + + monkeypatch.setattr(live_ingest, "get_background_session", _settlement_session) + + try: + reconciliation_task = asyncio.create_task( + AccountsRepository(reconciliation_session).upsert( + _make_account("acc_live_pg_current_identity_reauth", email, chatgpt_account_id=current_identity), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + ) + await asyncio.wait_for(reconciliation_commit_started.wait(), timeout=5.0) + + settlement_task = asyncio.create_task(ingestor._ingest(queued)) + await asyncio.wait_for(settlement_local_lookup_started.wait(), timeout=5.0) + assert not settlement_task.done() + + release_reconciliation_commit.set() + saved = await asyncio.wait_for(reconciliation_task, timeout=5.0) + await asyncio.wait_for(settlement_task, timeout=5.0) + assert saved.id == canonical_id + finally: + release_reconciliation_commit.set() + for task in (settlement_task, reconciliation_task): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather( + *(task for task in (settlement_task, reconciliation_task) if task is not None), + return_exceptions=True, + ) + await settlement_session.rollback() + await reconciliation_session.rollback() + await settlement_session.close() + await reconciliation_session.close() + + async with SessionLocal() as session: + accounts = list((await session.execute(select(Account).order_by(Account.id))).scalars().all()) + rows = list((await session.execute(select(UsageHistory).order_by(UsageHistory.id))).scalars().all()) + assert [account.id for account in accounts] == [canonical_id] + assert [account.chatgpt_account_id for account in accounts] == [current_identity] + assert [row.account_id for row in rows] == [canonical_id, canonical_id] + assert {row.window for row in rows} == {"primary", "secondary"} + assert {row.used_percent for row in rows} == {33.0, 44.0} + + +@pytest.mark.asyncio +async def test_postgresql_opposite_identity_moves_use_one_sorted_lock_order( + monkeypatch: pytest.MonkeyPatch, + db_setup, +) -> None: + del db_setup + bind = SessionLocal.kw["bind"] + if bind.dialect.name != "postgresql": + pytest.skip("PostgreSQL transaction-lock regression") + + first = _make_account("acc_identity_move_a", "identity-move-a@example.com", chatgpt_account_id="workspace-a") + second = _make_account("acc_identity_move_b", "identity-move-b@example.com", chatgpt_account_id="workspace-b") + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert(first, merge_by_email=False) + await repo.upsert(second, merge_by_email=False) + + real_identity_lock = accounts_repository_module.lock_postgresql_account_identities + arrival_guard = asyncio.Lock() + both_arrived = asyncio.Event() + arrival_count = 0 + + async def _synchronized_identity_lock(session: AsyncSession, identities): + nonlocal arrival_count + async with arrival_guard: + arrival_count += 1 + if arrival_count == 2: + both_arrived.set() + await asyncio.wait_for(both_arrived.wait(), timeout=5.0) + return await real_identity_lock(session, identities) + + monkeypatch.setattr(accounts_repository_module, "lock_postgresql_account_identities", _synchronized_identity_lock) + + async def _move(account: Account, incoming_identity: str) -> bool: + async with SessionLocal() as session: + return await AccountsRepository(session).rotate_tokens( + account.id, + account.access_token_encrypted, + account.refresh_token_encrypted, + account.id_token_encrypted, + utcnow(), + expected_refresh_token_encrypted=account.refresh_token_encrypted, + chatgpt_account_id=incoming_identity, + ) + + moved_first, moved_second = await asyncio.wait_for( + asyncio.gather(_move(first, "workspace-b"), _move(second, "workspace-a")), + timeout=5.0, + ) + assert moved_first is True + assert moved_second is True + + async with SessionLocal() as session: + identities = { + account_id: chatgpt_account_id + for account_id, chatgpt_account_id in ( + await session.execute(select(Account.id, Account.chatgpt_account_id)) + ).all() + } + assert identities == { + first.id: "workspace-b", + second.id: "workspace-a", + } + + +@pytest.mark.asyncio +async def test_live_ingestor_prefers_valid_local_owner_over_upstream_fallback(db_setup) -> None: + del db_setup + local_id = "acc_live_valid_local" + sibling_id = "acc_live_valid_local_sibling" + upstream_id = "workspace-live-shared" + async with SessionLocal() as session: + repo = AccountsRepository(session) + await repo.upsert( + _make_account(local_id, "live-valid-local@example.com", chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + await repo.upsert( + _make_account(sibling_id, "live-valid-sibling@example.com", chatgpt_account_id=upstream_id), + merge_by_email=False, + ) + + snapshot = _snapshot() + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(snapshot, account_id=local_id, chatgpt_account_id=upstream_id) + queued = ingestor._queue.get_nowait() + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(local_id, sibling_id) + assert len(rows) == 2 + assert {row.account_id for row in rows} == {local_id} + assert {row.window for row in rows} == {"primary", "secondary"} + + +@pytest.mark.asyncio +async def test_live_ingestor_resolves_chatgpt_account_id(db_setup) -> None: + del db_setup + account_id = "acc_live_resolved" + async with SessionLocal() as session: + await AccountsRepository(session).upsert( + _make_account(account_id, "live-resolved@example.com", chatgpt_account_id="workspace-live-1") + ) + + ingestor = live_ingest.LiveUsageIngestor(queue_size=8, write_min_interval_seconds=0.0) + ingestor.publish(_snapshot(), chatgpt_account_id="workspace-live-1") + queued = ingestor._queue.get_nowait() + + await ingestor._ingest(queued) + + rows = await _usage_rows_for(account_id) + assert len(rows) == 2 + assert {row.account_id for row in rows} == {account_id} + assert {row.window for row in rows} == {"primary", "secondary"} @pytest.mark.asyncio diff --git a/tests/integration/test_repositories.py b/tests/integration/test_repositories.py index 9d5e639015..08983ba27d 100644 --- a/tests/integration/test_repositories.py +++ b/tests/integration/test_repositories.py @@ -1240,6 +1240,40 @@ async def test_accounts_upsert_merge_by_chatgpt_identity_skips_without_upstream_ assert saved.id.startswith("acc_no_id__copy") +@pytest.mark.asyncio +async def test_identity_reconciliation_does_not_select_identityless_local_row_as_duplicate(db_setup): + async with SessionLocal() as session: + repo = AccountsRepository(session) + canonical = _make_account_with_chatgpt_id( + "acc_identity_canonical", + "identity-invariant@example.com", + "chatgpt_identity_invariant", + ) + identityless = _make_account("acc_identityless_local", "identity-invariant@example.com") + await repo.upsert(canonical, merge_by_email=False) + await repo.upsert(identityless, merge_by_email=False) + + saved = await repo.upsert( + _make_account_with_chatgpt_id( + "acc_identity_reauth", + "identity-invariant@example.com", + "chatgpt_identity_invariant", + ), + merge_by_email=False, + merge_by_chatgpt_identity=True, + ) + + assert saved.id == canonical.id + remaining = { + account.id: account.chatgpt_account_id + for account in (await session.execute(select(Account).order_by(Account.id))).scalars().all() + } + assert remaining == { + canonical.id: "chatgpt_identity_invariant", + identityless.id: None, + } + + @pytest.mark.asyncio async def test_usage_repository_aggregate(db_setup): async with SessionLocal() as session: diff --git a/tests/unit/test_accounts_repository_locks.py b/tests/unit/test_accounts_repository_locks.py index e4c6238abc..ce61f9ba7a 100644 --- a/tests/unit/test_accounts_repository_locks.py +++ b/tests/unit/test_accounts_repository_locks.py @@ -1,6 +1,7 @@ from __future__ import annotations from contextlib import asynccontextmanager +from typing import Any, cast from unittest.mock import AsyncMock, MagicMock import pytest @@ -8,8 +9,9 @@ import app.modules.accounts.repository as repository_module from app.core.crypto import TokenEncryptor from app.core.utils.time import utcnow +from app.db.account_identity_lock import account_identity_lock_key, lock_postgresql_account_identities from app.db.models import Account, AccountStatus -from app.modules.accounts.repository import AccountsRepository +from app.modules.accounts.repository import AccountIdentityRelockError, AccountsRepository def _stub_account(account_id: str, email: str, chatgpt_id: str | None = None) -> Account: @@ -41,18 +43,40 @@ def _make_postgres_repo(monkeypatch: pytest.MonkeyPatch) -> tuple[AccountsReposi session.execute = AsyncMock() session.commit = AsyncMock() session.refresh = AsyncMock() + session.rollback = AsyncMock() session.add = MagicMock() session.get = AsyncMock(return_value=None) repo = AccountsRepository(session) - recorded: dict[str, list[str]] = {"identity": [], "email": []} + recorded: dict[str, list[str]] = {"upstream": [], "identity": [], "email": [], "order": []} async def fake_identity_lock(key: str) -> None: recorded["identity"].append(key) + recorded["order"].append(f"identity:{key}") async def fake_email_lock(email: str) -> None: recorded["email"].append(email) + recorded["order"].append(f"email:{email}") + + async def fake_upstream_identity_locks(account: Account, *, include_email: bool) -> frozenset[str]: + del include_email + if account.chatgpt_account_id: + recorded["upstream"].append(account.chatgpt_account_id) + recorded["order"].append(f"upstream:{account.chatgpt_account_id}") + return frozenset((account.chatgpt_account_id,)) + return frozenset() + + async def fake_candidates_are_locked( + account: Account, + *, + include_email: bool, + locked_identities: frozenset[str], + ) -> bool: + del account + del include_email + del locked_identities + return True async def fake_merge_by_email_enabled() -> bool: # only used when merge_by_email is None return True @@ -76,9 +100,13 @@ async def fake_next_available_account_id(account_id: str) -> str: monkeypatch.setattr(repo, "_dialect_name", lambda: "postgresql") monkeypatch.setattr(repo, "_acquire_postgresql_identity_lock", fake_identity_lock) monkeypatch.setattr(repo, "_acquire_postgresql_merge_lock", fake_email_lock) + monkeypatch.setattr(repo, "_lock_postgresql_upsert_identity_candidates", fake_upstream_identity_locks) + monkeypatch.setattr(repo, "_postgresql_upsert_identity_candidates_are_locked", fake_candidates_are_locked) monkeypatch.setattr(repo, "_merge_by_email_enabled", fake_merge_by_email_enabled) monkeypatch.setattr(repo, "_account_by_chatgpt_identity", fake_account_by_chatgpt_identity) + monkeypatch.setattr(repo, "_account_by_slot_identity", AsyncMock(return_value=None)) monkeypatch.setattr(repo, "_single_account_by_email", fake_single_account_by_email) + monkeypatch.setattr(repo, "_single_unknown_workspace_account_by_email", fake_single_account_by_email) monkeypatch.setattr(repo, "_next_available_account_id", fake_next_available_account_id) return repo, recorded @@ -90,6 +118,35 @@ def _make_result(value: str | None = "acc") -> MagicMock: return result +@pytest.mark.asyncio +async def test_postgresql_upstream_identity_locks_use_existing_namespace_in_sorted_order() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock() + + lock_keys = await lock_postgresql_account_identities(session, ("workspace-z", None, "workspace-a", "workspace-z")) + + expected = tuple(sorted((account_identity_lock_key("workspace-a"), account_identity_lock_key("workspace-z")))) + assert lock_keys == expected + assert session.execute.await_args_list[0].args[1] == {"timeout": "30000ms"} + assert [call.args[1]["lock_key"] for call in session.execute.await_args_list[1:]] == list(expected) + + +@pytest.mark.asyncio +async def test_postgresql_upstream_identity_lock_failure_rolls_back_and_propagates() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + lock_error = RuntimeError("injected lock timeout") + session.execute = AsyncMock(side_effect=[MagicMock(), lock_error]) + session.rollback = AsyncMock() + + with pytest.raises(RuntimeError) as exc_info: + await lock_postgresql_account_identities(session, ("workspace-timeout",)) + + assert exc_info.value is lock_error + session.rollback.assert_awaited_once() + + @pytest.mark.asyncio async def test_account_update_status_uses_sqlite_writer_section(monkeypatch): session = MagicMock() @@ -183,9 +240,10 @@ async def test_upsert_takes_identity_lock_even_when_merge_by_email_enabled(monke await repo.upsert(account, merge_by_email=True, merge_by_chatgpt_identity=True) - assert recorded["identity"] == ["chatgpt:chatgpt_xyz"], ( - "identity lock must be acquired even when merge_by_email is True" + assert recorded["upstream"] == ["chatgpt_xyz"], ( + "upstream identity lock must be acquired even when merge_by_email is True" ) + assert recorded["identity"] == [] assert recorded["email"] == ["a@example.com"], "email lock must still be acquired when merge_by_email is True" @@ -200,7 +258,8 @@ async def test_upsert_takes_identity_lock_when_merge_by_email_disabled(monkeypat await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=True) - assert recorded["identity"] == ["chatgpt:chatgpt_zzz"] + assert recorded["upstream"] == ["chatgpt_zzz"] + assert recorded["identity"] == [] assert recorded["email"] == [] @@ -216,6 +275,7 @@ async def test_upsert_falls_back_to_id_lock_without_identity(monkeypatch): await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=False) + assert recorded["upstream"] == [] assert recorded["identity"] == ["acc_c"] assert recorded["email"] == [] @@ -231,5 +291,148 @@ async def test_upsert_email_only_when_identity_not_in_play(monkeypatch): await repo.upsert(account, merge_by_email=True, merge_by_chatgpt_identity=False) - assert recorded["identity"] == [], "no identity lock when merge_by_chatgpt_identity is False" + assert recorded["upstream"] == ["chatgpt_qqq"] + assert recorded["identity"] == [] assert recorded["email"] == ["d@example.com"] + assert recorded["order"] == ["upstream:chatgpt_qqq", "email:d@example.com"] + + +@pytest.mark.asyncio +async def test_ordinary_identity_upsert_uses_upstream_membership_lock(monkeypatch): + repo, recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_e", "e@example.com", chatgpt_id="chatgpt_ordinary") + + await repo.upsert(account, merge_by_email=False, merge_by_chatgpt_identity=False) + + assert recorded["upstream"] == ["chatgpt_ordinary"] + assert recorded["order"] == ["upstream:chatgpt_ordinary"] + + +@pytest.mark.asyncio +async def test_account_slot_upsert_locks_upstream_before_slot_keys(monkeypatch): + repo, recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_slot", "slot@example.com", chatgpt_id="chatgpt_slot") + account.workspace_id = "workspace-slot" + + await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + + assert recorded["upstream"] == ["chatgpt_slot"] + assert recorded["order"][0] == "upstream:chatgpt_slot" + assert all(item.startswith("identity:") for item in recorded["order"][1:]) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("slot_upsert", [False, True]) +async def test_identity_candidate_revalidation_restarts_once_then_succeeds(monkeypatch, slot_upsert: bool): + repo, _recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_retry", "retry@example.com", chatgpt_id="chatgpt_retry") + candidates_are_locked = AsyncMock(side_effect=[False, True]) + monkeypatch.setattr(repo, "_postgresql_upsert_identity_candidates_are_locked", candidates_are_locked) + + if slot_upsert: + saved = await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + else: + saved = await repo.upsert(account, merge_by_email=False) + + assert saved is account + assert candidates_are_locked.await_count == 2 + assert cast(Any, repo.session.rollback).await_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("slot_upsert", [False, True]) +async def test_identity_candidate_revalidation_raises_typed_error_after_second_change( + monkeypatch, + slot_upsert: bool, +): + repo, _recorded = _make_postgres_repo(monkeypatch) + account = _stub_account("acc_terminal", "terminal@example.com", chatgpt_id="chatgpt_terminal") + monkeypatch.setattr( + repo, + "_postgresql_upsert_identity_candidates_are_locked", + AsyncMock(side_effect=[False, False]), + ) + + with pytest.raises(AccountIdentityRelockError): + if slot_upsert: + await repo.upsert_account_slot(account, preserve_unknown_workspace_duplicates=False) + else: + await repo.upsert(account, merge_by_email=False) + + assert cast(Any, repo.session.rollback).await_count == 2 + + +@pytest.mark.asyncio +async def test_local_identity_membership_relocks_after_observed_identity_changes(monkeypatch): + session = MagicMock() + changed = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed") + session.scalar = AsyncMock(side_effect=["chatgpt_old", changed, "chatgpt_changed", changed]) + session.rollback = AsyncMock() + repo = AccountsRepository(session) + identity_locks = AsyncMock() + monkeypatch.setattr(repository_module, "lock_postgresql_account_identities", identity_locks) + + locked = await repo._lock_postgresql_account_identity_membership("acc_relock", "chatgpt_incoming") + + assert locked is changed + assert [call.args[1] for call in identity_locks.await_args_list] == [ + ("chatgpt_old", "chatgpt_incoming"), + ("chatgpt_changed", "chatgpt_incoming"), + ] + session.rollback.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_local_identity_membership_raises_typed_error_after_second_change(monkeypatch): + session = MagicMock() + changed_once = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed") + changed_twice = _stub_account("acc_relock", "relock@example.com", chatgpt_id="chatgpt_changed_again") + session.scalar = AsyncMock(side_effect=["chatgpt_old", changed_once, "chatgpt_changed", changed_twice]) + session.rollback = AsyncMock() + repo = AccountsRepository(session) + monkeypatch.setattr(repository_module, "lock_postgresql_account_identities", AsyncMock()) + + with pytest.raises(AccountIdentityRelockError): + await repo._lock_postgresql_account_identity_membership("acc_relock", "chatgpt_incoming") + + assert session.rollback.await_count == 2 + + +@pytest.mark.asyncio +async def test_local_identity_writers_lock_old_and_incoming_membership(monkeypatch): + repo, _recorded = _make_postgres_repo(monkeypatch) + existing = _stub_account("acc_writer", "writer@example.com", chatgpt_id="chatgpt_old") + membership_locks: list[tuple[str, str | None]] = [] + cast(Any, repo.session.execute).return_value = _make_result("acc_writer") + + async def fake_membership_lock(account_id: str, incoming: str | None) -> Account: + membership_locks.append((account_id, incoming)) + return existing + + monkeypatch.setattr(repo, "_lock_postgresql_account_identity_membership", fake_membership_lock) + monkeypatch.setattr(repo, "_apply_account_replacement", AsyncMock()) + monkeypatch.setattr(repository_module, "lock_fold_state", AsyncMock()) + monkeypatch.setattr(repository_module, "mirror_account_soft_delete_into_time_rollups", AsyncMock()) + + await repo.replace_reauthorized( + existing.id, + _stub_account("incoming", existing.email, chatgpt_id="chatgpt_new"), + ) + assert await repo.rotate_tokens( + existing.id, + b"access", + b"refresh", + b"id", + utcnow(), + expected_refresh_token_encrypted=b"expected", + chatgpt_account_id="chatgpt_new", + ) + assert await repo.update_account_metadata(existing.id, chatgpt_account_id="chatgpt_new") + assert await repo.delete(existing.id) + + assert membership_locks == [ + (existing.id, "chatgpt_new"), + (existing.id, "chatgpt_new"), + (existing.id, "chatgpt_new"), + (existing.id, None), + ] diff --git a/tests/unit/test_live_snapshot_owner_relock.py b/tests/unit/test_live_snapshot_owner_relock.py new file mode 100644 index 0000000000..11d59f28ab --- /dev/null +++ b/tests/unit/test_live_snapshot_owner_relock.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, call + +import pytest + +from app.db.account_identity_lock import lock_postgresql_account_identities +from app.modules.usage import repository as usage_repository_module +from app.modules.usage.repository import ( + LiveSnapshotOwnerIdentityRelockError, + UsageRepository, + UsageWindowWrite, +) + +pytestmark = pytest.mark.unit + + +def _identity_result(account_id: str | None, chatgpt_account_id: str | None = None) -> MagicMock: + result = MagicMock() + if account_id is None: + result.one_or_none.return_value = None + else: + result.one_or_none.return_value = MagicMock( + id=account_id, + chatgpt_account_id=chatgpt_account_id, + ) + return result + + +def _postgresql_session(results: list[MagicMock]) -> MagicMock: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock(side_effect=results) + session.add_all = MagicMock() + session.commit = AsyncMock() + session.rollback = AsyncMock() + return session + + +async def _settle(session: MagicMock) -> str | None: + return await UsageRepository(session).settle_live_account_snapshot( + account_id="acc-selected", + chatgpt_account_id="workspace-x", + windows=[UsageWindowWrite(window="primary", used_percent=25.0)], + should_skip=lambda _account_id: False, + ) + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_same_identity_does_not_relock( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-x"), + _identity_result("acc-selected", "workspace-x"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await _settle(session) + + assert resolved == "acc-selected" + identity_lock.assert_awaited_once_with(session, ("workspace-x",)) + session.rollback.assert_not_awaited() + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_relocks_once_for_current_owner_identity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-y"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await _settle(session) + + assert resolved == "acc-selected" + assert identity_lock.await_args_list == [ + call(session, ("workspace-x",)), + call(session, ("workspace-x", "workspace-y")), + ] + session.rollback.assert_awaited_once() + session.add_all.assert_called_once() + session.commit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_postgresql_live_snapshot_second_owner_identity_change_is_terminal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = _postgresql_session( + [ + _identity_result("acc-selected", "workspace-y"), + _identity_result("acc-selected", "workspace-z"), + ] + ) + identity_lock = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", identity_lock) + + with pytest.raises(LiveSnapshotOwnerIdentityRelockError): + await _settle(session) + + assert identity_lock.await_args_list == [ + call(session, ("workspace-x",)), + call(session, ("workspace-x", "workspace-y")), + ] + assert session.rollback.await_count == 2 + session.add_all.assert_not_called() + session.commit.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_postgresql_identity_lock_does_not_fabricate_none_key() -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + session.execute = AsyncMock() + + lock_keys = await lock_postgresql_account_identities(session, (None,)) + + assert lock_keys == () + session.execute.assert_not_awaited() diff --git a/tests/unit/test_live_usage_ingest.py b/tests/unit/test_live_usage_ingest.py index c1108e56f3..c37335d451 100644 --- a/tests/unit/test_live_usage_ingest.py +++ b/tests/unit/test_live_usage_ingest.py @@ -281,7 +281,6 @@ async def fake_lease(session: Any = None): assert (account_id, chatgpt_account_id) == (None, "workspace-live") assert snapshot.primary is not None assert snapshot.primary.used_percent == pytest.approx(55.0) - # When the caller knows the selected internal account, attribution - # prefers it so multi-seat workspaces are not dropped as ambiguous. + # Local attribution stays preferred while retaining its recovery identity. _, account_id_internal, chatgpt_internal = captured[1] - assert (account_id_internal, chatgpt_internal) == ("acc-internal", None) + assert (account_id_internal, chatgpt_internal) == ("acc-internal", "workspace-live") diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index e581ee7087..4477ec1ae8 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -779,7 +779,15 @@ def _make_bridge_session( kind=proxy_service.StickySessionKind.CODEX_SESSION, ), request_model="gpt-5.2", - account=cast(Any, SimpleNamespace(id="acc-bridge", status=AccountStatus.ACTIVE, plan_type="plus")), + account=cast( + Any, + SimpleNamespace( + id="acc-bridge", + chatgpt_account_id="workspace-bridge", + status=AccountStatus.ACTIVE, + plan_type="plus", + ), + ), upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), upstream_control=proxy_service._WebSocketUpstreamControl(), pending_requests=pending_requests or deque(), @@ -5358,6 +5366,7 @@ async def test_http_bridge_relay_publishes_live_rate_limit_events( service = proxy_service.ProxyService(cast(Any, nullcontext())) session = _make_bridge_session(key_value="bridge-live-rate-limits") + session.account.chatgpt_account_id = "workspace-bridge-live-rate-limits" rate_limit_text = ( '{"type":"codex.rate_limits","rate_limits":{"primary":' '{"used_percent":72,"window_minutes":300,"reset_at":1700000300}}}' @@ -5381,9 +5390,11 @@ async def test_http_bridge_relay_publishes_live_rate_limit_events( monkeypatch.setattr(service, "_fail_http_bridge_reader_and_maybe_retire", AsyncMock()) monkeypatch.setattr(service, "_fail_pending_websocket_requests", AsyncMock()) - captured: list[tuple[Any, str | None]] = [] + captured: list[tuple[Any, str | None, str | None]] = [] live_hub.register_live_usage_publisher( - lambda snapshot, *, account_id=None, chatgpt_account_id=None: captured.append((snapshot, account_id)) + lambda snapshot, *, account_id=None, chatgpt_account_id=None: captured.append( + (snapshot, account_id, chatgpt_account_id) + ) ) try: await service._relay_http_bridge_upstream_messages(session) @@ -5391,8 +5402,8 @@ async def test_http_bridge_relay_publishes_live_rate_limit_events( live_hub.register_live_usage_publisher(None) assert len(captured) == 1 - snapshot, account_id = captured[0] - assert account_id == session.account.id + snapshot, account_id, chatgpt_account_id = captured[0] + assert (account_id, chatgpt_account_id) == (session.account.id, session.account.chatgpt_account_id) assert snapshot.primary is not None assert snapshot.primary.used_percent == pytest.approx(72.0) diff --git a/tests/unit/test_usage_snapshot_repository.py b/tests/unit/test_usage_snapshot_repository.py index 117ab1142d..10b696fa32 100644 --- a/tests/unit/test_usage_snapshot_repository.py +++ b/tests/unit/test_usage_snapshot_repository.py @@ -3,13 +3,16 @@ from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager from datetime import datetime, timezone +from unittest.mock import AsyncMock, MagicMock import pytest from sqlalchemy import event, func, select +from sqlalchemy.dialects import postgresql from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine from app.db.models import Account, AccountStatus, Base, UsageHistory from app.modules.usage import background_repository as background_repository_module +from app.modules.usage import repository as usage_repository_module from app.modules.usage.background_repository import BackgroundUsageRepository from app.modules.usage.repository import UsageRepository, UsageWindowWrite @@ -26,6 +29,43 @@ async def session_factory() -> AsyncIterator[async_sessionmaker[AsyncSession]]: await engine.dispose() +@pytest.mark.asyncio +async def test_postgresql_settlement_lookups_compile_for_no_key_update( + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = MagicMock() + session.get_bind.return_value.dialect.name = "postgresql" + observed_result = MagicMock() + observed_result.one_or_none.return_value = MagicMock( + id="acc_current", + chatgpt_account_id="workspace-current", + ) + locked_result = MagicMock() + locked_result.one_or_none.return_value = MagicMock( + id="acc_current", + chatgpt_account_id="workspace-current", + ) + session.execute = AsyncMock(side_effect=[observed_result, locked_result]) + session.add_all = MagicMock() + session.commit = AsyncMock() + session.rollback = AsyncMock() + monkeypatch.setattr(usage_repository_module, "lock_postgresql_account_identities", AsyncMock()) + monkeypatch.setattr(usage_repository_module, "relax_commit_durability", AsyncMock()) + + resolved = await UsageRepository(session).settle_live_account_snapshot( + account_id="acc_stale", + chatgpt_account_id="workspace-current", + windows=[UsageWindowWrite(window="primary", used_percent=25.0)], + should_skip=lambda _account_id: False, + ) + + assert resolved == "acc_current" + observed_stmt = session.execute.await_args_list[0].args[0] + locked_stmt = session.execute.await_args_list[1].args[0] + assert "FOR NO KEY UPDATE" not in str(observed_stmt.compile(dialect=postgresql.dialect())) + assert "FOR NO KEY UPDATE" in str(locked_stmt.compile(dialect=postgresql.dialect())) + + def _account(account_id: str) -> Account: return Account( id=account_id, From f92bc906ee06079e307866cff548b75435b46c49 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:46:15 +0200 Subject: [PATCH 035/117] fix(proxy): normalize single-account warmup failures (#1774) * docs(openspec): define single-account warmup summaries * docs(openspec): sync warmup failure summary contract * fix(proxy): normalize single-account warmup failures * docs(openspec): record warmup summary verification --- app/modules/proxy/_service/warmup.py | 6 -- .../.openspec.yaml | 2 + .../design.md | 33 +++++++++++ .../proposal.md | 23 ++++++++ .../specs/proxy-warmup/spec.md | 26 +++++++++ .../tasks.md | 15 +++++ openspec/specs/proxy-warmup/spec.md | 10 +++- tests/integration/test_proxy_warmup.py | 58 ++++++++++++++++++- 8 files changed, 165 insertions(+), 8 deletions(-) create mode 100644 openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml create mode 100644 openspec/changes/normalize-single-account-warmup-summary/design.md create mode 100644 openspec/changes/normalize-single-account-warmup-summary/proposal.md create mode 100644 openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md create mode 100644 openspec/changes/normalize-single-account-warmup-summary/tasks.md diff --git a/app/modules/proxy/_service/warmup.py b/app/modules/proxy/_service/warmup.py index f6c611345c..862117e307 100644 --- a/app/modules/proxy/_service/warmup.py +++ b/app/modules/proxy/_service/warmup.py @@ -248,7 +248,6 @@ async def _submit_account_warmup(account: _WarmupAccountSnapshot) -> _WarmupSubm headers=filtered_headers, warmup_model=effective_model, prohibit_fast_mode=prohibit_fast_mode, - allow_pre_submit_errors_as_result=len(accounts_to_submit) > 1, ) submission_results = await asyncio.gather(*(_submit_account_warmup(account) for account in accounts_to_submit)) @@ -300,7 +299,6 @@ async def _submit_warmup_request( headers: Mapping[str, str], warmup_model: str, prohibit_fast_mode: bool, - allow_pre_submit_errors_as_result: bool = False, ) -> _WarmupSubmitResult: started_at = time.monotonic() useragent, useragent_group, conversation_id = _request_log_client_fields(headers) @@ -423,13 +421,9 @@ async def _submit_warmup_request( except ProxyAuthError as exc: error_code = "auth_error" error_message = str(exc) or "Warmup authentication failed" - if not allow_pre_submit_errors_as_result: - raise except ProxyRateLimitError as exc: error_code = "rate_limit_exceeded" error_message = str(exc) or "Warmup request was rate limited" - if not allow_pre_submit_errors_as_result: - raise except Exception as exc: error_code = "upstream_error" error_message = str(exc) or "Warmup request failed" diff --git a/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml b/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml new file mode 100644 index 0000000000..f161d5cc47 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-16 diff --git a/openspec/changes/normalize-single-account-warmup-summary/design.md b/openspec/changes/normalize-single-account-warmup-summary/design.md new file mode 100644 index 0000000000..227e60a15d --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/design.md @@ -0,0 +1,33 @@ +## Context + +Warmup already returns a structured result for each submitted account, but `_submit_warmup_request` conditionally re-raises `ProxyAuthError` and `ProxyRateLimitError` when the submission pool contains only one account. The production FastAPI exception handlers then return top-level 401 or 429 envelopes instead of the warmup response model. + +## Goals / Non-Goals + +**Goals:** +- Make ordinary auth and rate-limit failures use the existing failed-account representation for every pool cardinality. +- Preserve request logging, result ordering, bounded scheduling, and response schema. +- Prove the behavior through the production FastAPI route. + +**Non-Goals:** +- Change API-key authentication for calling the warmup endpoint. +- Change account selection, eligibility, concurrency, or upstream routing. +- Change invalid-mode or strict-eligibility `ValueError` handling. +- Change global auth or rate-limit exception envelopes for other endpoints. + +## Decisions + +### Decision: Normalize at the existing per-account submission boundary + +Always convert `ProxyAuthError` and `ProxyRateLimitError` inside `_submit_warmup_request`, where account identity and request-log fields are already available. This removes the cardinality-dependent re-raise without adding route-specific exception handling or changing global handlers. + +Alternative considered: catch these exceptions in `_run_v1_warmup`. This was rejected because the route no longer has the per-account result context and would duplicate service normalization. + +### Decision: Keep the existing response model unchanged + +Use the existing `WarmupFailedAccountData` mapping and error codes (`auth_error` and `rate_limit_exceeded`). No API schema or scheduling changes are required. + +## Risks / Trade-offs + +- **[Risk] A caller may have relied on the undocumented one-account 401/429 behavior** -> **Mitigation:** the cardinality-independent HTTP 200 summary is already the normative contract and existing multi-account behavior. +- **[Risk] Broad exception handling could accidentally change unrelated failures** -> **Mitigation:** remove only the conditional re-raise for the two named exception classes and cover both through FastAPI integration tests. diff --git a/openspec/changes/normalize-single-account-warmup-summary/proposal.md b/openspec/changes/normalize-single-account-warmup-summary/proposal.md new file mode 100644 index 0000000000..2efc75a488 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/proposal.md @@ -0,0 +1,23 @@ +## Why + +A warmup request with one eligible account currently returns a top-level 401 or 429 when that account fails authentication or rate limiting, while the same failure in a larger pool is represented in the documented HTTP 200 per-account summary. Pool cardinality should not change the endpoint contract or remove the account-level diagnostic. + +## What Changes + +- Normalize single-account `ProxyAuthError` and `ProxyRateLimitError` failures into the existing `failed` summary entries. +- Preserve the existing summary schema, multi-account behavior, invalid-request handling, account selection, scheduling, and global exception envelopes outside warmup. +- Add production FastAPI integration coverage for both single-account failure classes. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-warmup`: Clarify that ordinary per-account authentication and rate-limit failures return the structured HTTP 200 summary regardless of target-pool cardinality. + +## Impact + +The change is limited to the warmup service's pre-submit error normalization, its integration coverage, and the `proxy-warmup` contract. It adds no dependencies, settings, routes, or schema fields. diff --git a/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md b/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md new file mode 100644 index 0000000000..840d855436 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/specs/proxy-warmup/spec.md @@ -0,0 +1,26 @@ +## MODIFIED Requirements + +### Requirement: Warmup endpoint is exposed on the v1 proxy surface +The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return HTTP 200 with a structured JSON summary of submitted, skipped, and failed account warmups for every valid execution. Per-account `ProxyAuthError` and `ProxyRateLimitError` failures SHALL be represented in the `failed` summary regardless of the number of target accounts. + +The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated proxy surface. That route SHALL not require a request body and SHALL execute the same warmup behavior as the body-based route for the supplied `mode`. + +#### Scenario: Authenticated warmup request succeeds +- **WHEN** a client calls `POST /v1/warmup` with a valid API key and valid mode +- **THEN** the system returns 200 with a per-account warmup result summary + +#### Scenario: Single-account authentication failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyAuthError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `auth_error` + +#### Scenario: Single-account rate-limit failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyRateLimitError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `rate_limit_exceeded` + +#### Scenario: Invalid mode is rejected +- **WHEN** a client calls `POST /v1/warmup` with an unsupported mode value +- **THEN** the system returns a 400 invalid request error + +#### Scenario: Path-based warmup request succeeds without a body +- **WHEN** a client calls `POST /v1/warmup/normal` with a valid API key and no request body +- **THEN** the system returns 200 with the same per-account warmup result summary as the body-based route diff --git a/openspec/changes/normalize-single-account-warmup-summary/tasks.md b/openspec/changes/normalize-single-account-warmup-summary/tasks.md new file mode 100644 index 0000000000..8040b63230 --- /dev/null +++ b/openspec/changes/normalize-single-account-warmup-summary/tasks.md @@ -0,0 +1,15 @@ +## 1. Contract + +- [x] 1.1 Define the cardinality-independent warmup failure contract and implementation boundaries. +- [x] 1.2 Sync the clarified requirement to the main `proxy-warmup` specification. + +## 2. Regression and implementation + +- [x] 2.1 Add production FastAPI integration coverage for one-account auth and rate-limit failures and capture the failing baseline. +- [x] 2.2 Remove only the single-account conditional re-raise for `ProxyAuthError` and `ProxyRateLimitError`. + +## 3. Verification + +- [x] 3.1 Capture focused GREEN and adjacent warmup integration results. +- [x] 3.2 Run strict OpenSpec validation, affected lint/type checks, and production FastAPI surface proof. +- [x] 3.3 Review the committed diff independently and address in-scope findings. diff --git a/openspec/specs/proxy-warmup/spec.md b/openspec/specs/proxy-warmup/spec.md index 3bd97584ca..a176bb56fb 100644 --- a/openspec/specs/proxy-warmup/spec.md +++ b/openspec/specs/proxy-warmup/spec.md @@ -4,7 +4,7 @@ TBD - created by archiving change add-v1-warmup-endpoint. Update Purpose after archive. ## Requirements ### Requirement: Warmup endpoint is exposed on the v1 proxy surface -The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return a structured JSON summary of submitted, skipped, and failed account warmups. +The system SHALL expose `POST /v1/warmup` on the same authenticated proxy surface as other `/v1/*` routes. The endpoint SHALL accept a JSON body with `mode` and SHALL return HTTP 200 with a structured JSON summary of submitted, skipped, and failed account warmups for every valid execution. Per-account `ProxyAuthError` and `ProxyRateLimitError` failures SHALL be represented in the `failed` summary regardless of the number of target accounts. The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated proxy surface. That route SHALL not require a request body and SHALL execute the same warmup behavior as the body-based route for the supplied `mode`. @@ -12,6 +12,14 @@ The system SHALL also expose `POST /v1/warmup/{mode}` on the same authenticated - **WHEN** a client calls `POST /v1/warmup` with a valid API key and valid mode - **THEN** the system returns 200 with a per-account warmup result summary +#### Scenario: Single-account authentication failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyAuthError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `auth_error` + +#### Scenario: Single-account rate-limit failure returns summary +- **WHEN** a valid warmup request targets exactly one account and its submission raises `ProxyRateLimitError` +- **THEN** the system returns 200 with `total_accounts=1` and one `failed` entry with error code `rate_limit_exceeded` + #### Scenario: Invalid mode is rejected - **WHEN** a client calls `POST /v1/warmup` with an unsupported mode value - **THEN** the system returns a 400 invalid request error diff --git a/tests/integration/test_proxy_warmup.py b/tests/integration/test_proxy_warmup.py index 19260d713f..9d2d812d11 100644 --- a/tests/integration/test_proxy_warmup.py +++ b/tests/integration/test_proxy_warmup.py @@ -15,7 +15,7 @@ from app.core.clients.proxy import ProxyResponseError from app.core.config.settings import get_settings from app.core.errors import openai_error -from app.core.exceptions import ProxyRateLimitError +from app.core.exceptions import ProxyAuthError, ProxyRateLimitError from app.core.openai.models import CompactResponsePayload from app.core.upstream_proxy import ResolvedProxyEndpoint, ResolvedUpstreamRoute, UpstreamProxyRouteError from app.core.utils.time import utcnow @@ -962,6 +962,62 @@ async def _fake_compact(payload, headers, access_token, account_id, session=None assert peak_compact_calls == 5 +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("error_type", "error_code", "error_message"), + [ + (ProxyAuthError, "auth_error", "account unauthorized"), + (ProxyRateLimitError, "rate_limit_exceeded", "account limited"), + ], + ids=["auth", "rate-limit"], +) +async def test_single_account_pre_submit_failure_returns_summary( + async_client, + monkeypatch, + error_type, + error_code, + error_message, +): + await _enable_api_key_auth(async_client) + raw_account_id = "acc-warmup-single-failure" + account_id = await _import_account(async_client, raw_account_id, "warmup-single-failure@example.com") + await _add_primary_usage(account_id, used_percent=0.0, window_minutes=300) + _, key = await _create_api_key(async_client, name="warmup-single-failure") + + async def _fake_ensure_fresh(self, account, *, force=False, timeout_seconds=None): + del self, force, timeout_seconds + return account + + async def _fake_compact(payload, headers, access_token, upstream_account_id, session=None): + del payload, headers, access_token, session + assert upstream_account_id == raw_account_id + raise error_type(error_message) + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", _fake_ensure_fresh) + monkeypatch.setattr(proxy_module, "core_compact_responses", _fake_compact) + + response = await async_client.post( + "/v1/warmup", + headers={"Authorization": f"Bearer {key}"}, + json={"mode": "force"}, + ) + + assert response.status_code == 200 + assert response.json() == { + "mode": "force", + "total_accounts": 1, + "submitted": [], + "skipped": [], + "failed": [ + { + "account_id": account_id, + "error_code": error_code, + "error_message": error_message, + } + ], + } + + @pytest.mark.asyncio async def test_warmup_account_rate_limit_failure_does_not_abort_summary(async_client, monkeypatch): await _enable_api_key_auth(async_client) From 4e48f355b519fb20e16e89a5e5c6b2375bb08161 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 09:46:40 +0200 Subject: [PATCH 036/117] fix(proxy): settle terminal spool append failures (#1775) * fix(proxy): settle terminal spool append failures * fix(proxy): detach terminal fallback settlement * fix(proxy): queue terminal output before settlement * fix(proxy): preserve fallback settlement on cancellation * fix(proxy): fence delayed terminal settlement * fix(proxy): distinguish replay settlement identities * fix(proxy): preserve terminal append under cancellation * fix(proxy): preserve terminal settlement response identity * fix(proxy): queue terminal eof before settlement * fix(proxy): preserve terminal fanout delivery * fix(proxy): complete grouped terminal settlement * fix(proxy): complete terminal delivery authority * fix(proxy): preserve grouped cancellation outcome * fix(proxy): preserve terminal delivery claim * fix(proxy): fence partial replay persistence * fix(proxy): fence grouped terminal attempts * fix(proxy): retain recovery attempt generation * fix(proxy): retain persisted terminal response identity --- .../_service/http_bridge/request_submit.py | 6 + .../proxy/_service/http_bridge/streaming.py | 10 + .../_service/http_bridge/upstream_events.py | 347 ++++++++++++--- app/modules/proxy/_service/support.py | 7 + .../proxy/durable_bridge_coordinator.py | 28 ++ .../proxy/durable_bridge_repository.py | 70 +++ .../proxy/http_bridge_event_batcher.py | 62 ++- .../.openspec.yaml | 2 + .../design.md | 38 ++ .../proposal.md | 20 + .../specs/responses-api-compat/spec.md | 62 +++ .../tasks.md | 15 + tests/unit/test_bridge_ring_lifecycle.py | 284 ++++++++++++ tests/unit/test_http_bridge_event_batcher.py | 129 +++++- tests/unit/test_proxy_http_bridge.py | 412 ++++++++++++++++++ 15 files changed, 1417 insertions(+), 75 deletions(-) create mode 100644 openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml create mode 100644 openspec/changes/settle-terminal-spool-append-failure/design.md create mode 100644 openspec/changes/settle-terminal-spool-append-failure/proposal.md create mode 100644 openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/settle-terminal-spool-append-failure/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 9258d60bf3..29823ca125 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -1367,6 +1367,7 @@ async def lookup_operation() -> Any: ), ) request_state.operation_recovery_claimed = True + request_state.operation_attempt_generation = getattr(operation, "recovery_dispatch_count", 0) + 1 # The operation remains fenced to one durable identity. # One-shot mode consumes its existing replay-count budget; # indefinite mode may make further serialized attempts @@ -1406,6 +1407,11 @@ async def lookup_operation() -> Any: request_state.operation_registered = True request_state.operation_rebind_required = False request_state.operation_created = operation.created + request_state.operation_persisted_response_id = ( + None if request_state.operation_recovery_claimed else getattr(operation, "response_id", None) + ) + if not request_state.operation_recovery_claimed: + request_state.operation_attempt_generation = getattr(operation, "recovery_dispatch_count", 0) async def _cleanup_unsubmitted_recovery_claim() -> None: if ( diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index a4cbc518fd..c692de0809 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -1941,6 +1941,12 @@ def switch_to_account_neutral_replay() -> None: else None ) prior_operation_registered = request_state.operation_registered if preserve_operation_identity else False + prior_operation_attempt_generation = ( + request_state.operation_attempt_generation if preserve_operation_identity else 0 + ) + prior_operation_persisted_response_id = ( + request_state.operation_persisted_response_id if preserve_operation_identity else None + ) failed_owner_id = request_state.preferred_account_id _log_http_bridge_event( "owner_unavailable_fresh_resend", @@ -1970,6 +1976,8 @@ def switch_to_account_neutral_replay() -> None: request_state.operation_fingerprint = prior_operation_fingerprint request_state.operation_parent_response_id = prior_operation_parent_response_id request_state.operation_registered = prior_operation_registered + request_state.operation_attempt_generation = prior_operation_attempt_generation + request_state.operation_persisted_response_id = prior_operation_persisted_response_id request_state.operation_rebind_required = True request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity @@ -3436,6 +3444,8 @@ async def rollback_pre_dispatch_recovery_claim() -> None: retry_request_state.operation_fingerprint = request_state.operation_fingerprint retry_request_state.operation_parent_response_id = request_state.operation_parent_response_id retry_request_state.operation_registered = request_state.operation_registered + retry_request_state.operation_attempt_generation = request_state.operation_attempt_generation + retry_request_state.operation_persisted_response_id = request_state.operation_persisted_response_id retry_request_state.operation_rebind_required = request_state.operation_rebind_required if recovery_path == "local_previous_response_error": # The prior response.failed/error made the operation diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 351b3fb54f..98db21e125 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -3,6 +3,7 @@ import asyncio import logging import time +from collections.abc import Awaitable, Callable from dataclasses import replace from typing import Any, TypeVar, cast @@ -55,6 +56,7 @@ ) from app.modules.proxy._service.http_bridge.helpers import ( _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, + _await_task_deferring_cancellation, _http_bridge_durable_lease_ttl_seconds, _http_bridge_eventless_precreated_deadline, _http_bridge_request_budget_seconds, @@ -295,6 +297,8 @@ async def _update_http_bridge_operation_state( state=state, response_id=response_id, ) + if marked and response_id is not None: + request_state.operation_persisted_response_id = response_id if not marked: logger.info( "HTTP bridge operation outcome owner fence rejected operation_id=%s state=%s", @@ -328,38 +332,131 @@ async def _persist_http_bridge_operation_event( *, terminal: bool = False, terminal_state: str | None = None, -) -> None: - """Spool one downstream-visible SSE block for reconnect replay.""" + terminal_event_queue: Any | None = None, + terminal_delivery_scope: _HTTPBridgeCompletedDeliveryScope | None = None, + terminal_append_barrier: Callable[[], Awaitable[None]] | None = None, + terminal_delivery_barrier: Callable[[], Awaitable[None]] | None = None, +) -> bool: + """Spool one downstream-visible SSE block for reconnect replay. + + Return whether terminal failure handling already queued the block. + """ operation_id = getattr(request_state, "operation_id", None) session_id = getattr(session, "durable_session_id", None) owner_epoch = getattr(session, "durable_owner_epoch", None) batcher_enqueue = getattr(getattr(service, "_http_bridge_operation_event_batcher", None), "enqueue", None) append_event = getattr(getattr(service, "_durable_bridge", None), "append_operation_event", None) if not operation_id or session_id is None or owner_epoch is None: - return + return False try: batcher = getattr(service, "_http_bridge_operation_event_batcher", None) append_terminal_batch = getattr(batcher, "append_terminal_event", None) if terminal and terminal_state is not None and callable(append_terminal_batch): - persisted = await append_terminal_batch( - operation_id=operation_id, - session_id=session_id, - instance_id=_service_get_settings().http_responses_session_bridge_instance_id, - owner_epoch=owner_epoch, - event_text=event_block, - max_bytes=int( - getattr( - _service_get_settings(), - "http_responses_session_bridge_operation_event_spool_max_bytes", - 2 * 1024 * 1024, + instance_id = _service_get_settings().http_responses_session_bridge_instance_id + expected_response_ids = tuple( + dict.fromkeys( + response_identity + for response_identity in ( + request_state.response_id, + getattr(request_state, "operation_persisted_response_id", None), + request_state.replay_downstream_response_id, ) + if response_identity is not None + ) + ) + expected_response_id = expected_response_ids[0] if expected_response_ids else None + alternate_expected_response_id = expected_response_ids[1] if len(expected_response_ids) > 1 else None + response_id = _websocket_downstream_response_id(request_state) + + async def enqueue_terminal_delivery() -> bool: + if terminal_event_queue is None: + return False + await terminal_event_queue.put(event_block) + await terminal_event_queue.put(None) + if terminal_delivery_scope is not None: + async with session.pending_lock: + terminal_delivery_scope.terminal_enqueued = True + return True + + async def enqueue_terminal_delivery_deferring_cancellation() -> tuple[bool, asyncio.CancelledError | None]: + delivery_task = asyncio.create_task( + enqueue_terminal_delivery(), + name=f"http-bridge-terminal-delivery-{operation_id}", + ) + return await _await_task_deferring_cancellation(delivery_task) + + append_task = asyncio.create_task( + append_terminal_batch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + event_text=event_block, + max_bytes=int( + getattr( + _service_get_settings(), + "http_responses_session_bridge_operation_event_spool_max_bytes", + 2 * 1024 * 1024, + ) + ), + state=terminal_state, + expected_recovery_dispatch_count=request_state.operation_attempt_generation, + response_id=response_id, ), - state=terminal_state, - response_id=_websocket_downstream_response_id(request_state), + name=f"http-bridge-terminal-append-{operation_id}", ) + append_result, deferred_cancellation = await _await_task_deferring_cancellation(append_task) + if terminal_append_barrier is not None: + await terminal_append_barrier() + persisted = bool(append_result) if not persisted: logger.info("HTTP bridge terminal event spool became incomplete operation_id=%s", operation_id) - return + settlement_required = bool(getattr(append_result, "settlement_required", False)) + terminal_enqueued = False + if settlement_required: + terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation() + deferred_cancellation = deferred_cancellation or delivery_cancellation + if terminal_delivery_barrier is not None: + if not terminal_enqueued: + terminal_enqueued, delivery_cancellation = await enqueue_terminal_delivery_deferring_cancellation() + deferred_cancellation = deferred_cancellation or delivery_cancellation + await terminal_delivery_barrier() + if settlement_required: + settle_terminal_batch = getattr(batcher, "settle_terminal_event", None) + + async def settle_terminal_append_failure() -> None: + if callable(settle_terminal_batch): + await settle_terminal_batch( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=terminal_state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=request_state.operation_attempt_generation, + alternate_expected_response_id=alternate_expected_response_id, + response_id=response_id, + ) + else: + await _update_http_bridge_operation_state( + service, + session, + request_state, + state=terminal_state, + response_id=response_id, + ) + + settlement_task = asyncio.create_task( + settle_terminal_append_failure(), + name=f"http-bridge-terminal-settlement-{operation_id}", + ) + _, settlement_cancellation = await _await_task_deferring_cancellation(settlement_task) + deferred_cancellation = deferred_cancellation or settlement_cancellation + if deferred_cancellation is not None: + if not terminal_enqueued: + await enqueue_terminal_delivery_deferring_cancellation() + raise deferred_cancellation + return terminal_enqueued if callable(batcher_enqueue): await batcher_enqueue( operation_id=operation_id, @@ -369,9 +466,9 @@ async def _persist_http_bridge_operation_event( event_text=event_block, terminal=terminal, ) - return + return False if not callable(append_event): - return + return False persisted = await append_event( operation_id=operation_id, session_id=session_id, @@ -396,11 +493,13 @@ async def _persist_http_bridge_operation_event( state=terminal_state, response_id=_websocket_downstream_response_id(request_state), ) + return False except Exception: # The upstream result is still delivered. A reconnect can only replay # when every event was durably persisted, so never fail a live stream # because the optional spool is unavailable. logger.warning("Failed to persist HTTP bridge operation event operation_id=%s", operation_id, exc_info=True) + return False async def _wait_for_http_bridge_recovery_settlement_retry( @@ -1864,20 +1963,82 @@ async def _process_parsed_http_bridge_upstream_event( if is_missing_tool_output_event else "stream_incomplete" ) - try: - for grouped_request_state in grouped_previous_response_request_states: - grouped_request_state.error_http_status_override = 502 + grouped_terminal_events = [] + for grouped_request_state in grouped_previous_response_request_states: + grouped_request_state.error_http_status_override = 502 + ( + _grouped_downstream_text, + grouped_event_block, + grouped_event, + grouped_payload, + grouped_event_type, + ) = _build_stream_incomplete_terminal_event_for_request( + grouped_request_state, + reason=grouped_error_reason, + ) + grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type) + grouped_terminal_events.append( ( - _grouped_downstream_text, + grouped_request_state, grouped_event_block, grouped_event, grouped_payload, grouped_event_type, - ) = _build_stream_incomplete_terminal_event_for_request( - grouped_request_state, - reason=grouped_error_reason, + grouped_operation_state, ) - grouped_operation_state = _http_bridge_operation_state_for_event(grouped_event_type) + ) + + append_terminal_batch = getattr( + getattr(self, "_http_bridge_operation_event_batcher", None), + "append_terminal_event", + None, + ) + append_participants = { + id(grouped_request_state) + for grouped_request_state, *_rest in grouped_terminal_events + if grouped_request_state.operation_id + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and callable(append_terminal_batch) + } + append_ready = asyncio.Event() + append_lock = asyncio.Lock() + append_arrivals = 0 + if not append_participants: + append_ready.set() + + async def await_all_grouped_appends() -> None: + nonlocal append_arrivals + async with append_lock: + append_arrivals += 1 + if append_arrivals == len(append_participants): + append_ready.set() + await append_ready.wait() + + delivery_ready = asyncio.Event() + delivery_lock = asyncio.Lock() + delivery_arrivals = 0 + + async def await_all_grouped_deliveries() -> None: + nonlocal delivery_arrivals + async with delivery_lock: + delivery_arrivals += 1 + if delivery_arrivals == len(grouped_terminal_events): + delivery_ready.set() + await delivery_ready.wait() + + async def persist_one_grouped_terminal_event( + grouped_terminal_event: tuple[Any, str, OpenAIEvent | None, Any, str | None, str | None], + ) -> None: + ( + grouped_request_state, + grouped_event_block, + _grouped_event, + _grouped_payload, + _grouped_event_type, + grouped_operation_state, + ) = grouped_terminal_event + if id(grouped_request_state) in append_participants: await _persist_http_bridge_operation_event( self, session, @@ -1885,35 +2046,94 @@ async def _process_parsed_http_bridge_upstream_event( grouped_event_block, terminal=True, terminal_state=grouped_operation_state, + terminal_event_queue=grouped_request_state.event_queue, + terminal_append_barrier=await_all_grouped_appends, + terminal_delivery_barrier=await_all_grouped_deliveries, ) + else: + await append_ready.wait() if grouped_request_state.event_queue is not None: await grouped_request_state.event_queue.put(grouped_event_block) await grouped_request_state.event_queue.put(None) - if grouped_operation_state is not None and grouped_operation_state != "failed": - await _update_http_bridge_operation_state( - self, - session, - grouped_request_state, - state=grouped_operation_state, - response_id=_websocket_downstream_response_id(grouped_request_state), - ) - await self._finalize_websocket_request_state( + await await_all_grouped_deliveries() + await _persist_http_bridge_operation_event( + self, + session, + grouped_request_state, + grouped_event_block, + terminal=True, + terminal_state=grouped_operation_state, + ) + if grouped_operation_state is not None and grouped_operation_state != "failed": + await _update_http_bridge_operation_state( + self, + session, grouped_request_state, - account=session.account, - account_id_value=session.account.id, - event=grouped_event, - event_type=grouped_event_type, - payload=grouped_payload, - api_key=grouped_request_state.api_key, - upstream_control=session.upstream_control, - response_create_gate=session.response_create_gate, + state=grouped_operation_state, + response_id=_websocket_downstream_response_id(grouped_request_state), ) - finally: - # Grouped terminal errors settle detached/abandoned requests - # (event_queue is None) with no downstream stream finalizer - # left to run, so release the now-idle session's account - # stream lease here just like the single terminal path below. - await self._maybe_release_idle_http_bridge_session_lease(session) + + async def persist_grouped_terminal_events() -> Exception | None: + first_error: Exception | None = None + persistence_results = await asyncio.gather( + *(persist_one_grouped_terminal_event(item) for item in grouped_terminal_events), + return_exceptions=True, + ) + for persistence_result in persistence_results: + if isinstance(persistence_result, Exception) and first_error is None: + first_error = persistence_result + try: + for ( + grouped_request_state, + _grouped_event_block, + grouped_event, + grouped_payload, + grouped_event_type, + _grouped_operation_state, + ) in grouped_terminal_events: + try: + await self._finalize_websocket_request_state( + grouped_request_state, + account=session.account, + account_id_value=session.account.id, + event=grouped_event, + event_type=grouped_event_type, + payload=grouped_payload, + api_key=grouped_request_state.api_key, + upstream_control=session.upstream_control, + response_create_gate=session.response_create_gate, + ) + except Exception as exc: + if first_error is None: + first_error = exc + except Exception as exc: + if first_error is None: + first_error = exc + try: + # Grouped terminal errors settle detached/abandoned requests + # (event_queue is None) with no downstream stream finalizer + # left to run, so release the now-idle session's account + # stream lease here just like the single terminal path below. + await self._maybe_release_idle_http_bridge_session_lease(session) + except Exception as exc: + if first_error is None: + first_error = exc + return first_error + + grouped_settlement_task = asyncio.create_task( + persist_grouped_terminal_events(), + name=f"http-bridge-grouped-terminal-settlement-{session.durable_session_id}", + ) + grouped_error, grouped_cancellation = await _await_task_deferring_cancellation(grouped_settlement_task) + if grouped_cancellation is not None: + if grouped_error is not None: + logger.warning( + "Grouped HTTP bridge terminal finalization failed while preserving cancellation error=%r", + grouped_error, + ) + raise grouped_cancellation + if grouped_error is not None: + raise grouped_error return if len(grouped_previous_response_request_states) == 1 and terminal_request_state is None: @@ -2693,18 +2913,27 @@ async def _process_parsed_http_bridge_upstream_event( deferred_text, terminal=False, ) + if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event: + for deferred_text in matched_deferred_texts: + await matched_event_queue.put(deferred_text) + matched_terminal_enqueued = False if matched_request_state is not None and not suppress_downstream_event: - await _persist_http_bridge_operation_event( + matched_terminal_enqueued = await _persist_http_bridge_operation_event( self, session, matched_request_state, event_block, terminal=event_type in {"response.completed", "response.failed", "response.incomplete", "error"}, terminal_state=matched_terminal_state, + terminal_event_queue=matched_event_queue, + terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None), ) - if matched_request_state is not None and matched_event_queue is not None and not suppress_downstream_event: - for deferred_text in matched_deferred_texts: - await matched_event_queue.put(deferred_text) + if ( + matched_request_state is not None + and matched_event_queue is not None + and not suppress_downstream_event + and matched_terminal_enqueued is not True + ): await matched_event_queue.put(event_block) if terminal_request_state is None: @@ -2713,6 +2942,7 @@ async def _process_parsed_http_bridge_upstream_event( terminal_event_queue = ( completed_event_queue if completed_event_queue_claimed else terminal_request_state.event_queue ) + terminal_enqueued = matched_terminal_enqueued if terminal_request_state is matched_request_state else False if terminal_request_state is not matched_request_state: deferred_texts = _pop_websocket_deferred_reasoning_downstream_texts(terminal_request_state) for deferred_text in deferred_texts: @@ -2727,7 +2957,7 @@ async def _process_parsed_http_bridge_upstream_event( if terminal_event_queue is not None: await terminal_event_queue.put(deferred_text) if not suppress_downstream_event: - await _persist_http_bridge_operation_event( + terminal_enqueued = await _persist_http_bridge_operation_event( self, session, terminal_request_state, @@ -2738,11 +2968,14 @@ async def _process_parsed_http_bridge_upstream_event( if continuity_persistence_failed_after_ack and terminal_request_state is matched_request_state else _http_bridge_operation_state_for_event(event_type) ), + terminal_event_queue=terminal_event_queue, + terminal_delivery_scope=(completed_delivery_scope if completed_event_queue_claimed else None), ) - if terminal_event_queue is not None: + if terminal_event_queue is not None and terminal_enqueued is not True: await terminal_event_queue.put(event_block) if terminal_event_queue is not None: - await terminal_event_queue.put(None) + if terminal_enqueued is not True: + await terminal_event_queue.put(None) if completed_event_queue_claimed and completed_delivery_scope is not None: async with session.pending_lock: # Keep the completed claim authoritative after its producer diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 80db95953f..e73dfa08cf 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -979,6 +979,13 @@ class _WebSocketRequestState: operation_created: bool = False operation_replay: bool = False operation_dispatched: bool = False + # Immutable durable attempt generation. Recovery claims increment the + # operation's dispatch count before sending a replacement attempt. + operation_attempt_generation: int = 0 + # Last response identity successfully written to the durable operation. + # Retry setup may clear the active response before a replacement is + # acknowledged, but fallback settlement must still fence against this ID. + operation_persisted_response_id: str | None = None # Responses-Lite model advertised by ``fresh_upstream_request_text``. A # fresh replay built from a trusted marker-only frame has the reserved # marker stripped, so swapping to the fresh body must also swap this onto diff --git a/app/modules/proxy/durable_bridge_coordinator.py b/app/modules/proxy/durable_bridge_coordinator.py index d0d98a381a..ad3c7df0c3 100644 --- a/app/modules/proxy/durable_bridge_coordinator.py +++ b/app/modules/proxy/durable_bridge_coordinator.py @@ -601,6 +601,7 @@ async def append_terminal_operation_event( event_text: str, max_bytes: int, state: str, + expected_recovery_dispatch_count: int = 0, response_id: str | None = None, ) -> bool: async with self._session() as session: @@ -612,6 +613,7 @@ async def append_terminal_operation_event( event_text=event_text, max_bytes=max_bytes, state=state, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, response_id=response_id, ) @@ -643,6 +645,32 @@ async def finalize_operation_event_spool( owner_epoch=owner_epoch, ) + async def settle_terminal_append_failure( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, + response_id: str | None = None, + ) -> bool: + async with self._session() as session: + return await DurableBridgeRepository(session).settle_terminal_append_failure( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, + alternate_expected_response_id=alternate_expected_response_id, + response_id=response_id, + ) + async def update_operation( self, *, diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index f9780ef329..7291222bd2 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -1784,6 +1784,7 @@ async def append_terminal_operation_event( event_text: str, max_bytes: int, state: str, + expected_recovery_dispatch_count: int = 0, response_id: str | None = None, ) -> bool: """Append a terminal event and expose its operation state atomically.""" @@ -1802,6 +1803,7 @@ async def append_terminal_operation_event( .where( HttpBridgeOperationRecord.operation_id == operation_id, HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count, ) .with_for_update() ) @@ -2010,6 +2012,74 @@ async def get_latest_completed_operation_any_session( ) return _to_operation_snapshot(operation) if operation is not None else None + async def settle_terminal_append_failure( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, + response_id: str | None = None, + ) -> bool: + """Settle only the terminal attempt whose append outcome was ambiguous.""" + async with sqlite_writer_section(): + owner_exists = await self._session.scalar( + select(HttpBridgeSessionRecord.id) + .where( + HttpBridgeSessionRecord.id == session_id, + HttpBridgeSessionRecord.owner_instance_id == instance_id, + HttpBridgeSessionRecord.owner_epoch == owner_epoch, + ) + .with_for_update() + ) + if owner_exists is None: + await self._session.rollback() + return False + acknowledged_response_matches = ( + HttpBridgeOperationRecord.response_id == expected_response_id + if expected_response_id is not None + else HttpBridgeOperationRecord.response_id.is_(None) + ) + if alternate_expected_response_id is not None: + acknowledged_response_matches = or_( + acknowledged_response_matches, + HttpBridgeOperationRecord.response_id == alternate_expected_response_id, + ) + terminal_response_matches = ( + HttpBridgeOperationRecord.response_id == response_id + if response_id is not None + else HttpBridgeOperationRecord.response_id.is_(None) + ) + values: dict[str, object] = { + "state": state, + "event_spool_complete": False, + "updated_at": utcnow(), + } + if response_id is not None: + values["response_id"] = response_id + result = await self._session.execute( + update(HttpBridgeOperationRecord) + .where( + HttpBridgeOperationRecord.operation_id == operation_id, + HttpBridgeOperationRecord.session_id == session_id, + HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count, + or_( + and_(HttpBridgeOperationRecord.state == "acknowledged", acknowledged_response_matches), + and_( + HttpBridgeOperationRecord.state == state, + or_(acknowledged_response_matches, terminal_response_matches), + ), + ), + ) + .values(**values) + ) + await self._session.commit() + return bool(getattr(result, "rowcount", 0)) + async def update_operation( self, *, diff --git a/app/modules/proxy/http_bridge_event_batcher.py b/app/modules/proxy/http_bridge_event_batcher.py index a22bc733d1..3ec3892ee2 100644 --- a/app/modules/proxy/http_bridge_event_batcher.py +++ b/app/modules/proxy/http_bridge_event_batcher.py @@ -20,6 +20,15 @@ class _PendingOperationEvent: event_text: str +@dataclass(frozen=True, slots=True) +class TerminalOperationEventAppendResult: + persisted: bool + settlement_required: bool = False + + def __bool__(self) -> bool: + return self.persisted + + class HttpBridgeOperationEventBatcher: """Best-effort in-memory event buffer for the HTTP bridge. @@ -240,8 +249,9 @@ async def append_terminal_event( event_text: str, max_bytes: int, state: str, + expected_recovery_dispatch_count: int = 0, response_id: str | None = None, - ) -> bool: + ) -> TerminalOperationEventAppendResult: """Drain queued events and atomically append the terminal outcome.""" async with self._lock: self._contexts.setdefault( @@ -260,7 +270,7 @@ async def append_terminal_event( context = self._contexts.get(operation_id) dropped = operation_id in self._dropped_operations if context is None: - return False + return TerminalOperationEventAppendResult(persisted=False) if dropped: try: await self._durable_bridge.update_operation( @@ -282,7 +292,7 @@ async def append_terminal_event( self._closing_operations.discard(operation_id) self._contexts.pop(operation_id, None) self._dropped_operations.discard(operation_id) - return False + return TerminalOperationEventAppendResult(persisted=False) try: persisted = await self._durable_bridge.append_terminal_operation_event( operation_id=operation_id, @@ -292,22 +302,64 @@ async def append_terminal_event( event_text=event_text, max_bytes=max_bytes, state=state, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, response_id=response_id, ) - return bool(persisted and not dropped) + return TerminalOperationEventAppendResult(persisted=bool(persisted and not dropped)) except Exception: logger.debug( "Failed to append terminal HTTP bridge event operation_id=%s", operation_id, exc_info=True, ) - return False + return TerminalOperationEventAppendResult( + persisted=False, + settlement_required=True, + ) finally: async with self._lock: self._closing_operations.discard(operation_id) self._contexts.pop(operation_id, None) self._dropped_operations.discard(operation_id) + async def settle_terminal_event( + self, + *, + operation_id: str, + session_id: str, + instance_id: str, + owner_epoch: int, + state: str, + expected_response_id: str | None, + expected_recovery_dispatch_count: int = 0, + alternate_expected_response_id: str | None = None, + response_id: str | None = None, + ) -> None: + """Settle a failed terminal append after its SSE block was queued.""" + try: + settled = await self._durable_bridge.settle_terminal_append_failure( + operation_id=operation_id, + session_id=session_id, + instance_id=instance_id, + owner_epoch=owner_epoch, + state=state, + expected_response_id=expected_response_id, + expected_recovery_dispatch_count=expected_recovery_dispatch_count, + alternate_expected_response_id=alternate_expected_response_id, + response_id=response_id, + ) + if not settled: + logger.warning( + "Terminal HTTP bridge operation fallback settlement was fenced operation_id=%s", + operation_id, + ) + except Exception: + logger.warning( + "Failed to settle terminal HTTP bridge operation after event append failure operation_id=%s", + operation_id, + exc_info=True, + ) + async def flush_pending_operation(self, *, operation_id: str) -> bool: """Drain queued events while retaining the operation context.""" while True: diff --git a/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml b/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml new file mode 100644 index 0000000000..f161d5cc47 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-16 diff --git a/openspec/changes/settle-terminal-spool-append-failure/design.md b/openspec/changes/settle-terminal-spool-append-failure/design.md new file mode 100644 index 0000000000..59f11dd605 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/design.md @@ -0,0 +1,38 @@ +## Context + +The terminal HTTP-bridge event path intentionally skips a separate operation-state update because `append_terminal_operation_event` normally stores the terminal event and authoritative state atomically. If that repository call raises, the batcher currently logs and returns `False`; the durable row can therefore remain `acknowledged` with an incomplete spool after the terminal event was delivered downstream. + +The durable bridge exposes operation settlement under the same operation, session, instance, and owner-epoch fence. Fallback additionally needs to identify the acknowledged terminal attempt so a delayed write cannot overwrite a newer retry admitted under the same owner epoch. + +## Goals / Non-Goals + +**Goals:** + +- Preserve an authoritative terminal operation state after a terminal append exception. +- Apply the existing session and owner-epoch fence plus an acknowledged-state and response-identity comparison to fallback settlement. +- Keep the incomplete spool ineligible for transcript replay. +- Prove the result through the production repository/coordinator seam. + +**Non-Goals:** + +- Drain queued events during graceful shutdown. +- Change successful terminal event persistence or replay eligibility. +- Change warmup, upstream delivery, retry policy, or public response shapes. + +## Decisions + +- On `append_terminal_operation_event` exception, return an incomplete append result that explicitly requires fallback settlement. The relay queues the selected terminal SSE block and end-of-stream marker before awaiting a dedicated conditional settlement with the same operation ID, session ID, instance ID, owner epoch, immutable recovery-dispatch generation, intended terminal state, persisted upstream response IDs, and client-visible response ID. The repository accepts only that acknowledged attempt or its already-committed terminal result, without keeping the terminal event behind a stalled fallback write. +- Keep append and fallback settlement structured in the relay task instead of detaching them. This bounds settlement concurrency to active relay operations, and the relay defers cancellation until the append and any required settlement finish before preserving the cancellation outcome. +- Force `event_spool_complete=false` in the same conditional fallback update. This keeps replay disabled even when terminal append committed but its commit acknowledgement was lost before the caller observed success. +- Log a rejected fence or fallback exception inside the batcher's settlement method and do not re-raise. The terminal event has already been queued for downstream delivery, so bookkeeping failure must not replace or delay that event. +- Do not invoke fallback for ordinary `False` returns. The repository's bounded-spool overflow path already settles terminal state atomically, while a false owner fence must not be bypassed. + +## Risks / Trade-offs + +- [A transient database failure can affect both append and fallback update] -> Queue the terminal event and end-of-stream marker before awaiting structured fallback settlement and emit a warning for operator diagnosis. +- [Relay cancellation can interrupt terminal append or the delivery-authority claim] -> Defer cancellation through append, delivery ownership, and any required fallback; mark completed delivery authoritative before preserving cancellation. +- [A grouped terminal fan-out can release owner authority too early or stall on its first fallback] -> Start every owner-fenced append concurrently and await all append outcomes before exposing any sibling queue, then queue every sibling terminal event and end-of-stream marker before fallback settlement, settle every sibling before finalization, continue later finalizers after one fails, and preserve cancellation as the final outcome. +- [A stale owner could attempt to settle another owner's operation] -> Pass the unchanged session/instance/epoch fence and treat rejection as non-settlement. +- [A delayed append or fallback could overwrite a newer retry under the same owner] -> Require the prior attempt's immutable recovery-dispatch generation plus acknowledged/terminal state and response identity in both persistence predicates. +- [Replay aliases can differ from the upstream response identity persisted at acknowledgement] -> Carry both the active upstream identity and retained replay identity as possible CAS values separately from the client-visible terminal identity, covering a failed replacement-acknowledgement write, and preserve the known identity when no new client-visible identity is supplied. +- [A failed terminal append leaves no replayable terminal event] -> Keep `event_spool_complete` false and report `persisted=false`; authoritative state and transcript completeness remain separate facts. diff --git a/openspec/changes/settle-terminal-spool-append-failure/proposal.md b/openspec/changes/settle-terminal-spool-append-failure/proposal.md new file mode 100644 index 0000000000..b9693c33ee --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/proposal.md @@ -0,0 +1,20 @@ +## Why + +A durable HTTP-bridge operation can remain `acknowledged` after its terminal event has already been delivered downstream when terminal transcript persistence raises. Reconnect and recovery must observe an authoritative terminal outcome rather than treating that operation as incomplete work. + +## What Changes + +- Settle the operation to its intended terminal state when atomic terminal-event append raises. +- Preserve the existing owner/session/epoch fence, reject settlement after a newer same-owner retry, and leave the event spool explicitly incomplete. +- Keep successful terminal append behavior unchanged. +- Add deterministic unit and production-repository recovery coverage for the failure path. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: terminal HTTP-bridge operation settlement remains authoritative when terminal transcript persistence fails. + +## Impact + +The HTTP-bridge event batcher, focused durable bridge tests, and reconnect/recovery semantics are affected. Public request and response shapes, graceful-shutdown draining, and warmup behavior are unchanged. diff --git a/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md b/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..bd0158a3f0 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/specs/responses-api-compat/spec.md @@ -0,0 +1,62 @@ +## ADDED Requirements + +### Requirement: Terminal append failure preserves authoritative settlement + +When durable append of a terminal HTTP-bridge event raises after the operation was acknowledged, the proxy MUST attempt to persist the intended terminal operation state through the same operation, session, instance, and owner-epoch fence. Cancellation MUST be deferred through the append and any required fallback settlement. The event spool MUST remain incomplete, and the persistence failure MUST NOT replace or block the terminal event and end-of-stream marker already selected for downstream delivery. A rejected or failed fallback settlement MUST be logged and MUST NOT bypass the owner fence or overwrite a newer operation attempt admitted under the same owner epoch. + +#### Scenario: Terminal append exception settles the current owner operation + +- **GIVEN** an acknowledged HTTP-bridge operation owned by the current session epoch +- **WHEN** durable terminal-event append raises +- **THEN** the operation is persisted in the intended terminal state +- **AND** its event spool remains incomplete +- **AND** the terminal event and end-of-stream marker are queued before fallback settlement can stall +- **AND** reconnect or recovery does not observe the operation as acknowledged work + +#### Scenario: Grouped failures deliver every sibling before settlement + +- **GIVEN** one upstream error selects terminal failures for multiple pending operations +- **WHEN** the first operation's fallback settlement stalls +- **THEN** every selected operation attempts its owner-fenced terminal append before any terminal queue is exposed +- **AND** every selected operation then receives its terminal event and end-of-stream marker before fallback settlement +- **AND** sibling delivery does not wait for the first fallback settlement +- **AND** cancellation is preserved as the final outcome only after every pre-delivered sibling finishes settlement and finalization +- **AND** one sibling's finalization failure does not prevent later siblings from settling or replace pending cancellation + +#### Scenario: Cancellation preserves terminal delivery authority + +- **GIVEN** terminal append finishes while relay cancellation is deferred +- **WHEN** the append result becomes available +- **THEN** the terminal event and end-of-stream marker are queued +- **AND** a completed-delivery scope is marked authoritative before cleanup can deactivate it +- **AND** cancellation during that delivery-authority claim does not skip required fallback settlement +- **AND** cancellation is preserved only after delivery and required settlement + +#### Scenario: Stale owner cannot settle after terminal append exception + +- **GIVEN** an HTTP-bridge operation whose owner epoch has advanced +- **WHEN** the stale batcher encounters a terminal-event append exception +- **THEN** fallback settlement is rejected by the durable owner fence +- **AND** the stale batcher does not mutate the operation state + +#### Scenario: Newer retry rejects delayed fallback settlement + +- **GIVEN** terminal append committed its operation state before reporting an exception +- **AND** a retry under the same owner epoch has since reset the operation to submitted +- **WHEN** fallback settlement for the prior attempt runs +- **THEN** the fallback is rejected by an immutable recovery-attempt generation plus operation-state and persisted upstream-response identity fence +- **AND** the newer submitted attempt remains unchanged + +#### Scenario: Replay alias preserves the acknowledged-attempt fence + +- **GIVEN** a replay whose client-visible response alias differs from its persisted upstream response ID or whose active upstream response ID was reset before a replacement response was created +- **WHEN** durable terminal-event append raises +- **THEN** fallback settlement compares the acknowledged or already terminal operation against every response identity that may remain persisted when a replacement acknowledgement update fails +- **AND** persists the intended client-visible terminal response ID when present +- **AND** otherwise preserves the known upstream response ID + +#### Scenario: Successful terminal append remains atomic and replayable + +- **WHEN** durable terminal-event append succeeds +- **THEN** the terminal event and intended operation state are persisted atomically +- **AND** the completed event spool remains eligible for replay diff --git a/openspec/changes/settle-terminal-spool-append-failure/tasks.md b/openspec/changes/settle-terminal-spool-append-failure/tasks.md new file mode 100644 index 0000000000..231106f0d3 --- /dev/null +++ b/openspec/changes/settle-terminal-spool-append-failure/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression + +- [x] 1.1 Add deterministic regressions proving a terminal append exception settles through the unchanged owner fence without overwriting a newer retry, blocking terminal EOF, or starving grouped siblings. +- [x] 1.2 Capture the focused regression failing before production code changes. + +## 2. Implementation + +- [x] 2.1 Add the minimum fenced fallback settlement for terminal append exceptions without claiming spool completeness. +- [x] 2.2 Add a production-repository process/recovery proof that a reconnect cannot observe the operation as acknowledged and delayed settlement cannot overwrite its retry. + +## 3. Verification + +- [x] 3.1 Capture focused GREEN and run adjacent HTTP-bridge unit and integration tests. +- [x] 3.2 Run changed-file formatting, lint, type diagnostics, strict OpenSpec validation, and package/build checks required by the repository. +- [x] 3.3 Review the committed diff independently and address every in-scope finding. diff --git a/tests/unit/test_bridge_ring_lifecycle.py b/tests/unit/test_bridge_ring_lifecycle.py index 6c39812afe..6cead41406 100644 --- a/tests/unit/test_bridge_ring_lifecycle.py +++ b/tests/unit/test_bridge_ring_lifecycle.py @@ -35,12 +35,14 @@ _http_bridge_durable_lookup_allows_turn_state_takeover, ) from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key +from app.modules.proxy.durable_bridge_coordinator import DurableBridgeSessionCoordinator from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeRepository, durable_bridge_hash, durable_bridge_operation_id, ) +from app.modules.proxy.http_bridge_event_batcher import HttpBridgeOperationEventBatcher from app.modules.proxy.ring_membership import RingMembershipService pytestmark = pytest.mark.unit @@ -808,6 +810,288 @@ async def test_terminal_failure_exposes_state_when_spool_overflows( await session.close() +@pytest.mark.asyncio +async def test_terminal_append_failure_settlement_is_visible_to_recovery( + async_session_factory: Callable[[], AsyncSession], + monkeypatch: pytest.MonkeyPatch, +) -> None: + session = async_session_factory() + try: + repository = DurableBridgeRepository(session) + claim = await _claim( + repository, + instance_id="inst-terminal-recovery", + session_key_value="sid-terminal-recovery", + ) + fingerprint = durable_bridge_hash("terminal-recovery") + operation_id = durable_bridge_operation_id(claim.id, fingerprint) + operation = await repository.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert operation is not None + assert await repository.append_operation_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.created"}\n\n', + max_bytes=1024, + ) + assert await repository.update_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-terminal-recovery", + ) + + replay_fingerprint = durable_bridge_hash("terminal-recovery-replay-alias") + replay_operation_id = durable_bridge_operation_id(claim.id, replay_fingerprint) + assert await repository.record_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=replay_fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id="resp-client-visible-replay", + ) + replay_operation = await repository.get_operation(operation_id=replay_operation_id) + assert replay_operation is not None + assert replay_operation.state == "failed" + assert replay_operation.response_id == "resp-client-visible-replay" + assert replay_operation.event_spool_complete is False + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id="resp-client-visible-replay", + ) + pre_settled_replay = await repository.get_operation(operation_id=replay_operation_id) + assert pre_settled_replay is not None + assert pre_settled_replay.state == "failed" + assert pre_settled_replay.response_id == "resp-client-visible-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-persisted-before-replacement", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-unpersisted-replacement", + alternate_expected_response_id="resp-persisted-before-replacement", + response_id="resp-client-visible-replay", + ) + partially_persisted_replay = await repository.get_operation(operation_id=replay_operation_id) + assert partially_persisted_replay is not None + assert partially_persisted_replay.state == "failed" + assert partially_persisted_replay.response_id == "resp-client-visible-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + response_id=None, + ) + null_alias_settlement = await repository.get_operation(operation_id=replay_operation_id) + assert null_alias_settlement is not None + assert null_alias_settlement.state == "failed" + assert null_alias_settlement.response_id == "resp-upstream-replay" + + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="unknown", + ) + assert await repository.claim_unknown_operation_for_recovery( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + ) + assert await repository.update_operation( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="acknowledged", + response_id="resp-upstream-replay", + ) + assert not await repository.append_terminal_operation_event( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + expected_recovery_dispatch_count=0, + response_id="resp-client-visible-replay", + ) + assert not await repository.settle_terminal_append_failure( + operation_id=replay_operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-upstream-replay", + expected_recovery_dispatch_count=0, + response_id="resp-client-visible-replay", + ) + newer_attempt = await repository.get_operation(operation_id=replay_operation_id) + assert newer_attempt is not None + assert newer_attempt.state == "acknowledged" + assert newer_attempt.recovery_dispatch_count == 1 + assert newer_attempt.event_spool_complete is False + finally: + await session.close() + + coordinator = DurableBridgeSessionCoordinator(async_session_factory) + append_terminal_operation_event = coordinator.append_terminal_operation_event + settle_terminal_append_failure = coordinator.settle_terminal_append_failure + settlement_finished = asyncio.Event() + + async def fail_terminal_append(**kwargs: Any) -> bool: + assert await append_terminal_operation_event(**kwargs) + raise RuntimeError("injected post-commit terminal append failure") + + async def track_terminal_settlement(**kwargs: Any) -> bool: + try: + return await settle_terminal_append_failure(**kwargs) + finally: + settlement_finished.set() + + monkeypatch.setattr(coordinator, "append_terminal_operation_event", fail_terminal_append) + monkeypatch.setattr(coordinator, "settle_terminal_append_failure", track_terminal_settlement) + batcher = HttpBridgeOperationEventBatcher( + coordinator, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + append_result = await batcher.append_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + event_text='data: {"type":"response.failed"}\n\n', + max_bytes=1024, + state="failed", + response_id="resp-terminal-recovery", + ) + assert append_result.persisted is False + assert append_result.settlement_required is True + await batcher.settle_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-terminal-recovery", + response_id="resp-terminal-recovery", + ) + await asyncio.wait_for(settlement_finished.wait(), timeout=1.0) + + recovery = DurableBridgeSessionCoordinator(async_session_factory) + observed = await recovery.get_operation_by_fingerprint(request_fingerprint=fingerprint) + assert observed is not None + assert observed.operation_id == operation_id + assert observed.session_id == claim.id + assert observed.account_id == "account-terminal-recovery" + assert observed.state == "failed" + assert observed.event_spool_complete is False + assert await recovery.get_operation_events(operation_id=operation_id) == [ + 'data: {"type":"response.created"}\n\n', + 'data: {"type":"response.failed"}\n\n', + ] + + retry = await recovery.record_operation( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + request_fingerprint=fingerprint, + account_id="account-terminal-recovery", + model="gpt-5.6", + parent_response_id="resp-parent", + ) + assert retry is not None + assert retry.state == "submitted" + await batcher.settle_terminal_event( + operation_id=operation_id, + session_id=claim.id, + instance_id="inst-terminal-recovery", + owner_epoch=claim.owner_epoch, + state="failed", + expected_response_id="resp-terminal-recovery", + response_id="resp-terminal-recovery", + ) + after_stale_settlement = await recovery.get_operation(operation_id=operation_id) + assert after_stale_settlement is not None + assert after_stale_settlement.state == "submitted" + assert after_stale_settlement.response_id is None + await batcher.close() + + @pytest.mark.asyncio async def test_consumed_recovery_checkpoint_does_not_rebind_failed_operation( async_session_factory: Callable[[], AsyncSession], diff --git a/tests/unit/test_http_bridge_event_batcher.py b/tests/unit/test_http_bridge_event_batcher.py index bf8eddea23..bb6462b936 100644 --- a/tests/unit/test_http_bridge_event_batcher.py +++ b/tests/unit/test_http_bridge_event_batcher.py @@ -8,8 +8,9 @@ class _FakeDurableBridge: - def __init__(self, *, append_result: bool = True) -> None: + def __init__(self, *, append_result: bool = True, update_result: bool = True) -> None: self.append_result = append_result + self.update_result = update_result self.batches: list[list[str]] = [] self.finalized: list[str] = [] self.updated: list[dict[str, object]] = [] @@ -25,7 +26,26 @@ async def finalize_operation_event_spool(self, **kwargs) -> bool: async def update_operation(self, **kwargs) -> bool: self.updated.append(kwargs) - return True + return self.update_result + + async def settle_terminal_append_failure(self, **kwargs) -> bool: + kwargs["event_spool_complete"] = False + return await self.update_operation(**kwargs) + + +class _TerminalAppendFailingDurableBridge(_FakeDurableBridge): + def __init__(self, *, append_result: bool = True, update_result: bool = True) -> None: + super().__init__(append_result=append_result, update_result=update_result) + self.update_called = asyncio.Event() + + async def append_terminal_operation_event(self, **kwargs) -> bool: + del kwargs + raise RuntimeError("injected terminal append failure") + + async def update_operation(self, **kwargs) -> bool: + result = await super().update_operation(**kwargs) + self.update_called.set() + return result async def _enqueue( @@ -103,18 +123,17 @@ async def test_dropped_batch_is_never_marked_replayable() -> None: if durable.batches: break await asyncio.sleep(0.01) - assert ( - await batcher.append_terminal_event( - operation_id="op-1", - session_id="session-1", - instance_id="instance-1", - owner_epoch=1, - event_text="terminal", - max_bytes=1024, - state="failed", - ) - is False + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=1, + event_text="terminal", + max_bytes=1024, + state="failed", ) + assert result.persisted is False + assert result.settlement_required is False assert durable.finalized == [] assert durable.updated[0]["state"] == "failed" assert batcher._contexts == {} @@ -123,6 +142,90 @@ async def test_dropped_batch_is_never_marked_replayable() -> None: await batcher.close() +@pytest.mark.asyncio +async def test_terminal_append_failure_settles_operation() -> None: + durable = _TerminalAppendFailingDurableBridge() + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=7, + event_text="terminal", + max_bytes=1024, + state="failed", + response_id="resp-1", + ) + + assert result.persisted is False + assert result.settlement_required is True + await batcher.settle_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="instance-1", + owner_epoch=7, + state="failed", + expected_response_id="resp-upstream-1", + response_id="resp-1", + ) + await asyncio.wait_for(durable.update_called.wait(), timeout=1.0) + assert durable.updated == [ + { + "operation_id": "op-1", + "session_id": "session-1", + "instance_id": "instance-1", + "owner_epoch": 7, + "state": "failed", + "expected_response_id": "resp-upstream-1", + "expected_recovery_dispatch_count": 0, + "alternate_expected_response_id": None, + "response_id": "resp-1", + "event_spool_complete": False, + } + ] + + +@pytest.mark.asyncio +async def test_terminal_append_failure_reports_fenced_settlement( + caplog: pytest.LogCaptureFixture, +) -> None: + durable = _TerminalAppendFailingDurableBridge(update_result=False) + batcher = HttpBridgeOperationEventBatcher( + durable, + max_bytes=1024, + flush_interval_seconds=60.0, + ) + + result = await batcher.append_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="stale-instance", + owner_epoch=6, + event_text="terminal", + max_bytes=1024, + state="failed", + ) + + assert result.persisted is False + assert result.settlement_required is True + await batcher.settle_terminal_event( + operation_id="op-1", + session_id="session-1", + instance_id="stale-instance", + owner_epoch=6, + state="failed", + expected_response_id=None, + ) + await asyncio.wait_for(durable.update_called.wait(), timeout=1.0) + assert durable.updated[0]["owner_epoch"] == 6 + assert "fallback settlement was fenced operation_id=op-1" in caplog.text + + @pytest.mark.asyncio async def test_discard_operation_releases_partial_nonterminal_context() -> None: durable = _FakeDurableBridge() diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 4477ec1ae8..761b631131 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -61,6 +61,7 @@ DurableBridgeAliasRegistrationReceipt, ) from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch +from app.modules.proxy.http_bridge_event_batcher import TerminalOperationEventAppendResult from app.modules.proxy.http_bridge_forwarding import OwnerForwardRelayFailure from app.modules.proxy.load_balancer import CONTINUITY_OWNER_UNAVAILABLE, CatalogOmissionQuotaAdmission @@ -5201,6 +5202,417 @@ async def append_terminal_event(*args: Any, **kwargs: Any) -> bool: assert order == ["terminal"] +@pytest.mark.asyncio +async def test_terminal_append_failure_retains_last_persisted_response_id_after_retry_reset() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = SimpleNamespace( + operation_id="op-terminal-retry-reset", + operation_attempt_generation=0, + operation_persisted_response_id=None, + request_id="req-terminal-retry-reset", + response_id="resp-before-retry", + replay_downstream_response_id=None, + ) + session = _make_bridge_session(key_value="terminal-retry-reset") + session.durable_session_id = "durable-terminal-retry-reset" + session.durable_owner_epoch = 2 + service._durable_bridge = cast(Any, SimpleNamespace(update_operation=AsyncMock(return_value=True))) + + await http_bridge_upstream_events_module._update_http_bridge_operation_state( + service, + session, + request_state, + state="acknowledged", + response_id="resp-before-retry", + ) + assert request_state.operation_persisted_response_id == "resp-before-retry" + + request_state.response_id = None + settle_terminal_event = AsyncMock() + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=AsyncMock( + return_value=TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + ), + settle_terminal_event=settle_terminal_event, + ), + ) + + await http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + 'data: {"type":"response.failed"}\n\n', + terminal=True, + terminal_state="failed", + ) + + assert settle_terminal_event.await_args is not None + assert settle_terminal_event.await_args.kwargs["expected_response_id"] == "resp-before-retry" + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("upstream_response_id", "replay_response_id", "expected_response_id", "alternate_expected_response_id"), + [ + ( + "resp-terminal-append-fallback-order", + "resp-client-visible-replay", + "resp-terminal-append-fallback-order", + "resp-client-visible-replay", + ), + (None, "resp-persisted-before-replay", "resp-persisted-before-replay", None), + ], +) +async def test_terminal_append_failure_queues_before_stalled_fallback_settlement( + upstream_response_id: str | None, + replay_response_id: str, + expected_response_id: str, + alternate_expected_response_id: str | None, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + event_queue: asyncio.Queue[str | None] = asyncio.Queue() + request_state = proxy_service._WebSocketRequestState( + request_id="req-terminal-append-fallback-order", + response_id=upstream_response_id, + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-terminal-append-fallback-order" + request_state.operation_attempt_generation = 2 + request_state.replay_downstream_response_id = replay_response_id + session = _make_bridge_session( + key_value="terminal-append-fallback-order", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-terminal-append-fallback-order" + session.durable_owner_epoch = 3 + append_started = asyncio.Event() + release_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + settlement_finished = asyncio.Event() + append_kwargs: dict[str, Any] = {} + settlement_kwargs: dict[str, Any] = {} + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args + append_kwargs.update(kwargs) + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args + settlement_kwargs.update(kwargs) + settlement_started.set() + await release_settlement.wait() + settlement_finished.set() + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, + ), + ) + event_block = 'data: {"type":"response.failed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="failed", + terminal_event_queue=event_queue, + ) + ) + + await asyncio.wait_for(append_started.wait(), timeout=1.0) + persist_task.cancel() + assert persist_task.cancelling() + assert persist_task.done() is False + release_append.set() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + assert append_kwargs["response_id"] == replay_response_id + assert append_kwargs["expected_recovery_dispatch_count"] == 2 + assert settlement_kwargs["expected_response_id"] == expected_response_id + assert settlement_kwargs["expected_recovery_dispatch_count"] == 2 + assert settlement_kwargs["alternate_expected_response_id"] == alternate_expected_response_id + assert settlement_kwargs["response_id"] == replay_response_id + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert persist_task.done() is False + release_settlement.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert settlement_finished.is_set() + + +@pytest.mark.asyncio +async def test_terminal_append_failure_defers_cancellation_through_delivery_claim() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + event_queue: asyncio.Queue[str | None] = asyncio.Queue() + request_state = proxy_service._WebSocketRequestState( + request_id="req-terminal-delivery-claim-cancellation", + response_id="resp-terminal-delivery-claim-cancellation", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-terminal-delivery-claim-cancellation" + session = _make_bridge_session( + key_value="terminal-delivery-claim-cancellation", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-terminal-delivery-claim-cancellation" + session.durable_owner_epoch = 4 + append_started = asyncio.Event() + release_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + settlement_finished = asyncio.Event() + completed_delivery_scope = proxy_support_module._HTTPBridgeCompletedDeliveryScope(active=True) + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args, kwargs + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args, kwargs + settlement_started.set() + await release_settlement.wait() + settlement_finished.set() + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, + ), + ) + event_block = 'data: {"type":"response.failed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="failed", + terminal_event_queue=event_queue, + terminal_delivery_scope=completed_delivery_scope, + ) + ) + + await asyncio.wait_for(append_started.wait(), timeout=1.0) + await session.pending_lock.acquire() + release_append.set() + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + persist_task.cancel() + assert persist_task.done() is False + session.pending_lock.release() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + assert completed_delivery_scope.terminal_enqueued is True + assert persist_task.done() is False + release_settlement.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert settlement_finished.is_set() + + +@pytest.mark.asyncio +async def test_terminal_append_success_queues_output_before_preserving_cancellation() -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + event_queue: asyncio.Queue[str | None] = asyncio.Queue() + request_state = proxy_service._WebSocketRequestState( + request_id="req-terminal-append-success-cancellation", + response_id="resp-terminal-append-success-cancellation", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + skip_request_log=True, + ) + request_state.operation_id = "op-terminal-append-success-cancellation" + session = _make_bridge_session( + key_value="terminal-append-success-cancellation", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.durable_session_id = "durable-terminal-append-success-cancellation" + session.durable_owner_epoch = 4 + append_started = asyncio.Event() + release_append = asyncio.Event() + completed_delivery_scope = proxy_support_module._HTTPBridgeCompletedDeliveryScope(active=True) + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args, kwargs + append_started.set() + await release_append.wait() + return TerminalOperationEventAppendResult(persisted=True, settlement_required=False) + + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace(append_terminal_event=append_terminal_event), + ) + event_block = 'data: {"type":"response.completed"}\n\n' + persist_task = asyncio.create_task( + http_bridge_upstream_events_module._persist_http_bridge_operation_event( + service, + session, + request_state, + event_block, + terminal=True, + terminal_state="completed", + terminal_event_queue=event_queue, + terminal_delivery_scope=completed_delivery_scope, + ) + ) + + await asyncio.wait_for(append_started.wait(), timeout=1.0) + persist_task.cancel() + release_append.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(persist_task, timeout=1.0) + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) == event_block + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert completed_delivery_scope.terminal_enqueued is True + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("cancel_during_settlement", "finalizer_fails"), + [(False, True), (True, False), (True, True)], +) +async def test_grouped_terminal_fanout_queues_all_siblings_before_stalled_settlement( + monkeypatch: pytest.MonkeyPatch, + cancel_during_settlement: bool, + finalizer_fails: bool, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + queues: list[asyncio.Queue[str | None]] = [asyncio.Queue(), asyncio.Queue()] + request_states: list[proxy_service._WebSocketRequestState] = [] + for index, event_queue in enumerate(queues): + request_state = proxy_service._WebSocketRequestState( + request_id=f"req-grouped-terminal-{index}", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=event_queue, + transport="http", + previous_response_id="resp-shared-grouped-terminal", + skip_request_log=True, + ) + request_state.operation_id = f"op-grouped-terminal-{index}" + request_states.append(request_state) + session = _make_bridge_session( + key_value="grouped-terminal-fanout", + pending_requests=deque(request_states), + queued_request_count=2, + ) + session.durable_session_id = "durable-grouped-terminal-fanout" + session.durable_owner_epoch = 5 + all_appends_started = asyncio.Event() + release_first_append = asyncio.Event() + settlement_started = asyncio.Event() + release_settlement = asyncio.Event() + append_calls: list[str] = [] + + async def append_terminal_event(*args: Any, **kwargs: Any) -> TerminalOperationEventAppendResult: + del args + operation_id = kwargs["operation_id"] + append_calls.append(operation_id) + if len(append_calls) == 2: + all_appends_started.set() + if operation_id == "op-grouped-terminal-0": + await release_first_append.wait() + return TerminalOperationEventAppendResult(persisted=False, settlement_required=True) + + async def settle_terminal_event(*args: Any, **kwargs: Any) -> None: + del args, kwargs + settlement_started.set() + await release_settlement.wait() + + finalize_request = AsyncMock( + side_effect=[RuntimeError("first sibling finalization failed"), None] if finalizer_fails else None + ) + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request) + monkeypatch.setattr(service, "_maybe_release_idle_http_bridge_session_lease", AsyncMock()) + service._http_bridge_operation_event_batcher = cast( + Any, + SimpleNamespace( + append_terminal_event=append_terminal_event, + settle_terminal_event=settle_terminal_event, + ), + ) + process_task = asyncio.create_task( + service._process_http_bridge_upstream_text( + session, + json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp-shared-grouped-terminal' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) + + await asyncio.wait_for(all_appends_started.wait(), timeout=1.0) + assert append_calls == ["op-grouped-terminal-0", "op-grouped-terminal-1"] + assert all(event_queue.empty() for event_queue in queues) + release_first_append.set() + await asyncio.wait_for(settlement_started.wait(), timeout=1.0) + for event_queue in queues: + terminal_event = await asyncio.wait_for(event_queue.get(), timeout=1.0) + assert terminal_event is not None + assert '"type":"response.failed"' in terminal_event + assert await asyncio.wait_for(event_queue.get(), timeout=1.0) is None + assert event_queue.empty() + assert process_task.done() is False + if cancel_during_settlement: + process_task.cancel() + assert process_task.cancelling() + release_settlement.set() + expected_error = asyncio.CancelledError if cancel_during_settlement else RuntimeError + with pytest.raises(expected_error): + await asyncio.wait_for(process_task, timeout=1.0) + assert append_calls == ["op-grouped-terminal-0", "op-grouped-terminal-1"] + assert finalize_request.await_count == 2 + + @pytest.mark.asyncio async def test_ordinary_completed_alias_rejection_preserves_successful_response( monkeypatch: pytest.MonkeyPatch, From ff89bc4302f636a86f8930133748a945a9a681d6 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Sun, 16 Aug 2026 16:47:48 +0900 Subject: [PATCH 037/117] chore(github): retire codex review label gate in favor of CodeRabbit (#1763) * chore(github): retire codex review label gate in favor of CodeRabbit * chore(openspec): move removal rationale out of normative delta spec --- .github/CONTRIBUTING.md | 23 +- .github/scripts/sync_codex_ok_labels.py | 1910 --------------- .github/workflows/ci.yml | 16 +- .github/workflows/codex-review-labels.yml | 124 - .github/workflows/simplicity-budgets.yml | 6 +- AGENTS.md | 16 +- .../proposal.md | 24 - .../specs/github-automation/spec.md | 25 - .../label-sync-rate-limit-fallback/tasks.md | 10 - .../.openspec.yaml | 2 + .../remove-codex-review-label-gate/design.md | 29 + .../proposal.md | 27 + .../specs/github-automation/spec.md | 348 +++ .../remove-codex-review-label-gate/tasks.md | 15 + tests/unit/test_sync_codex_ok_labels.py | 2106 ----------------- 15 files changed, 446 insertions(+), 4235 deletions(-) delete mode 100755 .github/scripts/sync_codex_ok_labels.py delete mode 100644 .github/workflows/codex-review-labels.yml delete mode 100644 openspec/changes/label-sync-rate-limit-fallback/proposal.md delete mode 100644 openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md delete mode 100644 openspec/changes/label-sync-rate-limit-fallback/tasks.md create mode 100644 openspec/changes/remove-codex-review-label-gate/.openspec.yaml create mode 100644 openspec/changes/remove-codex-review-label-gate/design.md create mode 100644 openspec/changes/remove-codex-review-label-gate/proposal.md create mode 100644 openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md create mode 100644 openspec/changes/remove-codex-review-label-gate/tasks.md delete mode 100644 tests/unit/test_sync_codex_ok_labels.py diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 5b126ab8af..af3f057368 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -193,7 +193,7 @@ PR titles must follow the same format — that's the title release-please reads. 2. Make atomic commits with Conventional Commit titles. 3. Run the lint/test gate locally (see above). 4. Open a PR using the template. Link the relevant issue. -5. Codex Review (and a human maintainer) will review. Address feedback by +5. CodeRabbit (and a human maintainer) will review. Address feedback by pushing follow-up commits — no force-pushing during active review. 6. Once approved and CI is green, a maintainer squash-merges with a clean Conventional Commits title. @@ -215,16 +215,11 @@ Before a PR is squash-merged into `main`: `CI Required` check is the branch-protection check to require: it depends on every CI job and also runs for merge queue synthetic merge groups, so a stale PR head cannot bypass a broken merge result. -2. **`@codex review` must be clean — or its findings addressed — on the - merge-target head.** Every PR triggers `@codex review` at least once - against the head that's about to be merged. Local `codex review - --base origin/main` runs are encouraged but don't substitute for the - cloud review (the cloud `@codex review` reliably catches things the - local run misses). - The `🤖 codex: ok` label is maintained by the trusted - `Codex review labels` workflow from current-head CI and current-head - Codex review evidence. Treat the label as an audit aid, not as a - substitute for branch protection or merge queue checks. +2. **Actionable CodeRabbit findings must be fixed or explicitly addressed + or dismissed in-thread on the merge-target head.** Review the current-head + CodeRabbit findings before merging; no finding may be silently skipped. + Local `codex review --base origin/main` runs remain an encouraged extra + tool, but they are not a merge gate and do not substitute for CodeRabbit. - **P1 findings**: fix in the PR, or justify in-thread with a short write-up of why the finding doesn't apply. No silent skipping. - **P2 findings**: fix in the PR, or open a follow-up issue and link @@ -297,7 +292,7 @@ self-merge escape hatch applies: - If a collaborator's PR has been waiting on a maintainer merge for **more than 14 days** with **all merge gates met** (CI green, - `@codex review` clean or findings addressed, `mergeable=CLEAN`, no + CodeRabbit findings addressed, `mergeable=CLEAN`, no outstanding requested-changes review, no objection from any other active collaborator in the thread), the PR author may self-merge. - Self-merge under this clause **must** include a comment on the PR @@ -311,8 +306,8 @@ self-merge escape hatch applies: These rules are intentionally lightweight. They don't require: -- A second human reviewer in addition to `@codex review` for every PR. - Codex review + the PR author + a maintainer merge is the baseline. +- A second human reviewer in addition to CodeRabbit for every PR. + CodeRabbit review + the PR author + a maintainer merge is the baseline. - Squash-merge commit message rewriting beyond the Conventional Commits title. The PR description ends up in the body; that's enough. - A formal escalation process for disagreements. If a P1 finding is diff --git a/.github/scripts/sync_codex_ok_labels.py b/.github/scripts/sync_codex_ok_labels.py deleted file mode 100755 index 17b3c1a275..0000000000 --- a/.github/scripts/sync_codex_ok_labels.py +++ /dev/null @@ -1,1910 +0,0 @@ -#!/usr/bin/env python3 -"""Synchronize GitHub Codex review labels from current-head Codex reviews.""" - -from __future__ import annotations - -import argparse -import json -import os -import re -import subprocess -import sys -import time -from collections.abc import Callable -from dataclasses import dataclass -from datetime import UTC, datetime, timedelta -from typing import Any -from urllib.parse import quote - -CODEX_OK_LABEL = "🤖 codex: ok" -CODEX_NEEDS_WORK_LABEL = "🤖 codex: needs work" -NEEDS_REBASE_LABEL = "needs rebase" -LEGACY_CODEX_LABELS = {"🤖 codex-ok"} -CODEX_REVIEW_AUTHORS = { - "chatgpt-codex-connector", - "chatgpt-codex-connector[bot]", - "openai-codex", - "openai-codex[bot]", -} -CODEX_CLEAN_RE = re.compile( - r"(didn['’]t find any major issues|no major issues found|no major issues)", - re.IGNORECASE, -) -CODEX_FINDING_RE = re.compile(r"(?:\bP[0-3]\s+Badge\b|badge/P[0-3]-|(?m:(?:^|\n)\s*(?:\*\*)?(?:\[P[0-3]\]|P[0-3]\b)))") -# Anchored to the real quota envelope ("You have reached your Codex usage limits -# for code reviews. ...") so ordinary reviews that merely discuss usage limits do -# not latch the backoff. -CODEX_USAGE_LIMIT_RE = re.compile( - r"^\s*You(?: have|['’]ve) reached your Codex usage limits", - re.IGNORECASE, -) -CLEAN_REACTION_CONTENTS = frozenset({"THUMBS_UP", "+1"}) -DEFAULT_CODEX_USAGE_LIMIT_BACKOFF_HOURS = 24.0 -DEFAULT_CODEX_REVIEW_RESPONSE_WAIT_SECONDS = 10.0 -SUCCESS_CHECK_STATES = {"SUCCESS", "NEUTRAL", "SKIPPED"} -FAIL_CHECK_STATES = {"ACTION_REQUIRED", "CANCELLED", "ERROR", "FAILURE", "STALE", "TIMED_OUT"} -PENDING_CHECK_STATES = {"EXPECTED", "IN_PROGRESS", "PENDING", "QUEUED", "REQUESTED", "WAITING"} -UNMERGEABLE_STATES = {"DIRTY", "BLOCKED"} -NEEDS_REBASE_STATES = {"CONFLICTING", "DIRTY"} -NO_REBASE_STATES = {"BEHIND", "BLOCKED", "CLEAN", "DRAFT", "HAS_HOOKS", "UNSTABLE"} -CODEX_LB_REQUIRED_CHECKS = frozenset( - { - "Frontend lint (eslint)", - "Frontend type check (tsc)", - "Frontend tests (vitest + coverage)", - "Frontend build (vite)", - "Lint (ruff)", - "Type check (ty)", - "Tests (pytest, unit)", - "Tests (pytest, integration-core)", - "Tests (pytest, integration-bridge)", - "Tests (pytest, e2e)", - "Tests (pytest, PostgreSQL)", - "Migration check (alembic)", - "Migration check (alembic, PostgreSQL)", - "Package (build)", - "Docker build", - "Helm lint + template + kubeconform", - "Helm smoke install (kind)", - "CI Required", - } -) -REQUIRED_CHECKS_BY_REPO = { - "Soju06/codex-lb": CODEX_LB_REQUIRED_CHECKS, -} -PR_TIMELINE_QUERY = """ -query($owner: String!, $name: String!, $number: Int!, $before: String) { - repository(owner: $owner, name: $name) { - pullRequest(number: $number) { - headRefOid - commits(last: 1) { - nodes { - commit { - oid - } - } - } - timelineItems( - last: 100 - before: $before - itemTypes: [ - PULL_REQUEST_COMMIT - ISSUE_COMMENT - PULL_REQUEST_REVIEW - HEAD_REF_FORCE_PUSHED_EVENT - ] - ) { - pageInfo { - hasPreviousPage - startCursor - } - nodes { - __typename - ... on PullRequestCommit { - commit { - oid - } - } - ... on HeadRefForcePushedEvent { - afterCommit { - oid - } - } - ... on IssueComment { - author { - login - } - bodyText - createdAt - url - reactions(first: 100) { - nodes { - content - createdAt - user { - login - } - } - } - } - ... on PullRequestReview { - databaseId - author { - login - } - bodyText - submittedAt - url - commit { - oid - } - } - } - } - } - } -} -""" - -PR_REVIEW_THREADS_QUERY = """ -query($owner: String!, $name: String!, $number: Int!, $after: String) { - repository(owner: $owner, name: $name) { - pullRequest(number: $number) { - reviewThreads(first: 100, after: $after) { - pageInfo { - hasNextPage - endCursor - } - nodes { - isResolved - isOutdated - comments(first: 20) { - nodes { - author { - login - } - body - url - commit { - oid - } - originalCommit { - oid - } - } - } - } - } - } - } -} -""" - - -class GhError(RuntimeError): - """A GitHub CLI call failed.""" - - -_RATE_LIMIT_MARKER = "API rate limit exceeded" -_TRANSIENT_GH_MARKERS = ("HTTP 500", "HTTP 502", "HTTP 503", "HTTP 504", "Unicorn!") -_GH_RETRY_ATTEMPTS = 5 -_GH_RETRY_BASE_DELAY_SECONDS = 2.0 -_fallback_token_active = False - - -def _activate_fallback_token() -> bool: - """Switch gh calls to GH_FALLBACK_TOKEN once after a rate-limit failure. - - The primary token is typically a user PAT whose quota is shared with - other consumers; github.token carries a separate per-repository quota, - so a single runtime switch keeps the sync alive through PAT exhaustion. - """ - global _fallback_token_active - if _fallback_token_active: - return False - fallback = os.environ.get("GH_FALLBACK_TOKEN", "").strip() - if not fallback or fallback == os.environ.get("GH_TOKEN", ""): - return False - os.environ["GH_TOKEN"] = fallback - _fallback_token_active = True - return True - - -def _gh_args_safe_to_retry(args: list[str]) -> bool: - """Return whether a failed gh invocation is safe to re-run automatically.""" - - if not args or args[0] != "api": - return False - if "graphql" in args: - return True - method = "GET" - for index, arg in enumerate(args): - if arg == "--method" and index + 1 < len(args): - method = args[index + 1].upper() - break - return method == "GET" - - -def _is_retryable_gh_failure(detail: str) -> bool: - return any(marker in detail for marker in _TRANSIENT_GH_MARKERS) - - -def _gh_retry_delay(attempt_index: int) -> float: - return min(_GH_RETRY_BASE_DELAY_SECONDS * (2**attempt_index), 30.0) - - -@dataclass(frozen=True) -class SyncDecision: - repo: str - number: int - head_sha: str - has_ok_label: bool - wants_ok_label: bool - ok_action: str - has_needs_work_label: bool - wants_needs_work_label: bool - needs_work_action: str - has_needs_rebase_label: bool - wants_needs_rebase_label: bool - needs_rebase_action: str - legacy_labels: frozenset[str] - reason: str - review_url: str | None - review_state: str - checks_state: str - merge_state: str - trigger_codex_review: bool - approve_workflow_run_ids: tuple[int, ...] - - -def run_gh( - args: list[str], - *, - input_json: Any | None = None, - timeout_seconds: int = 30, - fallback_retry: bool = True, -) -> Any: - command = ["gh", *args] - input_text = json.dumps(input_json) if input_json is not None else None - safe_to_retry = _gh_args_safe_to_retry(args) - attempts = _GH_RETRY_ATTEMPTS if safe_to_retry else 1 - for attempt_index in range(attempts): - try: - proc = subprocess.run( - command, - check=False, - input=input_text, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - text=True, - timeout=timeout_seconds, - ) - except subprocess.TimeoutExpired as exc: - raise GhError(f"{' '.join(command)}: timed out after {timeout_seconds}s") from exc - - if proc.returncode == 0: - text = proc.stdout.strip() - if not text: - return None - return json.loads(text) - - detail = proc.stderr.strip() or proc.stdout.strip() - if _RATE_LIMIT_MARKER in detail and _activate_fallback_token(): - if not fallback_retry: - # Identity-sensitive commands (e.g. the @codex review comment) - # must not silently retry under the fallback token's identity; - # later calls still benefit from the activated fallback. - raise GhError( - f"{' '.join(command)}: rate-limited; switched to GH_FALLBACK_TOKEN " - f"without retrying this identity-sensitive command ({detail})" - ) - print( - "warning: active token rate-limited; retrying with GH_FALLBACK_TOKEN", - file=sys.stderr, - ) - return run_gh(args, input_json=input_json, timeout_seconds=timeout_seconds) - if safe_to_retry and _is_retryable_gh_failure(detail) and attempt_index + 1 < attempts: - delay = _gh_retry_delay(attempt_index) - print( - f"warning: {' '.join(command)} failed transiently ({detail}); retrying in {delay:g}s", - file=sys.stderr, - ) - time.sleep(delay) - continue - raise GhError(f"{' '.join(command)}: {detail}") - - raise GhError(f"{' '.join(command)}: failed after retries") - - -def gh_api(path: str, *, method: str = "GET", input_json: Any | None = None) -> Any: - if not path.startswith("/"): - path = f"/{path}" - - args = ["api", "--method", method, path] - if input_json is not None: - args.append("--input") - args.append("-") - return run_gh(args, input_json=input_json) - - -def graphql(query: str, **fields: object) -> dict[str, Any]: - args = ["api", "graphql", "-f", f"query={query}"] - for key, value in fields.items(): - args.extend(["-F", f"{key}={value}"]) - payload = run_gh(args) - if not isinstance(payload, dict): - raise GhError("gh api graphql returned a non-object payload") - return payload - - -def paged_api(path: str) -> list[dict[str, Any]]: - page = 1 - items: list[dict[str, Any]] = [] - sep = "&" if "?" in path else "?" - while True: - payload = gh_api(f"{path}{sep}per_page=100&page={page}") - if not payload: - return items - if isinstance(payload, dict): - page_items = None - for key in ("items", "check_runs", "workflow_runs"): - if key in payload: - page_items = payload[key] - break - else: - page_items = payload - if not isinstance(page_items, list): - raise GhError(f"{path}: expected list payload, got {type(payload).__name__}") - items.extend(item for item in page_items if isinstance(item, dict)) - if len(page_items) < 100: - return items - page += 1 - - -def repo_path(repo: str) -> str: - parts = repo.strip().split("/") - if len(parts) != 2 or not all(parts): - raise ValueError(f"repo must be owner/name, got {repo!r}") - return f"{parts[0]}/{parts[1]}" - - -def list_open_pr_numbers(repo: str) -> list[int]: - pulls = paged_api(f"/repos/{repo}/pulls?state=open") - return [int(pr["number"]) for pr in pulls if isinstance(pr.get("number"), int)] - - -def issue_label_names(repo: str, number: int) -> set[str]: - labels = paged_api(f"/repos/{repo}/issues/{number}/labels") - return {str(label.get("name")) for label in labels if isinstance(label.get("name"), str)} - - -def author_login(item: dict[str, Any]) -> str: - user = item.get("user") - login = user.get("login") if isinstance(user, dict) else None - return str(login or "") - - -def is_clean_codex_body(body: object) -> bool: - return isinstance(body, str) and CODEX_CLEAN_RE.search(body) is not None - - -def is_needs_work_codex_body(body: object) -> bool: - return isinstance(body, str) and CODEX_FINDING_RE.search(body) is not None - - -def is_codex_usage_limit_body(body: object) -> bool: - return isinstance(body, str) and CODEX_USAGE_LIMIT_RE.search(body) is not None - - -def body_mentions_head(body: object, head_sha: str) -> bool: - if not isinstance(body, str): - return False - return head_sha in body or head_sha[:12] in body or head_sha[:8] in body - - -def review_node_commit_oid(node: dict[str, Any]) -> str | None: - commit = node.get("commit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - - -def review_node_database_id(node: dict[str, Any]) -> int | None: - value = node.get("databaseId") - return value if isinstance(value, int) else None - - -def node_body(node: dict[str, Any]) -> str: - body = node.get("bodyText") - if not isinstance(body, str): - body = node.get("body") - return body if isinstance(body, str) else "" - - -def node_url(node: dict[str, Any]) -> str | None: - url = node.get("url") or node.get("html_url") - return str(url) if isinstance(url, str) else None - - -def node_author_login(node: dict[str, Any]) -> str: - author = node.get("author") - login = author.get("login") if isinstance(author, dict) else None - if isinstance(login, str): - return login - return author_login(node) - - -def is_timeline_codex_author(node: dict[str, Any], allowed: set[str]) -> bool: - return node_author_login(node) in allowed - - -def is_codex_review_request_comment(node: dict[str, Any]) -> bool: - if node.get("__typename") != "IssueComment": - return False - return node_body(node).strip().casefold() == "@codex review" - - -def reaction_user_login(node: dict[str, Any]) -> str: - user = node.get("user") - login = user.get("login") if isinstance(user, dict) else None - return str(login or "") - - -def codex_request_reaction_state(node: dict[str, Any], *, allowed_authors: set[str]) -> str: - if not is_codex_review_request_comment(node): - return "none" - - reactions = node.get("reactions") - reaction_nodes = reactions.get("nodes") if isinstance(reactions, dict) else [] - if not isinstance(reaction_nodes, list): - return "none" - - state = "none" - for reaction in reaction_nodes: - if not isinstance(reaction, dict): - continue - if reaction_user_login(reaction) not in allowed_authors: - continue - content = str(reaction.get("content") or "").upper() - if content in CLEAN_REACTION_CONTENTS: - state = "clean" - elif content == "EYES" and state == "none": - state = "pending" - return state - - -def timeline_head_oid(node: dict[str, Any]) -> str | None: - if node.get("__typename") == "PullRequestCommit": - commit = node.get("commit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - if node.get("__typename") == "HeadRefForcePushedEvent": - commit = node.get("afterCommit") - oid = commit.get("oid") if isinstance(commit, dict) else None - return oid if isinstance(oid, str) else None - return None - - -def timeline_node_timestamp(node: dict[str, Any]) -> str | None: - for key in ("createdAt", "submittedAt", "committedDate"): - value = node.get(key) - if isinstance(value, str) and value: - return value - return None - - -def parse_github_timestamp(value: object) -> datetime | None: - if not isinstance(value, str) or not value: - return None - try: - return datetime.fromisoformat(value.replace("Z", "+00:00")) - except ValueError: - return None - - -def current_viewer_login() -> str: - payload = gh_api("/user") - login = payload.get("login") if isinstance(payload, dict) else None - if not isinstance(login, str) or not login: - raise GhError("gh api /user did not return a login") - return login - - -def resolve_codex_request_sender() -> str | None: - """Resolve the login GitHub records as the author of our `@codex review` comments. - - GitHub App installation tokens cannot call ``GET /user``, so prefer the app - slug the workflow exports (installation comments are authored as - ``[bot]``) and fall back to ``GET /user`` for PAT-backed runs. - The slug describes the primary token only, so it is ignored once the run - has switched to ``GH_FALLBACK_TOKEN``. Returns ``None`` when no path can - resolve a login. - """ - - slug = os.environ.get("GH_APP_SLUG", "").strip() - if slug and not _fallback_token_active: - return slug if slug.endswith("[bot]") else f"{slug}[bot]" - try: - return current_viewer_login() - except GhError as exc: - print(f"warning: cannot resolve @codex review sender via GET /user: {exc}", file=sys.stderr, flush=True) - return None - - -def is_normal_codex_response_node(node: dict[str, Any]) -> bool: - body = node_body(node) - if is_codex_usage_limit_body(body): - return False - return node.get("__typename") in {"IssueComment", "PullRequestReview", "PullRequestReviewComment"} and bool( - body.strip() - ) - - -@dataclass -class CodexReviewUsageBackoff: - request_author: str - allowed_authors: set[str] - window: timedelta - now: datetime - latest_usage_limit_at: datetime | None = None - latest_usage_limit_url: str | None = None - latest_normal_response_at: datetime | None = None - - def observe(self, timeline_nodes: list[dict[str, Any]]) -> None: - active_request_author: str | None = None - for node in timeline_nodes: - timestamp = parse_github_timestamp(timeline_node_timestamp(node)) - if is_codex_review_request_comment(node): - active_request_author = node_author_login(node) - if active_request_author == self.request_author: - self._observe_clean_request_reactions(node) - continue - if timestamp is None or timestamp < self.now - self.window: - continue - if active_request_author != self.request_author: - continue - if not is_timeline_codex_author(node, self.allowed_authors): - continue - if is_codex_usage_limit_body(node_body(node)): - if self.latest_usage_limit_at is None or timestamp > self.latest_usage_limit_at: - self.latest_usage_limit_at = timestamp - self.latest_usage_limit_url = node_url(node) - elif is_normal_codex_response_node(node): - if self.latest_normal_response_at is None or timestamp > self.latest_normal_response_at: - self.latest_normal_response_at = timestamp - - def _observe_clean_request_reactions(self, node: dict[str, Any]) -> None: - """Count clean THUMBS_UP reactions on the sender's requests as normal responses.""" - - reactions = node.get("reactions") - reaction_nodes = reactions.get("nodes") if isinstance(reactions, dict) else None - if not isinstance(reaction_nodes, list): - return - for reaction in reaction_nodes: - if not isinstance(reaction, dict): - continue - if reaction_user_login(reaction) not in self.allowed_authors: - continue - if str(reaction.get("content") or "").upper() not in CLEAN_REACTION_CONTENTS: - continue - timestamp = parse_github_timestamp(reaction.get("createdAt")) - if timestamp is None or timestamp < self.now - self.window: - continue - if self.latest_normal_response_at is None or timestamp > self.latest_normal_response_at: - self.latest_normal_response_at = timestamp - - def is_limited(self) -> bool: - if self.latest_usage_limit_at is None: - return False - return self.latest_normal_response_at is None or self.latest_normal_response_at < self.latest_usage_limit_at - - def skip_warning(self, decision: SyncDecision) -> str: - when = self.latest_usage_limit_at.isoformat() if self.latest_usage_limit_at else "unknown time" - suffix = f" ({self.latest_usage_limit_url})" if self.latest_usage_limit_url else "" - return ( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - f"{self.request_author} has a recent Codex usage-limit reply at {when}{suffix}" - ) - - -def recent_issue_comment_timelines(repo: str, *, since: datetime) -> list[list[dict[str, Any]]]: - """Return repo-wide recent issue comments grouped per issue as timeline nodes. - - Sender quota evidence can live on pull requests outside the current - selection (single --pr runs, closed PRs), so the backoff also observes - every issue comment in the window, grouped per issue to keep request -> - reply attribution intact. - """ - - since_text = since.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") - comments = paged_api(f"/repos/{repo}/issues/comments?since={quote(since_text, safe='')}") - nodes_by_issue: dict[str, list[dict[str, Any]]] = {} - for comment in comments: - body = comment.get("body") - if not isinstance(body, str): - continue - issue_url = str(comment.get("issue_url") or "") - nodes_by_issue.setdefault(issue_url, []).append( - { - "__typename": "IssueComment", - "author": {"login": author_login(comment)}, - "bodyText": body, - "createdAt": comment.get("created_at"), - "url": comment.get("html_url"), - } - ) - return [sorted(nodes, key=lambda node: str(node.get("createdAt") or "")) for nodes in nodes_by_issue.values()] - - -def backoff_timeline_observer(backoff: CodexReviewUsageBackoff) -> Callable[[int, list[dict[str, Any]]], None]: - def observe(_number: int, timeline_nodes: list[dict[str, Any]]) -> None: - backoff.observe(timeline_nodes) - - return observe - - -def unresolved_review_comment_urls(repo: str, number: int) -> set[str]: - owner, name = repo.split("/", 1) - after: str | None = None - urls: set[str] = set() - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if after is not None: - fields["after"] = after - payload = graphql(PR_REVIEW_THREADS_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - threads = pr.get("reviewThreads") - if not isinstance(threads, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return review threads") - nodes = threads.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return review thread nodes") - - for thread in nodes: - if not isinstance(thread, dict): - continue - if thread.get("isResolved") or thread.get("isOutdated"): - continue - comments = thread.get("comments") - comment_nodes = comments.get("nodes") if isinstance(comments, dict) else [] - if not isinstance(comment_nodes, list): - continue - for comment in comment_nodes: - if not isinstance(comment, dict): - continue - url = comment.get("url") - if isinstance(url, str): - urls.add(url) - - page_info = threads.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasNextPage"): - break - end_cursor = page_info.get("endCursor") - if not isinstance(end_cursor, str) or not end_cursor: - break - after = end_cursor - - return urls - - -def pull_review_comment_nodes(repo: str, number: int, *, head_sha: str) -> list[dict[str, Any]]: - comments = paged_api(f"/repos/{repo}/pulls/{number}/comments") - unresolved_urls = unresolved_review_comment_urls(repo, number) - nodes: list[dict[str, Any]] = [] - for comment in comments: - body = comment.get("body") - if not isinstance(body, str): - continue - commit_id = comment.get("commit_id") - original_commit_id = comment.get("original_commit_id") - review_id = comment.get("pull_request_review_id") - commit_matches_head = commit_id == head_sha - original_matches_head = original_commit_id == head_sha - body_mentions_current_head = body_mentions_head(body, head_sha) - if not commit_matches_head and not original_matches_head and not body_mentions_current_head: - continue - effective_commit_id = original_commit_id if original_matches_head else commit_id - effective_review_id = review_id if original_matches_head else None - html_url = comment.get("html_url") or comment.get("url") - if is_needs_work_codex_body(body) and html_url not in unresolved_urls: - continue - user = comment.get("user") - login = user.get("login") if isinstance(user, dict) else None - nodes.append( - { - "__typename": "PullRequestReviewComment", - "author": {"login": login} if isinstance(login, str) else None, - "bodyText": body, - "createdAt": comment.get("created_at"), - "url": html_url, - "commit": {"oid": effective_commit_id} if isinstance(effective_commit_id, str) else None, - "pullRequestReviewDatabaseId": effective_review_id if isinstance(effective_review_id, int) else None, - } - ) - return nodes - - -def merge_review_comment_nodes( - timeline_nodes: list[dict[str, Any]], - comment_nodes: list[dict[str, Any]], -) -> list[dict[str, Any]]: - if not comment_nodes: - return timeline_nodes - - comments_by_review_id: dict[int, list[dict[str, Any]]] = {} - unplaced: list[dict[str, Any]] = [] - for node in comment_nodes: - review_id = node.get("pullRequestReviewDatabaseId") - if isinstance(review_id, int): - comments_by_review_id.setdefault(review_id, []).append(node) - else: - unplaced.append(node) - - merged: list[dict[str, Any]] = [] - placed_ids: set[int] = set() - for node in timeline_nodes: - merged.append(node) - if node.get("__typename") != "PullRequestReview": - continue - review_id = review_node_database_id(node) - if review_id is None: - continue - for comment in comments_by_review_id.get(review_id, []): - merged.append(comment) - placed_ids.add(id(comment)) - - for node in comment_nodes: - if id(node) not in placed_ids and node not in unplaced: - unplaced.append(node) - for node in unplaced: - timestamp = timeline_node_timestamp(node) - if timestamp is None: - merged.append(node) - continue - insert_at = len(merged) - for index, candidate in enumerate(merged): - candidate_timestamp = timeline_node_timestamp(candidate) - if candidate_timestamp is not None and candidate_timestamp > timestamp: - insert_at = index - break - merged.insert(insert_at, node) - return merged - - -def find_current_head_codex_review_state( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> tuple[str, dict[str, Any] | None]: - head_index = None - for index, node in enumerate(timeline_nodes): - if timeline_head_oid(node) == head_sha: - head_index = index - - if head_index is None: - return "none", None - - latest_state = "none" - latest_node: dict[str, Any] | None = None - for node in timeline_nodes[head_index + 1 :]: - reaction_state = codex_request_reaction_state(node, allowed_authors=allowed_authors) - if reaction_state == "clean": - latest_state = "clean" - latest_node = node - continue - if reaction_state == "pending" and latest_state == "none": - latest_state = "pending" - latest_node = node - continue - - if not is_timeline_codex_author(node, allowed_authors): - continue - - if node.get("__typename") == "PullRequestReview": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - if is_needs_work_codex_body(body): - latest_state = "needs_work" - latest_node = node - continue - latest_state = "clean" - latest_node = node - continue - - if node.get("__typename") == "PullRequestReviewComment": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - if is_needs_work_codex_body(body): - latest_state = "needs_work" - latest_node = node - continue - - if node.get("__typename") == "IssueComment" and is_clean_codex_body(node_body(node)): - latest_state = "clean" - latest_node = node - - return latest_state, latest_node - - -def find_current_head_clean_review( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> dict[str, Any] | None: - state, node = find_current_head_codex_review_state( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - return node if state == "clean" else None - - -def has_codex_news_after_current_head( - timeline_nodes: list[dict[str, Any]], - *, - head_sha: str, - allowed_authors: set[str], -) -> bool: - head_index = None - for index, node in enumerate(timeline_nodes): - if timeline_head_oid(node) == head_sha: - head_index = index - - if head_index is None: - return False - - for node in timeline_nodes[head_index + 1 :]: - if is_codex_review_request_comment(node): - return True - if not is_timeline_codex_author(node, allowed_authors): - continue - if node.get("__typename") == "PullRequestReview": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - return True - if node.get("__typename") == "PullRequestReviewComment": - body = node_body(node) - commit_oid = review_node_commit_oid(node) - if commit_oid != head_sha and not body_mentions_head(body, head_sha): - continue - return True - if node.get("__typename") == "IssueComment": - return True - - return False - - -def pr_timeline_evidence(repo: str, number: int) -> tuple[str, list[dict[str, Any]]]: - owner, name = repo.split("/", 1) - before: str | None = None - head_sha: str | None = None - timeline_nodes: list[dict[str, Any]] = [] - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if before is not None: - fields["before"] = before - payload = graphql(PR_TIMELINE_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - page_head_sha = pr.get("headRefOid") - commit_nodes = pr.get("commits", {}).get("nodes", []) - last_commit = commit_nodes[-1].get("commit", {}) if commit_nodes else {} - commit_sha = last_commit.get("oid") - if not isinstance(page_head_sha, str) or not page_head_sha: - raise GhError(f"{repo}#{number}: GraphQL did not return headRefOid") - if commit_sha != page_head_sha: - raise GhError(f"{repo}#{number}: headRefOid {page_head_sha} disagrees with commits.last {commit_sha}") - if head_sha is None: - head_sha = page_head_sha - elif head_sha != page_head_sha: - raise GhError(f"{repo}#{number}: headRefOid changed while paging timeline") - - timeline = pr.get("timelineItems") - if not isinstance(timeline, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return timeline items") - nodes = timeline.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return timeline nodes") - page_nodes = [node for node in nodes if isinstance(node, dict)] - timeline_nodes = page_nodes + timeline_nodes - if any(timeline_head_oid(node) == head_sha for node in page_nodes): - break - - page_info = timeline.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasPreviousPage"): - break - start_cursor = page_info.get("startCursor") - if not isinstance(start_cursor, str) or not start_cursor: - break - before = start_cursor - - if head_sha is None: - raise GhError(f"{repo}#{number}: GraphQL did not return headRefOid") - return head_sha, merge_review_comment_nodes( - timeline_nodes, - pull_review_comment_nodes(repo, number, head_sha=head_sha), - ) - - -def classify_check_state( - check_runs: list[dict[str, Any]], - combined_status: dict[str, Any], - *, - required_check_names: frozenset[str] = frozenset(), -) -> str: - states: list[str] = [] - seen_check_names: set[str] = set() - named_check_runs: dict[str, dict[str, Any]] = {} - unnamed_check_runs: list[dict[str, Any]] = [] - - authoritative_ci_workflow = authoritative_ci_workflow_id(check_runs) - authoritative_ci_run = authoritative_ci_workflow_run_id( - check_runs, - workflow_id=authoritative_ci_workflow, - ) - if authoritative_ci_run is not None and authoritative_ci_workflow is not None: - check_runs = [ - item - for item in check_runs - if github_actions_workflow_id(item) != authoritative_ci_workflow - or github_actions_workflow_run_id(item) in {None, authoritative_ci_run} - ] - - for item in check_runs: - name = item.get("name") - if isinstance(name, str): - previous = named_check_runs.get(name) - if previous is None or check_run_recency_key(item) >= check_run_recency_key(previous): - named_check_runs[name] = item - else: - unnamed_check_runs.append(item) - - for item in [*named_check_runs.values(), *unnamed_check_runs]: - name = item.get("name") - if isinstance(name, str): - seen_check_names.add(name) - conclusion = str(item.get("conclusion") or "").upper() - status = str(item.get("status") or "").upper() - states.append(conclusion or status or "UNKNOWN") - - for item in combined_status.get("statuses", []) if isinstance(combined_status, dict) else []: - if isinstance(item, dict): - states.append(str(item.get("state") or "").upper()) - - if not states: - return "none" - if any(state in FAIL_CHECK_STATES for state in states): - return "failure" - if any(state in PENDING_CHECK_STATES for state in states): - return "pending" - if required_check_names and not required_check_names <= seen_check_names: - return "pending" - if all(state in SUCCESS_CHECK_STATES for state in states): - return "success" - return "unknown" - - -def check_run_recency_key(item: dict[str, Any]) -> tuple[str, str]: - return ( - str( - item.get("started_at") - or item.get("startedAt") - or item.get("created_at") - or item.get("createdAt") - or item.get("completed_at") - or item.get("completedAt") - or "" - ), - str(item.get("completed_at") or item.get("completedAt") or ""), - ) - - -def github_actions_workflow_run_id(item: dict[str, Any]) -> str | None: - details_url = item.get("details_url") or item.get("detailsUrl") - if not isinstance(details_url, str): - return None - match = re.search(r"/actions/runs/(\d+)(?:/|$)", details_url) - return match.group(1) if match is not None else None - - -def authoritative_ci_workflow_run_id( - check_runs: list[dict[str, Any]], - *, - workflow_id: str | None, -) -> str | None: - if workflow_id is None: - return None - workflow_runs = [ - item - for item in check_runs - if github_actions_workflow_id(item) == workflow_id and github_actions_workflow_run_id(item) is not None - ] - if not workflow_runs: - return None - latest_run = max(workflow_runs, key=github_actions_workflow_run_recency_key) - return github_actions_workflow_run_id(latest_run) - - -def github_actions_workflow_run_recency_key(item: dict[str, Any]) -> tuple[str, tuple[str, str], int]: - run_id = github_actions_workflow_run_id(item) - return ( - str(item.get("_github_actions_run_started_at") or item.get("_github_actions_run_created_at") or ""), - check_run_recency_key(item), - int(run_id) if isinstance(run_id, str) and run_id.isdigit() else 0, - ) - - -def github_actions_workflow_id(item: dict[str, Any]) -> str | None: - workflow_id = item.get("_github_actions_workflow_id") - return str(workflow_id) if isinstance(workflow_id, (int, str)) else None - - -def authoritative_ci_workflow_id(check_runs: list[dict[str, Any]]) -> str | None: - required_runs = [item for item in check_runs if item.get("name") == "CI Required"] - if not required_runs: - return None - latest_required = max(required_runs, key=check_run_recency_key) - return github_actions_workflow_id(latest_required) - - -def annotate_github_actions_workflow_ids(repo: str, check_runs: list[dict[str, Any]]) -> list[dict[str, Any]]: - workflow_metadata_by_run: dict[str, tuple[str, str | None]] = {} - for run_id in {github_actions_workflow_run_id(item) for item in check_runs} - {None}: - assert run_id is not None - try: - workflow_run = gh_api(f"/repos/{repo}/actions/runs/{run_id}") - except GhError: - continue - workflow_id = workflow_run.get("workflow_id") if isinstance(workflow_run, dict) else None - if isinstance(workflow_id, (int, str)): - # GitHub preserves ``created_at`` when an existing run id is rerun, - # while ``run_started_at`` advances to the current attempt. - run_started_at = workflow_run.get("run_started_at") or workflow_run.get("created_at") - workflow_metadata_by_run[run_id] = ( - str(workflow_id), - str(run_started_at) if isinstance(run_started_at, str) else None, - ) - - annotated: list[dict[str, Any]] = [] - for item in check_runs: - run_id = github_actions_workflow_run_id(item) - metadata = workflow_metadata_by_run.get(run_id) if run_id is not None else None - if metadata is None: - annotated.append(item) - continue - workflow_id, run_started_at = metadata - annotated_item = {**item, "_github_actions_workflow_id": workflow_id} - if run_started_at is not None: - annotated_item["_github_actions_run_started_at"] = run_started_at - annotated.append(annotated_item) - return annotated - - -def commit_checks_state(repo: str, head_sha: str) -> str: - check_runs = paged_api(f"/repos/{repo}/commits/{head_sha}/check-runs") - check_runs = annotate_github_actions_workflow_ids(repo, check_runs) - combined_status = gh_api(f"/repos/{repo}/commits/{head_sha}/status") - return classify_check_state( - check_runs, - combined_status if isinstance(combined_status, dict) else {}, - required_check_names=REQUIRED_CHECKS_BY_REPO.get(repo, frozenset()), - ) - - -def decision_requires_writes(decision: SyncDecision) -> bool: - """Return whether applying this decision would mutate GitHub state.""" - - return ( - decision.ok_action != "keep" - or decision.needs_work_action != "keep" - or decision.needs_rebase_action != "keep" - or bool(decision.legacy_labels) - or bool(decision.approve_workflow_run_ids) - ) - - -def pr_merge_state(repo: str, number: int) -> str: - payload = run_gh( - ["pr", "view", str(number), "--repo", repo, "--json", "mergeStateStatus,mergeable"], - timeout_seconds=30, - ) - if not isinstance(payload, dict): - raise GhError(f"{repo}#{number}: expected pull request object") - mergeable = str(payload.get("mergeable") or "").upper() - merge_state = str(payload.get("mergeStateStatus") or "").upper() - if mergeable == "CONFLICTING": - return "CONFLICTING" - return merge_state or "UNKNOWN" - - -def needs_rebase_label_target(merge_state: str, *, has_label: bool) -> bool: - """Sync confirmed conflicts and preserve the label when GitHub is ambiguous.""" - - if merge_state in NEEDS_REBASE_STATES: - return True - if merge_state in NO_REBASE_STATES: - return False - return has_label - - -def workflow_runs_requiring_approval(repo: str, head_sha: str) -> tuple[int, ...]: - runs = paged_api(f"/repos/{repo}/actions/runs?event=pull_request&head_sha={head_sha}") - run_ids: list[int] = [] - for run in runs: - status = str(run.get("status") or "").lower() - conclusion = str(run.get("conclusion") or "").lower() - run_id = run.get("id") - if not isinstance(run_id, int): - continue - if status == "action_required" or conclusion == "action_required": - run_ids.append(run_id) - return tuple(run_ids) - - -def unresolved_codex_finding_thread_urls( - repo: str, - number: int, - *, - head_sha: str, - allowed_authors: set[str], -) -> tuple[str, ...]: - owner, name = repo.split("/", 1) - after: str | None = None - urls: list[str] = [] - - while True: - fields: dict[str, object] = {"owner": owner, "name": name, "number": number} - if after is not None: - fields["after"] = after - payload = graphql(PR_REVIEW_THREADS_QUERY, **fields) - pr = payload.get("data", {}).get("repository", {}).get("pullRequest") - if not isinstance(pr, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return a pull request") - - threads = pr.get("reviewThreads") - if not isinstance(threads, dict): - raise GhError(f"{repo}#{number}: GraphQL did not return review threads") - nodes = threads.get("nodes", []) - if not isinstance(nodes, list): - raise GhError(f"{repo}#{number}: GraphQL did not return review thread nodes") - - for thread in nodes: - if not isinstance(thread, dict): - continue - if thread.get("isResolved") or thread.get("isOutdated"): - continue - comments = thread.get("comments") - comment_nodes = comments.get("nodes") if isinstance(comments, dict) else [] - if not isinstance(comment_nodes, list): - continue - for comment in comment_nodes: - if not isinstance(comment, dict): - continue - author = comment.get("author") - login = author.get("login") if isinstance(author, dict) else None - if login not in allowed_authors: - continue - if not is_needs_work_codex_body(comment.get("body")): - continue - body = comment.get("body") - commit = comment.get("commit") - commit_oid = commit.get("oid") if isinstance(commit, dict) else None - original_commit = comment.get("originalCommit") - original_oid = original_commit.get("oid") if isinstance(original_commit, dict) else None - body_mentions_current_head = body_mentions_head(body, head_sha) - if body_mentions_current_head: - pass - elif commit_oid == head_sha: - pass - elif original_oid == head_sha: - pass - else: - continue - url = comment.get("url") - urls.append(str(url) if isinstance(url, str) else "unresolved Codex review thread") - - page_info = threads.get("pageInfo") - if not isinstance(page_info, dict) or not page_info.get("hasNextPage"): - break - end_cursor = page_info.get("endCursor") - if not isinstance(end_cursor, str) or not end_cursor: - break - after = end_cursor - - return tuple(urls) - - -def is_github_app_write_denial(exc: BaseException) -> bool: - """Return True when GitHub rejected a write from the current token.""" - - text = str(exc) - return "Resource not accessible by integration" in text and "HTTP 403" in text - - -def write_warning(action: str, exc: BaseException) -> str: - return f"{action}: skipped because the GitHub token cannot write this resource ({exc})" - - -def is_missing_issue_label(exc: BaseException) -> bool: - """Return True when GitHub reports that an issue label is already absent.""" - - text = str(exc) - return "HTTP 404" in text and "Label does not exist" in text - - -def gh_api_write( - path: str, - *, - method: str = "GET", - input_json: Any | None = None, - tolerate_permission_errors: bool, - tolerate_missing: bool = False, - action: str, -) -> str | None: - try: - gh_api(path, method=method, input_json=input_json) - except GhError as exc: - if tolerate_missing and is_missing_issue_label(exc): - return None - if tolerate_permission_errors and is_github_app_write_denial(exc): - return write_warning(action, exc) - raise - return None - - -def run_gh_write( - args: list[str], - *, - timeout_seconds: int, - tolerate_permission_errors: bool, - action: str, - fallback_retry: bool = True, -) -> str | None: - try: - run_gh(args, timeout_seconds=timeout_seconds, fallback_retry=fallback_retry) - except GhError as exc: - if tolerate_permission_errors and is_github_app_write_denial(exc): - return write_warning(action, exc) - raise - return None - - -def ensure_label( - repo: str, - label: str, - *, - color: str, - description: str, - apply: bool, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - if not apply: - return () - try: - gh_api(f"/repos/{repo}/labels/{quote(label, safe='')}") - return () - except GhError as exc: - if "HTTP 404" not in str(exc): - raise - - try: - warning = gh_api_write( - f"/repos/{repo}/labels", - method="POST", - input_json={ - "name": label, - "color": color, - "description": description, - }, - tolerate_permission_errors=tolerate_permission_errors, - action=f"create label {repo}:{label}", - ) - return (warning,) if warning else () - except GhError as exc: - if "already_exists" not in str(exc) and "already exists" not in str(exc).lower(): - raise - return () - - -def decide_pr( - repo: str, - number: int, - *, - allowed_authors: set[str], - ignore_checks: bool, - timeline_observer: Callable[[int, list[dict[str, Any]]], None] | None = None, -) -> SyncDecision: - head_sha, timeline_nodes = pr_timeline_evidence(repo, number) - if timeline_observer is not None: - timeline_observer(number, timeline_nodes) - labels = issue_label_names(repo, number) - checks_state = commit_checks_state(repo, head_sha) - merge_state = pr_merge_state(repo, number) - review_state, review_node = find_current_head_codex_review_state( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - unresolved_finding_urls = unresolved_codex_finding_thread_urls( - repo, - number, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - has_codex_news = has_codex_news_after_current_head( - timeline_nodes, - head_sha=head_sha, - allowed_authors=allowed_authors, - ) - has_ok_label = CODEX_OK_LABEL in labels - has_needs_work_label = CODEX_NEEDS_WORK_LABEL in labels - has_needs_rebase_label = NEEDS_REBASE_LABEL in labels - wants_needs_rebase_label = needs_rebase_label_target( - merge_state, - has_label=has_needs_rebase_label, - ) - legacy_labels = frozenset(label for label in labels if label in LEGACY_CODEX_LABELS) - - reason_parts: list[str] = [] - wants_ok_label = review_state == "clean" - wants_needs_work_label = review_state == "needs_work" - if review_state == "none": - reason_parts.append("no provable clean Codex review for current head") - elif review_state == "pending": - reason_parts.append("Codex review request is acknowledged but still pending") - elif review_state == "needs_work": - reason_parts.append("Codex raised current-head review issues") - else: - reason_parts.append("clean Codex review matches current head") - - if unresolved_finding_urls: - wants_ok_label = False - wants_needs_work_label = True - reason_parts.append(f"unresolved Codex review threads: {len(unresolved_finding_urls)}") - - if not ignore_checks and checks_state != "success": - wants_ok_label = False - reason_parts.append(f"checks are {checks_state}") - if not ignore_checks and merge_state in UNMERGEABLE_STATES | {"CONFLICTING"}: - wants_ok_label = False - reason_parts.append(f"merge state is {merge_state.lower()}") - if not ignore_checks and merge_state == "UNKNOWN" and not has_ok_label: - wants_ok_label = False - reason_parts.append("merge state is still unknown") - - trigger_codex_review = ( - review_state == "none" - and checks_state == "success" - and merge_state not in UNMERGEABLE_STATES | {"CONFLICTING"} - and merge_state != "UNKNOWN" - and not has_codex_news - ) - if trigger_codex_review: - reason_parts.append("current-head CI is green and no Codex news exists after head") - - approve_workflow_run_ids: tuple[int, ...] = () - if ( - review_state == "clean" - and not unresolved_finding_urls - and merge_state not in UNMERGEABLE_STATES | {"CONFLICTING"} - and merge_state != "UNKNOWN" - ): - approve_workflow_run_ids = workflow_runs_requiring_approval(repo, head_sha) - if approve_workflow_run_ids: - reason_parts.append( - "workflow runs need approval: " + ",".join(str(run_id) for run_id in approve_workflow_run_ids) - ) - - if wants_ok_label and not has_ok_label: - ok_action = "add" - elif not wants_ok_label and has_ok_label: - ok_action = "remove" - else: - ok_action = "keep" - if wants_needs_work_label and not has_needs_work_label: - needs_work_action = "add" - elif not wants_needs_work_label and has_needs_work_label: - needs_work_action = "remove" - else: - needs_work_action = "keep" - if wants_needs_rebase_label and not has_needs_rebase_label: - needs_rebase_action = "add" - elif not wants_needs_rebase_label and has_needs_rebase_label: - needs_rebase_action = "remove" - else: - needs_rebase_action = "keep" - - review_url = unresolved_finding_urls[0] if unresolved_finding_urls else None - if review_url is None and isinstance(review_node, dict): - review_url = node_url(review_node) - - return SyncDecision( - repo=repo, - number=number, - head_sha=head_sha, - has_ok_label=has_ok_label, - wants_ok_label=wants_ok_label, - ok_action=ok_action, - has_needs_work_label=has_needs_work_label, - wants_needs_work_label=wants_needs_work_label, - needs_work_action=needs_work_action, - has_needs_rebase_label=has_needs_rebase_label, - wants_needs_rebase_label=wants_needs_rebase_label, - needs_rebase_action=needs_rebase_action, - legacy_labels=legacy_labels, - reason="; ".join(reason_parts), - review_url=review_url, - review_state=review_state, - checks_state=checks_state, - merge_state=merge_state, - trigger_codex_review=trigger_codex_review, - approve_workflow_run_ids=approve_workflow_run_ids, - ) - - -def apply_decision(decision: SyncDecision, *, tolerate_permission_errors: bool = False) -> tuple[str, ...]: - warnings: list[str] = [] - - def record(warning: str | None) -> None: - if warning: - warnings.append(warning) - - if decision.ok_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [CODEX_OK_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {CODEX_OK_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.ok_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(CODEX_OK_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {CODEX_OK_LABEL} from {decision.repo}#{decision.number}", - ) - ) - if decision.needs_work_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [CODEX_NEEDS_WORK_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {CODEX_NEEDS_WORK_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.needs_work_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(CODEX_NEEDS_WORK_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {CODEX_NEEDS_WORK_LABEL} from {decision.repo}#{decision.number}", - ) - ) - if decision.needs_rebase_action == "add": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels", - method="POST", - input_json={"labels": [NEEDS_REBASE_LABEL]}, - tolerate_permission_errors=tolerate_permission_errors, - action=f"add {NEEDS_REBASE_LABEL} to {decision.repo}#{decision.number}", - ) - ) - elif decision.needs_rebase_action == "remove": - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(NEEDS_REBASE_LABEL, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove {NEEDS_REBASE_LABEL} from {decision.repo}#{decision.number}", - ) - ) - for label in decision.legacy_labels: - record( - gh_api_write( - f"/repos/{decision.repo}/issues/{decision.number}/labels/{quote(label, safe='')}", - method="DELETE", - tolerate_permission_errors=tolerate_permission_errors, - tolerate_missing=True, - action=f"remove legacy {label} from {decision.repo}#{decision.number}", - ) - ) - return tuple(warnings) - - -def trigger_codex_review( - decision: SyncDecision, - *, - body: str, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - warning = run_gh_write( - [ - "api", - "--method", - "POST", - f"/repos/{decision.repo}/issues/{decision.number}/comments", - "-f", - f"body={body}", - ], - timeout_seconds=30, - tolerate_permission_errors=tolerate_permission_errors, - action=f"request Codex review on {decision.repo}#{decision.number}", - fallback_retry=False, - ) - return (warning,) if warning else () - - -def approve_workflow_runs( - decision: SyncDecision, - *, - tolerate_permission_errors: bool = False, -) -> tuple[str, ...]: - warnings: list[str] = [] - for run_id in decision.approve_workflow_run_ids: - warning = gh_api_write( - f"/repos/{decision.repo}/actions/runs/{run_id}/approve", - method="POST", - tolerate_permission_errors=tolerate_permission_errors, - action=f"approve workflow run {run_id} for {decision.repo}#{decision.number}", - ) - if warning: - warnings.append(warning) - return tuple(warnings) - - -def parse_args(argv: list[str]) -> argparse.Namespace: - parser = argparse.ArgumentParser( - description=("Idempotently sync GitHub Codex review labels based on current-head Codex review state.") - ) - parser.add_argument("--repo", action="append", required=True, help="GitHub repo as owner/name. May repeat.") - parser.add_argument("--pr", action="append", type=int, help="PR number to sync. May repeat.") - parser.add_argument("--all-open", action="store_true", help="Sync all open PRs in each --repo.") - parser.add_argument("--apply", action="store_true", help="Actually write labels. Default is dry-run.") - parser.add_argument( - "--no-trigger-missing-codex", - action="store_true", - help="Do not post @codex review when current-head CI is green and Codex has no current-head news.", - ) - parser.add_argument( - "--no-approve-workflow-runs", - action="store_true", - help=( - "Do not approve action_required fork workflow runs after a current-head " - "clean Codex review on a mergeable PR." - ), - ) - parser.add_argument( - "--codex-review-command", - default="@codex review", - help="Issue comment body used to request a missing Codex review.", - ) - parser.add_argument( - "--codex-usage-limit-backoff-hours", - type=float, - default=DEFAULT_CODEX_USAGE_LIMIT_BACKOFF_HOURS, - help=( - "Skip new @codex review comments when the same comment sender account received a Codex usage-limit " - "reply within this many hours, unless that same account has a newer normal Codex response." - ), - ) - parser.add_argument( - "--codex-review-response-wait-seconds", - type=float, - default=DEFAULT_CODEX_REVIEW_RESPONSE_WAIT_SECONDS, - help=( - "After posting the first @codex review without recent quota evidence, wait this long, reread the PR " - "timeline, and stop further review requests if Codex replied with a usage limit." - ), - ) - parser.add_argument( - "--ignore-checks", - action="store_true", - help="Ignore current-head CI state when deciding the ok label. Normally do not use this.", - ) - parser.add_argument( - "--tolerate-write-permission-errors", - action="store_true", - help=( - "Log and continue when GitHub returns Resource not accessible by integration " - "for label/comment/approval writes. Read/classification errors still fail." - ), - ) - parser.add_argument( - "--tolerate-read-errors", - action="store_true", - help=( - "Log and continue when a selected PR cannot be classified because of a GitHub " - "read/API error. Intended for broad --all-open best-effort maintenance runs." - ), - ) - parser.add_argument( - "--reviewer-login", - action="append", - default=[], - help="Allowed Codex reviewer login. May repeat; defaults include chatgpt-codex-connector[bot].", - ) - return parser.parse_args(argv) - - -def main(argv: list[str] | None = None) -> int: - args = parse_args(argv or sys.argv[1:]) - repos = [repo_path(repo) for repo in args.repo] - allowed_authors = CODEX_REVIEW_AUTHORS | set(args.reviewer_login) - had_error = False - # Per-sender quota state is shared across every --repo in the run: a usage - # limit observed in one repository suppresses review requests in the rest. - # Timelines classified before the backoff exists are retained so evidence - # from repositories without their own triggers still counts. - usage_backoff: CodexReviewUsageBackoff | None = None - unobserved_timelines: list[list[dict[str, Any]]] = [] - codex_sender_unresolved = False - - for repo in repos: - setup_warnings: list[str] = [] - setup_warnings.extend( - ensure_label( - repo, - CODEX_OK_LABEL, - color="0e8a16", - description="Current PR head has green CI and a clean Codex review", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - setup_warnings.extend( - ensure_label( - repo, - CODEX_NEEDS_WORK_LABEL, - color="d93f0b", - description="Codex raised issues on the current PR head that still need work", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - setup_warnings.extend( - ensure_label( - repo, - NEEDS_REBASE_LABEL, - color="fbca04", - description="Needs rebase or conflict repair against current main", - apply=args.apply, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - for warning in setup_warnings: - print(f"warning: {warning}", file=sys.stderr, flush=True) - numbers = list_open_pr_numbers(repo) if args.all_open else list(args.pr or []) - if not numbers: - print(f"{repo}: no PRs selected; pass --pr or --all-open", file=sys.stderr) - had_error = True - continue - - timeline_nodes_by_number: dict[int, list[dict[str, Any]]] = {} - - def observe_timeline(_number: int, timeline_nodes: list[dict[str, Any]]) -> None: - timeline_nodes_by_number[_number] = timeline_nodes - - decisions: list[SyncDecision] = [] - classified_count = 0 - for number in sorted(set(numbers)): - try: - decision = decide_pr( - repo, - number, - allowed_authors=allowed_authors, - ignore_checks=args.ignore_checks, - timeline_observer=observe_timeline, - ) - except GhError as exc: - if not args.tolerate_read_errors: - had_error = True - print(f"{repo}#{number}: {exc}", file=sys.stderr, flush=True) - continue - except Exception as exc: # noqa: BLE001 - had_error = True - print(f"{repo}#{number}: {exc}", file=sys.stderr, flush=True) - continue - - classified_count += 1 - decisions.append(decision) - - if args.tolerate_read_errors and classified_count == 0: - had_error = True - print( - f"{repo}: all selected PRs failed classification; refusing a false-green tolerant run", - file=sys.stderr, - flush=True, - ) - - if args.apply and not args.no_trigger_missing_codex: - unobserved_timelines.extend(timeline_nodes_by_number.values()) - repo_has_triggers = any(decision.trigger_codex_review for decision in decisions) - if repo_has_triggers: - # Quota evidence may live outside the selected PRs (single - # --pr runs, closed PRs), so also observe the repo's recent - # issue comments before posting anything here. - try: - unobserved_timelines.extend( - recent_issue_comment_timelines( - repo, - since=datetime.now(UTC) - timedelta(hours=args.codex_usage_limit_backoff_hours), - ) - ) - except GhError as exc: - print( - f"warning: {repo}: could not gather repo-wide Codex quota evidence: {exc}", - file=sys.stderr, - flush=True, - ) - if usage_backoff is None and not codex_sender_unresolved and repo_has_triggers: - sender = resolve_codex_request_sender() - if sender is None: - # Only the trigger/backoff path depends on the sender; - # label sync and workflow approvals proceed regardless. - codex_sender_unresolved = True - print( - f"{repo}: cannot determine @codex review sender; " - "skipping review triggers but continuing label sync", - file=sys.stderr, - flush=True, - ) - else: - usage_backoff = CodexReviewUsageBackoff( - request_author=sender, - allowed_authors=allowed_authors, - window=timedelta(hours=args.codex_usage_limit_backoff_hours), - now=datetime.now(UTC), - ) - if usage_backoff is not None: - for timeline_nodes in unobserved_timelines: - usage_backoff.observe(timeline_nodes) - unobserved_timelines.clear() - - for decision in decisions: - try: - write_warnings: tuple[str, ...] = () - trigger_codex_review_now = decision.trigger_codex_review and not args.no_trigger_missing_codex - if args.apply and (decision_requires_writes(decision) or trigger_codex_review_now): - # Under --all-open every PR is classified before any is - # applied; evidence (head, checks, reviews, mergeability) - # may have moved meanwhile. Reclassify immediately before - # writing and act on the fresh decision only. The fresh - # timeline also feeds the shared backoff so a quota reply - # that arrived after bulk classification suppresses the - # remaining review requests. - try: - fresh_decision = decide_pr( - decision.repo, - decision.number, - allowed_authors=allowed_authors, - ignore_checks=args.ignore_checks, - timeline_observer=( - backoff_timeline_observer(usage_backoff) if usage_backoff is not None else None - ), - ) - except GhError as exc: - if not args.tolerate_read_errors: - had_error = True - print( - f"{decision.repo}#{decision.number}: apply-time reclassification failed: {exc}", - file=sys.stderr, - flush=True, - ) - continue - if fresh_decision.head_sha != decision.head_sha: - print( - f"warning: {decision.repo}#{decision.number}: head moved from " - f"{decision.head_sha[:12]} to {fresh_decision.head_sha[:12]} after classification; " - "skipping stale decision", - file=sys.stderr, - flush=True, - ) - continue - trigger_codex_review_now = trigger_codex_review_now and fresh_decision.trigger_codex_review - decision = fresh_decision - if args.apply: - accumulated_warnings: list[str] = [] - accumulated_warnings.extend( - apply_decision( - decision, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - if decision.approve_workflow_run_ids and not args.no_approve_workflow_runs: - accumulated_warnings.extend( - approve_workflow_runs( - decision, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - ) - if trigger_codex_review_now and _fallback_token_active: - # Comments would now be authored by the fallback token's - # identity, not the resolved sender, so quota replies - # could no longer be attributed. Stop posting. - accumulated_warnings.append( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - "the run switched to GH_FALLBACK_TOKEN and the resolved sender no longer " - "matches the active token" - ) - trigger_codex_review_now = False - if trigger_codex_review_now and codex_sender_unresolved: - accumulated_warnings.append( - f"request Codex review on {decision.repo}#{decision.number}: skipped because " - "the @codex review sender could not be resolved" - ) - trigger_codex_review_now = False - if trigger_codex_review_now and usage_backoff is not None and usage_backoff.is_limited(): - accumulated_warnings.append(usage_backoff.skip_warning(decision)) - trigger_codex_review_now = False - if trigger_codex_review_now: - trigger_warnings = trigger_codex_review( - decision, - body=args.codex_review_command, - tolerate_permission_errors=args.tolerate_write_permission_errors, - ) - accumulated_warnings.extend(trigger_warnings) - review_request_posted = not trigger_warnings - if ( - review_request_posted - and usage_backoff is not None - and usage_backoff.latest_normal_response_at is None - ): - if args.codex_review_response_wait_seconds > 0: - time.sleep(args.codex_review_response_wait_seconds) - _head_sha, timeline_nodes = pr_timeline_evidence(decision.repo, decision.number) - usage_backoff.observe(timeline_nodes) - write_warnings = tuple(accumulated_warnings) - mode = "apply" if args.apply else "dry-run" - print( - f"{mode} {decision.repo}#{decision.number}: " - f"head={decision.head_sha[:12]} checks={decision.checks_state} " - f"merge={decision.merge_state} review={decision.review_state} " - f"ok={decision.has_ok_label}->{decision.wants_ok_label}/{decision.ok_action} " - f"needs_work={decision.has_needs_work_label}->{decision.wants_needs_work_label}/" - f"{decision.needs_work_action} " - f"needs_rebase={decision.has_needs_rebase_label}->{decision.wants_needs_rebase_label}/" - f"{decision.needs_rebase_action} " - f"legacy={','.join(sorted(decision.legacy_labels)) or '-'} " - f"approve_runs={','.join(str(run_id) for run_id in decision.approve_workflow_run_ids) or '-'} " - f"trigger_codex={trigger_codex_review_now} " - f"reason={decision.reason}", - flush=True, - ) - if decision.review_url: - print(f" review_url={decision.review_url}", flush=True) - for warning in write_warnings: - print(f" write_warning={warning}", flush=True) - except Exception as exc: # noqa: BLE001 - had_error = True - print(f"{decision.repo}#{decision.number}: {exc}", file=sys.stderr, flush=True) - - return 1 if had_error else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd8009942a..0f88b4c6b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,10 +190,9 @@ jobs: run: make frontend-typecheck frontend-test: - # Keep this exact job name: the repository ruleset and - # .github/scripts/sync_codex_ok_labels.py treat "Frontend tests - # (vitest + coverage)" as a required check context. PR runs skip the - # coverage instrumentation below, but the check name must not change. + # Keep this exact job name: the repository ruleset requires "Frontend + # tests (vitest + coverage)" as a check context. PR runs skip the coverage + # instrumentation below, but the check name must not change. name: Frontend tests (vitest + coverage) runs-on: ubuntu-24.04 needs: changes @@ -464,11 +463,10 @@ jobs: run: make test-integration-core-${{ matrix.shard }} # Aggregate for the shards above. Keep this exact job name: the repository - # ruleset and .github/scripts/sync_codex_ok_labels.py treat - # "Tests (pytest, integration-core)" as a required check context. The shard - # jobs never skip at job level (they use the placeholder-step pattern), so - # anything other than an all-success matrix result must fail here — - # skipped or cancelled shards are not laundered into a passing check. + # ruleset requires "Tests (pytest, integration-core)" as a check context. + # The shard jobs never skip at job level (they use the placeholder-step + # pattern), so anything other than an all-success matrix result must fail + # here — skipped or cancelled shards are not laundered into a passing check. test-integration-core-required: name: Tests (pytest, integration-core) runs-on: ubuntu-24.04 diff --git a/.github/workflows/codex-review-labels.yml b/.github/workflows/codex-review-labels.yml deleted file mode 100644 index 62eef30fbe..0000000000 --- a/.github/workflows/codex-review-labels.yml +++ /dev/null @@ -1,124 +0,0 @@ -name: Codex review labels - -on: - pull_request_target: - types: [opened, synchronize, reopened, ready_for_review, converted_to_draft] - workflow_run: - workflows: ["CI"] - types: [completed] - issue_comment: - types: [created, edited] - pull_request_review: - types: [submitted, edited, dismissed] - schedule: - - cron: "*/15 * * * *" - -permissions: - actions: write - checks: read - contents: read - issues: write - pull-requests: read - statuses: read - -concurrency: - group: codex-review-labels-${{ github.event.pull_request.number || github.event.issue.number || github.event.workflow_run.head_sha || github.run_id }} - cancel-in-progress: false - -jobs: - sync-pr: - name: Sync Codex labels for PR - runs-on: ubuntu-24.04 - if: >- - ${{ - github.event_name == 'pull_request_target' || - github.event_name == 'pull_request_review' || - (github.event_name == 'issue_comment' && github.event.issue.pull_request) - }} - - steps: - - name: Checkout trusted base - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - ref: ${{ github.event.repository.default_branch }} - - - name: Mint label sync App token - id: app-token - if: ${{ vars.CODEX_LABEL_SYNC_APP_ID != '' }} - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 - with: - app-id: ${{ vars.CODEX_LABEL_SYNC_APP_ID }} - private-key: ${{ secrets.CODEX_LABEL_SYNC_APP_PRIVATE_KEY }} - permission-actions: write - permission-checks: read - permission-contents: read - permission-issues: write - permission-pull-requests: read - permission-statuses: read - - - name: Sync labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token }} - GH_FALLBACK_TOKEN: ${{ github.token }} - # App installation tokens cannot call GET /user; the script derives the - # @codex review sender login from this slug as "[bot]" instead. - GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }} - PR_NUMBER: ${{ github.event.pull_request.number || github.event.issue.number }} - REPO: ${{ github.repository }} - run: | - if [ ! -f .github/scripts/sync_codex_ok_labels.py ]; then - echo ".github/scripts/sync_codex_ok_labels.py is not available on the default branch yet; skipping bootstrap run" - exit 0 - fi - python3 .github/scripts/sync_codex_ok_labels.py --repo "$REPO" --pr "$PR_NUMBER" --apply --tolerate-write-permission-errors - - sync-after-ci: - name: Sync Codex labels after CI - runs-on: ubuntu-24.04 - if: >- - ${{ - github.event_name == 'schedule' || - ( - github.event_name == 'workflow_run' && - github.event.workflow_run.event == 'pull_request' - ) - }} - - steps: - - name: Checkout trusted base - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 - with: - persist-credentials: false - ref: ${{ github.event.repository.default_branch }} - - - name: Mint label sync App token - id: app-token - if: ${{ vars.CODEX_LABEL_SYNC_APP_ID != '' }} - continue-on-error: true - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 - with: - app-id: ${{ vars.CODEX_LABEL_SYNC_APP_ID }} - private-key: ${{ secrets.CODEX_LABEL_SYNC_APP_PRIVATE_KEY }} - permission-actions: write - permission-checks: read - permission-contents: read - permission-issues: write - permission-pull-requests: read - permission-statuses: read - - - name: Sync labels - env: - GH_TOKEN: ${{ steps.app-token.outputs.token || secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token }} - GH_FALLBACK_TOKEN: ${{ github.token }} - # App installation tokens cannot call GET /user; the script derives the - # @codex review sender login from this slug as "[bot]" instead. - GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }} - REPO: ${{ github.repository }} - run: | - if [ ! -f .github/scripts/sync_codex_ok_labels.py ]; then - echo ".github/scripts/sync_codex_ok_labels.py is not available on the default branch yet; skipping bootstrap run" - exit 0 - fi - python3 .github/scripts/sync_codex_ok_labels.py --repo "$REPO" --all-open --apply --tolerate-write-permission-errors --tolerate-read-errors diff --git a/.github/workflows/simplicity-budgets.yml b/.github/workflows/simplicity-budgets.yml index bf91705b60..b66baa6814 100644 --- a/.github/workflows/simplicity-budgets.yml +++ b/.github/workflows/simplicity-budgets.yml @@ -6,10 +6,8 @@ name: Simplicity budgets # This is deliberately a SEPARATE workflow from ci.yml, with different # pull_request trigger types: `labeled`/`unlabeled` re-evaluate the # simplicity-budget-approved override the moment a maintainer toggles it. -# Adding those types to ci.yml instead would re-run the full CI matrix on -# every label churn from codex-review-labels.yml (15-minute cron + -# workflow_run syncs of the codex labels), which is why they live here on a -# seconds-long job. +# They remain scoped to this seconds-long workflow so override-label changes +# do not re-run the full CI matrix. # # The override label is fetched live from the API (not read from the event # payload): fork PR payloads and re-runs of old runs would otherwise see a diff --git a/AGENTS.md b/AGENTS.md index 49ddd745f9..84a616afde 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -67,7 +67,7 @@ in [`.github/CONTRIBUTING.md`](.github/CONTRIBUTING.md). The sections an AI assistant most often needs are: - [Merge gates](.github/CONTRIBUTING.md#merge-gates) — CI green + - `@codex review` clean (or findings addressed) + `mergeable=CLEAN` + + actionable CodeRabbit findings addressed + `mergeable=CLEAN` + OpenSpec change folder for behavior changes + `Fixes #N` / `Closes #N` for issue cover + the five simplicity rules (PRINCIPLES.md P1-P5; see @@ -80,8 +80,8 @@ an AI assistant most often needs are: comment invoking the clause. An assistant preparing a merge MUST verify the gates against the -actual GitHub state (status check rollup, codex review submissions, -`mergeable` field) rather than asserting them from local history. +actual GitHub state (status check rollup, current-head CodeRabbit review +threads, `mergeable` field) rather than asserting them from local history. Local `uv run pytest` / `uv run ruff` / `codex review --base origin/main` are encouraged but not substitutes for the cloud gates. @@ -96,12 +96,10 @@ These rules encode recurring review blockers observed across codex-lb PRs. examples in `context.md` or change notes, and run strict OpenSpec validation before calling the PR ready. Code/tests alone are not enough when OpenSpec is required. -- Codex review state must come from current-head GitHub evidence. Check labels, - latest Codex review/comment/reaction, and GraphQL review threads before using - or claiming `🤖 codex: ok`. Usage-limit, environment, or missing-review - results mean missing evidence, not approval. Unresolved non-outdated P-level - Codex threads block readiness even when a top-level review comment looks - clean. +- CodeRabbit review state must come from current-head GitHub evidence. + Unresolved, non-outdated actionable review threads block readiness until + their findings are fixed or explicitly addressed or dismissed in-thread; + a top-level summary does not override active thread evidence. - Proxy failover and retry patches must prove account ownership and settlement invariants. File-pinned requests must not cross accounts; API-key reservations must settle before error-health writes; excluded accounts must actually leave diff --git a/openspec/changes/label-sync-rate-limit-fallback/proposal.md b/openspec/changes/label-sync-rate-limit-fallback/proposal.md deleted file mode 100644 index 0beb2bd7ac..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/proposal.md +++ /dev/null @@ -1,24 +0,0 @@ -## Why - -The Codex label sync workflow authenticates with a user PAT (`CODEX_LABEL_SYNC_TOKEN`), whose 5,000/hr REST quota is shared with every other consumer of that user's token (interactive sessions, agents, other automations). During busy review cycles the quota exhausts and every label sync run fails with `API rate limit exceeded (HTTP 403)`, painting spurious CI failures on open PRs for up to an hour — observed repeatedly on 2026-07-13 during the adaptive-windows review cycle (#1266/#1267/#1268). - -## What Changes - -- The sync script detects rate-limit exhaustion on any `gh` call and switches once to a fallback token (`GH_FALLBACK_TOKEN`), retrying the failed call; the workflow provides `github.token` as that fallback, which carries a separate per-repository Actions quota. -- When no distinct fallback is available (or it is also exhausted), behavior is unchanged: the run fails per the read/classification failure contract. - -## Capabilities - -### New Capabilities - -None. - -### Modified Capabilities - -- `github-automation`: label sync gains a runtime rate-limit token fallback on top of the existing configuration-time token preference. - -## Impact - -- Code: `.github/scripts/sync_codex_ok_labels.py`, `.github/workflows/codex-review-labels.yml` -- Tests: `tests/unit/test_sync_codex_ok_labels.py` -- Specs: `openspec/specs/github-automation/spec.md` diff --git a/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md b/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md deleted file mode 100644 index 4a5d3a38cd..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/specs/github-automation/spec.md +++ /dev/null @@ -1,25 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Codex review label sync write-token fallback - -The `Codex review labels` workflow MUST execute the label synchronization script from the trusted default branch and MUST prefer a repository-provided write token before falling back to the default `github.token`. When the active token's API quota is exhausted at runtime, the script MUST switch once to a configured fallback token and retry the failed call instead of failing the run outright. - -#### Scenario: Privileged token is configured - -- **WHEN** the workflow synchronizes Codex review labels -- **THEN** it uses `CODEX_LABEL_SYNC_TOKEN` when present -- **AND** it falls back to `RELEASE_PLEASE_TOKEN` before `github.token` -- **AND** it checks out the default branch with persisted checkout credentials disabled - -#### Scenario: Active token hits its rate limit - -- **GIVEN** the workflow provides `github.token` as `GH_FALLBACK_TOKEN` -- **WHEN** a gh call fails with `API rate limit exceeded` -- **THEN** the script switches to the fallback token once and retries the failed call -- **AND** subsequent calls in the run keep using the fallback token - -#### Scenario: No usable fallback token - -- **WHEN** a gh call fails with `API rate limit exceeded` -- **AND** no fallback token is configured, or it matches the active token, or it is also exhausted -- **THEN** the run fails as a read/classification failure per the existing contract diff --git a/openspec/changes/label-sync-rate-limit-fallback/tasks.md b/openspec/changes/label-sync-rate-limit-fallback/tasks.md deleted file mode 100644 index 6bf9b6c4e7..0000000000 --- a/openspec/changes/label-sync-rate-limit-fallback/tasks.md +++ /dev/null @@ -1,10 +0,0 @@ -## 1. Runtime token fallback - -- [x] 1.1 Detect `API rate limit exceeded` failures in the gh wrapper and switch once to `GH_FALLBACK_TOKEN`, retrying the failed call. -- [x] 1.2 Provide `github.token` as `GH_FALLBACK_TOKEN` in both label-sync workflow jobs. -- [x] 1.3 Unit coverage: fallback activates once and retries; no-op when the fallback is absent or identical; exhausted fallback still fails. - -## 2. Validation - -- [x] 2.1 Run the sync-script unit suite. -- [x] 2.2 Validate with `openspec validate label-sync-rate-limit-fallback --strict`. diff --git a/openspec/changes/remove-codex-review-label-gate/.openspec.yaml b/openspec/changes/remove-codex-review-label-gate/.openspec.yaml new file mode 100644 index 0000000000..0c73c8f54e --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-15 diff --git a/openspec/changes/remove-codex-review-label-gate/design.md b/openspec/changes/remove-codex-review-label-gate/design.md new file mode 100644 index 0000000000..0a60fe427d --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/design.md @@ -0,0 +1,29 @@ +## Context + +The retiring gate is implemented as one GitHub Actions workflow, one synchronization script, and its dedicated unit test module. Contributor guidance and workflow comments also describe that automation. Branch protection requires stable CI check contexts, not Codex labels, and the simplicity-budget workflow still needs label events for its independent override label. + +## Goals / Non-Goals + +**Goals:** + +- Remove the Codex review label automation as one coherent unit. +- Replace its merge-gate documentation with current-head CodeRabbit evidence. +- Preserve required CI check names and simplicity-budget override behavior. + +**Non-Goals:** + +- Changing branch-protection rules or required CI jobs. +- Removing the local Codex review harness or optional local review command. +- Replacing the removed `needs rebase` label with another synchronization workflow. + +## Decisions + +- Delete the workflow, script, and dedicated tests instead of disabling them. This prevents dormant automation from remaining a maintenance surface; retaining a disabled compatibility shim was rejected because there are no protected status checks or consumers to preserve. +- Remove every main-spec requirement whose behavior is implemented by the deleted synchronizer, including apply-time reclassification. The unrelated CI path-filtering and simplicity-budget requirements remain outside the delta. +- Keep `labeled` and `unlabeled` events on the simplicity-budget workflow because they re-evaluate `simplicity-budget-approved`, independent of the deleted label churn. +- Treat GitHub's live `mergeable` API field as triage evidence instead of replacing the removed `needs rebase` label sync. + +## Risks / Trade-offs + +- [Risk] Stale `needs rebase` labels may remain after automation removal. → Triage must use the live `mergeable` field, which is already the accepted source of truth. +- [Risk] Documentation could imply that optional local Codex review is still mandatory. → State consistently that CodeRabbit is the gate and local Codex review is only encouraged. diff --git a/openspec/changes/remove-codex-review-label-gate/proposal.md b/openspec/changes/remove-codex-review-label-gate/proposal.md new file mode 100644 index 0000000000..e85b9e1237 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/proposal.md @@ -0,0 +1,27 @@ +## Why + +The repository is adopting CodeRabbit on its OSS plan as the always-on mechanical reviewer under issue #1756. The auto-posted `@codex review` and `🤖 codex: ok` label gate is therefore redundant and should be retired. + +## What Changes + +- Replace the documented Codex cloud-review merge gate with a CodeRabbit gate that requires actionable findings to be fixed or explicitly addressed or dismissed in-thread on the merge-target head. +- Remove the Codex review label synchronization workflow, script, and unit tests. +- Remove the associated `needs rebase` label synchronization as accepted collateral; triage uses GitHub's live `mergeable` API field as its source of truth. +- Discard the in-flight `label-sync-rate-limit-fallback` change because it only patches the machinery being removed. +- Keep local `codex review --base origin/main` runs as an encouraged extra tool, not a merge gate. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `github-automation`: remove the requirements governing the retired Codex review and needs-rebase label synchronization machinery. + +## Impact + +- GitHub automation no longer auto-posts Codex review requests or maintains Codex review and needs-rebase labels. +- Contributor guidance uses CodeRabbit review evidence for the mechanical-review merge gate. +- Branch-protection status checks and simplicity-budget override label behavior remain unchanged. diff --git a/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md b/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md new file mode 100644 index 0000000000..0fe96196e1 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/specs/github-automation/spec.md @@ -0,0 +1,348 @@ +## REMOVED Requirements + +### Requirement: Needs-rebase label sync + +The Codex label synchronization script MUST add `needs rebase` when GitHub +reports a confirmed merge conflict, MUST remove it when GitHub reports a known +mergeable or non-conflict state, and MUST preserve its current value only when +GitHub reports `UNKNOWN`. It MUST NOT infer a conflict from the pull request +merely being behind the base branch. + +#### Scenario: Confirmed conflict gains the label + +- **WHEN** GitHub reports the pull request as `CONFLICTING` or `DIRTY` +- **THEN** the synchronizer adds `needs rebase` + +#### Scenario: Review-blocked pull request loses a stale label + +- **GIVEN** a pull request has `needs rebase` +- **WHEN** GitHub reports it as `BLOCKED` by review or status requirements +- **THEN** the synchronizer removes `needs rebase` + +#### Scenario: Base lag alone removes a stale label + +- **WHEN** GitHub reports a pull request as `BEHIND` without a confirmed conflict +- **THEN** the synchronizer removes `needs rebase` when present +- **AND** it does not add the label to an unlabelled pull request + +#### Scenario: Other mergeable statuses remove a stale label + +- **GIVEN** a pull request has `needs rebase` +- **WHEN** GitHub reports `CLEAN`, `DRAFT`, `HAS_HOOKS`, or `UNSTABLE` +- **THEN** the synchronizer removes `needs rebase` + +#### Scenario: Unknown state preserves current evidence + +- **WHEN** GitHub reports `UNKNOWN` +- **THEN** the synchronizer preserves the current `needs rebase` value + +### Requirement: Codex review label sync write-token fallback + +The `Codex review labels` workflow MUST execute the label synchronization script from the trusted default branch and MUST prefer a dedicated GitHub App installation token, then a repository-provided write token, before falling back to the default `github.token`. + +#### Scenario: GitHub App credentials are configured + +- **WHEN** the repository defines the `CODEX_LABEL_SYNC_APP_ID` variable and the `CODEX_LABEL_SYNC_APP_PRIVATE_KEY` secret +- **THEN** the workflow mints a short-lived installation token for that App before the sync step +- **AND** the mint requests only the label-sync permission subset (actions write, checks read, contents read, issues write, pull requests read, statuses read) rather than inheriting all installation permissions +- **AND** the sync step uses that token ahead of `CODEX_LABEL_SYNC_TOKEN`, `RELEASE_PLEASE_TOKEN`, and `github.token` + +#### Scenario: App token mint fails or is not configured + +- **WHEN** the mint step fails, or the `CODEX_LABEL_SYNC_APP_ID` variable is absent +- **THEN** the job does not fail because of the mint step +- **AND** the sync step falls back to the next available token in the chain + +#### Scenario: Privileged token is configured + +- **WHEN** the workflow synchronizes Codex review labels and no App token was minted +- **THEN** it uses `CODEX_LABEL_SYNC_TOKEN` when present +- **AND** it falls back to `RELEASE_PLEASE_TOKEN` before `github.token` +- **AND** it checks out the default branch with persisted checkout credentials disabled + +### Requirement: Codex review label sync write-denial resilience + +The Codex label synchronization script MUST distinguish GitHub write-permission denials from classification/read failures. + +#### Scenario: GitHub App token cannot mutate a PR resource + +- **WHEN** a label, comment, or workflow-run approval write returns `Resource not accessible by integration (HTTP 403)` +- **THEN** the workflow logs a per-PR warning for the skipped mutation +- **AND** it continues processing remaining selected PRs +- **AND** it exits successfully if no read/classification errors occurred + +#### Scenario: PR state cannot be read or classified + +- **WHEN** the script cannot read required PR state, check state, merge state, or Codex review evidence +- **THEN** the workflow fails rather than silently treating the PR as synchronized + +### Requirement: Codex review label sync review-thread state + +The Codex label synchronization script MUST grant `🤖 codex: ok` only when the +current pull-request head has green required checks, a clean Codex review for +that head, and no unresolved current-head Codex finding threads. It MUST treat +unresolved, non-outdated Codex inline review findings on the current head as +needs-work evidence, and MUST NOT treat inline Codex findings from resolved or +outdated review threads as active needs-work evidence. It MUST attribute an +unresolved thread to the current head only when the thread's current commit, +original commit, or body text ties it to the current head, and MUST treat +stale unresolved Codex inline threads as non-blocking when none of those tie +them to the current head. + +#### Scenario: Resolved inline finding no longer blocks the ok label + +- **WHEN** a current-head inline Codex finding comment belongs to a resolved + review thread +- **AND** a clean current-head Codex review exists +- **THEN** the script does not classify that inline finding as active + needs-work evidence + +#### Scenario: Unresolved inline finding still blocks the ok label + +- **WHEN** a current-head inline Codex finding comment belongs to an unresolved, + non-outdated review thread +- **THEN** the script classifies that inline finding as active needs-work + evidence + +#### Scenario: stale rebased inline thread remains unresolved + +- **GIVEN** a pull request was rebased after a Codex inline finding +- **AND** the unresolved GraphQL review thread still reports `isOutdated=false` +- **AND** the thread's current commit is not the current head +- **AND** the thread's original commit is not the current head +- **AND** the thread body does not mention the current head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread does not force `🤖 codex: needs work` + +#### Scenario: reanchored unresolved inline thread belongs to the current head + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread's current commit is the pull request head +- **AND** the thread's original commit is older than the pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: unresolved inline thread belongs to the current head + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread's original commit is the pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: unresolved inline thread mentions the current head explicitly + +- **GIVEN** an unresolved Codex inline finding thread +- **AND** the thread body mentions the current pull request head +- **WHEN** the label synchronizer evaluates the pull request +- **THEN** that thread blocks `🤖 codex: ok` +- **AND** the synchronizer records a needs-work reason that links to the thread + +#### Scenario: resolved inline thread is resynchronized by the scheduled fallback + +- **GIVEN** a pull request has a `🤖 codex: needs work` label from an unresolved Codex inline finding +- **WHEN** that review thread is resolved +- **THEN** the scheduled Codex label synchronization run resynchronizes the open pull request's labels + +### Requirement: Codex review labels use the authoritative current-head CI suite + +The Codex review label synchronizer SHALL identify the CI workflow from the +most recent `CI Required` check and SHALL treat the newest same-head run of +that workflow (ordered by the current attempt's start time, falling back to +workflow-run creation time, then check recency and run id) as the authoritative +CI suite when multiple runs of the same GitHub Actions CI workflow exist for +one pull-request head, even when that run has not yet produced its own +`CI Required` check. It MUST ignore Actions checks — +including stale required contexts — only from superseded (older) runs of that +workflow, while checks from the authoritative run, checks that cannot be +attributed to a workflow run, non-Actions status evidence, and failures from +independent workflows remain blocking evidence. + +#### Scenario: Cancelled duplicate leaves a unique failed placeholder + +- **GIVEN** an older CI workflow run for the current head was cancelled +- **AND** that run left a uniquely named non-required matrix placeholder in failure +- **AND** a newer run for the same head completed every required check including `CI Required` successfully +- **WHEN** Codex review labels are synchronized +- **THEN** the stale placeholder does not make the current head failed +- **AND** the synchronizer may request or accept current-head Codex review evidence + +#### Scenario: Authoritative CI run has an optional failure + +- **GIVEN** the newest run of the CI workflow identified by the latest `CI Required` check is the authoritative run +- **AND** another check in that same run failed +- **WHEN** Codex review labels are synchronized +- **THEN** the current head remains classified as failed + +#### Scenario: A newer run stays pending until its own CI Required completes + +- **GIVEN** an older run of the CI workflow completed `CI Required` successfully for the current head +- **AND** a newer run of the same CI workflow was created for the same head +- **AND** the newer run has started early checks but has not yet completed its own `CI Required` check +- **WHEN** Codex review labels are synchronized +- **THEN** the newer run is the authoritative CI suite and the older run's completed checks are ignored +- **AND** the current head remains classified as pending until the newer run's `CI Required` completes + +#### Scenario: An older workflow run id is manually rerun + +- **GIVEN** a newer-created CI workflow run completed successfully for the current head +- **AND** an older workflow `run_id` is manually rerun afterward +- **WHEN** the older run's new attempt has the latest `run_started_at` +- **THEN** that rerun is the authoritative CI suite +- **AND** its pending or failed checks remain blocking evidence + +#### Scenario: Independent workflow on the same head fails + +- **GIVEN** the authoritative CI workflow run is successful +- **AND** a different GitHub Actions workflow has a failed check on the same head +- **WHEN** Codex review labels are synchronized +- **THEN** the independent workflow failure remains blocking + +### Requirement: Codex label sync MUST use check-run recency evidence + +When multiple check runs have the same context name on a pull-request head, the label synchronizer MUST classify the current context from the newest run by +start or creation time. Completion time MUST NOT let an older superseded run +override a newer rerun that has already started. + +#### Scenario: older duplicate run completes after a newer rerun starts + +- **GIVEN** two check runs share the same name +- **AND** the older run started first but completes after the newer run starts +- **WHEN** the label synchronizer deduplicates check runs +- **THEN** it keeps the newer run +- **AND** a pending newer run keeps the pull request check state pending instead of failed + +### Requirement: Codex review trigger usage-limit backoff + +The Codex label synchronization script MUST NOT post a new `@codex review` comment while the comment sender's latest Codex response within the configured backoff window is a usage-limit reply. A usage-limit reply is a Codex response whose body starts (after optional leading whitespace) with the quota envelope "You have reached your Codex usage limits"; Codex reviews that merely discuss usage limits MUST NOT latch the backoff. Usage-limit evidence MUST be attributed to the sender whose request comment preceded the reply, and a newer normal Codex response for that same sender MUST lift the backoff; a clean THUMBS_UP reaction by a Codex reviewer on the sender's request comment counts as a normal response. Backoff state MUST be shared across all repositories processed in one run, so a usage limit observed in one repository suppresses the remaining review requests in the run; classified timelines from repositories without their own triggers MUST still contribute evidence. Before posting review requests in a repository, the script MUST also gather the repository's recent issue comments (within the backoff window, grouped per issue) as evidence, so quota replies on pull requests outside the current selection — including single `--pr` runs and closed pull requests — still latch the backoff; a failure to gather this evidence degrades to the classified-timeline evidence with a warning. When no quota evidence exists for the sender, the script MUST post the first `@codex review`, wait briefly, reread that pull request's timeline, and suppress the remaining review requests in the run if that probe observed a usage-limit reply; probing MUST stop once a normal Codex response has been observed, and MUST NOT run when the review request was not actually posted (for example after a tolerated write denial). Apply-loop status lines and error reports MUST reference the pull request of the decision being applied. + +The script MUST resolve the sender identity in a way that works with GitHub App installation tokens (which cannot call `GET /user`): it prefers the app slug exported by the workflow (`GH_APP_SLUG`, yielding `[bot]`) and falls back to `GET /user` for PAT-backed runs. Once the run has switched to the fallback token, the app slug no longer describes the active identity: sender resolution MUST ignore it, and review triggers MUST be suppressed with a warning because posted comments would no longer be authored by the resolved sender. The review-request POST itself MUST NOT be silently retried under the fallback token after a rate-limit response: the fallback activates for subsequent calls, but the identity-sensitive comment fails instead of posting under the wrong author. If the sender cannot be resolved, only the review-trigger path is disabled (with a warning per affected decision); label synchronization and workflow-run approvals MUST proceed. + +#### Scenario: Recent usage-limit reply latches the backoff + +- **GIVEN** the sender's `@codex review` comment was answered by a Codex usage-limit reply within the backoff window +- **AND** the sender has no newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post and surfaces a write warning naming the usage-limit evidence + +#### Scenario: Reviews that merely discuss usage limits do not latch + +- **GIVEN** a Codex review whose body discusses usage limits but does not start with the quota envelope +- **WHEN** the script classifies Codex responses for the backoff +- **THEN** the response is treated as a normal Codex response, not a usage-limit reply + +#### Scenario: Newer normal response lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** the same sender has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Newer clean reaction lifts the backoff + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** a Codex reviewer later reacted with THUMBS_UP to the sender's `@codex review` comment +- **WHEN** the script would trigger a missing Codex review +- **THEN** it posts the `@codex review` comment + +#### Scenario: Senders are attributed independently + +- **GIVEN** the sender received a Codex usage-limit reply within the backoff window +- **AND** only a different account has a newer normal Codex response +- **WHEN** the script would trigger a missing Codex review +- **THEN** it still skips the `@codex review` post for the sender + +#### Scenario: Backoff persists across repositories in one run + +- **GIVEN** a run selecting multiple repositories +- **AND** the sender's usage limit was observed while processing an earlier repository +- **WHEN** the script would trigger a missing Codex review in a later repository +- **THEN** it skips the `@codex review` post there as well + +#### Scenario: Evidence from a non-triggering repository still counts + +- **GIVEN** an earlier repository whose classified timelines contain the sender's usage-limit reply but whose decisions need no review trigger +- **WHEN** a later repository in the same run would trigger a missing Codex review +- **THEN** the earlier repository's evidence latches the backoff and the post is skipped + +#### Scenario: Quota evidence outside the selected pull requests still counts + +- **GIVEN** a single `--pr` run where the sender's usage-limit reply lives on a different (possibly closed) pull request of the repository +- **WHEN** the script would trigger a missing Codex review +- **THEN** the repository's recent issue comments provide the evidence and the post is skipped + +#### Scenario: Probe requires an actual post + +- **GIVEN** the review-request comment was not posted because the write was denied and tolerated +- **WHEN** the script would otherwise probe for a quota reply +- **THEN** it neither waits nor rereads the pull request timeline for that decision + +#### Scenario: No-data probe latches off remaining triggers + +- **GIVEN** no Codex quota evidence exists for the sender in the classified timelines +- **WHEN** the script posts the first `@codex review` of the run +- **THEN** it waits the configured probe interval, rereads that pull request's timeline, and skips the remaining review requests if the probe observed a usage-limit reply + +#### Scenario: Probing stops after a normal response + +- **GIVEN** a normal Codex response for the sender has already been observed +- **WHEN** the script posts further `@codex review` comments in the run +- **THEN** it does not wait or reread pull request timelines for those posts + +#### Scenario: Installation tokens resolve the sender from the app slug + +- **GIVEN** the run authenticates with a GitHub App installation token and the workflow exports the app slug +- **WHEN** the script resolves the `@codex review` sender +- **THEN** it derives `[bot]` without calling `GET /user` + +#### Scenario: Sender resolution failure only disables review triggers + +- **GIVEN** the sender cannot be resolved from either the app slug or `GET /user` +- **WHEN** the script applies decisions +- **THEN** label synchronization proceeds and each suppressed review trigger surfaces a warning naming the unresolved sender + +#### Scenario: Fallback token activation suppresses review triggers + +- **GIVEN** the run has switched to `GH_FALLBACK_TOKEN` after rate-limit exhaustion +- **WHEN** the script would trigger a missing Codex review +- **THEN** it skips the `@codex review` post with a warning, because the comment author would no longer match the resolved sender + +#### Scenario: The review-request POST is not retried under the fallback identity + +- **GIVEN** the review-request comment POST itself hits the primary token's rate limit +- **WHEN** the fallback token activates +- **THEN** the POST fails instead of being silently retried under the fallback identity, while later calls use the fallback token + +#### Scenario: Apply status is attributed to the applied pull request + +- **WHEN** the script applies decisions for multiple pull requests in one run +- **THEN** each status line and error report references the pull request of the decision being applied + +### Requirement: Apply-time reclassification + +Before performing writes for a classified decision (label changes, legacy label removal, workflow-run approvals, or review triggers), the Codex label synchronization script MUST reclassify the pull request and act on the fresh evidence only. If the head SHA no longer matches the SHA the decision was classified against, the decision MUST be skipped with a warning. If the head is unchanged but the evidence changed (checks, reviews, mergeability), the writes MUST follow the fresh decision, and a review trigger MUST only fire when both the original and the fresh classification want it. The freshly read timeline MUST feed the shared usage-limit backoff so quota replies that arrived after bulk classification suppress the remaining review requests. Reclassification read failures MUST honor `--tolerate-read-errors` (log and skip the decision without failing the run). Decisions without pending writes need not be reclassified. + +#### Scenario: Stale decision is skipped after a head move + +- **GIVEN** a pull request whose head changed between classification and apply +- **WHEN** the script reaches that decision in the apply loop +- **THEN** it skips all writes for the decision and warns that the head moved + +#### Scenario: Same-head evidence changes are applied fresh + +- **GIVEN** a pull request whose head is unchanged but where Codex raised a new finding after classification +- **WHEN** the script reaches that decision in the apply loop +- **THEN** the writes reflect the fresh classification instead of the superseded one + +#### Scenario: Fresh quota evidence suppresses later triggers + +- **GIVEN** a quota reply that arrived between bulk classification and apply-time reclassification of one pull request +- **WHEN** later decisions in the run would trigger missing Codex reviews +- **THEN** the reclassified timeline has latched the backoff and those posts are skipped + +#### Scenario: Reclassification honors tolerant reads + +- **GIVEN** a run with `--tolerate-read-errors` +- **WHEN** apply-time reclassification of one pull request fails with a GitHub read error +- **THEN** the decision is logged and skipped without failing the run diff --git a/openspec/changes/remove-codex-review-label-gate/tasks.md b/openspec/changes/remove-codex-review-label-gate/tasks.md new file mode 100644 index 0000000000..c6b12e3679 --- /dev/null +++ b/openspec/changes/remove-codex-review-label-gate/tasks.md @@ -0,0 +1,15 @@ +## 1. Retire label synchronization + +- [x] 1.1 Delete the Codex review label workflow, synchronization script, and dedicated unit tests. +- [x] 1.2 Discard the in-flight `label-sync-rate-limit-fallback` OpenSpec change. + +## 2. Update repository contracts and guidance + +- [x] 2.1 Replace the documented cloud Codex merge gate with the current-head CodeRabbit finding gate while retaining local Codex review as optional guidance. +- [x] 2.2 Remove stale label-sync references from CI and simplicity-budget workflow comments without changing check names or label-event triggers. +- [x] 2.3 Record removal of every label-sync requirement in the `github-automation` delta while preserving unrelated CI and simplicity-budget requirements. + +## 3. Verification + +- [x] 3.1 Validate the OpenSpec change and classify every remaining retired gate-term hit as historical, local-harness, or OpenSpec removal evidence. +- [x] 3.2 Run repository lint, type checks, and the unit test suite. diff --git a/tests/unit/test_sync_codex_ok_labels.py b/tests/unit/test_sync_codex_ok_labels.py deleted file mode 100644 index de6cc200a6..0000000000 --- a/tests/unit/test_sync_codex_ok_labels.py +++ /dev/null @@ -1,2106 +0,0 @@ -from __future__ import annotations - -import importlib.util -import sys -from pathlib import Path -from types import ModuleType -from typing import Any - -import pytest - - -def load_sync_module() -> ModuleType: - script_path = Path(__file__).resolve().parents[2] / ".github" / "scripts" / "sync_codex_ok_labels.py" - spec = importlib.util.spec_from_file_location("sync_codex_ok_labels", script_path) - assert spec is not None - assert spec.loader is not None - module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module - spec.loader.exec_module(module) - return module - - -def decision(module: ModuleType, **overrides: Any) -> Any: - values = { - "repo": "Soju06/codex-lb", - "number": 714, - "head_sha": "a" * 40, - "has_ok_label": True, - "wants_ok_label": False, - "ok_action": "remove", - "has_needs_work_label": False, - "wants_needs_work_label": False, - "needs_work_action": "keep", - "has_needs_rebase_label": False, - "wants_needs_rebase_label": False, - "needs_rebase_action": "keep", - "legacy_labels": frozenset(), - "reason": "checks are pending", - "review_url": None, - "review_state": "clean", - "checks_state": "pending", - "merge_state": "CLEAN", - "trigger_codex_review": False, - "approve_workflow_run_ids": (), - } - values.update(overrides) - return module.SyncDecision(**values) - - -def codex_review_request(author: str, created_at: str) -> dict[str, Any]: - return { - "__typename": "IssueComment", - "author": {"login": author}, - "bodyText": "@codex review", - "createdAt": created_at, - "url": f"https://github.test/request/{created_at}", - } - - -def codex_issue_comment(body: str, created_at: str) -> dict[str, Any]: - return { - "__typename": "IssueComment", - "author": {"login": "chatgpt-codex-connector"}, - "bodyText": body, - "createdAt": created_at, - "url": f"https://github.test/codex/{created_at}", - } - - -@pytest.mark.parametrize("merge_state", ["CONFLICTING", "DIRTY"]) -def test_needs_rebase_label_target_adds_for_confirmed_conflicts(merge_state: str) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target(merge_state, has_label=False) is True - - -@pytest.mark.parametrize("merge_state", ["BEHIND", "BLOCKED", "CLEAN", "DRAFT", "HAS_HOOKS", "UNSTABLE"]) -def test_needs_rebase_label_target_removes_for_known_non_conflict_states(merge_state: str) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target(merge_state, has_label=True) is False - - -@pytest.mark.parametrize("has_label", [False, True]) -def test_needs_rebase_label_target_preserves_unknown_state(has_label: bool) -> None: - module = load_sync_module() - - assert module.needs_rebase_label_target("UNKNOWN", has_label=has_label) is has_label - - -def test_apply_decision_adds_needs_rebase_label(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[tuple[str, str, Any | None]] = [] - - def capture_write(path: str, *, method: str = "GET", input_json: Any | None = None) -> None: - calls.append((method, path, input_json)) - - monkeypatch.setattr(module, "gh_api", capture_write) - - warnings = module.apply_decision( - decision( - module, - ok_action="keep", - has_needs_rebase_label=False, - wants_needs_rebase_label=True, - needs_rebase_action="add", - ) - ) - - assert warnings == () - assert calls == [ - ( - "POST", - "/repos/Soju06/codex-lb/issues/714/labels", - {"labels": ["needs rebase"]}, - ) - ] - - -def test_apply_decision_removes_stale_needs_rebase_label(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[tuple[str, str, Any | None]] = [] - - def capture_write(path: str, *, method: str = "GET", input_json: Any | None = None) -> None: - calls.append((method, path, input_json)) - - monkeypatch.setattr(module, "gh_api", capture_write) - - warnings = module.apply_decision( - decision( - module, - ok_action="keep", - has_needs_rebase_label=True, - wants_needs_rebase_label=False, - needs_rebase_action="remove", - ) - ) - - assert warnings == () - assert calls == [ - ( - "DELETE", - "/repos/Soju06/codex-lb/issues/714/labels/needs%20rebase", - None, - ) - ] - - -def test_classify_check_state_uses_latest_run_for_duplicate_check_names() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "completed_at": "2026-06-11T07:40:59Z", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:45:35Z", - }, - { - "name": "Type check (ty)", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:41:20Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required", "Type check (ty)"}), - ) - == "success" - ) - - -def test_classify_check_state_keeps_latest_pending_duplicate_pending() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "completed_at": "2026-06-11T07:40:59Z", - }, - { - "name": "CI Required", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-06-11T07:45:35Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_ignores_stale_duplicate_that_finishes_late() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-06-11T07:40:59Z", - "completed_at": "2026-06-11T07:50:00Z", - }, - { - "name": "CI Required", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-06-11T07:45:35Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_ignores_unique_failure_from_superseded_ci_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "Tests (pytest, ${{ matrix.slice.name }})", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:00:37Z", - "completed_at": "2026-07-10T06:00:37Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/1", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:00:38Z", - "completed_at": "2026-07-10T06:00:41Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/2", - "_github_actions_workflow_id": "ci", - }, - { - "name": "Tests (pytest, unit)", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:01:00Z", - "completed_at": "2026-07-10T06:05:00Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/3", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required", "Tests (pytest, unit)"}), - ) - == "success" - ) - - -def test_classify_check_state_keeps_optional_failure_from_authoritative_ci_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "optional security scan", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:09:00Z", - "completed_at": "2026-07-10T06:09:04Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/3", - "_github_actions_workflow_id": "ci", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_classify_check_state_keeps_newer_same_workflow_run_pending_before_required_job_exists() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:43Z", - }, - { - "name": "Detect changes", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-07-10T06:50:20Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:20Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_keeps_manual_rerun_of_older_run_pending() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:50:20Z", - "completed_at": "2026-07-10T06:59:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:00Z", - "_github_actions_run_started_at": "2026-07-10T06:50:00Z", - }, - { - "name": "Detect changes", - "status": "in_progress", - "conclusion": None, - "started_at": "2026-07-10T07:10:20Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/1", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:00Z", - "_github_actions_run_started_at": "2026-07-10T07:10:00Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "pending" - ) - - -def test_classify_check_state_keeps_failure_from_manual_rerun_of_older_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:50:20Z", - "completed_at": "2026-07-10T06:59:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:50:00Z", - "_github_actions_run_started_at": "2026-07-10T06:50:00Z", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T07:10:20Z", - "completed_at": "2026-07-10T07:15:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/100/job/4", - "_github_actions_workflow_id": "ci", - "_github_actions_run_created_at": "2026-07-10T06:00:00Z", - "_github_actions_run_started_at": "2026-07-10T07:10:00Z", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_classify_check_state_keeps_failure_from_independent_workflow_run() -> None: - module = load_sync_module() - - check_runs = [ - { - "name": "independent security scan", - "status": "completed", - "conclusion": "failure", - "started_at": "2026-07-10T06:08:00Z", - "completed_at": "2026-07-10T06:08:30Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - "_github_actions_workflow_id": "security", - }, - { - "name": "CI Required", - "status": "completed", - "conclusion": "success", - "started_at": "2026-07-10T06:09:01Z", - "completed_at": "2026-07-10T06:09:05Z", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - "_github_actions_workflow_id": "ci", - }, - ] - - assert ( - module.classify_check_state( - check_runs, - {"statuses": []}, - required_check_names=frozenset({"CI Required"}), - ) - == "failure" - ) - - -def test_annotate_github_actions_workflow_ids_is_conservative_when_metadata_lookup_fails( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - check_runs = [ - { - "name": "CI Required", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/200/job/4", - }, - { - "name": "independent scan", - "details_url": "https://github.com/Soju06/codex-lb/actions/runs/300/job/1", - }, - ] - - def workflow_run(path: str) -> dict[str, int | str]: - if path.endswith("/200"): - return { - "workflow_id": 10, - "created_at": "2026-07-10T06:00:43Z", - "run_started_at": "2026-07-10T07:10:00Z", - } - raise module.GhError("metadata unavailable") - - monkeypatch.setattr(module, "gh_api", workflow_run) - - annotated = module.annotate_github_actions_workflow_ids("Soju06/codex-lb", check_runs) - - assert annotated[0]["_github_actions_workflow_id"] == "10" - assert annotated[0]["_github_actions_run_started_at"] == "2026-07-10T07:10:00Z" - assert "_github_actions_workflow_id" not in annotated[1] - - -def test_apply_decision_tolerates_github_app_write_denial(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "gh_api", deny_write) - - warnings = module.apply_decision(decision(module), tolerate_permission_errors=True) - - assert len(warnings) == 1 - assert "remove 🤖 codex: ok from Soju06/codex-lb#714" in warnings[0] - assert "Resource not accessible by integration" in warnings[0] - - -def test_apply_decision_still_fails_on_write_denial_without_tolerance(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "gh_api", deny_write) - - with pytest.raises(module.GhError): - module.apply_decision(decision(module), tolerate_permission_errors=False) - - -def test_apply_decision_treats_missing_label_delete_as_done(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - calls: list[tuple[str, str]] = [] - - def missing_label(path: str, *, method: str = "GET", **_kwargs: Any) -> None: - calls.append((method, path)) - raise module.GhError("gh: Label does not exist (HTTP 404)") - - monkeypatch.setattr(module, "gh_api", missing_label) - - warnings = module.apply_decision(decision(module), tolerate_permission_errors=False) - - assert warnings == () - assert calls == [ - ( - "DELETE", - "/repos/Soju06/codex-lb/issues/714/labels/%F0%9F%A4%96%20codex%3A%20ok", - ) - ] - - -def test_apply_decision_does_not_swallow_unrelated_delete_404(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def missing_resource(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Not Found (HTTP 404)") - - monkeypatch.setattr(module, "gh_api", missing_resource) - - with pytest.raises(module.GhError): - module.apply_decision(decision(module), tolerate_permission_errors=False) - - -def test_trigger_codex_review_tolerates_github_app_write_denial(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - def deny_write(*_args: Any, **_kwargs: Any) -> None: - raise module.GhError("gh: Resource not accessible by integration (HTTP 403)") - - monkeypatch.setattr(module, "run_gh", deny_write) - request_review = decision(module, trigger_codex_review=True, ok_action="keep") - - warnings = module.trigger_codex_review( - request_review, - body="@codex review", - tolerate_permission_errors=True, - ) - - assert len(warnings) == 1 - assert "request Codex review on Soju06/codex-lb#714" in warnings[0] - - -def test_codex_usage_backoff_blocks_recent_limit_for_same_sender() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T15:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T15:01:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -def test_codex_usage_backoff_allows_after_newer_normal_reply() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request("Komzpa", "2026-07-31T15:00:00Z"), - codex_issue_comment("Codex Review: Didn't find any major issues.", "2026-07-31T15:02:00Z"), - ] - ) - - assert backoff.is_limited() is False - - -def test_codex_usage_backoff_keeps_accounts_independent() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request("OtherUser", "2026-07-31T15:00:00Z"), - codex_issue_comment("Codex Review: Didn't find any major issues.", "2026-07-31T15:02:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -@pytest.mark.parametrize( - "body", - [ - "You have reached your Codex usage limits for code reviews. " - "You can see your limits in the [Codex usage dashboard](https://chatgpt.com/codex/settings/usage).", - " \nYou have reached your Codex usage limits for code reviews.", - "You've reached your Codex usage limits.", - ], -) -def test_usage_limit_body_matches_real_quota_envelope(body: str) -> None: - module = load_sync_module() - - assert module.is_codex_usage_limit_body(body) is True - - -@pytest.mark.parametrize( - "body", - [ - "**[P1]** The unanchored `usage limit` pattern also matches reviews discussing usage limits.", - "Codex Review: the backoff should latch on a Codex usage limit reply. Didn't find any major issues.", - "This PR adds a usage-limit backoff. You have reached your Codex usage limits is the trigger phrase.", - None, - "", - ], -) -def test_usage_limit_body_ignores_reviews_discussing_usage_limits(body: object) -> None: - module = load_sync_module() - - assert module.is_codex_usage_limit_body(body) is False - - -def codex_review_request_with_reaction( - author: str, - created_at: str, - *, - reaction_user: str, - reaction_content: str, - reaction_created_at: str, -) -> dict[str, Any]: - request = codex_review_request(author, created_at) - request["reactions"] = { - "nodes": [ - { - "content": reaction_content, - "createdAt": reaction_created_at, - "user": {"login": reaction_user}, - } - ] - } - return request - - -def test_codex_usage_backoff_unlatches_on_newer_clean_reaction() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T15:00:00Z", - reaction_user="chatgpt-codex-connector", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T15:05:00Z", - ), - ] - ) - - assert backoff.is_limited() is False - - -def test_codex_usage_backoff_ignores_reactions_from_non_codex_users() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T15:00:00Z", - reaction_user="SomeoneElse", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T15:05:00Z", - ), - ] - ) - - assert backoff.is_limited() is True - - -def test_codex_usage_backoff_ignores_clean_reaction_older_than_limit() -> None: - module = load_sync_module() - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - - backoff.observe( - [ - codex_review_request_with_reaction( - "Komzpa", - "2026-07-31T13:00:00Z", - reaction_user="chatgpt-codex-connector", - reaction_content="THUMBS_UP", - reaction_created_at="2026-07-31T13:05:00Z", - ), - codex_review_request("Komzpa", "2026-07-31T14:00:00Z"), - codex_issue_comment("You've reached your Codex usage limits.", "2026-07-31T14:01:00Z"), - ] - ) - - assert backoff.is_limited() is True - - -def test_resolve_codex_request_sender_prefers_app_slug(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - - def fail_viewer_login() -> str: - raise AssertionError("GET /user must not be called when GH_APP_SLUG is set") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - - assert module.resolve_codex_request_sender() == "codex-label-sync[bot]" - - -def test_resolve_codex_request_sender_keeps_explicit_bot_suffix(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync[bot]") - - assert module.resolve_codex_request_sender() == "codex-label-sync[bot]" - - -def test_resolve_codex_request_sender_falls_back_to_viewer_login(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - - assert module.resolve_codex_request_sender() == "Komzpa" - - -def test_resolve_codex_request_sender_returns_none_when_unresolvable( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - monkeypatch.delenv("GH_APP_SLUG", raising=False) - - def fail_viewer_login() -> str: - raise module.GhError("gh api /user: HTTP 403 (installation token)") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - - assert module.resolve_codex_request_sender() is None - assert "cannot resolve @codex review sender" in capsys.readouterr().err - - -def recent_timestamp(module: ModuleType, *, minutes_ago: int) -> str: - moment = module.datetime.now(module.UTC) - module.timedelta(minutes=minutes_ago) - return moment.strftime("%Y-%m-%dT%H:%M:%SZ") - - -def test_main_stops_codex_review_triggers_after_probe_hits_usage_limit( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - observer( - number, - [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ], - ) - return decision(module, number=number, trigger_codex_review=True, ok_action="keep", checks_state="success") - - posted: list[int] = [] - - def fake_trigger(decision: Any, **_kwargs: Any) -> tuple[str, ...]: - posted.append(decision.number) - return () - - def fake_timeline(_repo: str, _number: int) -> tuple[str, list[dict[str, Any]]]: - return ( - "a" * 40, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=2)), - codex_issue_comment( - "You've reached your Codex usage limits.", - recent_timestamp(module, minutes_ago=1), - ), - ], - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "trigger_codex_review", fake_trigger) - monkeypatch.setattr(module, "pr_timeline_evidence", fake_timeline) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [710] - apply_lines = [line for line in captured.out.splitlines() if line.startswith("apply ")] - assert len(apply_lines) == 2 - assert apply_lines[0].startswith("apply Soju06/codex-lb#710: ") - assert "trigger_codex=True" in apply_lines[0] - assert apply_lines[1].startswith("apply Soju06/codex-lb#714: ") - assert "trigger_codex=False" in apply_lines[1] - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_skips_probe_after_normal_codex_response_observed( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - observer( - number, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "Codex Review: Didn't find any major issues.", - recent_timestamp(module, minutes_ago=29), - ), - ], - ) - return decision(module, number=number, trigger_codex_review=True, ok_action="keep", checks_state="success") - - posted: list[int] = [] - - def fake_trigger(decision: Any, **_kwargs: Any) -> tuple[str, ...]: - posted.append(decision.number) - return () - - probe_calls: list[int] = [] - - def fake_timeline(_repo: str, number: int) -> tuple[str, list[dict[str, Any]]]: - probe_calls.append(number) - return ("a" * 40, []) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "trigger_codex_review", fake_trigger) - monkeypatch.setattr(module, "pr_timeline_evidence", fake_timeline) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [710, 714] - assert probe_calls == [] - assert "apply Soju06/codex-lb#710: " in captured.out - assert "apply Soju06/codex-lb#714: " in captured.out - - -def test_main_continues_label_sync_when_sender_is_unresolvable( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fail_viewer_login() -> str: - raise module.GhError("gh api /user: HTTP 403 (installation token)") - - monkeypatch.setattr(module, "current_viewer_login", fail_viewer_login) - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, number=number, trigger_codex_review=True, checks_state="success" - ), - ) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [710, 714] - assert posted == [] - assert "cannot determine @codex review sender; skipping review triggers" in captured.err - assert captured.out.count("sender could not be resolved") == 2 - assert captured.out.count("trigger_codex=False") == 2 - - -def test_main_skips_apply_when_head_moved_after_classification( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls_by_number: dict[int, int] = {} - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls_by_number[number] = calls_by_number.get(number, 0) + 1 - # The classification pass sees head a...a; by the time #710 is - # re-classified in the apply loop its head has moved to b...b. - if number == 710 and calls_by_number[number] > 1: - return decision(module, number=number, head_sha="b" * 40) - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [714] - assert calls_by_number == {710: 2, 714: 2} - assert "Soju06/codex-lb#710: head moved from" in captured.err - assert "skipping stale decision" in captured.err - assert "apply Soju06/codex-lb#710" not in captured.out - assert "apply Soju06/codex-lb#714" in captured.out - - -def test_main_applies_freshly_reclassified_decision_for_same_head( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) == 1: - return decision( - module, - number=number, - has_ok_label=False, - wants_ok_label=True, - ok_action="add", - checks_state="success", - ) - # Same head, but by apply time Codex raised a new finding. - return decision( - module, - number=number, - has_ok_label=False, - wants_ok_label=False, - ok_action="keep", - wants_needs_work_label=True, - needs_work_action="add", - review_state="needs_work", - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied_actions: list[tuple[str, str]] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: ( - applied_actions.append((applied_decision.ok_action, applied_decision.needs_work_action)), - (), - )[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied_actions == [("keep", "add")] - assert "review=needs_work" in captured.out - - -def test_main_usage_backoff_state_persists_across_repos( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None: - timeline: list[dict[str, Any]] = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ] - if repo == "Soju06/codex-lb": - timeline.extend( - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ] - ) - observer(number, timeline) - return decision(module, repo=repo, number=number, trigger_codex_review=True, checks_state="success") - - posted: list[str] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.repo), ())[1], - ) - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--repo", - "Soju06/other-repo", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "apply Soju06/codex-lb#710" in captured.out - assert "apply Soju06/other-repo#710" in captured.out - assert "request Codex review on Soju06/other-repo#710: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_usage_backoff_counts_evidence_from_non_triggering_repo( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - def fake_decide_pr(repo: str, number: int, **kwargs: Any) -> Any: - observer = kwargs.get("timeline_observer") - if observer is not None and repo == "Soju06/codex-lb": - # The first repo has quota evidence but no trigger of its own. - observer( - number, - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ], - ) - elif observer is not None: - observer(number, []) - trigger = repo == "Soju06/other-repo" - return decision( - module, - repo=repo, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=trigger, - checks_state="success", - ) - - posted: list[str] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.repo), ())[1], - ) - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main( - [ - "--repo", - "Soju06/codex-lb", - "--repo", - "Soju06/other-repo", - "--all-open", - "--apply", - "--codex-review-response-wait-seconds", - "0", - ] - ) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/other-repo#710: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_stops_codex_review_triggers_after_fallback_token_activates( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - monkeypatch.setattr(module, "_fallback_token_active", True) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "fallback-user") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "switched to GH_FALLBACK_TOKEN" in captured.out - assert "trigger_codex=False" in captured.out - - -def test_resolve_codex_request_sender_ignores_app_slug_after_fallback(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_APP_SLUG", "codex-label-sync") - monkeypatch.setattr(module, "_fallback_token_active", True) - monkeypatch.setattr(module, "current_viewer_login", lambda: "fallback-user") - - assert module.resolve_codex_request_sender() == "fallback-user" - - -def test_recent_issue_comment_timelines_groups_by_issue(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - comments = [ - { - "body": "@codex review", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/700", - "created_at": "2026-07-31T14:00:00Z", - "html_url": "https://github.test/pull/700#issuecomment-1", - "user": {"login": "Komzpa"}, - }, - { - "body": "unrelated comment on another issue", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/701", - "created_at": "2026-07-31T14:00:30Z", - "html_url": "https://github.test/pull/701#issuecomment-2", - "user": {"login": "someone"}, - }, - { - "body": "You have reached your Codex usage limits for code reviews.", - "issue_url": "https://api.github.test/repos/Soju06/codex-lb/issues/700", - "created_at": "2026-07-31T14:01:00Z", - "html_url": "https://github.test/pull/700#issuecomment-3", - "user": {"login": "chatgpt-codex-connector"}, - }, - ] - paths: list[str] = [] - - def fake_paged_api(path: str) -> list[dict[str, Any]]: - paths.append(path) - return comments - - monkeypatch.setattr(module, "paged_api", fake_paged_api) - - timelines = module.recent_issue_comment_timelines( - "Soju06/codex-lb", - since=module.datetime.fromisoformat("2026-07-30T16:00:00+00:00"), - ) - - assert len(paths) == 1 - assert paths[0].startswith("/repos/Soju06/codex-lb/issues/comments?since=2026-07-30T16") - assert len(timelines) == 2 - grouped = {timeline[0]["url"].split("#")[0]: timeline for timeline in timelines} - pr_700 = grouped["https://github.test/pull/700"] - assert [node["bodyText"] for node in pr_700] == [ - "@codex review", - "You have reached your Codex usage limits for code reviews.", - ] - assert all(node["__typename"] == "IssueComment" for node in pr_700) - - backoff = module.CodexReviewUsageBackoff( - request_author="Komzpa", - allowed_authors={"chatgpt-codex-connector"}, - window=module.timedelta(hours=24), - now=module.datetime.fromisoformat("2026-07-31T16:00:00+00:00"), - ) - for timeline in timelines: - backoff.observe(timeline) - assert backoff.is_limited() is True - - -def test_main_gathers_repo_wide_quota_evidence_for_single_pr_run( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - # Quota evidence lives on another (already closed) PR of the repo. - monkeypatch.setattr( - module, - "recent_issue_comment_timelines", - lambda _repo, **_kwargs: [ - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=30)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=29), - ), - ] - ], - ) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--pr", "714", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/codex-lb#714: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_observes_quota_evidence_from_apply_time_reclassification( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls_by_number: dict[int, int] = {} - - def fake_decide_pr(_repo: str, number: int, **kwargs: Any) -> Any: - calls_by_number[number] = calls_by_number.get(number, 0) + 1 - reclassification = calls_by_number[number] > 1 - observer = kwargs.get("timeline_observer") - if observer is not None: - timeline: list[dict[str, Any]] = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": "a" * 40}, - "committedDate": recent_timestamp(module, minutes_ago=70), - } - ] - if reclassification and number == 710: - # A quota reply arrived between bulk classification and apply. - timeline.extend( - [ - codex_review_request("Komzpa", recent_timestamp(module, minutes_ago=3)), - codex_issue_comment( - "You have reached your Codex usage limits for code reviews.", - recent_timestamp(module, minutes_ago=2), - ), - ] - ) - observer(number, timeline) - trigger = not (reclassification and number == 710) - return decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=trigger, - checks_state="success", - ) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - posted: list[int] = [] - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: (posted.append(request_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert posted == [] - assert "request Codex review on Soju06/codex-lb#714: skipped" in captured.out - assert "recent Codex usage-limit reply" in captured.out - - -def test_main_tolerates_apply_time_reclassification_read_errors( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) > 1: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - applied: list[int] = [] - monkeypatch.setattr( - module, - "apply_decision", - lambda applied_decision, **_kwargs: (applied.append(applied_decision.number), ())[1], - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 0 - assert applied == [] - assert "Soju06/codex-lb#714: apply-time reclassification failed" in captured.err - - -def test_main_fails_apply_time_reclassification_read_errors_without_tolerance( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - - calls: list[int] = [] - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - calls.append(number) - if len(calls) > 1: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - - assert module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) == 1 - - -def test_main_skips_probe_when_trigger_post_was_denied( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.delenv("GH_APP_SLUG", raising=False) - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "current_viewer_login", lambda: "Komzpa") - monkeypatch.setattr(module, "recent_issue_comment_timelines", lambda *_args, **_kwargs: []) - monkeypatch.setattr(module, "apply_decision", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "approve_workflow_runs", lambda *_args, **_kwargs: ()) - monkeypatch.setattr( - module, - "decide_pr", - lambda _repo, number, **_kwargs: decision( - module, - number=number, - ok_action="keep", - has_ok_label=False, - trigger_codex_review=True, - checks_state="success", - ), - ) - monkeypatch.setattr( - module, - "trigger_codex_review", - lambda request_decision, **_kwargs: ( - f"request Codex review on {request_decision.repo}#{request_decision.number}: " - "skipped because the GitHub token cannot write this resource", - ), - ) - - probe_calls: list[int] = [] - monkeypatch.setattr( - module, - "pr_timeline_evidence", - lambda _repo, number: (probe_calls.append(number), ("a" * 40, []))[1], - ) - sleeps: list[float] = [] - monkeypatch.setattr(module.time, "sleep", sleeps.append) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply"]) - - captured = capsys.readouterr() - assert result == 0 - assert probe_calls == [] - assert sleeps == [] - assert "write_warning=request Codex review on Soju06/codex-lb#714" in captured.out - - -def test_workflow_prefers_privileged_token_and_enables_tolerant_apply() -> None: - workflow = Path(".github/workflows/codex-review-labels.yml").read_text(encoding="utf-8") - - assert "secrets.CODEX_LABEL_SYNC_TOKEN || secrets.RELEASE_PLEASE_TOKEN || github.token" in workflow - app_slug_env = "GH_APP_SLUG: ${{ steps.app-token.outputs.token && steps.app-token.outputs.app-slug || '' }}" - assert workflow.count(app_slug_env) == 2 - assert "pull_request_review_thread:" not in workflow - assert "github.event_name == 'pull_request_review_thread'" not in workflow - assert 'cron: "*/15 * * * *"' in workflow - assert workflow.count("--tolerate-write-permission-errors") == 2 - assert workflow.count("--tolerate-read-errors") == 1 - - -def test_main_tolerates_read_errors_when_requested( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - - def fake_decide_pr(_repo: str, number: int, **_kwargs: Any) -> Any: - if number == 710: - raise module.GhError("gh: HTTP 502") - return decision(module, number=number) - - monkeypatch.setattr(module, "decide_pr", fake_decide_pr) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 0 - assert "Soju06/codex-lb#710: gh: HTTP 502" in captured.err - assert "dry-run Soju06/codex-lb#714" in captured.out - - -def test_main_fails_tolerant_run_when_every_pr_read_fails( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710, 714]) - monkeypatch.setattr( - module, - "decide_pr", - lambda *_args, **_kwargs: (_ for _ in ()).throw(module.GhError("gh: HTTP 502")), - ) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 1 - assert "all selected PRs failed classification" in captured.err - - -def test_main_fails_read_errors_without_tolerance(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [710]) - monkeypatch.setattr( - module, - "decide_pr", - lambda *_args, **_kwargs: (_ for _ in ()).throw(module.GhError("gh: HTTP 502")), - ) - - assert module.main(["--repo", "Soju06/codex-lb", "--all-open"]) == 1 - - -def test_main_fails_apply_errors_even_with_read_error_tolerance( - monkeypatch: pytest.MonkeyPatch, - capsys: pytest.CaptureFixture[str], -) -> None: - module = load_sync_module() - - monkeypatch.setattr(module, "ensure_label", lambda *_args, **_kwargs: ()) - monkeypatch.setattr(module, "list_open_pr_numbers", lambda _repo: [714]) - monkeypatch.setattr(module, "decide_pr", lambda *_args, **_kwargs: decision(module)) - - def fail_apply(*_args: Any, **_kwargs: Any) -> tuple[str, ...]: - raise module.GhError("gh: HTTP 500 while writing labels") - - monkeypatch.setattr(module, "apply_decision", fail_apply) - - result = module.main(["--repo", "Soju06/codex-lb", "--all-open", "--apply", "--tolerate-read-errors"]) - - captured = capsys.readouterr() - assert result == 1 - assert "Soju06/codex-lb#714: gh: HTTP 500 while writing labels" in captured.err - - -def test_pull_review_comment_nodes_uses_original_commit_or_head_reference(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - head_sha = "a" * 40 - old_sha = "b" * 40 - comment_data = [ - { - "body": "reanchored current-head inline review", - "commit_id": head_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 1, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r1", - "user": {"login": "openai-codex"}, - }, - { - "body": f"stale but mentions head commit {head_sha[:12]}", - "commit_id": head_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 2, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r2", - "user": {"login": "openai-codex"}, - }, - { - "body": "actual current-head inline review", - "commit_id": old_sha, - "original_commit_id": head_sha, - "pull_request_review_id": 3, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r3", - "user": {"login": "openai-codex"}, - }, - { - "body": "older unrelated comment", - "commit_id": old_sha, - "original_commit_id": old_sha, - "pull_request_review_id": 4, - "created_at": "2026-06-11T00:00:00Z", - "html_url": "https://github.com/Soju06/codex-lb/pull/714#discussion_r4", - "user": {"login": "openai-codex"}, - }, - ] - - monkeypatch.setattr(module, "paged_api", lambda _path: comment_data) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: set()) - - nodes = module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha=head_sha) - - assert [node.get("commit", {}).get("oid") for node in nodes] == [head_sha, head_sha, head_sha] - assert [node.get("pullRequestReviewDatabaseId") for node in nodes] == [None, None, 3] - - -def test_head_mentioned_fallback_comment_keeps_timeline_chronology() -> None: - module = load_sync_module() - head_sha = "a" * 40 - review_id = 2 - timeline_nodes = [ - { - "__typename": "PullRequestCommit", - "commit": {"oid": head_sha}, - "committedDate": "2026-06-11T06:30:00Z", - }, - { - "__typename": "PullRequestReview", - "databaseId": review_id, - "author": {"login": "openai-codex"}, - "bodyText": "Reviewed older commit.", - "submittedAt": "2026-06-11T06:32:00Z", - "commit": {"oid": "b" * 40}, - }, - { - "__typename": "IssueComment", - "author": {"login": "openai-codex"}, - "bodyText": "Codex Review: Didn't find any major issues.", - "createdAt": "2026-06-11T06:40:00Z", - }, - ] - comment_nodes = [ - { - "__typename": "PullRequestReviewComment", - "author": {"login": "openai-codex"}, - "bodyText": f"**[P2]** stale finding mentioning {head_sha[:12]}", - "createdAt": "2026-06-11T06:34:00Z", - "commit": {"oid": head_sha}, - "pullRequestReviewDatabaseId": None, - } - ] - - merged = module.merge_review_comment_nodes(timeline_nodes, comment_nodes) - assert [node["__typename"] for node in merged] == [ - "PullRequestCommit", - "PullRequestReview", - "PullRequestReviewComment", - "IssueComment", - ] - - state, node = module.find_current_head_codex_review_state( - merged, - head_sha=head_sha, - allowed_authors={"openai-codex"}, - ) - - assert state == "clean" - assert node is timeline_nodes[-1] - - -def test_unresolved_codex_threads_filter_to_current_head(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - head_sha = "a" * 40 - old_sha = "b" * 40 - - pages = [ - { - "data": { - "repository": { - "pullRequest": { - "reviewThreads": { - "pageInfo": {"hasNextPage": False, "endCursor": None}, - "nodes": [ - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** reanchored current-head finding", - "url": "https://example.invalid/reanchored-current", - "commit": {"oid": head_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** current finding", - "url": "https://example.invalid/current", - "commit": {"oid": head_sha}, - "originalCommit": {"oid": head_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": f"**[P2]** stale fallback for {head_sha[:12]}", - "url": "https://example.invalid/fallback", - "commit": {"oid": old_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P2]** stale old commit finding", - "url": "https://example.invalid/stale", - "commit": {"oid": old_sha}, - "originalCommit": {"oid": old_sha}, - } - ] - }, - }, - { - "isResolved": False, - "isOutdated": False, - "comments": { - "nodes": [ - { - "author": {"login": "openai-codex"}, - "body": "**[P1]** unresolved stale thread without commit metadata", - "url": "https://example.invalid/no-commit-metadata", - "commit": None, - "originalCommit": None, - } - ] - }, - }, - ], - } - } - } - } - } - ] - - monkeypatch.setattr(module, "graphql", lambda *_args, **_kwargs: pages[0]) - - urls = module.unresolved_codex_finding_thread_urls( - "Soju06/codex-lb", - 714, - head_sha=head_sha, - allowed_authors={"openai-codex"}, - ) - - assert urls == ( - "https://example.invalid/reanchored-current", - "https://example.invalid/current", - "https://example.invalid/fallback", - ) - - -def test_resolved_inline_codex_finding_does_not_count_as_review_news( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - - monkeypatch.setattr( - module, - "paged_api", - lambda _path: [ - { - "body": "**P1 Badge** resolved finding", - "commit_id": "a" * 40, - "original_commit_id": "a" * 40, - "pull_request_review_id": 123, - "html_url": "https://github.test/review/resolved", - "created_at": "2026-06-14T00:00:00Z", - "user": {"login": "chatgpt-codex-connector"}, - } - ], - ) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: set()) - - assert module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha="a" * 40) == [] - - -def test_unresolved_inline_codex_finding_counts_as_review_news( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - url = "https://github.test/review/unresolved" - - monkeypatch.setattr( - module, - "paged_api", - lambda _path: [ - { - "body": "**P1 Badge** unresolved finding", - "commit_id": "a" * 40, - "original_commit_id": "a" * 40, - "pull_request_review_id": 123, - "html_url": url, - "created_at": "2026-06-14T00:00:00Z", - "user": {"login": "chatgpt-codex-connector"}, - } - ], - ) - monkeypatch.setattr(module, "unresolved_review_comment_urls", lambda *_args: {url}) - - nodes = module.pull_review_comment_nodes("Soju06/codex-lb", 714, head_sha="a" * 40) - - assert len(nodes) == 1 - assert nodes[0]["url"] == url - - -def _rate_limited_proc() -> Any: - class _Proc: - returncode = 1 - stdout = "" - stderr = "gh: API rate limit exceeded for user ID 34199905 (HTTP 403)" - - return _Proc() - - -def _ok_proc(payload: str = "{}") -> Any: - class _Proc: - returncode = 0 - stdout = payload - stderr = "" - - return _Proc() - - -def _transient_gh_proc() -> Any: - class _Proc: - returncode = 1 - stdout = "" - stderr = "gh: HTTP 503" - - return _Proc() - - -def test_run_gh_switches_to_fallback_token_on_rate_limit(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - calls: list[str] = [] - - def fake_run(command: Any, **kwargs: Any) -> Any: - import os - - calls.append(os.environ["GH_TOKEN"]) - if len(calls) == 1: - return _rate_limited_proc() - return _ok_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - - result = module.run_gh(["api", "/rate-limited-path"]) - - assert result == {} - assert calls == ["primary-token", "fallback-token"] - - -def test_run_gh_retries_transient_read_only_api_failure(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[list[str]] = [] - sleeps: list[float] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - if len(calls) == 1: - return _transient_gh_proc() - return _ok_proc('{"ok": true}') - - monkeypatch.setattr(module.subprocess, "run", fake_run) - monkeypatch.setattr(module.time, "sleep", sleeps.append) - - assert module.run_gh(["api", "/repos/example/project/issues/1/labels"]) == {"ok": True} - assert len(calls) == 2 - assert sleeps == [2.0] - - -def test_run_gh_does_not_retry_mutating_pr_comment(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - calls: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - return _transient_gh_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - monkeypatch.setattr(module.time, "sleep", lambda _: None) - - with pytest.raises(module.GhError): - module.run_gh(["pr", "comment", "1344", "--body", "@codex review"]) - assert len(calls) == 1 - - -def test_run_gh_activates_fallback_without_retrying_identity_sensitive_command( - monkeypatch: pytest.MonkeyPatch, -) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - calls: list[list[str]] = [] - - def fake_run(command: list[str], **kwargs: Any) -> Any: - del kwargs - calls.append(command) - return _rate_limited_proc() - - monkeypatch.setattr(module.subprocess, "run", fake_run) - - with pytest.raises(module.GhError, match="without retrying this identity-sensitive command"): - module.run_gh( - ["api", "--method", "POST", "/repos/example/project/issues/1/comments"], - fallback_retry=False, - ) - - assert len(calls) == 1 - # The fallback still activates so the rest of the run stays alive. - assert module._fallback_token_active is True - import os - - assert os.environ["GH_TOKEN"] == "fallback-token" - - -def test_trigger_codex_review_posts_without_fallback_retry(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - captured_kwargs: list[dict[str, Any]] = [] - - def capture_run_gh(_args: list[str], **kwargs: Any) -> None: - captured_kwargs.append(kwargs) - - monkeypatch.setattr(module, "run_gh", capture_run_gh) - - warnings = module.trigger_codex_review( - decision(module, trigger_codex_review=True, ok_action="keep"), - body="@codex review", - ) - - assert warnings == () - assert len(captured_kwargs) == 1 - assert captured_kwargs[0]["fallback_retry"] is False - - -def test_run_gh_fails_without_distinct_fallback_token(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "primary-token") - - monkeypatch.setattr(module.subprocess, "run", lambda command, **kwargs: _rate_limited_proc()) - - with pytest.raises(module.GhError): - module.run_gh(["api", "/rate-limited-path"]) - - -def test_run_gh_fails_when_fallback_token_is_also_exhausted(monkeypatch: pytest.MonkeyPatch) -> None: - module = load_sync_module() - monkeypatch.setenv("GH_TOKEN", "primary-token") - monkeypatch.setenv("GH_FALLBACK_TOKEN", "fallback-token") - - monkeypatch.setattr(module.subprocess, "run", lambda command, **kwargs: _rate_limited_proc()) - - with pytest.raises(module.GhError): - module.run_gh(["api", "/rate-limited-path"]) From 6ff51cd69c564499e4371ee755eecab8d80b262d Mon Sep 17 00:00:00 2001 From: Kevin Lin <86810837+kevinsslin@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:24:07 +0800 Subject: [PATCH 038/117] fix(proxy): hold fenced hard turns through cooldown (#1739) * fix(proxy): hold fenced hard turns through cooldown * docs: add @kevinsslin as a contributor * fix(proxy): harden operation-fenced cooldown wait * fix(proxy): fail closed on cooldown lease renewal errors --------- Co-authored-by: Darafei Praliaskouski --- .all-contributorsrc | 10 + README.md | 1 + .../proxy/_service/http_bridge/streaming.py | 147 +++++- .../.openspec.yaml | 2 + .../design.md | 41 ++ .../proposal.md | 27 ++ .../specs/responses-api-compat/spec.md | 67 +++ .../tasks.md | 8 + tests/integration/test_proxy_api_extended.py | 60 +++ tests/unit/test_proxy_http_bridge.py | 436 ++++++++++++++++++ 10 files changed, 798 insertions(+), 1 deletion(-) create mode 100644 openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml create mode 100644 openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md create mode 100644 openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md create mode 100644 openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index 02b873a9de..2624725fd9 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1232,6 +1232,16 @@ "code", "test" ] + }, + { + "login": "kevinsslin", + "name": "Kevin Lin", + "avatar_url": "https://avatars.githubusercontent.com/u/86810837?v=4", + "profile": "https://github.com/kevinsslin", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index e7ba0594a1..1d63923e57 100644 --- a/README.md +++ b/README.md @@ -286,6 +286,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e DuyBui
DuyBui
💻 ⚠️ + Kevin Lin
Kevin Lin
💻 ⚠️ diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index c692de0809..1285996fad 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -3642,10 +3642,149 @@ async def retry_precreated_for_idle_recovery( ), ) + def operation_fenced_cooldown_wait_enabled() -> bool: + """Allow a hard turn to wait until its durable fence can arbitrate recovery.""" + return ( + getattr( + _service_get_settings(), + "http_responses_session_bridge_ambiguous_continuation_recovery_mode", + "fail_closed", + ) + in {"server_anchored_replay_once", "server_indefinite_recovery"} + and getattr(_service_get_settings(), "http_responses_session_bridge_operation_ledger_enabled", True) + and request_state.hard_continuity_anchor + and session.durable_session_id is not None + and session.durable_owner_epoch is not None + and request_state.previous_response_id is None + and request_state.response_id is None + and request_state.response_event_count == 0 + ) + def continuity_bound_without_safe_replay() -> bool: """Do not hold a client stream through a cooldown we cannot use.""" return _http_bridge_continuity_bound_without_safe_replay(request_state) and not ( - _http_bridge_server_anchored_replay_enabled(request_state) + _http_bridge_server_anchored_replay_enabled(request_state) or operation_fenced_cooldown_wait_enabled() + ) + + async def wait_through_operation_fenced_startup_cooldown() -> bool: + if session.key.strength != "hard" or not operation_fenced_cooldown_wait_enabled(): + return False + retry_cooldown_seconds = await self._http_bridge_precreated_retry_cooldown_seconds(session) + if retry_cooldown_seconds <= 0: + return False + remaining_budget_seconds = request_deadline - _service_time().monotonic() + if remaining_budget_seconds <= 0: + return False + wait_seconds = min(retry_cooldown_seconds, remaining_budget_seconds) + async with session.pending_lock: + if session.queued_request_count >= queue_limit: + raise ProxyResponseError( + 429, + openai_error( + "bridge_queue_full", + "HTTP responses session bridge queue is full", + error_type="rate_limit_error", + ), + ) + session.queued_request_count += 1 + _log_http_bridge_event( + "wait_operation_fenced_cooldown", + session.key, + account_id=session.account.id, + model=session.request_model, + detail="hard_turn_operation_fence", + cache_key_family=session.key.affinity_kind, + ) + logger.info( + "HTTP bridge waiting through retry-circuit cooldown before durable hard-turn arbitration " + "request_id=%s wait_seconds=%.1f remaining_budget_seconds=%.1f", + request_state.request_id, + wait_seconds, + remaining_budget_seconds, + ) + # No upstream request has been dispatched on this path. After the + # cooldown, normal submission still has to create or claim the + # durable operation fence before response.create can be sent. + try: + current_instance = _service_get_settings().http_responses_session_bridge_instance_id + lease_refresh_interval_seconds = max( + 1.0, + min( + _http_bridge_durable_lease_ttl_seconds() / 3.0, + wait_seconds, + ), + ) + remaining_wait_seconds = wait_seconds + while remaining_wait_seconds > 0: + sleep_seconds = min(remaining_wait_seconds, lease_refresh_interval_seconds) + await asyncio.sleep(sleep_seconds) + remaining_wait_seconds = max(0.0, remaining_wait_seconds - sleep_seconds) + if remaining_wait_seconds <= 0: + break + try: + owner_lookup = await self._durable_bridge.renew_live_session( + session_id=session.durable_session_id, + api_key_id=session.key.api_key_id, + instance_id=current_instance, + owner_epoch=session.durable_owner_epoch, + lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), + latest_turn_state=session.downstream_turn_state, + latest_response_id=None, + ) + except Exception as exc: + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership could not be renewed; retry the request.", + ), + ) from exc + if ( + owner_lookup is None + or owner_lookup.owner_instance_id != current_instance + or owner_lookup.owner_epoch != session.durable_owner_epoch + ): + session.closed = True + session.upstream_control.reconnect_requested = True + session.upstream_control.retire_after_drain = True + raise ProxyResponseError( + 502, + openai_error( + "bridge_continuity_persistence_failed", + "HTTP responses session ownership changed during cooldown; retry the request.", + ), + ) + finally: + async with session.pending_lock: + session.queued_request_count = max(0, session.queued_request_count - 1) + return True + + async def operation_fenced_request_budget_terminal_event() -> str | None: + if not operation_fenced_cooldown_wait_enabled() or _service_time().monotonic() < request_deadline: + return None + await self._release_websocket_request_state_reservation(request_state) + request_state.api_key_reservation = None + if propagate_http_errors: + raise ProxyResponseError( + 503, + openai_error( + "upstream_request_timeout", + "HTTP responses session bridge recovery exceeded the request budget.", + error_type="server_error", + ), + ) + return format_sse_event( + cast( + Mapping[str, JsonValue], + response_failed_event( + "stream_idle_timeout", + "HTTP responses session bridge recovery exceeded the request budget", + response_id=_websocket_downstream_response_id(request_state), + ), + ) ) async def startup_continuity_cooldown_terminal_event() -> str | None: @@ -3716,6 +3855,12 @@ async def startup_continuity_cooldown_terminal_event() -> str | None: ) while True: + budget_terminal_event = await operation_fenced_request_budget_terminal_event() + if budget_terminal_event is not None: + yield budget_terminal_event + return + if await wait_through_operation_fenced_startup_cooldown(): + continue startup_terminal_event = await startup_continuity_cooldown_terminal_event() if startup_terminal_event is not None: yield startup_terminal_event diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md new file mode 100644 index 0000000000..11b8db08ef --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/design.md @@ -0,0 +1,41 @@ +## Context + +Hard turn-state requests can omit `previous_response_id` while still carrying a +real Codex turn-state continuity anchor. Their replay identity is protected by +the durable operation ledger, but the startup cooldown guard runs before +operation registration. It therefore classifies the request as continuity-bound +without safe replay and returns 503 before the ledger can serialize recovery. + +The HTTP response already includes `Retry-After`, and an already-started SSE +failure includes an SSE `retry:` directive. Production telemetry shows Codex +Desktop retrying in milliseconds anyway, so another client hint does not address +the observed failure mode. + +## Decision + +Treat a turn-state-only hard request as eligible to wait through cooldown only +when all of the following hold: + +- recovery mode is `server_anchored_replay_once` or + `server_indefinite_recovery`; +- the durable operation ledger is enabled; +- the request has a real hard continuity anchor; +- the bridge has both a durable session id and current owner epoch; +- no response id or upstream response event has been observed; and +- request budget remains. + +The wait is clamped to the smaller of cooldown remaining and request budget. +It does not reserve a replay, mutate the operation journal, or send upstream. +When the cooldown expires, normal submission performs the existing operation +fingerprint lookup and atomic recovery claim. One-shot mode keeps its existing +maximum of one recovery dispatch; indefinite mode retains its existing explicit +opt-in semantics. + +## Explicit exclusions + +- No change to the default `fail_closed` mode. +- No transparent replay without a durable session and owner fence. +- No cross-account, file-pinned, image, soft-affinity, or eventful recovery. +- No weakening of operation fingerprint, ownership, or replay-count checks. +- No infinite retry added by this change; bounded one-shot mode is the + recommended deployment setting for this incident class. diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md new file mode 100644 index 0000000000..5c6d83bd73 --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/proposal.md @@ -0,0 +1,27 @@ +## Why + +When two eventless upstream attempts open the HTTP bridge retry circuit, Codex +Desktop immediately retries the same hard turn-state request. The bridge +currently returns a startup 503 before consulting the durable operation ledger. +Codex does not honor the full retry-circuit delay and can exhaust its client +retry budget during the cooldown, pausing the task even though the bridge and +VPS remain healthy. + +## What Changes + +- In an explicitly enabled server recovery mode, hold a turn-state-only hard + continuation through the active retry-circuit cooldown before submission. +- Require a live durable session id and owner epoch, zero response events, and + no response id before waiting. +- Dispatch nothing while waiting. After cooldown, use the existing durable + operation ledger and one-shot/indefinite recovery policy to arbitrate whether + the request may be created, claimed, replayed, or failed closed. +- Preserve the current immediate 503 for the default `fail_closed` mode, + in-memory fallback sessions, soft affinity, and eventful requests. +- Emit a low-cardinality bridge event when the operation-fenced wait begins. + +## Impact + +- HTTP Responses bridge startup behavior during retry-circuit cooldown. +- No database schema, public API, account routing, or default configuration + change. diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..a487f23e1f --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/specs/responses-api-compat/spec.md @@ -0,0 +1,67 @@ +## ADDED Requirements + +### Requirement: Operation-fenced hard turns preserve client retry budget during cooldown + +A hard turn-state HTTP bridge request arriving during retry-circuit cooldown MUST remain pending until cooldown expires only if an explicit server recovery mode is enabled, the request has not observed a response id or response event, and the bridge has a live durable session and owner epoch. The proxy MUST NOT dispatch upstream while waiting. After the wait, the request MUST pass through the existing durable operation-ledger admission before any `response.create` is sent. + +#### Scenario: One-shot hard turn waits before durable arbitration + +- **GIVEN** `server_anchored_replay_once` is enabled +- **AND** a turn-state-only hard continuation has a live durable owner +- **AND** its retry circuit is cooling down before submission +- **WHEN** the request reaches bridge startup +- **THEN** the proxy waits for the bounded cooldown instead of returning 503 +- **AND** it sends no upstream request during the wait +- **AND** normal durable operation admission runs after cooldown + +#### Scenario: Missing durable fence remains fail closed + +- **GIVEN** a turn-state-only hard continuation has no durable session or owner + epoch +- **WHEN** its retry circuit is cooling down +- **THEN** the proxy does not wait or dispatch upstream +- **AND** it returns the existing cooldown failure with a retry hint + +#### Scenario: Operation ledger disabled remains fail closed + +- **GIVEN** ambiguous continuation recovery mode is enabled +- **AND** a turn-state-only hard continuation has a live durable session and + owner epoch +- **AND** the durable operation ledger is disabled +- **WHEN** its retry circuit is cooling down before submission +- **THEN** the proxy preserves the existing cooldown failure +- **AND** it does not wait or dispatch upstream + +#### Scenario: Default mode remains fail closed + +- **GIVEN** ambiguous continuation recovery mode is `fail_closed` +- **WHEN** any continuity-bound hard request arrives during cooldown +- **THEN** the proxy preserves the existing immediate cooldown failure +- **AND** it does not create or claim a durable recovery operation + +#### Scenario: Request budget expires while waiting + +- **GIVEN** an operation-fenced hard turn is allowed to wait through cooldown +- **AND** its request budget expires before the cooldown does +- **WHEN** the bounded wait reaches the request deadline +- **THEN** the proxy releases the request reservation and returns a terminal + timeout +- **AND** it does not submit `response.create` after the deadline + +#### Scenario: Cooldown waiter stays within the per-session queue limit + +- **GIVEN** an operation-fenced hard turn is eligible to wait through cooldown +- **AND** the bridge session is already at its configured queue limit +- **WHEN** the request reaches the cooldown wait point before submission +- **THEN** the proxy rejects the request with the existing bridge queue full + error +- **AND** it does not sleep or dispatch upstream + +#### Scenario: Durable ownership is renewed while the cooldown wait is pending + +- **GIVEN** an operation-fenced hard turn is waiting through startup cooldown +- **AND** the cooldown exceeds one durable lease refresh cadence +- **WHEN** the wait continues before submission +- **THEN** the proxy renews and revalidates the durable owner lease before the + wait completes +- **AND** it fails closed if durable ownership changes during the wait diff --git a/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md new file mode 100644 index 0000000000..3f7ccce4ae --- /dev/null +++ b/openspec/changes/hold-operation-fenced-hard-turn-cooldown/tasks.md @@ -0,0 +1,8 @@ +- [x] 1. Reproduce the production turn-state-only startup cooldown as a unit + regression that currently returns 503 before submission. +- [x] 2. Hold only explicitly enabled, zero-event, durable operation-fenced hard + turns through the bounded cooldown. +- [x] 3. Preserve fail-closed behavior when the durable session/owner proof is + absent and keep one-shot recovery bounded by the existing atomic claim. +- [x] 4. Run focused tests, relevant bridge suites, Ruff, type/architecture + checks, whitespace checks, and strict OpenSpec validation. diff --git a/tests/integration/test_proxy_api_extended.py b/tests/integration/test_proxy_api_extended.py index 99431da8fe..a52bfce166 100644 --- a/tests/integration/test_proxy_api_extended.py +++ b/tests/integration/test_proxy_api_extended.py @@ -2965,6 +2965,66 @@ async def stream_responses(self, *args, **kwargs): assert any("response.completed" in chunk for chunk in chunks) +@pytest.mark.asyncio +async def test_codex_route_stream_responses_keeps_client_alive_while_bridge_cooldown_delays_first_event( + monkeypatch, +): + upstream_started = asyncio.Event() + release_upstream = asyncio.Event() + + class _FakeService: + async def rate_limit_headers(self): + return {} + + async def stream_responses(self, *args, **kwargs): + del args, kwargs + upstream_started.set() + _signal_propagated_capacity_startup_ready() + await release_upstream.wait() + yield _sse_event({"type": "response.in_progress", "response": {"id": "resp_cooldown_wait"}}) + yield _sse_event({"type": "response.completed", "response": {"id": "resp_cooldown_wait"}}) + + settings = SimpleNamespace( + http_responses_session_bridge_enabled=False, + sse_keepalive_interval_seconds=0.01, + proxy_account_stream_recovery_reserve=1, + proxy_api_key_fair_share_congestion_threshold_pct=0, + ) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_api_module.proxy_service_module, "get_settings", lambda: settings) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/backend-api/codex/responses", + "headers": [], + } + ) + payload = proxy_api_module.ResponsesRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True} + ) + + response = await proxy_api_module._stream_responses( + request, + payload, + ProxyContext(service=cast(proxy_module.ProxyService, _FakeService())), + api_key=None, + enforce_openai_sdk_contract=False, + ) + + assert isinstance(response, StreamingResponse) + assert upstream_started.is_set() is True + iterator = response.body_iterator.__aiter__() + first_chunk = await asyncio.wait_for(iterator.__anext__(), timeout=0.2) + assert first_chunk == CODEX_KEEPALIVE_FRAME + release_upstream.set() + second_chunk = cast(str, await asyncio.wait_for(iterator.__anext__(), timeout=0.2)) + third_chunk = cast(str, await asyncio.wait_for(iterator.__anext__(), timeout=0.2)) + assert "response.in_progress" in second_chunk + assert "response.completed" in third_chunk + + @pytest.mark.asyncio async def test_proxy_stream_retries_rate_limit_then_success(async_client, monkeypatch): expected_account_id_1 = await _import_account(async_client, "acc_1", "one@example.com") diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 761b631131..9d4ea02384 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -6046,6 +6046,442 @@ async def test_http_bridge_startup_cooldown_releases_api_key_reservation( assert request_state.api_key_reservation is None +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_waits_through_startup_cooldown( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-cooldown-wait") + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-cooldown-wait", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-hard-anchor", + hard_continuity_anchor=True, + ) + session.durable_session_id = "durable-hard-turn-cooldown-wait" + session.durable_owner_epoch = 7 + cooldown = AsyncMock(side_effect=[0.01, 0.0]) + submit = AsyncMock(side_effect=RuntimeError("submitted after cooldown")) + sleeps: list[float] = [] + + async def sleep(delay: float) -> None: + sleeps.append(delay) + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(RuntimeError, match="submitted after cooldown"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-hard-anchor", + ): + pass + + assert sleeps == [pytest.approx(0.01)] + assert cooldown.await_count == 2 + submit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_http_bridge_previous_response_anchor_bypasses_hard_turn_cooldown_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-anchored-replay-cooldown-bypass") + session.durable_session_id = "durable-anchored-replay-cooldown-bypass" + session.durable_owner_epoch = 8 + request_state = proxy_service._WebSocketRequestState( + request_id="req-anchored-replay-cooldown-bypass", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + previous_response_id="resp-anchor-before-cooldown", + hard_continuity_anchor=True, + ) + cooldown = AsyncMock(return_value=30.0) + submit = AsyncMock(side_effect=RuntimeError("submitted without cooldown wait")) + sleep = AsyncMock() + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(RuntimeError, match="submitted without cooldown wait"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create","previous_response_id":"resp-anchor-before-cooldown"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state=None, + ): + pass + + cooldown.assert_not_awaited() + sleep.assert_not_awaited() + submit.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_without_durable_fence_fails_closed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-no-durable-fence") + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-no-durable-fence", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-without-durable-fence", + hard_continuity_anchor=True, + ) + submit = AsyncMock() + sleep = AsyncMock() + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-without-durable-fence", + ): + pass + + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + sleep.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_requires_operation_ledger_for_cooldown_wait( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-no-ledger") + session.durable_session_id = "durable-hard-turn-no-ledger" + session.durable_owner_epoch = 11 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-no-ledger", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-no-ledger", + hard_continuity_anchor=True, + ) + submit = AsyncMock() + sleep = AsyncMock() + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_operation_ledger_enabled=False, + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-no-ledger", + ): + pass + + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + sleep.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_cooldown_wait_rejects_when_queue_is_full( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-cooldown-queue-full", queued_request_count=8) + session.durable_session_id = "durable-hard-turn-cooldown-queue-full" + session.durable_owner_epoch = 12 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-cooldown-queue-full", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-cooldown-queue-full", + hard_continuity_anchor=True, + ) + submit = AsyncMock() + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-cooldown-queue-full", + ): + pass + + assert exc_info.value.status_code == 429 + assert exc_info.value.payload["error"]["code"] == "bridge_queue_full" + submit.assert_not_awaited() + assert session.queued_request_count == 8 + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_renews_durable_lease_while_waiting( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-renew-wait") + session.durable_session_id = "durable-hard-turn-renew-wait" + session.durable_owner_epoch = 13 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-renew-wait", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-renew-wait", + hard_continuity_anchor=True, + ) + renew_live_session = AsyncMock( + return_value=SimpleNamespace( + owner_instance_id="instance-hard-turn-renew", + owner_epoch=13, + ) + ) + service._durable_bridge = cast(Any, SimpleNamespace(renew_live_session=renew_live_session)) + cooldown = AsyncMock(side_effect=[25.0, 0.0]) + submit = AsyncMock(side_effect=RuntimeError("submitted after renewed cooldown")) + slept: list[float] = [] + + async def sleep(delay: float) -> None: + slept.append(delay) + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_instance_id="instance-hard-turn-renew", + ), + ) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", cooldown) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(RuntimeError, match="submitted after renewed cooldown"): + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-renew-wait", + ): + pass + + assert slept == [pytest.approx(10.0), pytest.approx(10.0), pytest.approx(5.0)] + assert renew_live_session.await_count == 2 + submit.assert_awaited_once() + assert session.queued_request_count == 0 + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_fails_closed_when_lease_renewal_raises( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-renew-failure") + session.durable_session_id = "durable-hard-turn-renew-failure" + session.durable_owner_epoch = 14 + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-renew-failure", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-renew-failure", + hard_continuity_anchor=True, + ) + renew_live_session = AsyncMock(side_effect=RuntimeError("durable store unavailable")) + service._durable_bridge = cast(Any, SimpleNamespace(renew_live_session=renew_live_session)) + submit = AsyncMock() + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + http_responses_session_bridge_instance_id="instance-hard-turn-renew-failure", + ), + ) + monkeypatch.setattr( + service, + "_http_bridge_precreated_retry_cooldown_seconds", + AsyncMock(return_value=25.0), + ) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", AsyncMock()) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-renew-failure", + ): + pass + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "bridge_continuity_persistence_failed" + renew_live_session.assert_awaited_once() + submit.assert_not_awaited() + assert session.closed is True + assert session.upstream_control.reconnect_requested is True + assert session.upstream_control.retire_after_drain is True + assert session.queued_request_count == 0 + + +@pytest.mark.asyncio +async def test_http_bridge_one_shot_hard_turn_does_not_submit_after_wait_budget( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="sid-hard-turn-wait-budget") + session.durable_session_id = "durable-hard-turn-wait-budget" + session.durable_owner_epoch = 9 + reservation = cast(Any, object()) + request_state = proxy_service._WebSocketRequestState( + request_id="req-hard-turn-wait-budget", + model="gpt-5.6", + service_tier=None, + reasoning_effort=None, + api_key_reservation=reservation, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + session_id="turn-state-wait-budget", + hard_continuity_anchor=True, + ) + clock = SimpleNamespace(now=100.0) + submit = AsyncMock() + release = AsyncMock() + + async def sleep(delay: float) -> None: + clock.now += delay + + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: _make_app_settings( + http_responses_session_bridge_ambiguous_continuation_recovery_mode="server_anchored_replay_once", + ), + ) + monkeypatch.setattr(http_bridge_streaming_module._service_time(), "monotonic", lambda: clock.now) + monkeypatch.setattr(service, "_http_bridge_precreated_retry_cooldown_seconds", AsyncMock(return_value=30.0)) + monkeypatch.setattr(service, "_submit_http_bridge_request", submit) + monkeypatch.setattr(service, "_release_websocket_request_state_reservation", release) + monkeypatch.setattr(http_bridge_streaming_module.asyncio, "sleep", sleep) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_http_bridge_session_events( + session, + request_state=request_state, + text_data='{"type":"response.create"}', + queue_limit=8, + propagate_http_errors=True, + downstream_turn_state="turn-state-wait-budget", + request_deadline=105.0, + ): + pass + + assert exc_info.value.status_code == 503 + assert exc_info.value.payload["error"]["code"] == "upstream_request_timeout" + submit.assert_not_awaited() + release.assert_awaited_once_with(request_state) + assert request_state.api_key_reservation is None + + @pytest.mark.asyncio async def test_http_bridge_replay_detach_releases_reservation_without_pending_ownership( monkeypatch: pytest.MonkeyPatch, From 57618c87528aaaac4fecc4c6ad57dd07fbfa108b Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 16 Aug 2026 12:24:10 +0400 Subject: [PATCH 039/117] fix(proxy): stop abandoning an unresolved inflight session-creation future (#1644) * fix(http-bridge): avoid abandoned inflight session future * test(http-bridge): cover inflight cleanup through responses route * test(http-bridge): collect inflight reuse regression * test(http-bridge): accept optional session creation kwargs * fix(api-keys): retain durability compatibility hook * fix(api-keys): drop stale last-used durability hook * fix(websocket): shield response-create lease release * fix(websocket): finish shielded lease cleanup * fix(websocket): collect cancellation cleanup coverage * fix(websocket): satisfy lease cleanup type checks * fix: preserve websocket gate cancellation cleanup * fix: drop stale durability drift from folded codex-lb branch * test(http-bridge): prove anchored inflight reuse * test(http-bridge): force registered-anchor reuse path --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- .../proxy/_service/http_bridge/mixin.py | 2 +- .../proxy/_service/websocket/helpers.py | 28 +++- .../.openspec.yaml | 2 + .../fix-inflight-future-abandoned/design.md | 27 ++++ .../fix-inflight-future-abandoned/proposal.md | 25 ++++ .../specs/responses-api-compat/spec.md | 51 ++++++++ .../fix-inflight-future-abandoned/tasks.md | 10 ++ .../design.md | 7 + .../proposal.md | 13 ++ .../specs/proxy-admission-control/spec.md | 16 +++ .../tasks.md | 6 + .../integration/test_http_responses_bridge.py | 120 ++++++++++++++++-- tests/unit/test_proxy_security_work.py | 71 +++++++++++ tests/unit/test_proxy_utils.py | 51 ++++++++ 14 files changed, 414 insertions(+), 15 deletions(-) create mode 100644 openspec/changes/fix-inflight-future-abandoned/.openspec.yaml create mode 100644 openspec/changes/fix-inflight-future-abandoned/design.md create mode 100644 openspec/changes/fix-inflight-future-abandoned/proposal.md create mode 100644 openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/fix-inflight-future-abandoned/tasks.md create mode 100644 openspec/changes/fix-websocket-response-create-lease-cancellation/design.md create mode 100644 openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md create mode 100644 openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md create mode 100644 openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index b7593607aa..d628dd1a86 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -1293,7 +1293,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: model_class=_extract_model_class(request_model) if request_model else None, owner_check_applied=owner_check_required, ) - elif inflight_future is None: + elif session_to_return_after_close is None and inflight_future is None: # Detached generations remain globally capacity-owned # until close finalization. This request may discount # only the idle generations it has committed to close diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 3a0ee3d4c7..1d4eec183f 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -5,7 +5,7 @@ import sys import time from collections import deque -from collections.abc import Sequence +from collections.abc import Awaitable, Sequence from dataclasses import dataclass from typing import Any, cast @@ -1661,6 +1661,7 @@ async def _release_websocket_response_create_gate( request_state: _WebSocketRequestState, response_create_gate: asyncio.Semaphore, ) -> None: + cancellation: asyncio.CancelledError | None = None account_response_create_lease = request_state.account_response_create_lease account_response_create_release = request_state.account_response_create_release request_state.account_response_create_lease = None @@ -1669,13 +1670,36 @@ async def _release_websocket_response_create_gate( request_state.response_create_admission.release() request_state.response_create_admission = None if account_response_create_lease is not None and account_response_create_release is not None: - await account_response_create_release(account_response_create_lease) + cancellation = await _await_cleanup_deferring_cancellation( + account_response_create_release(account_response_create_lease) + ) request_state.awaiting_response_created = False request_state.response_create_gate = None if not request_state.response_create_gate_acquired: + if cancellation is not None: + raise cancellation return request_state.response_create_gate_acquired = False response_create_gate.release() + if cancellation is not None: + raise cancellation + + +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> asyncio.CancelledError | None: + """Finish response-create lease cleanup before propagating cancellation.""" + + task = asyncio.ensure_future(awaitable) + cancellation: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + break + except asyncio.CancelledError as exc: + cancellation = cancellation or exc + if task.cancelled(): + raise + return cancellation def _pop_terminal_websocket_request_state( diff --git a/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml b/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml new file mode 100644 index 0000000000..84cfc12459 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-06 diff --git a/openspec/changes/fix-inflight-future-abandoned/design.md b/openspec/changes/fix-inflight-future-abandoned/design.md new file mode 100644 index 0000000000..77f0f2602e --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/design.md @@ -0,0 +1,27 @@ +## Context + +The bridge lookup loop first resolves a reusable previous-response session and records it in `session_to_return_after_close`. The generic create arm is selected independently by `inflight_future is None`, so it can register a new pending future for the already-resolved key before the function returns the reused session. The existing cleanup/janitor intentionally removes only completed futures and is covered by a unit test. + +## Goals / Non-Goals + +**Goals:** + +- Make the reuse decision terminal for session creation in that loop. +- Leave no unresolved future registered for a key whose existing session is returned. +- Preserve all create, waiter, handoff, timeout, and janitor behavior for paths that do not reuse a previous-response session. + +**Non-Goals:** + +- Do not change janitor eligibility or restart-blocking semantics for genuinely live creation futures. +- Do not redesign durable ownership, session closing, or response routing. + +## Decisions + +Guard the generic `inflight_future is None` creation arm with `session_to_return_after_close is None`. This is the smallest local invariant: once reuse has selected a session, the loop may still close detached sessions, then returns the selected session without publishing a creation future. An early `continue` or future resolution would add lifecycle behavior without benefit and could interfere with the existing create-chain arms. + +The regression uses the real `_get_or_create_http_bridge_session` previous-response lookup path and asserts both registry state and a second successful reuse. The existing janitor test remains unchanged as a negative control. + +## Risks / Trade-offs + +- [Risk] A future branch might set `session_to_return_after_close` for a case that still needs creation. → Mitigation: the symbol has one assignment, in the validated live-session reuse arm; all other arms leave it `None` and retain the original create condition. +- [Risk] A future remains from an earlier concurrent creator. → Mitigation: reuse already requires the canonical previous-key inflight lookup to be empty; waiter behavior remains guarded by `inflight_future is not None`. diff --git a/openspec/changes/fix-inflight-future-abandoned/proposal.md b/openspec/changes/fix-inflight-future-abandoned/proposal.md new file mode 100644 index 0000000000..80e65c5e21 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/proposal.md @@ -0,0 +1,25 @@ +## Why + +The HTTP bridge's previous-response reuse path can return an already-live session after first publishing a new, unresolved session-creation future for that same anchor. That orphaned future permanently marks the bridge as restart-blocking and makes later requests fail with a continuity 502, so the create chain must not run when reuse has already selected a session. + +## What Changes + +- Prevent the generic HTTP bridge session-creation arm from publishing an inflight future when the previous-response path has selected an existing session for return. +- Preserve the existing done-future janitor contract and all other session-creation and handoff arms. +- Keep regression coverage for both registry cleanup and a successful second request on the same previous-response anchor. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: HTTP bridge previous-response reuse must not leave an unresolved session-creation future registered for the reused anchor. + +## Impact + +- Affected code: `app/modules/proxy/_service/http_bridge/mixin.py` session lookup/create chain. +- Affected tests: focused HTTP bridge bughunt regression and existing unit/integration bridge suites. +- No API schema, persistence, or janitor behavior changes. diff --git a/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md b/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..76ed8af65e --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/specs/responses-api-compat/spec.md @@ -0,0 +1,51 @@ +## MODIFIED Requirements + +### Requirement: Continuity-dependent Responses follow-ups fail closed with retryable errors +When a Responses follow-up depends on previously established continuity state, the service MUST return a retryable continuity error if that continuity cannot be reconstructed safely. The service MUST NOT expose raw `previous_response_not_found` for bridge-local metadata loss or similar internal continuity gaps. When forwarding a turn-state-anchored follow-up to its bridge owner fails with `bridge_owner_unreachable` and a fresh durable lookup shows the owner no longer holds an active lease (released, expired, or the row is missing or CLOSED), the service MUST recover the follow-up locally through durable takeover instead of returning the retryable error. The fresh durable lookup MUST use the same resolution semantics as request routing, including the latest-turn-state fallback, so a row originally resolved without a registered alias remains takeover-eligible. When the durable lease is still actively held by another instance — including DRAINING rows whose lease has not been released or expired — the service MUST keep failing closed with the retryable error. + +#### Scenario: HTTP bridge loses local continuity metadata for a follow-up request +- **WHEN** an HTTP `/v1/responses` or `/backend-api/codex/responses` follow-up request depends on `previous_response_id` or a hard continuity turn-state +- **AND** the bridge cannot reconstruct the matching live continuity state from local or durable metadata +- **THEN** the service returns a retryable OpenAI-format error +- **AND** the error code is not `previous_response_not_found` + +#### Scenario: in-flight bridge follower loses continuity while waiting on the same canonical session +- **WHEN** a follow-up request waits on an in-flight HTTP bridge session for the same hard continuity key +- **AND** the bridge still cannot reconstruct safe continuity state once the leader finishes +- **THEN** the service returns a retryable OpenAI-format error +- **AND** the error code is not `previous_response_not_found` + +#### Scenario: multiplexed follow-ups fail closed only for the matching continuity anchor +- **WHEN** a websocket or HTTP bridge session has multiple pending follow-up requests with different `previous_response_id` anchors +- **AND** continuity loss is detected for exactly one of those anchors +- **THEN** the service applies the retryable fail-closed continuity error only to the matching follow-up request +- **AND** it does not expose raw `previous_response_not_found` +- **AND** unrelated pending requests continue on their own response lifecycle + +#### Scenario: multiplexed follow-ups sharing one anchor fail closed together without leaking raw continuity errors +- **WHEN** a websocket or HTTP bridge session has multiple pending follow-up requests that share the same `previous_response_id` anchor +- **AND** upstream emits an anonymous continuity loss event such as `previous_response_not_found` for that shared anchor +- **THEN** the service rewrites each affected follow-up into a retryable continuity error +- **AND** no affected follow-up exposes raw `previous_response_not_found` +- **AND** the run remains usable for subsequent requests after the rewritten failures + +#### Scenario: single pre-created follow-up still fails closed when continuity loss omits explicit response id in message +- **WHEN** a websocket follow-up request is pending with `previous_response_id` and has not received a stable upstream `response.id` yet +- **AND** upstream emits `previous_response_not_found` with `param=previous_response_id` +- **AND** the upstream error message omits the literal previous response identifier +- **THEN** the service still maps that continuity loss to the pending follow-up +- **AND** it rewrites the downstream terminal event to a retryable continuity error +- **AND** it does not surface raw `previous_response_not_found` to the client + +#### Scenario: turn-state follow-up recovers locally after the owner released its lease +- **WHEN** a turn-state-anchored follow-up without `previous_response_id` is forwarded to its bridge owner during the post-shutdown ring grace window +- **AND** the forward fails with `bridge_owner_unreachable` +- **AND** a fresh durable lookup using the request-routing resolution semantics (registered alias or latest-turn-state fallback) shows the lease is released or expired +- **THEN** the service retries the follow-up locally through durable takeover instead of returning the retryable 503 +- **AND** the takeover retry carries the fresh durable lookup as its continuity anchor even when the turn-state alias registration was lost +- **AND** a fresh durable lookup showing a live lease held by another instance — even for a DRAINING row — still fails closed with the retryable `bridge_owner_unreachable` error + +#### Scenario: previous-response reuse does not register an abandoned creation future +- **WHEN** an HTTP bridge request resolves a live compatible session through `previous_response_id` +- **THEN** the bridge returns that existing session without registering an unresolved inflight session-creation future for its canonical key +- **AND** a subsequent request on the same previous-response anchor can reuse the session successfully diff --git a/openspec/changes/fix-inflight-future-abandoned/tasks.md b/openspec/changes/fix-inflight-future-abandoned/tasks.md new file mode 100644 index 0000000000..9bf866d275 --- /dev/null +++ b/openspec/changes/fix-inflight-future-abandoned/tasks.md @@ -0,0 +1,10 @@ +## 1. Implementation + +- [x] 1.1 Guard the generic HTTP bridge session-creation arm so a previous-response reuse selection cannot publish an inflight future. +- [x] 1.2 Keep the existing janitor and all non-reuse create-chain arms unchanged. + +## 2. Verification + +- [x] 2.1 Run the F1 bughunt regression and confirm it fails on the baseline and passes after the fix, including the second-request reuse assertion. +- [x] 2.2 Run the HTTP bridge unit and integration suites, including the existing live-inflight janitor test. +- [x] 2.3 Validate OpenSpec artifacts and inspect the final diff/status before committing. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md new file mode 100644 index 0000000000..796035a099 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/design.md @@ -0,0 +1,7 @@ +# Design + +`_release_websocket_response_create_gate` keeps its existing state-clearing and +gate-release ordering, but awaits the captured account lease release through +`asyncio.shield`. The release operation therefore continues after cancellation +of the surrounding WebSocket task, returning the account slot without changing +the existing response-create gate semantics. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md new file mode 100644 index 0000000000..1427bfa9ef --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/proposal.md @@ -0,0 +1,13 @@ +# Change: Make WebSocket response-create lease cleanup cancellation-safe + +## Why + +WebSocket terminal cleanup clears the request state's account response-create +lease before awaiting its asynchronous release. Cancellation at that await can +leave the account slot counted until stale-lease reclamation. + +## What Changes + +- Shield the account response-create lease release in WebSocket gate cleanup. +- Add regression coverage for cancellation under load-balancer runtime-lock + contention and retain coverage for genuine stale-lease reclamation. diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..344b5c9470 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/specs/proxy-admission-control/spec.md @@ -0,0 +1,16 @@ +## ADDED Requirements + +### Requirement: WebSocket response-create lease cleanup is cancellation-safe + +When WebSocket terminal cleanup has captured an account response-create lease, it MUST complete the asynchronous lease release even if the surrounding task is cancelled while waiting for the load-balancer runtime lock. Cleanup MUST retain the existing response-create gate release semantics. + +#### Scenario: Cancellation under lease-release contention returns the account slot + +- **GIVEN** a WebSocket request owns an account response-create lease and its + response-create gate +- **AND** the load-balancer runtime lock is held by another task +- **WHEN** terminal cleanup is cancelled while releasing the account lease +- **THEN** the account response-create slot MUST be returned after the lock is + freed +- **AND** the request state does not retain the released lease +- **AND** the response-create gate cleanup semantics remain unchanged diff --git a/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md b/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md new file mode 100644 index 0000000000..84d999b223 --- /dev/null +++ b/openspec/changes/fix-websocket-response-create-lease-cancellation/tasks.md @@ -0,0 +1,6 @@ +# Tasks + +- [x] Make WebSocket response-create lease release cancellation-safe. +- [x] Add cancellation and stale-reclaim regression coverage. +- [x] Run targeted WebSocket, HTTP bridge, and load-balancer lease tests. +- [x] Validate the OpenSpec documents. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index fa108d04dc..bdc858dc77 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -731,6 +731,7 @@ class _AnonymousPreviousResponseNotFoundWithInflightUpstreamWebSocket(_FakeBridg def __init__(self) -> None: super().__init__() self.first_request_created = asyncio.Event() + self._anchored_followup_failed = False async def send_text(self, text: str) -> None: self.sent_text.append(text) @@ -756,6 +757,57 @@ async def send_text(self, text: str) -> None: payload = json.loads(text) previous_response_id = payload.get("previous_response_id") + if self._anchored_followup_failed: + response_id = f"{self.response_id_prefix}_{len(self.sent_text)}" + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.created", + "response": { + "id": response_id, + "object": "response", + "status": "in_progress", + }, + }, + separators=(",", ":"), + ), + ) + ) + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "response.completed", + "response": { + "id": response_id, + "object": "response", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "OK"}], + } + ], + "usage": { + "input_tokens": 24, + "output_tokens": 2, + "total_tokens": 26, + "input_tokens_details": {"cached_tokens": 20}, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + }, + separators=(",", ":"), + ), + ) + ) + return + + self._anchored_followup_failed = True await self._messages.put( _FakeUpstreamMessage( "text", @@ -10688,6 +10740,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -10789,6 +10842,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -10918,6 +10972,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11038,6 +11093,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11139,6 +11195,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11486,6 +11543,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11599,6 +11657,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11732,6 +11791,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11824,6 +11884,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -11919,6 +11980,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -12014,6 +12076,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -12111,6 +12174,7 @@ async def fake_create_http_bridge_session( preferred_account_id=None, require_preferred_account=False, fallback_on_preferred_account_unavailable=True, + **_kwargs, ): del ( self, @@ -13791,7 +13855,8 @@ async def test_v1_responses_http_bridge_masks_anonymous_previous_response_not_fo monkeypatch, ): _install_bridge_settings(monkeypatch, enabled=True) - upstream = _AnonymousPreviousResponseNotFoundWithInflightUpstreamWebSocket() + service = get_proxy_service_for_app(app_instance) + upstream = _TwoSameAnchorFollowupsPreviousResponseNotFoundUpstreamWebSocket() connect_count = 0 async def fake_select_account_with_budget( @@ -13859,6 +13924,8 @@ async def fake_connect_responses_websocket( AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as admin_client, AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as first_client, AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as second_client, + AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as third_client, + AsyncClient(transport=ASGITransport(app=app_instance), base_url="http://testserver") as fourth_client, ): account_id = await _import_account( admin_client, @@ -13867,28 +13934,39 @@ async def fake_connect_responses_websocket( ) account = await _get_account(account_id) + anchor_response = await first_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-seed", + "prompt_cache_key": "previous-response-anchor-seed", + }, + ) + first = asyncio.create_task( - first_client.post( + second_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello", - "prompt_cache_key": "previous-response-inflight-mixed", + "input": "hello-a", + "prompt_cache_key": "previous-response-inflight-origin", + "previous_response_id": anchor_response.json()["id"], }, ) ) - await _wait_for_event(upstream.first_request_created) + await _wait_for_event(upstream.first_followup_created) second = asyncio.create_task( - second_client.post( + third_client.post( "/v1/responses", json={ "model": "gpt-5.1", "instructions": "Return exactly OK.", - "input": "hello-again", - "prompt_cache_key": "previous-response-inflight-mixed", - "previous_response_id": "resp_bridge_prev_anchor", + "input": "hello-b", + "prompt_cache_key": "previous-response-inflight-anchor", + "previous_response_id": anchor_response.json()["id"], }, ) ) @@ -13898,13 +13976,31 @@ async def fake_connect_responses_websocket( timeout=_TEST_SYNC_TIMEOUT_SECONDS, ) - assert first_response.status_code == 200 - assert first_response.json()["output"][0]["content"][0]["text"] == "OK" + third_response = await fourth_client.post( + "/v1/responses", + json={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "hello-on-anchor-again", + "prompt_cache_key": "previous-response-after-error", + }, + ) + + assert not any(not future.done() for future in service._http_bridge_inflight_sessions.values()) + + assert anchor_response.status_code == 200 + assert anchor_response.json()["output"][0]["content"][0]["text"] == "OK" + assert first_response.status_code >= 400 + assert first_response.json()["error"]["code"] == "stream_incomplete" assert second_response.status_code >= 400 assert second_response.json()["error"]["code"] == "stream_incomplete" + assert "previous_response_not_found" not in first_response.json()["error"].get("code", "") + assert "previous_response_not_found" not in first_response.json()["error"].get("message", "") assert "previous_response_not_found" not in second_response.json()["error"].get("code", "") assert "previous_response_not_found" not in second_response.json()["error"].get("message", "") - assert connect_count == 1 + assert third_response.status_code == 200 + assert third_response.json()["output"][0]["content"][0]["text"] == "OK" + assert connect_count == 2 @pytest.mark.asyncio diff --git a/tests/unit/test_proxy_security_work.py b/tests/unit/test_proxy_security_work.py index d419686792..b2f61fd45f 100644 --- a/tests/unit/test_proxy_security_work.py +++ b/tests/unit/test_proxy_security_work.py @@ -88,6 +88,77 @@ async def test_process_websocket_security_retry_releases_response_create_gate() gate.release() +@pytest.mark.asyncio +async def test_websocket_security_cleanup_finishes_after_cancellation() -> None: + """The externally exercised WebSocket path cannot orphan a response lease.""" + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_security_gate_cancel") + gate = asyncio.Semaphore(1) + await gate.acquire() + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_security_gate_cancel", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + transport="websocket", + request_text='{"type":"response.create","model":"gpt-5.1","input":[]}', + ) + request_state.response_create_gate = gate + request_state.response_create_gate_acquired = True + lease = proxy_service.AccountLease("lease-ws-security-gate-cancel", account.id, "response_create", 1.0) + release_started = asyncio.Event() + release_finished = asyncio.Event() + + async def release_account_lease(value): + assert value is lease + release_started.set() + await release_finished.wait() + + request_state.account_response_create_lease = lease + request_state.account_response_create_release = release_account_lease + pending_requests = deque([request_state]) + upstream_control = proxy_service._WebSocketUpstreamControl() + text = json.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_ws_security_gate_cancel", + "status": "failed", + "error": {"code": "invalid_request_error", "type": "invalid_request_error", "message": "cancel"}, + }, + }, + separators=(",", ":"), + ) + + task = asyncio.create_task( + service._process_upstream_websocket_text( + text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=gate, + ) + ) + await release_started.wait() + task.cancel() + release_finished.set() + with pytest.raises(asyncio.CancelledError): + await task + + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + assert request_state.response_create_gate_acquired is False + assert request_state.response_create_gate is None + await asyncio.wait_for(gate.acquire(), timeout=0.1) + gate.release() + + def test_http_bridge_deferred_reasoning_blocks_previsible_replay() -> None: request_state = proxy_service._WebSocketRequestState( request_id="http_security_deferred_reasoning", diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 6561ee9582..bb4fe2c472 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -38169,6 +38169,57 @@ async def release_account_lease(received_lease: AccountLease | None) -> None: assert request_state.account_response_create_release is None +@pytest.mark.asyncio +async def test_response_create_gate_release_reraises_caller_cancellation_after_cleanup(): + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_gate_cancel", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + ) + response_create_gate = asyncio.Semaphore(1) + await response_create_gate.acquire() + request_state.response_create_gate_acquired = True + request_state.response_create_gate = response_create_gate + lease = AccountLease( + lease_id="lease_gate_cancel", + account_id="acc_gate_cancel", + kind="response_create", + acquired_at=0.0, + ) + request_state.account_response_create_lease = lease + release_started = asyncio.Event() + release_allowed = asyncio.Event() + + async def release_account_lease(received_lease: AccountLease | None) -> None: + assert received_lease == lease + release_started.set() + await release_allowed.wait() + + request_state.account_response_create_release = release_account_lease + + release_task = asyncio.create_task( + proxy_service._release_websocket_response_create_gate(request_state, response_create_gate) + ) + await release_started.wait() + release_task.cancel() + await asyncio.sleep(0) + + assert response_create_gate.locked() is True + assert request_state.response_create_gate_acquired is True + + release_allowed.set() + with pytest.raises(asyncio.CancelledError): + await release_task + + assert response_create_gate.locked() is False + assert request_state.response_create_gate_acquired is False + assert request_state.account_response_create_lease is None + assert request_state.account_response_create_release is None + + @pytest.mark.asyncio async def test_compact_selection_budget_exhaustion_returns_request_timeout(monkeypatch): settings = _make_proxy_settings() From 585611522478e109f9de0521312a4d1c9bac8c8f Mon Sep 17 00:00:00 2001 From: Kevin Lin <86810837+kevinsslin@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:49:58 +0800 Subject: [PATCH 040/117] chore(tooling): run Makefile Python scripts through uv (#1741) * chore(tooling): run Makefile Python scripts through uv * chore: add kevinsslin to .all-contributorsrc Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 9e5c571958..efee90715e 100644 --- a/Makefile +++ b/Makefile @@ -90,7 +90,7 @@ lint: architecture-check uv run ruff format --check . architecture-check: - python scripts/check_proxy_architecture.py + uv run python scripts/check_proxy_architecture.py typecheck: uv sync --dev --frozen @@ -114,9 +114,9 @@ test-integration-core: frontend-build # guards that the shards always partition the full selection exactly. test-integration-core-shard: frontend-build uv sync --dev --frozen - python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --verify + uv run python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --verify PYTHONFAULTHANDLER=1 uv run pytest $(PYTEST_ARGS) \ - $$(python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --shard $(SHARD)) + $$(uv run python .github/scripts/pytest_shards.py --shard-count $(INTEGRATION_CORE_SHARD_COUNT) --shard $(SHARD)) test-integration-core-1: $(MAKE) test-integration-core-shard SHARD=1 @@ -163,7 +163,7 @@ package: frontend-build uv run python -c "import app; import app.main; print('import ok')" rm -rf build dist *.egg-info uvx --from build==1.3.0 python -m build - python scripts/verify-wheel-assets.py + uv run python scripts/verify-wheel-assets.py .PHONY: docker docker: From fd97cb856970e46a5e6e4265e0064fca4f24fb02 Mon Sep 17 00:00:00 2001 From: Kevin Lin <86810837+kevinsslin@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:50:02 +0800 Subject: [PATCH 041/117] fix(proxy): separate websocket scope cleanup budget (#1723) * fix(proxy): separate websocket scope cleanup budget Keep the one-second generic task cancellation bound while allowing normal direct Responses WebSocket scope finalization to use a bounded five-second observation window. Preserve the shared shutdown deadline and add a regression plus OpenSpec contract. Refs #1711 * test(proxy): harden cleanup budget regression * test(proxy): assert websocket cleanup warning is absent * chore: add kevinsslin to .all-contributorsrc Co-Authored-By: Claude Fable 5 * fix(proxy): align websocket cleanup subtask budget --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- app/modules/proxy/_service/websocket/mixin.py | 23 ++++-- .../.openspec.yaml | 2 + .../websocket-scope-cleanup-budget/design.md | 68 ++++++++++++++++ .../proposal.md | 41 ++++++++++ .../specs/responses-api-compat/spec.md | 39 +++++++++ .../websocket-scope-cleanup-budget/tasks.md | 23 ++++++ .../test_websocket_terminal_cancellation.py | 79 +++++++++++++++++++ 7 files changed, 268 insertions(+), 7 deletions(-) create mode 100644 openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml create mode 100644 openspec/changes/websocket-scope-cleanup-budget/design.md create mode 100644 openspec/changes/websocket-scope-cleanup-budget/proposal.md create mode 100644 openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/websocket-scope-cleanup-budget/tasks.md diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 0e1894e559..7f02097e69 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -494,6 +494,9 @@ def _facade() -> Any: logger = logging.getLogger(__name__) _WEBSOCKET_PINNED_REFRESH_UNAVAILABLE_MESSAGE = "Account refresh is temporarily unavailable; retry later." +# Scope teardown coordinates several request/lease finalizers; keep its normal +# observation budget separate from the short generic child-task cancel bound. +_WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS = 5.0 _CAPABILITY_REQUIRED_NO_AUTHORIZED_ACCOUNTS_MESSAGE = ( "This request requires Trusted Access for Cyber, but no eligible account is marked as " "security-work-authorized. codex-lb did not fall back to an ordinary account." @@ -2615,9 +2618,15 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: scope_cancelled = True raise finally: - cleanup_timeout = shutdown_state.remaining_drain_timeout_seconds() - if cleanup_timeout is None: - cleanup_timeout = _facade()._TASK_CANCEL_TIMEOUT_SECONDS + remaining_drain_timeout = shutdown_state.remaining_drain_timeout_seconds() + cleanup_timeout = ( + _WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS + if remaining_drain_timeout is None + else max(float(remaining_drain_timeout), 0.0) + ) + task_cleanup_timeout = ( + _facade()._TASK_CANCEL_TIMEOUT_SECONDS if remaining_drain_timeout is None else cleanup_timeout + ) async def finalize_websocket_scope() -> None: nonlocal replay_request_state @@ -2635,7 +2644,7 @@ async def finalize_websocket_scope() -> None: await _close_websocket_upstream_for_cleanup( proxy, upstream, - timeout_seconds=cleanup_timeout, + timeout_seconds=task_cleanup_timeout, ) if reader_to_await is not None: try: @@ -2657,7 +2666,7 @@ async def finalize_websocket_scope() -> None: try: await _facade()._await_cancelled_task( retired_create_lease_release_task, - timeout_seconds=cleanup_timeout, + timeout_seconds=task_cleanup_timeout, label="proxy websocket retired create lease release", cancel=False, ) @@ -2671,7 +2680,7 @@ async def finalize_websocket_scope() -> None: try: await _facade()._await_cancelled_task( request_state_failure_task, - timeout_seconds=cleanup_timeout, + timeout_seconds=task_cleanup_timeout, label="proxy websocket unsent request finalization", cancel=False, ) @@ -2774,7 +2783,7 @@ def log_scope_cleanup_failure(done_task: asyncio.Task[None]) -> None: ) if not done: _facade().logger.warning( - "Websocket scope cleanup exceeded its remaining drain budget " + "Websocket scope cleanup exceeded its cleanup budget " "timeout_seconds=%.3f background_cleanup_tasks=%d", max(float(cleanup_timeout), 0.0), sum(1 for task in proxy._background_cleanup_tasks if not task.done()), diff --git a/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml b/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/websocket-scope-cleanup-budget/design.md b/openspec/changes/websocket-scope-cleanup-budget/design.md new file mode 100644 index 0000000000..eb96d1d45b --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/design.md @@ -0,0 +1,68 @@ +## Context + +The WebSocket handler publishes one tracked `finalize_websocket_scope()` task +from its `finally` block. The task must preserve cancellation while completing +the terminal request cleanup sequence. During process drain, +`shutdown_state.remaining_drain_timeout_seconds()` provides the shared absolute +deadline. Outside drain it returns `None`, so the current code falls back to +the generic one-second `_TASK_CANCEL_TIMEOUT_SECONDS` value. + +The generic timeout is used across HTTP bridge and proxy child-task cancellation +paths. Increasing it globally would slow unrelated cancellation and would +change the semantics of a helper that is intentionally a short cancellation +observation bound. The scope finalizer needs a different bounded allowance +because its work is a sequence of request-state and lease finalization steps. + +## Goals / Non-Goals + +**Goals:** + +- Give normal direct WebSocket scope cleanup enough bounded time for the + existing request finalization and lease-release sequence. +- Preserve the existing tracked-task ownership and cancellation behavior when + the bound is reached. +- Keep shutdown cleanup governed by the one shared remaining drain deadline. +- Prove the behavior through the real WebSocket route finalizer. + +**Non-Goals:** + +- Changing the `response.created` watchdog or any upstream request budget. +- Retrying, replaying, or moving an interrupted request to another account. +- Increasing the generic `_TASK_CANCEL_TIMEOUT_SECONDS` value. +- Adding an operator setting, environment variable, database state, or a new + background cleanup registry. + +## Decisions + +1. **Use one internal five-second scope budget.** The value is deliberately + fixed and bounded because this is a lifecycle safety allowance, not an + operator tuning surface. Five seconds is long enough to absorb ordinary + persistence/lease scheduling variance while still returning promptly when + teardown is stuck. + +2. **Prefer the active drain deadline.** When shutdown drain is active, the + finalizer continues to use the remaining shared deadline exactly as today. + The normal-operation budget is only the fallback for the no-drain case and + cannot extend process shutdown. + +3. **Keep child cancellation semantics separate.** Individual task waits keep + the one-second generic cancellation bound during normal operation. During + drain they remain capped by the shared remaining deadline. Only the outer + scope-finalization wait receives the five-second normal-operation allowance. + +4. **Retain tracked cleanup after the bound.** `asyncio.wait()` continues to + observe the finalizer without cancelling it at the scope budget. The + existing `_background_cleanup_tasks` registry and persistence drain remain + the owner of unfinished cleanup, so a timeout is honest and does not cause + lease or request finalization to be abandoned. + +## Verification Strategy + +- Run the focused WebSocket terminal-cancellation tests, including a regression + that lowers the generic task timeout and delays finalization beyond it while + allowing completion within the separate scope budget. +- Run Ruff check/format on changed Python files, the proxy architecture check, + and the applicable type/test targets. +- Validate the OpenSpec delta if the CLI is available; otherwise record the + unavailable local CLI as a handoff limitation and keep the artifacts in the + repository for CI validation. diff --git a/openspec/changes/websocket-scope-cleanup-budget/proposal.md b/openspec/changes/websocket-scope-cleanup-budget/proposal.md new file mode 100644 index 0000000000..6666fe1962 --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/proposal.md @@ -0,0 +1,41 @@ +## Why + +Direct Responses WebSocket scope teardown currently reuses the generic +`_TASK_CANCEL_TIMEOUT_SECONDS` value as its entire normal-operation cleanup +budget. That value is intentionally one second for individual task +cancellation, but scope teardown can also have to finalize request logs, +release response-create ownership, and release the account connection lease. +Under ordinary load those operations can exceed one second, producing +`Websocket scope cleanup exceeded its remaining drain budget` even when the +server is not draining. The cleanup task remains tracked, but the warning and +unfinished teardown increase the chance of follow-up reconnect churn. + +## What Changes + +- Give normal-operation WebSocket scope teardown its own fixed five-second + bounded budget. +- Keep the existing one-second generic task-cancellation timeout for ordinary + child-task waits. +- Continue using the remaining shared shutdown deadline whenever process drain + is active; the new budget must not extend shutdown. +- Add a route-level cancellation regression proving that cleanup which takes + longer than the generic task timeout can still finish within the scope budget + and does not leave an orphaned cleanup task. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Direct Responses WebSocket scope cleanup has a + separate bounded normal-operation budget while preserving the shared + shutdown deadline and task ownership guarantees. + +## Impact + +The change is limited to the direct Responses WebSocket finalizer, its focused +unit coverage, and the OpenSpec contract. It adds no setting, dependency, +database migration, API shape, upstream watchdog change, or retry policy. diff --git a/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md b/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..35fb23b2db --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/specs/responses-api-compat/spec.md @@ -0,0 +1,39 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Direct WebSocket scope cleanup has a bounded normal-operation budget + +When a direct Responses WebSocket scope exits while the process is not using an +active shutdown drain deadline, the proxy MUST allow its existing scope +finalization task a fixed five-second bounded observation budget, separate from +the one-second generic child-task cancellation timeout. The finalizer MUST +continue to own request finalization and lease cleanup through that budget. If +the budget expires, the proxy MUST preserve the existing cancellation result, +leave unfinished cleanup tracked by the existing cleanup-task registry, and +MUST NOT cancel or silently abandon that cleanup solely because the observation +budget expired. + +When an active shutdown drain deadline exists, the proxy MUST use the remaining +shared drain deadline instead of the normal-operation budget, so normal cleanup +allowance MUST NOT extend process shutdown. + +#### Scenario: normal scope cleanup outlives generic child cancellation + +- **GIVEN** a direct Responses WebSocket scope is cancelled while its existing + request finalization takes longer than the generic one-second child-task + cancellation timeout +- **AND** the finalization completes within the five-second normal-operation + scope budget +- **WHEN** scope cleanup runs +- **THEN** the finalizer completes and request/lease ownership is released +- **AND** the scope preserves its cancellation result +- **AND** no cleanup task remains orphaned after the finalizer completes + +#### Scenario: shutdown drain remains the upper bound + +- **GIVEN** a direct Responses WebSocket scope is cancelled while an active + shutdown drain deadline has less than five seconds remaining +- **WHEN** scope cleanup runs +- **THEN** the remaining shared drain deadline remains the upper bound +- **AND** the normal-operation five-second budget does not extend shutdown diff --git a/openspec/changes/websocket-scope-cleanup-budget/tasks.md b/openspec/changes/websocket-scope-cleanup-budget/tasks.md new file mode 100644 index 0000000000..1d68bdb7e5 --- /dev/null +++ b/openspec/changes/websocket-scope-cleanup-budget/tasks.md @@ -0,0 +1,23 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a real direct Responses WebSocket cancellation regression that + distinguishes the generic task timeout from the scope cleanup budget. +- [x] 1.2 Confirm the regression fails against the baseline implementation and + passes with the scoped budget. + +## 2. Scope Cleanup Budget + +- [x] 2.1 Add the fixed normal-operation WebSocket scope cleanup budget. +- [x] 2.2 Preserve the active shared shutdown deadline and one-second generic + child-task cancellation behavior. +- [x] 2.3 Keep unfinished cleanup tracked and prevent cancellation/lease + ownership regressions when the bound expires. + +## 3. Verification + +- [x] 3.1 Run focused WebSocket terminal-cancellation tests. +- [x] 3.2 Run changed-file Ruff check/format, proxy architecture checks, and + applicable type checks. +- [x] 3.3 Validate the OpenSpec delta and inspect the final diff/status. +- [x] 3.4 Open a Draft PR targeting upstream `main` and add the live 1.23.0 + evidence to issue #1711. diff --git a/tests/unit/test_websocket_terminal_cancellation.py b/tests/unit/test_websocket_terminal_cancellation.py index 45122c622e..c8fb423f5f 100644 --- a/tests/unit/test_websocket_terminal_cancellation.py +++ b/tests/unit/test_websocket_terminal_cancellation.py @@ -240,6 +240,85 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: assert service._background_cleanup_tasks == set() +@pytest.mark.asyncio +async def test_normal_websocket_scope_cleanup_uses_separate_scope_budget( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield SimpleNamespace(request_logs=_RequestLogsRecorder(), api_keys=object()) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + settings = SimpleNamespace( + prefer_earlier_reset_accounts=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + prohibit_fast_mode=False, + ) + + class _SettingsCache: + async def get(self) -> SimpleNamespace: + return settings + + receive_started = asyncio.Event() + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + + class _BlockingDownstreamWebSocket: + async def receive(self) -> dict[str, object]: + receive_started.set() + await asyncio.Event().wait() + raise AssertionError("unreachable") + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + + async def block_cleanup(*_args: object, **_kwargs: object) -> None: + cleanup_started.set() + await release_cleanup.wait() + + monkeypatch.setattr(proxy_service, "_TASK_CANCEL_TIMEOUT_SECONDS", 0.01) + monkeypatch.setattr(websocket_mixin, "_WEBSOCKET_SCOPE_CLEANUP_TIMEOUT_SECONDS", 0.08) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache()) + monkeypatch.setattr( + proxy_service, + "get_settings", + lambda: SimpleNamespace(proxy_downstream_websocket_idle_timeout_seconds=30.0), + ) + monkeypatch.setattr(proxy_service, "_routing_strategy", lambda _settings: "usage_weighted") + monkeypatch.setattr(service, "_websocket_continuity_state_for_request", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", block_cleanup) + monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) + caplog.set_level(logging.WARNING) + + scope_task = asyncio.create_task( + service.proxy_responses_websocket( + cast(WebSocket, _BlockingDownstreamWebSocket()), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + ) + await asyncio.wait_for(receive_started.wait(), timeout=1) + + scope_task.cancel() + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + await asyncio.sleep(0.03) + assert scope_task.done() is False + release_cleanup.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(scope_task, timeout=0.5) + await asyncio.sleep(0) + + assert not any( + message.startswith("Websocket scope cleanup exceeded its cleanup budget") for message in caplog.messages + ) + assert service._background_cleanup_tasks == set() + + @pytest.mark.asyncio @pytest.mark.parametrize( "failing_child", From 08b84a95ad5d4de88b3ac4ebb37185781a603f81 Mon Sep 17 00:00:00 2001 From: Borealin <41241077+Borealin@users.noreply.github.com> Date: Sun, 16 Aug 2026 16:54:13 +0800 Subject: [PATCH 042/117] fix(proxy): route source-owned models off the WebSocket transport (#1659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy): route source-owned models off the WebSocket transport Model sources were only consulted by the HTTP request handlers. The WebSocket session path went straight to subscription-account selection, so a model served by an enabled OpenAI-compatible source was dispatched to a ChatGPT account and rejected upstream with "The '' model is not supported when using Codex with a ChatGPT account." Extract the shared source-resolution helpers into app/modules/model_sources/selection.py so both transports agree on which models belong to a source, and fail the WebSocket connect with model_source_requires_http_transport when the requested model resolves to one. The failure is emitted as a 503 connect failure on purpose: Codex clients fall back to the HTTP transport only on service-level connect failures, while a 4xx is treated as terminal. After the fallback the HTTP path routes to the model source normally, so no client configuration change is required. Refs #1658 * style: fix import ordering flagged by ruff * docs(contributors): add Borealin * test(proxy): cover the websocket model-source guard Adds the coverage requested in review: - `tests/unit/test_proxy_websocket_model_source_guard.py` asserts the guard emits `model_source_requires_http_transport` with status 503, selects no account, and never reaches account selection. - `tests/integration/test_model_source_routing.py` covers `responses_model_is_source_owned` positively and negatively, the `require_streaming` edge, and an API key whose `enforced_model` resolves to a source. Also considers the API key's `enforced_model` in the guard. The HTTP handlers build their candidate list from both the enforced model and the requested one; the guard only looked at the requested model, so an enforced source model could still fall through to subscription-account selection. The enforced-model rule moves to `app/modules/model_sources/selection.py` with `api.py` delegating, keeping one definition. Refs #1658 * test(proxy): satisfy ty in the websocket guard test Drop the redundant _WebSocketMixin cast and narrow the request_state cast to _WebSocketRequestState so the call matches the method signature. * fix(proxy): guard source models on prepared response.create too The connect-path guard is skipped when a WebSocket already has an open subscription-account upstream, so a later response.create that switches to a source-owned model (including via a refreshed API-key enforced_model) was still forwarded to the ChatGPT account and rejected upstream with the unsupported-model error. Apply the source-ownership check to every prepared response.create, before socket reuse. On an established session the turn fails with model_source_requires_http_transport as a terminal error, and its usage reservation is released first so the reuse path cannot leak one. Addresses the Codex review P2 on #1659. * fix(proxy): contain and narrow the websocket model-source guard Follow-up to the review findings on the guard itself. Fail open when source resolution raises. The lookup runs after a turn's usage reservation is acquired but before it is registered for cleanup, so a propagating database error tore down the whole session and stranded the reservation until the 6h stale reaper. Failing open degrades to the pre-guard behaviour, and source forwarding could not have worked anyway since it needs the same database for the source's credentials. Scoped to the WebSocket-only helper: the HTTP handlers keep surfacing resolution errors. Evaluate the connect guard once per connect series instead of once per failover attempt, by hoisting it out of _select_websocket_connect_account into its only caller. Judge it with the per-request api key rather than the connect-time session key, so a policy refresh mid-session cannot make it disagree with the equivalent check on the prepared-request path. Gate the reuse guard on a live upstream reader. The cleanup that nulls a dead upstream runs after the guard, so a socket that died between turns would take the reuse path (terminal error) when it should reconnect and take the connect path (503, which the client transparently falls back from). Finalize the request-log row on the reuse-guard path; previously the same logical failure was only recorded when it happened on the first turn. Tests: the reuse guard's positive path, the connect guard's negative path, key freshness, and the fail-open behaviour were all unpinned — each new test was verified to fail when its behaviour is reverted. * fix(proxy): judge WebSocket source ownership on the raw client model apply_api_key_enforcement normalizes model aliases as its first statement, before the api_key is None early return, so gpt-5-high becomes gpt-5 during request preparation. Both source-ownership guards read request_state.model, which is built from the enforced payload, so a source exposing an alias-named model was invisible to them. select_responses_model_source filters candidates against allowed_models exactly, with no alias resolution, so an API key that allowlists gpt-5-high discards the normalized gpt-5 candidate -- leaving the raw alias as the only one that could have matched, and the guards never supplied it. The HTTP handlers route the identical request correctly because they snapshot raw_source_model before enforcement. Carry that snapshot through preparation on request_state and feed it to both guards. The connect guard needed it too: its only call site passes the post-enforcement request_state.model, so it was never checking pre-enforcement either. The snapshot rides on the request state rather than on _PreparedWebSocketRequest because the replay path swaps in the state popped from pending_requests; a value threaded separately would desynchronize there, while a field on the state keeps each turn paired with its own raw model. States not built by preparation leave it None and keep today's derivation. Mirror the HTTP fast-mode correction as well. Without it, a source exposing a fast alias would be rejected on WebSocket while HTTP deliberately collapses the raw candidate and serves the request from a subscription account -- a transport divergence visible to the client. The resulting candidate list now matches the HTTP construction exactly, including enforced_model taking the raw slot. The regression tests run the real ownership helper and the real selector; only the catalog I/O is faked, and the fake records every candidate offered to it so the tests assert the raw alias physically reached source selection rather than trusting a stub. Reverting app/ with the tests in place fails both. Addresses the Codex review P2 and the maintainer's remaining blocker on #1659. * fix(proxy): keep the HTTP source-routing exclusions in the WebSocket guards The HTTP route skips model-source selection for two kinds of request: a terminal compaction_trigger, which the upstream compact flow serves on the turn's owner account, and a request referencing uploaded files, which is pinned to the subscription account that received the upload. The WebSocket guards judged model ownership alone, so such a turn whose model was also exposed by a source got a terminal model_source_requires_http_transport instead of reaching the owner-routing logic that would have dispatched it to the pinned account. The raw-alias fix widened the reach of this by making more requests match as source-owned. Extract the HTTP gate into responses_source_route_excluded so the route and the guards cannot drift, stamp the verdict on the prepared request state, and skip both guards when it is set. The verdict is computed on the full client input, before the WebSocket-specific trimming and anchor injection rewrite it, so it sees the same payload the HTTP route evaluates. It rides on the request state for the same reason raw_source_model does: the replay path swaps in the state popped from pending_requests, and states not built by preparation default to False, keeping the guards active for them. A malformed compaction trigger keeps the guards active rather than inheriting HTTP's 400. The WebSocket path has always forwarded those frames verbatim, and changing that belongs in its own change. /v1/responses is deliberately left alone: it excludes file references but not compaction triggers, and the WebSocket transport is codex-native, so the codex-native route is the right parity target. Tests pin each guard separately -- removing the exclusion from the reuse guard fails exactly the two reuse tests and leaves the connect ones passing, and vice versa -- plus unit coverage of the shared predicate including the malformed trigger raising. Addresses the Codex review P2 on the 38405534 head. * test(proxy): set up the DB for the file-pin WebSocket guard tests _pin_file_account now persists through FileAccountPinRepository (#1521), so the two guard tests that pin or resolve file accounts need the db_setup fixture like their siblings. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- .all-contributorsrc | 10 + README.md | 1 + app/modules/model_sources/selection.py | 133 +++ app/modules/proxy/_service/support.py | 18 + app/modules/proxy/_service/websocket/mixin.py | 163 ++++ app/modules/proxy/api.py | 59 +- app/modules/proxy/request_policy.py | 19 + .../proposal.md | 36 + .../specs/responses-api-compat/spec.md | 114 +++ .../tasks.md | 36 + .../integration/test_model_source_routing.py | 116 +++ ...test_proxy_websocket_model_source_guard.py | 913 ++++++++++++++++++ tests/unit/test_request_policy.py | 48 +- 13 files changed, 1628 insertions(+), 38 deletions(-) create mode 100644 app/modules/model_sources/selection.py create mode 100644 openspec/changes/route-model-sources-off-websocket/proposal.md create mode 100644 openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/route-model-sources-off-websocket/tasks.md create mode 100644 tests/unit/test_proxy_websocket_model_source_guard.py diff --git a/.all-contributorsrc b/.all-contributorsrc index 2624725fd9..eac13c51dc 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1242,6 +1242,16 @@ "code", "test" ] + }, + { + "login": "Borealin", + "name": "Borealin", + "avatar_url": "https://avatars.githubusercontent.com/u/41241077?v=4", + "profile": "https://github.com/Borealin", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 1d63923e57..e6f1100e9b 100644 --- a/README.md +++ b/README.md @@ -287,6 +287,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e DuyBui
DuyBui
💻 ⚠️ Kevin Lin
Kevin Lin
💻 ⚠️ + Borealin
Borealin
💻 ⚠️ diff --git a/app/modules/model_sources/selection.py b/app/modules/model_sources/selection.py new file mode 100644 index 0000000000..849c597599 --- /dev/null +++ b/app/modules/model_sources/selection.py @@ -0,0 +1,133 @@ +"""Shared model-source selection helpers. + +Both the HTTP request handlers and the WebSocket session path need to decide +whether a requested model is served by an OpenAI-compatible model source. +Keeping that decision in one module stops the two transports from drifting +apart: previously only the HTTP handlers consulted model sources, so a +source-owned model requested over WebSocket fell through to subscription +account selection and was rejected upstream. +""" + +from __future__ import annotations + +import logging + +from app.core.openai.model_registry import get_model_registry +from app.db.models import ModelSource +from app.db.session import detach_session_objects, get_background_session +from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.repository import ModelSourcesRepository + +logger = logging.getLogger(__name__) + + +def allowed_source_ids_for_api_key(api_key: ApiKeyData | None) -> set[str] | None: + """Source ids an API key may use, or ``None`` when scoping is disabled.""" + if api_key is None or not api_key.source_assignment_scope_enabled: + return None + return set(api_key.assigned_source_ids) + + +async def select_responses_model_source( + model: str, + api_key: ApiKeyData | None, + *, + raw_model: str | None = None, + require_streaming: bool = False, +) -> tuple[ModelSource, str] | None: + """Resolve ``model`` to a Responses-capable model source, if any.""" + assigned_source_ids = allowed_source_ids_for_api_key(api_key) + exact_allowed_models = set(api_key.allowed_models) if api_key and api_key.allowed_models else None + candidates = [candidate for candidate in (raw_model, model) if candidate] + if not candidates: + return None + deduped_candidates = list(dict.fromkeys(candidates)) + registry_models = get_model_registry().get_models_with_fallback() + async with get_background_session() as session: + repository = ModelSourcesRepository(session) + for candidate in deduped_candidates: + if exact_allowed_models is not None and candidate not in exact_allowed_models: + continue + subscription_model = registry_models.get(candidate) + if assigned_source_ids is None and subscription_model is not None: + continue + source = await repository.find_responses_source_for_model( + candidate, + allowed_source_ids=assigned_source_ids, + require_streaming=require_streaming, + ) + if source is not None: + break + else: + source = None + # ``close_session`` rolls back the read transaction, which would + # expire the loaded row; detach it so the forwarding path can read + # its attributes after this session boundary. + detach_session_objects(session) + return (source, candidate) if source is not None else None + + +def effective_model_for_api_key(api_key: ApiKeyData | None, requested_model: str | None) -> str | None: + """The model an API key forces, falling back to the requested one.""" + if api_key is None or api_key.enforced_model is None: + return requested_model + return api_key.enforced_model + + +async def responses_model_is_source_owned( + model: str | None, + api_key: ApiKeyData | None, + *, + raw_model: str | None = None, +) -> bool: + """True when ``model`` is served by an enabled Responses-capable source. + + Used by the WebSocket path, which cannot forward to a model source and must + fail the session so the client falls back to the HTTP transport. + + The API key's ``enforced_model`` is considered alongside the requested + model, matching how the HTTP handlers build their candidate list: an + enforced model that resolves to a source must not slip through to + subscription-account selection. + + ``raw_model`` is the client's requested model captured before request + preparation normalized aliases (``gpt-5-high`` -> ``gpt-5``), mirroring the + HTTP path's ``raw_source_model``: the caller has already substituted the + API key's ``enforced_model`` and applied the fast-mode correction, so it is + used verbatim as the leading source candidate. When omitted (request states + that predate preparation, e.g. replayed turns), the raw candidate is + derived from ``enforced_model``/``model`` as before. + + Resolution failures fail open to ``False``. This helper only gates the + WebSocket transport, where the alternative is worse: the lookup runs after + the turn's usage reservation is acquired but before it is registered for + cleanup, so a propagating database error tears the whole session down and + strands the reservation until the stale reaper runs. Failing open degrades + to the pre-guard behaviour (the subscription upstream rejects the model), + and source forwarding could not have worked anyway — it needs the same + database for the source's credentials. The HTTP handlers deliberately do + not use this helper: they call ``select_responses_model_source`` directly + and must keep surfacing resolution errors rather than silently routing + source traffic to a subscription account. + """ + raw = raw_model if raw_model is not None else effective_model_for_api_key(api_key, model) + if not model and not raw: + return False + try: + return ( + await select_responses_model_source( + model or raw or "", + api_key, + raw_model=raw, + require_streaming=True, + ) + is not None + ) + except Exception: + logger.warning( + "model_source_resolution_failed_open model=%s raw_model=%s", + model, + raw, + exc_info=True, + ) + return False diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index e73dfa08cf..404c63487e 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -893,6 +893,24 @@ class _WebSocketRequestState: connection_request_kind: str | None = None generate_false_prewarm: bool = False api_key: ApiKeyData | None = None + # The client's requested model captured before api-key enforcement + # normalized aliases (``gpt-5-high`` -> ``gpt-5``), with the key's + # ``enforced_model`` substituted and the fast-mode correction applied, + # exactly like the HTTP path's ``raw_source_model``. Consumed only by the + # WebSocket source-ownership guards; it must never reach the upstream + # wire payload. ``None`` on request states that were not built by + # ``_prepare_websocket_response_create_request`` (replays, archives), + # which keeps those on the normalized-model check. + raw_source_model: str | None = None + # True when the HTTP route would exclude this request from model-source + # routing (``responses_source_route_excluded``: a terminal compaction + # trigger, or ``input_file`` references pinned to the uploading + # subscription account). The WebSocket source-ownership guards skip such + # requests so the owner-routing logic can dispatch them to a subscription + # account, exactly like HTTP. ``False`` on request states that were not + # built by ``_prepare_websocket_response_create_request``, which keeps + # the guards active for those. + source_route_excluded: bool = False request_usage_budget: ApiKeyRequestUsageBudget | None = None request_text: str | None = None replay_count: int = 0 diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 7f02097e69..3179aa8855 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -96,6 +96,10 @@ ApiKeyInvalidError, ApiKeysService, ) +from app.modules.model_sources.selection import ( + effective_model_for_api_key, + responses_model_is_source_owned, +) from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -471,10 +475,12 @@ from app.modules.proxy.request_policy import ( apply_api_key_enforcement, apply_enforced_service_tier_model_fallback, + model_alias_requests_fast_mode, normalize_responses_request_payload, openai_client_payload_error, openai_invalid_payload_error, openai_validation_error, + responses_source_route_excluded, validate_model_access, ) from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response @@ -1708,6 +1714,81 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: request_state = prepared_request.request_state request_affinity = prepared_request.affinity_policy text_data = prepared_request.text_data + if ( + upstream is not None + and account is not None + # A reader that has already finished means the upstream is + # gone but the cleanup that nulls it runs further below, so + # without this the turn would take the reuse path (terminal + # error) when it should reconnect and take the connect path + # (503, which the client transparently falls back from). + and upstream_reader is not None + and not upstream_reader.done() + # Requests the HTTP route excludes from + # source routing (a terminal compaction + # trigger, ``input_file`` references) + # must stay on subscription accounts even + # when their model is also source-owned; + # the owner-routing below dispatches them + # to the pinned account instead of this + # guard failing the turn. + and not request_state.source_route_excluded + and await responses_model_is_source_owned( + request_state.model, + request_state.api_key or api_key, + # The raw client model, before enforcement + # normalized aliases: an alias-only source + # (``gpt-5-high``) is invisible in the + # normalized ``request_state.model``. + raw_model=request_state.raw_source_model, + ) + ): + # Socket reuse bypasses connect-time selection, so a later + # response.create that switches to a source-owned model + # would otherwise be forwarded to the subscription account + # already attached to the open upstream. Model sources are + # only reachable from the HTTP request path. + # + # Gated on an existing upstream on purpose: a first turn has + # no socket yet and must fall through to the connect guard, + # which fails with a service-level 503 so the client falls + # back to HTTP. Emitting a terminal error here would preempt + # that fallback and make source models unreachable. + source_model = request_state.raw_source_model or request_state.model + source_message = ( + f"Model {source_model!r} is served by an " + "OpenAI-compatible model source, which is only reachable " + "over the HTTP transport; retry the request over HTTPS." + ) + _facade().logger.info( + "Websocket model source requires http transport " + "request_id=%s model=%s raw_model=%s stage=response_create", + request_state.request_log_id or request_state.request_id, + request_state.model, + request_state.raw_source_model, + ) + await proxy._release_websocket_request_state_reservation(request_state) + # The prepared request already owns a request-log row; without + # this the row is never finalized, so the same logical failure + # is only visible in request logs when it happens on the first + # turn (where the connect path writes it). + await proxy._write_websocket_connect_failure( + account_id=account.id, + api_key=request_state.api_key or api_key, + request_state=request_state, + error_code="model_source_requires_http_transport", + error_message=source_message, + ) + await proxy._emit_websocket_terminal_error( + websocket, + client_send_lock=client_send_lock, + request_state=request_state, + error_code="model_source_requires_http_transport", + error_message=source_message, + error_type="invalid_request_error", + downstream_activity=downstream_activity, + ) + continue except ProxyResponseError as exc: ( status_code, @@ -2823,15 +2904,36 @@ async def _prepare_websocket_response_create_request( payload, openai_compat=openai_cache_affinity, ) + # The client's raw model, captured before enforcement normalizes + # aliases (``gpt-5-high`` -> ``gpt-5``). The source-ownership guards + # must judge the raw alias too, or an alias-only model source is + # missed on the WebSocket paths while the HTTP path routes the same + # request via ``raw_source_model``. Mirrors ``api.py::responses`` + # exactly, including the enforced-model substitution here and the + # fast-mode correction after enforcement below. + raw_source_model = effective_model_for_api_key(refreshed_api_key, responses_payload.model) service_tier_was_enforced = apply_api_key_enforcement( responses_payload, refreshed_api_key, prohibit_fast_mode=prohibit_fast_mode, ) + if prohibit_fast_mode and model_alias_requests_fast_mode(raw_source_model): + raw_source_model = responses_payload.model apply_enforced_service_tier_model_fallback( responses_payload, service_tier_was_enforced=service_tier_was_enforced, ) + # Judged on the full client input, before the websocket-specific + # trimming and anchor injection below rewrite it — the same payload + # the HTTP route evaluates for its source-selection gate. + try: + source_route_excluded = responses_source_route_excluded(responses_payload) + except ClientPayloadError: + # HTTP rejects a malformed compaction trigger with a 400; the + # WebSocket path has always forwarded such frames verbatim, so a + # parse failure keeps the source guards active instead of + # changing that behavior here. + source_route_excluded = False normalized_payload = responses_payload.to_payload() stripped_client_metadata = strip_capability_metadata(normalized_payload.get("client_metadata")) if stripped_client_metadata is not normalized_payload.get("client_metadata"): @@ -2997,6 +3099,8 @@ async def _prepare_websocket_response_create_request( request_state.useragent_group = useragent_group request_state.conversation_id = conversation_id request_state.client_ip = client_ip + request_state.raw_source_model = raw_source_model + request_state.source_route_excluded = source_route_excluded request_state.responses_lite_model = next_responses_lite_model request_state.expose_stale_previous_response_classifier = codex_session_affinity request_state.require_security_work_authorized = capability_route.require_security_work_authorized @@ -3231,6 +3335,65 @@ async def _record_or_defer_confirmed_route_backoff(account: Account) -> None: request_transport="websocket", ), ) + # Model sources are only reachable from the HTTP request path. Fail the + # WebSocket connect instead of dispatching a source-owned model to a + # subscription account, which the upstream rejects with "The '' + # model is not supported when using Codex with a ChatGPT account." + # Codex clients fall back to the HTTP transport when a WebSocket + # connect fails, and that path routes to the source correctly. + # + # Evaluated once per connect series rather than inside the failover + # loop below: source ownership is a property of the requested model, so + # re-resolving it per attempt would only repeat the same lookup. The + # per-request api key is used (rather than the session key) so a policy + # refresh mid-session cannot make this disagree with the equivalent + # check on the prepared-request path. + # + # Requests the HTTP route excludes from source routing (a terminal + # compaction trigger, ``input_file`` references pinned to the + # uploading account) skip the guard: they must land on a subscription + # account either way, and the owner-required selection below routes + # them there instead of bouncing the turn to HTTP. + if not request_state.source_route_excluded and await responses_model_is_source_owned( + model, + request_state.api_key or api_key, + # ``model`` is the session loop's post-enforcement + # ``request_state.model``; the raw client alias captured at + # preparation is what an alias-only source is registered under. + raw_model=request_state.raw_source_model, + ): + source_model = request_state.raw_source_model or model + message = ( + f"Model {source_model!r} is served by an OpenAI-compatible model source, which is only " + "reachable over the HTTP transport; retry the request over HTTPS." + ) + _facade().logger.info( + "Websocket model source requires http transport request_id=%s model=%s raw_model=%s api_key_present=%s", + request_state.request_log_id or request_state.request_id, + model, + request_state.raw_source_model, + (request_state.api_key or api_key) is not None, + ) + await proxy._emit_websocket_connect_failure( + websocket, + client_send_lock=client_send_lock, + account_id=None, + api_key=request_state.api_key or api_key, + request_state=request_state, + # 503 (not 4xx) is deliberate: Codex clients only fall back to + # the HTTP transport when a WebSocket connect fails at the + # service level. A 4xx is treated as terminal and surfaces to + # the user instead of retrying over HTTPS. + status_code=503, + payload=openai_error( + "model_source_requires_http_transport", + message, + error_type="server_error", + ), + error_code="model_source_requires_http_transport", + error_message=message, + ) + return None, None max_attempts = _facade()._WEBSOCKET_MAX_ACCOUNT_ATTEMPTS excluded_account_ids: set[str] = set(request_state.excluded_account_ids) last_failover_exc: ProxyResponseError | None = None diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 531506480e..b0930fad0e 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -203,6 +203,11 @@ stream_responses as stream_source_responses, ) from app.modules.model_sources.repository import ModelSourcesRepository +from app.modules.model_sources.selection import ( + allowed_source_ids_for_api_key, + effective_model_for_api_key, + select_responses_model_source, +) from app.modules.proxy import affinity as proxy_affinity_module from app.modules.proxy import images_service as images_service_module from app.modules.proxy import service as proxy_service_module @@ -244,6 +249,7 @@ openai_client_payload_error, openai_validation_error, resolve_model_alias, + responses_source_route_excluded, sanitize_source_chat_payload, strip_terminal_compaction_trigger_input, validate_model_access, @@ -1070,12 +1076,16 @@ async def responses( raw_source_model = responses_payload.model validate_model_access(api_key, responses_payload.model) try: - compact_trigger_input = strip_terminal_compaction_trigger_input(responses_payload) + # Terminal compaction triggers run the upstream compact flow on the + # turn's owner account, and file-referencing requests are pinned to + # the account that received the upload; the shared predicate keeps + # this gate and the WebSocket source-ownership guards in agreement. + source_route_excluded = responses_source_route_excluded(responses_payload) except ClientPayloadError as exc: error = openai_client_payload_error(exc) return _logged_error_json_response(request, 400, error) source = None - if compact_trigger_input is None and not extract_input_file_ids(responses_payload.input): + if not source_route_excluded: source_selection = await _select_responses_model_source( responses_payload.model, api_key, @@ -4210,35 +4220,14 @@ async def _select_responses_model_source( raw_model: str | None = None, require_streaming: bool = False, ) -> tuple[ModelSource, str] | None: - assigned_source_ids = _allowed_source_ids_for_api_key(api_key) - exact_allowed_models = set(api_key.allowed_models) if api_key and api_key.allowed_models else None - candidates = [candidate for candidate in (raw_model, model) if candidate] - if not candidates: - return None - deduped_candidates = list(dict.fromkeys(candidates)) - registry_models = get_model_registry().get_models_with_fallback() - async with get_background_session() as session: - repository = ModelSourcesRepository(session) - for candidate in deduped_candidates: - if exact_allowed_models is not None and candidate not in exact_allowed_models: - continue - subscription_model = registry_models.get(candidate) - if assigned_source_ids is None and subscription_model is not None: - continue - source = await repository.find_responses_source_for_model( - candidate, - allowed_source_ids=assigned_source_ids, - require_streaming=require_streaming, - ) - if source is not None: - break - else: - source = None - # ``close_session`` rolls back the read transaction, which would - # expire the loaded row; detach it so the forwarding path can read - # its attributes after this session boundary. - detach_session_objects(session) - return (source, candidate) if source is not None else None + # Shared with the WebSocket path so both transports agree on which models + # belong to a model source. + return await select_responses_model_source( + model, + api_key, + raw_model=raw_model, + require_streaming=require_streaming, + ) async def _select_audio_transcriptions_model_source(model: str, api_key: ApiKeyData | None) -> ModelSource | None: @@ -4258,9 +4247,7 @@ async def _select_audio_transcriptions_model_source(model: str, api_key: ApiKeyD def _allowed_source_ids_for_api_key(api_key: ApiKeyData | None) -> set[str] | None: - if api_key is None or not api_key.source_assignment_scope_enabled: - return None - return set(api_key.assigned_source_ids) + return allowed_source_ids_for_api_key(api_key) async def _parse_transcription_multipart( @@ -7552,9 +7539,7 @@ def _effective_model_for_api_key(api_key: ApiKeyData | None, requested_model: st def _effective_optional_model_for_api_key(api_key: ApiKeyData | None, requested_model: str | None) -> str | None: - if api_key is None or api_key.enforced_model is None: - return requested_model - return api_key.enforced_model + return effective_model_for_api_key(api_key, requested_model) def _compact_request_service_tier(payload: ResponsesCompactRequest) -> str | None: diff --git a/app/modules/proxy/request_policy.py b/app/modules/proxy/request_policy.py index 6769493b21..b33b6864d4 100644 --- a/app/modules/proxy/request_policy.py +++ b/app/modules/proxy/request_policy.py @@ -12,6 +12,7 @@ ResponsesCompactRequest, ResponsesReasoning, ResponsesRequest, + extract_input_file_ids, responses_input_uses_lite_tools, ) from app.core.openai.strict_schema import ( @@ -598,6 +599,24 @@ def strip_terminal_compaction_trigger_input(payload: ResponsesRequest) -> list[J return stripped_input +def responses_source_route_excluded(payload: ResponsesRequest) -> bool: + """True when a Responses request must stay on subscription accounts. + + A terminal compaction trigger is served by the upstream compact flow on the + turn's owner account, and an ``input_file``/``input_image`` file reference + is pinned to the subscription account that received the upload — neither + can be dispatched to an OpenAI-compatible model source. The HTTP + ``/responses`` route and the WebSocket source-ownership guards share this + predicate so their notion of source-route eligibility cannot drift. + + Raises ``ClientPayloadError`` for a malformed compaction trigger, exactly + like ``strip_terminal_compaction_trigger_input``. + """ + if strip_terminal_compaction_trigger_input(payload) is not None: + return True + return bool(extract_input_file_ids(payload.input)) + + def enforce_strict_text_format(request: ResponsesRequest) -> None: """Reject strict-mode JSON schemas that violate OpenAI structured-outputs rules. diff --git a/openspec/changes/route-model-sources-off-websocket/proposal.md b/openspec/changes/route-model-sources-off-websocket/proposal.md new file mode 100644 index 0000000000..2abab883a2 --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/proposal.md @@ -0,0 +1,36 @@ +## Why + +Model sources are only consulted by the HTTP request handlers. The WebSocket +session path goes straight to subscription-account selection, so a model served +by an enabled OpenAI-compatible source is dispatched to a ChatGPT account and +rejected upstream with: + +``` +The '' model is not supported when using Codex with a ChatGPT account. +``` + +`docs/client-setup.md` documents `supports_websockets = true`, so model sources +are unusable with the documented Codex client configuration. See #1658. + +## What Changes + +- Extract the shared model-source resolution helpers into + `app/modules/model_sources/selection.py` so the HTTP and WebSocket paths agree + on which models belong to a source. +- Fail the WebSocket connect with `model_source_requires_http_transport` when the + requested model resolves to an enabled Responses-capable source, instead of + selecting a subscription account. +- Emit the failure as a `503` connect failure. Codex clients fall back to the + HTTP transport only on service-level connect failures; a `4xx` is treated as + terminal and surfaces to the user. After the fallback, the HTTP path routes to + the model source normally. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat` diff --git a/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md b/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..c035c8c1ca --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/specs/responses-api-compat/spec.md @@ -0,0 +1,114 @@ +## ADDED Requirements + +### Requirement: Source-owned models are not served over the WebSocket transport + +Model sources are reachable only from the HTTP request path. When a WebSocket +Responses session requests a model that resolves to an enabled, +Responses-capable OpenAI-compatible model source, the system SHALL NOT dispatch +the request to a subscription account. + +The check SHALL be applied on the connect path before account selection, and +SHALL also be applied to every prepared `response.create`, so that a turn which +switches to a source-owned model on an already-open subscription upstream is +also rejected instead of being forwarded. + +Both checks SHALL evaluate the client's raw requested model, captured before +API-key enforcement normalizes model aliases (for example `gpt-5-high` to +`gpt-5`), alongside the normalized model — the same candidate list the HTTP +handlers build from `raw_source_model`, including substituting the API key's +`enforced_model` and, when fast mode is prohibited and the raw model is a +fast-mode alias, replacing the raw candidate with the normalized model. A +source that exposes an alias-named model MUST be matched on the WebSocket +transport whenever the HTTP path would route to it. + +Both checks SHALL apply only to requests that are eligible for model-source +routing on the HTTP request path, judged on the full client input before any +WebSocket-specific trimming or anchor injection. A request whose input ends +with a terminal `compaction_trigger` item, or that references uploaded files +(`input_file` / file-backed `input_image` items), is excluded from source +routing over HTTP — the former is served by the upstream compact flow on the +turn's owner account, the latter is pinned to the subscription account that +received the upload — and MUST NOT be failed by either WebSocket guard even +when its model also resolves to an enabled source. Such requests proceed to +subscription account selection and the owner-routing rules, exactly as they +would after the HTTP route skips source selection. A malformed compaction +trigger (repeated, or not the final top-level input item) SHALL keep the +guards active: the HTTP route rejects that payload with a 400, the WebSocket +path forwards it verbatim, and the exclusion changes neither. + +Both failures MUST use error code `model_source_requires_http_transport`. On the +connect path the failure MUST be emitted as a service-level connect failure +(HTTP status `503`), so that Codex clients fall back to the HTTP transport, +where source routing is applied. For a prepared `response.create` on an +established session the failure MUST be emitted as a terminal error for that +turn, and any usage reservation held for the turn MUST be released. + +When source resolution is unavailable, the WebSocket transport MUST fall back to +subscription account selection rather than failing the request. The resolution +runs after a turn's usage reservation is acquired but before it is registered +for cleanup, so a propagating failure would end the session and strand the +reservation; the degraded behaviour is the pre-change one, where the +subscription upstream rejects the model. This applies to the WebSocket transport +only — the HTTP request path MUST continue to surface resolution failures, since +silently routing source traffic to a subscription account would be worse there. + +#### Scenario: Source-owned model over WebSocket fails the connect + +- **GIVEN** an enabled OpenAI-compatible model source exposes model `m` with Responses support +- **WHEN** a client opens a WebSocket Responses session requesting model `m` +- **THEN** the system fails the connect with error code `model_source_requires_http_transport` +- **AND** no subscription account is selected for the request + +#### Scenario: Later turn switching to a source-owned model is rejected + +- **GIVEN** a WebSocket Responses session already has an open subscription-account upstream +- **AND** an enabled OpenAI-compatible model source exposes model `m` with Responses support +- **WHEN** a subsequent `response.create` requests model `m` +- **THEN** the system emits a terminal error with code `model_source_requires_http_transport` +- **AND** the frame is not forwarded to the subscription account on the open upstream +- **AND** the turn's usage reservation is released + +#### Scenario: An alias-named source model is rejected despite normalization + +- **GIVEN** an enabled OpenAI-compatible model source exposes model `gpt-5-high` with Responses support +- **AND** an API key whose `allowed_models` contains exactly `gpt-5-high` +- **WHEN** the key sends a WebSocket `response.create` for `gpt-5-high`, which enforcement normalizes to `gpt-5` +- **THEN** the source-ownership check also considers the raw `gpt-5-high` candidate +- **AND** the request is rejected with `model_source_requires_http_transport` on the connect path and on socket reuse alike + +#### Scenario: A file-referencing turn is dispatched to its pinned account, not failed + +- **GIVEN** a WebSocket Responses session already has an open subscription-account upstream +- **AND** a later `response.create` references an uploaded `input_file` pinned to that account +- **AND** the request's model is also exposed by an enabled model source +- **WHEN** the turn is prepared for the open socket +- **THEN** the reuse guard does not fail the turn with `model_source_requires_http_transport` +- **AND** the turn is forwarded to the pinned subscription account + +#### Scenario: A terminal compaction trigger is not failed by the WebSocket guards + +- **GIVEN** a `response.create` whose final top-level input item is a `compaction_trigger` +- **AND** the request's model is also exposed by an enabled model source +- **WHEN** the request reaches the connect path or an already-open subscription upstream +- **THEN** neither WebSocket guard fails the request with `model_source_requires_http_transport` +- **AND** the connect path proceeds to subscription account selection, and an open upstream receives the turn + +#### Scenario: An API key that enforces a source-owned model is rejected + +- **GIVEN** an API key whose `enforced_model` resolves to an enabled model source +- **WHEN** the key opens a WebSocket Responses session requesting any model +- **THEN** the enforced model is resolved against the model sources +- **AND** the session fails with `model_source_requires_http_transport` + +#### Scenario: Subscription models are unaffected + +- **GIVEN** a model that is not served by any enabled model source +- **WHEN** a client opens a WebSocket Responses session requesting that model +- **THEN** account selection proceeds unchanged + +#### Scenario: Source resolution failure falls back to subscription selection + +- **GIVEN** the model-source catalog cannot be read +- **WHEN** a client opens a WebSocket Responses session +- **THEN** account selection proceeds as it did before the guard existed +- **AND** the session is not terminated by the resolution failure diff --git a/openspec/changes/route-model-sources-off-websocket/tasks.md b/openspec/changes/route-model-sources-off-websocket/tasks.md new file mode 100644 index 0000000000..9dd91f6f03 --- /dev/null +++ b/openspec/changes/route-model-sources-off-websocket/tasks.md @@ -0,0 +1,36 @@ +## Tasks + +- [x] Extract `select_responses_model_source` / `allowed_source_ids_for_api_key` + into `app/modules/model_sources/selection.py`; delegate from + `app/modules/proxy/api.py`. +- [x] Add `responses_model_is_source_owned` for transport-level checks. +- [x] Guard `_select_websocket_connect_account` so source-owned models fail the + WebSocket connect instead of selecting a subscription account. +- [x] Return `503` so the Codex client falls back to the HTTP transport. +- [x] Consider the API key's `enforced_model` in the guard, matching the + candidate list the HTTP handlers build. +- [x] Add spec delta for `responses-api-compat`. +- [x] Cover the guard and the source-owned check with unit and integration + tests, including the `require_streaming` edge and the enforced-model case. +- [x] Apply the guard to every prepared `response.create` so socket reuse cannot + forward a source-owned model to the open subscription upstream (Codex P2). +- [x] Fail open to subscription selection when source resolution raises, so a + database failure cannot end the session or strand a usage reservation. +- [x] Evaluate the connect guard once per connect series instead of per failover + attempt, and judge it with the per-request api key rather than the + session key. +- [x] Gate the reuse guard on a live upstream reader so a socket that died + between turns reconnects into the 503 fallback instead of receiving a + terminal error. +- [x] Finalize the request-log row on the reuse-guard path. +- [x] Carry the client's raw model (pre alias normalization) through request + preparation and feed it to the source-ownership check on both the + connect and reuse paths, so an alias-only source behind an alias + allowlist matches like it does over HTTP (Codex P2). +- [x] Preserve the HTTP source-routing exclusions in both WebSocket guards: + extract the HTTP gate into `responses_source_route_excluded`, stamp it + on the prepared request state, and skip the guards for terminal + compaction triggers and `input_file`-referencing requests so they + dispatch to their (owner-pinned) subscription account instead of + failing with `model_source_requires_http_transport` (Codex P2). + diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index caf86b630e..7368447162 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -526,6 +526,81 @@ async def test_responses_source_selector_can_require_streaming(async_client): assert streaming is None +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_detects_streaming_source(async_client): + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-streaming-model" + await _create_model_source( + async_client, + name="ws-guard-streaming", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + + assert await responses_model_is_source_owned(model, None) is True + # A subscription model must stay on the WebSocket path. + assert await responses_model_is_source_owned("gpt-5.6-sol", None) is False + assert await responses_model_is_source_owned(None, None) is False + + +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_requires_streaming(async_client): + """The guard mirrors the HTTP selector, which requires a streaming source.""" + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-non-streaming-model" + await _create_model_source( + async_client, + name="ws-guard-non-streaming", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=False, + ) + + assert await responses_model_is_source_owned(model, None) is False + + +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_honors_enforced_model(async_client): + """An API key that forces a source-owned model must also be caught. + + The HTTP handlers build their candidate list from the enforced model as + well as the requested one; the WebSocket guard has to match or an enforced + source model would fall through to subscription-account selection. + """ + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "ws-guard-enforced-model" + await _create_model_source( + async_client, + name="ws-guard-enforced", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + enforcing_key = ApiKeyData( + id="key_ws_guard_enforced", + name="ws guard enforced", + key_prefix="sk-test-ws-enforced", + allowed_models=[], + enforced_model=model, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + # The client asked for a subscription model, but the key forces the source. + assert await responses_model_is_source_owned("gpt-5.6-sol", enforcing_key) is True + + @pytest.mark.asyncio async def test_responses_source_raw_alias_lookup_requires_exact_allowlist(async_client): import app.modules.proxy.api as proxy_api @@ -583,6 +658,47 @@ async def test_responses_source_raw_alias_lookup_requires_exact_allowlist(async_ assert selected_model == model +@pytest.mark.asyncio +async def test_responses_model_is_source_owned_prefers_the_raw_alias(async_client): + """WebSocket parity for the raw-alias candidate (see the HTTP test above). + + Request preparation normalizes ``gpt-5-high`` to ``gpt-5`` before the + WebSocket guards run, so the guard helper must accept the client's raw + model and offer it to source selection ahead of the normalized one — the + HTTP path routes the identical request via ``raw_source_model``. + """ + from app.modules.model_sources.selection import responses_model_is_source_owned + + model = "gpt-5-high" + await _create_model_source( + async_client, + name="ws-guard-raw-alias", + model=model, + base_url="http://127.0.0.1:9/v1", + supports_responses=True, + supports_streaming=True, + ) + exact_key = ApiKeyData( + id="key_ws_raw_alias", + name="ws raw alias", + key_prefix="sk-test-ws-raw-alias", + allowed_models=[model], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + assert await responses_model_is_source_owned("gpt-5", exact_key, raw_model=model) is True + # Without the raw candidate the exact allowlist hides the source. This is + # the pre-fix WebSocket behaviour; keeping it false proves the assertion + # above matched through the raw candidate, not some other fallback. + assert await responses_model_is_source_owned("gpt-5", exact_key) is False + + @pytest.mark.asyncio async def test_chat_source_selector_can_require_streaming(async_client): from app.modules.model_sources.repository import ModelSourcesRepository diff --git a/tests/unit/test_proxy_websocket_model_source_guard.py b/tests/unit/test_proxy_websocket_model_source_guard.py new file mode 100644 index 0000000000..8488f0a087 --- /dev/null +++ b/tests/unit/test_proxy_websocket_model_source_guard.py @@ -0,0 +1,913 @@ +"""Tests for the WebSocket model-source guard. + +Model sources are only reachable from the HTTP request path, so the WebSocket +transport must refuse them. Two guards cover the two ways a turn can reach an +upstream: + +* the connect guard, which fails the connect with a service-level ``503`` that + Codex clients transparently fall back from onto HTTP; +* the reuse guard, which fails a later ``response.create`` that switches to a + source-owned model on an already-open subscription upstream. +""" + +from __future__ import annotations + +import asyncio +import json +from contextlib import asynccontextmanager +from types import SimpleNamespace +from typing import cast +from unittest.mock import AsyncMock + +import anyio +import pytest +from fastapi import WebSocket + +import app.modules.model_sources.selection as source_selection +import app.modules.proxy._service.websocket.mixin as ws_mixin +from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.selection import ( + effective_model_for_api_key, + responses_model_is_source_owned, +) +from app.modules.proxy import service as proxy_service +from tests.unit.test_proxy_utils import ( + _make_account, + _make_proxy_settings, + _QueuedTestUpstreamWebSocket, + _repo_factory, + _RequestLogsRecorder, + _SettingsCache, +) + +pytestmark = pytest.mark.unit + + +def _api_key(*, enforced_model: str | None = None) -> ApiKeyData: + from datetime import datetime + + return ApiKeyData( + id="key_ws_guard", + name="ws guard", + key_prefix="sk-test-ws-guard", + allowed_models=[], + enforced_model=enforced_model, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2026, 1, 1), + last_used_at=None, + ) + + +def _request_state(model: str) -> ws_mixin._WebSocketRequestState: + return proxy_service._WebSocketRequestState( + request_id="req-ws-guard", + model=model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=anyio.current_time(), + ) + + +def test_effective_model_prefers_enforced_model() -> None: + assert effective_model_for_api_key(None, "gpt-5.6-sol") == "gpt-5.6-sol" + assert effective_model_for_api_key(_api_key(), "gpt-5.6-sol") == "gpt-5.6-sol" + assert effective_model_for_api_key(_api_key(enforced_model="qwen3.8-max"), "gpt-5.6-sol") == "qwen3.8-max" + + +@pytest.mark.asyncio +async def test_source_ownership_fails_open_when_resolution_raises(monkeypatch: pytest.MonkeyPatch) -> None: + """A database failure must not be able to reject a subscription turn. + + The lookup runs after the turn's usage reservation is acquired but before it + is registered for cleanup, so a propagating error would tear the session + down and strand the reservation. Failing open degrades to the behaviour that + existed before the guard. + """ + + async def boom(*args, **kwargs): # noqa: ANN002, ANN003, ANN202 + raise RuntimeError("model_sources table is unavailable") + + monkeypatch.setattr(source_selection, "select_responses_model_source", boom) + + assert await responses_model_is_source_owned("qwen3.8-max", None) is False + + +async def _run_connect_guard( + monkeypatch: pytest.MonkeyPatch, + *, + is_source_owned: bool, + api_key: ApiKeyData | None = None, + request_state_api_key: ApiKeyData | None = None, +): + """Drive ``_connect_proxy_websocket`` far enough to observe the connect guard. + + ``_select_websocket_connect_account`` stands in for the failover loop the + guard short-circuits, so reaching it means the guard did not fire. + """ + emitted: dict[str, object] = {} + selection_calls = 0 + seen_api_keys: list[ApiKeyData | None] = [] + + async def fake_is_source_owned(model, key, *, raw_model=None): # noqa: ANN001 + seen_api_keys.append(key) + return is_source_owned + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", fake_is_source_owned) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + request_state = _request_state("qwen3.8-max") + request_state.api_key = request_state_api_key + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + model="qwen3.8-max", + request_state=request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + return account, upstream, emitted, selection_calls, seen_api_keys + + +@pytest.mark.asyncio +async def test_connect_guard_fails_session_for_source_owned_model(monkeypatch: pytest.MonkeyPatch) -> None: + account, upstream, emitted, selection_calls, _ = await _run_connect_guard(monkeypatch, is_source_owned=True) + + assert account is None + assert upstream is None + assert selection_calls == 0, "the guard must short-circuit before account selection" + assert emitted["error_code"] == "model_source_requires_http_transport" + assert emitted["status_code"] == 503, "a 4xx is terminal client-side and would strand the fallback" + assert emitted["account_id"] is None + + +@pytest.mark.asyncio +async def test_connect_guard_ignores_subscription_models(monkeypatch: pytest.MonkeyPatch) -> None: + account, _upstream, emitted, selection_calls, _ = await _run_connect_guard(monkeypatch, is_source_owned=False) + + assert account is None # the stubbed selector returns no account + assert selection_calls >= 1, "subscription models must proceed to account selection" + assert emitted == {} + + +@pytest.mark.asyncio +async def test_connect_guard_uses_the_per_request_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + """A policy refresh mid-session must not be judged against the stale session key. + + ``request_state.api_key`` is refreshed per request; the session key captured + at connect time can be arbitrarily old on a long-lived socket, and the reuse + guard already consults the fresh one. + """ + session_key = _api_key() + refreshed_key = _api_key(enforced_model="qwen3.8-max") + + *_, seen_api_keys = await _run_connect_guard( + monkeypatch, + is_source_owned=True, + api_key=session_key, + request_state_api_key=refreshed_key, + ) + + assert seen_api_keys == [refreshed_key] + + +def _text_frame(payload: dict[str, object]) -> SimpleNamespace: + return SimpleNamespace( + kind="text", + text=json.dumps(payload, separators=(",", ":")), + data=None, + close_code=None, + error=None, + error_code=None, + ) + + +def _completed_turn(response_id: str) -> list[SimpleNamespace]: + return [ + _text_frame({"type": "response.created", "response": {"id": response_id, "status": "in_progress"}}), + _text_frame( + { + "type": "response.completed", + "response": { + "id": response_id, + "status": "completed", + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + } + ), + ] + + +class _Downstream: + """A downstream socket that replays a scripted sequence of client frames.""" + + def __init__(self, request_texts: list[str]) -> None: + self.pending = list(request_texts) + self.done = asyncio.Event() + self.sent_text: list[str] = [] + self.turn_completed = asyncio.Event() + + async def receive(self) -> dict[str, object]: + if self.pending: + # Wait for the previous turn to settle so the frames stay ordered. + if len(self.pending) < 1 or self.sent_text: + await self.turn_completed.wait() + self.turn_completed.clear() + return {"type": "websocket.receive", "text": self.pending.pop(0)} + await self.done.wait() + return {"type": "websocket.disconnect"} + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + if payload.get("type") in {"response.completed", "response.failed", "error"}: + self.turn_completed.set() + if not self.pending: + self.done.set() + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + self.done.set() + + +def _create_frame(model: str) -> str: + return json.dumps( + { + "type": "response.create", + "model": model, + "instructions": "", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "stream": True, + }, + separators=(",", ":"), + ) + + +@pytest.mark.asyncio +async def test_first_turn_reaches_connect_guard_not_the_reuse_guard( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A fresh socket must reach the connect guard, which emits the 503 that + makes Codex clients fall back to HTTP. + + The per-frame reuse guard runs before connection, so if it were not gated on + an already-open upstream it would emit a terminal ``invalid_request_error`` + for the very first ``response.create`` and preempt the fallback, leaving + model sources unreachable. + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_first_turn") + upstream = _QueuedTestUpstreamWebSocket(_completed_turn("resp_first_turn")) + + connect_called = False + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal connect_called + connect_called = True + return account, upstream + + async def always_source_owned(*args, **kwargs) -> bool: # noqa: ANN002, ANN003 + return True + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", always_source_owned) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert connect_called, "first turn must reach the connect path, not the per-frame reuse guard" + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the reuse guard must not preempt the connect-path 503 on a fresh socket" + ) + + +@pytest.mark.asyncio +async def test_reuse_guard_rejects_a_later_source_owned_turn(monkeypatch: pytest.MonkeyPatch) -> None: + """A second turn that switches to a source-owned model must not be forwarded. + + Socket reuse skips connection entirely, so without the reuse guard the frame + would go to the subscription account already attached to the open upstream + and be rejected by the backend with the unsupported-model error. + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_reuse") + upstream = _QueuedTestUpstreamWebSocket(_completed_turn("resp_turn_one")) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + # Only the second turn's model is source-owned. + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + released = AsyncMock() + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(proxy_service.ProxyService, "_release_websocket_request_state_reservation", released) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("gpt-5.6-sol"), _create_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert any("resp_turn_one" in text for text in downstream.sent_text), "the subscription turn must complete" + assert any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the source-owned turn must be rejected by the reuse guard" + ) + assert len(upstream.sent_text) == 1, "the rejected turn must not be forwarded upstream" + assert released.await_count >= 1, "the rejected turn must release its usage reservation" + + +def _alias_allowlist_api_key() -> ApiKeyData: + """A key that allowlists exactly the alias an alias-named source exposes. + + ``validate_model_access`` resolves aliases on both sides, so this key also + admits plain ``gpt-5`` requests — but ``select_responses_model_source`` + filters candidates against the allowlist *exactly*, which keeps the + normalized ``gpt-5`` candidate away from source lookup. That makes the raw + alias the only candidate that can match the source, in the unit fake, the + integration database, and production alike. + """ + from datetime import datetime + + return ApiKeyData( + id="key_ws_guard_alias", + name="ws guard alias", + key_prefix="sk-test-ws-alias", + allowed_models=["gpt-5-high"], + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=datetime(2026, 1, 1), + last_used_at=None, + ) + + +class _AliasSourceCatalog: + """Fake only the I/O seams underneath ``select_responses_model_source``. + + The candidate construction in ``responses_model_is_source_owned`` and the + allowlist/registry filtering in ``select_responses_model_source`` stay + real; this stands in for the database session/repository and records which + candidates were actually offered to the catalog, so tests can assert the + raw client alias physically reached source selection (monkeypatching + ``responses_model_is_source_owned`` itself would test the stub instead). + """ + + def __init__(self, source_models: set[str]) -> None: + self.source_models = source_models + self.seen_candidates: list[str] = [] + + def install(self, monkeypatch: pytest.MonkeyPatch) -> None: + catalog = self + + class _FakeRepository: + def __init__(self, _session: object) -> None: + pass + + async def find_responses_source_for_model( + self, + candidate: str, + *, + allowed_source_ids=None, # noqa: ANN001 + require_streaming: bool = False, + ): # noqa: ANN202 + catalog.seen_candidates.append(candidate) + if candidate in catalog.source_models: + return SimpleNamespace(id="src_alias", name="alias-source", enabled=True) + return None + + @asynccontextmanager + async def fake_session(): # noqa: ANN202 + yield object() + + monkeypatch.setattr(source_selection, "ModelSourcesRepository", _FakeRepository) + monkeypatch.setattr(source_selection, "get_background_session", fake_session) + monkeypatch.setattr(source_selection, "detach_session_objects", lambda _session: None) + + +class _TurnDrivenUpstream: + """An upstream that releases each scripted turn only after its request. + + ``_QueuedTestUpstreamWebSocket`` queues every frame up front, which would + let a second turn's events race ahead of the second ``response.create``. + Here turn N's events become readable only after the Nth upstream send, so + a turn that is (correctly) rejected before forwarding leaves its events + unread, and a (buggy) forwarded turn completes cleanly instead of hanging + the session — the pre-fix failure stays a crisp assertion failure. + """ + + def __init__(self, turns: list[list[SimpleNamespace]]) -> None: + self._turns = list(turns) + self._messages: asyncio.Queue[SimpleNamespace] = asyncio.Queue() + self.close_seen = asyncio.Event() + self.sent_text: list[str] = [] + + def response_header(self, name: str) -> str | None: + del name + return None + + async def receive(self) -> SimpleNamespace: + message = await self._messages.get() + if message.kind == "close": + self.close_seen.set() + return message + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + if self._turns: + for event in self._turns.pop(0): + self._messages.put_nowait(event) + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self) -> None: + self.close_seen.set() + + +@pytest.mark.asyncio +async def test_reuse_guard_sees_the_raw_model_alias(monkeypatch: pytest.MonkeyPatch) -> None: + """A later turn asking for an alias-only source model must be rejected. + + ``apply_api_key_enforcement`` normalizes ``gpt-5-high`` to ``gpt-5`` + during request preparation, so a guard that judges only + ``request_state.model`` misses a source that exposes exactly + ``gpt-5-high`` — while the HTTP path routes the identical request to the + source via its pre-enforcement ``raw_source_model``. The real + ``responses_model_is_source_owned`` and ``select_responses_model_source`` + run here; only the catalog I/O is faked (regression for the raw-alias P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_alias") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + released = AsyncMock() + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_release_websocket_request_state_reservation", released) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _Downstream([_create_frame("gpt-5"), _create_frame("gpt-5-high")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=api_key, + ) + + assert any("resp_turn_one" in text for text in downstream.sent_text), "the subscription turn must complete" + assert "gpt-5-high" in catalog.seen_candidates, ( + "the client's raw alias must reach source selection; the normalized " + "'gpt-5' is filtered out by the key's exact allowlist" + ) + assert any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "the alias-owned turn must be rejected by the reuse guard" + ) + assert len(upstream.sent_text) == 1, "the alias turn must not be forwarded to the subscription upstream" + assert released.await_count >= 1, "the rejected turn must release its usage reservation" + + +@pytest.mark.asyncio +async def test_connect_guard_sees_the_raw_model_alias(monkeypatch: pytest.MonkeyPatch) -> None: + """The connect guard must judge the pre-enforcement alias too. + + The session loop hands ``_connect_proxy_websocket`` the post-enforcement + ``request_state.model``, so the raw alias must ride on the prepared + request state itself for the connect-time check to see it. This drives the + real ``_prepare_websocket_response_create_request`` (where enforcement + normalizes the alias) into the real connect guard. + """ + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + + emitted: dict[str, object] = {} + selection_calls = 0 + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001, ANN003, ANN202 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + prepared = await service._prepare_websocket_response_create_request( + json.loads(_create_frame("gpt-5-high")), + headers={}, + codex_session_affinity=False, + openai_cache_affinity=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + api_key=api_key, + ) + assert prepared.request_state.model == "gpt-5", "enforcement is expected to normalize the alias" + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + # Exactly what the session loop passes: the normalized model. + model=prepared.request_state.model, + request_state=prepared.request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + + assert account is None + assert upstream is None + assert selection_calls == 0, "the connect guard must short-circuit before account selection" + assert emitted.get("error_code") == "model_source_requires_http_transport" + assert emitted.get("status_code") == 503 + assert "gpt-5-high" in catalog.seen_candidates, "the raw alias must reach source selection on the connect path" + + +def _create_frame_with_input(model: str, input_items: list[dict[str, object]]) -> str: + return json.dumps( + { + "type": "response.create", + "model": model, + "instructions": "", + "input": input_items, + "stream": True, + }, + separators=(",", ":"), + ) + + +def _input_file_frame(model: str, file_id: str) -> str: + return _create_frame_with_input( + model, + [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "summarize the attachment"}, + {"type": "input_file", "file_id": file_id}, + ], + } + ], + ) + + +def _compaction_trigger_frame(model: str) -> str: + return _create_frame_with_input( + model, + [ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"type": "compaction_trigger"}, + ], + ) + + +class _TurnSerializedDownstream: + """A downstream that sends each next frame only after the prior turn ends. + + ``_Downstream`` hands the session loop the next frame as soon as it asks, + so a second turn can be dispatched while the first turn's events are still + in flight; the first turn's terminal frame then arrives with no pending + client frames left and ends the session before the second turn's events + come back. The rejection tests never notice — the guard fails the second + turn synchronously inside the message loop — but the forwarding + regressions below need the second turn's scripted upstream events to reach + the client, so this downstream serializes turns the way a real Codex + client does: it waits for a terminal frame before sending the next + ``response.create``. + """ + + def __init__(self, request_texts: list[str]) -> None: + self.pending = list(request_texts) + self.done = asyncio.Event() + self.sent_text: list[str] = [] + self.turn_completed = asyncio.Event() + self._dispatched_any = False + + async def receive(self) -> dict[str, object]: + if self.pending: + if self._dispatched_any: + await self.turn_completed.wait() + self.turn_completed.clear() + self._dispatched_any = True + return {"type": "websocket.receive", "text": self.pending.pop(0)} + await self.done.wait() + return {"type": "websocket.disconnect"} + + async def send_text(self, text: str) -> None: + self.sent_text.append(text) + payload = json.loads(text) + if payload.get("type") in {"response.completed", "response.failed", "error"}: + self.turn_completed.set() + if not self.pending: + self.done.set() + + async def send_bytes(self, _data: bytes) -> None: + return None + + async def close(self, code: int = 1000, reason: str | None = None) -> None: + del code, reason + self.done.set() + + +@pytest.mark.asyncio +async def test_reuse_guard_forwards_a_pinned_input_file_turn(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """A later source-owned turn that references an uploaded file must be forwarded. + + The HTTP route skips source selection whenever the input references an + ``input_file`` — the upload is account-scoped, so the request is pinned to + the subscription account that received it. The equivalent WebSocket turn + must reach that account through the owner-routing path instead of being + failed by the reuse guard (regression for the source-routing-exclusions + P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_file_pin") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + # The second turn's model is source-owned, like the reuse-guard rejection test. + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + await service._pin_file_account("file_ws_guard_pin", account.id) + downstream = _TurnSerializedDownstream( + [_create_frame("gpt-5.6-sol"), _input_file_frame("qwen3.8-max", "file_ws_guard_pin")] + ) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "HTTP excludes file-referencing requests from source routing, so the " + "reuse guard must not fail the file-pinned turn" + ) + assert len(upstream.sent_text) == 2, "the file-pinned turn must be forwarded to the pinned subscription account" + assert any("resp_turn_two" in text for text in downstream.sent_text), "the file-pinned turn must complete" + + +@pytest.mark.asyncio +async def test_reuse_guard_forwards_a_terminal_compaction_trigger_turn( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A terminal compaction-trigger turn must stay on the subscription upstream. + + The HTTP route serves terminal compaction triggers through the upstream + compact flow on the turn's owner account and never source-routes them, so + the reuse guard must not fail the equivalent WebSocket turn even when its + model is also exposed by a source (regression for the + source-routing-exclusions P2). + """ + settings = _make_proxy_settings() + settings.stream_idle_timeout_seconds = 300.0 + settings.proxy_downstream_websocket_idle_timeout_seconds = 120.0 + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_source_guard_compact") + upstream = _TurnDrivenUpstream([_completed_turn("resp_turn_one"), _completed_turn("resp_turn_two")]) + + async def fake_connect(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + return account, upstream + + async def source_owned_for_qwen(model, _api_key, *, raw_model=None): # noqa: ANN001 + return model == "qwen3.8-max" + + monkeypatch.setattr(ws_mixin, "responses_model_is_source_owned", source_owned_for_qwen) + monkeypatch.setattr(proxy_service.ProxyService, "_connect_proxy_websocket", fake_connect) + monkeypatch.setattr(service, "_resolve_compact_turn_state_owner", AsyncMock(return_value=None)) + + downstream = _TurnSerializedDownstream([_create_frame("gpt-5.6-sol"), _compaction_trigger_frame("qwen3.8-max")]) + + await service.proxy_responses_websocket( + cast(WebSocket, downstream), + {}, + codex_session_affinity=False, + openai_cache_affinity=False, + api_key=None, + ) + + assert not any("model_source_requires_http_transport" in text for text in downstream.sent_text), ( + "HTTP excludes terminal compaction triggers from source routing, so " + "the reuse guard must not fail the compaction turn" + ) + assert len(upstream.sent_text) == 2, "the compaction turn must be forwarded to the subscription upstream" + assert any("resp_turn_two" in text for text in downstream.sent_text), "the compaction turn must complete" + + +async def _drive_prepared_request_into_connect_guard( + monkeypatch: pytest.MonkeyPatch, + frame: str, +): + """Prepare ``frame`` for real and drive it into the real connect guard. + + Mirrors ``test_connect_guard_sees_the_raw_model_alias``: the alias catalog + makes ``gpt-5-high`` genuinely source-owned, so before the exclusions fix + the guard demonstrably fires for these frames — the stubbed account + selector standing in for the failover loop proves the guard was skipped. + """ + settings = _make_proxy_settings() + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + + catalog = _AliasSourceCatalog({"gpt-5-high"}) + catalog.install(monkeypatch) + + api_key = _alias_allowlist_api_key() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + + emitted: dict[str, object] = {} + selection_calls = 0 + + async def fake_refresh(key): # noqa: ANN001, ANN202 + return api_key + + async def fake_emit(self, websocket, **kwargs): # noqa: ANN001, ANN003, ANN202 + emitted.update(kwargs) + + async def fake_select(self, *args, **kwargs): # noqa: ANN001, ANN002, ANN003, ANN202 + nonlocal selection_calls + selection_calls += 1 + return None + + monkeypatch.setattr(service, "_refresh_websocket_api_key_policy", fake_refresh) + monkeypatch.setattr(service, "_reserve_websocket_api_key_usage", AsyncMock(return_value=None)) + monkeypatch.setattr(proxy_service.ProxyService, "_emit_websocket_connect_failure", fake_emit) + monkeypatch.setattr(proxy_service.ProxyService, "_select_websocket_connect_account", fake_select) + + prepared = await service._prepare_websocket_response_create_request( + json.loads(frame), + headers={}, + codex_session_affinity=False, + openai_cache_affinity=False, + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=0, + api_key=api_key, + ) + + account, upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + routing_strategy="capacity_weighted", + model=prepared.request_state.model, + request_state=prepared.request_state, + api_key=api_key, + client_send_lock=anyio.Lock(), + websocket=AsyncMock(), + ) + return prepared, account, upstream, emitted, lambda: selection_calls + + +@pytest.mark.asyncio +async def test_connect_guard_skips_input_file_requests(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """The connect guard must not bounce a file-referencing request to HTTP. + + An ``input_file`` reference excludes the request from source routing on + the HTTP path — pinned or not, the upload lives on a subscription + account — so the connect path must proceed to (owner-required) account + selection instead of emitting the 503 fallback. + """ + prepared, account, upstream, emitted, selection_calls = await _drive_prepared_request_into_connect_guard( + monkeypatch, + _input_file_frame("gpt-5-high", "file_ws_connect_unpinned"), + ) + + assert emitted == {}, "the connect guard must not emit the 503 HTTP-fallback failure" + assert selection_calls() >= 1, "file-referencing requests must proceed to account selection" + assert account is None # the stubbed selector returns no account + assert upstream is None + assert prepared.request_state.source_route_excluded is True, ( + "preparation must record the HTTP source-route exclusion on the request state" + ) + + +@pytest.mark.asyncio +async def test_connect_guard_skips_terminal_compaction_trigger_requests( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The connect guard must not bounce a terminal compaction trigger to HTTP. + + HTTP serves these through the compact flow on the owner account and never + source-routes them; the WebSocket connect path must likewise proceed to + account selection. + """ + prepared, account, upstream, emitted, selection_calls = await _drive_prepared_request_into_connect_guard( + monkeypatch, + _compaction_trigger_frame("gpt-5-high"), + ) + + assert emitted == {}, "the connect guard must not emit the 503 HTTP-fallback failure" + assert selection_calls() >= 1, "compaction-trigger requests must proceed to account selection" + assert account is None # the stubbed selector returns no account + assert upstream is None + assert prepared.request_state.source_route_excluded is True, ( + "preparation must record the HTTP source-route exclusion on the request state" + ) diff --git a/tests/unit/test_request_policy.py b/tests/unit/test_request_policy.py index 93f5ffaefa..69c30391be 100644 --- a/tests/unit/test_request_policy.py +++ b/tests/unit/test_request_policy.py @@ -6,10 +6,15 @@ import pytest from app.core.exceptions import ProxyModelNotAllowed +from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import ModelRegistry from app.core.openai.requests import ResponsesRequest from app.modules.api_keys.service import ApiKeyData -from app.modules.proxy.request_policy import apply_api_key_enforcement, validate_model_access +from app.modules.proxy.request_policy import ( + apply_api_key_enforcement, + responses_source_route_excluded, + validate_model_access, +) @pytest.mark.parametrize( @@ -243,3 +248,44 @@ def test_model_access_rejects_alias_when_canonical_model_not_allowed() -> None: with pytest.raises(ProxyModelNotAllowed): validate_model_access(api_key, "gpt-5.5-extra") + + +def _responses_request_with_input(input_value: object) -> ResponsesRequest: + return ResponsesRequest.model_validate({"model": "gpt-5", "instructions": "", "input": input_value}) + + +def test_source_route_excluded_is_false_for_plain_turns() -> None: + request = _responses_request_with_input([{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}]) + + assert responses_source_route_excluded(request) is False + + +def test_source_route_excluded_for_input_file_references() -> None: + request = _responses_request_with_input( + [{"role": "user", "content": [{"type": "input_file", "file_id": "file_123"}]}] + ) + + assert responses_source_route_excluded(request) is True + + +def test_source_route_excluded_for_terminal_compaction_trigger() -> None: + request = _responses_request_with_input( + [ + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + {"type": "compaction_trigger"}, + ] + ) + + assert responses_source_route_excluded(request) is True + + +def test_source_route_excluded_raises_for_malformed_compaction_trigger() -> None: + request = _responses_request_with_input( + [ + {"type": "compaction_trigger"}, + {"role": "user", "content": [{"type": "input_text", "text": "hi"}]}, + ] + ) + + with pytest.raises(ClientPayloadError): + responses_source_route_excluded(request) From 0c8d921906735352ef60c0a445be455daa35249d Mon Sep 17 00:00:00 2001 From: Kevin Lin <86810837+kevinsslin@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:10:57 +0800 Subject: [PATCH 043/117] feat(proxy): report websocket cleanup phase (#1726) * feat(proxy): report websocket cleanup phase * docs(contributors): add kevinsslin attribution * feat(proxy): report websocket cleanup phase * docs(contributors): add kevinsslin attribution * test(proxy): prove pending cleanup phase attribution * style(tests): format merged caplog assertion Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Darafei Praliaskouski Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- app/modules/proxy/_service/websocket/mixin.py | 14 ++- .../proposal.md | 33 +++++++ .../specs/proxy-runtime-observability/spec.md | 35 +++++++ .../tasks.md | 12 +++ .../test_websocket_terminal_cancellation.py | 96 +++++++++++++++++-- 5 files changed, 180 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md create mode 100644 openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md create mode 100644 openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 3179aa8855..954f8aea59 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -2708,8 +2708,10 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: task_cleanup_timeout = ( _facade()._TASK_CANCEL_TIMEOUT_SECONDS if remaining_drain_timeout is None else cleanup_timeout ) + cleanup_phase = "not_started" async def finalize_websocket_scope() -> None: + nonlocal cleanup_phase nonlocal replay_request_state nonlocal request_state_failure_task nonlocal request_state_to_fail @@ -2722,6 +2724,7 @@ async def finalize_websocket_scope() -> None: # release that wait. reader_to_await.cancel() if upstream is not None: + cleanup_phase = "upstream_close" await _close_websocket_upstream_for_cleanup( proxy, upstream, @@ -2729,6 +2732,7 @@ async def finalize_websocket_scope() -> None: ) if reader_to_await is not None: try: + cleanup_phase = "upstream_reader" await _facade()._await_cancelled_task( reader_to_await, label="proxy websocket upstream reader", @@ -2745,6 +2749,7 @@ async def finalize_websocket_scope() -> None: upstream_reader = None if retired_create_lease_release_task is not None: try: + cleanup_phase = "retired_create_lease" await _facade()._await_cancelled_task( retired_create_lease_release_task, timeout_seconds=task_cleanup_timeout, @@ -2759,6 +2764,7 @@ async def finalize_websocket_scope() -> None: retired_create_lease_release_task = None if request_state_failure_task is not None: try: + cleanup_phase = "unsent_request" await _facade()._await_cancelled_task( request_state_failure_task, timeout_seconds=task_cleanup_timeout, @@ -2775,6 +2781,7 @@ async def finalize_websocket_scope() -> None: replay_request_state = upstream_control.replay_request_state upstream_control.replay_request_state = None if request_state_to_fail is not None: + cleanup_phase = "unsent_request" await proxy._fail_pending_websocket_requests( account=None, account_id_value=account.id if account is not None else upstream_account_id, @@ -2793,6 +2800,7 @@ async def finalize_websocket_scope() -> None: ) request_state_to_fail = None if replay_request_state is not None: + cleanup_phase = "replay_request" await proxy._fail_pending_websocket_requests( account=None, account_id_value=account.id if account is not None else upstream_account_id, @@ -2810,6 +2818,7 @@ async def finalize_websocket_scope() -> None: penalize_account=False, ) client_disconnected = downstream_activity.disconnected + cleanup_phase = "pending_requests" await proxy._fail_pending_websocket_requests( account=None if client_disconnected or scope_cancelled else account, account_id_value=account.id if account is not None else upstream_account_id, @@ -2832,6 +2841,7 @@ async def finalize_websocket_scope() -> None: penalize_account=not (client_disconnected or scope_cancelled), ) try: + cleanup_phase = "connection_lease" await release_current_account_lease() except Exception: # Connection-lease cleanup must never replace cancellation @@ -2840,6 +2850,7 @@ async def finalize_websocket_scope() -> None: "Failed to release websocket connection lease during scope cleanup", exc_info=True, ) + cleanup_phase = "complete" cleanup_task = asyncio.create_task( finalize_websocket_scope(), @@ -2865,8 +2876,9 @@ def log_scope_cleanup_failure(done_task: asyncio.Task[None]) -> None: if not done: _facade().logger.warning( "Websocket scope cleanup exceeded its cleanup budget " - "timeout_seconds=%.3f background_cleanup_tasks=%d", + "timeout_seconds=%.3f cleanup_phase=%s background_cleanup_tasks=%d", max(float(cleanup_timeout), 0.0), + cleanup_phase, sum(1 for task in proxy._background_cleanup_tasks if not task.done()), ) diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md new file mode 100644 index 0000000000..4ade0403fa --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/proposal.md @@ -0,0 +1,33 @@ +## Why + +When WebSocket scope cleanup exceeds its cleanup budget, the warning reports the +timeout and total background cleanup task count but not the operation that is +still blocked. Operators cannot distinguish an upstream-close stall from +reader observation, request finalization, or lease release without reproducing +the incident under instrumentation. + +## What Changes + +- Track the current WebSocket scope cleanup phase locally while the existing + finalization sequence runs. +- Add that fixed, low-cardinality phase to the existing timeout warning. +- Keep cleanup ordering, timeout budgets, retries, and ownership unchanged. +- Do not log request ids, account ids, payloads, credentials, or exception + content in the phase field. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `proxy-runtime-observability`: WebSocket scope cleanup timeout warnings MUST + identify the blocked cleanup phase with a fixed low-cardinality value. + +## Impact + +`app/modules/proxy/_service/websocket/mixin.py` and its route-level WebSocket +cleanup regression coverage. No API, schema, setting, timeout, or dashboard +change. diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..20feb5e9d7 --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,35 @@ +# proxy-runtime-observability Delta + +## ADDED Requirements + +### Requirement: WebSocket scope cleanup timeout identifies its blocked phase + +When WebSocket scope finalization exceeds its cleanup budget, the proxy MUST +include the current cleanup phase in the existing warning. The phase MUST be a +fixed low-cardinality value that identifies the cleanup operation and MUST NOT +contain request ids, account ids, request payloads, credentials, or exception +content. This diagnostic MUST NOT change cleanup ordering, timeout budgets, +retry behavior, or task ownership. + +The phase MUST be one of `not_started`, `upstream_close`, `upstream_reader`, +`retired_create_lease`, `unsent_request`, `replay_request`, `pending_requests`, +`connection_lease`, or `complete`. `not_started` is the fallback before the +first cleanup operation begins. `complete` records finished cleanup and MUST NOT +appear in a timeout warning. Missing or unrecognized phases MUST fall back to +`not_started`; implementations MUST NOT derive a phase from request or exception +data. + +#### Scenario: Pending request finalization exceeds the cleanup budget + +- **GIVEN** a cancelled WebSocket scope whose pending request finalization does + not finish within the cleanup budget +- **WHEN** the proxy emits the cleanup-budget warning +- **THEN** the warning includes `cleanup_phase=pending_requests` +- **AND** the cleanup remains owned by the existing background drain + +#### Scenario: Diagnostic phase remains low-cardinality + +- **WHEN** any WebSocket scope cleanup phase exceeds the cleanup budget +- **THEN** the warning identifies only a fixed cleanup phase +- **AND** the phase contains no request id, account id, payload, credential, or + exception content diff --git a/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md b/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md new file mode 100644 index 0000000000..668b52e9e1 --- /dev/null +++ b/openspec/changes/attribute-websocket-scope-cleanup-phase/tasks.md @@ -0,0 +1,12 @@ +## 1. Implementation + +- [x] 1.1 Track the current fixed WebSocket scope cleanup phase. +- [x] 1.2 Include the phase in the existing cleanup-budget warning without + changing cleanup control flow or timeout behavior. + +## 2. Validation + +- [x] 2.1 Add a route-level regression proving a blocked request-finalization + cleanup is attributed to `pending_requests`. +- [x] 2.2 Run focused WebSocket tests, proxy integration tests, lint, type + checks, architecture checks, and strict OpenSpec validation. diff --git a/tests/unit/test_websocket_terminal_cancellation.py b/tests/unit/test_websocket_terminal_cancellation.py index c8fb423f5f..d285e0edff 100644 --- a/tests/unit/test_websocket_terminal_cancellation.py +++ b/tests/unit/test_websocket_terminal_cancellation.py @@ -152,6 +152,7 @@ async def test_transport_end_replay_requires_send_boundary_only_for_direct_webso @pytest.mark.asyncio async def test_cancelled_websocket_scope_cleanup_is_deadline_bounded_and_remains_drain_owned( monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, ) -> None: @asynccontextmanager async def repo_factory() -> AsyncIterator[SimpleNamespace]: @@ -163,27 +164,94 @@ async def repo_factory() -> AsyncIterator[SimpleNamespace]: sticky_threads_enabled=False, openai_cache_affinity_max_age_seconds=0, prohibit_fast_mode=False, + proxy_downstream_websocket_idle_timeout_seconds=30.0, + proxy_request_budget_seconds=30.0, + stream_idle_timeout_seconds=30.0, + sse_keepalive_interval_seconds=0.0, ) class _SettingsCache: async def get(self) -> SimpleNamespace: return settings - receive_started = asyncio.Event() + request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.6-sol", + "input": "pending cleanup", + }, + separators=(",", ":"), + ) + request_state = _request_state("request_pending_cleanup") + request_state.request_text = request_text + request_sent = asyncio.Event() cleanup_started = asyncio.Event() cleanup_cancelled = asyncio.Event() release_cleanup = asyncio.Event() + cleanup_request_ids: list[str] = [] class _BlockingDownstreamWebSocket: + def __init__(self) -> None: + self._received = False + async def receive(self) -> dict[str, object]: - receive_started.set() + if not self._received: + self._received = True + return {"type": "websocket.receive", "text": request_text} await asyncio.Event().wait() raise AssertionError("unreachable") + async def send_text(self, _text: str) -> None: + return None + + async def send_bytes(self, _data: bytes) -> None: + return None + async def close(self, code: int = 1000, reason: str | None = None) -> None: del code, reason - async def block_cleanup(*_args: object, **_kwargs: object) -> None: + class _PendingUpstream: + async def send_text(self, _text: str) -> None: + request_sent.set() + + async def send_bytes(self, _data: bytes) -> None: + raise AssertionError("binary send is not expected") + + async def close(self) -> None: + return None + + upstream = _PendingUpstream() + + async def prepare_request(*_args: object, **_kwargs: object) -> proxy_service._PreparedWebSocketRequest: + return proxy_service._PreparedWebSocketRequest( + text_data=request_text, + request_state=request_state, + affinity_policy=proxy_service._AffinityPolicy(), + ) + + async def acquire_admission( + state: proxy_service._WebSocketRequestState, + *, + response_create_gate: asyncio.Semaphore, + ) -> None: + state.response_create_gate = response_create_gate + await response_create_gate.acquire() + state.response_create_gate_acquired = True + state.awaiting_response_created = True + + async def connect_upstream(*_args: object, **_kwargs: object) -> tuple[Account, UpstreamWebSocket]: + account = cast(Account, SimpleNamespace(id="account_pending_cleanup", codex_installation_id=None)) + return account, cast(UpstreamWebSocket, upstream) + + async def relay_until_cancelled(*_args: object, **_kwargs: object) -> None: + await asyncio.Event().wait() + + async def block_cleanup( + *_args: object, + pending_requests: deque[proxy_service._WebSocketRequestState], + **_kwargs: object, + ) -> None: + cleanup_request_ids.extend(state.request_id for state in pending_requests) cleanup_started.set() try: await release_cleanup.wait() @@ -192,13 +260,17 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: raise monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache()) - monkeypatch.setattr( - proxy_service, - "get_settings", - lambda: SimpleNamespace(proxy_downstream_websocket_idle_timeout_seconds=30.0), - ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) monkeypatch.setattr(proxy_service, "_routing_strategy", lambda _settings: "usage_weighted") + monkeypatch.setattr(proxy_service, "_enforce_response_create_size_limit", lambda _request_state: None) + monkeypatch.setattr(websocket_mixin, "effective_account_concurrency_caps", lambda _settings: object()) monkeypatch.setattr(service, "_websocket_continuity_state_for_request", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_prepare_websocket_response_create_request", prepare_request) + monkeypatch.setattr(service, "_start_request_state_api_key_reservation_heartbeat", lambda *_args, **_kwargs: None) + monkeypatch.setattr(service, "_acquire_request_state_response_create_admission", acquire_admission) + monkeypatch.setattr(service, "_connect_proxy_websocket", connect_upstream) + monkeypatch.setattr(service, "_relay_upstream_websocket_messages", relay_until_cancelled) + monkeypatch.setattr(service, "_acquire_account_response_create_lease_or_overload", AsyncMock(return_value=object())) monkeypatch.setattr(service, "_fail_pending_websocket_requests", block_cleanup) monkeypatch.setattr(service._load_balancer, "release_account_lease", AsyncMock()) @@ -211,8 +283,9 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: api_key=None, ) ) - await asyncio.wait_for(receive_started.wait(), timeout=1) + await asyncio.wait_for(request_sent.wait(), timeout=1) + caplog.set_level(logging.WARNING) shutdown_state.commit_shutdown(timeout_seconds=0.1) started_at = asyncio.get_running_loop().time() scope_task.cancel() @@ -225,12 +298,17 @@ async def block_cleanup(*_args: object, **_kwargs: object) -> None: elapsed = asyncio.get_running_loop().time() - started_at assert cleanup_cancelled.is_set() is False + assert cleanup_request_ids == [request_state.request_id] assert 0.05 <= elapsed < 0.3 assert any( task.get_name() == "proxy-websocket-finalization-scope-cleanup" for task in service._background_cleanup_tasks if not task.done() ) + assert any( + "Websocket scope cleanup exceeded its cleanup budget" in message and "cleanup_phase=pending_requests" in message + for message in caplog.messages + ) persistence_drain = asyncio.create_task(service.drain_persistence_tasks(timeout_seconds=1)) await asyncio.sleep(0) From 6cf7e61d7719654a62fb3132a21ae5f19ad8dba1 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Sun, 16 Aug 2026 19:57:23 +0400 Subject: [PATCH 044/117] =?UTF-8?q?fix(proxy):=20complete=20disconnect=20c?= =?UTF-8?q?leanup=20=E2=80=94=20pool=20leak,=20charged=20reservation,=20mu?= =?UTF-8?q?table=20terminal=20reason=20(#1645)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy): complete disconnect cleanup and terminal settlement * fix(proxy): defer streamed reservation cleanup on cancellation * fix: mark terminal streams after frame preparation * fix: settle terminal usage before cancellation * test: exercise repeated session cancellation * fix(proxy): preserve terminal stream settlement on disconnect * fix(proxy): preserve source cancellation settlement * fix(proxy): finish disconnect settlement cleanup * fix(proxy): harden terminal disconnect cleanup * test(proxy): align visible failure request-log expectation * fix(proxy): preserve source setup disconnect logging * fix(proxy): harden remaining PR1645 disconnect cleanup * fix(proxy): classify source stream setup disconnect as cancelled The stream-setup disconnect path logged status="error" with error_code="client_disconnected" while every sibling disconnect path (request setup, stream buffering, usage settlement, mid-stream) records status="cancelled". The usage aggregation keys on status, so this path alone inflated the dashboard error rate for ordinary client disconnects. Align it with the sibling convention and update the regression test to assert the cancelled status. Co-Authored-By: Claude Fable 5 * fix(proxy): keep disconnect logging when reservation release fails The stream-setup disconnect path isolates reservation-release failures so the visible disconnect log is always written, but the sibling request-setup and buffered-stream cancellation paths let a release failure replace the CancelledError and skip the request-log write. Align both paths with the stream-setup pattern: capture the release exception, always attempt the disconnect log, and surface the release failure as a warning instead of swallowing the disconnect record. Co-Authored-By: Claude Fable 5 * fix(proxy): finalize post-refresh terminal streams on downstream close The disconnect-cleanup hardening gave the primary stream attempt an owned cancellation-safe close and terminal settlement finalization, but the post-refresh path (after a 401 token refresh) still consumed _stream_once() with a bare async-for. Closing the outer response generator after a post-refresh terminal frame left the child generator suspended, so its cleanup/request-log finally never ran and the outer finalizer released the reservation as abandoned, losing terminal usage accounting and the account-health result. Mirror the primary-attempt pattern at all three post-refresh levels: own the aclose() of the inner _stream_once() stream, of the _iter_stream_once() wrapper, and of the post-refresh generator at its call site, and finalize delivered terminal settlements before propagating cancellation. Co-Authored-By: Claude Fable 5 * fix(proxy): shield source stream body teardown and cleanup wait loop Two residual cancellation-delivery gaps in the disconnect cleanup: - The source stream body generators unwound their exit stack with a plain "async with", so a second cancellation delivered while __aexit__ was unwinding could interrupt the pooled HTTP session lease release and leak the lease. Close the stack from the body's finally via the cancellation-deferring helper, matching the setup handlers. - retry.py's _await_task_deferring_cancellation lacked the anyio shield its api.py counterpart has, so a level-cancelled Starlette scope re-raised into every await and busy-spun the loop until the owned cleanup task completed. Wrap the loop in anyio.CancelScope(shield=True). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- app/db/session.py | 26 +- app/modules/model_sources/forwarding.py | 110 ++- app/modules/proxy/_service/streaming/mixin.py | 18 +- app/modules/proxy/_service/streaming/retry.py | 184 +++- app/modules/proxy/api.py | 284 ++++-- .../.openspec.yaml | 2 + .../fix-disconnect-cleanup-leak/design.md | 5 + .../fix-disconnect-cleanup-leak/proposal.md | 5 + .../specs/api-keys/spec.md | 10 + .../specs/responses-api-compat/spec.md | 11 + .../fix-disconnect-cleanup-leak/tasks.md | 5 + .../integration/test_model_source_routing.py | 823 ++++++++++++++++++ tests/integration/test_proxy_responses.py | 307 +++++++ tests/unit/test_db_session.py | 48 + tests/unit/test_proxy_utils.py | 157 +++- 15 files changed, 1855 insertions(+), 140 deletions(-) create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/design.md create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/proposal.md create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/fix-disconnect-cleanup-leak/tasks.md diff --git a/app/db/session.py b/app/db/session.py index d7d6ae7f9d..48e68a9cae 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -343,11 +343,18 @@ def _startup_sqlite_check_mode(raw_mode: str) -> SqliteIntegrityCheckMode | None async def _shielded(awaitable: Awaitable[object]) -> None: task = asyncio.ensure_future(awaitable) - try: - await asyncio.shield(task) - except asyncio.CancelledError: - await task - raise + cancellation: asyncio.CancelledError | None = None + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + break + except asyncio.CancelledError as exc: + if task.cancelled(): + raise + cancellation = cancellation or exc + if cancellation is not None: + raise cancellation async def _safe_rollback(session: AsyncSession) -> None: @@ -367,9 +374,12 @@ async def _safe_close(session: AsyncSession) -> None: async def close_session(session: AsyncSession) -> None: - if session.in_transaction(): - await _safe_rollback(session) - await _safe_close(session) + async def _close() -> None: + if session.in_transaction(): + await _safe_rollback(session) + await _safe_close(session) + + await _shielded(_close()) def detach_session_objects(session: AsyncSession) -> None: diff --git a/app/modules/model_sources/forwarding.py b/app/modules/model_sources/forwarding.py index cd30a7bbc0..8fa2ed496c 100644 --- a/app/modules/model_sources/forwarding.py +++ b/app/modules/model_sources/forwarding.py @@ -1,7 +1,8 @@ from __future__ import annotations +import asyncio import json -from collections.abc import AsyncIterator, Mapping +from collections.abc import AsyncIterator, Awaitable, Mapping from contextlib import AsyncExitStack from dataclasses import dataclass from json import JSONDecodeError @@ -9,6 +10,7 @@ from typing import cast import aiohttp +import anyio from app.core.clients.http import lease_http_session from app.core.crypto import TokenEncryptor @@ -101,42 +103,84 @@ class SourceUsageHolder: timings: SourceTimings | None = None +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> None: + """Finish owned upstream cleanup even if the caller is cancelled again.""" + + task = asyncio.ensure_future(awaitable) + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + return + except asyncio.CancelledError: + if task.cancelled(): + raise + + +async def _await_result_deferring_cancellation(awaitable: Awaitable[object]) -> bool: + """Finish owned cleanup and report whether cancellation arrived mid-flight.""" + + task = asyncio.ensure_future(awaitable) + cancellation_deferred = False + with anyio.CancelScope(shield=True): + while True: + try: + await asyncio.shield(task) + return cancellation_deferred + except asyncio.CancelledError: + if task.cancelled(): + raise + cancellation_deferred = True + + async def forward_chat_completion( source: ModelSource, payload: dict[str, JsonValue], *, encryptor: TokenEncryptor | None = None, ) -> SourceChatCompletion: + stack = AsyncExitStack() try: - async with lease_http_session() as session: - timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) - async with session.post( + session = await stack.enter_async_context(lease_http_session()) + timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) + response = await stack.enter_async_context( + session.post( _source_url(source, "/chat/completions"), headers=_source_headers(source, encryptor=encryptor), json=payload, timeout=timeout, - ) as response: - data = await _response_json(response) - if response.status >= 400: - raise ModelSourceForwardingError( - status_code=response.status, - payload=_redact_source_error_payload( - _error_payload(data), - source, - encryptor=encryptor, - ), - upstream_status_code=response.status, - ) - if data is None: - raise _invalid_upstream_response_error(response.status) - return SourceChatCompletion( - payload=data, - usage=_usage_from_chat_payload(data), - timings=_timings_from_payload(data), - upstream_status_code=response.status, - ) + ) + ) + data = await _response_json(response) + if response.status >= 400: + raise ModelSourceForwardingError( + status_code=response.status, + payload=_redact_source_error_payload( + _error_payload(data), + source, + encryptor=encryptor, + ), + upstream_status_code=response.status, + ) + if data is None: + raise _invalid_upstream_response_error(response.status) + result = SourceChatCompletion( + payload=data, + usage=_usage_from_chat_payload(data), + timings=_timings_from_payload(data), + upstream_status_code=response.status, + ) except (aiohttp.ClientError, TimeoutError) as exc: + await _await_cleanup_deferring_cancellation(stack.aclose()) raise _unreachable_error(exc) from exc + except BaseException: + await _await_cleanup_deferring_cancellation(stack.aclose()) + raise + + cleanup_cancelled = await _await_result_deferring_cancellation(stack.aclose()) + if cleanup_cancelled: + raise asyncio.CancelledError + return result async def stream_chat_completion( @@ -150,10 +194,15 @@ async def stream_chat_completion( stack, response = await _open_source_stream(source, "/chat/completions", payload, encryptor=encryptor) async def body() -> AsyncIterator[bytes]: - async with stack: + try: async for chunk in response.content.iter_chunked(4096): usage_parser.feed(chunk) yield chunk + finally: + # A plain ``async with stack`` unwinds unshielded: repeated + # cancellation delivery can interrupt ``__aexit__`` mid-unwind and + # leak the pooled HTTP session lease. + await _await_cleanup_deferring_cancellation(stack.aclose()) return SourceChatStream(body=body(), usage_holder=usage_holder, upstream_status_code=response.status) @@ -260,10 +309,15 @@ async def stream_responses( stack, response = await _open_source_stream(source, "/responses", payload, encryptor=encryptor) async def body() -> AsyncIterator[bytes]: - async with stack: + try: async for chunk in response.content.iter_chunked(4096): usage_parser.feed(chunk) yield chunk + finally: + # A plain ``async with stack`` unwinds unshielded: repeated + # cancellation delivery can interrupt ``__aexit__`` mid-unwind and + # leak the pooled HTTP session lease. + await _await_cleanup_deferring_cancellation(stack.aclose()) return SourceResponsesStream(body=body(), usage_holder=usage_holder, upstream_status_code=response.status) @@ -309,10 +363,10 @@ async def _open_source_stream( ) return stack, response except (aiohttp.ClientError, TimeoutError) as exc: - await stack.aclose() + await _await_cleanup_deferring_cancellation(stack.aclose()) raise _unreachable_error(exc) from exc except BaseException: - await stack.aclose() + await _await_cleanup_deferring_cancellation(stack.aclose()) raise diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 470e7a9376..92a96f7fc5 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -577,6 +577,7 @@ async def _stream_once( settlement.record_success = False settlement.account_health_error = True settlement.error = {"message": error_message} + terminal_event_seen = True yield format_sse_event( response_failed_event( error_code, @@ -595,6 +596,7 @@ async def _stream_once( settlement.record_success = False settlement.account_health_error = True settlement.error = {"message": error_message} + terminal_event_seen = True yield format_sse_event( response_failed_event( error_code, @@ -609,12 +611,7 @@ async def _stream_once( first_payload = parse_sse_data_json(first) event = parse_sse_event_payload(first_payload) event_type = _event_type_from_payload(event, first_payload) - terminal_event_seen = event_type in { - "response.completed", - "response.failed", - "response.incomplete", - "error", - } + terminal_event_seen = False preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( @@ -754,6 +751,8 @@ async def _stream_once( else: if first_payload is not None and not preserve_raw_sse_line: first = format_sse_event(first_payload) + if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: + terminal_event_seen = True if latency_first_token_ms is None: latency_first_token_ms = _ttft_event_latency_ms( event_type, first_payload, ttft_reasoning_deltas, attempt_started_at @@ -768,8 +767,6 @@ async def _stream_once( event_payload = parse_sse_data_json(line) event = parse_sse_event_payload(event_payload) event_type = _event_type_from_payload(event, event_payload) - if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: - terminal_event_seen = True preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" if ( enforce_openai_sdk_contract @@ -940,6 +937,8 @@ async def _stream_once( settlement.downstream_visible = True if event_type in _facade()._TEXT_DELTA_EVENT_TYPES: settlement.downstream_text_visible = True + if event_type in {"response.completed", "response.failed", "response.incomplete", "error"}: + terminal_event_seen = True yield line if not terminal_event_seen: status, error_code, error_message, failure_metadata = _mark_upstream_stream_incomplete(settlement) @@ -1005,7 +1004,8 @@ async def _stream_once( except _TerminalStreamError: raise except (asyncio.CancelledError, GeneratorExit): - status, error_code, error_message, failure_metadata = _mark_downstream_stream_cancelled(settlement) + if not terminal_event_seen: + status, error_code, error_message, failure_metadata = _mark_downstream_stream_cancelled(settlement) raise except Exception: if settlement.downstream_visible: diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index e114e9648e..0b28853a44 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -7,9 +7,10 @@ import sys import time from dataclasses import replace -from typing import Any, AsyncIterator, Mapping, cast +from typing import Any, AsyncGenerator, AsyncIterator, Mapping, TypeVar, cast import aiohttp +import anyio from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind from app.core.balancer import failover_decision @@ -86,6 +87,26 @@ _HTTP_DOWNSTREAM_TRANSPORT_POLICIES = frozenset({"smart", "always_http", "always_websocket", "pinned"}) logger = logging.getLogger(__name__) +_TaskResultT = TypeVar("_TaskResultT") + + +async def _await_task_deferring_cancellation( + task: asyncio.Task[_TaskResultT], +) -> tuple[_TaskResultT, asyncio.CancelledError | None]: + """Finish critical cleanup while preserving the caller's cancellation.""" + + cancellation: asyncio.CancelledError | None = None + # The anyio shield keeps a level-cancelled Starlette scope from re-raising + # into every ``await``, which would otherwise busy-spin this loop until the + # owned task completes. + with anyio.CancelScope(shield=True): + while True: + try: + return await asyncio.shield(task), cancellation + except asyncio.CancelledError as exc: + if task.cancelled(): + raise + cancellation = cancellation or exc def _facade() -> Any: @@ -484,6 +505,32 @@ async def _drain_pending_post_refresh_penalty_on_terminal( return settled return True + async def _finalize_terminal_settlement_after_downstream_close( + current_settlement: _StreamSettlement, + account: Account, + ) -> None: + nonlocal settled + + async def _finalize() -> None: + nonlocal settled + if not settled: + settled = await _settle_stream_usage_before_pending_penalty(current_settlement) + if not settled: + return + if current_settlement.account_health_error: + await proxy._handle_stream_error( + account, + _stream_settlement_error_payload(current_settlement), + current_settlement.error_code or "upstream_error", + ) + elif current_settlement.record_success: + await proxy._load_balancer.record_success(account) + + finalize_task = asyncio.create_task(_finalize(), name=f"stream-terminal-settlement-{request_id}") + _, cancellation = await _await_task_deferring_cancellation(finalize_task) + if cancellation is not None: + raise cancellation + async def _wait_for_process_network_recovery( account: Account, *, @@ -561,35 +608,45 @@ async def _stream_post_refresh_with_capacity_recovery( settlement: _StreamSettlement, can_try_other_account: bool, tool_call_dedupe: _WebSocketUpstreamControl, - ) -> AsyncIterator[str]: + ) -> AsyncGenerator[str, None]: nonlocal last_transient_exc transient_retries = 0 - async def _iter_stream_once() -> AsyncIterator[str]: + async def _iter_stream_once() -> AsyncGenerator[str, None]: + inner_stream = proxy._stream_once( + account, + payload, + headers, + request_id, + False, + request_started_at=start, + allow_transient_retry=True, + api_key=api_key, + api_key_reservation=api_key_reservation, + settlement=settlement, + suppress_text_done_events=suppress_text_done_events, + upstream_stream_transport=upstream_stream_transport, + request_transport=request_transport, + concurrency_caps=concurrency_caps, + useragent=useragent, + useragent_group=useragent_group, + conversation_id=conversation_id, + client_ip=client_ip, + tool_call_dedupe=tool_call_dedupe, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + ) try: - async for line in proxy._stream_once( - account, - payload, - headers, - request_id, - False, - request_started_at=start, - allow_transient_retry=True, - api_key=api_key, - api_key_reservation=api_key_reservation, - settlement=settlement, - suppress_text_done_events=suppress_text_done_events, - upstream_stream_transport=upstream_stream_transport, - request_transport=request_transport, - concurrency_caps=concurrency_caps, - useragent=useragent, - useragent_group=useragent_group, - conversation_id=conversation_id, - client_ip=client_ip, - tool_call_dedupe=tool_call_dedupe, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - yield line + try: + async for line in inner_stream: + yield line + finally: + close_task = asyncio.create_task( + inner_stream.aclose(), + name=f"stream-post-refresh-inner-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation except ProxyResponseError as exc: if is_confirmed_pre_dispatch_transport_error(exc): # Keep dispatch provenance intact for the outer account @@ -625,8 +682,28 @@ async def _iter_stream_once() -> AsyncIterator[str]: _facade()._remaining_budget_seconds(deadline) ) try: - async for line in _iter_stream_once(): - yield line + attempt_stream = _iter_stream_once() + try: + try: + async for line in attempt_stream: + yield line + finally: + close_task = asyncio.create_task( + attempt_stream.aclose(), + name=f"stream-post-refresh-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation + except (asyncio.CancelledError, GeneratorExit): + # A terminal frame may already have been yielded when + # downstream cancellation is delivered on the next + # generator resume. Finalize that terminal usage and + # health result before propagating cancellation so the + # reservation is not released as abandoned. + if settlement.status in {"success", "error"} and not settled: + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise network_recovery.log_recovered() return except _TerminalStreamError: @@ -1769,7 +1846,7 @@ async def _retry_account_model_rejection( ) try: settlement = _StreamSettlement() - async for line in proxy._stream_once( + inner_stream = proxy._stream_once( account, payload, headers, @@ -1808,8 +1885,27 @@ async def _retry_account_model_rejection( ), tool_call_dedupe=tool_call_dedupe, enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - yield line + ) + try: + async for line in inner_stream: + yield line + finally: + close_task = asyncio.create_task( + inner_stream.aclose(), + name=f"stream-inner-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation + except (asyncio.CancelledError, GeneratorExit): + # A terminal frame may already have been yielded when + # downstream cancellation is delivered on the next + # generator resume. Finalize that terminal usage and + # health result before propagating cancellation so the + # reservation is not released as abandoned. + if settlement.status in {"success", "error"} and not settled: + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise except (_TransientStreamError, ProxyResponseError) as tex: if account.id == account_model_replacement_account_id: # Account/model routing gets exactly one selected @@ -1873,7 +1969,6 @@ async def _retry_account_model_rejection( account.id, error_code, ) - yield format_sse_event(event) settlement.record_success = False settlement.error_code = error_code settlement.error_message = error_message @@ -1882,6 +1977,11 @@ async def _retry_account_model_rejection( else: settlement.error = tex.error settlement.account_health_error = _facade()._should_penalize_stream_error(error_code) + try: + yield format_sse_event(event) + except (asyncio.CancelledError, GeneratorExit): + await _finalize_terminal_settlement_after_downstream_close(settlement, account) + raise settled = await _settle_stream_usage_before_pending_penalty(settlement) if settled and settlement.account_health_error: await proxy._handle_stream_error( @@ -2475,13 +2575,27 @@ async def _retry_account_model_rejection( and account.id != file_preferred_account_id and attempt < max_attempts - 1 ) - async for line in _stream_post_refresh_with_capacity_recovery( + post_refresh_stream = _stream_post_refresh_with_capacity_recovery( account, settlement=settlement, can_try_other_account=can_try_other_account, tool_call_dedupe=tool_call_dedupe, - ): - yield line + ) + try: + async for line in post_refresh_stream: + yield line + finally: + # Closing this generator runs its internal + # cancellation-safe close/terminal finalization; + # without an owned aclose() the child would stay + # suspended after a downstream disconnect. + close_task = asyncio.create_task( + post_refresh_stream.aclose(), + name=f"stream-post-refresh-outer-close-{request_id}", + ) + _, close_cancellation = await _await_task_deferring_cancellation(close_task) + if close_cancellation is not None: + raise close_cancellation except ProxyResponseError as retry_exc: if _facade()._is_proxy_budget_exhausted_error(retry_exc): await _settle_process_network_budget_exhaustion(account, settlement) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index b0930fad0e..b63a672621 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, replace from datetime import datetime, timezone from json import JSONDecodeError -from typing import Any, Final, Literal, Protocol, cast +from typing import Any, Final, Literal, Protocol, TypeVar, cast from uuid import uuid4 import anyio @@ -294,6 +294,7 @@ from app.modules.usage.updater import UsageUpdater logger = logging.getLogger(__name__) +_T = TypeVar("_T") _REASONING_SUMMARY_DELTA_TYPES = frozenset({"response.reasoning_summary_text.delta"}) _REASONING_SUMMARY_DONE_TYPES = frozenset( @@ -2060,15 +2061,28 @@ async def _rate_limit_headers_for_request( async def _release_reservation_deferring_cancellation( reservation: ApiKeyUsageReservationData, ) -> None: + await _await_cleanup_deferring_cancellation(_release_reservation(reservation)) + + +async def _await_result_deferring_cancellation(awaitable: Awaitable[_T]) -> tuple[_T, bool]: + """Finish an owned awaitable despite repeated cancellation and report whether cancellation arrived.""" + + task = asyncio.ensure_future(awaitable) + cancellation_deferred = False with anyio.CancelScope(shield=True): - task = asyncio.create_task(_release_reservation(reservation)) while True: try: - await asyncio.shield(task) - return + return await asyncio.shield(task), cancellation_deferred except asyncio.CancelledError: if task.cancelled(): raise + cancellation_deferred = True + + +async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> None: + """Finish a required cleanup operation despite repeated cancellation delivery.""" + + await _await_result_deferring_cancellation(awaitable) async def _rate_limit_headers_with_reservation_cleanup( @@ -4736,6 +4750,36 @@ async def _source_chat_completion_response( upstream_status_code=exc.upstream_status_code, ) return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + except asyncio.CancelledError: + release_exc: BaseException | None = None + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + error_code="client_disconnected", + error_message="client disconnected during source stream setup", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release source stream setup reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + raise + except BaseException: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) + raise if _reservation_requires_usage(reservation): return await _buffered_limited_source_chat_stream_response( request, @@ -4777,6 +4821,36 @@ async def _source_chat_completion_response( upstream_status_code=exc.upstream_status_code, ) return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + except asyncio.CancelledError: + release_exc: BaseException | None = None + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + error_code="client_disconnected", + error_message="client disconnected during source request setup", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release source request setup reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + raise + except BaseException: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) + raise if result.usage is None and _reservation_requires_usage(reservation): await _release_reservation(reservation) @@ -4797,34 +4871,60 @@ async def _source_chat_completion_response( ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=result.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=result.usage) + ) + if settlement_deferred_cancellation: + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=result.usage, + timings=result.timings, + error_code="client_disconnected", + error_message="client disconnected during source usage settlement", + upstream_status_code=result.upstream_status_code, + ) + ) + raise asyncio.CancelledError if not settled: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="error", - error_code="usage_settlement_failed", - error_message="source usage settlement failed", - upstream_status_code=result.upstream_status_code, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + upstream_status_code=result.upstream_status_code, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return _logged_error_json_response( request, 502, _source_usage_settlement_failed_error(), headers=rate_limit_headers, ) - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="success", - usage=result.usage, - timings=result.timings, - upstream_status_code=result.upstream_status_code, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=result.usage, + timings=result.timings, + upstream_status_code=result.upstream_status_code, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return JSONResponse(content=result.payload, status_code=200, headers=rate_limit_headers) @@ -4870,13 +4970,44 @@ async def _buffered_limited_source_chat_stream_response( error_message="source stream buffer limit exceeded", ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - except asyncio.CancelledError: + except asyncio.CancelledError as cancel_exc: # Starlette cancels this task when the downstream client disconnects; # CancelledError is a BaseException, so without this branch the # reservation would stay charged until stale-reservation cleanup. - await _aclose_stream(stream) - await _release_reservation(reservation) - raise + close_exc: BaseException | None = None + release_exc: BaseException | None = None + try: + await _await_cleanup_deferring_cancellation(_aclose_stream(stream)) + except BaseException as exc: + close_exc = exc + if reservation is not None: + try: + await _release_reservation_deferring_cancellation(reservation) + except BaseException as exc: + release_exc = exc + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code="client_disconnected", + error_message="client disconnected during source stream buffering", + ) + ) + if release_exc is not None: + logger.warning( + "Failed to release buffered source stream reservation after client disconnect source_id=%s model=%s", + source.id, + model, + exc_info=release_exc, + ) + if close_exc is not None: + raise close_exc + raise cancel_exc except ModelSourceForwardingError as exc: await _release_reservation(reservation) await _log_source_chat_completion( @@ -4926,32 +5057,57 @@ async def _buffered_limited_source_chat_stream_response( ) return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + ) + if settlement_deferred_cancellation: + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="cancelled", + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code="client_disconnected", + error_message="client disconnected during source stream usage settlement", + ) + ) + raise asyncio.CancelledError if not settled: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="error", - error_code="usage_settlement_failed", - error_message="source usage settlement failed", + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError return _logged_error_json_response( request, 502, _source_usage_settlement_failed_error(), headers=rate_limit_headers, ) - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status="success", - usage=usage_holder.usage, - timings=usage_holder.timings, + _, log_deferred_cancellation = await _await_result_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=usage_holder.usage, + timings=usage_holder.timings, + ) ) + if log_deferred_cancellation: + raise asyncio.CancelledError async def body() -> AsyncIterator[bytes]: for chunk in chunks: @@ -4991,8 +5147,11 @@ async def _source_chat_stream_with_settlement( status = "cancelled" error_code = "client_disconnected" error_message = "client disconnected before stream completed" - await _aclose_stream(stream) - await _release_reservation(reservation) + try: + await _await_cleanup_deferring_cancellation(_aclose_stream(stream)) + finally: + if reservation is not None: + await _release_reservation_deferring_cancellation(reservation) raise except ModelSourceForwardingError as exc: status = "error" @@ -5007,7 +5166,14 @@ async def _source_chat_stream_with_settlement( await _release_reservation(reservation) raise else: - settled = await _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + settled, settlement_deferred_cancellation = await _await_result_deferring_cancellation( + _settle_source_reservation(reservation, source=source, model=model, usage=usage_holder.usage) + ) + if settlement_deferred_cancellation: + status = "cancelled" + error_code = "client_disconnected" + error_message = "client disconnected during source usage settlement" + raise asyncio.CancelledError if not settled: status = "error" error_code = "usage_settlement_failed" @@ -5023,17 +5189,19 @@ async def _source_chat_stream_with_settlement( model, ) finally: - await _log_source_chat_completion( - request, - source=source, - api_key=api_key, - model=model, - status=status, - usage=usage_holder.usage, - timings=usage_holder.timings, - error_code=error_code, - error_message=error_message, - upstream_status_code=None, + await _await_cleanup_deferring_cancellation( + _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status=status, + usage=usage_holder.usage, + timings=usage_holder.timings, + error_code=error_code, + error_message=error_message, + upstream_status_code=None, + ) ) diff --git a/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml b/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml new file mode 100644 index 0000000000..84cfc12459 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-06 diff --git a/openspec/changes/fix-disconnect-cleanup-leak/design.md b/openspec/changes/fix-disconnect-cleanup-leak/design.md new file mode 100644 index 0000000000..970214b6d6 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/design.md @@ -0,0 +1,5 @@ +# Design + +- Run required teardown in an owned asyncio task inside an anyio shield and tolerate repeated cancellation delivery until that task completes. +- Use the same cancellation-deferring primitive for source-chat upstream closure, API-key reservation release, and request-log persistence. +- Track whether a Responses terminal event was observed; only classify cancellation as `client_disconnected` when no terminal event has been observed. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/proposal.md b/openspec/changes/fix-disconnect-cleanup-leak/proposal.md new file mode 100644 index 0000000000..d7bad1a580 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/proposal.md @@ -0,0 +1,5 @@ +# Fix disconnect cleanup and terminal settlement + +Streaming and source-chat disconnect cleanup must complete even while Starlette/anyio is cancelling the request task. A completed terminal Responses event must remain authoritative after a later downstream disconnect. + +This change hardens database/session teardown, source-chat reservation and request-log cleanup, and Responses stream settlement classification. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md b/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md new file mode 100644 index 0000000000..8ee6fc5221 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/specs/api-keys/spec.md @@ -0,0 +1,10 @@ +## ADDED Requirements + +### Requirement: Disconnect cleanup settles source-chat reservations + +When a source-chat request is cancelled or its streaming body is closed, the proxy MUST close the upstream iterator, release its API-key reservation, and write or explicitly abort the source request-log row despite repeated cancellation delivery. + +#### Scenario: Client disconnects during source stream + +- **WHEN** the downstream client disconnects before source-stream completion +- **THEN** the reservation is released and the source request is logged as an aborted/error request. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md b/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..0c0aba73f5 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/specs/responses-api-compat/spec.md @@ -0,0 +1,11 @@ +## ADDED Requirements + +### Requirement: Terminal stream settlement is immutable after delivery + +When a Responses stream has observed and delivered a terminal event (`response.completed`, `response.failed`, `response.incomplete`, or `error`), a later downstream cancellation MUST NOT rewrite the terminal status, error, usage, or account-health settlement. + +#### Scenario: Disconnect after terminal event + +- **WHEN** the downstream closes after receiving a terminal event +- **THEN** the request log and settlement retain the terminal event's outcome +- **AND** the proxy does not record `client_disconnected` for that stream. diff --git a/openspec/changes/fix-disconnect-cleanup-leak/tasks.md b/openspec/changes/fix-disconnect-cleanup-leak/tasks.md new file mode 100644 index 0000000000..e473d9fe28 --- /dev/null +++ b/openspec/changes/fix-disconnect-cleanup-leak/tasks.md @@ -0,0 +1,5 @@ +- [x] Harden pooled session rollback/close against repeated cancellation delivery. +- [x] Complete source-chat stream close, reservation release, and request-log persistence on disconnect. +- [x] Release non-stream source-chat reservations on cancellation and unexpected upstream failures. +- [x] Preserve terminal Responses settlement after downstream disconnect. +- [x] Run regression and named proxy/integration suites. diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index 7368447162..0405933703 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -1060,6 +1060,741 @@ async def cancelled_stream() -> AsyncIterator[bytes]: assert stream_closed is True +@pytest.mark.asyncio +async def test_cancelled_buffered_stream_releases_reservation_when_close_fails(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + + released: list[object] = [] + + async def record_release(reservation: object) -> None: + released.append(reservation) + + async def fail_close(_stream: object) -> None: + raise RuntimeError("close failed") + + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_aclose_stream", fail_close) + + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_cancelled_close_fails", + name="cancelled-close-fails", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_cancelled_close_fails", + key_id="key_cancelled_close_fails", + model="cancelled-model", + ) + + with pytest.raises(RuntimeError, match="close failed"): + await proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="cancelled-model", + reservation=reservation, + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) + + assert released == [reservation] + + +@pytest.mark.asyncio +async def test_cancelled_buffered_stream_finishes_usage_settlement(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsage, SourceUsageHolder + + settlement_started = asyncio.Event() + settlement_can_finish = asyncio.Event() + settled: list[object] = [] + logs: list[dict[str, object]] = [] + + async def settle(reservation: object, **_kwargs: object) -> bool: + settlement_started.set() + await settlement_can_finish.wait() + settled.append(reservation) + return True + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) + + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def complete_stream() -> AsyncIterator[bytes]: + yield b"data: done\n\n" + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_settlement_cancelled", + name="settlement-cancelled", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_settlement_cancelled", + key_id="key_settlement_cancelled", + model="cancelled-model", + ) + usage_holder = SourceUsageHolder(usage=SourceUsage(input_tokens=3, output_tokens=5)) + + task = asyncio.create_task( + proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="cancelled-model", + reservation=reservation, + stream=complete_stream(), + usage_holder=usage_holder, + rate_limit_headers={}, + ) + ) + await asyncio.wait_for(settlement_started.wait(), timeout=1) + task.cancel() + settlement_can_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert settled == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["usage"] == usage_holder.usage + + +@pytest.mark.asyncio +async def test_cancelled_buffered_stream_logs_disconnect(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + + released: list[object] = [] + logs: list[dict[str, object]] = [] + + async def record_release(reservation: object) -> None: + released.append(reservation) + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) + + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_buffered_cancelled_log", + name="buffered-cancelled-log", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_buffered_cancelled_log", + key_id="key_buffered_cancelled_log", + model="cancelled-model", + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="cancelled-model", + reservation=reservation, + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) + + assert released == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source stream buffering" + + +@pytest.mark.asyncio +async def test_source_completion_success_log_finishes_after_cancellation(async_client, monkeypatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceChatCompletion, SourceUsage + + log_started = asyncio.Event() + allow_log_finish = asyncio.Event() + logs: list[dict[str, object]] = [] + + async def fake_forward(*_args: object, **_kwargs: object) -> SourceChatCompletion: + return SourceChatCompletion( + payload={"id": "chatcmpl_cancelled_after_settlement"}, + usage=SourceUsage(input_tokens=3, output_tokens=5), + timings=None, + upstream_status_code=200, + ) + + async def settle(*_args: object, **_kwargs: object) -> bool: + return True + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(kwargs) + log_started.set() + await allow_log_finish.wait() + + monkeypatch.setattr(proxy_api, "forward_chat_completion", fake_forward) + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_completion_cancelled_log", + name="completion-cancelled-log", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "completion-cancelled-log", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + } + ) + + task = asyncio.create_task( + proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="completion-cancelled-log", + api_key=None, + reservation=None, + rate_limit_headers={}, + ) + ) + await asyncio.wait_for(log_started.wait(), timeout=1) + task.cancel() + allow_log_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert logs[-1]["status"] == "success" + assert logs[-1]["usage"] == SourceUsage(input_tokens=3, output_tokens=5) + + +@pytest.mark.asyncio +async def test_source_stream_setup_cancellation_logs_visible_error_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def cancel_during_open(*_args: object, **_kwargs: object) -> object: + raise asyncio.CancelledError + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "stream_source_chat_completion", cancel_during_open) + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_stream_setup_cancel", + name="stream-setup-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_stream_setup_cancel", + key_id="key_stream_setup_cancel", + model="stream-setup-cancel", + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "stream-setup-cancel", + "messages": [{"role": "user", "content": "hello"}], + "stream": True, + } + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="stream-setup-cancel", + api_key=None, + reservation=reservation, + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs == [ + { + "source": source, + "api_key": None, + "model": "stream-setup-cancel", + "status": "cancelled", + "error_code": "client_disconnected", + "error_message": "client disconnected during source stream setup", + } + ] + + +@pytest.mark.asyncio +async def test_source_request_setup_cancellation_logs_disconnect_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.core.openai.chat_requests import ChatCompletionsRequest + from app.db.models import ModelSource + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def cancel_during_forward(*_args: object, **_kwargs: object) -> object: + raise asyncio.CancelledError + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "forward_chat_completion", cancel_during_forward) + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_request_setup_cancel_release_fail", + name="request-setup-cancel-release-fail", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_request_setup_cancel_release_fail", + key_id="key_request_setup_cancel_release_fail", + model="request-setup-cancel-release-fail", + ) + payload = ChatCompletionsRequest.model_validate( + { + "model": "request-setup-cancel-release-fail", + "messages": [{"role": "user", "content": "hello"}], + "stream": False, + } + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._source_chat_completion_response( + request, + payload, + source=source, + model="request-setup-cancel-release-fail", + api_key=None, + reservation=reservation, + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs == [ + { + "source": source, + "api_key": None, + "model": "request-setup-cancel-release-fail", + "status": "cancelled", + "error_code": "client_disconnected", + "error_message": "client disconnected during source request setup", + } + ] + + +@pytest.mark.asyncio +async def test_buffered_stream_cancellation_logs_disconnect_even_if_release_fails( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsageHolder + + logs: list[dict[str, object]] = [] + release_attempts: list[object] = [] + + async def fail_release(reservation: object) -> None: + release_attempts.append(reservation) + raise RuntimeError("sqlite busy") + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "_release_reservation_deferring_cancellation", fail_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def cancelled_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + raise asyncio.CancelledError() + + request = Request( + { + "type": "http", + "method": "POST", + "path": "/v1/chat/completions", + "headers": [], + "client": ("127.0.0.1", 1234), + "query_string": b"", + } + ) + source = ModelSource( + id="src_buffered_cancel_release_fail", + name="buffered-cancel-release-fail", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_buffered_cancel_release_fail", + key_id="key_buffered_cancel_release_fail", + model="buffered-cancel-release-fail", + ) + + with pytest.raises(asyncio.CancelledError): + await proxy_api._buffered_limited_source_chat_stream_response( + request, + source=source, + api_key=None, + model="buffered-cancel-release-fail", + reservation=reservation, + stream=cancelled_stream(), + usage_holder=SourceUsageHolder(), + rate_limit_headers={}, + ) + + assert release_attempts == [reservation] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source stream buffering" + + +@pytest.mark.asyncio +async def test_source_stream_body_teardown_survives_repeated_cancellation(monkeypatch: pytest.MonkeyPatch): + from contextlib import AsyncExitStack + + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + stream_blocked = asyncio.Event() + release_started = asyncio.Event() + allow_release = asyncio.Event() + release_finished = asyncio.Event() + + class _SlowLease: + async def __aenter__(self) -> None: + return None + + async def __aexit__(self, exc_type: object, exc: object, tb: object) -> bool: + release_started.set() + await allow_release.wait() + release_finished.set() + return False + + stack = AsyncExitStack() + await stack.enter_async_context(_SlowLease()) + + class _FakeContent: + def iter_chunked(self, _size: int) -> AsyncIterator[bytes]: + async def gen() -> AsyncIterator[bytes]: + yield b"data: chunk\n\n" + stream_blocked.set() + await asyncio.Event().wait() + + return gen() + + class _FakeResponse: + status = 200 + content = _FakeContent() + + async def fake_open(*_args: object, **_kwargs: object) -> object: + return stack, _FakeResponse() + + monkeypatch.setattr(forwarding_module, "_open_source_stream", fake_open) + + source = ModelSource( + id="src_body_teardown_repeated_cancel", + name="body-teardown-repeated-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + stream = await forwarding_module.stream_chat_completion(source, {"model": "body-teardown"}) + + async def consume() -> None: + async for _chunk in stream.body: + pass + + task = asyncio.create_task(consume()) + await asyncio.wait_for(stream_blocked.wait(), timeout=1) + await asyncio.sleep(0) + + task.cancel() + await asyncio.wait_for(release_started.wait(), timeout=1) + # Second cancellation delivery while the exit stack is unwinding: teardown + # must still return the pooled HTTP lease. + task.cancel() + await asyncio.sleep(0) + allow_release.set() + + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(task, timeout=1) + + assert release_finished.is_set() + + +@pytest.mark.asyncio +async def test_open_source_stream_cleanup_finishes_after_cancellation(monkeypatch: pytest.MonkeyPatch): + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + cleanup_started = asyncio.Event() + allow_cleanup_finish = asyncio.Event() + cleanup_finished = asyncio.Event() + + class _FailingPostContext: + async def __aenter__(self): + raise asyncio.CancelledError() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + return False + + class _Session: + def post(self, *_args: object, **_kwargs: object) -> _FailingPostContext: + return _FailingPostContext() + + class _SessionLease: + async def __aenter__(self) -> _Session: + return _Session() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + cleanup_started.set() + await allow_cleanup_finish.wait() + cleanup_finished.set() + return False + + monkeypatch.setattr(forwarding_module, "lease_http_session", lambda: _SessionLease()) + + source = ModelSource( + id="src_open_cancelled_cleanup", + name="open-cancelled-cleanup", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + + task = asyncio.create_task( + forwarding_module._open_source_stream( + source, + "/chat/completions", + {"model": "open-cancelled-cleanup"}, + encryptor=None, + ) + ) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + task.cancel() + allow_cleanup_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert cleanup_finished.is_set() is True + + +@pytest.mark.asyncio +async def test_forward_chat_completion_cleanup_finishes_after_cancellation(monkeypatch: pytest.MonkeyPatch): + import app.modules.model_sources.forwarding as forwarding_module + from app.db.models import ModelSource + + cleanup_started = asyncio.Event() + allow_cleanup_finish = asyncio.Event() + cleanup_finished = asyncio.Event() + + class _Response: + status = 200 + + async def json(self, content_type=None): + del content_type + return { + "id": "chatcmpl_forward_cancelled_cleanup", + "usage": {"prompt_tokens": 3, "completion_tokens": 5}, + } + + class _PostContext: + async def __aenter__(self) -> _Response: + return _Response() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + return False + + class _Session: + def post(self, *_args: object, **_kwargs: object) -> _PostContext: + return _PostContext() + + class _SessionLease: + async def __aenter__(self) -> _Session: + return _Session() + + async def __aexit__(self, exc_type, exc, tb) -> bool: + del exc_type, exc, tb + cleanup_started.set() + await allow_cleanup_finish.wait() + cleanup_finished.set() + return False + + monkeypatch.setattr(forwarding_module, "lease_http_session", lambda: _SessionLease()) + + source = ModelSource( + id="src_forward_cancelled_cleanup", + name="forward-cancelled-cleanup", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + + task = asyncio.create_task( + forwarding_module.forward_chat_completion( + source, + {"model": "forward-cancelled-cleanup", "messages": [{"role": "user", "content": "hello"}]}, + ) + ) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + task.cancel() + allow_cleanup_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert cleanup_finished.is_set() is True + + @pytest.mark.asyncio async def test_downstream_disconnect_closes_source_stream(async_client, monkeypatch): from starlette.requests import Request @@ -1205,6 +1940,94 @@ async def source_stream() -> AsyncIterator[bytes]: assert aggregate.top_error is None +@pytest.mark.asyncio +async def test_source_stream_settlement_cancellation_logs_cancelled_not_success(monkeypatch: pytest.MonkeyPatch): + from starlette.requests import Request + + import app.modules.proxy.api as proxy_api + from app.db.models import ModelSource + from app.modules.model_sources.forwarding import SourceUsage, SourceUsageHolder + + settle_started = asyncio.Event() + allow_settle_finish = asyncio.Event() + released: list[object] = [] + logs: list[dict[str, object]] = [] + + async def settle(*_args: object, **_kwargs: object) -> bool: + settle_started.set() + await allow_settle_finish.wait() + return True + + async def record_release(reservation: object) -> None: + released.append(reservation) + + async def record_log(*_args: object, **kwargs: object) -> None: + logs.append(dict(kwargs)) + + monkeypatch.setattr(proxy_api, "_settle_source_reservation", settle) + monkeypatch.setattr(proxy_api, "_release_reservation", record_release) + monkeypatch.setattr(proxy_api, "_log_source_chat_completion", record_log) + + async def source_stream() -> AsyncIterator[bytes]: + yield b"data: partial\n\n" + + request = Request( + { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/v1/chat/completions", + "raw_path": b"/v1/chat/completions", + "query_string": b"", + "headers": [], + "client": ("127.0.0.1", 0), + "server": ("testserver", 80), + } + ) + source = ModelSource( + id="src_stream_settlement_cancel", + name="stream-settlement-cancel", + kind="openai_compatible", + base_url="http://127.0.0.1:9/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=False, + ) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_stream_settlement_cancel", + key_id="key_stream_settlement_cancel", + model="stream-settlement-cancel", + ) + usage_holder = SourceUsageHolder(usage=SourceUsage(input_tokens=3, output_tokens=5)) + + async def consume_stream() -> None: + async for _chunk in proxy_api._source_chat_stream_with_settlement( + source_stream(), + usage_holder=usage_holder, + request=request, + source=source, + api_key=None, + model="stream-settlement-cancel", + reservation=reservation, + ): + pass + + task = asyncio.create_task(consume_stream()) + await asyncio.wait_for(settle_started.wait(), timeout=1) + task.cancel() + allow_settle_finish.set() + + with pytest.raises(asyncio.CancelledError): + await task + + assert released == [] + assert logs[-1]["status"] == "cancelled" + assert logs[-1]["error_code"] == "client_disconnected" + assert logs[-1]["error_message"] == "client disconnected during source usage settlement" + assert logs[-1]["usage"] == usage_holder.usage + + @pytest.mark.asyncio async def test_opportunistic_key_routes_to_source_without_account_pool(async_client, source_upstream): await _enable_api_key_auth(async_client) diff --git a/tests/integration/test_proxy_responses.py b/tests/integration/test_proxy_responses.py index 66bed5957f..fb8df513c8 100644 --- a/tests/integration/test_proxy_responses.py +++ b/tests/integration/test_proxy_responses.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import base64 import json from collections.abc import Mapping @@ -20,6 +21,7 @@ from app.core.utils.time import utcnow from app.db.models import Account, DashboardSettings, RequestLog from app.db.session import SessionLocal +from app.modules.api_keys.service import ApiKeyUsageReservationData from app.modules.proxy._service.streaming import retry as streaming_retry_module from app.modules.request_logs.repository import RequestLogsRepository from app.modules.usage.repository import AdditionalUsageRepository @@ -2055,6 +2057,311 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert log.transport == "http" +@pytest.mark.asyncio +async def test_backend_responses_terminal_disconnect_finalizes_settlement_and_success( + async_client, + app_instance, + monkeypatch, +): + email = "terminal-disconnect@example.com" + raw_account_id = "acc_terminal_disconnect" + expected_account_id = generate_unique_account_id(raw_account_id, email) + auth_json = _make_auth_json(raw_account_id, email) + response = await async_client.post( + "/api/accounts/import", + files={"auth_json": ("auth.json", json.dumps(auth_json), "application/json")}, + ) + assert response.status_code == 200 + + reservation = ApiKeyUsageReservationData( + reservation_id="resv_terminal_disconnect", + key_id="key_terminal_disconnect", + model="gpt-5.1", + ) + stream_closed = asyncio.Event() + settle_calls: list[dict[str, object]] = [] + success_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + try: + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_terminal_disconnect",' + '"object":"response","status":"completed","output":[],"usage":' + '{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n' + ) + await asyncio.Event().wait() + finally: + stream_closed.set() + + async def fake_enforce_request_limits(*_args: object, **_kwargs: object): + return reservation + + async def fake_settle_stream_api_key_usage(self, api_key, api_key_reservation, settlement, request_id, **kwargs): + del self, api_key + settle_calls.append( + { + "reservation": api_key_reservation, + "status": settlement.status, + "request_id": request_id, + "input_tokens": settlement.input_tokens, + "output_tokens": settlement.output_tokens, + "wait_for_settlement": kwargs.get("wait_for_settlement", False), + } + ) + return True + + async def fake_record_success(self, account): + del self + success_account_ids.append(account.id) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_api_module, "_enforce_request_limits", fake_enforce_request_limits) + monkeypatch.setattr( + proxy_module.ProxyService, + "_settle_stream_api_key_usage", + fake_settle_stream_api_key_usage, + ) + monkeypatch.setattr(proxy_module.LoadBalancer, "record_success", fake_record_success) + + request_id = "req_terminal_disconnect" + request_body = json.dumps( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}, + separators=(",", ":"), + ).encode("utf-8") + request_sent = False + disconnect_allowed = asyncio.Event() + first_terminal_sent = asyncio.Event() + response_started: list[dict[str, object]] = [] + + async def receive() -> dict[str, object]: + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": request_body, "more_body": False} + await disconnect_allowed.wait() + return {"type": "http.disconnect"} + + async def send(message: dict[str, object]) -> None: + if message["type"] == "http.response.start": + response_started.append(message) + return + body = message.get("body") + if ( + message["type"] == "http.response.body" + and isinstance(body, bytes) + and b'"type":"response.completed"' in body + ): + first_terminal_sent.set() + disconnect_allowed.set() + + await app_instance( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/backend-api/codex/responses", + "raw_path": b"/backend-api/codex/responses", + "query_string": b"", + "headers": [ + (b"host", b"testserver"), + (b"content-type", b"application/json"), + (b"x-request-id", request_id.encode("ascii")), + ], + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + }, + receive, + send, + ) + assert response_started and response_started[0]["status"] == 200 + assert first_terminal_sent.is_set() is True + + await asyncio.wait_for(stream_closed.wait(), timeout=1.0) + await app_instance.state.proxy_service.drain_persistence_tasks(timeout_seconds=5) + + assert settle_calls == [ + { + "reservation": reservation, + "status": "success", + "request_id": request_id, + "input_tokens": 1, + "output_tokens": 1, + "wait_for_settlement": False, + } + ] + assert success_account_ids == [expected_account_id] + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.request_id == "resp_terminal_disconnect")) + log = result.scalars().one() + assert log.archive_request_id == request_id + assert log.account_id == expected_account_id + assert log.status == "success" + + +@pytest.mark.asyncio +async def test_backend_responses_post_refresh_terminal_disconnect_finalizes_settlement( + async_client, + app_instance, + monkeypatch, +): + email = "post-refresh-terminal-disconnect@example.com" + raw_account_id = "acc_post_refresh_terminal_disconnect" + expected_account_id = generate_unique_account_id(raw_account_id, email) + auth_json = _make_auth_json(raw_account_id, email) + response = await async_client.post( + "/api/accounts/import", + files={"auth_json": ("auth.json", json.dumps(auth_json), "application/json")}, + ) + assert response.status_code == 200 + + reservation = ApiKeyUsageReservationData( + reservation_id="resv_post_refresh_terminal_disconnect", + key_id="key_post_refresh_terminal_disconnect", + model="gpt-5.1", + ) + stream_calls: list[int] = [] + stream_closed = asyncio.Event() + settle_calls: list[dict[str, object]] = [] + success_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + stream_calls.append(len(stream_calls) + 1) + if len(stream_calls) == 1: + raise proxy_module.ProxyResponseError( + 401, + {"error": {"code": "invalid_api_key", "message": "token invalidated"}}, + ) + try: + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_post_refresh_disconnect",' + '"object":"response","status":"completed","output":[],"usage":' + '{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}\n\n' + ) + await asyncio.Event().wait() + finally: + stream_closed.set() + + async def fake_ensure_fresh(self, account, **kwargs): + del self, kwargs + return account + + async def fake_enforce_request_limits(*_args: object, **_kwargs: object): + return reservation + + async def fake_settle_stream_api_key_usage(self, api_key, api_key_reservation, settlement, request_id, **kwargs): + del self, api_key + settle_calls.append( + { + "reservation": api_key_reservation, + "status": settlement.status, + "request_id": request_id, + "input_tokens": settlement.input_tokens, + "output_tokens": settlement.output_tokens, + "wait_for_settlement": kwargs.get("wait_for_settlement", False), + } + ) + return True + + async def fake_record_success(self, account): + del self + success_account_ids.append(account.id) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + monkeypatch.setattr(proxy_api_module, "_enforce_request_limits", fake_enforce_request_limits) + monkeypatch.setattr( + proxy_module.ProxyService, + "_settle_stream_api_key_usage", + fake_settle_stream_api_key_usage, + ) + monkeypatch.setattr(proxy_module.LoadBalancer, "record_success", fake_record_success) + + request_id = "req_post_refresh_terminal_disconnect" + request_body = json.dumps( + {"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}, + separators=(",", ":"), + ).encode("utf-8") + request_sent = False + disconnect_allowed = asyncio.Event() + first_terminal_sent = asyncio.Event() + response_started: list[dict[str, object]] = [] + + async def receive() -> dict[str, object]: + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": request_body, "more_body": False} + await disconnect_allowed.wait() + return {"type": "http.disconnect"} + + async def send(message: dict[str, object]) -> None: + if message["type"] == "http.response.start": + response_started.append(message) + return + body = message.get("body") + if ( + message["type"] == "http.response.body" + and isinstance(body, bytes) + and b'"type":"response.completed"' in body + ): + first_terminal_sent.set() + disconnect_allowed.set() + + await app_instance( + { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/backend-api/codex/responses", + "raw_path": b"/backend-api/codex/responses", + "query_string": b"", + "headers": [ + (b"host", b"testserver"), + (b"content-type", b"application/json"), + (b"x-request-id", request_id.encode("ascii")), + ], + "client": ("127.0.0.1", 1234), + "server": ("testserver", 80), + }, + receive, + send, + ) + assert response_started and response_started[0]["status"] == 200 + assert stream_calls == [1, 2] + assert first_terminal_sent.is_set() is True + + await asyncio.wait_for(stream_closed.wait(), timeout=1.0) + await app_instance.state.proxy_service.drain_persistence_tasks(timeout_seconds=5) + + assert settle_calls == [ + { + "reservation": reservation, + "status": "success", + "request_id": request_id, + "input_tokens": 1, + "output_tokens": 1, + "wait_for_settlement": False, + } + ] + assert success_account_ids == [expected_account_id] + + async with SessionLocal() as session: + result = await session.execute( + select(RequestLog).where(RequestLog.request_id == "resp_post_refresh_disconnect") + ) + log = result.scalars().one() + assert log.archive_request_id == request_id + assert log.account_id == expected_account_id + assert log.status == "success" + + @pytest.mark.asyncio async def test_proxy_responses_forwards_native_codex_headers(async_client, monkeypatch): email = "stream-headers@example.com" diff --git a/tests/unit/test_db_session.py b/tests/unit/test_db_session.py index e75bf781db..31d24b552b 100644 --- a/tests/unit/test_db_session.py +++ b/tests/unit/test_db_session.py @@ -376,6 +376,54 @@ async def close(self) -> None: assert calls == ["rollback", "close"] +@pytest.mark.asyncio +async def test_close_session_outlives_caller_cancellation() -> None: + rollback_started = asyncio.Event() + rollback_release = asyncio.Event() + close_started = asyncio.Event() + close_release = asyncio.Event() + cleanup_done = asyncio.Event() + calls: list[str] = [] + + class FakeSession: + def in_transaction(self) -> bool: + return True + + async def rollback(self) -> None: + calls.append("rollback-start") + rollback_started.set() + await rollback_release.wait() + calls.append("rollback-end") + + async def close(self) -> None: + calls.append("close-start") + close_started.set() + await close_release.wait() + calls.append("close-end") + + async def run_cleanup() -> None: + try: + await session_module.close_session(cast(session_module.AsyncSession, FakeSession())) + finally: + cleanup_done.set() + + async with asyncio.TaskGroup() as group: + task = group.create_task(run_cleanup()) + await rollback_started.wait() + task.cancel() + await asyncio.sleep(0) + task.cancel() + await asyncio.sleep(0) + assert calls == ["rollback-start"] + assert not cleanup_done.is_set() + rollback_release.set() + await close_started.wait() + close_release.set() + + assert calls == ["rollback-start", "rollback-end", "close-start", "close-end"] + assert cleanup_done.is_set() + + @pytest.mark.asyncio async def test_detach_session_objects_keeps_loaded_fields_available_after_rollback() -> None: engine = create_async_engine("sqlite+aiosqlite:///:memory:") diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index bb4fe2c472..fd1779be02 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -63,7 +63,7 @@ from app.modules.accounts import auth_manager as auth_manager_module from app.modules.accounts.repository import AccountsRepository from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyData +from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData from app.modules.proxy import affinity as proxy_affinity from app.modules.proxy import api as proxy_api from app.modules.proxy import request_policy as proxy_request_policy @@ -11737,6 +11737,89 @@ async def fake_stream( assert settlement.account_health_error is False +@pytest.mark.asyncio +async def test_stream_once_keeps_first_terminal_frame_success_after_downstream_close(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_stream_first_terminal_close") + settlement = proxy_service._StreamSettlement() + completed_line = ( + 'data: {"type":"response.completed","response":{"id":"resp_first_terminal_close","status":"completed"}}\n\n' + ) + + async def fake_stream( + payload, + headers, + access_token, + account_id, + base_url=None, + raise_for_status=False, + enforce_openai_sdk_contract=True, + ): + del payload, headers, access_token, account_id, base_url, raise_for_status, enforce_openai_sdk_contract + yield completed_line + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.4", "instructions": "hi", "input": [], "stream": True}) + stream = service._stream_once( + account, + payload, + {"session_id": "sid-stream"}, + "req_stream_first_terminal_close", + False, + request_started_at=0.0, + api_key=None, + api_key_reservation=None, + settlement=settlement, + suppress_text_done_events=False, + upstream_stream_transport=None, + request_transport="http", + ) + + first_chunk = await anext(stream) + assert "event: response.completed" in first_chunk + assert '"id":"resp_first_terminal_close"' in first_chunk + await cast(Any, stream).aclose() + + assert settlement.status == "success" + assert settlement.error is None + assert settlement.account_health_error is False + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["error_code"] is None + assert request_logs.calls[0]["request_id"] == "resp_first_terminal_close" + + +@pytest.mark.asyncio +async def test_streaming_retry_cleanup_helper_finishes_task_before_reporting_cancellation(): + cleanup_started = asyncio.Event() + release_cleanup = asyncio.Event() + cleanup_finished = asyncio.Event() + + async def cleanup() -> str: + cleanup_started.set() + await release_cleanup.wait() + cleanup_finished.set() + return "settled" + + cleanup_task = asyncio.create_task(cleanup()) + waiter_task = asyncio.create_task(streaming_retry_module._await_task_deferring_cancellation(cleanup_task)) + await asyncio.wait_for(cleanup_started.wait(), timeout=1) + + waiter_task.cancel() + await asyncio.sleep(0) + assert not cleanup_task.done() + assert not waiter_task.done() + + release_cleanup.set() + result, cancellation = await asyncio.wait_for(waiter_task, timeout=1) + + assert result == "settled" + assert cancellation is not None + assert cleanup_finished.is_set() + + @pytest.mark.asyncio async def test_stream_once_marks_downstream_cancel_before_first_event(monkeypatch): request_logs = _RequestLogsRecorder() @@ -15696,6 +15779,73 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, record_success.assert_not_awaited() +@pytest.mark.asyncio +async def test_stream_with_retry_finalizes_generated_terminal_failure_before_downstream_close(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_retry_generated_terminal_close") + record_error = AsyncMock() + record_success = AsyncMock() + settle_stream_usage = AsyncMock(return_value=True) + reservation = ApiKeyUsageReservationData( + reservation_id="resv_retry_generated_terminal_close", + key_id="key_retry_generated_terminal_close", + model="gpt-5.1", + ) + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(streaming_retry_module.ProcessNetworkRecovery, "wait", AsyncMock(return_value=None)) + monkeypatch.setattr(service._load_balancer, "record_error", record_error) + monkeypatch.setattr(service._load_balancer, "record_success", record_success) + monkeypatch.setattr(service, "_settle_stream_api_key_usage", settle_stream_usage) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_stream_once(*args: object, **kwargs: object): + settlement = cast(proxy_service._StreamSettlement, kwargs["settlement"]) + settlement.downstream_visible = True + settlement.response_id = "resp_retry_generated_terminal_close" + yield ( + 'data: {"type":"response.created","response":{"id":"resp_retry_generated_terminal_close",' + '"status":"in_progress","output":[]}}\n\n' + ) + raise streaming_retry_module._TransientStreamError( + "upstream_unavailable", + {"message": "transport exploded after first event"}, + ) + + monkeypatch.setattr(service, "_stream_once", fake_stream_once) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + stream = service._stream_with_retry( + payload, + {"session_id": "sid-retry-generated-terminal-close"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=reservation, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override=None, + ) + + first_chunk = await anext(stream) + terminal_chunk = await anext(stream) + assert "response.created" in first_chunk + assert "response.failed" in terminal_chunk + await cast(Any, stream).aclose() + + assert await service.drain_persistence_tasks(timeout_seconds=1) + settle_stream_usage.assert_awaited_once() + record_success.assert_not_awaited() + + @pytest.mark.asyncio async def test_stream_responses_first_event_connection_reset_surfaces_without_replay(monkeypatch): settings = _make_proxy_settings() @@ -16002,7 +16152,10 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert event["response"]["id"] == "resp_reset_event" assert event["response"]["error"]["code"] == "upstream_unavailable" assert seen_excluded_account_ids == [set()] - assert request_logs.calls == [] + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + assert request_logs.calls[0]["account_id"] == account_a.id + assert request_logs.calls[0]["error_code"] == "upstream_unavailable" record_error.assert_not_awaited() record_errors.assert_not_awaited() record_success.assert_not_awaited() From 5d27f7f0cc2fea37ce2e3786ea08660f008dd694 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Sun, 16 Aug 2026 17:57:34 +0200 Subject: [PATCH 045/117] fix(dashboard): show cancellation totals in reports (#1772) * fix(dashboard): show cancellation totals in reports * test(reports): enforce cancellation schema contracts * docs(openspec): preserve dashboard cancellation contract --- .../reports-date-range-flow.test.tsx | 1 + .../components/cost-per-day-chart.test.tsx | 3 + .../components/daily-detail-table.test.tsx | 133 ++++++++++++------ .../reports/components/daily-detail-table.tsx | 41 ++++-- .../components/queue-wait-chart.test.tsx | 1 + .../reports/components/reports-page.test.tsx | 1 + .../components/reports-summary-cards.test.tsx | 64 ++++++++- .../components/reports-summary-cards.tsx | 12 +- .../components/tokens-per-day-chart.test.tsx | 3 + frontend/src/features/reports/daily-series.ts | 1 + frontend/src/features/reports/schemas.test.ts | 63 +++++++-- frontend/src/features/reports/schemas.ts | 2 + frontend/src/i18n/locales/en.json | 5 + frontend/src/i18n/locales/ko.json | 5 + frontend/src/i18n/locales/zh-CN.json | 5 + .../design.md | 54 +++++++ .../proposal.md | 25 ++++ .../specs/usage-error-metrics/spec.md | 69 +++++++++ .../tasks.md | 25 ++++ 19 files changed, 452 insertions(+), 61 deletions(-) create mode 100644 openspec/changes/surface-reports-cancellation-totals/design.md create mode 100644 openspec/changes/surface-reports-cancellation-totals/proposal.md create mode 100644 openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md create mode 100644 openspec/changes/surface-reports-cancellation-totals/tasks.md diff --git a/frontend/src/__integration__/reports-date-range-flow.test.tsx b/frontend/src/__integration__/reports-date-range-flow.test.tsx index 0b5695d24f..4b068b18b3 100644 --- a/frontend/src/__integration__/reports-date-range-flow.test.tsx +++ b/frontend/src/__integration__/reports-date-range-flow.test.tsx @@ -16,6 +16,7 @@ const EMPTY_REPORT: ReportsResponse = { totalOutputTokens: 0, totalCachedTokens: 0, totalRequests: 0, + totalCancelled: 0, totalErrors: 0, totalConversations: 0, activeAccounts: 0, diff --git a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx index 6b212d8472..26f5b0c490 100644 --- a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx @@ -44,6 +44,7 @@ describe("CostPerDayChart", () => { cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} @@ -68,6 +69,7 @@ describe("CostPerDayChart", () => { cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, { @@ -79,6 +81,7 @@ describe("CostPerDayChart", () => { cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} diff --git a/frontend/src/features/reports/components/daily-detail-table.test.tsx b/frontend/src/features/reports/components/daily-detail-table.test.tsx index 2968522c7f..87ace864ca 100644 --- a/frontend/src/features/reports/components/daily-detail-table.test.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.test.tsx @@ -3,8 +3,29 @@ import { act, cleanup, render, screen, within } from "@testing-library/react"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; import { formatReportBucketDate } from "../date"; - -import { DailyDetailTable } from "./daily-detail-table"; +import { buildContinuousDailyRows } from "../daily-series"; +import type { DailyReportRow } from "../schemas"; + +import { + DailyDetailTable as DailyDetailTableImpl, + type DailyDetailTableProps, +} from "./daily-detail-table"; + +type DailyDetailTableFixtureRow = Omit & { + cancelledCount?: number; +}; + +function DailyDetailTable({ + data, + ...props +}: Omit & { data: DailyDetailTableFixtureRow[] }) { + return ( + ({ cancelledCount: 0, ...row }))} + /> + ); +} beforeEach(() => { useDateDisplayFormatStore.setState({ dateDisplayFormat: "default" }); @@ -98,6 +119,28 @@ describe("DailyDetailTable", () => { ); }); + it("zero-fills cancelled counts for dates missing from the response", () => { + const rows = buildContinuousDailyRows("2026-06-05", "2026-06-06", [ + { + date: "2026-06-05", + requests: 4, + conversations: 0, + inputTokens: 100, + outputTokens: 20, + cachedInputTokens: 0, + costUsd: 1, + activeAccounts: 1, + errorCount: 1, + cancelledCount: 2, + }, + ]); + + expect(Reflect.get(rows[0] ?? {}, "cancelledCount")).toBe(2); + expect(Reflect.get(rows[1] ?? {}, "cancelledCount")).toBe(0); + expect(rows[0]?.requests).toBe(4); + expect(rows[0]?.errorCount).toBe(1); + }); + it("renders existing rows when a date bound is cleared", () => { render( { ]); }); - it("exports csv rows in chronological order regardless of visible sort", async () => { + it("renders requests, cancelled, and errors as distinct daily values", () => { + render( + , + ); + + expect.soft(screen.queryByRole("columnheader", { name: "Reqs" })).toBeInTheDocument(); + expect.soft(screen.queryByRole("columnheader", { name: "Cancelled" })).toBeInTheDocument(); + expect.soft(screen.queryByRole("columnheader", { name: "Errors" })).toBeInTheDocument(); + + const row = screen.getByTestId("daily-breakdown-row-2026-06-05"); + const cells = Array.from(row.querySelectorAll("td"), (cell) => cell.textContent?.trim()); + expect.soft(cells).toContain("4"); + expect.soft(cells).toContain("2"); + expect.soft(cells).toContain("1"); + }); + + it("exports localized cancellation values and preserves requests and errors", async () => { const user = userEvent.setup(); const blobText = vi.fn(async () => ""); const createObjectURL = vi.spyOn(URL, "createObjectURL").mockImplementation((blob) => { @@ -246,46 +322,24 @@ describe("DailyDetailTable", () => { render( , ); - await user.click(screen.getByRole("button", { name: /reqs/i })); await user.click(screen.getByRole("button", { name: /csv/i })); expect(createObjectURL).toHaveBeenCalledOnce(); @@ -293,10 +347,9 @@ describe("DailyDetailTable", () => { expect(revokeObjectURL).toHaveBeenCalledWith("blob:daily-breakdown"); await expect(blobText()).resolves.toBe( [ - "Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Errors", - "2026-06-05,8,0,100,20,1,1.0000,3,0", - "2026-06-06,2,0,200,30,2,2.0000,1,0", - "2026-06-07,5,0,300,40,3,3.0000,2,0", + "Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors", + "2026-06-05,4,0,100,20,1,1.0000,3,2,1", + "2026-06-06,0,0,0,0,0,0.0000,0,0,0", ].join("\n"), ); }); @@ -510,9 +563,9 @@ describe("DailyDetailTable", () => { startDate="2026-06-05" endDate="2026-06-07" data={[ - { date: "2026-06-05", requests: 8, conversations: 1, inputTokens: 100, outputTokens: 20, cachedInputTokens: 0, costUsd: 1, activeAccounts: 1, errorCount: 0 }, - { date: "2026-06-06", requests: 2, conversations: 5, inputTokens: 200, outputTokens: 30, cachedInputTokens: 1, costUsd: 2, activeAccounts: 1, errorCount: 0 }, - { date: "2026-06-07", requests: 5, conversations: 3, inputTokens: 300, outputTokens: 40, cachedInputTokens: 2, costUsd: 3, activeAccounts: 1, errorCount: 0 }, + { date: "2026-06-05", requests: 8, conversations: 1, inputTokens: 100, outputTokens: 20, cachedInputTokens: 0, costUsd: 1, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, + { date: "2026-06-06", requests: 2, conversations: 5, inputTokens: 200, outputTokens: 30, cachedInputTokens: 1, costUsd: 2, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, + { date: "2026-06-07", requests: 5, conversations: 3, inputTokens: 300, outputTokens: 40, cachedInputTokens: 2, costUsd: 3, activeAccounts: 1, cancelledCount: 0, errorCount: 0 }, ]} />, ); @@ -530,15 +583,15 @@ describe("DailyDetailTable", () => { const headerRow = screen.getAllByRole("row")[0]; const headerCells = Array.from(headerRow?.querySelectorAll("th") ?? []); const labels = headerCells.map((c) => c.textContent?.trim() ?? ""); - expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Cost", "Accounts"]); + expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Cost", "Accounts", "Cancelled", "Errors"]); // CSV: full header + first data row with Conversations between Requests and Input Tokens await user.click(screen.getByRole("button", { name: /csv/i })); const csv = await blobText(); const csvLines = csv.split("\n"); - expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Errors"); + expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors"); // First data row in CSV (chronological: 06-05 first, conversations=1) - expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,1\.0000,1,0/); + expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,1\.0000,1,0,0/); }); it("zero-filled gap rows have conversations=0 in column 2", () => { @@ -572,7 +625,7 @@ describe("DailyDetailTable", () => { />, ); - const tables = document.querySelectorAll("table.min-w-\\[700px\\]"); + const tables = document.querySelectorAll("table.min-w-\\[900px\\]"); expect(tables.length).toBe(2); }); }); diff --git a/frontend/src/features/reports/components/daily-detail-table.tsx b/frontend/src/features/reports/components/daily-detail-table.tsx index 32a4d08eb2..21dd78fd7e 100644 --- a/frontend/src/features/reports/components/daily-detail-table.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.tsx @@ -16,7 +16,7 @@ export type DailyDetailTableProps = { const DAILY_BREAKDOWN_SCROLL_HEIGHT_CLASS = "max-h-[17.5rem]"; -type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "costUsd" | "activeAccounts"; +type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "costUsd" | "activeAccounts" | "cancelledCount" | "errorCount"; type SortDirection = "asc" | "desc"; function formatTokens(v: number): string { @@ -59,7 +59,7 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP
- +
@@ -106,6 +106,18 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP direction={sort.direction} onClick={() => toggleSort("activeAccounts")} /> + toggleSort("cancelledCount")} + /> + toggleSort("errorCount")} + />
@@ -113,7 +125,7 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP data-testid="daily-breakdown-scroll-body" className={`${DAILY_BREAKDOWN_SCROLL_HEIGHT_CLASS} overflow-y-auto [scrollbar-width:none] [&::-webkit-scrollbar]:hidden`} > - +
{rows.map((row) => ( @@ -143,9 +155,15 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP - + + ))} @@ -201,13 +219,15 @@ function SortableHeader({ function ColumnGroup() { return ( - - - - - + + + + + + + ); } @@ -242,10 +262,11 @@ function exportCSV(rows: DailyReportRow[], t: TFunction) { t("reports.dailyBreakdown.csvColumns.cachedTokens"), t("reports.dailyBreakdown.csvColumns.costUsd"), t("reports.dailyBreakdown.csvColumns.activeAccounts"), + t("reports.dailyBreakdown.csvColumns.cancelled"), t("reports.dailyBreakdown.csvColumns.errors"), ]; const lines = rows.map((r) => - [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.errorCount].join(","), + [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.cancelledCount, r.errorCount].join(","), ); const csv = [headers.join(","), ...lines].join("\n"); const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); diff --git a/frontend/src/features/reports/components/queue-wait-chart.test.tsx b/frontend/src/features/reports/components/queue-wait-chart.test.tsx index 7de125173a..91ba329ccc 100644 --- a/frontend/src/features/reports/components/queue-wait-chart.test.tsx +++ b/frontend/src/features/reports/components/queue-wait-chart.test.tsx @@ -32,6 +32,7 @@ const BASE_ROW = { cachedInputTokens: 0, costUsd: 0.5, activeAccounts: 1, + cancelledCount: 0, errorCount: 0, medianTtftMs: 200, medianTps: 25, diff --git a/frontend/src/features/reports/components/reports-page.test.tsx b/frontend/src/features/reports/components/reports-page.test.tsx index a5acd14ed3..ca6deabcb0 100644 --- a/frontend/src/features/reports/components/reports-page.test.tsx +++ b/frontend/src/features/reports/components/reports-page.test.tsx @@ -46,6 +46,7 @@ const EMPTY_REPORT: ReportsResponse = { totalOutputTokens: 0, totalCachedTokens: 0, totalRequests: 0, + totalCancelled: 0, totalErrors: 0, totalConversations: 0, activeAccounts: 0, diff --git a/frontend/src/features/reports/components/reports-summary-cards.test.tsx b/frontend/src/features/reports/components/reports-summary-cards.test.tsx index d282b10ac1..d2eb22f71c 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.test.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.test.tsx @@ -2,7 +2,27 @@ import { render, screen, within } from "@testing-library/react"; import { describe, expect, it } from "vitest"; -import { ReportsSummaryCards } from "./reports-summary-cards"; +import type { ReportSummary } from "../schemas"; +import { + ReportsSummaryCards as ReportsSummaryCardsImpl, + type ReportsSummaryCardsProps, +} from "./reports-summary-cards"; + +type ReportsSummaryFixture = Omit & { + totalCancelled?: number; +}; + +function ReportsSummaryCards({ + summary, + ...props +}: Omit & { summary: ReportsSummaryFixture }) { + return ( + + ); +} describe("ReportsSummaryCards", () => { it("renders inline comparison badges for cost, tokens, and requests", () => { @@ -165,6 +185,48 @@ describe("ReportsSummaryCards", () => { expect(requestsCard.nextElementSibling).toBe(conversationsCard); }); + it("renders requests, cancelled, and errors as distinct summary totals", () => { + render( + , + ); + + const requestsCard = screen.getByTestId("report-summary-card-requests"); + expect(within(requestsCard).getByText("Requests")).toBeInTheDocument(); + expect(within(requestsCard).getByText("4")).toBeInTheDocument(); + + const cancelledCard = screen.queryByTestId("report-summary-card-cancelled"); + expect.soft(cancelledCard).toBeInTheDocument(); + if (cancelledCard) { + expect.soft(within(cancelledCard).getByText("Cancelled")).toBeInTheDocument(); + expect.soft(within(cancelledCard).getByText("2")).toBeInTheDocument(); + } + + const errorsCard = screen.queryByTestId("report-summary-card-errors"); + expect.soft(errorsCard).toBeInTheDocument(); + if (errorsCard) { + expect.soft(within(errorsCard).getByText("Errors")).toBeInTheDocument(); + expect.soft(within(errorsCard).getByText("1")).toBeInTheDocument(); + } + }); + it("preserves trailing zeroes for unrelated whole K and B values", () => { render( +
{cards.map((card) => (
{ cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} @@ -68,6 +69,7 @@ describe("TokensPerDayChart", () => { cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, { @@ -79,6 +81,7 @@ describe("TokensPerDayChart", () => { cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, + cancelledCount: 0, errorCount: 0, }, ]} diff --git a/frontend/src/features/reports/daily-series.ts b/frontend/src/features/reports/daily-series.ts index dd539d0435..999d9166b3 100644 --- a/frontend/src/features/reports/daily-series.ts +++ b/frontend/src/features/reports/daily-series.ts @@ -44,6 +44,7 @@ function createZeroRow(date: string): DailyReportRow { cachedInputTokens: 0, costUsd: 0, activeAccounts: 0, + cancelledCount: 0, errorCount: 0, medianTtftMs: 0, medianTps: 0, diff --git a/frontend/src/features/reports/schemas.test.ts b/frontend/src/features/reports/schemas.test.ts index 13f66196ad..0d7e9d9ab9 100644 --- a/frontend/src/features/reports/schemas.test.ts +++ b/frontend/src/features/reports/schemas.test.ts @@ -2,40 +2,76 @@ import { describe, expect, it } from "vitest"; import { ReportsResponseSchema } from "./schemas"; +function validReportsPayload() { + return { + summary: { + totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalCachedTokens: 0, totalRequests: 4, totalCancelled: 2, + totalErrors: 1, totalConversations: 7, activeAccounts: 3, + avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, + }, + comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 2, errorCount: 1 }], + byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], + byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], + byAccount: [], + }; +} + describe("ReportsResponseSchema", () => { - it("parses totalConversations on summary and conversations on daily rows", () => { + it("preserves conversation and cancellation totals from the reports payload", () => { const parsed = ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, - totalCachedTokens: 0, totalRequests: 25, totalErrors: 1, - totalConversations: 7, activeAccounts: 3, + totalCachedTokens: 0, totalRequests: 4, totalErrors: 1, + totalCancelled: 2, totalConversations: 7, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 10, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 0 }], - byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 25, percentage: 100 }], - byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 25, percentage: 100 }], + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 1, cancelledCount: 2 }], + byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], + byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], byAccount: [], }); + expect(parsed.summary.totalRequests).toBe(4); + expect(parsed.summary.totalErrors).toBe(1); + expect.soft(Reflect.get(parsed.summary, "totalCancelled")).toBe(2); expect(parsed.summary.totalConversations).toBe(7); + expect(parsed.daily[0]?.requests).toBe(4); + expect(parsed.daily[0]?.errorCount).toBe(1); + expect.soft(Reflect.get(parsed.daily[0] ?? {}, "cancelledCount")).toBe(2); expect(parsed.daily[0]?.conversations).toBe(3); }); + it("rejects omitted totalCancelled on summary", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.summary, "totalCancelled"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/totalCancelled/i); + }); + + it("rejects omitted cancelledCount on daily rows", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.daily[0] ?? {}, "cancelledCount"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/cancelledCount/i); + }); + it("rejects omitted totalConversations on summary", () => { expect(() => ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, - totalCachedTokens: 0, totalRequests: 25, totalErrors: 1, + totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, totalErrors: 1, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 0 }], + daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 0, errorCount: 0 }], byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 25, percentage: 100 }], byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 25, percentage: 100 }], byAccount: [], }), - ).toThrow(); + ).toThrow(/totalConversations/i); }); @@ -47,6 +83,7 @@ describe("ReportsResponseSchema", () => { totalOutputTokens: 200, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, totalConversations: 0, activeAccounts: 3, @@ -98,7 +135,9 @@ describe("ReportsResponseSchema", () => { totalOutputTokens: 200, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -120,7 +159,9 @@ describe("ReportsResponseSchema", () => { totalOutputTokens: 200, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -145,7 +186,9 @@ describe("ReportsResponseSchema", () => { totalOutputTokens: 200, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, @@ -181,7 +224,9 @@ describe("ReportsResponseSchema", () => { totalOutputTokens: 200, totalCachedTokens: 0, totalRequests: 25, + totalCancelled: 0, totalErrors: 1, + totalConversations: 0, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, diff --git a/frontend/src/features/reports/schemas.ts b/frontend/src/features/reports/schemas.ts index 795e671577..33e5b65397 100644 --- a/frontend/src/features/reports/schemas.ts +++ b/frontend/src/features/reports/schemas.ts @@ -9,6 +9,7 @@ const DailyReportRowSchema = z.object({ cachedInputTokens: z.number(), costUsd: z.number(), activeAccounts: z.number(), + cancelledCount: z.number(), errorCount: z.number(), medianTtftMs: z.number().optional().default(0), medianTps: z.number().optional().default(0), @@ -42,6 +43,7 @@ const ReportSummarySchema = z.object({ totalOutputTokens: z.number(), totalCachedTokens: z.number(), totalRequests: z.number(), + totalCancelled: z.number(), totalErrors: z.number(), totalConversations: z.number(), activeAccounts: z.number(), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 7d3c983df5..7e1d8f96aa 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -954,14 +954,17 @@ "reports.charts.tokensByDay": "Tokens by Day", "reports.charts.tokensPerSecond": "Tokens per Second", "reports.dailyBreakdown.columns.accounts": "Accounts", + "reports.dailyBreakdown.columns.cancelled": "Cancelled", "reports.dailyBreakdown.columns.cost": "Cost", "reports.dailyBreakdown.columns.day": "Day", + "reports.dailyBreakdown.columns.errors": "Errors", "reports.dailyBreakdown.columns.inputTokens": "Input Tokens", "reports.dailyBreakdown.columns.outputTokens": "Output Tokens", "reports.dailyBreakdown.columns.reqs": "Reqs", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "Active Accounts", "reports.dailyBreakdown.csvColumns.cachedTokens": "Cached Tokens", + "reports.dailyBreakdown.csvColumns.cancelled": "Cancelled", "reports.dailyBreakdown.columns.conversations": "Conversations", "reports.dailyBreakdown.csvColumns.conversations": "Conversations", "reports.dailyBreakdown.csvColumns.costUsd": "Cost USD", @@ -987,6 +990,8 @@ "reports.page.subtitle": "Usage history by date range", "reports.page.title": "Cost Report", "reports.summary.avgCostPerDay": "avg {{cost}}/day", + "reports.summary.cancelled": "Cancelled", + "reports.summary.errors": "Errors", "reports.summary.requests": "Requests", "reports.summary.requestsSub": "avg {{requests}}/day · {{accounts}} accounts", "reports.summary.tokens": "Tokens", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 04c91ebe4b..d8ed58337b 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -954,14 +954,17 @@ "reports.charts.tokensByDay": "일별 token", "reports.charts.tokensPerSecond": "Tokens per Second", "reports.dailyBreakdown.columns.accounts": "Accounts", + "reports.dailyBreakdown.columns.cancelled": "취소됨", "reports.dailyBreakdown.columns.cost": "비용", "reports.dailyBreakdown.columns.day": "일자", + "reports.dailyBreakdown.columns.errors": "오류", "reports.dailyBreakdown.columns.inputTokens": "Input token", "reports.dailyBreakdown.columns.outputTokens": "Output token", "reports.dailyBreakdown.columns.reqs": "요청", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "활성 Accounts", "reports.dailyBreakdown.csvColumns.cachedTokens": "Cached token", + "reports.dailyBreakdown.csvColumns.cancelled": "취소됨", "reports.dailyBreakdown.csvColumns.costUsd": "Cost USD", "reports.dailyBreakdown.csvColumns.date": "날짜", "reports.dailyBreakdown.csvColumns.errors": "오류", @@ -985,7 +988,9 @@ "reports.page.subtitle": "기간별 사용 기록", "reports.page.title": "비용 리포트", "reports.summary.avgCostPerDay": "평균 {{cost}}/일", + "reports.summary.cancelled": "취소됨", "reports.summary.conversations": "활성 대화", + "reports.summary.errors": "오류", "reports.summary.requests": "요청", "reports.summary.requestsSub": "평균 {{requests}}/일 · {{accounts}} Accounts", "reports.summary.tokens": "Token", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 2049f4a403..5cc730652c 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -954,14 +954,17 @@ "reports.charts.tokensByDay": "按天 token", "reports.charts.tokensPerSecond": "每秒 token", "reports.dailyBreakdown.columns.accounts": "账户", + "reports.dailyBreakdown.columns.cancelled": "已取消", "reports.dailyBreakdown.columns.cost": "费用", "reports.dailyBreakdown.columns.day": "日期", + "reports.dailyBreakdown.columns.errors": "错误", "reports.dailyBreakdown.columns.inputTokens": "输入 token", "reports.dailyBreakdown.columns.outputTokens": "输出 token", "reports.dailyBreakdown.columns.reqs": "请求", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "活跃账户", "reports.dailyBreakdown.csvColumns.cachedTokens": "缓存 token", + "reports.dailyBreakdown.csvColumns.cancelled": "已取消", "reports.dailyBreakdown.csvColumns.costUsd": "费用 USD", "reports.dailyBreakdown.csvColumns.date": "日期", "reports.dailyBreakdown.csvColumns.errors": "错误", @@ -985,7 +988,9 @@ "reports.page.subtitle": "按日期范围查看使用历史", "reports.page.title": "费用报表", "reports.summary.avgCostPerDay": "平均 {{cost}}/天", + "reports.summary.cancelled": "已取消", "reports.summary.conversations": "活跃对话", + "reports.summary.errors": "错误", "reports.summary.requests": "请求", "reports.summary.requestsSub": "平均 {{requests}}/天 · {{accounts}} 个账户", "reports.summary.tokens": "Token", diff --git a/openspec/changes/surface-reports-cancellation-totals/design.md b/openspec/changes/surface-reports-cancellation-totals/design.md new file mode 100644 index 0000000000..fc3279b81c --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/design.md @@ -0,0 +1,54 @@ +# Design: Surface reports cancellation totals + +## Context + +The raw Reports backend models define cancellation data as `ReportSummary.total_cancelled` and `DailyReportRow.cancelled_count`. Dashboard API serialization exposes those fields as `summary.totalCancelled` and `daily[].cancelledCount`, which are also the names consumed by the frontend. The frontend's strict response schemas omit both camelCase properties, so parsing strips the values before the report model reaches rendering and export. The date-range completion path also creates synthetic daily rows without a cancellation field. As a result, cancellation data is absent from the summary, daily table, and downloaded CSV despite being available at the system boundary. + +This change is limited to the Reports frontend. The existing `usage-error-metrics` specification remains the owner of request terminal classification and cancellation accounting. + +## Goals and Non-goals + +### Goals + +- Preserve the backend cancellation fields through frontend parsing. +- Treat a synthesized no-activity day as having zero cancellations. +- Present cancellations beside requests and errors in the visible summary, table, and CSV. +- Keep labels localized, including English, Korean, and Simplified Chinese. +- Prove existing request and error values do not regress. + +### Non-goals + +- Changing backend aggregation, response casing, storage, or terminal classification. +- Recomputing cancellations in the browser. +- Redesigning the Reports page or introducing a new visual primitive. + +## Decisions + +### Preserve cancellation values in the typed report model + +The response schemas will explicitly parse `summary.totalCancelled` and each `daily[].cancelledCount`. The UI and export paths will consume these parsed fields rather than deriving cancellation counts from requests and errors. Derivation would be incorrect because requests may include successful, cancelled, and genuinely failed terminals, and future terminal classes may exist. + +### Zero-fill only synthesized empty days + +The date-range completion path will assign `cancelledCount: 0` to synthetic rows, matching the existing zero-fill semantics for other count metrics. A cancellation value returned by the API will be preserved, including an explicit zero. + +### Extend existing Reports presentation patterns + +The cancellation summary item and daily-table column will compose the page's existing summary and table primitives, spacing, typography, responsive behavior, and semantic design tokens. The CSV will add a localized cancellation header in the same column order used by the visible daily table. No one-off visual values or separate desktop/mobile markup will be introduced. + +### Use the existing localization boundary + +Visible labels and the CSV header will use the Reports translation namespace. English, Korean, and Simplified Chinese resources will receive equivalent cancellation labels; CSV generation will use the active locale just as existing headers do. + +## Failure Modes and Mitigations + +- **Schema strips valid backend data:** parser tests assert both cancellation fields survive parsing. +- **Synthetic rows expose `undefined` or a blank CSV cell:** zero-fill tests assert a numeric `0` in the model, table, and export. +- **Cancellation is accidentally folded into errors:** regression fixtures retain distinct requests, cancellations, and errors and assert all three independently. +- **A label is readable in one locale only:** locale coverage and real zh-CN browser evidence verify the visible label and CSV header. +- **The added column overflows or hides key values:** desktop and 390px mobile browser evidence verifies the existing responsive table behavior with the added column. +- **QA leaves local state behind:** the browser QA task records teardown of servers, ports, browser sessions, downloads, fixtures, and temporary data. + +## Example + +Given a parsed frontend report response with `totalRequests: 4`, `totalCancelled: 2`, and `totalErrors: 1`, plus a daily row with `requests: 4`, `cancelledCount: 2`, and `errorCount: 1`, parsing preserves all six values. The summary visibly shows Requests 4, Cancelled 2, and Errors 1; the daily table shows the same breakdown; and the localized CSV contains a cancellation column with value `2`. A missing date synthesized into the selected range displays and exports cancellation value `0`. diff --git a/openspec/changes/surface-reports-cancellation-totals/proposal.md b/openspec/changes/surface-reports-cancellation-totals/proposal.md new file mode 100644 index 0000000000..43d478aba6 --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/proposal.md @@ -0,0 +1,25 @@ +# Change: Surface reports cancellation totals + +## Why + +The reports API already returns cancellation totals, but the frontend parser drops them and the reports summary, daily table, and CSV export omit them. Operators therefore cannot distinguish cancelled requests from genuine errors on the Reports surface even though the owning metric contract requires cancellation counts alongside errors. + +## What Changes + +- Preserve `summary.totalCancelled` and `daily[].cancelledCount` when parsing reports responses. +- Zero-fill missing daily cancellation counts as `0` when constructing a complete date range. +- Show localized cancellation totals in the reports summary and daily table. +- Include a localized cancellation column and values in reports CSV exports. +- Add deterministic parser, rendering, export, localization, responsive-layout, and regression evidence while preserving existing request and error totals. + +## Capabilities + +### Modified Capabilities + +- `usage-error-metrics`: require the Reports frontend to preserve and visibly surface the cancellation fields already supplied by the reports API. + +## Impact + +- Affected area: Reports frontend response parsing, date-range zero-fill, summary cards, daily detail table, CSV export, and report translations. +- Compatibility: additive presentation only; existing requests and errors values and CSV semantics remain unchanged apart from the new cancellation column. +- No backend, database, or API contract changes are required. diff --git a/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md b/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md new file mode 100644 index 0000000000..a649c9e7e7 --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/specs/usage-error-metrics/spec.md @@ -0,0 +1,69 @@ +## MODIFIED Requirements + +### Requirement: Cancelled counts surface alongside error counts + +Metric surfaces that expose an error count MUST also expose the window's +cancelled count as an additive field: the dashboard overview metrics +(`cancelledCount`), the usage summary metrics (`cancelled7d`), the raw Reports +backend daily rows (`cancelled_count`) and summary (`total_cancelled`), and the +fleet pressure metrics (`cancelledCount`). The dashboard overview cancelled total +MUST be sourced from the demand quarter rollup (status grain) for the folded +segment plus the raw tail, so it stays accurate across history already folded +without the hourly `cancelled_count` measure. The dashboard frontend MUST +preserve `cancelledCount` when parsing the overview response. + +The Reports dashboard API MUST serialize the raw backend `total_cancelled` and +`cancelled_count` fields as `summary.totalCancelled` and +`daily[].cancelledCount`, respectively. The Reports frontend MUST preserve +those parsed camelCase values from the reports response. A daily row +synthesized to fill a missing date in the selected range MUST set +`cancelledCount` to `0`. The Reports summary and daily table MUST visibly show +the cancellation values with localized labels, and the Reports CSV export MUST +include a localized cancellation header and each daily row's cancellation +value. Adding cancellation presentation MUST NOT change the parsed, visible, or +exported request and error values. + +#### Scenario: Dashboard overview reports the status breakdown + +- **GIVEN** a window containing 1 successful, 2 cancelled, and 1 error rows + that are partially folded into the rollups +- **WHEN** the dashboard overview metrics are computed +- **THEN** the metrics expose `requests=4`, `errorCount=1`, and + `cancelledCount=2` + +#### Scenario: Dashboard overview preserves the status breakdown + +- **GIVEN** the dashboard overview API returns `requests=4`, `errorCount=1`, + and `cancelledCount=2` +- **WHEN** the frontend parses the overview response +- **THEN** the parsed metrics expose all three values unchanged + +#### Scenario: Reports preserve and display cancellation values + +- **GIVEN** a reports response whose summary has `totalRequests=4`, + `totalCancelled=2`, and `totalErrors=1` and whose frontend daily row has + `requests=4`, `cancelledCount=2`, and `errorCount=1` +- **WHEN** the Reports frontend parses and displays the response +- **THEN** the parsed summary has `totalCancelled=2` and the parsed daily row + has `cancelledCount=2` +- **AND** the localized summary visibly shows requests `4`, cancellations `2`, + and errors `1` +- **AND** the localized daily table visibly shows requests `4`, cancellations + `2`, and errors `1` + +#### Scenario: Reports zero-fill cancellations for a missing date + +- **GIVEN** a selected report range containing a date absent from the reports + response +- **WHEN** the Reports frontend synthesizes the daily row for that date +- **THEN** the synthesized row has `cancelledCount=0` +- **AND** the daily table visibly shows cancellation value `0` for that row + +#### Scenario: Reports CSV exports localized cancellation values + +- **GIVEN** parsed report rows with cancellation values `2` and `0` +- **WHEN** a user exports the report while a supported locale is active +- **THEN** the CSV contains the locale's cancellation header and the values `2` + and `0` in that column +- **AND** the exported request and error headers and values remain present and + unchanged diff --git a/openspec/changes/surface-reports-cancellation-totals/tasks.md b/openspec/changes/surface-reports-cancellation-totals/tasks.md new file mode 100644 index 0000000000..dc9a70985b --- /dev/null +++ b/openspec/changes/surface-reports-cancellation-totals/tasks.md @@ -0,0 +1,25 @@ +# Tasks + +## 1. Deterministic regression coverage + +- [x] 1.1 Add parser tests proving `summary.totalCancelled` and `daily[].cancelledCount` survive reports response parsing. +- [x] 1.2 Add date-range completion coverage proving a synthesized daily row has `cancelledCount: 0`. +- [x] 1.3 Add summary and daily-table tests proving Requests 4, Cancelled 2, and Errors 1 remain distinct and visible. +- [x] 1.4 Add CSV coverage proving the localized cancellation header, cancellation value `2`, zero-filled value `0`, and existing request/error values. +- [x] 1.5 Run the focused parser/table/export tests before implementation and record the expected cancellation-specific failures. + +## 2. Reports cancellation presentation + +- [x] 2.1 Extend the typed Reports schemas and zero-fill model to preserve cancellation values. +- [x] 2.2 Add a cancellation item to the existing Reports summary composition. +- [x] 2.3 Add a cancellation column to the existing daily detail table and CSV export. +- [x] 2.4 Add equivalent Reports cancellation labels for every supported locale, including English, Korean, and Simplified Chinese. +- [x] 2.5 Re-run the focused tests and frontend diagnostics, then run the affected frontend test, type-check, lint, and build gates. + +## 3. User-visible verification and cleanup + +- [x] 3.1 Exercise a real Reports fixture with Requests 4, Cancelled 2, and Errors 1 in Chromium and verify the visible summary, daily table, and downloaded CSV. +- [x] 3.2 Capture desktop, 390px mobile, and zh-CN evidence showing cancellation alongside unchanged request and error values. +- [x] 3.3 Verify a zero-filled date visibly shows and exports cancellation value `0`. +- [x] 3.4 Tear down browser sessions, frontend/backend processes, ports, fixture databases, downloads, and temporary QA artifacts, and record the cleanup receipt. +- [x] 3.5 Run `openspec validate surface-reports-cancellation-totals --strict` and retain the exact successful output. From f1c8d5cd19947d76191fea8da8c07a69df493f83 Mon Sep 17 00:00:00 2001 From: Borealin <41241077+Borealin@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:20:08 +0800 Subject: [PATCH 046/117] feat(model-sources): advertise operator-declared reasoning efforts (#1661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(model-sources): advertise operator-declared reasoning efforts Source-model Codex catalog entries hardcoded `supported_reasoning_levels=()`, `default_reasoning_level=None` and `supports_reasoning_summaries=False`, so a reasoning-capable backend could not advertise its efforts and Codex clients offered no reasoning-effort options for model-source models. Every other client-capability field on those entries is already an operator-overridable `raw_metadata_json` default. Derive the three reasoning fields from `raw_metadata_json`, accepting either effort slugs or `{"effort", "description"}` objects, ignoring malformed entries, and reporting a declared default only when it matches an advertised effort. Models without reasoning metadata keep the previous behavior. Backends differ in the efforts they accept (Model Studio exposes none/minimal/low/medium/high/xhigh/max, DeepSeek and Kimi expose low/high/max), so the advertised set has to be operator declared. * style: apply ruff format to reasoning metadata helpers * docs(contributors): add Borealin * fix(proxy): keep declared reasoning efforts for model-source models `normalize_unsupported_reasoning_effort` rewrites `minimal` because the ChatGPT/Codex backend drops it and the stream hangs. The fallback resolves against the subscription registry snapshot, which never contains model-source models, so a source request carrying `minimal` was silently downgraded to `low` — including when the operator had declared `minimal` in `supported_reasoning_levels`. Skip the rewrite when a populated snapshot has no entry for the model: that model is not served by the backend the workaround targets. An absent snapshot still takes the conservative rewrite, since the request cannot be attributed to a model source without registry data. Also document how `supported_reasoning_levels` relates to the pre-existing `supports_reasoning` chat-path opt-in. Refs #1660, #1672 * fix(model-sources): decide the reasoning-effort exemption by route, not registry The previous revision skipped the minimal rewrite when a populated registry snapshot had no entry for the model, treating absence as proof of a model source. That inference fails in both directions. A populated snapshot can omit a genuine subscription model -- a partial refresh, an account unavailable during refresh, or an operator-mapped slug outside the bootstrap set -- and those requests still reach the ChatGPT backend through the unfiltered fallback, where an unrewritten minimal restores the no-completion hang the workaround exists to prevent (#493). It was never right on the WebSocket transport at all, since no source is reachable from there. Conversely a source model whose slug shadows a subscription slug is present in the snapshot yet source-routed, so its declared effort was downgraded anyway. Apply the rewrite unconditionally at enforcement time and report the replaced effort, then undo it at the source-routing branch -- the first point where the route is known. Restoration is gated on the source declaring that effort, so sources without reasoning metadata keep their previous behaviour. The reported value is post-enforcement, so restoring it cannot resurrect an effort an API key overrode, and it is not stashed on the payload because ResponsesReasoning allows extra fields and would serialize it onto the wire. Also clamp declared efforts to the vocabulary the client surfaces understand: the Codex catalog is deserialized as a whole, so an operator typo must not be able to affect other entries. And treat a non-empty declared level set as the chat-path reasoning opt-in -- /v1/models derives supports_reasoning from it, so without this the same capability was advertised and silently stripped. Addresses the Codex review P1 and the maintainer's first review point on #1661. * test(model-sources): accept the restore argument in the source-response stub _source_responses_response now takes pre_normalization_effort; the prompt-cache integration stub replaces it, so it has to accept the keyword. * fix(model-sources): keep the ultra alias out of the source-routed restore The restore added for the minimal workaround reported every effort the normalizer rewrote, including the ultra -> max wire alias. That alias is not a backend workaround: it mirrors what the reference Codex client sends and is required on every upstream surface, so a source declaring ultra had max quietly turned back into ultra on the wire. Report only fallback rewrites. Restore the normalized effort rather than the caller's string. The gate already compared the trimmed, lowercased form while the assignment used the argument verbatim; that only held because the sole producer normalizes first, which is an invariant no reader of the function can see. Make pre_normalization_effort a required keyword on _source_responses_response so a future route cannot reach a source while silently dropping the effort, and treat supports_reasoning_summaries as the chat-path reasoning opt-in alongside declared levels -- /v1/models derives supports_reasoning from either key, so declaring summaries alone advertised a capability the chat sanitizer stripped. Pin both route call sites. The codex-native /backend-api/codex/responses threading was covered by nothing: dropping its argument left the whole suite green, while that route is the one Codex CLI uses and the origin of --reasoning-effort minimal. Each route now has its own test, verified by mutation per call site. Correct the chat endpoint's comment. It claimed the endpoint does not reach the source-routing branch, which is false -- v1_chat_completions does source-route. The discard is safe for a different reason: that branch forwards the untouched original chat payload, never the enforced Responses payload, so there is nothing to restore. A maintainer trusting the old wording could reintroduce the downgrade. Also record that a declared capability outranks an explicit "supports_reasoning": false, matching the /v1/models derivation, and drop the proposal's claim that an operator can advertise none -- the clamp excludes it, as the subscription catalog does. * fix(model-sources): validate declared efforts by shape, not a fixed enum The clamp added with the parser dropped any effort outside a seven-value set, which silently discarded `none`. The #1660 backend survey shows `none` is a real value -- GLM exposes max/high/none and Model Studio includes it -- and it is already first-class for API-key enforced efforts in app/modules/api_keys/service.py, so filtering it out of source catalogs alone left the two vocabularies disagreeing about the same word. Effort sets diverge per provider, so any enum here is a guess that costs operators efforts their backend really accepts. Validate shape instead: a string or a mapping with a string `effort`, normalized, non-empty, deduplicated. Malformed entries are still dropped, and the default level is still restricted to the advertised set. Also drop the proposal's pointer to #1672, which is closed -- the dashboard work now lands as the UI-only rebase of #1675 on this parser -- and correct a comment in _v1_supports_reasoning that still claimed source models advertise no reasoning levels, which stopped being true when levels began deriving from raw_metadata_json. * fix(model-sources): gate reasoning derivation on the operator's switch The previous revision made declared levels imply the chat-path reasoning opt-in. That inverted the relationship between the two keys: levels describe which efforts an opted-in backend accepts, not whether reasoning is permitted at all, and treating them as permission meant a model the backend considered opted in still read as "Reasoning: off" in the dashboard, whose single checkbox is bound to supports_reasoning alone. Gate the other way instead. supports_reasoning is the switch; the levels, the default and the summary flag are detail that only applies once it is on. With the switch off a model advertises no efforts, /v1/models reports supports_reasoning false, the chat sanitizer strips, and the restore has no declared effort to act on. With it on, all four agree. The visible-and-inert state the implication was meant to avoid becomes unreachable rather than papered over, and the maintainer's call that the chat-path opt-in stays supports_reasoning-only is honoured. source_model_reasoning_levels is gated too, not just the catalog: it is what the unsupported-effort restore consults, so without it a switched-off model would still have its rewritten effort restored and forwarded upstream. The Responses path keeps forwarding reasoning regardless. reasoning is a first-class field of the schema a source opts into via supports_responses, whereas on the chat path three of the five stripped keys are vendor extensions that only survive because the request model allows extra fields. Making Responses strip as well would reverse that existing decision and belongs in its own change. The two tests that asserted the implication were rewritten rather than deleted: as fixtures gained the switch they would have kept passing while asserting nothing. They now pin the negative, and a third pins all four surfaces together. Mutation-checked: ungating the derivation, the levels accessor, or the opt-in each fails them. * fix(proxy): sanitize codex catalog reasoning efforts --------- Co-authored-by: Darafei Praliaskouski Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- app/modules/model_sources/catalog.py | 118 +++++++++- app/modules/proxy/_service/websocket/mixin.py | 5 +- app/modules/proxy/api.py | 84 +++++-- app/modules/proxy/request_policy.py | 96 +++++++- .../proposal.md | 80 +++++++ .../specs/model-catalog-compat/spec.md | 193 ++++++++++++++++ .../source-model-reasoning-metadata/tasks.md | 43 ++++ .../integration/test_model_source_routing.py | 91 ++++++++ .../test_openai_compat_features.py | 4 +- tests/integration/test_v1_models.py | 31 +++ tests/unit/test_model_sources_catalog.py | 206 ++++++++++++++++++ tests/unit/test_proxy_api_websocket_auth.py | 5 +- .../unit/test_proxy_load_balancer_refresh.py | 8 +- tests/unit/test_proxy_utils.py | 194 +++++++++++++++++ 14 files changed, 1111 insertions(+), 47 deletions(-) create mode 100644 openspec/changes/source-model-reasoning-metadata/proposal.md create mode 100644 openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md create mode 100644 openspec/changes/source-model-reasoning-metadata/tasks.md diff --git a/app/modules/model_sources/catalog.py b/app/modules/model_sources/catalog.py index da7b25bbaa..39f846e25a 100644 --- a/app/modules/model_sources/catalog.py +++ b/app/modules/model_sources/catalog.py @@ -4,6 +4,7 @@ from app.core.openai.model_registry import ( MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + ReasoningLevel, UpstreamModel, ) from app.core.types import JsonValue @@ -58,15 +59,26 @@ def _to_upstream_model(source: ModelSource, source_model: ModelSourceModel) -> U input_modalities = ("text", "image") if source_model.supports_vision else ("text",) display_name = source_model.display_name or source_model.model + # The dashboard's single Reasoning switch is the master gate: it is the + # only reasoning control an operator has in the UI, so a model with it off + # must not advertise efforts it will never be allowed to use. Keeping the + # switch authoritative is what lets the Codex catalog, /v1/models and the + # dashboard checkbox agree; deriving levels regardless would advertise a + # capability the chat sanitizer then strips. + reasoning_opted_in = raw.get("supports_reasoning") is True + reasoning_levels = _reasoning_levels_from_metadata(raw) if reasoning_opted_in else () + default_reasoning_level = ( + _default_reasoning_level_from_metadata(raw, reasoning_levels) if reasoning_opted_in else None + ) return UpstreamModel( slug=source_model.model, display_name=display_name, description=display_name, context_window=context_window, input_modalities=input_modalities, - supported_reasoning_levels=(), - default_reasoning_level=None, - supports_reasoning_summaries=False, + supported_reasoning_levels=reasoning_levels, + default_reasoning_level=default_reasoning_level, + supports_reasoning_summaries=reasoning_opted_in and raw.get("supports_reasoning_summaries") is True, support_verbosity=False, default_verbosity=None, prefer_websockets=False, @@ -81,18 +93,104 @@ def _to_upstream_model(source: ModelSource, source_model: ModelSourceModel) -> U ) -def source_model_supports_reasoning(source: ModelSource, model: str) -> bool: - """Whether the source model opted into reasoning via raw catalog metadata. +def _reasoning_levels_from_metadata(raw: dict[str, JsonValue]) -> tuple[ReasoningLevel, ...]: + """Reasoning efforts advertised for a source model. + + Source catalogs have no first-class reasoning schema, so operators declare + the efforts their backend accepts under ``supported_reasoning_levels`` in + ``raw_metadata_json``. Both shapes are accepted:: + + ["low", "high", "max"] + [{"effort": "low", "description": "Low reasoning effort"}] - Source catalog entries have no first-class reasoning flag; a model that - genuinely supports reasoning can opt in with ``"supports_reasoning": true`` - in ``raw_metadata_json``. Everything else is treated as non-reasoning so - client-sent reasoning toggles are stripped before forwarding. + Efforts are normalized (trimmed and lowercased) and deduplicated. + Validation is on shape, not on membership of a fixed vocabulary: backends + disagree on which efforts exist (GLM exposes ``none``, Model Studio + includes it, others stop at ``low``/``high``/``max``), so an enum here + would silently drop efforts a provider really accepts. Malformed entries -- + a non-string, a mapping without a string ``effort``, an empty slug -- are + ignored, keeping the previous no-reasoning default for models that never + opted in. """ - entry = next( + declared = raw.get("supported_reasoning_levels") + if not is_json_list(declared): + return () + levels: list[ReasoningLevel] = [] + seen: set[str] = set() + for item in declared: + if isinstance(item, str): + effort = item + description = f"{item.strip().lower()} reasoning effort" + elif is_json_mapping(item): + effort_value = item.get("effort") + if not isinstance(effort_value, str): + continue + effort = effort_value + description_value = item.get("description") + description = ( + description_value + if isinstance(description_value, str) + else f"{effort.strip().lower()} reasoning effort" + ) + else: + continue + effort = effort.strip().lower() + if not effort or effort in seen: + continue + seen.add(effort) + levels.append(ReasoningLevel(effort=effort, description=description)) + return tuple(levels) + + +def _default_reasoning_level_from_metadata( + raw: dict[str, JsonValue], + levels: tuple[ReasoningLevel, ...], +) -> str | None: + """Operator-declared default effort, restricted to the advertised levels.""" + declared = raw.get("default_reasoning_level") + if not isinstance(declared, str): + return None + normalized = declared.strip().lower() + if not any(level.effort == normalized for level in levels): + return None + return normalized + + +def _enabled_source_model(source: ModelSource, model: str) -> ModelSourceModel | None: + return next( (candidate for candidate in source.models if candidate.model == model and candidate.is_enabled), None, ) + + +def source_model_reasoning_levels(source: ModelSource, model: str) -> tuple[ReasoningLevel, ...]: + """Reasoning efforts an opted-in source model declared. + + Gated on ``supports_reasoning`` like the catalog derivation, so the + unsupported-effort restore cannot hand a declared effort to a model whose + operator left the Reasoning switch off. + """ + entry = _enabled_source_model(source, model) + if entry is None: + return () + raw = _raw_metadata(entry) + if raw.get("supports_reasoning") is not True: + return () + return _reasoning_levels_from_metadata(raw) + + +def source_model_supports_reasoning(source: ModelSource, model: str) -> bool: + """Whether the operator turned the model's Reasoning switch on. + + ``"supports_reasoning": true`` in ``raw_metadata_json`` is the single + opt-in, written by the dashboard's Reasoning checkbox. Declared levels do + not imply it: they describe *which* efforts an opted-in backend accepts, + not *whether* reasoning is allowed at all, and the catalog derivation is + gated on this same flag so the two can never disagree. Everything else is + treated as non-reasoning, so client-sent reasoning toggles are stripped + before forwarding on the chat path. + """ + entry = _enabled_source_model(source, model) if entry is None: return False return _raw_metadata(entry).get("supports_reasoning") is True diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 954f8aea59..bdd122a2e0 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -2924,11 +2924,14 @@ async def _prepare_websocket_response_create_request( # exactly, including the enforced-model substitution here and the # fast-mode correction after enforcement below. raw_source_model = effective_model_for_api_key(refreshed_api_key, responses_payload.model) + # The effort the normalizer replaced is discarded here on purpose: the + # WebSocket transport never reaches a model source, so the rewrite that + # works around the backend hang must stick. service_tier_was_enforced = apply_api_key_enforcement( responses_payload, refreshed_api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced if prohibit_fast_mode and model_alias_requests_fast_mode(raw_source_model): raw_source_model = responses_payload.model apply_enforced_service_tier_model_fallback( diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index b63a672621..c2a361720b 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -250,6 +250,7 @@ openai_validation_error, resolve_model_alias, responses_source_route_excluded, + restore_source_reasoning_effort, sanitize_source_chat_payload, strip_terminal_compaction_trigger_input, validate_model_access, @@ -1070,9 +1071,11 @@ async def responses( return _logged_error_json_response(request, 400, error) raw_source_model = _effective_optional_model_for_api_key(api_key, responses_payload.model) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( - responses_payload, api_key - ) + ( + prohibit_fast_mode, + service_tier_was_enforced, + pre_normalization_effort, + ) = await _apply_api_key_enforcement_with_fast_mode_policy(responses_payload, api_key) if prohibit_fast_mode and _is_fast_mode_model_alias(raw_source_model): raw_source_model = responses_payload.model validate_model_access(api_key, responses_payload.model) @@ -1108,6 +1111,7 @@ async def responses( source=source, api_key=api_key, rate_limit_headers=rate_limit_headers, + pre_normalization_effort=pre_normalization_effort, ) apply_enforced_service_tier_model_fallback( @@ -1219,9 +1223,11 @@ async def v1_responses( error = openai_validation_error(exc) return _logged_error_json_response(request, 400, error) raw_source_model = _effective_optional_model_for_api_key(api_key, responses_payload.model) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( - responses_payload, api_key - ) + ( + prohibit_fast_mode, + service_tier_was_enforced, + pre_normalization_effort, + ) = await _apply_api_key_enforcement_with_fast_mode_policy(responses_payload, api_key) if prohibit_fast_mode and _is_fast_mode_model_alias(raw_source_model): raw_source_model = responses_payload.model validate_model_access(api_key, responses_payload.model) @@ -1252,6 +1258,7 @@ async def v1_responses( source=source, api_key=api_key, rate_limit_headers=rate_limit_headers, + pre_normalization_effort=pre_normalization_effort, ) apply_enforced_service_tier_model_fallback( responses_payload, @@ -2030,14 +2037,18 @@ async def _hide_upstream_quota_for_api_key_clients(api_key: ApiKeyData | None) - async def _apply_api_key_enforcement_with_fast_mode_policy( payload: ResponsesRequest | ResponsesCompactRequest, api_key: ApiKeyData | None, -) -> tuple[bool, bool]: +) -> tuple[bool, bool, str | None]: prohibit_fast_mode = await _prohibit_fast_mode_enabled() - service_tier_was_enforced = apply_api_key_enforcement( + enforcement = apply_api_key_enforcement( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, ) - return prohibit_fast_mode, service_tier_was_enforced + return ( + prohibit_fast_mode, + enforcement.service_tier_was_enforced, + enforcement.pre_normalization_reasoning_effort, + ) async def _prohibit_fast_mode_enabled() -> bool: @@ -3793,6 +3804,9 @@ def _is_codex_backend_catalog_model(model: UpstreamModel) -> bool: return model.raw.get("shell_type") == "shell_command" +_CODEX_WIRE_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) + + def _codex_model_truncation_policy(model: UpstreamModel) -> CodexTruncationPolicy: if "truncation_policy" in model.raw: try: @@ -3810,8 +3824,24 @@ def _codex_model_experimental_supported_tools(model: UpstreamModel) -> list[str] return [tool for tool in tools if isinstance(tool, str)] +def _codex_wire_reasoning_levels(model: UpstreamModel) -> list[ReasoningLevelSchema]: + return [ + ReasoningLevelSchema(effort=level.effort, description=level.description) + for level in model.supported_reasoning_levels + if level.effort in _CODEX_WIRE_REASONING_EFFORTS + ] + + +def _codex_wire_default_reasoning_level(model: UpstreamModel) -> str | None: + default = model.default_reasoning_level + if default in _CODEX_WIRE_REASONING_EFFORTS: + return default + return None + + def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None) -> CodexModelEntry: raw = model.raw + reasoning_levels = _codex_wire_reasoning_levels(model) extra: dict[str, JsonValue] = {} skip_keys = { @@ -3850,11 +3880,8 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None display_name=model.display_name, description=model.description, base_instructions=model.base_instructions, - default_reasoning_level=model.default_reasoning_level, - supported_reasoning_levels=[ - ReasoningLevelSchema(effort=rl.effort, description=rl.description) - for rl in model.supported_reasoning_levels - ], + default_reasoning_level=_codex_wire_default_reasoning_level(model), + supported_reasoning_levels=reasoning_levels, supported_in_api=model.supported_in_api, priority=model.priority, minimal_client_version=model.minimal_client_version, @@ -3917,8 +3944,8 @@ def _v1_model_capabilities(model: UpstreamModel) -> dict[str, JsonValue]: def _v1_supports_reasoning(model: UpstreamModel) -> bool: if bool(model.supported_reasoning_levels) or model.supports_reasoning_summaries: return True - # OpenAI-compatible source models advertise no reasoning levels; their - # catalog entries opt in via raw metadata so /v1/models reflects reality. + # Source models whose operator declared no levels and no summary support + # opt in via raw metadata instead, so /v1/models reflects reality. return model.raw.get("supports_reasoning") is True @@ -4047,7 +4074,11 @@ async def v1_chat_completions( except ValidationError as exc: error = openai_validation_error(exc) return _logged_error_json_response(request, 400, error, headers=rate_limit_headers) - prohibit_fast_mode, service_tier_was_enforced = await _apply_api_key_enforcement_with_fast_mode_policy( + # The replaced effort is discarded: the enforced Responses payload built + # here is only ever forwarded to a subscription. This endpoint does + # source-route, but that branch forwards the untouched original chat + # payload, so there is nothing for a restore to undo. + prohibit_fast_mode, service_tier_was_enforced, _ = await _apply_api_key_enforcement_with_fast_mode_policy( responses_payload, api_key ) if prohibit_fast_mode and _is_fast_mode_model_alias(effective_model): @@ -4407,7 +4438,16 @@ async def _source_responses_response( source: ModelSource, api_key: ApiKeyData | None, rate_limit_headers: Mapping[str, str], + pre_normalization_effort: str | None, ) -> Response: + # This is the first point where the request is known to be served by a + # model source rather than a subscription account, so it is the only place + # the reasoning-effort workaround can be undone safely. + restore_source_reasoning_effort( + payload, + source, + pre_normalization_effort=pre_normalization_effort, + ) reservation = await _enforce_request_limits( api_key, request_model=payload.model, @@ -5241,7 +5281,7 @@ async def _stream_responses( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced if forwarded_request: payload.service_tier = forwarded_effective_service_tier else: @@ -5692,11 +5732,13 @@ async def _collect_responses( prefer_http_bridge: bool = False, prohibit_fast_mode: bool = False, ) -> Response: + # The replaced effort is discarded: this path is subscription-only, so the + # rewrite that works around the backend hang must stick. service_tier_was_enforced = apply_api_key_enforcement( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced apply_enforced_service_tier_model_fallback( payload, service_tier_was_enforced=service_tier_was_enforced, @@ -5918,11 +5960,13 @@ async def _compact_responses( openai_cache_affinity: bool = False, prohibit_fast_mode: bool = False, ) -> JSONResponse: + # The replaced effort is discarded: this path is subscription-only, so the + # rewrite that works around the backend hang must stick. service_tier_was_enforced = apply_api_key_enforcement( payload, api_key, prohibit_fast_mode=prohibit_fast_mode, - ) + ).service_tier_was_enforced apply_enforced_service_tier_model_fallback( payload, service_tier_was_enforced=service_tier_was_enforced, diff --git a/app/modules/proxy/request_policy.py b/app/modules/proxy/request_policy.py index b33b6864d4..1c5547bf75 100644 --- a/app/modules/proxy/request_policy.py +++ b/app/modules/proxy/request_policy.py @@ -1,6 +1,7 @@ from __future__ import annotations import logging +from typing import NamedTuple from pydantic import ValidationError @@ -23,7 +24,9 @@ from app.core.types import JsonValue from app.core.utils.json_guards import is_json_list, is_json_mapping from app.core.utils.request_id import get_request_id +from app.db.models import ModelSource from app.modules.api_keys.service import ApiKeyData +from app.modules.model_sources.catalog import source_model_reasoning_levels logger = logging.getLogger(__name__) @@ -126,13 +129,27 @@ def validate_model_access(api_key: ApiKeyData | None, model: str | None) -> None raise ProxyModelNotAllowed(f"This API key does not have access to model '{model}'") +class ApiKeyEnforcementResult(NamedTuple): + """What :func:`apply_api_key_enforcement` observed while mutating the payload. + + ``pre_normalization_reasoning_effort`` carries the effort that + :func:`normalize_unsupported_reasoning_effort` replaced, so a caller that + later routes the request to an OpenAI-compatible model source can restore + it. It is the post-enforcement value: restoring it cannot resurrect an + effort an API key overrode. + """ + + service_tier_was_enforced: bool + pre_normalization_reasoning_effort: str | None + + def apply_api_key_enforcement( payload: ResponsesRequest | ResponsesCompactRequest, api_key: ApiKeyData | None, *, registry: ModelRegistry | None = None, prohibit_fast_mode: bool = False, -) -> bool: +) -> ApiKeyEnforcementResult: """Apply API-key policy and report whether it supplied the service tier. The returned provenance is captured before mutating ``payload``. Callers @@ -143,8 +160,8 @@ def apply_api_key_enforcement( normalize_upstream_model_alias(payload, prohibit_fast_mode=prohibit_fast_mode) if api_key is None: - normalize_unsupported_reasoning_effort(payload) - return False + pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) + return ApiKeyEnforcementResult(False, pre_normalization_effort) if api_key.enforced_model: requested_model = payload.model @@ -186,7 +203,7 @@ def apply_api_key_enforcement( api_key.enforced_reasoning_effort, ) - normalize_unsupported_reasoning_effort(payload) + pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) service_tier_was_enforced = False if api_key.enforced_service_tier is not None: @@ -216,7 +233,7 @@ def apply_api_key_enforcement( api_key.enforced_service_tier, effective_service_tier, ) - return service_tier_was_enforced + return ApiKeyEnforcementResult(service_tier_was_enforced, pre_normalization_effort) def apply_enforced_service_tier_model_fallback( @@ -449,7 +466,7 @@ def normalize_unsupported_reasoning_effort( payload: ResponsesRequest | ResponsesCompactRequest, *, registry: ModelRegistry | None = None, -) -> None: +) -> str | None: """Rewrite ``reasoning.effort`` values the upstream backend rejects. Some efforts that codex-lb accepts at the API surface (notably @@ -462,10 +479,25 @@ def normalize_unsupported_reasoning_effort( Client-plane efforts the reference Codex client aliases before sending (``ultra`` -> ``max``) are rewritten the same way here. + + Returns the effort that the unsupported-effort fallback replaced, in + normalized (trimmed, lowercased) form, or ``None`` when nothing restorable + was rewritten. Model sources do not have the backend quirk that fallback + works around, but whether a request is served by one is only known after + source selection, which happens later; callers that can reach a source + carry this value forward and restore it there (see + ``restore_source_reasoning_effort``). + + The ``ultra`` -> ``max`` wire alias is never reported. That aliasing mirrors + the reference client and is required on every upstream surface, so it must + survive source routing too. + + The reported value is the post-enforcement effort rather than the client's + original, so restoring it cannot resurrect an effort an API key overrode. """ if payload.reasoning is None or payload.reasoning.effort is None: - return + return None requested_effort = payload.reasoning.effort normalized_effort = requested_effort.strip().lower() @@ -480,10 +512,12 @@ def normalize_unsupported_reasoning_effort( requested_effort, wire_alias, ) - return + # Deliberately not reported as restorable: the ultra -> max alias must + # hold on every surface, source-routed payloads included. + return None if normalized_effort not in _UNSUPPORTED_UPSTREAM_REASONING_EFFORTS: - return + return None fallback = _resolve_reasoning_effort_fallback( payload.model, @@ -497,6 +531,50 @@ def normalize_unsupported_reasoning_effort( requested_effort, fallback, ) + return normalized_effort + + +def restore_source_reasoning_effort( + payload: ResponsesRequest | ResponsesCompactRequest, + source: ModelSource, + *, + pre_normalization_effort: str | None, +) -> None: + """Undo :func:`normalize_unsupported_reasoning_effort` for a source-routed request. + + The rewrite exists solely to work around a ChatGPT/Codex backend quirk, so + it must not reach an OpenAI-compatible model source. This runs at the point + where the source has actually been selected, which is the only place the + routing outcome is known -- inferring it earlier from registry membership + misfires in both directions (a subscription model missing from a populated + snapshot, and a source model whose slug shadows a subscription one). + + The restore is gated on the operator having declared the effort for this + model: sources without reasoning metadata keep the pre-existing behaviour, + and an effort the backend never advertised is not sent to it. + """ + if pre_normalization_effort is None or payload.reasoning is None: + return + if payload.model is None: + return + restored_effort = pre_normalization_effort.strip().lower() + declared = {level.effort for level in source_model_reasoning_levels(source, payload.model)} + if restored_effort not in declared: + return + current_effort = payload.reasoning.effort + # Normalized on assignment rather than trusting the caller: the sole + # producer already reports the normalized form, but that invariant is + # non-local and a casing variant must never reach the wire. + payload.reasoning.effort = restored_effort + logger.info( + "reasoning_effort_restored_for_source request_id=%s model=%s source_id=%s " + "normalized_effort=%s restored_effort=%s", + get_request_id(), + payload.model, + source.id, + current_effort, + restored_effort, + ) def _resolve_reasoning_effort_fallback( diff --git a/openspec/changes/source-model-reasoning-metadata/proposal.md b/openspec/changes/source-model-reasoning-metadata/proposal.md new file mode 100644 index 0000000000..b5a7367afc --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/proposal.md @@ -0,0 +1,80 @@ +## Why + +Source-model Codex catalog entries hardcode `supported_reasoning_levels=()`, +`default_reasoning_level=None`, and `supports_reasoning_summaries=False`. Every +other client-capability field on those entries is an operator-overridable +`raw_metadata_json` default, so a reasoning-capable backend has no way to +advertise its efforts and Codex clients show no reasoning-effort options for +model-source models. + +The efforts themselves already reach the source: the Responses path forwards +`reasoning` unchanged, so an operator who hardcodes `model_reasoning_effort` in +`config.toml` gets working reasoning today. Only the advertisement is missing, +which makes the capability undiscoverable in the client UI. + +Backends differ in the efforts they accept — for example Alibaba Model Studio +exposes `none`/`minimal`/`low`/`medium`/`high`/`xhigh`/`max`, while DeepSeek and +Kimi expose `low`/`high`/`max` — so the advertised set has to be operator +declared rather than inferred, and validated by shape rather than against a +fixed enum. That includes `none`, which the #1660 backend survey shows is a +real value (GLM `max`/`high`/`none`), and which is already first-class for +API-key enforced efforts in `app/modules/api_keys/service.py`; filtering it out +of source catalogs alone would have left the two vocabularies disagreeing. + +## What Changes + +- Read `supported_reasoning_levels`, `default_reasoning_level`, and + `supports_reasoning_summaries` for source-model catalog entries from + `raw_metadata_json` instead of hardcoding them. +- Accept both effort slugs (`["low", "high"]`) and objects + (`[{"effort": "low", "description": "..."}]`), ignoring malformed entries. +- Gate all of it on the existing `"supports_reasoning"` switch, the only + reasoning control the dashboard exposes, so a model with it off advertises + nothing and keeps the existing no-reasoning behavior. +- Normalize and deduplicate declared efforts, validating shape rather than + membership of a fixed vocabulary. +- Undo the unsupported-effort rewrite for requests that are actually routed to + a model source and that declared the effort, instead of inferring the route + from registry membership. This covers only the `minimal` workaround; the + `ultra` -> `max` wire alias mirrors the reference client and stays applied on + every surface. + +## Relationship to `supports_reasoning` + +`raw_metadata_json` carries reasoning keys with different jobs: + +- `supports_reasoning` is the **switch**. It is written by the dashboard's + single `Reasoning` checkbox and gates `sanitize_source_chat_payload`, which + strips `reasoning`, `reasoning_effort` and related toggles on the Chat + Completions path. +- `supported_reasoning_levels` / `default_reasoning_level` / + `supports_reasoning_summaries` are the **detail**: which efforts an opted-in + backend accepts. They are set through the API today; the dashboard UI for + them is the UI-only rebase of #1675 on this parser. + +Detail is gated on the switch. An earlier revision of this change instead made +declared levels imply the switch, which inverted the relationship: it turned a +description of *which* efforts into permission for reasoning at all, and left +the dashboard checkbox reading `false` for a model the backend treated as +opted in. Gating the other way keeps every surface consistent — `/v1/models` +derives `supports_reasoning` from the levels and the summary flag before +consulting the raw key, so a model advertising levels while the sanitizer +strips its chat requests would be visible and inert at once. With the gate, that +state is unreachable. + +The Responses path forwards `reasoning` regardless, as it does today: it is a +first-class field of the Responses schema that a source opts into with +`supports_responses`, unlike the chat path where three of the stripped keys are +vendor extensions that only survive because the request model allows extra +fields. Making the Responses path strip as well would reverse that existing +design decision and is out of scope here. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `model-catalog-compat` diff --git a/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md b/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..71af684608 --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/specs/model-catalog-compat/spec.md @@ -0,0 +1,193 @@ +## ADDED Requirements + +### Requirement: Source-model catalog entries advertise operator-declared reasoning efforts + +Codex catalog entries built for OpenAI-compatible source models MUST derive +`supported_reasoning_levels`, `default_reasoning_level`, and +`supports_reasoning_summaries` from the source model's `raw_metadata_json` +rather than reporting a fixed no-reasoning capability. + +Derivation MUST be gated on `"supports_reasoning": true`. That flag is the only +reasoning control the dashboard exposes, so a model whose operator left it off +MUST advertise no efforts, no default, and no summary support regardless of what +else the metadata declares. Levels say *which* efforts an opted-in backend +accepts, not *whether* reasoning is permitted, and gating them on the same flag +that gates the chat-completions sanitizer is what keeps the Codex catalog, +`/v1/models` and the dashboard checkbox in agreement. + +`supported_reasoning_levels` MUST accept a list of effort slugs and a list of +`{"effort", "description"}` objects. Entries that are neither a string nor a +mapping with a string `effort`, and duplicate efforts, MUST be ignored. A +non-list value MUST yield no advertised efforts. `default_reasoning_level` MUST +be reported only when it matches one of the advertised efforts. A source model +without reasoning metadata MUST continue to advertise no efforts, no default, +and no summary support. + +Declared efforts MUST be normalized (trimmed and lowercased) and +deduplicated. They MUST NOT be filtered against a fixed vocabulary: backends +disagree on which efforts exist -- `none` is real on GLM and Alibaba Model +Studio, while others stop at `low`/`high`/`max` -- so an enum would drop +efforts a provider genuinely accepts. Only shape is validated; an entry that +is not a string, a mapping without a string `effort`, or an empty slug MUST be +dropped. + +#### Scenario: Effort slugs are advertised in declaration order + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["low", "medium", "high", "xhigh"]` and + `"default_reasoning_level": "high"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises efforts `low`, `medium`, `high`, `xhigh` in that order +- **AND** `default_reasoning_level` is `high` + +#### Scenario: Effort objects carry operator descriptions and summary support + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": [{"effort": "low", "description": "Low effort"}]` + and `"supports_reasoning_summaries": true` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the `low` effort is advertised with description `Low effort` +- **AND** `supports_reasoning_summaries` is `true` + +#### Scenario: Malformed entries and out-of-range defaults are dropped + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["low", "low", {"description": "x"}, 7, {"effort": "high"}]` + and `"default_reasoning_level": "ultra"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises exactly `low` and `high` +- **AND** `default_reasoning_level` is absent + +#### Scenario: Casing variants are normalized, unknown efforts are kept + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": [" Low ", "HIGH", "provider-specific"]` and + `"default_reasoning_level": " HIGH "` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises `low`, `high`, and `provider-specific` +- **AND** `default_reasoning_level` is `high` + +#### Scenario: An operator-declared `none` survives + +- **GIVEN** a source model whose `raw_metadata_json` sets + `"supported_reasoning_levels": ["none", "high", "max"]` and + `"default_reasoning_level": "none"` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises `none`, `high`, and `max` +- **AND** `default_reasoning_level` is `none` + +#### Scenario: Models without reasoning metadata keep the previous behavior + +- **GIVEN** a source model with no `raw_metadata_json` +- **WHEN** a client fetches the Codex model catalog +- **THEN** the entry advertises no reasoning efforts, no default effort, and no + reasoning-summary support + +### Requirement: The reasoning switch is the single opt-in across every surface + +`"supports_reasoning": true` MUST remain the only reasoning opt-in for a source +model. Declared levels or `supports_reasoning_summaries` MUST NOT imply it. + +Because catalog derivation is gated on the same flag, the surfaces cannot +disagree: with the switch off the model advertises no efforts, `/v1/models` +reports `supports_reasoning: false`, the chat-completions sanitizer strips the +client's reasoning fields, and the unsupported-effort restore has no declared +effort to act on. With it on, the operator's declared efforts reach all of them. + +#### Scenario: The switch is off + +- **GIVEN** a source model that declares `supported_reasoning_levels` and + `supports_reasoning_summaries` but not `"supports_reasoning": true` +- **WHEN** its catalog entry is built and a chat-completions request for it + carries reasoning fields +- **THEN** the entry advertises no efforts, no default, and no summary support +- **AND** `/v1/models` reports `supports_reasoning: false` +- **AND** the request's reasoning fields are stripped + +#### Scenario: The switch is on + +- **GIVEN** the same source model with `"supports_reasoning": true` added +- **WHEN** its catalog entry is built and a chat-completions request for it + carries reasoning fields +- **THEN** the entry advertises the declared efforts and summary support +- **AND** the request's reasoning fields are forwarded + +#### Scenario: The switch alone still opts in + +- **GIVEN** a source model that sets only `"supports_reasoning": true` +- **WHEN** a chat-completions request for that model carries reasoning fields +- **THEN** the fields are forwarded, and the entry advertises no specific efforts + +### Requirement: The unsupported-effort rewrite is undone for source-routed requests + +The `minimal` normalization works around a ChatGPT/Codex backend that drops the +value, hanging the stream. Model sources do not have that defect, so a request +served by one MUST NOT be downgraded by it. + +Whether a request is served by a model source is known only after source +selection, which runs after enforcement. The rewrite MUST therefore be applied +unconditionally at enforcement time, and the replaced effort MUST be reported to +the caller so it can be restored once a source has actually been selected. +Restoration MUST occur only when a source was selected and the replaced effort +is among the efforts that source declares for the model. Declared efforts are +read through the same `"supports_reasoning"` gate as the catalog, so a model +whose switch is off has none and is never restored. The reported effort +MUST be the post-enforcement value, so restoring it cannot resurrect an effort +an API key overrode, and MUST be the normalized (trimmed, lowercased) form, so +restoration cannot reintroduce a casing variant the normalizer removed. + +Restoration MUST apply only to efforts replaced by the unsupported-effort +fallback. The `ultra` -> `max` rewrite is a wire alias rather than a workaround: +it mirrors the reference client and is required on every upstream surface, so it +MUST remain applied to source-routed payloads even when the source declares +`ultra`. + +Registry membership MUST NOT be used to decide this. A populated snapshot can +omit a genuine subscription model — a partial refresh, an account unavailable +during refresh, or an operator-mapped slug outside the bootstrap set — and those +requests still reach the ChatGPT backend, where skipping the rewrite restores +the hang. Conversely a source model whose slug shadows a subscription slug is +present in the snapshot yet source-routed. + +#### Scenario: A source that declared the effort receives it unchanged + +- **GIVEN** a source model declaring `["minimal", "low", "high"]` +- **AND** a request for that model with `reasoning.effort` of `minimal` +- **WHEN** the request is routed to the source +- **THEN** the source receives `minimal` + +#### Scenario: A source that did not declare the effort keeps the safe value + +- **GIVEN** a source model declaring `["low", "high"]` +- **AND** a request for that model with `reasoning.effort` of `minimal` +- **WHEN** the request is routed to the source +- **THEN** the source receives the rewritten effort + +#### Scenario: A source declaring ultra still receives the max alias + +- **GIVEN** a source model declaring `["ultra", "max"]` +- **AND** a request for that model with `reasoning.effort` of `ultra` +- **WHEN** the request is routed to the source +- **THEN** the source receives `max` + +#### Scenario: Subscription requests keep the workaround + +- **GIVEN** a request with `reasoning.effort` of `minimal` that is not routed to + a model source, including one whose model is absent from a populated registry + snapshot +- **WHEN** the request is forwarded +- **THEN** the effort is rewritten to the model's lowest supported effort + +#### Scenario: WebSocket requests keep the workaround + +- **GIVEN** a WebSocket Responses request with `reasoning.effort` of `minimal` +- **WHEN** the request is forwarded +- **THEN** the effort is rewritten, because the WebSocket transport never + reaches a model source + +#### Scenario: An enforced effort is not resurrected by restoration + +- **GIVEN** an API key that enforces a reasoning effort +- **AND** a request for a source model that declares the client's original effort +- **WHEN** the request is routed to the source +- **THEN** the source receives the enforced effort diff --git a/openspec/changes/source-model-reasoning-metadata/tasks.md b/openspec/changes/source-model-reasoning-metadata/tasks.md new file mode 100644 index 0000000000..b54332b000 --- /dev/null +++ b/openspec/changes/source-model-reasoning-metadata/tasks.md @@ -0,0 +1,43 @@ +## 1. Catalog metadata + +- [x] 1.1 Derive source-model reasoning levels, default level, and summary + support from `raw_metadata_json`. +- [x] 1.2 Restrict the declared default to one of the advertised efforts. + +## 2. Effort delivery + +- [x] 2.1 Apply the unsupported-effort rewrite unconditionally at enforcement + time and restore it at the source-routing branch, gated on the effort + being declared for that source model. Route membership is not inferred + from the model registry. +- [x] 2.2 Report the replaced effort from the normalizer and thread it through + enforcement. Paths whose enforced Responses payload only ever reaches a + subscription (WebSocket, stream, collect, compact, and chat -- whose own + source branch forwards the untouched original chat payload) discard it, so + the workaround still applies there. +- [x] 2.3 Report only fallback rewrites, so the `ultra` -> `max` wire alias + survives source routing, and restore the normalized effort form. +- [x] 2.4 Normalize and deduplicate declared efforts, validating shape rather + than membership of a fixed vocabulary, so operator-declared `none` and + other provider-specific efforts survive. +- [x] 2.5 Gate catalog derivation, the declared-levels accessor and the + chat-path opt-in on the `supports_reasoning` switch, so the Codex + catalog, `/v1/models`, the chat sanitizer and the restore agree. + +## 3. Verification + +- [x] 3.1 Unit coverage for slug lists, object lists, malformed entries, an + out-of-range default, and the no-metadata default. +- [x] 3.2 Manual end-to-end check that `/backend-api/codex/models` advertises the + declared efforts and that forwarding behavior is unchanged. +- [x] 3.3 Unit coverage for the restore matrix (declared, undeclared, enforced), + the never-restored `ultra` alias, and the normalized restored form. +- [x] 3.4 Integration coverage that a source declaring `minimal` receives it, + via both `/v1/responses` and the codex-native `/backend-api/codex/responses` + route. Mutation-checked per call site: dropping the restore call, or the + threading at either route, fails the corresponding test. One test per route + is required -- the codex-native threading is invisible to the `/v1` test. +- [ ] 3.5 The WebSocket scenario is verified by inspection only: the WebSocket + service tree contains no model-source references, so there is no restore to + suppress. Left unchecked rather than claimed as tested. + diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index 0405933703..f90e133527 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -2647,3 +2647,94 @@ async def stream_handler(request: web.Request) -> web.StreamResponse: assert b'"content":"hello"' in received assert b"[DONE]" in received + + +@pytest.mark.asyncio +async def test_source_responses_payload_restores_declared_minimal_effort(async_client, source_upstream): + """The minimal rewrite must be undone for a source that declared the effort. + + This pins the wiring, not just the helper: the restore lives inside + _source_responses_response, and both the call and the threading of the + replaced effort through enforcement have to survive for the source to see + ``minimal`` instead of the ``low`` fallback. + """ + captured: dict[str, object] = {} + + async def capture(request: web.Request) -> web.Response: + captured.update(await request.json()) + return web.json_response({"id": "resp_source_reasoning", "status": "completed", "output": []}) + + base_url = await source_upstream(capture) + model = "reasoning-levels-model" + await _create_model_source( + async_client, + name="reasoning-levels", + model=model, + base_url=base_url, + supports_responses=True, + raw_metadata_json='{"supports_reasoning": true, "supported_reasoning_levels": ["minimal", "low", "high"]}', + ) + + response = await async_client.post( + "/v1/responses", + json={ + "model": model, + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "reasoning": {"effort": "minimal"}, + }, + ) + + assert response.status_code == 200 + reasoning = captured["reasoning"] + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "minimal" + + +@pytest.mark.asyncio +async def test_codex_responses_payload_restores_declared_minimal_effort(async_client, source_upstream): + """The codex-native route must thread the replaced effort too. + + Codex CLI talks to this route, and ``--reasoning-effort minimal`` is where + the rewrite originates, so this call site matters more than the /v1 one. + It forces streaming for source-routed requests, hence the SSE upstream. + """ + captured: dict[str, object] = {} + frames = b'data: {"type":"response.completed","response":{"id":"resp_codex","status":"completed"}}\n\n' + + async def capture(request: web.Request) -> web.StreamResponse: + captured.update(await request.json()) + response = web.StreamResponse(status=200, headers={"Content-Type": "text/event-stream"}) + await response.prepare(request) + await response.write(frames) + await response.write_eof() + return response + + base_url = await source_upstream(capture) + model = "codex-reasoning-levels-model" + await _create_model_source( + async_client, + name="codex-reasoning-levels", + model=model, + base_url=base_url, + supports_responses=True, + raw_metadata_json='{"supports_reasoning": true, "supported_reasoning_levels": ["minimal", "low", "high"]}', + ) + + async with async_client.stream( + "POST", + "/backend-api/codex/responses", + json={ + "model": model, + "instructions": "hi", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "hi"}]}], + "stream": True, + "reasoning": {"effort": "minimal"}, + }, + ) as response: + assert response.status_code == 200 + async for _ in response.aiter_bytes(): + pass + + reasoning = captured["reasoning"] + assert isinstance(reasoning, dict) + assert reasoning["effort"] == "minimal" diff --git a/tests/integration/test_openai_compat_features.py b/tests/integration/test_openai_compat_features.py index c3c44dd615..586c1b5ae9 100644 --- a/tests/integration/test_openai_compat_features.py +++ b/tests/integration/test_openai_compat_features.py @@ -299,7 +299,9 @@ async def test_v1_responses_preserves_explicit_prompt_cache_for_model_source(asy async def fake_select(model, api_key, *, raw_model=None, require_streaming=False): return source, model - async def fake_source_response(request, payload, *, source, api_key, rate_limit_headers): + async def fake_source_response( + request, payload, *, source, api_key, rate_limit_headers, pre_normalization_effort=None + ): seen["payload"] = payload.model_dump_for_forwarding() return JSONResponse({"id": "resp_prompt_cache_source", "status": "completed", "output": []}) diff --git a/tests/integration/test_v1_models.py b/tests/integration/test_v1_models.py index 7d268f8cb1..be1d673a62 100644 --- a/tests/integration/test_v1_models.py +++ b/tests/integration/test_v1_models.py @@ -798,6 +798,37 @@ async def test_backend_codex_models_unions_service_tiers_across_accounts(async_c assert "fast" in (model.get("additional_speed_tiers") or []) +@pytest.mark.asyncio +async def test_backend_codex_models_filters_unknown_reasoning_efforts(async_client): + registry = get_model_registry() + model = replace( + _make_upstream_model( + "source-gpt", + raw={ + "shell_type": "shell_command", + "visibility": "list", + }, + ), + supported_reasoning_levels=tuple( + ReasoningLevel(effort=effort, description=effort) + for effort in ("none", "high", "provider-specific", "ultra") + ), + default_reasoning_level="provider-specific", + ) + await registry.update({"plus": [model], "pro": [model]}) + + resp = await async_client.get("/backend-api/codex/models") + + assert resp.status_code == 200 + entry = next(m for m in resp.json()["models"] if m["slug"] == "source-gpt") + assert [level["effort"] for level in entry["supported_reasoning_levels"]] == [ + "none", + "high", + "ultra", + ] + assert entry["default_reasoning_level"] is None + + @pytest.mark.asyncio async def test_backend_codex_models_does_not_reunion_stale_global_service_tiers(async_client): registry = get_model_registry() diff --git a/tests/unit/test_model_sources_catalog.py b/tests/unit/test_model_sources_catalog.py index fb25a3c94d..db7e63a2b8 100644 --- a/tests/unit/test_model_sources_catalog.py +++ b/tests/unit/test_model_sources_catalog.py @@ -9,8 +9,10 @@ from app.modules.model_sources.catalog import ( DEFAULT_SOURCE_CONTEXT_WINDOW, source_model_audio_cost_usd, + source_model_reasoning_levels, source_model_request_overrides, source_model_supported_tool_types, + source_model_supports_reasoning, source_models_to_upstream_models, ) @@ -220,3 +222,207 @@ def test_source_models_force_codex_lb_provider_metadata() -> None: assert len(models) == 1 assert models[0].raw["model_provider"] == "codex-lb" + + +def _reasoning_source(raw_metadata_json: str | None) -> ModelSource: + return ModelSource( + id="src_reasoning", + name="Reasoning", + kind=MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + base_url="http://127.0.0.1:8000/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=True, + supports_audio_transcriptions=False, + models=[ + ModelSourceModel( + model="reasoning-model", + is_enabled=True, + supports_streaming=True, + raw_metadata_json=raw_metadata_json, + ) + ], + ) + + +def test_source_model_without_metadata_advertises_no_reasoning_levels() -> None: + [model] = source_models_to_upstream_models([_reasoning_source(None)]) + assert model.supported_reasoning_levels == () + assert model.default_reasoning_level is None + assert model.supports_reasoning_summaries is False + + +def test_source_model_reasoning_levels_accept_effort_slugs() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["low", "medium", "high", "xhigh"], + "default_reasoning_level": "high", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == [ + "low", + "medium", + "high", + "xhigh", + ] + assert model.default_reasoning_level == "high" + + +def test_source_model_reasoning_levels_accept_objects_and_summaries() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": [ + {"effort": "low", "description": "Low effort"}, + {"effort": "max", "description": "Max effort"}, + ], + "default_reasoning_level": "max", + "supports_reasoning_summaries": True, + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [(level.effort, level.description) for level in model.supported_reasoning_levels] == [ + ("low", "Low effort"), + ("max", "Max effort"), + ] + assert model.default_reasoning_level == "max" + assert model.supports_reasoning_summaries is True + + +def test_source_model_reasoning_levels_ignore_invalid_entries_and_defaults() -> None: + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["low", "low", {"description": "no effort key"}, 7, {"effort": "high"}], + # Not one of the advertised efforts, so it must not be surfaced. + "default_reasoning_level": "ultra", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == ["low", "high"] + assert model.default_reasoning_level is None + + +def test_source_model_reasoning_levels_ignore_non_list_metadata() -> None: + raw = json.dumps({"supports_reasoning": True, "supported_reasoning_levels": "high"}) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert model.supported_reasoning_levels == () + + +def test_source_model_reasoning_levels_are_normalized_and_deduplicated() -> None: + """Efforts are normalized and deduplicated, but not filtered by vocabulary. + + Backends disagree on which efforts exist, so an effort this proxy has + never heard of is still the operator's to declare; only shape is checked. + """ + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": [" Low ", "HIGH", " ", "low", "provider-specific"], + "default_reasoning_level": " HIGH ", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == [ + "low", + "high", + "provider-specific", + ] + assert model.default_reasoning_level == "high" + + +def test_source_model_can_declare_none_as_a_reasoning_level() -> None: + """``none`` is a real effort on GLM and Model Studio (see #1660). + + It is also already first-class for API-key enforced efforts, so dropping + it from source catalogs would have made the two vocabularies disagree. + """ + raw = json.dumps( + { + "supports_reasoning": True, + "supported_reasoning_levels": ["none", "high", "max"], + "default_reasoning_level": "none", + } + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert [level.effort for level in model.supported_reasoning_levels] == ["none", "high", "max"] + assert model.default_reasoning_level == "none" + + +def test_source_model_default_level_outside_declared_set_is_dropped() -> None: + raw = json.dumps( + {"supports_reasoning": True, "supported_reasoning_levels": ["low"], "default_reasoning_level": "max"} + ) + [model] = source_models_to_upstream_models([_reasoning_source(raw)]) + assert model.default_reasoning_level is None + + +def test_declared_levels_do_not_imply_the_reasoning_opt_in() -> None: + """Levels describe *which* efforts an opted-in backend takes, not whether + reasoning is allowed. The dashboard's Reasoning switch is the only opt-in, + and the catalog derivation is gated on it too, so a model with the switch + off advertises nothing rather than advertising an inert capability.""" + raw = json.dumps({"supported_reasoning_levels": ["low", "high"]}) + source = _reasoning_source(raw) + assert source_model_supports_reasoning(source, "reasoning-model") is False + [model] = source_models_to_upstream_models([source]) + assert model.supported_reasoning_levels == () + assert model.default_reasoning_level is None + assert source_model_reasoning_levels(source, "reasoning-model") == () + + +def test_declared_summaries_do_not_imply_the_reasoning_opt_in() -> None: + """Summary support is gated by the same switch, for the same reason.""" + summaries_only = _reasoning_source(json.dumps({"supports_reasoning_summaries": True})) + assert source_model_supports_reasoning(summaries_only, "reasoning-model") is False + [model] = source_models_to_upstream_models([summaries_only]) + assert model.supports_reasoning_summaries is False + + +def test_the_reasoning_switch_gates_every_surface() -> None: + """The Codex catalog, the chat gate and the restore must never disagree. + + They are read by different call sites, so this pins them together: with + the switch off nothing is advertised or restorable, with it on the + operator's declared levels reach all three. + """ + declared = {"supported_reasoning_levels": ["low", "high"], "supports_reasoning_summaries": True} + off = _reasoning_source(json.dumps(declared)) + on = _reasoning_source(json.dumps({"supports_reasoning": True, **declared})) + + [off_model] = source_models_to_upstream_models([off]) + assert off_model.supported_reasoning_levels == () + assert off_model.supports_reasoning_summaries is False + assert source_model_supports_reasoning(off, "reasoning-model") is False + assert source_model_reasoning_levels(off, "reasoning-model") == () + + [on_model] = source_models_to_upstream_models([on]) + assert [level.effort for level in on_model.supported_reasoning_levels] == ["low", "high"] + assert on_model.supports_reasoning_summaries is True + assert source_model_supports_reasoning(on, "reasoning-model") is True + assert [level.effort for level in source_model_reasoning_levels(on, "reasoning-model")] == ["low", "high"] + + +def test_no_declared_levels_keeps_the_explicit_reasoning_opt_in() -> None: + assert source_model_supports_reasoning(_reasoning_source(None), "reasoning-model") is False + explicit = _reasoning_source(json.dumps({"supports_reasoning": True})) + assert source_model_supports_reasoning(explicit, "reasoning-model") is True + + +def test_source_model_reasoning_levels_accessor_matches_the_catalog() -> None: + raw = json.dumps({"supports_reasoning": True, "supported_reasoning_levels": ["minimal", "low"]}) + source = _reasoning_source(raw) + assert [level.effort for level in source_model_reasoning_levels(source, "reasoning-model")] == [ + "minimal", + "low", + ] + assert source_model_reasoning_levels(source, "unknown-model") == () + + +def test_declared_summaries_imply_the_chat_path_reasoning_opt_in() -> None: + """``supports_reasoning_summaries`` is surfaced as ``supports_reasoning`` on + /v1/models, so declaring it alone must not leave the chat path stripping.""" + summaries_only = _reasoning_source(json.dumps({"supports_reasoning": True, "supports_reasoning_summaries": True})) + assert source_model_supports_reasoning(summaries_only, "reasoning-model") is True diff --git a/tests/unit/test_proxy_api_websocket_auth.py b/tests/unit/test_proxy_api_websocket_auth.py index 57976701e8..13517982fd 100644 --- a/tests/unit/test_proxy_api_websocket_auth.py +++ b/tests/unit/test_proxy_api_websocket_auth.py @@ -15,6 +15,7 @@ import app.core.auth.dependencies as auth_dependencies import app.core.request_locality as request_locality import app.modules.proxy.api as proxy_api_module +import app.modules.proxy.request_policy as proxy_request_policy from app.core.clients.proxy import ProxyResponseError from app.core.errors import openai_error from app.core.exceptions import ProxyAuthError @@ -393,7 +394,7 @@ async def test_stream_responses_prefers_forwarded_downstream_turn_state(monkeypa def fake_apply_api_key_enforcement(_payload, _api_key, *, prohibit_fast_mode=False): assert prohibit_fast_mode is False - return None + return proxy_request_policy.ApiKeyEnforcementResult(False, None) def fake_validate_model_access(_api_key, _model): return None @@ -557,7 +558,7 @@ async def test_stream_responses_does_not_release_forwarded_reservation_on_intern def fake_apply_api_key_enforcement(_payload, _api_key, *, prohibit_fast_mode=False): assert prohibit_fast_mode is False - return None + return proxy_request_policy.ApiKeyEnforcementResult(False, None) def fake_validate_model_access(_api_key, _model): return None diff --git a/tests/unit/test_proxy_load_balancer_refresh.py b/tests/unit/test_proxy_load_balancer_refresh.py index 365c3181af..f82d9b5fa8 100644 --- a/tests/unit/test_proxy_load_balancer_refresh.py +++ b/tests/unit/test_proxy_load_balancer_refresh.py @@ -4147,7 +4147,7 @@ def test_enforced_service_tier_provenance_treats_default_aliases_as_omitted( service_tier_was_enforced = apply_api_key_enforcement( payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert service_tier_was_enforced is True assert payload.service_tier == "priority" @@ -4186,7 +4186,7 @@ async def test_select_account_ignores_enforced_service_tier_the_model_never_adve service_tier_was_enforced = apply_api_key_enforcement( payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert service_tier_was_enforced is True assert apply_enforced_service_tier_model_fallback( payload, @@ -4215,7 +4215,7 @@ async def test_select_account_ignores_enforced_service_tier_the_model_never_adve explicitly_requested = apply_api_key_enforcement( explicit_payload, _service_tier_enforcement_key("priority"), - ) + ).service_tier_was_enforced assert explicitly_requested is False assert not apply_enforced_service_tier_model_fallback( explicit_payload, @@ -4314,7 +4314,7 @@ async def test_api_key_enforced_priority_tier_still_routes_a_model_without_prior last_used_at=None, ) payload = ResponsesRequest(model=model, instructions="ping", input=[]) - service_tier_was_enforced = apply_api_key_enforcement(payload, api_key) + service_tier_was_enforced = apply_api_key_enforcement(payload, api_key).service_tier_was_enforced assert payload.service_tier == "priority" assert service_tier_was_enforced is True assert apply_enforced_service_tier_model_fallback( diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index fd1779be02..6d5207a616 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -2093,6 +2093,200 @@ def test_normalize_unsupported_reasoning_effort_rewrites_minimal_to_low(caplog): assert any("reasoning_effort_normalized" in record.message for record in caplog.records) +def test_normalize_unsupported_reasoning_effort_rewrites_a_model_absent_from_the_snapshot(): + """Snapshot membership must not decide whether the workaround applies. + + A populated snapshot can omit a genuine subscription model -- a partial + refresh, an account unavailable during refresh, or an operator-mapped slug + outside the bootstrap set -- and those requests still reach the ChatGPT + backend through the unfiltered fallback. Skipping the rewrite for them + would restore the no-completion hang it exists to prevent, so the rewrite + is unconditional here and only undone once a source is actually selected. + """ + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate( + { + "model": "qwen3.8-max", + "instructions": "hello", + "input": [], + } + ) + payload.reasoning = ResponsesReasoning(effort="minimal") + # Snapshot is populated, but only with an unrelated subscription model. + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None + assert payload.reasoning.effort == "low" + assert replaced == "minimal", "the replaced effort must be reported so a source route can restore it" + + +def test_normalize_unsupported_reasoning_effort_still_rewrites_known_subscription_model(): + """The workaround must stay in place for models the registry does know.""" + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.5", + "instructions": "hello", + "input": [], + } + ) + payload.reasoning = ResponsesReasoning(effort="minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None + assert payload.reasoning.effort == "low" + + +def _reasoning_model_source(levels: list[str]) -> "ModelSource": + import json as _json + + from app.core.openai.model_registry import MODEL_SOURCE_KIND_OPENAI_COMPATIBLE + from app.db.models import ModelSource, ModelSourceModel + + return ModelSource( + id="src_restore", + name="Restore", + kind=MODEL_SOURCE_KIND_OPENAI_COMPATIBLE, + base_url="http://127.0.0.1:8000/v1", + is_enabled=True, + supports_chat_completions=True, + supports_responses=True, + supports_audio_transcriptions=False, + models=[ + ModelSourceModel( + model="qwen3.8-max", + is_enabled=True, + supports_streaming=True, + raw_metadata_json=_json.dumps({"supports_reasoning": True, "supported_reasoning_levels": levels}), + ) + ], + ) + + +def _payload_with_effort(model: str, effort: str): + from app.core.openai.requests import ResponsesReasoning + + payload = ResponsesRequest.model_validate({"model": model, "instructions": "hello", "input": []}) + payload.reasoning = ResponsesReasoning(effort=effort) + return payload + + +def test_restore_source_reasoning_effort_undoes_the_rewrite_for_a_declared_effort(): + """The workaround targets a ChatGPT backend quirk that model sources do not + have, so a source that declared the effort must receive it unchanged.""" + payload = _payload_with_effort("qwen3.8-max", "minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + assert payload.reasoning is not None and payload.reasoning.effort == "low" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "minimal" + + +def test_restore_source_reasoning_effort_skips_an_undeclared_effort(): + """Sources without the effort in their declared set keep the safe value, so + a source that never advertised ``minimal`` is not sent it.""" + payload = _payload_with_effort("qwen3.8-max", "minimal") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning is not None and payload.reasoning.effort == "low" + + +def test_ultra_alias_is_never_restored_for_a_source(): + """The ultra -> max alias must hold on every upstream surface. + + An existing requirement makes the proxy forward ``ultra`` as ``max`` on any + outbound Responses payload, with no source carve-out, and real Codex clients + already rewrite it client-side. So the normalizer must not report the alias + as restorable, even for a source that declares ``ultra``. + """ + payload = _payload_with_effort("qwen3.8-max", "ultra") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + + assert payload.reasoning is not None and payload.reasoning.effort == "max" + assert replaced is None, "the wire alias must not be reported as restorable" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["ultra", "max", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "max" + + +def test_restore_source_reasoning_effort_uses_the_normalized_effort(): + """The restored value must be normalized, not the raw client string. + + Pre-PR a source would have received the normalized rewrite, so forwarding + ``" MINIMAL "`` verbatim would be a new behaviour with no operator opt-in. + """ + payload = _payload_with_effort("qwen3.8-max", " MINIMAL ") + registry = _build_registry_with_model("gpt-5.5", ["low", "medium", "high", "xhigh"]) + + replaced = proxy_request_policy.normalize_unsupported_reasoning_effort(payload, registry=registry) + assert replaced == "minimal" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning is not None and payload.reasoning.effort == "minimal" + + +def test_restore_source_reasoning_effort_cannot_resurrect_an_enforced_effort(): + """The captured value is post-enforcement, so an API key that pinned an + effort still wins after the restore.""" + from app.core.openai.requests import ResponsesReasoning + + payload = _payload_with_effort("qwen3.8-max", "minimal") + api_key = proxy_service.ApiKeyData( + id="key_effort", + name="effort-enforcement-key", + key_prefix="sk-clb-test", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort="high", + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + + replaced = proxy_request_policy.apply_api_key_enforcement(payload, api_key).pre_normalization_reasoning_effort + + assert payload.reasoning is not None and payload.reasoning.effort == "high" + assert replaced is None, "nothing was rewritten, so there is nothing to restore" + + proxy_request_policy.restore_source_reasoning_effort( + payload, + _reasoning_model_source(["minimal", "low", "high"]), + pre_normalization_effort=replaced, + ) + assert payload.reasoning.effort == "high" + assert isinstance(payload.reasoning, ResponsesReasoning) + + def test_normalize_unsupported_reasoning_effort_falls_back_to_low_without_registry(): from app.core.openai.model_registry import ModelRegistry from app.core.openai.requests import ResponsesReasoning From 138aa9f15c6ebea998335afae06475d8834fe7d6 Mon Sep 17 00:00:00 2001 From: BrenticusMaximus <32489248+BrenticusMaximus@users.noreply.github.com> Date: Sun, 16 Aug 2026 14:20:11 -0400 Subject: [PATCH 047/117] feat(ui): customize dashboard request-log columns (#1503) * feat(ui): customize dashboard request-log columns * fix(ui): pin request-log table width and reset stale column layouts Codex review follow-ups on the column-layout feature: - Use the configured column-width sum as the table's explicit width (not merely its minimum) so surplus container space is no longer redistributed across columns; resizing one column now never shifts its siblings, even when few columns are visible. - When stored visibility preferences are stale or malformed, discard the saved widths too so the dashboard restores the complete default layout, matching the openspec recovery scenario. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- .all-contributorsrc | 10 + README.md | 1 + .../components/dashboard-page.test.tsx | 63 ++++- .../dashboard/components/dashboard-page.tsx | 68 ++++- .../components/recent-requests-table.test.tsx | 163 ++++++++++++ .../components/recent-requests-table.tsx | 239 +++++++++++++++--- ...use-request-log-table-preferences.test.tsx | 122 +++++++++ .../use-request-log-table-preferences.ts | 164 ++++++++++++ .../features/dashboard/request-log-columns.ts | 50 ++++ frontend/src/i18n/locales/en.json | 6 + frontend/src/i18n/locales/ko.json | 6 + frontend/src/i18n/locales/zh-CN.json | 6 + .../.openspec.yaml | 2 + .../design.md | 44 ++++ .../proposal.md | 35 +++ .../specs/frontend-architecture/spec.md | 53 ++++ .../tasks.md | 24 ++ 17 files changed, 1013 insertions(+), 43 deletions(-) create mode 100644 frontend/src/features/dashboard/hooks/use-request-log-table-preferences.test.tsx create mode 100644 frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts create mode 100644 frontend/src/features/dashboard/request-log-columns.ts create mode 100644 openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml create mode 100644 openspec/changes/customize-dashboard-request-log-columns/design.md create mode 100644 openspec/changes/customize-dashboard-request-log-columns/proposal.md create mode 100644 openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/customize-dashboard-request-log-columns/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index eac13c51dc..f9b43d2947 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1252,6 +1252,16 @@ "code", "test" ] + }, + { + "login": "BrenticusMaximus", + "name": "BrenticusMaximus", + "avatar_url": "https://avatars.githubusercontent.com/u/32489248?v=4", + "profile": "https://github.com/BrenticusMaximus", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index e6f1100e9b..21f2305e00 100644 --- a/README.md +++ b/README.md @@ -288,6 +288,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e
+
${row.costUsd.toFixed(2)} + {row.activeAccounts} + {row.cancelledCount} + + {row.errorCount} +
DuyBui
DuyBui
💻 ⚠️		 Kevin Lin
Kevin Lin
💻 ⚠️		 Borealin
Borealin
💻 ⚠️		BrenticusMaximus
BrenticusMaximus
💻 ⚠️
diff --git a/frontend/src/features/dashboard/components/dashboard-page.test.tsx b/frontend/src/features/dashboard/components/dashboard-page.test.tsx index 7890243d39..a7a2203b4e 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.test.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.test.tsx @@ -9,17 +9,26 @@ import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDashboard, useDashboardProjections } from "@/features/dashboard/hooks/use-dashboard"; import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; import { useConversations } from "@/features/dashboard/hooks/use-conversations"; +import { REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY } from "@/features/dashboard/hooks/use-request-log-table-preferences"; import { buildDashboardView } from "@/features/dashboard/utils"; -import { useDashboardPreferencesStore } from "@/hooks/use-dashboard-preferences"; import type { AccountListSort } from "@/features/dashboard/components/account-list"; +import type { RecentRequestsTableProps } from "@/features/dashboard/components/recent-requests-table"; +import { useDashboardPreferencesStore } from "@/hooks/use-dashboard-preferences"; import { DashboardPage } from "./dashboard-page"; -const { accountCardsSpy, accountListSpy, accountSummaryLineSpy, conversationsViewSpy } = vi.hoisted(() => ({ +const { + accountCardsSpy, + accountListSpy, + accountSummaryLineSpy, + conversationsViewSpy, + recentRequestsTableSpy, +} = vi.hoisted(() => ({ accountCardsSpy: vi.fn(), accountListSpy: vi.fn(), accountSummaryLineSpy: vi.fn(), conversationsViewSpy: vi.fn(), + recentRequestsTableSpy: vi.fn(), })); vi.mock("@/features/accounts/hooks/use-accounts", () => ({ @@ -119,7 +128,10 @@ vi.mock("@/features/dashboard/components/filters/request-filters", async () => { }); vi.mock("@/features/dashboard/components/recent-requests-table", () => ({ - RecentRequestsTable: () =>
, + RecentRequestsTable: (props: RecentRequestsTableProps) => { + recentRequestsTableSpy(props); + return
; + }, })); vi.mock("@/features/dashboard/components/stats-grid", () => ({ @@ -166,12 +178,14 @@ describe("DashboardPage", () => { accountListSpy.mockReset(); accountSummaryLineSpy.mockReset(); conversationsViewSpy.mockReset(); + recentRequestsTableSpy.mockReset(); useAccountMutationsMock.mockReset(); useDashboardMock.mockReset(); useDashboardProjectionsMock.mockReset(); useRequestLogsMock.mockReset(); useConversationsMock.mockReset(); buildDashboardViewMock.mockReset(); + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); useDashboardPreferencesStore.setState({ accountBurnrateEnabled: true, accountViewMode: "cards", @@ -485,6 +499,49 @@ describe("DashboardPage", () => { expect(useConversationsMock.mock.calls.every(([options]) => options !== undefined && options.enabled === false)).toBe(true); }); + it("customizes and restores the request-log table without a global width control", async () => { + const user = userEvent.setup(); + mockReadyDashboard(); + + renderWithProviders(); + + expect(screen.getByRole("button", { name: "Columns (12)" })).toBeInTheDocument(); + expect(screen.queryByRole("slider")).not.toBeInTheDocument(); + expect(screen.queryByText(/^Width$/)).not.toBeInTheDocument(); + + await user.click(screen.getByRole("button", { name: "Columns (12)" })); + expect(screen.getByText("Visible columns")).toBeInTheDocument(); + await user.click(screen.getByRole("menuitemcheckbox", { name: "Plan" })); + + expect(screen.getByRole("menu", { name: "Columns (11)" })).toBeInTheDocument(); + const selectedProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(selectedProps?.visibleColumns).not.toContain("plan"); + + act(() => { + selectedProps?.onColumnWidthChange?.("account", 240); + }); + const resizedProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(resizedProps?.columnWidths?.account).toBe(240); + + await user.keyboard("{Escape}"); + await user.click( + screen.getByRole("button", { name: "Restore default column layout" }), + ); + + const restoredProps = recentRequestsTableSpy.mock.lastCall?.[0] as + | RecentRequestsTableProps + | undefined; + expect(restoredProps?.visibleColumns).toHaveLength(12); + expect(restoredProps?.columnWidths).toEqual({}); + expect( + window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY), + ).toBeNull(); + }); + it("renders the account summary line in the Accounts header using overview accounts", () => { const overview = mockReadyDashboard(); diff --git a/frontend/src/features/dashboard/components/dashboard-page.tsx b/frontend/src/features/dashboard/components/dashboard-page.tsx index 6f28be006f..fbdb5ecf48 100644 --- a/frontend/src/features/dashboard/components/dashboard-page.tsx +++ b/frontend/src/features/dashboard/components/dashboard-page.tsx @@ -2,10 +2,18 @@ import { useCallback, useEffect, useMemo } from "react"; import { Trans, useTranslation } from "react-i18next"; import { useNavigate, useSearchParams } from "react-router-dom"; import { useQueryClient } from "@tanstack/react-query"; -import { RefreshCw } from "lucide-react"; +import { Columns3, RefreshCw, RotateCcw } from "lucide-react"; import { AlertMessage } from "@/components/alert-message"; import { Button } from "@/components/ui/button"; +import { + DropdownMenu, + DropdownMenuCheckboxItem, + DropdownMenuContent, + DropdownMenuLabel, + DropdownMenuSeparator, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { SpinnerBlock } from "@/components/ui/spinner"; import { useDialogState } from "@/hooks/use-dialog-state"; import { useAccountMutations } from "@/features/accounts/hooks/use-accounts"; @@ -27,7 +35,9 @@ import { WeeklyCreditsPaceCard } from "@/features/dashboard/components/weekly-cr import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDashboard, useDashboardProjections } from "@/features/dashboard/hooks/use-dashboard"; import { useConversations } from "@/features/dashboard/hooks/use-conversations"; +import { useRequestLogTablePreferences } from "@/features/dashboard/hooks/use-request-log-table-preferences"; import { useRequestLogs } from "@/features/dashboard/hooks/use-request-logs"; +import { REQUEST_LOG_COLUMN_OPTIONS } from "@/features/dashboard/request-log-columns"; import { buildDashboardView } from "@/features/dashboard/utils"; import { DEFAULT_OVERVIEW_TIMEFRAME, @@ -48,6 +58,13 @@ const MODEL_OPTION_DELIMITER = ":::"; export function DashboardPage() { const { t, i18n } = useTranslation(); + const { + visibleColumns, + columnWidths, + toggleColumn, + setColumnWidth, + restoreDefaultLayout, + } = useRequestLogTablePreferences(); const resolvedLanguage = i18n.resolvedLanguage; const navigate = useNavigate(); const [searchParams, setSearchParams] = useSearchParams(); @@ -439,7 +456,51 @@ export function DashboardPage() { onChange={handleDashboardViewChange} showConversations={isAdmin} /> -
+
+ {dashboardView === "request-logs" ? ( + <> + + + + + + + {t("dashboard.requests.columnLayout.visibleColumns")} + + + {REQUEST_LOG_COLUMN_OPTIONS.map((column) => { + const isVisible = visibleColumns.includes(column.id); + return ( + toggleColumn(column.id)} + onSelect={(event) => event.preventDefault()} + > + {t(column.translationKey)} + + ); + })} + + + + + ) : null}
{isAdmin && dashboardView === "conversations" ? : logsQuery.isPending && !logPage ? (
@@ -502,6 +563,9 @@ export function DashboardPage() { requests={view.requestLogs} accounts={overview?.accounts ?? []} total={logPage?.total ?? 0} + visibleColumns={visibleColumns} + columnWidths={columnWidths} + onColumnWidthChange={setColumnWidth} limit={filters.limit} offset={filters.offset} hasMore={logPage?.hasMore ?? false} diff --git a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx index 2190acb1ce..88bd0f8083 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx @@ -3,6 +3,13 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { RecentRequestsTable } from "@/features/dashboard/components/recent-requests-table"; +import { + ALL_REQUEST_LOG_COLUMNS, + MAX_REQUEST_LOG_COLUMN_WIDTH, + MIN_REQUEST_LOG_COLUMN_WIDTH, + REQUEST_LOG_COLUMN_WIDTH_STEP, +} from "@/features/dashboard/request-log-columns"; +import type { RequestLog } from "@/features/dashboard/schemas"; const ISO = "2026-01-01T12:00:00+00:00"; const NULL_FAILURE_METADATA = { @@ -48,6 +55,41 @@ const PAGINATION_PROPS = { onOffsetChange: vi.fn(), }; +const LAYOUT_REQUEST = { + requestedAt: ISO, + accountId: "acc-layout", + planType: "plus", + apiKeyName: "Layout Key", + apiKeyId: "key-layout", + requestId: "req-layout", + conversationId: null, + requestKind: "normal", + model: "gpt-5.1", + source: null, + serviceTier: null, + requestedServiceTier: null, + actualServiceTier: null, + transport: "http", + upstreamTransport: "http", + status: "ok", + errorCode: null, + errorMessage: null, + ...NULL_FAILURE_METADATA, + ...NULL_USERAGENT_METADATA, + tokens: 1200, + inputTokens: 1000, + outputTokens: 200, + outputTokensRaw: 200, + reasoningTokens: 0, + latencyFirstTokenMs: 200, + latencyQueueMs: null, + cachedInputTokens: 0, + reasoningEffort: null, + costUsd: 0.01, + costBreakdown: null, + latencyMs: 1000, +} satisfies RequestLog; + function openRequestDetails() { fireEvent.click(screen.getByRole("button", { name: "View Details" })); return screen.getByRole("dialog"); @@ -74,6 +116,127 @@ describe("RecentRequestsTable", () => { } }); + it("renders every existing column when layout props are omitted", () => { + render( + , + ); + + expect(screen.getAllByRole("columnheader")).toHaveLength(ALL_REQUEST_LOG_COLUMNS.length); + expect(screen.getByText("Layout Key")).toBeInTheDocument(); + expect(screen.getByRole("button", { name: "View Details" })).toBeInTheDocument(); + }); + + it("renders only selected headers and matching row cells", () => { + render( + , + ); + + expect(screen.getAllByRole("columnheader")).toHaveLength(2); + expect(screen.getByRole("columnheader", { name: "Time" })).toBeInTheDocument(); + expect(screen.getByRole("columnheader", { name: "Model" })).toBeInTheDocument(); + expect(screen.queryByRole("columnheader", { name: "API Key" })).not.toBeInTheDocument(); + expect(screen.queryByText("Layout Key")).not.toBeInTheDocument(); + expect(screen.getByText("gpt-5.1")).toBeInTheDocument(); + }); + + it("resizes only the selected column by pointer and clamps it to bounds", () => { + const onColumnWidthChange = vi.fn(); + render( + , + ); + + const accountSeparator = screen.getByRole("separator", { + name: "Resize Account column", + }); + fireEvent.pointerDown(accountSeparator, { pointerId: 7, clientX: 100 }); + fireEvent.pointerMove(accountSeparator, { pointerId: 7, clientX: 164 }); + fireEvent.pointerUp(accountSeparator, { pointerId: 7, clientX: 164 }); + + expect(onColumnWidthChange).toHaveBeenCalledWith("account", 224); + expect(onColumnWidthChange).not.toHaveBeenCalledWith("time", expect.any(Number)); + + onColumnWidthChange.mockClear(); + fireEvent.pointerDown(accountSeparator, { pointerId: 8, clientX: 100 }); + fireEvent.pointerMove(accountSeparator, { pointerId: 8, clientX: 10_000 }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "account", + MAX_REQUEST_LOG_COLUMN_WIDTH, + ); + }); + + it("resizes with arrow keys within bounds and sums visible widths", () => { + const onColumnWidthChange = vi.fn(); + render( + , + ); + + expect(screen.getByRole("table")).toHaveStyle({ + width: `${MIN_REQUEST_LOG_COLUMN_WIDTH + 200}px`, + minWidth: `${MIN_REQUEST_LOG_COLUMN_WIDTH + 200}px`, + }); + + const timeSeparator = screen.getByRole("separator", { + name: "Resize Time column", + }); + fireEvent.keyDown(timeSeparator, { key: "ArrowLeft" }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "time", + MIN_REQUEST_LOG_COLUMN_WIDTH, + ); + + const accountSeparator = screen.getByRole("separator", { + name: "Resize Account column", + }); + fireEvent.keyDown(accountSeparator, { key: "ArrowRight" }); + expect(onColumnWidthChange).toHaveBeenLastCalledWith( + "account", + 200 + REQUEST_LOG_COLUMN_WIDTH_STEP, + ); + }); + + it("pins the table to the configured width sum so surplus space is not redistributed", () => { + render( + , + ); + + // An explicit width (not merely a minimum) keeps configured column widths + // independent when their sum is smaller than the container. + expect(screen.getByRole("table")).toHaveStyle({ + width: "272px", + minWidth: "272px", + }); + }); + it("renders rows with status badges and supports request details and copy actions", async () => { const longError = "Rate limit reached while processing this request ".repeat(3); const writeText = vi.fn().mockResolvedValue(undefined); diff --git a/frontend/src/features/dashboard/components/recent-requests-table.tsx b/frontend/src/features/dashboard/components/recent-requests-table.tsx index 8965abe220..6c54bb04f7 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.tsx @@ -1,5 +1,11 @@ import { Inbox } from "lucide-react"; -import { useMemo, useState } from "react"; +import { + useMemo, + useRef, + useState, + type KeyboardEvent, + type PointerEvent, +} from "react"; import { useTranslation } from "react-i18next"; import { isEmailLabel } from "@/components/blur-email"; @@ -26,9 +32,20 @@ import { } from "@/components/ui/table"; import { PaginationControls } from "@/features/dashboard/components/filters/pagination-controls"; import { RequestArchivePanel } from "@/features/conversation-archive/components/request-archive-panel"; +import { + ALL_REQUEST_LOG_COLUMNS, + MAX_REQUEST_LOG_COLUMN_WIDTH, + MIN_REQUEST_LOG_COLUMN_WIDTH, + REQUEST_LOG_COLUMN_DEFAULT_WIDTHS, + REQUEST_LOG_COLUMN_WIDTH_STEP, + clampRequestLogColumnWidth, + type RequestLogColumnId, + type RequestLogColumnWidths, +} from "@/features/dashboard/request-log-columns"; import type { AccountSummary, RequestLog } from "@/features/dashboard/schemas"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; +import { cn } from "@/lib/utils"; import { REQUEST_STATUS_LABELS } from "@/utils/constants"; import { formatDateTimeInline, @@ -84,11 +101,132 @@ export type RecentRequestsTableProps = { offset: number; hasMore: boolean; filtersApplied?: boolean; + visibleColumns?: readonly RequestLogColumnId[]; + columnWidths?: RequestLogColumnWidths; + onColumnWidthChange?: (column: RequestLogColumnId, width: number) => void; onLimitChange: (limit: number) => void; onOffsetChange: (offset: number) => void; onConversationClick?: (conversationId: string) => void; }; +type RequestLogTableHeadProps = { + column: RequestLogColumnId; + label: string; + resizeLabel: string; + className?: string; + width?: number; + onWidthChange?: (column: RequestLogColumnId, width: number) => void; +}; + +function RequestLogTableHead({ + column, + label, + resizeLabel, + className, + width, + onWidthChange, +}: RequestLogTableHeadProps) { + const resizeState = useRef<{ + pointerId: number; + startX: number; + startWidth: number; + } | null>(null); + const resolvedWidth = clampRequestLogColumnWidth( + width ?? REQUEST_LOG_COLUMN_DEFAULT_WIDTHS[column], + ); + + const handlePointerDown = (event: PointerEvent) => { + if (!onWidthChange) { + return; + } + + event.preventDefault(); + const measuredWidth = event.currentTarget.parentElement?.getBoundingClientRect().width; + resizeState.current = { + pointerId: event.pointerId, + startX: event.clientX, + startWidth: measuredWidth && measuredWidth > 0 ? measuredWidth : resolvedWidth, + }; + event.currentTarget.setPointerCapture(event.pointerId); + }; + + const handlePointerMove = (event: PointerEvent) => { + const state = resizeState.current; + if (!state || state.pointerId !== event.pointerId || !onWidthChange) { + return; + } + + onWidthChange( + column, + clampRequestLogColumnWidth(state.startWidth + event.clientX - state.startX), + ); + }; + + const handlePointerEnd = (event: PointerEvent) => { + if (resizeState.current?.pointerId !== event.pointerId) { + return; + } + + resizeState.current = null; + if (event.currentTarget.hasPointerCapture(event.pointerId)) { + event.currentTarget.releasePointerCapture(event.pointerId); + } + }; + + const handleKeyDown = (event: KeyboardEvent) => { + if (!onWidthChange || (event.key !== "ArrowLeft" && event.key !== "ArrowRight")) { + return; + } + + event.preventDefault(); + const direction = event.key === "ArrowLeft" ? -1 : 1; + onWidthChange( + column, + clampRequestLogColumnWidth( + resolvedWidth + direction * REQUEST_LOG_COLUMN_WIDTH_STEP, + ), + ); + }; + + return ( + + {label} + {onWidthChange ? ( +
{ + resizeState.current = null; + }} + onKeyDown={handleKeyDown} + > +
+ ) : null} + + ); +} + function formatRequestCostSummary(request: RequestLog | null, t: ReturnType["t"]): string | null { if (!request || request.status !== "ok") { return null; @@ -173,6 +311,9 @@ export function RecentRequestsTable({ offset, hasMore, filtersApplied = false, + visibleColumns: configuredVisibleColumns, + columnWidths, + onColumnWidthChange, onLimitChange, onOffsetChange, onConversationClick, @@ -183,6 +324,25 @@ export function RecentRequestsTable({ const isAdmin = useAuthStore((state) => state.role === "admin"); const dateDisplayFormat = useDateDisplayFormatStore((state) => state.dateDisplayFormat); const selectedRequestCostSummary = formatRequestCostSummary(selectedRequest, t); + const visibleColumns = configuredVisibleColumns ?? ALL_REQUEST_LOG_COLUMNS; + const visibleColumnSet = useMemo(() => new Set(visibleColumns), [visibleColumns]); + const hasConfiguredLayout = + configuredVisibleColumns !== undefined || + columnWidths !== undefined || + onColumnWidthChange !== undefined; + const tableWidth = hasConfiguredLayout + ? visibleColumns.reduce( + (totalWidth, column) => + totalWidth + + clampRequestLogColumnWidth( + columnWidths?.[column] ?? REQUEST_LOG_COLUMN_DEFAULT_WIDTHS[column], + ), + 0, + ) + : undefined; + const isColumnVisible = (column: RequestLogColumnId) => visibleColumnSet.has(column); + const resizeLabel = (label: string) => + t("dashboard.requests.resizeColumn", { column: label }); const accountLabelMap = useMemo(() => { const index = new Map(); @@ -227,21 +387,24 @@ export function RecentRequestsTable({
- +
- {t("dashboard.requests.columns.time")} - {t("dashboard.requests.columns.account")} - {t("dashboard.requests.columns.plan")} - {t("dashboard.requests.columns.apiKey")} - {t("dashboard.requests.columns.model")} - {t("dashboard.requests.columns.transport")} - {t("dashboard.requests.columns.status")} - TTFT - TPS - {t("dashboard.requests.columns.tokens")} - {t("dashboard.requests.columns.cost")} - {t("dashboard.requests.columns.details")} + {isColumnVisible("time") ? : null} + {isColumnVisible("account") ? : null} + {isColumnVisible("plan") ? : null} + {isColumnVisible("apiKey") ? : null} + {isColumnVisible("model") ? : null} + {isColumnVisible("transport") ? : null} + {isColumnVisible("status") ? : null} + {isColumnVisible("ttft") ? : null} + {isColumnVisible("tps") ? : null} + {isColumnVisible("tokens") ? : null} + {isColumnVisible("cost") ? : null} + {isColumnVisible("details") ? : null} @@ -261,20 +424,20 @@ export function RecentRequestsTable({ return ( - + {isColumnVisible("time") ?
{time.primary}
{time.secondary}
- - + : null} + {isColumnVisible("account") ? {isEmailLabel && blurred ? ( {accountLabel} ) : ( accountLabel )} - - + : null} + {isColumnVisible("plan") ? {planType ? ( -- )} - - + : null} + {isColumnVisible("apiKey") ? {request.apiKeyName || "--"} - - + : null} + {isColumnVisible("model") ?
{formatModelLabel(request.model, request.reasoningEffort, visibleServiceTier)} @@ -305,8 +468,8 @@ export function RecentRequestsTable({
) : null} - - + : null} + {isColumnVisible("transport") ? {request.transport ? (
-- )} - - + : null} + {isColumnVisible("status") ? {t(`dashboard.requestStatus.${request.status}`, { defaultValue: REQUEST_STATUS_LABELS[request.status] ?? request.status })} - - + : null} + {isColumnVisible("ttft") ? {formatCompactElapsed(request.latencyFirstTokenMs) ?? "--"} - - + : null} + {isColumnVisible("tps") ? {generationSpeed ?? "--"} - - + : null} + {isColumnVisible("tokens") ?
{formatCompactNumber(request.tokens)}
{request.cachedInputTokens != null && request.cachedInputTokens > 0 && ( @@ -349,11 +512,11 @@ export function RecentRequestsTable({
)}
- - + : null} + {isColumnVisible("cost") ? {formatCurrency(request.costUsd)} - - + : null} + {isColumnVisible("details") ? {hasError ? (
{request.errorCode ? ( @@ -387,7 +550,7 @@ export function RecentRequestsTable({ {t("dashboard.requests.viewDetails")} )} - + : null} ); })} diff --git a/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.test.tsx b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.test.tsx new file mode 100644 index 0000000000..9cdc50655c --- /dev/null +++ b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.test.tsx @@ -0,0 +1,122 @@ +import { act, renderHook } from "@testing-library/react"; +import { beforeEach, describe, expect, it } from "vitest"; + +import { + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + useRequestLogTablePreferences, +} from "@/features/dashboard/hooks/use-request-log-table-preferences"; +import { + ALL_REQUEST_LOG_COLUMNS, + MAX_REQUEST_LOG_COLUMN_WIDTH, + MIN_REQUEST_LOG_COLUMN_WIDTH, +} from "@/features/dashboard/request-log-columns"; + +describe("useRequestLogTablePreferences", () => { + beforeEach(() => { + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + }); + + it("shows every request-log column by default", () => { + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + }); + + it("persists visible columns and individual widths across remounts", () => { + const { result, unmount } = renderHook(() => useRequestLogTablePreferences()); + + act(() => { + result.current.toggleColumn("plan"); + result.current.setColumnWidth("account", 284); + }); + + expect(result.current.visibleColumns).not.toContain("plan"); + expect(result.current.columnWidths.account).toBe(284); + + unmount(); + const restored = renderHook(() => useRequestLogTablePreferences()); + expect(restored.result.current.visibleColumns).not.toContain("plan"); + expect(restored.result.current.columnWidths.account).toBe(284); + }); + + it("clamps finite widths and ignores malformed width entries", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ + visibleColumns: ["time", "account"], + columnWidths: { + time: 1, + account: "wide", + details: 10_000, + unknown: 200, + }, + }), + ); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.columnWidths).toEqual({ + time: MIN_REQUEST_LOG_COLUMN_WIDTH, + details: MAX_REQUEST_LOG_COLUMN_WIDTH, + }); + }); + + it("keeps the final visible column selected", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ visibleColumns: ["time"], columnWidths: {} }), + ); + const { result } = renderHook(() => useRequestLogTablePreferences()); + + act(() => { + result.current.toggleColumn("time"); + }); + + expect(result.current.visibleColumns).toEqual(["time"]); + }); + + it.each([ + "{not-json", + JSON.stringify({ visibleColumns: [], columnWidths: {} }), + JSON.stringify({ visibleColumns: ["time", "retired-column"], columnWidths: {} }), + ])("falls back safely for malformed or stale preferences", (stored) => { + window.localStorage.setItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, stored); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + }); + + it("restores default widths too when stored visibility is stale", () => { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify({ + visibleColumns: ["time", "retired-column"], + columnWidths: { time: 240, account: 320 }, + }), + ); + + const { result } = renderHook(() => useRequestLogTablePreferences()); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + }); + + it("restores all columns, clears widths, and removes stored customization", () => { + const { result } = renderHook(() => useRequestLogTablePreferences()); + act(() => { + result.current.toggleColumn("plan"); + result.current.setColumnWidth("account", 240); + }); + act(() => { + result.current.restoreDefaultLayout(); + }); + + expect(result.current.visibleColumns).toEqual(ALL_REQUEST_LOG_COLUMNS); + expect(result.current.columnWidths).toEqual({}); + expect( + window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY), + ).toBeNull(); + }); +}); diff --git a/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts new file mode 100644 index 0000000000..b94fbbed82 --- /dev/null +++ b/frontend/src/features/dashboard/hooks/use-request-log-table-preferences.ts @@ -0,0 +1,164 @@ +import { useCallback, useState } from "react"; + +import { + ALL_REQUEST_LOG_COLUMNS, + DEFAULT_REQUEST_LOG_COLUMNS, + clampRequestLogColumnWidth, + type RequestLogColumnId, + type RequestLogColumnWidths, +} from "@/features/dashboard/request-log-columns"; + +export const REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY = + "codex-lb-dashboard-request-log-columns:v1"; + +type RequestLogTablePreferences = { + visibleColumns: RequestLogColumnId[]; + columnWidths: RequestLogColumnWidths; +}; + +const supportedColumns = new Set(ALL_REQUEST_LOG_COLUMNS); + +function isRequestLogColumnId(value: unknown): value is RequestLogColumnId { + return typeof value === "string" && supportedColumns.has(value as RequestLogColumnId); +} + +function normalizeColumnWidths(value: unknown): RequestLogColumnWidths { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return {}; + } + + const stored = value as Record; + const widths: RequestLogColumnWidths = {}; + for (const column of ALL_REQUEST_LOG_COLUMNS) { + const width = stored[column]; + if (typeof width === "number" && Number.isFinite(width)) { + widths[column] = clampRequestLogColumnWidth(width); + } + } + return widths; +} + +function defaultPreferences(): RequestLogTablePreferences { + return { + visibleColumns: [...DEFAULT_REQUEST_LOG_COLUMNS], + columnWidths: {}, + }; +} + +function normalizeVisibleColumns(value: unknown): RequestLogColumnId[] | null { + if ( + !Array.isArray(value) || + value.length === 0 || + value.some((column) => !isRequestLogColumnId(column)) + ) { + return null; + } + + const selected = new Set(value); + return ALL_REQUEST_LOG_COLUMNS.filter((column) => selected.has(column)); +} + +function loadPreferences(): RequestLogTablePreferences { + if (typeof window === "undefined") { + return defaultPreferences(); + } + + try { + const stored = window.localStorage.getItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + if (!stored) { + return defaultPreferences(); + } + + const parsed = JSON.parse(stored) as { + visibleColumns?: unknown; + columnWidths?: unknown; + }; + const visibleColumns = normalizeVisibleColumns(parsed.visibleColumns); + if (visibleColumns === null) { + // Stale or malformed visibility invalidates the whole stored layout so the + // dashboard restores the complete default layout, widths included. + return defaultPreferences(); + } + return { + visibleColumns, + columnWidths: normalizeColumnWidths(parsed.columnWidths), + }; + } catch { + return defaultPreferences(); + } +} + +function persistPreferences(preferences: RequestLogTablePreferences): void { + if (typeof window === "undefined") { + return; + } + + try { + window.localStorage.setItem( + REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY, + JSON.stringify(preferences), + ); + } catch { + // Browser storage may be unavailable in private or locked-down sessions. + } +} + +export function useRequestLogTablePreferences() { + const [preferences, setPreferences] = useState(loadPreferences); + + const toggleColumn = useCallback((column: RequestLogColumnId) => { + setPreferences((current) => { + const isVisible = current.visibleColumns.includes(column); + if (isVisible && current.visibleColumns.length === 1) { + return current; + } + + const selected = new Set(current.visibleColumns); + if (isVisible) { + selected.delete(column); + } else { + selected.add(column); + } + const next = { + ...current, + visibleColumns: ALL_REQUEST_LOG_COLUMNS.filter((candidate) => selected.has(candidate)), + }; + persistPreferences(next); + return next; + }); + }, []); + + const setColumnWidth = useCallback((column: RequestLogColumnId, width: number) => { + setPreferences((current) => { + const next = { + ...current, + columnWidths: { + ...current.columnWidths, + [column]: clampRequestLogColumnWidth(width), + }, + }; + persistPreferences(next); + return next; + }); + }, []); + + const restoreDefaultLayout = useCallback(() => { + const next = defaultPreferences(); + if (typeof window !== "undefined") { + try { + window.localStorage.removeItem(REQUEST_LOG_TABLE_PREFERENCES_STORAGE_KEY); + } catch { + // Browser storage may be unavailable in private or locked-down sessions. + } + } + setPreferences(next); + }, []); + + return { + visibleColumns: preferences.visibleColumns, + columnWidths: preferences.columnWidths, + toggleColumn, + setColumnWidth, + restoreDefaultLayout, + }; +} diff --git a/frontend/src/features/dashboard/request-log-columns.ts b/frontend/src/features/dashboard/request-log-columns.ts new file mode 100644 index 0000000000..db9dc094d2 --- /dev/null +++ b/frontend/src/features/dashboard/request-log-columns.ts @@ -0,0 +1,50 @@ +export const REQUEST_LOG_COLUMN_OPTIONS = [ + { id: "time", translationKey: "dashboard.requests.columns.time" }, + { id: "account", translationKey: "dashboard.requests.columns.account" }, + { id: "plan", translationKey: "dashboard.requests.columns.plan" }, + { id: "apiKey", translationKey: "dashboard.requests.columns.apiKey" }, + { id: "model", translationKey: "dashboard.requests.columns.model" }, + { id: "transport", translationKey: "dashboard.requests.columns.transport" }, + { id: "status", translationKey: "dashboard.requests.columns.status" }, + { id: "ttft", translationKey: "dashboard.requests.columns.ttft" }, + { id: "tps", translationKey: "dashboard.requests.columns.tps" }, + { id: "tokens", translationKey: "dashboard.requests.columns.tokens" }, + { id: "cost", translationKey: "dashboard.requests.columns.cost" }, + { id: "details", translationKey: "dashboard.requests.columns.details" }, +] as const; + +export type RequestLogColumnId = (typeof REQUEST_LOG_COLUMN_OPTIONS)[number]["id"]; + +export const ALL_REQUEST_LOG_COLUMNS: readonly RequestLogColumnId[] = + REQUEST_LOG_COLUMN_OPTIONS.map((column) => column.id); + +export const DEFAULT_REQUEST_LOG_COLUMNS: readonly RequestLogColumnId[] = [ + ...ALL_REQUEST_LOG_COLUMNS, +]; + +export const MIN_REQUEST_LOG_COLUMN_WIDTH = 64; +export const MAX_REQUEST_LOG_COLUMN_WIDTH = 720; +export const REQUEST_LOG_COLUMN_WIDTH_STEP = 8; + +export type RequestLogColumnWidths = Partial>; + +export const REQUEST_LOG_COLUMN_DEFAULT_WIDTHS: Record = { + time: 112, + account: 160, + plan: 96, + apiKey: 144, + model: 180, + transport: 128, + status: 96, + ttft: 80, + tps: 80, + tokens: 96, + cost: 64, + details: 288, +}; + +export function clampRequestLogColumnWidth(width: number): number { + return Math.round( + Math.max(MIN_REQUEST_LOG_COLUMN_WIDTH, Math.min(MAX_REQUEST_LOG_COLUMN_WIDTH, width)), + ); +} diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 7e1d8f96aa..3f29af876a 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -702,12 +702,18 @@ "dashboard.requests.columns.time": "Time", "dashboard.requests.columns.tokens": "Tokens", "dashboard.requests.columns.transport": "Transport", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "Columns ({{count}})", + "dashboard.requests.columnLayout.restoreDefault": "Restore default column layout", + "dashboard.requests.columnLayout.visibleColumns": "Visible columns", "dashboard.requests.downstreamTransport": "Downstream client transport", "dashboard.requests.emptyDescription": "Requests will appear here after clients start using the proxy.", "dashboard.requests.emptyFilteredDescription": "No request logs match the current filters.", "dashboard.requests.emptyFilteredTitle": "No matching requests", "dashboard.requests.emptyTitle": "No requests yet", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.resizeColumn": "Resize {{column}} column", "dashboard.requests.title": "Request Logs", "dashboard.requests.unassigned": "Unassigned", "dashboard.requests.upstreamTransport": "Up {{transport}}", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index d8ed58337b..32debd4408 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -702,12 +702,18 @@ "dashboard.requests.columns.time": "시간", "dashboard.requests.columns.tokens": "token", "dashboard.requests.columns.transport": "Transport", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "열 ({{count}})", + "dashboard.requests.columnLayout.restoreDefault": "기본 열 레이아웃 복원", + "dashboard.requests.columnLayout.visibleColumns": "표시할 열", "dashboard.requests.downstreamTransport": "Downstream transport", "dashboard.requests.emptyDescription": "클라이언트가 프록시를 사용하면 요청이 여기에 표시됩니다.", "dashboard.requests.emptyFilteredDescription": "현재 필터와 일치하는 request log가 없습니다.", "dashboard.requests.emptyFilteredTitle": "일치하는 요청이 없습니다", "dashboard.requests.emptyTitle": "아직 요청이 없습니다", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.resizeColumn": "{{column}} 열 크기 조절", "dashboard.requests.title": "요청 로그", "dashboard.requests.unassigned": "미할당", "dashboard.requests.upstreamTransport": "Up {{transport}}", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 5cc730652c..a684848fb2 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -702,12 +702,18 @@ "dashboard.requests.columns.time": "时间", "dashboard.requests.columns.tokens": "token", "dashboard.requests.columns.transport": "传输", + "dashboard.requests.columns.tps": "TPS", + "dashboard.requests.columns.ttft": "TTFT", + "dashboard.requests.columnLayout.columns": "列（{{count}}）", + "dashboard.requests.columnLayout.restoreDefault": "恢复默认列布局", + "dashboard.requests.columnLayout.visibleColumns": "可见列", "dashboard.requests.downstreamTransport": "下游传输", "dashboard.requests.emptyDescription": "客户端开始使用代理后，请求将显示在此处。", "dashboard.requests.emptyFilteredDescription": "没有符合当前筛选条件的 request log。", "dashboard.requests.emptyFilteredTitle": "没有匹配的请求", "dashboard.requests.emptyTitle": "暂无请求", "dashboard.requests.requestedTier": "请求 {{tier}}", + "dashboard.requests.resizeColumn": "调整“{{column}}”列宽", "dashboard.requests.title": "请求日志", "dashboard.requests.unassigned": "未分配", "dashboard.requests.upstreamTransport": "上游 {{transport}}", diff --git a/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml b/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml new file mode 100644 index 0000000000..8e7013b8b1 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-07-27 diff --git a/openspec/changes/customize-dashboard-request-log-columns/design.md b/openspec/changes/customize-dashboard-request-log-columns/design.md new file mode 100644 index 0000000000..2ba9649b5e --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/design.md @@ -0,0 +1,44 @@ +## Context + +The dashboard renders request logs through `RecentRequestsTable`, with a fixed +set of columns and browser-managed horizontal overflow. The feature is entirely +presentational and requires no backend contract changes. + +## Goals / Non-Goals + +**Goals:** + +- Let operators select visible request-log columns from the existing dashboard. +- Let pointer and keyboard users resize each visible column independently. +- Persist and safely restore the layout per browser. +- Preserve existing request filtering, pagination, row details, and defaults. + +**Non-Goals:** + +- Creating another dashboard route or navigation item. +- Changing request-log APIs, schemas, database records, or server settings. +- Synchronizing layout preferences between browsers or users. + +## Decisions + +- Keep column metadata and bounded default widths in one typed frontend module + so the chooser, table, and preference validation share a single source of + truth. +- Store only column identifiers and widths in versioned `localStorage`. + Defensive parsing ignores unknown columns and malformed widths. +- Extend `RecentRequestsTable` with optional presentation props. Its existing + defaults remain all columns, preserving other callers and tests. +- Render an accessible separator in each visible header. Pointer movement sets + the width continuously; Left/Right arrow keys adjust it by a fixed step. +- Set table minimum width to the sum of visible widths so the existing + horizontal scroll container handles overflow without a global width slider. + +## Risks / Trade-offs + +- Browser-local settings can become stale after future columns are added. + → Validate stored identifiers and provide a restore-default action. +- Very wide user-selected columns require horizontal scrolling. + → Keep bounded widths and retain the existing overflow container. +- Drag handles can interfere with header content. + → Restrict pointer behavior to a narrow trailing-edge separator with an + explicit resize cursor and accessible name. diff --git a/openspec/changes/customize-dashboard-request-log-columns/proposal.md b/openspec/changes/customize-dashboard-request-log-columns/proposal.md new file mode 100644 index 0000000000..df5cc96cb7 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/proposal.md @@ -0,0 +1,35 @@ +## Why + +The dashboard request log contains many fields, but operators cannot currently +prioritize the fields they use or allocate more horizontal space to values that +need it. Configurable visibility and per-column sizing make the existing table +usable across different workflows and screen sizes. + +## What Changes + +- Add a column chooser to the existing dashboard Request Logs section. +- Allow each visible request-log column to be resized by dragging its header + separator, with keyboard adjustment for accessibility. +- Persist visible columns and individual widths in browser-local storage. +- Preserve at least one visible column and provide a control that restores the + default column layout. +- Derive the table's minimum width from its visible columns so wide layouts + remain horizontally scrollable. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Define configurable visibility, resizable headers, + persistence, reset behavior, and accessibility for dashboard request logs. + +## Impact + +- Dashboard-only frontend changes under `frontend/src/features/dashboard/`. +- A small browser-local preference module; no API, database, authentication, + routing, or deployment changes. +- Focused component, preference, and dashboard integration tests. diff --git a/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md b/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..9bb4f7cea5 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/specs/frontend-architecture/spec.md @@ -0,0 +1,53 @@ +## ADDED Requirements + +### Requirement: Configurable dashboard request-log columns + +The dashboard SHALL let operators show or hide request-log columns and MUST +preserve at least one visible column. Column choices MUST be stored locally per +browser and restored on later visits. Malformed or stale stored choices MUST +fall back to supported defaults without preventing the dashboard from +rendering. A restore-default action MUST clear customized visibility and width +values. + +#### Scenario: Choose visible request-log columns + +- **WHEN** an operator selects or deselects columns in the dashboard request-log column chooser +- **THEN** the corresponding request-log headers and cells are shown or hidden +- **AND** the choice is restored when that browser revisits the dashboard + +#### Scenario: Preserve a usable table + +- **WHEN** only one request-log column remains visible +- **THEN** the dashboard prevents that final column from being hidden + +#### Scenario: Recover from invalid stored preferences + +- **WHEN** stored request-log preferences are malformed or contain unsupported column identifiers +- **THEN** the dashboard renders with supported default columns and widths + +### Requirement: Resizable dashboard request-log columns + +The dashboard SHALL render a vertical resize separator at the trailing edge of +each visible request-log header. Dragging a separator MUST adjust that column +within bounded minimum and maximum widths without changing other configured +columns. Individual widths MUST be stored locally per browser and restored on +later visits. Separators MUST support keyboard adjustment, and the table's +minimum width MUST be derived from its visible column widths so overflow +remains horizontally scrollable without a global table-width control. + +#### Scenario: Resize a request-log column by dragging + +- **WHEN** an operator drags a request-log header separator horizontally +- **THEN** the corresponding header and body column change width +- **AND** the selected width is restored on a later dashboard visit in the same browser + +#### Scenario: Resize a request-log column with the keyboard + +- **WHEN** a focused request-log header separator receives a Left or Right arrow key +- **THEN** the corresponding column width decreases or increases by the documented step within its bounds + +#### Scenario: Wide columns remain reachable + +- **WHEN** the sum of visible request-log column widths exceeds the available viewport +- **THEN** the table remains horizontally scrollable +- **AND** no separate global table-width control is displayed diff --git a/openspec/changes/customize-dashboard-request-log-columns/tasks.md b/openspec/changes/customize-dashboard-request-log-columns/tasks.md new file mode 100644 index 0000000000..2a7e4e17e4 --- /dev/null +++ b/openspec/changes/customize-dashboard-request-log-columns/tasks.md @@ -0,0 +1,24 @@ +## 1. Column layout model + +- [x] 1.1 Add typed request-log column metadata, default widths, and width bounds. +- [x] 1.2 Add a versioned browser-local preference hook with defensive parsing, persistence, final-column protection, and restore-default behavior. +- [x] 1.3 Add focused preference tests for persistence, malformed data, bounds, and reset behavior. + +## 2. Resizable request-log table + +- [x] 2.1 Extend `RecentRequestsTable` with optional visible-column and column-width props while preserving all-column defaults. +- [x] 2.2 Render only selected headers and cells, and derive table minimum width from visible column widths. +- [x] 2.3 Add accessible pointer and keyboard resize separators to visible headers. +- [x] 2.4 Add component tests for visibility, pointer resizing, keyboard resizing, bounds, and horizontal overflow sizing. + +## 3. Dashboard integration + +- [x] 3.1 Add the column chooser and restore-default action to the existing dashboard Request Logs section. +- [x] 3.2 Connect saved visibility and width preferences to `RecentRequestsTable` without changing filters, pagination, or account/dashboard content. +- [x] 3.3 Add dashboard integration coverage for column selection, resizing, persistence, and absence of a global width control. + +## 4. Validation + +- [x] 4.1 Run frontend type checking, lint, focused tests, and the full frontend suite. +- [x] 4.2 Run the production frontend build and strict OpenSpec validation. +- [x] 4.3 Review the final diff to confirm it contains no Compact route, navigation, backend, deployment, secret, or machine-specific changes. From 6c97ad6265100c4ac3489e15a247d73ab45866fb Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:32:42 +0900 Subject: [PATCH 048/117] fix(proxy): keep abrupt eventless websocket drops account-neutral (#1777) * fix(proxy): keep abrupt eventless websocket drops account-neutral An upstream websocket that dies without a close frame before any application-layer response event was always charged to the account: the HTTP bridge reader only consulted _classify_upstream_close when a close frame arrived, so the frame-less reset fell through to penalize_account=True while a graceful 1000 close with zero events was exempted. Three such drops crossed ERROR_BACKOFF_THRESHOLD and 502ed every continuity-bound follow-up (previous_response_owner_unavailable) for 30 seconds while healthy pool siblings idled. Classify the no-close-frame, zero-event drop as account-neutral for the individual health write, keep the penalty whenever a close frame arrived (including non-clean codes) or response events already streamed, and record neutral drops into the existing windowed eventless account drain signal so repeated drops on one account still drain it. The per-bridge retry circuit keeps recording the failure at bridge scope. The [routed-receive-error] pin is flipped intentionally per the issue triage; the direct-close 1011 pin keeps its penalty. Fixes #1754 Co-Authored-By: Claude Fable 5 * fix(proxy): restrict drop neutrality to terminal transport messages Codex review round 1: a protocol-invalid binary frame also carries no close code but does not end the socket, so gate the account-neutral drop exemption on terminal close/error messages and keep the existing penalty for binary frames. Narrow the OpenSpec requirement to unclassified stream_incomplete drops so it does not promise windowed signaling for transport codes already covered by the account-neutral contract. Co-Authored-By: Claude Fable 5 * fix(proxy): treat synthetic 1006 as frame-less and observed output as non-eventless Codex review round 2: - RFC 6455 reserves close code 1006; aiohttp synthesizes it locally when the socket dies without a close frame, so treat it as frame-less in the account-neutral drop predicate (the main routed reset shape). - A buffered reasoning prelude is intentionally excluded from response_event_count but is application-layer output: a following frame-less drop keeps the account penalty. - Scope the OpenSpec windowed-signal requirement to drops that settle pending requests as failures; replay-recovered drops keep their existing behavior. Co-Authored-By: Claude Fable 5 * docs(openspec): fix heading level in keep-abrupt-eventless-drop-account-neutral proposal Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../_service/http_bridge/service_stubs.py | 4 + .../_service/http_bridge/upstream_events.py | 66 +++- .../proxy/_service/streaming/helpers.py | 25 ++ app/modules/proxy/service.py | 3 + .../context.md | 23 ++ .../proposal.md | 50 +++ .../specs/responses-api-compat/spec.md | 25 ++ .../tasks.md | 31 ++ tests/unit/test_proxy_http_bridge.py | 322 +++++++++++++++++- tests/unit/test_proxy_utils.py | 15 + 10 files changed, 554 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md create mode 100644 openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md create mode 100644 openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/service_stubs.py b/app/modules/proxy/_service/http_bridge/service_stubs.py index b4f7ac4a27..3db2e16c15 100644 --- a/app/modules/proxy/_service/http_bridge/service_stubs.py +++ b/app/modules/proxy/_service/http_bridge/service_stubs.py @@ -376,6 +376,10 @@ def _classify_upstream_close(*args: Any, **kwargs: Any) -> Any: return _service_global("_classify_upstream_close")(*args, **kwargs) +def _is_account_neutral_transport_drop(*args: Any, **kwargs: Any) -> Any: + return _service_global("_is_account_neutral_transport_drop")(*args, **kwargs) + + def _websocket_auth_failure_permanent_code(*args: Any, **kwargs: Any) -> Any: return _service_global("_websocket_auth_failure_permanent_code")(*args, **kwargs) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 98db21e125..1f7e55b67c 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -78,6 +78,7 @@ _classify_upstream_close, _find_websocket_request_state_by_response_id, _http_error_status_from_payload, + _is_account_neutral_transport_drop, _is_missing_tool_output_error, _is_previous_response_not_found_error, _is_security_work_authorization_required_error, @@ -227,13 +228,16 @@ async def _record_http_bridge_account_timeout_signal( service: Any, session: "_HTTPBridgeSession", + *, + detail: str = _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL, ) -> None: - """Drain an account after repeated eventless upstream timeouts. + """Drain an account after repeated eventless upstream failures. This is deliberately separate from the per-session retry circuit. A - timeout cannot be replayed safely for a continuity-bound turn, but three - independent eventless failures are enough evidence to keep *new* turns - away from that account until its normal health probe succeeds. + timeout or abrupt eventless transport drop cannot be replayed safely for a + continuity-bound turn, but three independent eventless failures are enough + evidence to keep *new* turns away from that account until its normal + health probe succeeds. """ account_id = session.account.id @@ -265,9 +269,10 @@ async def _record_http_bridge_account_timeout_signal( ) else: logger.warning( - "HTTP bridge account temporarily drained after repeated eventless upstream timeouts " - "account_id=%s threshold=%s window_seconds=%.0f", + "HTTP bridge account temporarily drained after repeated eventless upstream failures " + "account_id=%s detail=%s threshold=%s window_seconds=%.0f", account_id, + detail, _HTTP_BRIDGE_ACCOUNT_TIMEOUT_EJECTION_THRESHOLD, _HTTP_BRIDGE_ACCOUNT_TIMEOUT_WINDOW_SECONDS, ) @@ -1012,6 +1017,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( response_events_seen: int | None = None, transport_classification: str | None = None, retry_circuit_attempt_selection: _HTTPBridgeRetryCircuitAttemptSelection | None = None, + account_neutral_transport_drop: bool = False, ) -> bool: session.closed = True async with session.pending_lock: @@ -1128,12 +1134,27 @@ async def _fail_http_bridge_reader_and_maybe_retire( failed_pending_count > 0 and reservations_settled is not False and observed_response_events == 0 - and retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + and ( + retire_detail == _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + or account_neutral_transport_drop + ) ): # Only penalize the account after pending-request cleanup has # settled its API-key reservations. A failed release must not # be hidden behind an already-recorded timeout health signal. - await _record_http_bridge_account_timeout_signal(self, session) + # Account-neutral abrupt drops share the same windowed signal: + # one drop is infrastructure noise, but repeated eventless + # drops on the same account remain evidence of an account-side + # fault and must still drain it (issue #1754). + await _record_http_bridge_account_timeout_signal( + self, + session, + detail=( + "eventless_transport_drop" + if account_neutral_transport_drop + else _HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL + ), + ) finally: poison_after_deferred_failures = False if session.admission_waiter_count > 0 and not force_retire: @@ -1481,6 +1502,14 @@ async def _relay_http_bridge_upstream_messages( (request_state.response_event_count for request_state in session.pending_requests), default=0, ) + # Buffered reasoning preludes are suppressed from + # response_event_count on purpose, but they are still + # application-layer output: a drop after one is not an + # eventless drop for account-health purposes. + upstream_output_observed = any( + getattr(request_state, "upstream_model_output_seen", False) + for request_state in session.pending_requests + ) reader_failure_retry_circuit_attempt_selection = ( _http_bridge_retry_circuit_attempt_selection_for_pending_requests( tuple(session.pending_requests) @@ -1505,6 +1534,22 @@ async def _relay_http_bridge_upstream_messages( if message.close_code is not None else None ) + # An abrupt drop with no close frame and no response events is + # weaker account-health evidence than a graceful pre-created + # close, which is already exempted below. Keep the individual + # drop account-neutral; repeated eventless drops still feed + # the windowed account drain signal inside the failure path. + # Only terminal transport messages qualify: a protocol-invalid + # binary frame also carries no close code but did not end the + # socket, so it keeps the existing penalty semantics. + account_neutral_transport_drop = ( + message.kind in ("close", "error") + and not account_neutral + and not upstream_output_observed + and _is_account_neutral_transport_drop( + message.close_code, response_events_seen=response_events_seen + ) + ) async with session.lifecycle_lock: if ( session.liveness_settlement_owner == "send" @@ -1528,8 +1573,11 @@ async def _relay_http_bridge_upstream_messages( ), retry_circuit_attempt_selection=reader_failure_retry_circuit_attempt_selection, penalize_account=( - not account_neutral and not (message.kind == "close" and close_classification == "clean") + not account_neutral + and not account_neutral_transport_drop + and not (message.kind == "close" and close_classification == "clean") ), + account_neutral_transport_drop=account_neutral_transport_drop, **( # An admission waiter must not inherit a socket whose # heartbeat already proved it dead. Other failures diff --git a/app/modules/proxy/_service/streaming/helpers.py b/app/modules/proxy/_service/streaming/helpers.py index 6599eb516d..0d720916e4 100644 --- a/app/modules/proxy/_service/streaming/helpers.py +++ b/app/modules/proxy/_service/streaming/helpers.py @@ -477,6 +477,31 @@ def _classify_upstream_close( return "transient" +def _is_account_neutral_transport_drop( + close_code: int | None, + *, + response_events_seen: int, +) -> bool: + """Return whether an upstream websocket ending is account-neutral evidence. + + An abrupt transport drop that carries no close frame and arrived before + any application-layer response event is the weakest possible evidence of + account ill-health: the account never spoke at the application layer for + this request. Charging the account lets a few infrastructure resets push + it into error backoff and 502 continuity-bound follow-ups while healthy + pool siblings idle (issue #1754). Any close frame — even a non-clean one — + is upstream-authored evidence and keeps the existing penalty semantics, as + does a drop after response events started streaming. + + Close code 1006 (abnormal closure) is reserved by RFC 6455 and can never + appear in an actual close frame: adapters synthesize it locally when the + socket dies without one (aiohttp stores 1006 on ``close_code`` for an + abnormal CLOSED), so it counts as frame-less here. + """ + + return close_code in (None, 1006) and response_events_seen == 0 + + def _should_infer_upstream_status_from_proxy_error(exc: ProxyResponseError, upstream_error_code: str | None) -> bool: if exc.failure_phase == "status": return True diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 196a69964f..898b41d88e 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -513,6 +513,9 @@ from app.modules.proxy._service.streaming.helpers import ( _classify_upstream_close as _classify_upstream_close, ) +from app.modules.proxy._service.streaming.helpers import ( + _is_account_neutral_transport_drop as _is_account_neutral_transport_drop, +) from app.modules.proxy._service.streaming.helpers import ( _push_stream_attempt_timeout_overrides as _push_stream_attempt_timeout_overrides, ) diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md new file mode 100644 index 0000000000..39d600e4fb --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/context.md @@ -0,0 +1,23 @@ +A websocket that dies without a close frame before any application-layer +response event is the weakest possible evidence of account ill-health. The +reader failure path nevertheless penalized it because +`close_classification` was computed only for `close_code is not None`, so +the frame-less case fell through to the default `penalize_account=True`. +Meanwhile the strictly stronger signal — a graceful 1000 close with zero +events — was already exempted, and #1718 established the same precedent for +stream idle timeouts. + +The fix adds `_is_account_neutral_transport_drop` beside +`_classify_upstream_close` (no close frame AND zero response events) and +consults it in the reader failure path. To avoid masking a genuine account +ban that manifests as repeated drops, the neutral drop is recorded into the +existing `_record_http_bridge_account_timeout_signal` accumulator: three +eventless failures within the 300-second window still apply the minimum +drain penalty. No new settings are introduced. + +Incident shape from #1754: three drops ~12 minutes apart never meet the +300-second window, so the owner stays routable and continuity-bound +follow-ups keep working; before the fix they crossed +`ERROR_BACKOFF_THRESHOLD` and produced eight +`previous_response_owner_unavailable` 502s in eight seconds while the other +pool account idled. diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md new file mode 100644 index 0000000000..0ef5273b1d --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/proposal.md @@ -0,0 +1,50 @@ +# Why + +An abrupt upstream websocket drop with no close frame and zero response +events is charged to the account: the HTTP bridge reader only consults +`_classify_upstream_close` when a close frame arrived, so a frame-less +transport reset always sets `penalize_account=True` while a graceful 1000 +close before any response event is exempted. That is inverted with respect +to the available evidence — the account never spoke at the application +layer. Three such drops cross the error-backoff threshold and 502 every +continuity-bound follow-up (`previous_response_owner_unavailable`) for 30 +seconds while healthy pool siblings idle (issue #1754). + +# What Changes + +- Classify an abrupt upstream websocket ending — a terminal close or receive + error with no upstream-authored close frame (the synthetic abnormal-closure + code 1006 counts as frame-less), no established account-neutral transport + classification, and no observed application-layer output — as + account-neutral in the HTTP bridge reader failure path: no `record_error` + health write for the individual drop. +- Keep the existing penalty when an upstream-authored close frame arrived + (including non-clean codes such as 1008/1011), when application-layer + output was already observed (streamed response events or a buffered + reasoning prelude), or when a non-terminal protocol-invalid frame (for + example a binary message) triggered the failure, and keep all established + account-neutral transport codes on their existing contract. +- Feed account-neutral eventless drops that settle their pending requests as + failures into the existing windowed eventless account drain signal so + repeated drops on the same account still drain it (same threshold/window + as repeated eventless upstream timeouts), keeping genuine account faults + visible. Drops recovered by the bounded pre-created replay keep their + existing behavior. +- The per-bridge retry circuit continues to record the failure at bridge + scope, unchanged. + +# Capabilities + +## Modified Capabilities + +- `responses-api-compat`: HTTP bridge abrupt eventless upstream drops must + stay account-neutral for per-drop health writes while repeated drops still + drain the account through the windowed eventless failure signal. + +# Impact + +Continuity-bound conversations survive sporadic infrastructure resets +instead of 502-storming against a self-inflicted 30-second owner backoff. +Accounts whose sockets repeatedly drop eventlessly are still drained by the +existing windowed signal. Clean-close, close-frame, and mid-stream drop +semantics are unchanged, as is the per-bridge retry circuit. diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..9fafb71553 --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/specs/responses-api-compat/spec.md @@ -0,0 +1,25 @@ +## ADDED Requirements + +### Requirement: Abrupt eventless upstream websocket drops remain account-neutral + +When an HTTP bridge upstream websocket ends with a terminal transport message (a close or receive error) that carries no upstream-authored close frame, no established account-neutral transport classification (process-network, liveness-timeout, keepalive-timeout), and no application-layer output was observed for the pending requests (zero response events and no buffered reasoning prelude), the proxy MUST NOT write per-drop account error-health (`record_error`) for that unclassified `stream_incomplete` drop. The synthetic abnormal-closure code 1006, which RFC 6455 reserves and which adapters synthesize locally when the socket dies without a close frame, MUST be treated as frame-less. When such a drop settles its pending requests as failures, the proxy MUST record it into the windowed eventless account failure signal so that repeated eventless drops on the same account within the window still apply the drain penalty; drops recovered by the bounded pre-created replay keep their existing behavior, and drops already covered by an established account-neutral transport classification keep their existing contract and are not added to the signal. A failure that carries an upstream-authored close frame (including non-clean codes), occurs after application-layer output was observed, or arrives as a non-terminal protocol-invalid frame (for example a binary message) MUST keep the existing account penalty semantics. The per-bridge retry circuit MUST still record the failure at bridge scope. + +#### Scenario: Sporadic frame-less drops do not strand a continuity-bound conversation + +- **GIVEN** a conversation continuity-bound to account A via `previous_response_id` +- **AND** account A's upstream websocket drops three times with no close frame and zero response events, spread wider than the eventless failure window +- **WHEN** the client sends the next continuity-bound follow-up +- **THEN** account A's `error_count` receives no per-drop increment and stays below the error-backoff threshold +- **AND** the follow-up still routes to account A instead of failing with `previous_response_owner_unavailable` + +#### Scenario: Repeated eventless drops inside the window still drain the account + +- **GIVEN** an account whose upstream websocket drops with no close frame and zero response events on three separate bridge failures within the eventless failure window +- **WHEN** the third drop is recorded +- **THEN** the windowed eventless failure signal applies the minimum drain penalty so new turns avoid the account until its health probe succeeds + +#### Scenario: Close frames and observed-output drops keep the account penalty + +- **GIVEN** an upstream websocket ending that carries an upstream-authored close frame (for example 1008 or 1011) before any response event, or a frame-less drop after application-layer output was observed (streamed response events or a buffered reasoning prelude), or a non-terminal protocol-invalid binary frame +- **WHEN** the reader failure path settles the pending requests +- **THEN** the account penalty semantics are unchanged from before this change diff --git a/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md new file mode 100644 index 0000000000..b03daf3a4f --- /dev/null +++ b/openspec/changes/keep-abrupt-eventless-drop-account-neutral/tasks.md @@ -0,0 +1,31 @@ +## 1. Implementation + +- [x] 1.1 Add `_is_account_neutral_transport_drop` (no close frame AND zero + response events) beside `_classify_upstream_close`. +- [x] 1.2 Consult it in the HTTP bridge reader failure path so the frame-less + eventless drop no longer sets `penalize_account=True`. +- [x] 1.3 Record account-neutral drops into the windowed eventless account + drain signal (`_record_http_bridge_account_timeout_signal`) so repeated + drops still drain the account. + +## 2. Regression coverage + +- [x] 2.1 Flip the `[routed-receive-error]` pin intentionally: + `penalize_account is False` for a frame-less eventless drop. +- [x] 2.2 Assert the drop records the windowed drain signal with + `detail=eventless_transport_drop`. +- [x] 2.3 Assert a drop after streamed response events still penalizes. +- [x] 2.4 Assert a non-clean close frame (1008/1011) with zero events still + penalizes. +- [x] 2.5 Assert a non-terminal protocol-invalid binary frame still penalizes + and records no drop signal. +- [x] 2.6 Assert the synthetic abnormal-closure code 1006 counts as + frame-less and stays account-neutral. +- [x] 2.7 Assert a drop after a buffered reasoning prelude (output observed, + zero response events) still penalizes. +- [x] 2.8 Helper unit coverage for `_is_account_neutral_transport_drop`. + +## 3. Validation + +- [x] 3.1 Run the HTTP bridge unit suite and proxy utils suite. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 9d4ea02384..3e91a8f60b 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -26711,16 +26711,336 @@ async def fail_reader( assert session.last_upstream_close_code == (None if routed else 1011) assert len(failure_calls) == 1 assert failure_calls[0]["error_code"] == "stream_incomplete" - assert failure_calls[0]["penalize_account"] is True assert failure_calls[0]["response_events_seen"] == 0 if routed: + # Intentional flip for issue #1754: an abrupt drop with no close frame + # and zero response events must stay account-neutral instead of + # feeding error backoff and stranding continuity-bound follow-ups. + assert failure_calls[0]["penalize_account"] is False + assert failure_calls[0]["account_neutral_transport_drop"] is True assert failure_calls[0]["upstream_close_code"] is None assert failure_calls[0]["transport_classification"] == "websocket_transport_error" else: + # A close frame — even a non-clean 1011 — is upstream-authored + # evidence and keeps the existing account penalty. + assert failure_calls[0]["penalize_account"] is True + assert failure_calls[0]["account_neutral_transport_drop"] is False assert failure_calls[0]["upstream_close_code"] == 1011 assert failure_calls[0]["transport_classification"] == "websocket_close_transient" +@pytest.mark.asyncio +async def test_http_bridge_abrupt_eventless_drop_stays_account_neutral_and_records_drop_signal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Issue #1754: an abrupt upstream drop with no close frame and zero + response events must not feed per-drop account error backoff (which 502s + continuity-bound follow-ups), but repeated eventless drops must still feed + the windowed account drain signal so genuine account faults surface.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-abrupt-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + session = _make_bridge_session( + key_value="bridge-abrupt-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock( + return_value=UpstreamWebSocketMessage( + kind="error", + error="Upstream websocket closed before response.completed: no close frame received or sent", + ) + ), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + assert target_session is session + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["error_code"] == "stream_incomplete" + assert fail_pending.await_args.kwargs["penalize_account"] is False + assert drop_signals == [{"detail": "eventless_transport_drop"}] + retire.assert_awaited_once() + + +@pytest.mark.asyncio +async def test_http_bridge_abrupt_drop_after_response_events_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A drop after the upstream already streamed response events remains + account-attributable: only the eventless no-close-frame case is neutral.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-mid-stream-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + request_state.response_event_count = 8 + session = _make_bridge_session( + key_value="bridge-mid-stream-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="error", error="upstream reset")), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] + + +@pytest.mark.asyncio +async def test_http_bridge_non_clean_close_frame_before_response_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An upstream-authored close frame (e.g. policy 1008) with zero response + events keeps the existing account penalty; only the frame-less drop is + account-neutral.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-policy-close") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1008)), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + + +@pytest.mark.asyncio +async def test_http_bridge_synthetic_1006_close_stays_account_neutral( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """aiohttp reports an abnormal socket loss as CLOSED with the synthesized + reserved code 1006 (never sent in an actual close frame): it is the same + frame-less eventless drop and must stay account-neutral (issue #1754).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-1006-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + session = _make_bridge_session( + key_value="bridge-1006-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="close", close_code=1006)), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is False + assert drop_signals == [{"detail": "eventless_transport_drop"}] + + +@pytest.mark.asyncio +async def test_http_bridge_abrupt_drop_after_buffered_reasoning_prelude_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A buffered reasoning prelude is deliberately excluded from + response_event_count, but it is application-layer output: a following + frame-less drop is not eventless and keeps the account penalty.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + request_state = proxy_service._WebSocketRequestState( + request_id="req-reasoning-prelude-drop", + model="gpt-5.2", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + transport="http", + ) + request_state.upstream_model_output_seen = True + assert request_state.response_event_count == 0 + session = _make_bridge_session( + key_value="bridge-reasoning-prelude-drop", + pending_requests=deque([request_state]), + queued_request_count=1, + ) + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="error", error="upstream reset")), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] + + +@pytest.mark.asyncio +async def test_http_bridge_protocol_invalid_binary_frame_still_penalizes_account( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A non-terminal protocol-invalid frame (binary payload) also carries no + close code, but the socket did not end: it must keep the existing account + penalty instead of being classified as an abrupt transport drop.""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session(key_value="bridge-binary-frame") + session.upstream = cast( + UpstreamWebSocket, + SimpleNamespace( + receive=AsyncMock(return_value=UpstreamWebSocketMessage(kind="binary", data=b"\x00\x01")), + close=AsyncMock(), + ), + ) + fail_pending = AsyncMock(return_value=True) + retire = AsyncMock() + drop_signals: list[dict[str, object]] = [] + + async def record_signal( + target_service: object, + target_session: proxy_service._HTTPBridgeSession, + **kwargs: object, + ) -> None: + drop_signals.append(dict(kwargs)) + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service, "_retry_http_bridge_precreated_request", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + monkeypatch.setattr( + http_bridge_upstream_events_module, + "_record_http_bridge_account_timeout_signal", + record_signal, + ) + + await service._relay_http_bridge_upstream_messages(session) + + assert fail_pending.await_args is not None + assert fail_pending.await_args.kwargs["penalize_account"] is True + assert drop_signals == [] + + @pytest.mark.asyncio async def test_http_bridge_response_create_gate_timeout_logs_pending_bridge_context( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 6d5207a616..9cd887ba2e 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -40285,6 +40285,21 @@ def test_classify_upstream_close_clean_for_clean_close_before_any_response_event assert proxy_service._classify_upstream_close(1011, response_events_seen=0) == "transient" +def test_account_neutral_transport_drop_requires_no_close_frame_and_no_response_events(): + # Issue #1754: only a frame-less drop before any application-layer + # response event is account-neutral; any close frame or streamed events + # keep the account penalty semantics. + assert proxy_service._is_account_neutral_transport_drop(None, response_events_seen=0) is True + assert proxy_service._is_account_neutral_transport_drop(None, response_events_seen=8) is False + assert proxy_service._is_account_neutral_transport_drop(1000, response_events_seen=0) is False + assert proxy_service._is_account_neutral_transport_drop(1008, response_events_seen=0) is False + assert proxy_service._is_account_neutral_transport_drop(1011, response_events_seen=0) is False + # RFC 6455 reserves 1006: it never travels in an actual close frame, so a + # synthesized abnormal-closure code counts as frame-less. + assert proxy_service._is_account_neutral_transport_drop(1006, response_events_seen=0) is True + assert proxy_service._is_account_neutral_transport_drop(1006, response_events_seen=1) is False + + @pytest.mark.asyncio async def test_open_upstream_websocket_dns_failure_recovers_on_same_account(monkeypatch): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) From 9eedb2c8f4aa809715484e1c7607e6f0219b4ccf Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:33:11 +0900 Subject: [PATCH 049/117] fix(db): bound wedged SQLite session teardown and reclaim the connection (#1778) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(db): bound wedged SQLite session teardown and reclaim the connection Part 2 of the issue #1682 plan. Session teardown shielded rollback/close unboundedly, so a wedged teardown pinned SQLite's single writer slot with nothing to reclaim it: every writer — including the scheduler_leader INSERT needed to re-establish leadership — surfaced 'database is locked' until the wedge spontaneously resolved (~17 minutes in the report). The teardown now gets a hard deadline on file-backed SQLite (busy timeout / 6 = 5s), still shielded from the caller's cancellation. Abandoning the wedged await alone would release nothing — the aiosqlite worker thread keeps holding the lock — so a missed deadline reclaims the connection: the driver is interrupted (aborting the C-level call the worker is stuck in) and the connection is invalidated, which terminates it at the pool via aiosqlite's stop(), hard-closing the underlying sqlite3 connection. That releases the writer slot and guarantees the connection is never handed out again. The reclaim log carries part 1's long-write watchdog identifiers, including ones already deferred into its pending report (invalidation would otherwise suppress that report). Wedged sessions are fenced from further teardown and closed for bookkeeping via a cleanup task owned until completion and drained at close_db. PostgreSQL teardown is untouched, and in-memory SQLite keeps the unbounded path: its one shared connection is the whole database and cannot starve other writers. Refs #1682 (part 2 of 3; part 1 was #1752). Co-Authored-By: Claude Fable 5 * fix(db): track abandoned wedged teardowns through the bounded close_db drain The reclaimed rollback/close task was never registered in _wedged_teardown_cleanup_tasks — only the late bookkeeping close was — so close_db could return while the abandoned teardown was still pending, and its one-shot gather snapshot missed the bookkeeping close scheduled after an abandoned task completed mid-drain. Register the abandoned task in the registry immediately, and make close_db drain the registry until stable under one explicit deadline (2x the teardown bound) so a teardown still wedged despite the reclaim cannot wedge shutdown either. Co-Authored-By: Claude Fable 5 * fix(db): harden wedged-teardown reclaim per review - register the abandoned teardown in the cleanup registry before the reclaim's first await so a concurrent close_db can never observe an empty registry while the rollback is pending - call driver.interrupt() and await the result only when awaitable, pinning nothing on aiosqlite's coroutine shape; raise the swallowed interrupt failure from debug to warning - detect mode=memory in the parsed SQLite URL query (file: URI forms) as in-memory, keeping the unbounded teardown there - tests: exercise the real aiosqlite interrupt contract without a spy, make the interrupt spy shape-preserving, cover in-memory vs file-backed URL forms, and give _FakeBind a file-backed url stub Co-Authored-By: Claude Fable 5 * fix(db): require uri=true before classifying mode=memory SQLite URLs as in-memory Without a truthy uri query parameter the pysqlite/aiosqlite dialects never enable SQLite URI mode: sqlite+aiosqlite:///file:shared?mode=memory&cache=shared connects to a file literally named "file:shared", so the teardown classifier must keep it on the bounded (file-backed) path instead of granting it the unbounded in-memory shield. URI-form in-memory detection is now gated on uri=true plus SQLite's own file: prefix requirement. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/db/session.py | 354 +++++++++++- .../bound-sqlite-wedged-teardown/proposal.md | 21 + .../specs/database-backends/spec.md | 40 ++ .../bound-sqlite-wedged-teardown/tasks.md | 14 + tests/unit/test_db_session.py | 514 ++++++++++++++++++ 5 files changed, 939 insertions(+), 4 deletions(-) create mode 100644 openspec/changes/bound-sqlite-wedged-teardown/proposal.md create mode 100644 openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md create mode 100644 openspec/changes/bound-sqlite-wedged-teardown/tasks.md diff --git a/app/db/session.py b/app/db/session.py index 48e68a9cae..1c4ad3c4cb 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import inspect import logging import os import sqlite3 @@ -8,12 +9,13 @@ from contextlib import asynccontextmanager from enum import StrEnum from pathlib import Path -from typing import TYPE_CHECKING, AsyncIterator, Awaitable, Callable, Protocol, TypeVar +from typing import TYPE_CHECKING, Any, AsyncIterator, Awaitable, Callable, Protocol, TypeVar import anyio from anyio import to_thread from sqlalchemy import event, text -from sqlalchemy.engine import Engine +from sqlalchemy import util as sqlalchemy_util +from sqlalchemy.engine import Connection, Engine from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool @@ -55,6 +57,26 @@ "begin exclusive", ) _SQLITE_WATCHDOG_STATEMENT_PREVIEW_CHARS = 300 +# Hard deadline for the shielded rollback/close teardown on SQLite (part 2 of +# the issue #1682 plan). A teardown wedged behind a stuck aiosqlite worker +# keeps holding the single writer slot, so it must be reclaimed well before +# other writers exhaust their busy timeout and surface "database is locked"; +# one-sixth of the busy timeout (5s) matches the leader-gated shielded-drain +# grace. Abandoning the wedged await alone would NOT release the lock — the +# aiosqlite worker thread still holds it — so on timeout the reclaim below +# interrupts the driver and invalidates the connection, disposing the worker. +_SQLITE_TEARDOWN_TIMEOUT_SECONDS = _SQLITE_BUSY_TIMEOUT_SECONDS / 6 +# Session.info marker set once a teardown step was abandoned as wedged: the +# session must never be driven by another coroutine again (the abandoned +# greenlet may still resume), and the deferred cleanup takes over. +_SQLITE_TEARDOWN_WEDGED_INFO_KEY = "sqlite_teardown_wedged" +# Abandoned wedged teardown tasks and the deferred bookkeeping closes they +# schedule on late completion, owned until completion so shutdown (close_db) +# drains them instead of closing the event loop over pending tasks. The +# bookkeeping closes are bounded by _SQLITE_TEARDOWN_TIMEOUT_SECONDS; an +# abandoned teardown may outlive its reclaim (the interrupt is best-effort), +# so the close_db drain is explicitly bounded as a whole. +_wedged_teardown_cleanup_tasks: set[asyncio.Task[Any]] = set() # PostgreSQL pool checkout timeout and connection recycle window. Fixed # application constants (issue #1340): recycle keeps pooled connections @@ -357,20 +379,320 @@ async def _shielded(awaitable: Awaitable[object]) -> None: raise cancellation +async def _shielded_bounded(awaitable: Awaitable[object], timeout: float) -> asyncio.Task[object] | None: + """Shield ``awaitable`` from the caller's cancellation, waiting at most ``timeout``. + + Returns ``None`` when the awaitable finished inside the bound (re-raising + its exception like ``_shielded``); returns the still-running task when the + deadline passed — the caller must treat the underlying connection as + wedged and reclaim it, because the abandoned await does not release + anything the aiosqlite worker thread holds (issue #1682). + """ + task = asyncio.ensure_future(awaitable) + waiter = asyncio.ensure_future(asyncio.wait({task}, timeout=timeout)) + while not waiter.done(): + try: + await asyncio.shield(waiter) + except asyncio.CancelledError: + # Teardown runs in ``finally`` blocks: the bound, not the caller's + # cancellation, decides abandonment. ``asyncio.wait`` cannot + # outlive its timeout, so this drain stays bounded. + continue + if task.done(): + task.result() + return None + return task + + +def _sqlite_uri_mode_active(url: Any) -> bool: + """Whether the pysqlite/aiosqlite dialect will connect in URI mode. + + Mirrors the dialect's ``create_connect_args``: URI mode is enabled only + when the URL query carries a ``uri`` value that coerces to true. + """ + query = getattr(url, "query", None) + if query is None: + return False + try: + value = query.get("uri") + except Exception: + return False + values = value if isinstance(value, (tuple, list)) else (value,) + for item in values: + if item is None: + continue + try: + if sqlalchemy_util.asbool(str(item)): + return True + except Exception: + # An unrecognized value would fail at connect time anyway; + # classify conservatively as file-backed (bounded). + continue + return False + + +def _session_teardown_bound_seconds(session: AsyncSession) -> float | None: + """Teardown deadline for this session, or None for the unbounded path. + + Only file-backed SQLite gets a bound: its single writer slot turns a + wedged teardown into a database-wide write stall (issue #1682). + PostgreSQL teardown semantics are deliberately untouched, and in-memory + SQLite shares one StaticPool connection with the whole process — the + reclaim's invalidation would destroy the entire database (the + database-backends spec requires preserving shared in-memory state), and + with a single shared connection there is no cross-connection writer + contention to starve in the first place. + """ + try: + bind = session.get_bind() + except Exception: + return None + if getattr(getattr(bind, "dialect", None), "name", None) != "sqlite": + return None + url = getattr(bind, "url", None) + if url is not None: + database = getattr(url, "database", None) + if not database: + return None + database_text = str(database) + if database_text == ":memory:": + return None + # SQLite URI forms (``sqlite:///file:name?mode=memory&cache=shared&uri=true``) + # are in-memory only when the pysqlite/aiosqlite dialect actually + # passes the database string to the driver as a URI, which it does + # only when the URL query carries a truthy ``uri`` — and SQLite itself + # parses a filename as a URI only when it starts with ``file:``. + # Without ``uri=true``, ``file:name?mode=memory`` is a *file-backed* + # database whose filename literally contains those characters, so it + # must keep the bounded teardown. + if _sqlite_uri_mode_active(url) and database_text.startswith("file:"): + # ``mode=memory`` normally rides the parsed URL's query; it only + # appears inside ``url.database`` when the URL escaped the query + # into the database portion. + if ":memory:" in database_text or "mode=memory" in database_text: + return None + query = getattr(url, "query", None) + if query is not None: + try: + mode = query.get("mode") + except Exception: + mode = None + modes = mode if isinstance(mode, (tuple, list)) else (mode,) + if any(str(value).lower() == "memory" for value in modes if value is not None): + return None + return _SQLITE_TEARDOWN_TIMEOUT_SECONDS + + +def _session_is_teardown_wedged(session: AsyncSession) -> bool: + try: + return bool(session.info.get(_SQLITE_TEARDOWN_WEDGED_INFO_KEY)) + except Exception: + return False + + +def _session_sync_connections(session: AsyncSession) -> tuple[Connection, ...]: + """Best-effort snapshot of the sync Connections held by the session's transaction. + + Captured before a teardown attempt so a wedged rollback can be attributed + and its connection reclaimed. Diagnostics only — never raises. + """ + try: + transaction = session.sync_session.get_transaction() + if transaction is None: + return () + connections = getattr(transaction, "_connections", None) + if not isinstance(connections, dict): + return () + # The transaction tracks each Connection under two keys (the + # Connection itself and its Engine); deduplicate by identity. + unique: dict[int, Connection] = {} + for value in connections.values(): + if isinstance(value, tuple) and value and isinstance(value[0], Connection): + unique[id(value[0])] = value[0] + return tuple(unique.values()) + except Exception: + return () + + +def _sqlite_watchdog_identifiers(connection: Connection) -> str: + """Render the long-write watchdog's identifiers for the wedged connection. + + Invalidation prevents the connection from ever reaching the watchdog's + deferred report (next begin / pool checkin), so the reclaim log carries + the same attribution instead. + """ + try: + info = connection.info + started_at = info.get("sqlite_write_started_at") + first_statement = info.get("sqlite_first_write_statement") + last_statement = info.get("sqlite_last_write_statement") + task_name = info.get("sqlite_write_task") + if started_at is None: + # The watchdog's commit/rollback listener already moved the + # identifiers into the deferred report — the wedge happened inside + # the transaction-ending call itself, exactly the issue #1682 + # shape. + pending = info.get("sqlite_write_pending_report") + if isinstance(pending, tuple) and len(pending) == 5: + started_at, _, first_statement, last_statement, task_name = pending + held = f"{time.monotonic() - started_at:.1f}" if isinstance(started_at, float) else "unknown" + return ( + f"write_held_seconds={held} write_task={task_name!r} " + f"first_statement={first_statement!r} last_statement={last_statement!r}" + ) + except Exception: + return "write_held_seconds=unknown" + + +async def _reclaim_wedged_sqlite_session( + session: AsyncSession, + abandoned: asyncio.Task[object], + connections: tuple[Connection, ...], + *, + phase: str, +) -> None: + """Release what a wedged SQLite teardown still holds and fence the session. + + Abandoning the wedged rollback/close is not enough: the aiosqlite worker + thread keeps holding the write lock (issue #1682). Interrupting the driver + aborts the C-level call the worker is stuck in, and invalidating the + connection terminates it at the pool — aiosqlite's ``stop()`` queues a + hard close of the underlying ``sqlite3`` connection, which releases the + writer slot and disposes the worker thread — so leader election and every + other writer recover instead of stalling behind the wedge. The invalidated + connection can never be handed out again. + """ + try: + session.info[_SQLITE_TEARDOWN_WEDGED_INFO_KEY] = True + except Exception: + logger.exception("Failed to fence a wedged SQLite session during teardown reclaim") + # Own the abandoned teardown before this coroutine's first await: if + # close_db runs concurrently with the reclaim, it must already see the + # pending task in the registry instead of returning while the rollback is + # still pending. The completion callbacks are attached only after the + # connection is invalidated below, so the deferred bookkeeping close can + # never touch a live connection. + _wedged_teardown_cleanup_tasks.add(abandoned) + for connection in connections: + logger.warning( + "sqlite_wedged_teardown phase=%s bound_seconds=%.1f %s — interrupting and invalidating the " + "connection so the writer slot is released instead of stalling every writer (issue #1682)", + phase, + _SQLITE_TEARDOWN_TIMEOUT_SECONDS, + _sqlite_watchdog_identifiers(connection), + ) + try: + driver = connection.connection.driver_connection + if driver is not None: + # aiosqlite's ``interrupt`` runs sqlite3_interrupt inline on + # this task — it never enters the (wedged) worker queue. In + # the pinned aiosqlite (0.22.x) it is a coroutine function; + # await the result only when it is awaitable so a driver that + # makes ``interrupt`` synchronous keeps working. + result = driver.interrupt() + if inspect.isawaitable(result): + await result + except Exception: + logger.warning( + "Interrupting a wedged SQLite connection failed — the invalidation below still " + "reclaims the writer slot, but the stuck statement may run to completion first", + exc_info=True, + ) + try: + connection.invalidate() + except Exception: + logger.debug("Invalidating a wedged SQLite connection failed", exc_info=True) + if not connections: + logger.warning( + "sqlite_wedged_teardown phase=%s bound_seconds=%.1f — no held connection to reclaim; " + "abandoning the wedged %s (issue #1682)", + phase, + _SQLITE_TEARDOWN_TIMEOUT_SECONDS, + phase, + ) + # The abandoned teardown is owned until completion (registered above, so + # close_db drains it and shutdown waits for — or boundedly abandons — the + # reclaimed rollback/close instead of returning while it is still + # pending). The discard callback is registered first so that when the task + # completes during the drain, deregistration happens before + # _finish_abandoned_teardown registers the follow-up bookkeeping close. + abandoned.add_done_callback(_wedged_teardown_cleanup_tasks.discard) + abandoned.add_done_callback(lambda task: _finish_abandoned_teardown(session, task, phase=phase)) + + +def _finish_abandoned_teardown(session: AsyncSession, task: asyncio.Task[object], *, phase: str) -> None: + if not task.cancelled(): + # The wedged teardown resuming into an interrupted/invalidated + # connection is expected to error; consume it so the abandoned task + # never logs "exception was never retrieved". + task.exception() + logger.info("Wedged SQLite teardown finished late phase=%s", phase) + if phase != "rollback": + return + + # The session was abandoned before ``close`` ran. Now that no other + # coroutine can be driving it, close it for bookkeeping — the connection + # is already invalidated, so this cannot touch the database. + async def _close_late() -> None: + try: + await asyncio.wait_for(session.close(), timeout=_SQLITE_TEARDOWN_TIMEOUT_SECONDS) + except BaseException: + logger.debug("Late close of a wedged SQLite session failed", exc_info=True) + + try: + cleanup_task = asyncio.get_running_loop().create_task(_close_late()) + except RuntimeError: + # Event loop already gone (shutdown); the invalidated connection was + # closed at the pool, nothing is leaked. + return + # Own the task until completion: close_db drains it so shutdown cannot + # skip the promised bookkeeping close or leave a pending-task warning. + _wedged_teardown_cleanup_tasks.add(cleanup_task) + cleanup_task.add_done_callback(_wedged_teardown_cleanup_tasks.discard) + + async def _safe_rollback(session: AsyncSession) -> None: if not session.in_transaction(): return + if _session_is_teardown_wedged(session): + # A previous bounded teardown abandoned a wedged rollback; the + # abandoned greenlet may still resume, so never drive this session + # concurrently. The reclaim already released the connection. + return + bound = _session_teardown_bound_seconds(session) + if bound is None: + try: + await _shielded(session.rollback()) + except BaseException: + return + return + held_connections = _session_sync_connections(session) try: - await _shielded(session.rollback()) + abandoned = await _shielded_bounded(session.rollback(), bound) except BaseException: return + if abandoned is not None: + await _reclaim_wedged_sqlite_session(session, abandoned, held_connections, phase="rollback") async def _safe_close(session: AsyncSession) -> None: + if _session_is_teardown_wedged(session): + # Deferred cleanup owns the session now; see _finish_abandoned_teardown. + return + bound = _session_teardown_bound_seconds(session) + if bound is None: + try: + await _shielded(session.close()) + except BaseException: + return + return + held_connections = _session_sync_connections(session) try: - await _shielded(session.close()) + abandoned = await _shielded_bounded(session.close(), bound) except BaseException: return + if abandoned is not None: + await _reclaim_wedged_sqlite_session(session, abandoned, held_connections, phase="close") async def close_session(session: AsyncSession) -> None: @@ -630,6 +952,30 @@ async def init_db() -> None: async def close_db() -> None: + if _wedged_teardown_cleanup_tasks: + # Abandoned wedged teardowns plus their deferred bookkeeping closes. + # Drain until the registry is stable — an abandoned teardown that + # completes during the drain schedules its bookkeeping close only + # after any one-time snapshot — and bound the whole drain so a + # teardown still wedged despite the reclaim (the interrupt is + # best-effort) cannot wedge shutdown too: one deadline covers the + # abandoned teardown and the bounded close it chains. + loop = asyncio.get_running_loop() + deadline = loop.time() + 2 * _SQLITE_TEARDOWN_TIMEOUT_SECONDS + while _wedged_teardown_cleanup_tasks: + remaining = deadline - loop.time() + if remaining <= 0: + logger.warning( + "close_db abandoned %d still-pending wedged-teardown task(s) after the bounded " + "drain; their connections were already reclaimed (issue #1682)", + len(_wedged_teardown_cleanup_tasks), + ) + break + await asyncio.wait(tuple(_wedged_teardown_cleanup_tasks), timeout=remaining) + # Completion callbacks (deregistration and scheduling of the + # deferred bookkeeping close) run via call_soon; yield once so + # the registry reflects them before the next stability check. + await asyncio.sleep(0) await engine.dispose() if _background_engine is not None: await _background_engine.dispose() diff --git a/openspec/changes/bound-sqlite-wedged-teardown/proposal.md b/openspec/changes/bound-sqlite-wedged-teardown/proposal.md new file mode 100644 index 0000000000..f6a1e4dfd6 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/proposal.md @@ -0,0 +1,21 @@ +## Why + +Issue #1682, part 2 of the plan. Session teardown shields rollback/close unboundedly (`app/db/session.py`), so a wedged teardown pins SQLite's single writer slot with nothing to reclaim it: every writer — including the `scheduler_leader` INSERT that would re-establish leadership — surfaces `database is locked` until the wedge spontaneously resolves (~17 minutes in the report). Part 1 (`report-sqlite-long-write-holders`) made the holder attributable; the teardown itself must now be bounded. Crucially, abandoning the wedged await alone releases nothing — the aiosqlite worker thread still holds the lock — so the bound must come with reclaiming the connection. + +## What Changes + +- The shielded rollback/close teardown gets a hard deadline on file-backed SQLite (one-sixth of the busy timeout, 5s — reclaimed well before other writers exhaust their 30s busy timeout). PostgreSQL teardown semantics are untouched, and in-memory SQLite keeps the unbounded path: its one shared StaticPool connection is the whole database (invalidation would destroy it) and cannot starve other writers. +- A teardown that misses the deadline is reclaimed, not merely abandoned: the driver connection is interrupted (aborting the C-level call the aiosqlite worker is stuck in) and the connection is invalidated — terminating it at the pool disposes the worker and hard-closes the underlying `sqlite3` connection, which releases the writer slot and guarantees the connection is never handed out again. +- The reclaim report carries part 1's watchdog identifiers (held duration, owning task, first/last write statements), including when the watchdog had already deferred them into its pending report because the wedge is inside the transaction-ending call itself — invalidation would otherwise suppress that deferred report. +- A wedged session is fenced: later teardown attempts return immediately instead of driving the session concurrently with the abandoned work, and once the abandoned teardown finishes late the session is closed for bookkeeping — a deferred task owned until completion and drained at `close_db`, never fire-and-forget. +- No new settings: the deadline derives from the existing busy timeout. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `database-backends`: a wedged SQLite session teardown is bounded and its connection reclaimed so the writer slot is released. diff --git a/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md b/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md new file mode 100644 index 0000000000..b234160df7 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/specs/database-backends/spec.md @@ -0,0 +1,40 @@ +## ADDED Requirements + +### Requirement: Wedged SQLite session teardown is bounded and reclaimed + +Session teardown (rollback and close) on file-backed SQLite MUST complete within a hard deadline derived from the busy timeout while remaining shielded from the caller's cancellation. A teardown that misses the deadline MUST NOT be merely abandoned — the aiosqlite worker thread would keep holding the writer slot — it MUST be reclaimed: the driver connection is interrupted to abort the call the worker is stuck in, and the connection is invalidated so the worker is disposed, the underlying `sqlite3` connection is hard-closed releasing the writer slot, and the connection can never be handed out again. The reclaim MUST be reported with the long-write watchdog's identifiers where available (held duration, owning task, first and last write statements), including identifiers the watchdog already deferred into its pending report. A session whose teardown was abandoned MUST be fenced from further teardown attempts, the abandoned work finishing late MUST NOT surface unretrieved errors, and the deferred bookkeeping close MUST be owned until completion (drained at database shutdown, never fire-and-forget). PostgreSQL teardown semantics MUST remain unchanged, and in-memory SQLite — whose single shared connection is the entire database and cannot starve other writers — MUST keep the unbounded teardown and never be reclaimed. + +#### Scenario: A wedged rollback no longer starves every other writer + +- **GIVEN** a session holding an open SQLite write transaction whose rollback wedges during teardown +- **WHEN** the teardown deadline passes +- **THEN** teardown returns, the connection is interrupted and invalidated, and another writer — such as the leader-election `scheduler_leader` INSERT — acquires the writer slot immediately instead of surfacing `database is locked` + +#### Scenario: The reclaim is attributed with the watchdog's identifiers + +- **GIVEN** a wedged teardown whose transaction ran write statements tracked by the long-write watchdog +- **WHEN** the connection is reclaimed +- **THEN** the report names the held duration, owning task, and first/last write statements, even though invalidation prevents the watchdog's own deferred report from firing + +#### Scenario: A wedged session cannot be driven concurrently + +- **GIVEN** a session whose teardown was abandoned as wedged +- **WHEN** teardown is attempted again +- **THEN** it returns immediately, and the session is closed for bookkeeping only after the abandoned teardown finishes late + +#### Scenario: PostgreSQL teardown is untouched + +- **GIVEN** a session bound to a non-SQLite dialect +- **WHEN** its rollback or close outlives the SQLite deadline +- **THEN** the teardown still awaits completion unboundedly and no connection is reclaimed + +#### Scenario: The shared in-memory SQLite connection is never reclaimed + +- **GIVEN** a session bound to an in-memory SQLite database, whose one shared connection is the entire database +- **WHEN** its teardown outlives the deadline +- **THEN** the teardown still awaits completion unboundedly and the connection is never invalidated, preserving schema and data for later sessions + +#### Scenario: The bound never abandons healthy teardown + +- **WHEN** rollback and close complete within the deadline +- **THEN** teardown behaves exactly as before, including re-raising the completed call's exception to the existing swallow points diff --git a/openspec/changes/bound-sqlite-wedged-teardown/tasks.md b/openspec/changes/bound-sqlite-wedged-teardown/tasks.md new file mode 100644 index 0000000000..0ccedd5515 --- /dev/null +++ b/openspec/changes/bound-sqlite-wedged-teardown/tasks.md @@ -0,0 +1,14 @@ +## 1. Bounded teardown and reclaim + +- [x] 1.1 Bound the shielded rollback/close teardown for file-backed SQLite sessions with a deadline derived from the busy timeout, preserving the shield against caller cancellation; PostgreSQL and in-memory SQLite (one shared StaticPool connection is the whole database — reclaim would destroy it, and it cannot starve other writers) keep the unbounded path +- [x] 1.2 On a missed deadline, interrupt the driver connection and invalidate it so the aiosqlite worker is disposed, the writer slot is released, and the connection can never be handed out again; report the reclaim with the long-write watchdog's identifiers (including ones already deferred into its pending report) +- [x] 1.3 Fence the wedged session against further teardown, consume the abandoned task's late failure, and close the session for bookkeeping once the abandoned teardown finishes; own the deferred close until completion and drain it at close_db so shutdown cannot abandon it + +## 2. Tests + +- [x] 2.1 A wedged sqlite rollback: close_session returns within the bound (fails on pre-fix unbounded teardown), the driver is interrupted, the connection invalidated, the reclaim log carries the watchdog identifiers, and an independent writer succeeds immediately while the wedge is still pending +- [x] 2.2 A wedged session is fenced from further teardown; the abandoned teardown finishing late is observed and followed by the bookkeeping close +- [x] 2.3 The bounded shield completes fast work, abandons at the deadline without cancelling, and absorbs caller cancellation like the unbounded shield +- [x] 2.4 Non-sqlite sessions keep the unbounded teardown: a slow rollback/close beyond the sqlite bound still runs to completion and is never reclaimed +- [x] 2.5 A wedged close without a transaction is bounded and fenced too +- [x] 2.6 In-memory SQLite keeps the unbounded teardown: a slow teardown is never reclaimed, the shared connection is never invalidated, and schema/data survive for later sessions diff --git a/tests/unit/test_db_session.py b/tests/unit/test_db_session.py index 31d24b552b..cd3382a024 100644 --- a/tests/unit/test_db_session.py +++ b/tests/unit/test_db_session.py @@ -13,6 +13,7 @@ import pytest from sqlalchemy import event as sa_event from sqlalchemy import text as sa_text +from sqlalchemy.engine import make_url from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine from sqlalchemy.pool import NullPool @@ -1196,3 +1197,516 @@ def failing_commit(self) -> None: assert "outcome=commit " not in records[0].getMessage() finally: await engine.dispose() + + +@pytest.mark.asyncio +async def test_shielded_bounded_returns_none_when_the_awaitable_finishes_in_time() -> None: + async def _fast() -> str: + return "done" + + assert await session_module._shielded_bounded(_fast(), 1.0) is None + + async def _boom() -> None: + raise RuntimeError("teardown failed") + + with pytest.raises(RuntimeError, match="teardown failed"): + await session_module._shielded_bounded(_boom(), 1.0) + + +@pytest.mark.asyncio +async def test_shielded_bounded_abandons_a_wedged_awaitable_at_the_deadline() -> None: + release = asyncio.Event() + + async def _wedged() -> None: + await release.wait() + + abandoned = await session_module._shielded_bounded(_wedged(), 0.05) + assert abandoned is not None + assert not abandoned.done(), "the wedged awaitable must be left running, not cancelled" + release.set() + await abandoned + + +@pytest.mark.asyncio +async def test_shielded_bounded_absorbs_caller_cancellation_like_the_unbounded_shield() -> None: + """Teardown runs in ``finally`` blocks: the bound, not the caller's + cancellation, must decide abandonment (matching ``_shielded`` + the + swallow in ``_safe_rollback``/``_safe_close``).""" + started = asyncio.Event() + release = asyncio.Event() + finished: list[bool] = [] + + async def _work() -> None: + started.set() + await release.wait() + finished.append(True) + + async def _caller() -> asyncio.Task[object] | None: + return await session_module._shielded_bounded(_work(), 5.0) + + caller = asyncio.ensure_future(_caller()) + await started.wait() + caller.cancel() + await asyncio.sleep(0.05) + assert not caller.done(), "cancellation must not abandon the shielded teardown" + release.set() + assert await caller is None + assert finished, "the shielded work must run to completion despite the cancellation" + + +@pytest.mark.asyncio +async def test_close_session_reclaims_a_wedged_sqlite_rollback_so_other_writers_recover( + tmp_path, monkeypatch, caplog +) -> None: + """Issue #1682 part 2: a wedged rollback used to be awaited forever while + the aiosqlite worker kept the single writer slot — a self-sustaining + 'database is locked' stall that starved leader election itself. The + teardown must be bounded, and the bound alone is not enough: the wedged + connection must be interrupted and invalidated so the writer slot is + actually released and the connection is never handed out again.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.2) + db_path = tmp_path / "wedged-rollback.db" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + # An independent writer with a short busy timeout: if the reclaim fails to + # release the writer slot, its INSERT surfaces 'database is locked' fast. + other_writer = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 1.0}, + ) + release_wedge = asyncio.Event() + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + caplog.clear() + + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + # Take the writer slot with an uncommitted write. + await session.execute(sa_text("DELETE FROM accounts")) + + held = session_module._session_sync_connections(session) + assert held, "the open write transaction must expose its sync connection" + sync_connection = held[0] + driver = sync_connection.connection.driver_connection + assert driver is not None + + # Wedge this connection's rollback (a stuck aiosqlite worker queues + # the teardown behind itself exactly like this) and spy on interrupt. + original_rollback = driver.rollback + interrupted = asyncio.Event() + original_interrupt = driver.interrupt + + async def _wedged_rollback() -> None: + await release_wedge.wait() + await original_rollback() + + # Delegate without changing the installed driver's shape: the reclaim + # awaits ``interrupt()``'s result only when it is awaitable, so the + # spy hands back exactly what the real aiosqlite method returns and + # the production awaitable-handling is exercised against the installed + # contract (a coroutine in the pinned aiosqlite) instead of a stand-in. + def _spying_interrupt() -> object: + interrupted.set() + return original_interrupt() + + driver.rollback = _wedged_rollback + driver.interrupt = _spying_interrupt + + with caplog.at_level(logging.INFO, logger=session_module.__name__): + close_task = asyncio.ensure_future(session_module.close_session(session)) + done, _ = await asyncio.wait({close_task}, timeout=2.0) + # RED on the pre-fix teardown: the shielded rollback was awaited + # unboundedly, so close_session never returned. + assert done, "close_session must be bounded when the sqlite rollback wedges" + + assert interrupted.is_set(), "the wedged driver must be interrupted to unstick its worker" + assert sync_connection.invalidated, "the wedged connection must be invalidated, never reused" + assert session.info.get(session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY) is True + + reclaim_logs = [ + record + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_wedged_teardown" in record.getMessage() + ] + assert reclaim_logs, "the reclaim must be reported with the watchdog's identifiers" + message = reclaim_logs[0].getMessage() + assert "phase=rollback" in message + assert "DELETE FROM accounts" in message, "part 1 watchdog identifiers must attribute the holder" + + # The stall must not be self-sustaining: with the wedged rollback + # still pending, another writer takes the slot immediately. + async with other_writer.begin() as writer: + await writer.execute(sa_text("DELETE FROM accounts")) + + # A wedged session is fenced: further teardown returns immediately + # instead of driving the session concurrently with the abandoned + # greenlet. + await asyncio.wait_for(session_module.close_session(session), timeout=1.0) + + # Late completion: once the wedge resolves, the abandoned teardown + # finishes and the session is closed for bookkeeping. + release_wedge.set() + for _ in range(100): + if any("finished late" in record.getMessage() for record in caplog.records): + break + await asyncio.sleep(0.02) + assert any("finished late" in record.getMessage() for record in caplog.records), ( + "the abandoned teardown must be observed finishing late" + ) + # The deferred bookkeeping close is owned until completion (drained + # by close_db on shutdown), never fire-and-forget. + pending_cleanup = tuple(session_module._wedged_teardown_cleanup_tasks) + if pending_cleanup: + await asyncio.wait_for(asyncio.gather(*pending_cleanup, return_exceptions=True), timeout=2.0) + assert not session_module._wedged_teardown_cleanup_tasks, ( + "the deferred close must deregister itself once it completes" + ) + finally: + release_wedge.set() + await asyncio.sleep(0.05) + await engine.dispose() + await other_writer.dispose() + + +@pytest.mark.asyncio +async def test_close_session_keeps_the_unbounded_shield_for_non_sqlite_sessions(monkeypatch) -> None: + """PostgreSQL teardown semantics are untouched: a slow rollback/close far + beyond the SQLite bound is still awaited to completion, never reclaimed.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.01) + + class _FakeDialect: + name = "postgresql" + + class _FakeBind: + dialect = _FakeDialect() + + class _FakeSession: + def __init__(self) -> None: + self.info: dict[str, object] = {} + self.rolled_back = False + self.closed = False + + def get_bind(self) -> _FakeBind: + return _FakeBind() + + def in_transaction(self) -> bool: + return not self.rolled_back + + async def rollback(self) -> None: + await asyncio.sleep(0.1) + self.rolled_back = True + + async def close(self) -> None: + await asyncio.sleep(0.1) + self.closed = True + + fake = _FakeSession() + await session_module.close_session(cast(session_module.AsyncSession, fake)) + + assert fake.rolled_back, "the slow PostgreSQL rollback must be awaited to completion" + assert fake.closed, "the slow PostgreSQL close must be awaited to completion" + assert session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY not in fake.info + + +@pytest.mark.asyncio +async def test_close_session_bounds_a_wedged_sqlite_close_without_a_transaction(monkeypatch, caplog) -> None: + """The close step can wedge on its own (connection release goes through + the same aiosqlite worker); it must be bounded and fenced too.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.05) + release = asyncio.Event() + + class _FakeDialect: + name = "sqlite" + + class _FakeUrl: + database = "/tmp/wedged-close.db" + query: dict[str, str] = {} + + class _FakeBind: + dialect = _FakeDialect() + url = _FakeUrl() + + class _FakeSyncSession: + def get_transaction(self) -> None: + return None + + class _FakeSession: + def __init__(self) -> None: + self.info: dict[str, object] = {} + self.sync_session = _FakeSyncSession() + + def get_bind(self) -> _FakeBind: + return _FakeBind() + + def in_transaction(self) -> bool: + return False + + async def close(self) -> None: + await release.wait() + + fake = _FakeSession() + try: + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_session(cast(session_module.AsyncSession, fake)), timeout=2.0) + + assert fake.info.get(session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY) is True + messages = [ + record.getMessage() + for record in caplog.records + if record.levelno == logging.WARNING and "sqlite_wedged_teardown" in record.getMessage() + ] + assert messages + assert "phase=close" in messages[0] + finally: + release.set() + await asyncio.sleep(0.05) + + +@pytest.mark.asyncio +async def test_close_session_never_reclaims_the_shared_in_memory_sqlite_connection(monkeypatch) -> None: + """In-memory SQLite shares one StaticPool connection with the whole + process: invalidating it would destroy the entire database (the + database-backends spec preserves shared in-memory state), and a single + shared connection cannot starve other writers. The teardown must keep the + unbounded shield there.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.01) + engine = create_async_engine("sqlite+aiosqlite:///:memory:") + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + assert session_module._session_teardown_bound_seconds(session) is None + + held = session_module._session_sync_connections(session) + assert held + driver = held[0].connection.driver_connection + assert driver is not None + original_rollback = driver.rollback + + async def _slow_rollback() -> None: + await asyncio.sleep(0.1) + await original_rollback() + + driver.rollback = _slow_rollback + await session_module.close_session(session) + driver.rollback = original_rollback + + assert not held[0].invalidated, "the shared in-memory connection must never be invalidated" + assert session_module._SQLITE_TEARDOWN_WEDGED_INFO_KEY not in session.info + + # The database survives: schema and connection are intact. + verify = factory() + (await verify.execute(sa_text("SELECT count(*) FROM accounts"))).scalar_one() + await session_module.close_session(verify) + finally: + await engine.dispose() + + +@pytest.mark.parametrize( + "url_text", + [ + "sqlite+aiosqlite:///:memory:", + "sqlite+aiosqlite://", + "sqlite+aiosqlite:///file:shared?mode=memory&cache=shared&uri=true", + "sqlite+aiosqlite:///file::memory:?cache=shared&uri=true", + ], +) +def test_session_teardown_bound_skips_every_in_memory_sqlite_url_form(url_text: str) -> None: + """Every in-memory SQLite URL form must keep the unbounded teardown: the + SQLite URI forms carry ``mode=memory`` in the parsed URL's query, not in + ``url.database``, and a shared in-memory database reclaimed by invalidation + would be destroyed for the whole process. URI forms count only with + ``uri=true`` — that is what makes the dialect pass the string as a URI.""" + + class _FakeDialect: + name = "sqlite" + + class _FakeBind: + dialect = _FakeDialect() + url = make_url(url_text) + + class _FakeSession: + def get_bind(self) -> _FakeBind: + return _FakeBind() + + fake = _FakeSession() + assert session_module._session_teardown_bound_seconds(cast(session_module.AsyncSession, fake)) is None + + +@pytest.mark.parametrize( + "url_text", + [ + "sqlite+aiosqlite:////data/store.db", + "sqlite+aiosqlite:///file:/data/store.db?uri=true", + # Without ``uri=true`` the dialect never enables SQLite URI mode: this + # connects to a file literally named ``file:shared`` and must keep the + # bounded teardown despite carrying ``mode=memory`` in the query. + "sqlite+aiosqlite:///file:shared?mode=memory&cache=shared", + ], +) +def test_session_teardown_bound_applies_to_file_backed_sqlite_url_forms(url_text: str) -> None: + """File-backed SQLite (plain path, ``file:`` URI without ``mode=memory``, + or a ``mode=memory`` query without ``uri=true``) is exactly the + wedge-prone single-writer case and must stay bounded.""" + + class _FakeDialect: + name = "sqlite" + + class _FakeBind: + dialect = _FakeDialect() + url = make_url(url_text) + + class _FakeSession: + def get_bind(self) -> _FakeBind: + return _FakeBind() + + fake = _FakeSession() + assert ( + session_module._session_teardown_bound_seconds(cast(session_module.AsyncSession, fake)) + == session_module._SQLITE_TEARDOWN_TIMEOUT_SECONDS + ) + + +@pytest.mark.asyncio +async def test_reclaim_interrupts_the_real_aiosqlite_driver_without_a_spy(tmp_path, caplog) -> None: + """The reclaim invokes the driver's real ``interrupt()`` and awaits the + result only when it is awaitable. Exercise the production path against the + installed aiosqlite with no stand-in, so a driver signature change + surfaces as a failure here instead of being swallowed by the reclaim's + broad except (the failure is logged, and asserted absent).""" + engine = create_async_engine( + f"sqlite+aiosqlite:///{tmp_path / 'interrupt-contract.db'}", + poolclass=NullPool, + ) + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + held = session_module._session_sync_connections(session) + assert held, "the open write transaction must expose its sync connection" + + async def _already_finished_teardown() -> None: + return None + + abandoned = asyncio.ensure_future(_already_finished_teardown()) + with caplog.at_level(logging.DEBUG, logger=session_module.__name__): + await session_module._reclaim_wedged_sqlite_session(session, abandoned, held, phase="rollback") + + assert not any( + "Interrupting a wedged SQLite connection failed" in record.getMessage() for record in caplog.records + ), "the installed aiosqlite interrupt() contract must be handled without error" + assert held[0].invalidated, "the reclaim must still invalidate the connection" + + # Drain the bookkeeping the reclaim registered so no task outlives + # the test (mirrors the close_db drain). + for _ in range(100): + pending = tuple(session_module._wedged_teardown_cleanup_tasks) + if not pending: + break + await asyncio.wait_for(asyncio.gather(*pending, return_exceptions=True), timeout=2.0) + await asyncio.sleep(0) + assert not session_module._wedged_teardown_cleanup_tasks + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_close_db_drains_a_pending_reclaimed_rollback_and_its_bookkeeping_close( + tmp_path, monkeypatch, caplog +) -> None: + """A rollback reclaimed as wedged can still be pending when close_db runs. + The abandoned task is registered in the teardown registry immediately, so + close_db must wait for it — and for the bookkeeping close it schedules only + after any one-time snapshot — instead of returning while the event loop + still has pending teardown tasks.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.5) + db_path = tmp_path / "close-db-drain.db" + engine = create_async_engine( + f"sqlite+aiosqlite:///{db_path}", + poolclass=NullPool, + connect_args={"timeout": 5.0}, + ) + session_module._configure_sqlite_engine(engine.sync_engine, enable_wal=True) + release_wedge = asyncio.Event() + try: + async with engine.begin() as connection: + await connection.run_sync(Base.metadata.create_all) + + factory = async_sessionmaker(engine, expire_on_commit=False) + session = factory() + await session.execute(sa_text("DELETE FROM accounts")) + + held = session_module._session_sync_connections(session) + assert held + driver = held[0].connection.driver_connection + assert driver is not None + original_rollback = driver.rollback + + async def _wedged_rollback() -> None: + await release_wedge.wait() + await original_rollback() + + driver.rollback = _wedged_rollback + + with caplog.at_level(logging.INFO, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_session(session), timeout=5.0) + abandoned_pending = [task for task in session_module._wedged_teardown_cleanup_tasks if not task.done()] + # RED pre-fix: the reclaim only registered the deferred bookkeeping + # close (which does not exist yet), never the abandoned rollback. + assert abandoned_pending, "the reclaimed rollback must be registered while still pending" + + async def _release_soon() -> None: + await asyncio.sleep(0.05) + release_wedge.set() + + releaser = asyncio.ensure_future(_release_soon()) + await asyncio.wait_for(session_module.close_db(), timeout=5.0) + # RED pre-fix: close_db saw an empty registry and returned + # immediately, before the wedge was even released. + assert release_wedge.is_set(), "close_db must drain the pending reclaimed rollback" + assert all(task.done() for task in abandoned_pending), ( + "close_db must wait for the abandoned rollback itself" + ) + assert not session_module._wedged_teardown_cleanup_tasks, ( + "close_db must also drain the bookkeeping close scheduled after its first snapshot" + ) + await releaser + + assert any("finished late" in record.getMessage() for record in caplog.records) + finally: + release_wedge.set() + await asyncio.sleep(0.05) + await engine.dispose() + + +@pytest.mark.asyncio +async def test_close_db_bounds_the_wedged_teardown_drain(monkeypatch, caplog) -> None: + """A teardown that stays wedged despite the reclaim (the interrupt is + best-effort) must not wedge shutdown too: the registry drain is explicitly + bounded and abandons whatever remains after the deadline.""" + monkeypatch.setattr(session_module, "_SQLITE_TEARDOWN_TIMEOUT_SECONDS", 0.05) + never = asyncio.Event() + stuck: asyncio.Task[bool] = asyncio.ensure_future(never.wait()) + session_module._wedged_teardown_cleanup_tasks.add(stuck) + try: + with caplog.at_level(logging.WARNING, logger=session_module.__name__): + await asyncio.wait_for(session_module.close_db(), timeout=2.0) + assert any("still-pending wedged-teardown" in record.getMessage() for record in caplog.records), ( + "the bounded drain must report what it abandoned" + ) + assert stuck in session_module._wedged_teardown_cleanup_tasks + finally: + session_module._wedged_teardown_cleanup_tasks.discard(stuck) + never.set() + await stuck From 2e4a580c1c1834f00b6d4c62598caf1ffd0446ee Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:33:28 +0900 Subject: [PATCH 050/117] perf(proxy): disable permessage-deflate on direct-egress upstream websockets (#1786) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The direct-egress upstream websocket transport passed no compression kwarg, so the websockets library default silently negotiated permessage-deflate with upstream — an undocumented library default, not a chosen behavior. Both sibling upstream transports (routed aiohttp, raw handshake) already run uncompressed, and per-frame zlib decode of high-rate upstream event streams burns ~2.6% of profiled CPU on the proxy host. Pass compression=None at the single shared websocket_connect callsite (covers the Responses websocket and the realtime live sideband). The client-facing socket keeps negotiating permessage-deflate per the responses-api-compat downstream ingress requirement; uvicorn ws_per_message_deflate is untouched. OpenSpec: openspec/changes/disable-upstream-websocket-compression/ records the delta (direct-egress upstream sockets do not offer permessage-deflate; client-facing MUST unchanged) and the owner-visible caveats (Codex CLI handshake-fingerprint divergence, shared realtime sideband callsite, WAN ingress bandwidth trade). Co-authored-by: Claude Fable 5 --- app/core/clients/proxy_websocket.py | 7 +++ .../proposal.md | 63 +++++++++++++++++++ .../specs/responses-api-compat/spec.md | 22 +++++++ .../tasks.md | 13 ++++ tests/unit/test_proxy_websocket_client.py | 1 + 5 files changed, 106 insertions(+) create mode 100644 openspec/changes/disable-upstream-websocket-compression/proposal.md create mode 100644 openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/disable-upstream-websocket-compression/tasks.md diff --git a/app/core/clients/proxy_websocket.py b/app/core/clients/proxy_websocket.py index 1a9d6a8f5f..c14ee69c65 100644 --- a/app/core/clients/proxy_websocket.py +++ b/app/core/clients/proxy_websocket.py @@ -934,6 +934,13 @@ async def _connect_upstream_websocket( ping_timeout=ping_timeout, max_size=settings.max_sse_event_bytes, proxy=proxy_url, + # Do not offer permessage-deflate upstream: the websockets library + # enables it by default, but the sibling upstream transports (the + # routed aiohttp path and the raw-handshake transport) already run + # uncompressed, and per-frame zlib decode on high-rate event + # streams burns CPU on the proxy host. The client-facing socket + # keeps negotiating permessage-deflate per responses-api-compat. + compression=None, **subprotocol_kwargs, ) except asyncio.TimeoutError as exc: diff --git a/openspec/changes/disable-upstream-websocket-compression/proposal.md b/openspec/changes/disable-upstream-websocket-compression/proposal.md new file mode 100644 index 0000000000..c21121cdf2 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/proposal.md @@ -0,0 +1,63 @@ +# Proposal: disable-upstream-websocket-compression + +## Why + +The direct-egress upstream websocket transport (`websockets.asyncio.client.connect` in +`app/core/clients/proxy_websocket.py`) passes no `compression` kwarg, so the websockets +library default (`compression="deflate"`) silently offers and negotiates `permessage-deflate` +with the upstream endpoint. No commit, spec, or comment ever chose this: the two sibling +upstream transports already run uncompressed — the routed aiohttp path uses aiohttp's +default `compress=0` (off), and the raw-handshake transport in `app/core/clients/proxy.py` +sets `compress=False`/`compress=0` explicitly. Per-frame zlib decode of high-rate upstream +event streams shows up as a measurable CPU leaf (~2.6% of profiled CPU) on the +single-weak-core proxy host, where CPU — not LAN/WAN bandwidth — is the scarce resource. + +## What Changes + +- Pass `compression=None` at the single direct-egress `websocket_connect(...)` callsite, + so upstream direct-egress websockets (Responses websocket and realtime live sideband, + which share the callsite) no longer offer `permessage-deflate` in the handshake. +- The client-facing socket is untouched: the existing normative requirement that the + server MUST continue to negotiate `permessage-deflate` on the client-facing websocket + (responses-api-compat, downstream ingress budget requirement) is unchanged, and uvicorn's + `ws_per_message_deflate` default stays enabled. +- The routed aiohttp path and raw-handshake transport are untouched (already uncompressed). + +## Owner-visible caveats + +- **Codex CLI fingerprint divergence**: codex-lb impersonates the Codex CLI persona on + the upstream handshake, and the native Codex CLI (tungstenite with + `DeflateConfig::default()`) DOES offer `permessage-deflate`. Dropping the + `Sec-WebSocket-Extensions` offer makes the direct path's handshake differ from the + native client. Mitigating evidence: the routed aiohttp path and the raw-handshake path + already send no extension offer today and are accepted in production, so extension + parity is not currently maintained anywhere. +- **Realtime live sideband shares the callsite**: the change applies to both + `_RESPONSES_WEBSOCKET_POLICY` and `_LIVE_SIDEBAND_WEBSOCKET_POLICY` sockets. Live + sideband frames (base64 audio) compress poorly anyway; if per-policy compression is + ever wanted, the kwarg can be lifted into `_UpstreamWebSocketPolicy`. +- **WAN ingress bandwidth** from the upstream increases (JSON event streams compress + ~4-8x); this is a cost trade only, not a correctness change. +- **Falsification test**: if a post-deploy profile still shows the `permessage_deflate` + decode leaf, the cost was downstream uvicorn decode (spec-protected, must keep) and + this change is a CPU no-op; revert is one line. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: adds a requirement that direct-egress upstream websockets do + not offer `permessage-deflate`. The client-facing `permessage-deflate` MUST is + unchanged. + +## Impact + +- `app/core/clients/proxy_websocket.py`: one kwarg (`compression=None`) on the shared + direct-egress `websocket_connect` call; persona headers, subprotocols, ping-timeout + watchdog, max_size, and proxy resolution are unchanged. +- `tests/unit/test_proxy_websocket_client.py`: kwargs assertion pins + `compression is None` on the direct transport contract. diff --git a/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md b/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..63af95bfd7 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/specs/responses-api-compat/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Direct-egress upstream websockets do not offer permessage-deflate +When codex-lb opens a direct-egress upstream websocket (the Responses websocket or the +realtime live sideband over the `websockets` transport, used when no upstream proxy route +applies), it MUST NOT offer the `permessage-deflate` extension in the upstream handshake, +matching the routed and raw-handshake upstream transports, which already run uncompressed. +This requirement applies only to the proxy-to-upstream link: the server MUST continue to +negotiate `permessage-deflate` on the client-facing websocket, as required by the +downstream websocket ingress requirement. + +#### Scenario: Direct upstream handshake omits the compression extension offer + +- **WHEN** codex-lb connects an upstream websocket via the direct-egress `websockets` transport +- **THEN** the handshake does not offer `permessage-deflate` (the transport is invoked with compression disabled) +- **AND** persona headers, subprotocols, open-timeout, ping-timeout, message-size cap, and proxy resolution are unchanged + +#### Scenario: Client-facing compression negotiation is unchanged + +- **WHEN** a client connects to a Responses websocket route offering `permessage-deflate` +- **THEN** the server still negotiates `permessage-deflate` on the client-facing socket +- **AND** the downstream ingress budget continues to apply to the decompressed message size diff --git a/openspec/changes/disable-upstream-websocket-compression/tasks.md b/openspec/changes/disable-upstream-websocket-compression/tasks.md new file mode 100644 index 0000000000..1db495f828 --- /dev/null +++ b/openspec/changes/disable-upstream-websocket-compression/tasks.md @@ -0,0 +1,13 @@ +## 1. Implementation + +- [x] 1.1 Pass `compression=None` at the direct-egress `websocket_connect` callsite in + `app/core/clients/proxy_websocket.py` so upstream direct-egress sockets stop + offering `permessage-deflate`. Leave the routed aiohttp path, raw-handshake + transport, and downstream uvicorn `ws_per_message_deflate` untouched. + +## 2. Validation + +- [x] 2.1 Extend the direct-transport kwargs assertions in + `tests/unit/test_proxy_websocket_client.py` with `compression is None`. +- [x] 2.2 Run the proxy websocket client unit suite, lint, and strict OpenSpec + validation. diff --git a/tests/unit/test_proxy_websocket_client.py b/tests/unit/test_proxy_websocket_client.py index b0079f87c4..547eb7aa0e 100644 --- a/tests/unit/test_proxy_websocket_client.py +++ b/tests/unit/test_proxy_websocket_client.py @@ -416,6 +416,7 @@ async def fake_websocket_connect(url: str, **kwargs): assert "ping_interval" not in kwargs assert kwargs["ping_timeout"] == 120.0 assert kwargs["max_size"] == 4321 + assert kwargs["compression"] is None assert "subprotocols" not in kwargs additional_headers = cast(dict[str, str], kwargs["additional_headers"]) assert additional_headers["Authorization"] == "Bearer access-token" From 94057ccf91a7ed9f32eb15dcfa0487c969dc2a83 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:33:31 +0900 Subject: [PATCH 051/117] perf(middleware): convert BaseHTTPMiddleware layers to pure ASGI (#1787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(middleware): convert app_version, request_id, capability, firewall layers to pure ASGI Replace four @app.middleware("http") BaseHTTPMiddleware dispatch functions with pure ASGI classes registered at identical positions. Each BaseHTTPMiddleware layer builds a _CachedRequest, an anyio Event, an anyio memory object stream, and a task group per request, and every response body chunk crosses the memory stream with an extra task switch — on the dominant SSE path each converted layer removes one per-chunk hop. Behavior preserved: - app_version: X-App-Version appended via send-wrapper on 200-499 http.response.start only, route-owned values preserved (setdefault), 5xx/WebSocket excluded per api-response-metadata spec. - request_id: same inbound header precedence (x-request-id then request-id), contextvars now set/reset in the task running the downstream app, so they stay bound through response streaming (strictly wider visibility than the BaseHTTP dispatch window); x-request-id response header setdefault kept. - required_capability_http: cheap sync guards first (scope type, POST, header-bytes scan, deny-path check); Request built only on the cold deny path; identical openai_error envelope and images observability. - api_firewall: prefix check first, TTL cache before DB, identical 403 envelope; settings still captured at registration time. Also hoist InFlightMiddleware's per-request import_module('app.core.shutdown') into a lazily-resolved cached attribute (kept lazy for import-cycle safety). Co-Authored-By: Claude Fable 5 * perf(middleware): convert request_decompression to pure ASGI with replay receive Replace the last @app.middleware("http") BaseHTTPMiddleware layer with a pure ASGI class at the identical registration position. Requests without Content-Encoding (the common case) now pass straight through with a single header scan instead of paying BaseHTTPMiddleware's per-request task group, anyio memory stream, and per-chunk stream hop on every response. Encoded requests are drained through the upstream receive — which is the body-limit middleware's limited receive, so the wire-size cap and its _RequestBodyTooLarge propagation are unchanged (body_limit registers outside decompression) — then decompressed under the same per-path budget with identical 413/400 envelopes. The downstream app receives a replay receive that yields the decompressed body once and then delegates to the original receive, replacing the starlette _CachedRequest body replay: chunked uploads coalesce as before, a mid-body http.disconnect raises ClientDisconnect as Request.body() did, and post-body receives (e.g. StreamingResponse's disconnect listener during SSE) still observe http.disconnect. The multipart content-encoding gate flow is untouched: the multipart layer registers outside body_limit/decompression and strips content-encoding for its routes, so decompression keeps passing those through. Tests: rewrite the dispatch-extraction unit tests against the ASGI class, update the two production-order assertions to match the new class, and add chunked-compressed-upload plus client-disconnect-mid-SSE regressions (the BaseHTTP receive_or_disconnect short-circuit no longer exists). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/middleware/api_firewall.py | 71 ++++--- app/core/middleware/app_version.py | 43 +++-- app/core/middleware/inflight.py | 8 +- app/core/middleware/request_decompression.py | 118 ++++++++---- app/core/middleware/request_id.py | 63 +++++-- .../middleware/required_capability_http.py | 56 ++++-- tests/unit/test_app_version_middleware.py | 105 +++-------- ...t_multipart_content_encoding_middleware.py | 6 +- .../test_request_body_limit_middleware.py | 10 +- .../test_request_decompression_middleware.py | 178 ++++++++++++------ tests/unit/test_request_id_middleware.py | 130 ++++++------- 11 files changed, 469 insertions(+), 319 deletions(-) diff --git a/app/core/middleware/api_firewall.py b/app/core/middleware/api_firewall.py index 7c3df53c24..6e57d1a6df 100644 --- a/app/core/middleware/api_firewall.py +++ b/app/core/middleware/api_firewall.py @@ -1,15 +1,16 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable from ipaddress import IPv4Network, IPv6Network from typing import cast from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response +from fastapi.responses import JSONResponse +from starlette.datastructures import Headers +from starlette.types import ASGIApp, Receive, Scope, Send from app.core.config.settings import get_settings from app.core.errors import openai_error -from app.core.middleware.firewall_cache import get_firewall_ip_cache +from app.core.middleware.firewall_cache import FirewallIPCache, get_firewall_ip_cache from app.core.request_locality import ( FORWARDED_CHAIN_HEADER_NAMES, parse_trusted_proxy_networks, @@ -20,27 +21,41 @@ from app.modules.firewall.service import FirewallRepositoryPort, FirewallService -def add_api_firewall_middleware(app: FastAPI) -> None: - settings = get_settings() - trusted_proxy_networks = parse_trusted_proxy_networks(settings.firewall_trusted_proxy_cidrs) - firewall_cache = get_firewall_ip_cache() - - @app.middleware("http") - async def api_firewall_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - path = request.url.path - if not _is_protected_api_path(path): - return await call_next(request) +class ApiFirewallMiddleware: + """IP allowlist for ``/v1/*`` and ``/backend-api/codex/*`` HTTP requests. + + Pure ASGI: unprotected paths pass through with a single prefix check, and + allow decisions are answered from the in-memory TTL cache; the database is + only consulted on a cache miss. + """ + + def __init__( + self, + app: ASGIApp, + *, + trust_proxy_headers: bool, + trusted_proxy_networks: tuple[IPv4Network | IPv6Network, ...], + firewall_cache: FirewallIPCache, + ) -> None: + self.app = app + self._trust_proxy_headers = trust_proxy_headers + self._trusted_proxy_networks = trusted_proxy_networks + self._firewall_cache = firewall_cache + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http" or not _is_protected_api_path(scope["path"]): + await self.app(scope, receive, send) + return + client = scope.get("client") client_ip = resolve_connection_client_ip( - request.headers, - request.client.host if request.client else None, - trust_proxy_headers=settings.firewall_trust_proxy_headers, - trusted_proxy_networks=trusted_proxy_networks, + Headers(scope=scope), + client[0] if client else None, + trust_proxy_headers=self._trust_proxy_headers, + trusted_proxy_networks=self._trusted_proxy_networks, allowed_proxy_header_names=FORWARDED_CHAIN_HEADER_NAMES, ) + firewall_cache = self._firewall_cache cached_decision = await firewall_cache.is_allowed(client_ip) if client_ip is not None else None if cached_decision is not None: is_allowed = cached_decision @@ -54,12 +69,24 @@ async def api_firewall_middleware( await firewall_cache.set(client_ip, is_allowed, if_version=version_before_read) if is_allowed: - return await call_next(request) + await self.app(scope, receive, send) + return - return JSONResponse( + response = JSONResponse( status_code=403, content=openai_error("ip_forbidden", "Access denied for client IP", error_type="access_error"), ) + await response(scope, receive, send) + + +def add_api_firewall_middleware(app: FastAPI) -> None: + settings = get_settings() + app.add_middleware( + ApiFirewallMiddleware, + trust_proxy_headers=settings.firewall_trust_proxy_headers, + trusted_proxy_networks=parse_trusted_proxy_networks(settings.firewall_trusted_proxy_cidrs), + firewall_cache=get_firewall_ip_cache(), + ) def _is_protected_api_path(path: str) -> bool: diff --git a/app/core/middleware/app_version.py b/app/core/middleware/app_version.py index bffd195703..a20b96afbe 100644 --- a/app/core/middleware/app_version.py +++ b/app/core/middleware/app_version.py @@ -1,20 +1,37 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable - -from fastapi import FastAPI, Request -from fastapi.responses import Response +from fastapi import FastAPI +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app import __version__ +class AppVersionMiddleware: + """Append ``X-App-Version`` to every 200-499 HTTP response. + + Pure ASGI (no ``BaseHTTPMiddleware``) so streaming responses relay chunks + without an extra task and memory-stream hop. 5xx responses and WebSocket + scopes never carry the header, and a route-owned ``X-App-Version`` value is + preserved (see ``openspec/specs/api-response-metadata/spec.md``). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + async def send_with_app_version(message: Message) -> None: + if message["type"] == "http.response.start" and 200 <= message["status"] < 500: + headers = MutableHeaders(raw=message.setdefault("headers", [])) + headers.setdefault("X-App-Version", __version__) + await send(message) + + await self.app(scope, receive, send_with_app_version) + + def add_app_version_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def app_version_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - response = await call_next(request) - if 200 <= response.status_code < 500: - response.headers.setdefault("X-App-Version", __version__) - return response + app.add_middleware(AppVersionMiddleware) diff --git a/app/core/middleware/inflight.py b/app/core/middleware/inflight.py index c324ac7797..6f8faa3cdd 100644 --- a/app/core/middleware/inflight.py +++ b/app/core/middleware/inflight.py @@ -1,6 +1,7 @@ from __future__ import annotations from importlib import import_module +from types import ModuleType from starlette.responses import JSONResponse from starlette.types import ASGIApp, Receive, Scope, Send @@ -37,6 +38,9 @@ class InFlightMiddleware: def __init__(self, app: ASGIApp) -> None: self.app = app + # Resolved lazily on the first request (import-cycle safety) and cached + # so the hot path skips the per-request sys.modules lookup. + self._shutdown_state: ModuleType | None = None async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: scope_type = scope["type"] @@ -44,7 +48,9 @@ async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: await self.app(scope, receive, send) return - shutdown_state = import_module("app.core.shutdown") + shutdown_state = self._shutdown_state + if shutdown_state is None: + shutdown_state = self._shutdown_state = import_module("app.core.shutdown") if scope_type == "websocket": # Register before checking the drain barrier. A synchronous signal diff --git a/app/core/middleware/request_decompression.py b/app/core/middleware/request_decompression.py index 99b63aa503..7e8c4472b5 100644 --- a/app/core/middleware/request_decompression.py +++ b/app/core/middleware/request_decompression.py @@ -3,14 +3,14 @@ import gzip import io import zlib -from collections.abc import Awaitable, Callable from typing import Protocol import zstandard as zstd -from fastapi import FastAPI, Request -from fastapi.responses import Response +from fastapi import FastAPI from starlette._utils import get_route_path -from starlette.requests import ClientDisconnect +from starlette.datastructures import Headers +from starlette.requests import ClientDisconnect, Request +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app.core.middleware.request_body_limit import ( REQUEST_BODY_TOO_LARGE_MESSAGE, @@ -108,58 +108,110 @@ def _decompress_body(data: bytes, encodings: list[str], max_size: int) -> bytes: return result -def _replace_request_body(request: Request, body: bytes) -> None: - request._body = body +def _rewrite_scope_headers_for_body(scope: Scope, body_length: int) -> None: + """Drop content-encoding/content-length and declare the decompressed length.""" headers: list[tuple[bytes, bytes]] = [] - for key, value in request.scope.get("headers", []): + for key, value in scope.get("headers", []): if key.lower() in (b"content-encoding", b"content-length"): continue headers.append((key, value)) - headers.append((b"content-length", str(len(body)).encode("ascii"))) - request.scope["headers"] = headers - # Ensure subsequent request.headers reflects the updated scope headers. - request.__dict__.pop("_headers", None) + headers.append((b"content-length", str(body_length).encode("ascii"))) + scope["headers"] = headers -def add_request_decompression_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def request_decompression_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - content_encoding = request.headers.get("content-encoding") +async def _drain_request_body(receive: Receive) -> bytes: + """Read the full request body from ``receive``. + + Mirrors ``Request.body()`` semantics: a mid-body ``http.disconnect`` raises + ``ClientDisconnect``, and receive failures propagate. The caller's + ``receive`` is the body-limit middleware's limited receive, so the wire-size + cap (``_RequestBodyTooLarge``) propagates through this drain unchanged. + """ + chunks = bytearray() + while True: + message = await receive() + if message["type"] == "http.disconnect": + raise ClientDisconnect() + chunks.extend(message.get("body", b"")) + if not message.get("more_body", False): + return bytes(chunks) + + +class RequestDecompressionMiddleware: + """Decompress zstd/gzip/deflate request bodies with per-layer decode budgets. + + Pure ASGI replacement for the previous ``BaseHTTPMiddleware`` dispatch: + requests without ``Content-Encoding`` (the common case) pass straight + through. Encoded requests are drained through the upstream receive (the + body-limit middleware's limited receive, so the wire-size cap still applies + to the compressed bytes), decompressed under the per-path budget, and the + downstream app is given a replay receive that yields the decompressed body + once and then delegates to the original receive so ``http.disconnect`` is + still observed (this replaces the ``_CachedRequest`` body replay the + BaseHTTP wrapper used to provide). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + content_encoding = Headers(scope=scope).get("content-encoding") if not content_encoding: - return await call_next(request) + await self.app(scope, receive, send) + return encodings = [enc.strip().lower() for enc in content_encoding.split(",") if enc.strip()] if not encodings: - return await call_next(request) - max_size = request_body_limit_for_path(get_route_path(request.scope)) - try: - body = await request.body() - except ClientDisconnect: - raise + await self.app(scope, receive, send) + return + + max_size = request_body_limit_for_path(get_route_path(scope)) + body = await _drain_request_body(receive) try: decompressed = _decompress_body(body, encodings, max_size) except _DecompressedBodyTooLarge: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=413, code="payload_too_large", message=REQUEST_BODY_TOO_LARGE_MESSAGE, ) + await response(scope, receive, send) + return except ValueError: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=400, code="invalid_request", message="Unsupported Content-Encoding", ) + await response(scope, receive, send) + return except Exception: - return request_ingress_error_response( - request, + response = request_ingress_error_response( + Request(scope), status_code=400, code="invalid_request", message="Request body is compressed but could not be decompressed", ) - _replace_request_body(request, decompressed) - return await call_next(request) + await response(scope, receive, send) + return + + _rewrite_scope_headers_for_body(scope, len(decompressed)) + body_replayed = False + + async def replay_receive() -> Message: + nonlocal body_replayed + if not body_replayed: + body_replayed = True + return {"type": "http.request", "body": decompressed, "more_body": False} + return await receive() + + await self.app(scope, replay_receive, send) + + +def add_request_decompression_middleware(app: FastAPI) -> None: + app.add_middleware(RequestDecompressionMiddleware) diff --git a/app/core/middleware/request_id.py b/app/core/middleware/request_id.py index 1ee1af006a..d7ba92820a 100644 --- a/app/core/middleware/request_id.py +++ b/app/core/middleware/request_id.py @@ -1,10 +1,10 @@ from __future__ import annotations -from collections.abc import Awaitable, Callable from uuid import uuid4 -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse +from fastapi import FastAPI +from starlette.datastructures import MutableHeaders +from starlette.types import ASGIApp, Message, Receive, Scope, Send from app.core.utils.request_id import ( clear_request_id, @@ -16,22 +16,57 @@ ) -def add_request_id_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def request_id_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[JSONResponse]], - ) -> JSONResponse: - inbound_request_id = request.headers.get("x-request-id") or request.headers.get("request-id") - request_id = inbound_request_id or str(uuid4()) +def _inbound_request_id(scope: Scope) -> str | None: + """Return the first ``x-request-id`` value, else the first ``request-id`` value.""" + x_request_id: bytes | None = None + request_id: bytes | None = None + for name, value in scope.get("headers", []): + lowered = name.lower() + if x_request_id is None and lowered == b"x-request-id": + x_request_id = value + elif request_id is None and lowered == b"request-id": + request_id = value + inbound = x_request_id or request_id + if not inbound: + return None + return inbound.decode("latin-1") + + +class RequestIdMiddleware: + """Bind request-id contextvars around the request and echo ``x-request-id``. + + Pure ASGI: the contextvars are set and reset in the same task that runs the + downstream app, so they stay visible for the whole response stream (with + ``BaseHTTPMiddleware`` they were reset when the dispatch returned, before + the body finished streaming). + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + request_id = _inbound_request_id(scope) or str(uuid4()) request_id_token = set_request_id(request_id) request_scope_token = set_request_scope_id(str(uuid4())) + + async def send_with_request_id(message: Message) -> None: + if message["type"] == "http.response.start": + headers = MutableHeaders(raw=message.setdefault("headers", [])) + headers.setdefault("x-request-id", request_id) + await send(message) + try: - response = await call_next(request) - response.headers.setdefault("x-request-id", request_id) - return response + await self.app(scope, receive, send_with_request_id) finally: reset_request_scope_id(request_scope_token) reset_request_id(request_id_token) clear_request_scope_id() clear_request_id() + + +def add_request_id_middleware(app: FastAPI) -> None: + app.add_middleware(RequestIdMiddleware) diff --git a/app/core/middleware/required_capability_http.py b/app/core/middleware/required_capability_http.py index 5bb8a0f1f9..36ca7bffda 100644 --- a/app/core/middleware/required_capability_http.py +++ b/app/core/middleware/required_capability_http.py @@ -2,11 +2,11 @@ import logging import time -from collections.abc import Awaitable, Callable from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response +from fastapi.responses import JSONResponse from starlette._utils import get_route_path +from starlette.types import ASGIApp, Receive, Scope, Send from app.core.auth.dependencies import validate_required_proxy_api_key_authorization from app.core.clients.proxy import CODEX_LB_REQUIRED_CAPABILITY_HEADER @@ -20,6 +20,8 @@ logger = logging.getLogger(__name__) +_REQUIRED_CAPABILITY_HEADER_BYTES = CODEX_LB_REQUIRED_CAPABILITY_HEADER.lower().encode("latin-1") + _JSON_BODY_DENY_PATHS = frozenset( { "/backend-api/codex/responses", @@ -41,23 +43,45 @@ def _is_pre_body_deny_path(path: str) -> bool: return normalized in _JSON_BODY_DENY_PATHS or normalized.startswith("/v1/warmup/") -def add_required_capability_http_middleware(app: FastAPI) -> None: - @app.middleware("http") - async def required_capability_http_middleware( - request: Request, - call_next: Callable[[Request], Awaitable[Response]], - ) -> Response: - if request.method != "POST": - return await call_next(request) - if not request.headers.getlist(CODEX_LB_REQUIRED_CAPABILITY_HEADER): - return await call_next(request) - if not _is_pre_body_deny_path(get_route_path(request.scope)): - return await call_next(request) +def _has_required_capability_header(scope: Scope) -> bool: + for name, _value in scope.get("headers", []): + if name.lower() == _REQUIRED_CAPABILITY_HEADER_BYTES: + return True + return False + + +class RequiredCapabilityHttpMiddleware: + """Deny capability-marked POSTs on JSON-body proxy paths before the body is read. + + Pure ASGI with cheap synchronous guards first; the ``Request`` object is + only constructed on the cold deny path. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if ( + scope["type"] != "http" + or scope["method"] != "POST" + or not _has_required_capability_header(scope) + or not _is_pre_body_deny_path(get_route_path(scope)) + ): + await self.app(scope, receive, send) + return + + request = Request(scope) try: await validate_required_proxy_api_key_authorization(request.headers.get("authorization")) except ProxyAuthError as exc: - return _capability_error_response(request, exc) - return _capability_error_response(request, ProxyRequiredCapabilityTransportError()) + response = _capability_error_response(request, exc) + else: + response = _capability_error_response(request, ProxyRequiredCapabilityTransportError()) + await response(scope, receive, send) + + +def add_required_capability_http_middleware(app: FastAPI) -> None: + app.add_middleware(RequiredCapabilityHttpMiddleware) def _capability_error_response( diff --git a/tests/unit/test_app_version_middleware.py b/tests/unit/test_app_version_middleware.py index 80d2075f92..e45f7ccafe 100644 --- a/tests/unit/test_app_version_middleware.py +++ b/tests/unit/test_app_version_middleware.py @@ -1,14 +1,10 @@ from __future__ import annotations import asyncio -from collections.abc import Awaitable, Callable -from typing import cast import pytest -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response +from fastapi import FastAPI, Response from httpx import ASGITransport, AsyncClient -from starlette.types import Message import app.main as main from app import __version__ @@ -17,98 +13,43 @@ pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] - -@pytest.mark.asyncio -async def test_app_version_middleware_adds_header_to_2xx_response(): +def _build_app(status_code: int, *, headers: dict[str, str] | None = None) -> FastAPI: app = FastAPI() add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - async def call_next(_: Request) -> JSONResponse: - return JSONResponse({"ok": True}, status_code=204) + @app.get("/probe") + async def probe() -> Response: + return Response(status_code=status_code, headers=headers) + + return app + - response = await dispatch(request, call_next) +@pytest.mark.asyncio +async def test_app_version_middleware_adds_header_to_2xx_response(): + transport = ASGITransport(app=_build_app(204)) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") + assert response.status_code == 204 assert response.headers["X-App-Version"] == __version__ @pytest.mark.asyncio async def test_app_version_middleware_skips_header_on_5xx_response(): - app = FastAPI() - add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> JSONResponse: - return JSONResponse({"error": "boom"}, status_code=503) - - response = await dispatch(request, call_next) + transport = ASGITransport(app=_build_app(503)) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") + assert response.status_code == 503 assert "X-App-Version" not in response.headers @pytest.mark.asyncio async def test_app_version_middleware_preserves_existing_header_value(): - app = FastAPI() - add_app_version_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> Response: - return Response(status_code=200, headers={"X-App-Version": "route-owned-version"}) - - response = await dispatch(request, call_next) + transport = ASGITransport(app=_build_app(200, headers={"X-App-Version": "route-owned-version"})) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/probe") assert response.headers["X-App-Version"] == "route-owned-version" @@ -145,7 +86,3 @@ async def work(): assert overloaded.status_code == 429 assert overloaded.headers["X-App-Version"] == __version__ - - -async def _empty_receive() -> Message: - return {"type": "http.request", "body": b"", "more_body": False} diff --git a/tests/unit/test_multipart_content_encoding_middleware.py b/tests/unit/test_multipart_content_encoding_middleware.py index d427ae557e..5fac0d9889 100644 --- a/tests/unit/test_multipart_content_encoding_middleware.py +++ b/tests/unit/test_multipart_content_encoding_middleware.py @@ -6,7 +6,6 @@ import pytest from httpx import ASGITransport, AsyncByteStream, AsyncClient from starlette.datastructures import Headers -from starlette.middleware.base import BaseHTTPMiddleware from starlette.requests import Request from starlette.types import Message, Scope @@ -22,6 +21,7 @@ ) from app.core.middleware.path_rewrite import BackendApiCodexV1AliasMiddleware from app.core.middleware.request_body_limit import RequestBodyLimitMiddleware +from app.core.middleware.request_decompression import RequestDecompressionMiddleware from app.main import create_app pytestmark = pytest.mark.unit @@ -360,9 +360,7 @@ def test_production_middleware_order_composes_route_and_generic_ingress_guards() ) limit_index = next(index for index, item in enumerate(middleware) if item.cls is RequestBodyLimitMiddleware) decompression_index = next( - index - for index, item in enumerate(middleware) - if item.cls is BaseHTTPMiddleware and item.kwargs.get("dispatch").__name__ == "request_decompression_middleware" + index for index, item in enumerate(middleware) if item.cls is RequestDecompressionMiddleware ) assert alias_index < multipart_index < limit_index < decompression_index diff --git a/tests/unit/test_request_body_limit_middleware.py b/tests/unit/test_request_body_limit_middleware.py index cc6956b897..3206c9878a 100644 --- a/tests/unit/test_request_body_limit_middleware.py +++ b/tests/unit/test_request_body_limit_middleware.py @@ -8,14 +8,16 @@ import pytest from fastapi import Body, Depends, FastAPI, HTTPException from httpx import ASGITransport, AsyncByteStream, AsyncClient -from starlette.middleware.base import BaseHTTPMiddleware from starlette.types import Message, Receive, Scope, Send from app.core.config.settings import get_settings from app.core.handlers import add_exception_handlers from app.core.middleware.path_rewrite import BackendApiCodexV1AliasMiddleware from app.core.middleware.request_body_limit import RequestBodyLimitMiddleware, add_request_body_limit_middleware -from app.core.middleware.request_decompression import add_request_decompression_middleware +from app.core.middleware.request_decompression import ( + RequestDecompressionMiddleware, + add_request_decompression_middleware, +) from app.main import create_app pytestmark = pytest.mark.unit @@ -495,9 +497,7 @@ def test_production_middleware_order_keeps_alias_and_admission_outside_body_read alias_index = next(index for index, item in enumerate(middleware) if item.cls is BackendApiCodexV1AliasMiddleware) limit_index = next(index for index, item in enumerate(middleware) if item.cls is RequestBodyLimitMiddleware) decompression_index = next( - index - for index, item in enumerate(middleware) - if item.cls is BaseHTTPMiddleware and item.kwargs.get("dispatch").__name__ == "request_decompression_middleware" + index for index, item in enumerate(middleware) if item.cls is RequestDecompressionMiddleware ) assert alias_index < limit_index < decompression_index diff --git a/tests/unit/test_request_decompression_middleware.py b/tests/unit/test_request_decompression_middleware.py index e1240a28c5..e5978d3a25 100644 --- a/tests/unit/test_request_decompression_middleware.py +++ b/tests/unit/test_request_decompression_middleware.py @@ -1,25 +1,27 @@ from __future__ import annotations +import asyncio import gzip import json import zlib -from collections.abc import Awaitable, Callable -from typing import cast +from collections.abc import AsyncIterator import pytest import zstandard as zstd from fastapi import FastAPI, Request -from fastapi.responses import Response -from httpx import ASGITransport, AsyncClient +from fastapi.responses import StreamingResponse +from httpx import ASGITransport, AsyncByteStream, AsyncClient from starlette.requests import ClientDisconnect +from starlette.types import Message, Receive, Scope, Send from app.core.middleware.request_body_limit import add_request_body_limit_middleware -from app.core.middleware.request_decompression import add_request_decompression_middleware +from app.core.middleware.request_decompression import ( + RequestDecompressionMiddleware, + add_request_decompression_middleware, +) pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] - def _build_echo_app(*, touch_headers: bool = False) -> FastAPI: app = FastAPI() @@ -442,67 +444,131 @@ async def test_request_decompression_keeps_default_limit_for_other_routes(monkey assert response_data["error"]["code"] == "payload_too_large" +def _encoded_post_scope() -> Scope: + return { + "type": "http", + "asgi": {"version": "3.0", "spec_version": "2.3"}, + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": "/echo", + "raw_path": b"/echo", + "query_string": b"", + "root_path": "", + "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], + "client": ("testclient", 50000), + "server": ("testserver", 80), + "state": {}, + } + + +async def _unused_send(message: Message) -> None: + raise AssertionError(f"send should not be reached, got {message['type']}") + + @pytest.mark.asyncio async def test_request_decompression_propagates_client_disconnect(): - app = FastAPI() - add_request_decompression_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) + async def downstream(scope: Scope, receive: Receive, send: Send) -> None: + raise AssertionError("downstream app should not run after client disconnect") - async def receive() -> dict[str, object]: - return {"type": "http.disconnect"} + middleware = RequestDecompressionMiddleware(downstream) - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "POST", - "scheme": "http", - "path": "/echo", - "raw_path": b"/echo", - "query_string": b"", - "root_path": "", - "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=receive, - ) - - async def call_next(_: Request): - raise AssertionError("call_next should not run after client disconnect") + async def receive() -> Message: + return {"type": "http.disconnect"} with pytest.raises(ClientDisconnect): - await dispatch(request, call_next) + await middleware(_encoded_post_scope(), receive, _unused_send) @pytest.mark.asyncio async def test_request_decompression_propagates_body_read_failures(): + async def downstream(scope: Scope, receive: Receive, send: Send) -> None: + raise AssertionError("downstream app should not run when body read fails") + + middleware = RequestDecompressionMiddleware(downstream) + + async def receive() -> Message: + raise RuntimeError("receive failed") + + with pytest.raises(RuntimeError, match="receive failed"): + await middleware(_encoded_post_scope(), receive, _unused_send) + + +class _ChunkedBody(AsyncByteStream): + def __init__(self, *chunks: bytes) -> None: + self._chunks = chunks + + async def __aiter__(self) -> AsyncIterator[bytes]: + for chunk in self._chunks: + yield chunk + + +@pytest.mark.asyncio +async def test_request_decompression_supports_chunked_compressed_upload(): + app = _build_echo_app() + + payload = {"hello": "chunked"} + compressed = gzip.compress(json.dumps(payload).encode("utf-8")) + assert len(compressed) > 10 + + transport = ASGITransport(app=app) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + resp = await client.post( + "/echo", + content=_ChunkedBody(compressed[:10], compressed[10:]), + headers={"Content-Encoding": "gzip", "Content-Type": "application/json"}, + ) + + assert resp.status_code == 200 + response_data = resp.json() + assert response_data["content_encoding"] is None + assert response_data["data"] == payload + + +@pytest.mark.asyncio +async def test_client_disconnect_mid_sse_stops_stream_after_replayed_body(): + """Regression: without BaseHTTPMiddleware's receive wrapper, http.disconnect + must still reach StreamingResponse's disconnect listener through the + decompression middleware's replay receive.""" app = FastAPI() add_request_decompression_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) + add_request_body_limit_middleware(app) - async def receive() -> dict[str, object]: - raise RuntimeError("receive failed") + chunks_seen = asyncio.Event() + chunk_count = 0 - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "POST", - "scheme": "http", - "path": "/echo", - "raw_path": b"/echo", - "query_string": b"", - "root_path": "", - "headers": [(b"content-encoding", b"gzip"), (b"content-type", b"application/json")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=receive, - ) - - async def call_next(_: Request): - raise AssertionError("call_next should not run when body read fails") + @app.post("/stream") + async def stream(request: Request) -> StreamingResponse: + data = await request.json() + assert data == {"hello": "sse"} - with pytest.raises(RuntimeError, match="receive failed"): - await dispatch(request, call_next) + async def event_stream() -> AsyncIterator[bytes]: + while True: + yield b"data: tick\n\n" + await asyncio.sleep(0) + + return StreamingResponse(event_stream(), media_type="text/event-stream") + + compressed = gzip.compress(json.dumps({"hello": "sse"}).encode("utf-8")) + scope = _encoded_post_scope() + scope["path"] = "/stream" + scope["raw_path"] = b"/stream" + + body_messages = iter([{"type": "http.request", "body": compressed, "more_body": False}]) + + async def receive() -> Message: + for message in body_messages: + return message + # Simulate the client hanging up once a few SSE chunks have streamed. + await chunks_seen.wait() + return {"type": "http.disconnect"} + + async def send(message: Message) -> None: + nonlocal chunk_count + if message["type"] == "http.response.body" and message.get("body"): + chunk_count += 1 + if chunk_count >= 3: + chunks_seen.set() + + await asyncio.wait_for(app(scope, receive, send), timeout=5) + assert chunk_count >= 3 diff --git a/tests/unit/test_request_id_middleware.py b/tests/unit/test_request_id_middleware.py index 3a48b4c47b..17f952b615 100644 --- a/tests/unit/test_request_id_middleware.py +++ b/tests/unit/test_request_id_middleware.py @@ -1,98 +1,86 @@ from __future__ import annotations import asyncio -from collections.abc import Awaitable, Callable -from typing import cast import pytest -from fastapi import FastAPI, Request -from fastapi.responses import JSONResponse, Response -from starlette.types import Message +from fastapi import FastAPI +from httpx import ASGITransport, AsyncClient from app.core.middleware.request_id import add_request_id_middleware from app.core.utils.request_id import get_request_id, get_request_scope_id pytestmark = pytest.mark.unit -_Dispatch = Callable[[Request, Callable[[Request], Awaitable[Response]]], Awaitable[Response]] + +def _build_app(request_ids: list[str | None], scope_ids: list[str | None]) -> FastAPI: + app = FastAPI() + add_request_id_middleware(app) + + @app.get("/health") + async def health() -> dict[str, bool]: + request_ids.append(get_request_id()) + scope_ids.append(get_request_scope_id()) + return {"ok": True} + + return app @pytest.mark.asyncio async def test_request_id_middleware_resets_context_on_success(): - app = FastAPI() - add_request_id_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - - request = Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [(b"x-request-id", b"req-test-123")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) - - async def call_next(_: Request) -> JSONResponse: - assert get_request_id() == "req-test-123" - assert get_request_scope_id() not in {None, "req-test-123"} - return JSONResponse({"ok": True}) - - response = await dispatch(request, call_next) + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health", headers={"x-request-id": "req-test-123"}) assert response.headers["x-request-id"] == "req-test-123" + assert request_ids == ["req-test-123"] + assert scope_ids[0] not in {None, "req-test-123"} assert get_request_id() is None assert get_request_scope_id() is None @pytest.mark.asyncio -async def test_request_id_middleware_uses_distinct_server_scopes_for_duplicate_client_ids(): - app = FastAPI() - add_request_id_middleware(app) - dispatch = cast(_Dispatch, app.user_middleware[0].kwargs["dispatch"]) - scopes: list[str] = [] - - def make_request() -> Request: - return Request( - { - "type": "http", - "http_version": "1.1", - "method": "GET", - "scheme": "http", - "path": "/health", - "raw_path": b"/health", - "query_string": b"", - "root_path": "", - "headers": [(b"x-request-id", b"duplicate-client-id")], - "client": ("testclient", 50000), - "server": ("testserver", 80), - }, - receive=_empty_receive, - ) +async def test_request_id_middleware_generates_id_when_missing(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) - async def call_next(_: Request) -> JSONResponse: - assert get_request_id() == "duplicate-client-id" - scope = get_request_scope_id() - assert scope is not None - scopes.append(scope) - return JSONResponse({"ok": True}) + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health") - first, second = await asyncio.gather( - dispatch(make_request(), call_next), - dispatch(make_request(), call_next), - ) + generated = response.headers["x-request-id"] + assert generated + assert request_ids == [generated] - assert first.headers["x-request-id"] == "duplicate-client-id" - assert second.headers["x-request-id"] == "duplicate-client-id" - assert len(set(scopes)) == 2 + +@pytest.mark.asyncio +async def test_request_id_middleware_falls_back_to_request_id_header(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + response = await client.get("/health", headers={"request-id": "legacy-456"}) + + assert response.headers["x-request-id"] == "legacy-456" + assert request_ids == ["legacy-456"] -async def _empty_receive() -> Message: - return {"type": "http.request", "body": b"", "more_body": False} +@pytest.mark.asyncio +async def test_request_id_middleware_uses_distinct_server_scopes_for_duplicate_client_ids(): + request_ids: list[str | None] = [] + scope_ids: list[str | None] = [] + transport = ASGITransport(app=_build_app(request_ids, scope_ids)) + + async with AsyncClient(transport=transport, base_url="http://testserver") as client: + first, second = await asyncio.gather( + client.get("/health", headers={"x-request-id": "duplicate-client-id"}), + client.get("/health", headers={"x-request-id": "duplicate-client-id"}), + ) + + assert first.headers["x-request-id"] == "duplicate-client-id" + assert second.headers["x-request-id"] == "duplicate-client-id" + assert request_ids == ["duplicate-client-id", "duplicate-client-id"] + assert len(set(scope_ids)) == 2 From 120af7520cae7d51fb66c731d9d4578e416ce93f Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:34:06 +0900 Subject: [PATCH 052/117] fix(compact): recover previous-response-pinned compaction from quota-excluded owners (#1780) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(compact): recover previous-response-pinned compaction from quota-excluded owners A compaction pinned to a previous_response_id whose owner account is quota-excluded wedged the session: the pin hard-requires the exhausted owner at selection time, the failure surfaces straight to the client, and every retry re-resolves the same dead owner until its quota window resets. Normal turns already escape this exact state via account-neutral fresh replay; this gives the compact surface the same selection-time recovery, rescoped per the maintainer review on #1490. Recovery activates only when the pin is exclusively previous_response_id (no turn-state owner, no file owner, no session identity, no session-ownership affinity), the owner loss is quota-caused (persisted RATE_LIMITED/QUOTA_EXCEEDED status at selection time, or a pre-visible quota/rate-limit in-request failover), and the anchor-free upstream payload passes the existing account-neutral fresh-replay gates plus the shared retained-prior-output transcript proof. Everything outside the gate keeps today's fail-closed surface, now recorded on the existing continuity_fail_closed counter. compact.py-scoped: no durable-bridge or continuity-rebind machinery. Carries forward the core diagnosis and recovery shape from #1490. Co-authored-by: Iweisc <179300695+Iweisc@users.noreply.github.com> Co-authored-by: Komzpa <810638+Komzpa@users.noreply.github.com> Co-Authored-By: Claude Fable 5 * docs(compact): state the recovery evidence ceiling explicitly Codex review round 1 (P1): a delta resend that itself carries a completed assistant exchange is indistinguishable from a full resend at this scope. Completeness relative to the anchor is not provable from the payload alone, and the durable prefix metadata that could prove it is excluded by the maintainer rescope of #1490. Make the spec and the helper docstring claim exactly what the transcript walk can refute instead of overclaiming delta rejection. Co-Authored-By: Claude Fable 5 * fix(compact): recognize Lite full resends and refuse policy-skipped owners Codex review round 2: - P1: a Responses-Lite history opening with the canonical additional_tools bundle and its developer instruction was rejected by the retained-prior- output walk because the canonical developer index was not derived. Route the wire input through the shared account-neutral projection (an identity transform for gate-passing input) and pass its canonical index, exactly as the HTTP bridge caller does. - P2: an owner the selector skipped for routing policy (API-key assignment scope, single-account routing) could authorize replay through a coincidental RATE_LIMITED/QUOTA_EXCEEDED status. Gate on the selector's own rejection cause: a preferred_account_unavailable policy skip stays owner-bound unless the owner was excluded by a pre-visible quota failover. Co-Authored-By: Claude Fable 5 * docs(compact): scope replay validation claim to the serializer output Codex review round 3 flagged that the compact transport injects the Responses-Lite reasoning.context control (and inlines image URLs) after to_payload(), so the finalized egress form differs from the validated serialization. Those mutations are proxy-injected, account-agnostic, and applied identically to the owner send and the replay send — they carry no client or account state, and the HTTP bridge replay paths apply the same shared gate to pre-transport serializations. State that boundary explicitly in the spec and helper docstring instead of overclaiming validation of the finalized wire form. Co-Authored-By: Claude Fable 5 * fix(compact): record fail-closed outcome when additional owner pins block recovery A previous-response-pinned selection failure skipped the recovery branch entirely when the same owner was also pinned by turn-state or input-file ownership, so the unavailable owner surfaced without recording the compact continuity_fail_closed/owner_account_unavailable outcome the OpenSpec requirement promises for requests outside the recovery gates. Route additional owner pins through the common pinned-selection failure path as a new blocked reason (additional_owner_pins) — recording only; the recovery gate itself stays closed for those pins. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Iweisc <179300695+Iweisc@users.noreply.github.com> Co-authored-by: Komzpa <810638+Komzpa@users.noreply.github.com> Co-authored-by: Claude Fable 5 --- app/modules/proxy/_service/compact.py | 316 +++++++++++++- .../proposal.md | 47 +++ .../specs/responses-api-compat/spec.md | 173 ++++++++ .../tasks.md | 32 ++ tests/integration/test_proxy_compact.py | 387 ++++++++++++++++++ tests/unit/test_proxy_utils.py | 128 ++++++ 6 files changed, 1081 insertions(+), 2 deletions(-) create mode 100644 openspec/changes/fix-compact-previous-response-quota-failover/proposal.md create mode 100644 openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/fix-compact-previous-response-quota-failover/tasks.md diff --git a/app/modules/proxy/_service/compact.py b/app/modules/proxy/_service/compact.py index be46c9096a..682cfb4bb9 100644 --- a/app/modules/proxy/_service/compact.py +++ b/app/modules/proxy/_service/compact.py @@ -9,6 +9,7 @@ from typing import Any, NoReturn, Protocol, TypeVar, cast import aiohttp +from pydantic import ValidationError from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind from app.core.balancer import ResetPreferenceWindow, RoutingStrategy, failover_decision @@ -23,6 +24,7 @@ from app.core.config.settings import get_settings from app.core.config.settings_cache import get_settings_cache from app.core.errors import openai_error +from app.core.openai.exceptions import ClientPayloadError from app.core.openai.models import CompactResponsePayload from app.core.openai.requests import ResponsesCompactRequest from app.core.resilience.network_recovery import ProcessNetworkRecovery @@ -30,7 +32,7 @@ from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError from app.core.utils.request_id import ensure_request_id, get_request_id from app.core.utils.retry import backoff_seconds -from app.db.models import Account, DashboardSettings, StickySessionKind +from app.db.models import Account, AccountStatus, DashboardSettings, StickySessionKind from app.modules.api_keys.service import ( ApiKeyData, ApiKeyRequestUsageBudget, @@ -51,7 +53,10 @@ _thread_codex_session_affinity, ) from app.modules.proxy.api_key_usage import estimate_api_key_request_usage -from app.modules.proxy.continuity import resolve_required_account_id +from app.modules.proxy.continuity import ( + resolve_required_account_id, + without_http_bridge_session_affinity_headers, +) from app.modules.proxy.helpers import ( _header_account_id, _normalize_error_code, @@ -64,6 +69,11 @@ AccountSelection, effective_account_concurrency_caps, ) +from app.modules.proxy.replay_safety import ( + project_responses_input_for_account_neutral_fresh_replay, + responses_input_suffix_retains_prior_output, + responses_payload_is_account_neutral_fresh_replay, +) from app.modules.proxy.selection_errors import selection_failure_response from app.modules.proxy.work_admission import AdmissionLease, WorkAdmissionController @@ -139,6 +149,8 @@ async def _resolve_compact_turn_state_owner( fail_on_missing: bool = True, ) -> str | None: ... + async def _compact_owner_selection_loss_is_quota_caused(self, account_id: str) -> bool: ... + async def _ensure_fresh_with_budget( self, account: Account, *, force: bool = False, timeout_seconds: float | None = None ) -> Account: ... @@ -480,7 +492,160 @@ def _service_tier_from_compact_payload(payload: ResponsesCompactRequest) -> str return normalize(payload.service_tier) +# Account statuses that prove the pinned owner's selection-time loss is caused +# by upstream quota/rate-limit state rather than authentication, deactivation, +# or an operator pause. Only these authorize account-neutral replay recovery. +_COMPACT_OWNER_QUOTA_UNAVAILABLE_STATUSES = ( + AccountStatus.RATE_LIMITED, + AccountStatus.QUOTA_EXCEEDED, +) + + +def _compact_replay_history_retains_prior_output(input_items: list[JsonValue]) -> bool: + """Prove the carried history retains prior assistant output before new input. + + A self-contained account-neutral ``input`` is not by itself a full resend: + a client could send only the turns after ``previous_response_id`` (for + example two fresh user messages) and rely on the owner account to hold the + earlier conversation, so replaying without the anchor would compact a + truncated history. Without durable prefix metadata for the compact surface, + the strongest client-side evidence of a full resend is the same + retained-prior-output shape the HTTP bridge replay path trusts: the input + must parse as a clean transcript whose final segment is the previous + response's completed assistant output followed only by fresh client input. + The split is anchored at the last assistant message so the shared suffix + walk proves exactly that segment; anything it cannot prove stays + owner-bound. + + This is the evidence ceiling of the #1490 rescope: completeness relative to + the anchored conversation is not provable from the payload alone, and the + durable prefix metadata that could prove it is deliberately not consulted + here. A delta resend that itself carries a completed assistant exchange + ahead of the fresh input is indistinguishable from a full resend and is + recovered as the client's authoritative local history — the same trust the + shared account-neutral fresh-replay rules already grant a normal turn that + abandons an unavailable owner. The rejected shapes below are the ones the + transcript walk can actually refute. + """ + + last_assistant_index: int | None = None + for index in range(len(input_items) - 1, -1, -1): + item = input_items[index] + if isinstance(item, dict) and item.get("type") in (None, "message") and item.get("role") == "assistant": + last_assistant_index = index + break + # ``responses_input_suffix_retains_prior_output`` requires a non-empty + # stored prefix, so a history that opens with (or lacks) assistant output + # cannot be proven and stays owner-bound. + if last_assistant_index is None or last_assistant_index == 0: + return False + # The projection is an identity transform for input that already passed + # the account-neutral fresh-replay gate (no server-assigned ids, no + # reasoning or omitted bookkeeping types survive that gate), but it is the + # shared authority for recognizing the canonical Responses-Lite developer + # instruction behind an ``additional_tools`` bundle — without that index + # the suffix walk would reject every Lite full resend. + projection = project_responses_input_for_account_neutral_fresh_replay( + input_items, + stored_count=last_assistant_index, + ) + if projection is None: + return False + return responses_input_suffix_retains_prior_output( + projection.input_items, + stored_count=projection.stored_prefix_count, + canonical_lite_developer_index=projection.canonical_lite_developer_index, + ) + + +def _compact_account_neutral_replay_payload( + payload: ResponsesCompactRequest, +) -> ResponsesCompactRequest | None: + """Return the anchor-free replay payload for a verified full resend. + + A compact request pinned only by ``previous_response_id`` may move off an + unselectable owner account when the history it carries is provably + account-neutral: the upstream-bound payload without the anchor must pass + the shared fresh-replay validation, so no encrypted or compaction state, + server-assigned item ids, account-scoped file/container handles, + conversation/prompt handles, or hosted/MCP state can reach the replacement + account. + + Neutrality is checked on the serialized upstream-bound payload + (``to_payload``), never on the request model, matching how the HTTP bridge + replay paths apply the shared gate to pre-transport serializations. The + compact transport applies two further mutations after this serialization — + the Responses-Lite ``reasoning.context`` control and inline image + fetching — both proxy-injected, account-agnostic, and applied identically + to the owner send and the replay send, so they are not part of the client + payload being proven. The serialized history must additionally be a + complete resend. ``to_payload`` can still drop history on the wire: it + strips poisoned local-compact fallback messages together with their + trailing encrypted compaction item, and it trims oversized inputs down to a + head, a trim marker, and a tail. Both remain multi-item account-neutral + lists. Sending either to a replacement account without the anchor would + compact an incomplete conversation, because only the owner can resolve the + omitted context from the dropped anchor. So the wire input must be + item-for-item identical to the validated request input, must still carry + more than one item, and must retain prior assistant output ahead of the new + client input (see ``_compact_replay_history_retains_prior_output``). + """ + + previous_response_id = getattr(payload, "previous_response_id", None) + if not isinstance(previous_response_id, str) or not previous_response_id.strip(): + return None + if not isinstance(payload.input, list): + return None + replay_source = payload.model_dump(mode="json", exclude_none=True) + replay_source.pop("previous_response_id", None) + request_input = replay_source.get("input") + if not isinstance(request_input, list) or len(request_input) <= 1: + return None + try: + replay_payload = ResponsesCompactRequest.model_validate(replay_source) + replay_wire_payload = replay_payload.to_payload() + except (ValidationError, ClientPayloadError): + return None + replay_wire_input = replay_wire_payload.get("input") + if not isinstance(replay_wire_input, list) or len(replay_wire_input) <= 1: + return None + if replay_wire_input != request_input: + return None + if not responses_payload_is_account_neutral_fresh_replay(replay_wire_payload): + return None + if not _compact_replay_history_retains_prior_output(cast(list[JsonValue], replay_wire_input)): + return None + return replay_payload + + class _CompactMixin: + async def _compact_owner_selection_loss_is_quota_caused(self, account_id: str) -> bool: + """Return whether the pinned owner is unselectable because of quota state. + + Account-neutral replay off a pinned previous-response owner is legal + only for owner loss the owner's quota state caused. At selection time + that evidence is the owner's own persisted status: ``RATE_LIMITED`` or + ``QUOTA_EXCEEDED`` is the same upstream usage-exhaustion state the + selector consulted. Authentication loss (``REAUTH_REQUIRED``, + ``DEACTIVATED``), operator pauses, local capacity caps on an ``ACTIVE`` + account, and a failed lookup all stay owner-bound. + """ + + proxy = cast(_CompactServiceProtocol, self) + try: + async with proxy._repo_factory() as repos: + account = await repos.accounts.get_by_id_fresh(account_id) + # Read inside the repository scope: the session expires ORM + # attributes when it closes. + status = account.status if account is not None else None + except Exception: + logger.warning( + "Compact previous-response owner status lookup failed; keeping the request owner-bound", + exc_info=True, + ) + return False + return status in _COMPACT_OWNER_QUOTA_UNAVAILABLE_STATUSES + async def _resolve_compact_turn_state_owner( self, *, @@ -1063,6 +1228,15 @@ async def _call_compact( last_exc: ProxyResponseError | None = None network_recovery = ProcessNetworkRecovery(transport="compact", request_id=request_id) excluded_account_ids: set[str] = set() + # Account-neutral replay off a pinned previous-response owner is only + # legal for owner loss the owner's quota state caused: either the + # owner was never usable at selection time, or it was excluded + # mid-request by a pre-visible quota / rate-limit failure. Post- + # selection authentication, refresh, transport, and transient + # failures also exclude the owner, and those keep their existing + # owner-bound handling instead of moving the history to another + # account. + owner_quota_failover_eligible = False require_security_work_authorized = False estimated_lease_tokens = _estimated_lease_tokens_from_request_usage_budget( estimate_api_key_request_usage(payload) @@ -1118,6 +1292,136 @@ async def _call_compact( fallback_on_preferred_account_unavailable=preferred_account_id is None, ) account = selection.account + if ( + account is None + and previous_response_preferred_account_id is not None + and preferred_account_id == previous_response_preferred_account_id + ): + # Narrowed alias: the structural gate above proves the + # selection pin names the previous-response owner. + unavailable_owner_account_id = previous_response_preferred_account_id + # The pinned previous-response owner cannot be selected. + # A full resend that is provably account-neutral on the + # wire needs nothing from the owner, so the stale anchor + # can be dropped and the compact can move to a healthy + # account instead of wedging the session until the + # owner's quota window resets — the same selection-time + # escape normal turns already have. Turn-state and file + # pins keep the request owner-bound (the first blocked + # reason below), but their selection failure still + # records the fail-closed outcome on the common path. + recovery_blocked_reason: str | None = None + if turn_state_owner_account_id is not None or rewritten_file_account_id is not None: + # The previous-response owner is also pinned by a + # turn-state or input-file owner. Those pins are + # account ownership this recovery must never move, + # so the request stays owner-bound regardless of + # the owner's quota state — but the unavailable + # owner still fails closed and must be recorded. + recovery_blocked_reason = "additional_owner_pins" + elif previous_response_lookup_session_id is not None: + # A session/turn-state identity on the request can + # bind live or durable HTTP-bridge continuity rows + # that still name the lost owner. Without the + # rebinding machinery this recovery deliberately + # avoids, moving the history would strand that + # continuity, so session-scoped requests stay + # owner-bound. + recovery_blocked_reason = "session_scoped_continuity" + elif ( + affinity.kind == StickySessionKind.CODEX_SESSION + or affinity.legacy_selection_key is not None + or affinity.require_unambiguous_account + ): + # CODEX_SESSION affinity (turn-state, thread, or + # session-header keys, including raw legacy rows) + # is session ownership this recovery would have to + # rebind, so those requests stay owner-bound. + # PROMPT_CACHE / STICKY_THREAD keys are soft cache + # locality the sticky selection path already falls + # back from on an unavailable account — the compact + # routes derive one unconditionally — so they gate + # nothing here and the recovery reselection flows + # through that same existing sticky handling. + recovery_blocked_reason = "session_affinity" + elif ( + not owner_quota_failover_eligible + and selection.error_code == "preferred_account_unavailable" + ): + # The selector skipped the owner before evaluating + # its availability (API-key assignment scope, + # single-account routing, or an in-request + # exclusion that was not a pre-visible quota + # failover). Policy-caused loss must not become + # replay-eligible just because the owner's + # persisted status happens to be quota-exhausted. + recovery_blocked_reason = "owner_skipped_by_policy" + elif not ( + owner_quota_failover_eligible + or ( + unavailable_owner_account_id not in excluded_account_ids + and await proxy._compact_owner_selection_loss_is_quota_caused( + unavailable_owner_account_id + ) + ) + ): + recovery_blocked_reason = "non_quota_owner_loss" + replay_payload: ResponsesCompactRequest | None = None + if recovery_blocked_reason is None: + replay_payload = _compact_account_neutral_replay_payload(payload) + if replay_payload is None: + recovery_blocked_reason = "history_not_account_neutral" + if replay_payload is None: + logger.info( + "Compact previous-response owner unavailable; staying owner-bound " + "request_id=%s owner_account_id=%s blocked_reason=%s selection_error_code=%s", + request_id, + preferred_account_id, + recovery_blocked_reason, + selection.error_code, + ) + _record_continuity_fail_closed( + surface="compact", + reason="owner_account_unavailable", + previous_response_id=previous_response_id + if isinstance(previous_response_id, str) + else None, + session_id=previous_response_lookup_session_id, + upstream_error_code=selection.error_code, + ) + else: + logger.warning( + "Compact previous-response owner unavailable; replaying verified " + "account-neutral full resend request_id=%s owner_account_id=%s " + "selection_error_code=%s", + request_id, + preferred_account_id, + selection.error_code, + ) + excluded_account_ids.add(unavailable_owner_account_id) + payload = replay_payload + filtered = without_http_bridge_session_affinity_headers(filtered) + preferred_account_id = None + previous_response_preferred_account_id = None + selection = await proxy._select_account_with_budget_compatible( + deadline, + request_id=request_id, + kind="compact", + api_key=api_key, + affinity_policy=affinity, + prefer_earlier_reset_accounts=prefer_earlier_reset, + prefer_earlier_reset_window=_prefer_earlier_reset_window(settings), + routing_strategy=routing_strategy, + model=payload.model, + service_tier=payload.service_tier, + exclude_account_ids=excluded_account_ids, + preferred_account_id=None, + require_security_work_authorized=require_security_work_authorized, + lease_kind="response_create", + estimated_lease_tokens=estimated_lease_tokens, + fallback_on_preferred_account_unavailable=True, + ) + account = selection.account if account is not None: pass elif last_exc is not None: @@ -1713,6 +2017,14 @@ async def _call_compact( action, ) if action == "failover_next": + if account.id == preferred_account_id and classified["failure_class"] in ( + "rate_limit", + "quota", + ): + # Only a pre-visible quota / rate-limit exclusion + # of the pinned owner makes account-neutral replay + # recovery eligible for the remaining attempts. + owner_quota_failover_eligible = True last_exc = exc excluded_account_ids.add(account.id) await record_or_defer_stream_health( diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md b/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md new file mode 100644 index 0000000000..9194aafe58 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/proposal.md @@ -0,0 +1,47 @@ +# Fix Compact Previous-Response Quota Failover + +## Why + +When a long conversation's previous-response owner account is quota-excluded and the next +client action is a compaction, the compact request wedges the session. The compact path pins +selection to the resolved owner (`fallback_on_preferred_account_unavailable` is false whenever +a pin exists) and raises the selection failure straight to the client (`429 +usage_limit_reached` / 503, or the owner's in-request 429). Because the pin re-resolves the +same exhausted owner on every retry, the client cannot compact — and cannot shrink its history +to continue — until the owner's quota window resets. + +Normal turns already escape exactly this state through account-neutral fresh replay (strip the +stale `previous_response_id` anchor, verify the payload is a self-contained account-neutral +full resend, exclude the dead owner, reselect). This change gives the compact surface the same +selection-time recovery, rescoped per the maintainer review on PR #1490: activation only for +`previous_response_id`-only pins over self-contained histories, reusing the existing +account-neutral fresh-replay gates, gated on quota-caused owner exclusion, with no +continuity-rebind/CAS/fencing machinery. + +## What Changes + +- When a compact request is pinned **only** by `previous_response_id` (no turn-state owner, no + input-file owner, no session identity on the request, no session-ownership affinity) and + account selection cannot return the pinned owner, the proxy attempts account-neutral + fresh-replay recovery instead of failing: it verifies the anchor-free upstream compact + payload against the shared account-neutral fresh-replay rules plus the retained-prior-output + transcript shape, and on success removes `previous_response_id`, strips downstream + session/turn affinity aliases from upstream-bound headers, excludes the unavailable owner, + and reselects among the remaining eligible accounts. +- Recovery activates only for quota-caused owner loss: the owner's persisted status is + `RATE_LIMITED`/`QUOTA_EXCEEDED` at selection time, or the owner was excluded mid-request by + a pre-visible quota/rate-limit failover. Post-selection authentication, refresh, transport, + and transient exclusions keep their existing owner-bound surfaces. +- Every request outside the gate keeps today's fail-closed behavior, now also recorded on the + existing `continuity_fail_closed` observability counter (surface `compact`, reason + `owner_account_unavailable`). + +## Impact + +- Affected specs: `responses-api-compat` (one added requirement). +- Affected code: `app/modules/proxy/_service/compact.py` only (recovery branch in the account + selection loop, a payload-verification helper reusing `app/modules/proxy/replay_safety.py` + and `app/modules/proxy/continuity.py`, and an owner-status quota check). +- No new settings, endpoints, schemas, durable-bridge methods, or dashboard surfaces. + Reservation settlement is unchanged: recovery introduces no new terminal raise; existing + settle-before-raise sites still cover every exit. diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md b/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..df4798fcc6 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/specs/responses-api-compat/spec.md @@ -0,0 +1,173 @@ +## ADDED Requirements + +### Requirement: Compact requests recover from quota-caused previous-response owner loss + +When a compact request is pinned to a previous-response owner account, that pin is the only +continuity pin (no client-supplied turn-state owner and no input-file owner), and account +selection cannot return the pinned owner, the proxy MUST attempt account-neutral fresh-replay +recovery before surfacing the failure, provided every activation gate below holds. Outside the +gates the proxy MUST keep today's fail-closed failure for that request, MUST NOT send any part +of the payload to another account, and MUST record the fail-closed outcome on the continuity +fail-closed observability counter for the compact surface. + +Recovery MUST activate only for quota-caused owner loss. At selection time, before the owner +was ever used for the request, the owner's persisted account status MUST be rate-limited or +quota-exhausted; an owner unselectable for any other reason (re-authentication required, +deactivated, paused, local capacity caps on an active account, or a failed status lookup) +stays owner-bound. An owner the selector skipped for routing policy — API-key assignment +scope, single-account routing, or a prior in-request exclusion — MUST stay owner-bound +regardless of its persisted quota status, because policy, not quota, caused that selection +loss. Mid-request, only a pre-visible quota or rate-limit failure of the pinned owner that +permits failover makes recovery eligible; post-selection authentication, refresh, transport, +timeout, and transient exclusions of the pinned owner keep their existing owner-bound +handling. + +Recovery MUST NOT activate when the request carries a session identity (a turn-state or +session header) that can bind live or durable HTTP-bridge continuity, or when the resolved +affinity is session ownership (a Codex-session affinity key, a raw legacy session row, or a +conversation handle requiring an unambiguous owner): this recovery deliberately carries no +continuity-rebind machinery, so anything that would need rebinding stays owner-bound. +Prompt-cache and sticky-thread locality keys are advisory cache locality that ordinary sticky +selection already falls back from and do not block recovery. + +Local verification MUST run against the exact serialized upstream-bound compact payload +without `previous_response_id`, after every transformation the compact serializer applies. +Transport-stage mutations that follow that serialization are outside the client payload being +proven and MUST be limited to proxy-injected, account-agnostic controls that are applied +identically to the owner send and the replay send (the Responses-Lite +`reasoning.context` control and inline image fetching); they carry no client or account +state, so the shared account-neutral rules — which the HTTP bridge replay paths likewise +apply to pre-transport serializations — retain their meaning. It +MUST require that serialized `input` to be a list of more than one item that is item-for-item +identical to the validated request `input`, so that no request whose wire history is dropped +or trimmed — including single-item collapse and oversized-input trim markers — is ever +replayed on another account, which could not resolve the omitted owner-resident context. It +MUST validate that same serialized payload against the shared account-neutral fresh-replay +rules: self-contained tool call/output pairing, no server-assigned item ids, no encrypted or +compaction state, no nonblank conversation or prompt handles, no account-scoped +file/container/vector handles, no hosted/MCP call state, and only recognized account-neutral +fields and shapes. Because a self-contained payload may still be a delta that relies on the +owner to hold the earlier conversation, the serialized `input` MUST additionally parse as a +transcript whose final segment retains completed assistant output followed only by fresh +client input, using the shared retained-prior-output rule anchored at the last assistant +message. Histories those gates cannot prove — including delta-shaped inputs without retained +assistant output and transcripts without fresh follow-up input — stay owner-bound. + +This transcript-shape rule is the evidence ceiling of this scope: completeness relative to the +anchored conversation is not provable from the payload alone, and the durable prefix metadata +that could prove it is deliberately not consulted here (per the maintainer rescope of the +original change, which excluded durable-bridge plumbing from this recovery). A delta resend +that itself carries a completed assistant exchange ahead of the fresh input is therefore +indistinguishable from a full resend and MAY be recovered as the client's authoritative local +history — the same trust the shared account-neutral fresh-replay rules already grant a normal +turn that abandons an unavailable owner. Clients that resend partial histories under a +previous-response anchor accept summarization over that partial history when the owner is +quota-lost; the alternative surface is the current hard failure until the owner's quota +window resets. + +For an eligible recovery, the proxy MUST remove `previous_response_id` from the upstream +compact payload, strip downstream session/turn affinity aliases from the upstream-bound +headers, exclude the unavailable owner account from the remaining attempts, and reselect among +the remaining eligible accounts with fallback enabled. + +#### Scenario: Quota-excluded owner at selection time fails over with a verified full resend + +- **GIVEN** account A owns the previous response referenced by a compact request and account B is eligible +- **AND** account A's persisted status is rate-limited or quota-exhausted +- **AND** the compact payload carries an account-neutral full-resend `input` that retains prior assistant output ahead of the new client input +- **AND** the request carries no session identity and no session-ownership affinity +- **WHEN** pinned account selection cannot return account A +- **THEN** the proxy sends the compact upstream exactly once on account B without `previous_response_id` +- **AND** the compact response is returned successfully + +#### Scenario: Owner exhausts quota during the compact request + +- **GIVEN** the pinned previous-response owner is selected for a compact request +- **AND** the upstream compact fails with a pre-visible quota or rate-limit error that permits failover +- **WHEN** reselection cannot return the now-excluded owner +- **THEN** the proxy applies the same account-neutral fresh-replay recovery on another eligible account +- **AND** the owner's quota failure is not surfaced to the client when the recovery succeeds + +#### Scenario: Post-selection authentication failure on the pinned owner stays owner-bound + +- **GIVEN** the pinned previous-response owner is selected for a compact request with an account-neutral full-resend `input` +- **AND** the upstream compact fails with `401` again after the forced token refresh, which excludes the owner from the remaining attempts +- **WHEN** reselection cannot return the now-excluded owner +- **THEN** the proxy surfaces the owner's authentication failure +- **AND** account-neutral fresh-replay recovery does not activate and no part of the payload is sent to another account + +#### Scenario: Policy-skipped owner stays owner-bound despite quota status + +- **GIVEN** a previous-response-pinned compact request whose owner account is excluded by API-key assignment scope or single-account routing +- **AND** that owner's persisted status is coincidentally rate-limited or quota-exhausted +- **WHEN** pinned account selection skips the owner +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Responses-Lite full resend behind a canonical tool bundle is recoverable + +- **GIVEN** a quota-excluded previous-response-pinned compact request whose `input` opens with a canonical `additional_tools` bundle and its immediately following developer instruction +- **AND** the remaining transcript retains prior assistant output ahead of fresh client input and passes the account-neutral fresh-replay rules +- **WHEN** pinned account selection cannot return the owner +- **THEN** the shared canonical-Lite prefix handling recognizes the developer instruction +- **AND** the recovery replays the anchor-free payload on another eligible account + +#### Scenario: Non-quota owner loss at selection time stays owner-bound + +- **GIVEN** a previous-response-pinned compact request whose owner account is paused, deactivated, or requires re-authentication +- **WHEN** pinned account selection cannot return the owner +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Non-neutral compact payload stays fail-closed + +- **GIVEN** a pinned compact request whose `input` retains encrypted compaction state, server-assigned item ids, or account-scoped file handles +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** no part of the payload is sent to another account +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Delta-shaped history without retained output stays fail-closed + +- **GIVEN** a pinned compact request whose multi-item `input` carries no retained assistant output ahead of fresh client input +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** the proxy keeps the anchor and sends no part of the payload to another account + +#### Scenario: History the wire serializer shortens stays fail-closed + +- **GIVEN** a pinned compact request whose `input` loses history when serialized for upstream, either collapsing to a single item or being trimmed to a head, trim marker, and tail +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection or upstream error +- **AND** the proxy does not replay the shortened history on another account + +#### Scenario: Session-identified compact stays owner-bound + +- **GIVEN** a pinned compact request that carries a session or turn-state identity able to bind live or durable HTTP-bridge continuity +- **WHEN** the quota-excluded pinned owner cannot be selected +- **THEN** the request fails with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Turn-state-pinned and file-pinned compacts remain owner-bound + +- **GIVEN** a compact request pinned by a client-supplied turn-state owner or an input-file owner +- **WHEN** that owner account cannot be selected +- **THEN** the request fails closed with the existing continuity or selection error +- **AND** account-neutral fresh-replay recovery does not activate + +#### Scenario: Additional owner pins record the fail-closed outcome + +- **GIVEN** a compact request whose `previous_response_id` owner is also resolved by a turn-state or input-file pin naming the same account +- **WHEN** the pinned owner cannot be selected +- **THEN** the request fails closed with the existing selection error +- **AND** account-neutral fresh-replay recovery does not activate +- **AND** the continuity fail-closed counter records the compact-surface outcome + +#### Scenario: Unresolvable previous-response owner remains fail-closed + +- **GIVEN** a compact request whose `previous_response_id` owner cannot be resolved from any record +- **AND** more than one account is eligible +- **WHEN** the request is evaluated before account selection +- **THEN** the request fails with `previous_response_owner_unavailable` +- **AND** the proxy does not treat the missing owner as a selector result or replay on another account diff --git a/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md b/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md new file mode 100644 index 0000000000..1d584e8223 --- /dev/null +++ b/openspec/changes/fix-compact-previous-response-quota-failover/tasks.md @@ -0,0 +1,32 @@ +# Tasks + +- [x] 1. Add a compact account-neutral replay verification helper in + `app/modules/proxy/_service/compact.py` that returns the anchor-free + `ResponsesCompactRequest` only when the request carries `previous_response_id`, a + list-shaped `input` with more than one item, an upstream-bound serialization that is + item-for-item identical to the validated request `input`, passes + `responses_payload_is_account_neutral_fresh_replay`, and retains prior assistant output + ahead of new client input via `responses_input_suffix_retains_prior_output`. +- [x] 2. In the `compact_responses` account-selection loop, when selection returns no account + and the request is pinned only by the previous-response owner, activate recovery for a + verified payload when the owner loss is quota-caused (persisted + `RATE_LIMITED`/`QUOTA_EXCEEDED` status at selection time, or a pre-visible quota/rate-limit + in-request failover of the owner): exclude the owner, drop the pin, strip session/turn + affinity aliases from upstream-bound headers via + `without_http_bridge_session_affinity_headers`, and reselect with fallback enabled. +- [x] 3. Keep every other case fail-closed and record `continuity_fail_closed` (surface + `compact`, reason `owner_account_unavailable`) when the pinned selection failure is + surfaced: additional turn-state/input-file owner pins on the same owner, session identity + on the request, session-ownership affinity, non-quota owner loss, and unverifiable + histories. +- [x] 4. Add unit tests for the verification helper (eligible full resend; missing anchor; + single-item and string inputs; server-assigned ids; encrypted compaction state; delta + histories without retained output; transcripts without fresh follow-up input; wire-trimmed + oversized histories). +- [x] 5. Add integration regression tests at `POST /backend-api/codex/responses/compact`: + selection-time quota loss recovers on the other account without `previous_response_id`; + mid-request owner 429 recovers the same way; repeated post-refresh 401 stays owner-bound; + delta histories, account-scoped histories, session-identified requests, and paused owners + stay fail-closed with nothing sent to another account. +- [x] 6. Run `uv run ruff check`, `uv run ruff format --check`, `uv run ty check`, and the + unit + compact integration suites; validate the change with strict OpenSpec validation. diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index f6b17c0732..5810721243 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -3,6 +3,7 @@ import base64 import contextlib import json +import logging from datetime import timedelta, timezone from typing import cast from unittest.mock import AsyncMock @@ -22,6 +23,7 @@ from app.db.session import SessionLocal from app.modules.api_keys.repository import ApiKeysRepository from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService +from app.modules.proxy.account_cache import get_account_selection_cache from app.modules.proxy.rate_limit_cache import get_rate_limit_headers_cache from app.modules.usage.repository import AdditionalUsageRepository, UsageRepository @@ -1655,3 +1657,388 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert response.status_code == 200 assert seen_inputs == [compact_window["output"]] + + +_NEUTRAL_FULL_RESEND_INPUT: list[dict[str, object]] = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, +] + + +async def _import_account(async_client, *, email: str, raw_account_id: str) -> str: + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + return generate_unique_account_id(raw_account_id, email) + + +async def _mark_account_status(account_id: str, status: AccountStatus) -> None: + async with SessionLocal() as session: + account = await session.get(Account, account_id) + assert account is not None + account.status = status + if status in (AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED): + account.reset_at = int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + 3600 + await session.commit() + get_account_selection_cache().invalidate() + + +def _pin_previous_response_owner(monkeypatch, owner_account_id: str) -> None: + async def fake_owner(self, *, previous_response_id, api_key, session_id=None, surface, **kwargs): + del self, previous_response_id, api_key, session_id, surface, kwargs + return owner_account_id + + monkeypatch.setattr(proxy_module.ProxyService, "_resolve_websocket_previous_response_owner", fake_owner) + + +def _recording_compact( + calls: list[tuple[str | None, dict[str, object], dict[str, str]]], + *, + fail_accounts_with: dict[str, ProxyResponseError] | None = None, +): + async def fake_compact(payload, headers, access_token, account_id): + del access_token + calls.append((account_id, cast(dict[str, object], payload.to_payload()), dict(headers))) + if fail_accounts_with and account_id in fail_accounts_with: + raise fail_accounts_with[account_id] + return CompactResponsePayload.model_validate({"object": "response.compaction", "output": []}) + + return fake_compact + + +def _usage_limit_429() -> ProxyResponseError: + return ProxyResponseError( + 429, + { + "error": { + "type": "usage_limit_reached", + "message": "limit reached", + "plan_type": "plus", + "resets_at": int(utcnow().replace(tzinfo=timezone.utc).timestamp()) + 3600, + } + }, + ) + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_selection_time_quota_loss_replays_account_neutral_full_resend( + async_client, monkeypatch +): + """A previous-response-pinned compact whose owner is quota-excluded at + selection time recovers by dropping the anchor and replaying the verified + account-neutral full resend on a healthy account instead of wedging.""" + owner_account_id = await _import_account( + async_client, email="compact-quota-owner@example.com", raw_account_id="acc_quota_owner" + ) + await _import_account(async_client, email="compact-quota-alt@example.com", raw_account_id="acc_quota_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_quota_anchor", + }, + ) + + assert response.status_code == 200, response.text + assert [account_id for account_id, _payload, _headers in calls] == ["acc_quota_alt"] + replay_payload = calls[0][1] + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == _NEUTRAL_FULL_RESEND_INPUT + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_in_request_quota_429_replays_account_neutral_full_resend( + async_client, monkeypatch +): + """An owner that 429s mid-request (pre-visible quota failover) is excluded + and the verified account-neutral full resend recovers on the other account + instead of re-raising the owner's 429 forever.""" + owner_account_id = await _import_account( + async_client, email="compact-429-owner@example.com", raw_account_id="acc_429_owner" + ) + await _import_account(async_client, email="compact-429-alt@example.com", raw_account_id="acc_429_alt") + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr( + proxy_module, + "core_compact_responses", + _recording_compact(calls, fail_accounts_with={"acc_429_owner": _usage_limit_429()}), + ) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_429_anchor", + }, + ) + + assert response.status_code == 200, response.text + assert [account_id for account_id, _payload, _headers in calls] == ["acc_429_owner", "acc_429_alt"] + owner_payload, replay_payload = calls[0][1], calls[1][1] + assert owner_payload["previous_response_id"] == "resp_429_anchor" + assert "previous_response_id" not in replay_payload + assert replay_payload["input"] == owner_payload["input"] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_post_selection_401_stays_owner_bound(async_client, monkeypatch): + """A repeated 401 after the forced refresh excludes the owner for a + non-quota reason, so account-neutral replay must not activate and the + owner's authentication failure surfaces unchanged.""" + owner_account_id = await _import_account( + async_client, email="compact-401-owner@example.com", raw_account_id="acc_401_owner" + ) + await _import_account(async_client, email="compact-401-alt@example.com", raw_account_id="acc_401_alt") + _pin_previous_response_owner(monkeypatch, owner_account_id) + + async def fake_ensure_fresh(self, account, *, force=False, timeout_seconds=None): + del self, force, timeout_seconds + return account + + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + unauthorized = ProxyResponseError(401, openai_error("unauthorized", "token rejected")) + monkeypatch.setattr( + proxy_module, + "core_compact_responses", + _recording_compact(calls, fail_accounts_with={"acc_401_owner": unauthorized}), + ) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_401_anchor", + }, + ) + + assert response.status_code == 401 + assert [account_id for account_id, _payload, _headers in calls] == ["acc_401_owner", "acc_401_owner"] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_without_retained_prior_output( + async_client, monkeypatch +): + """A multi-item history without retained assistant output cannot be proven + a full resend (it may be a delta the owner account resolves through the + anchor), so nothing is sent to another account.""" + owner_account_id = await _import_account( + async_client, email="compact-delta-owner@example.com", raw_account_id="acc_delta_owner" + ) + await _import_account(async_client, email="compact-delta-alt@example.com", raw_account_id="acc_delta_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"role": "user", "content": "first delta turn"}, + {"role": "user", "content": "second delta turn"}, + ], + "previous_response_id": "resp_delta_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_with_account_scoped_history( + async_client, monkeypatch +): + """A history carrying account-scoped state (an encrypted compaction item) + fails the shared account-neutral fresh-replay gate and never crosses + accounts.""" + owner_account_id = await _import_account( + async_client, email="compact-scoped-owner@example.com", raw_account_id="acc_scoped_owner" + ) + await _import_account(async_client, email="compact-scoped-alt@example.com", raw_account_id="acc_scoped_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": [ + {"type": "compaction", "encrypted_content": "enc_owner_scoped_state"}, + *_NEUTRAL_FULL_RESEND_INPUT, + ], + "previous_response_id": "resp_scoped_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_quota_loss_stays_owner_bound_with_session_identity(async_client, monkeypatch): + """A session identity on the request can bind live or durable HTTP-bridge + continuity that still names the lost owner; without rebinding machinery the + request stays owner-bound.""" + owner_account_id = await _import_account( + async_client, email="compact-session-owner@example.com", raw_account_id="acc_session_owner" + ) + await _import_account(async_client, email="compact-session-alt@example.com", raw_account_id="acc_session_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_session_anchor", + }, + headers={"session_id": "sid-compact-session-bound"}, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_non_quota_loss_stays_owner_bound(async_client, monkeypatch): + """An owner unselectable for a non-quota reason (operator pause) keeps + today's fail-closed surface; recovery requires quota-caused owner loss.""" + owner_account_id = await _import_account( + async_client, email="compact-paused-owner@example.com", raw_account_id="acc_paused_owner" + ) + await _import_account(async_client, email="compact-paused-alt@example.com", raw_account_id="acc_paused_alt") + await _mark_account_status(owner_account_id, AccountStatus.PAUSED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_paused_anchor", + }, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_policy_skip_stays_owner_bound_despite_quota_status(async_client, monkeypatch): + """An owner the selector skips for routing policy (API-key assignment + scope) must stay owner-bound even when its persisted status happens to be + quota-exhausted: policy, not quota, caused the selection loss.""" + owner_account_id = await _import_account( + async_client, email="compact-policy-owner@example.com", raw_account_id="acc_policy_owner" + ) + alt_account_id = await _import_account( + async_client, email="compact-policy-alt@example.com", raw_account_id="acc_policy_alt" + ) + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + settings_resp = await async_client.put( + "/api/settings", + json={"totpRequiredOnLogin": False, "apiKeyAuthEnabled": True}, + ) + assert settings_resp.status_code == 200 + _key_id, key = await _create_api_key( + name="compact-policy-scope-key", + assigned_account_ids=[alt_account_id], + ) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_policy_anchor", + }, + headers={"authorization": f"Bearer {key}"}, + ) + + assert response.status_code >= 400 + assert calls == [] + + +@pytest.mark.asyncio +async def test_proxy_compact_pinned_owner_with_additional_turn_state_pin_records_fail_closed( + async_client, monkeypatch, caplog +): + """A previous-response pin accompanied by a turn-state pin on the same + owner stays owner-bound (recovery never activates for additional owner + pins), but the unavailable owner must still record the compact + continuity_fail_closed outcome on the common pinned-selection failure + path instead of skipping the recording branch entirely.""" + owner_account_id = await _import_account( + async_client, email="compact-multipin-owner@example.com", raw_account_id="acc_multipin_owner" + ) + await _import_account(async_client, email="compact-multipin-alt@example.com", raw_account_id="acc_multipin_alt") + await _mark_account_status(owner_account_id, AccountStatus.RATE_LIMITED) + _pin_previous_response_owner(monkeypatch, owner_account_id) + + async def fake_turn_state_owner(self, *, turn_state, api_key, fail_on_missing=True): + del self, turn_state, api_key, fail_on_missing + return owner_account_id + + monkeypatch.setattr(proxy_module.ProxyService, "_resolve_compact_turn_state_owner", fake_turn_state_owner) + + calls: list[tuple[str | None, dict[str, object], dict[str, str]]] = [] + monkeypatch.setattr(proxy_module, "core_compact_responses", _recording_compact(calls)) + + with caplog.at_level(logging.INFO): + response = await async_client.post( + "/backend-api/codex/responses/compact", + json={ + "model": "gpt-5.1", + "instructions": "hi", + "input": _NEUTRAL_FULL_RESEND_INPUT, + "previous_response_id": "resp_multipin_anchor", + }, + headers={"x-codex-turn-state": "ts-multipin-owner-bound"}, + ) + + assert response.status_code >= 400 + assert calls == [] + assert "blocked_reason=additional_owner_pins" in caplog.text + assert "continuity_fail_closed surface=compact reason=owner_account_unavailable" in caplog.text diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 9cd887ba2e..70f8db1e34 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -46105,3 +46105,131 @@ async def __aexit__(self, *args): assert exc_info.value.status_code == 400 assert "image_download_failed" in json.dumps(exc_info.value.payload) + + +_COMPACT_REPLAY_NEUTRAL_INPUT: list[dict[str, object]] = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, +] + + +def _compact_replay_request(**overrides: object) -> ResponsesCompactRequest: + source: dict[str, object] = { + "model": "gpt-5.1", + "instructions": "hi", + "input": _COMPACT_REPLAY_NEUTRAL_INPUT, + "previous_response_id": "resp_anchor", + } + source.update(overrides) + return ResponsesCompactRequest.model_validate(source) + + +def test_compact_account_neutral_replay_payload_accepts_verified_full_resend() -> None: + replay = proxy_compact_service._compact_account_neutral_replay_payload(_compact_replay_request()) + assert replay is not None + assert getattr(replay, "previous_response_id", None) is None + assert replay.input == _COMPACT_REPLAY_NEUTRAL_INPUT + assert "previous_response_id" not in replay.to_payload() + + +def test_compact_account_neutral_replay_payload_requires_previous_response_anchor() -> None: + payload = ResponsesCompactRequest.model_validate( + {"model": "gpt-5.1", "instructions": "hi", "input": _COMPACT_REPLAY_NEUTRAL_INPUT} + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_single_item_and_string_inputs() -> None: + single_item = _compact_replay_request(input=[{"role": "user", "content": "hello"}]) + assert proxy_compact_service._compact_account_neutral_replay_payload(single_item) is None + string_input = _compact_replay_request(input="hello there") + assert proxy_compact_service._compact_account_neutral_replay_payload(string_input) is None + + +def test_compact_account_neutral_replay_payload_rejects_server_assigned_item_ids() -> None: + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + { + "type": "message", + "id": "msg_server_assigned", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi there"}], + }, + {"role": "user", "content": "please compact"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_encrypted_compaction_state() -> None: + payload = _compact_replay_request( + input=[ + {"type": "compaction", "encrypted_content": "enc_owner_scoped"}, + *_COMPACT_REPLAY_NEUTRAL_INPUT, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_history_without_retained_output() -> None: + # Two fresh user turns may be a delta the owner resolves through the + # anchor; without retained assistant output the full resend is unproven. + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "first delta turn"}, + {"role": "user", "content": "second delta turn"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_history_without_fresh_followup() -> None: + # A transcript that ends on assistant output has no new client input after + # the retained output, so the retained-prior-output proof fails closed. + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_rejects_wire_trimmed_history() -> None: + # An oversized history is trimmed on the wire to a head, marker, and tail; + # replaying that shortened transcript would compact an incomplete + # conversation, so the wire input must stay item-for-item identical. + oversized = "x" * 600_000 + payload = _compact_replay_request( + input=[ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": oversized}]}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, + ] + ) + assert proxy_compact_service._compact_account_neutral_replay_payload(payload) is None + + +def test_compact_account_neutral_replay_payload_accepts_canonical_lite_full_resend() -> None: + # A Responses-Lite history opens with the additional_tools bundle and its + # canonical developer instruction; the shared projection must recognize + # that developer message so the Lite surface stays recoverable. + payload = _compact_replay_request( + input=[ + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "function", "name": "exec", "parameters": {"type": "object"}}], + }, + {"type": "message", "role": "developer", "content": [{"type": "input_text", "text": "instructions"}]}, + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "output_text", "text": "hi there"}]}, + {"role": "user", "content": "please compact"}, + ] + ) + replay = proxy_compact_service._compact_account_neutral_replay_payload(payload) + assert replay is not None + assert getattr(replay, "previous_response_id", None) is None From 076aab854ff0b334d77ab2104175d672384065c5 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:34:32 +0900 Subject: [PATCH 053/117] perf: coalesce same-owner sticky session TTL refresh upserts (#1790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(proxy): coalesce same-owner sticky refresh upserts within a bounded freshness window The sticky_sessions upsert is 44% of total DB execution time in production (1.2M calls / 10d, mean 32.1ms, stddev 259.9ms, max 23.2s, zero I/O): every request that retains its pinned prompt-cache owner rewrites the same (key, kind) row purely to advance updated_at, and concurrent requests of a hot session serialize on that row lock. The per-request owner lookup now reports whether the row was observed fresh (updated_at within min(15s, 1% of the mapping TTL) and no abandonment marker); selection skips the same-owner refresh write while that holds. The flag is DB-observed in the same request, so it stays correct across workers and replicas. Rebinds, deletes, restores, seed initialization, tombstone clears, and raw legacy owner paths keep writing immediately. Co-Authored-By: Claude Fable 5 * perf(proxy): revalidate refresh-skip deadline at persist time and never skip seed-initializing writes Address three review findings against the same-owner sticky refresh coalescing: - Carry the observed skip window as a deadline (StickyOwnerLookup.refresh_skip_deadline = observed_updated_at + window) instead of a fixed boolean, attach it to the retention mutation, and revalidate it against the clock at the persist site (_sticky_refresh_write_skippable) so admission latency can never move a mapping's effective expiry earlier by more than the documented bound. - Never skip a retention write that must initialize a missing process seed: that write is the seed-initialization carrier, and suppressing it let sibling threads select divergent owners until the window closed. Guarded both at deadline threading (seed_initialization_pending) and at the persist site (initialize_seed_key). - Reject future updated_at stamps (database clock ahead, restored rows): a negative age no longer satisfies the upper-bound-only window check. OpenSpec delta, proposal, and tasks updated to the deadline semantics with scenarios for all three behaviors; unit, balancer-level, and integration regression tests added. Co-Authored-By: Claude Fable 5 * perf(proxy): honor the refresh-skip deadline on the recovery-probe admission path The reserved-probe persistence branch called _persist_sticky_mutation unconditionally, so a fresh same-owner retention of a due-probing pinned owner still issued the redundant refresh upsert. Gate it with the same persist-time deadline revalidation as the non-probe path (the probe path never initializes a seed) and skip the compensating restore writes symmetrically when nothing was written, preserving reservation CAS commit/rollback bookkeeping. Adds a balancer-level regression test with a write-through control proving the scenario exercises the probe branch. Co-Authored-By: Claude Fable 5 * perf(proxy): keep thread-only affinity skippable when no seed key exists seed_initialization_pending treated every thread_header request with an unknown seed owner as seed-needing, but thread-only affinity (thread-id without a process session) carries no sticky_seed_key at all — there is nothing to initialize, and the persist site's initialize_seed_key is already None for it. Require an actual seed key in the predicate so those hot mappings coalesce their same-owner refreshes too, with a balancer-level regression test. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../proxy/_load_balancer/sticky_selection.py | 209 +++++++++++----- app/modules/proxy/load_balancer.py | 2 + app/modules/proxy/sticky_repository.py | 71 +++++- .../proposal.md | 65 +++++ .../specs/sticky-session-operations/spec.md | 82 ++++++ .../tasks.md | 47 ++++ .../integration/test_proxy_sticky_sessions.py | 197 ++++++++++++++- tests/unit/test_load_balancer_concurrency.py | 234 +++++++++++++++++- tests/unit/test_select_with_stickiness.py | 192 +++++++++++++- 9 files changed, 1031 insertions(+), 68 deletions(-) create mode 100644 openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md create mode 100644 openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md create mode 100644 openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index 1dc29cbf71..fa5235ecea 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -23,6 +23,7 @@ TrafficClass, select_account, ) +from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, AdditionalUsageHistory, StickySessionKind, UsageHistory from app.modules.accounts.repository import AccountsRepository from app.modules.proxy._load_balancer.types import ( @@ -194,6 +195,7 @@ async def _select_with_stickiness( ignore_standard_quota: bool, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: ... async def release_account_lease(self, lease: AccountLease | None) -> None: ... @@ -243,6 +245,12 @@ class _StickyMutation: # ``None`` is an intentional delete; absence of a mutation means preserve # the current mapping until final admission succeeds. account_id: str | None + # Set only when this mutation is a pure same-owner freshness rewrite of a + # row this request's lookup observed inside the repository's refresh-skip + # window. The persist site revalidates the deadline against the clock at + # write time and may then omit the statement entirely; a mutation that + # rebinds, deletes, or must initialize a seed mapping never carries it. + refresh_skip_deadline: datetime | None = None @dataclass(frozen=True, slots=True) @@ -327,6 +335,16 @@ def _direct_error( sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET sticky_continuity_abandoned = False + sticky_refresh_skip_deadline: datetime | None = None + # A thread row whose process seed exists but is still unowned must keep + # its retention write: that write doubles as the seed-initialization + # carrier (see the ``initialize_seed_key`` argument at the persist site + # below), and suppressing it would let sibling threads select divergent + # owners until the skip window closes. Thread-only affinity without a + # seed key has nothing to initialize and stays skippable. + seed_initialization_pending = ( + sticky_source == "thread_header" and sticky_seed_key is not None and sticky_seed_account_id is None + ) # A source-qualified marker can be observed before this call or after a # retirement CAS miss. In both cases its retained owner is authoritative # exclusion evidence even though it is no longer affinity ownership for @@ -357,6 +375,16 @@ def _direct_error( # always has, rather than silently bypassing the ambiguous # owner check below. sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True + # ``isinstance`` for the same test-double reason as above. The + # deadline is only ever an optimization hint: None always + # falls back to today's write-on-every-request refresh + # behavior, and seed-needing requests never skip. + observed_refresh_skip_deadline = sticky_owner_lookup.refresh_skip_deadline + sticky_refresh_skip_deadline = ( + observed_refresh_skip_deadline + if isinstance(observed_refresh_skip_deadline, datetime) and not seed_initialization_pending + else None + ) sticky_abandoned_account_id = sticky_owner_lookup.abandoned_account_id if sticky_owner_lookup.continuity_abandoned is True and isinstance( sticky_abandoned_account_id, @@ -370,6 +398,9 @@ def _direct_error( # turn-state ownership. sticky_existing_account_id = legacy_existing_account_id sticky_continuity_abandoned = False + # The freshness observation belongs to the namespaced row, + # not the raw legacy owner that now shadows it. + sticky_refresh_skip_deadline = None async with owner._runtime_lock: states, account_map = owner._prepare_sticky_selection_states( selection_inputs, @@ -643,6 +674,7 @@ def _direct_error( routing_costs_by_account_id=effective_routing_costs, allow_usage_exhaustion_error=allow_usage_exhaustion_error, usage_exhaustion_states=states, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, ) result = sticky_outcome.selection if ( @@ -909,25 +941,35 @@ def _direct_error( assert sticky_kind is not None sticky_mutation = sticky_outcome.mutation assert sticky_mutation is not None - try: - async with owner._repo_factory() as repos: - # A recovery-probe reservation is still reversible until - # the runtime CAS below succeeds. Persist its thread row so - # existing rollback machinery can restore it, but do not - # publish an immutable process seed that cannot be safely - # deleted after a concurrent sibling observes it. - await _persist_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - mutation=sticky_mutation, - ) - except BaseException: - await owner.release_account_lease(selected_lease) - selected_lease = None - async with owner._runtime_lock: - owner._release_due_probe_reservation_locked(probe_reservation) - raise + # A pure same-owner freshness rewrite may be omitted here exactly + # as on the non-probe path below (the row already holds this + # owner, so the rollback restores become no-ops and are skipped + # symmetrically). The probe path deliberately never initializes a + # seed, so only the deadline gates the skip. + probe_refresh_write_skipped = _sticky_refresh_write_skippable( + sticky_mutation, + initialize_seed_key=None, + ) + if not probe_refresh_write_skipped: + try: + async with owner._repo_factory() as repos: + # A recovery-probe reservation is still reversible until + # the runtime CAS below succeeds. Persist its thread row so + # existing rollback machinery can restore it, but do not + # publish an immutable process seed that cannot be safely + # deleted after a concurrent sibling observes it. + await _persist_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + mutation=sticky_mutation, + ) + except BaseException: + await owner.release_account_lease(selected_lease) + selected_lease = None + async with owner._runtime_lock: + owner._release_due_probe_reservation_locked(probe_reservation) + raise try: async with owner._runtime_lock: assert probe_reservation is not None @@ -945,14 +987,15 @@ def _direct_error( selected_lease = None async with owner._runtime_lock: owner._release_due_probe_reservation_locked(probe_reservation) - async with owner._repo_factory() as repos: - await _restore_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - expected_account_id=sticky_mutation.account_id, - sticky_existing_account_id=sticky_existing_account_id, - ) + if not probe_refresh_write_skipped: + async with owner._repo_factory() as repos: + await _restore_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + expected_account_id=sticky_mutation.account_id, + sticky_existing_account_id=sticky_existing_account_id, + ) raise if not reservation_committed: # Runtime health changed while account-state persistence @@ -962,14 +1005,15 @@ def _direct_error( # runtime snapshot. await owner.release_account_lease(selected_lease) selected_lease = None - async with owner._repo_factory() as repos: - await _restore_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - expected_account_id=sticky_mutation.account_id, - sticky_existing_account_id=sticky_existing_account_id, - ) + if not probe_refresh_write_skipped: + async with owner._repo_factory() as repos: + await _restore_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + expected_account_id=sticky_mutation.account_id, + sticky_existing_account_id=sticky_existing_account_id, + ) selected_snapshot = None error_message = None selected_states = [] @@ -1054,27 +1098,27 @@ def _direct_error( assert sticky_kind is not None sticky_mutation = sticky_outcome.mutation assert sticky_mutation is not None - try: - async with owner._repo_factory() as repos: - await _persist_sticky_mutation( - sticky_repo=repos.sticky_sessions, - sticky_key=sticky_key, - sticky_kind=sticky_kind, - mutation=sticky_mutation, - initialize_seed_key=( - sticky_seed_key - if sticky_source == "thread_header" and sticky_seed_account_id is None - else None - ), - initialize_seed_kind=sticky_seed_kind, - ) - except BaseException: - # Runtime admission may already be committed. Preserve - # its selection timestamp, but never leak the local - # concurrency lease when sticky persistence fails. - await owner.release_account_lease(selected_lease) - selected_lease = None - raise + initialize_seed_key = ( + sticky_seed_key if sticky_source == "thread_header" and sticky_seed_account_id is None else None + ) + if not _sticky_refresh_write_skippable(sticky_mutation, initialize_seed_key=initialize_seed_key): + try: + async with owner._repo_factory() as repos: + await _persist_sticky_mutation( + sticky_repo=repos.sticky_sessions, + sticky_key=sticky_key, + sticky_kind=sticky_kind, + mutation=sticky_mutation, + initialize_seed_key=initialize_seed_key, + initialize_seed_kind=sticky_seed_kind, + ) + except BaseException: + # Runtime admission may already be committed. Preserve + # its selection timestamp, but never leak the local + # concurrency lease when sticky persistence fails. + await owner.release_account_lease(selected_lease) + selected_lease = None + raise break return StickySelectionOutcome( @@ -1111,6 +1155,7 @@ async def _select_with_stickiness( ignore_standard_quota: bool = False, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: if not sticky_key or not sticky_repo: return _StickySelectionOutcome( @@ -1138,10 +1183,14 @@ def finish_selection( selection: SelectionResult, *, persist_account_id: str | None = None, + refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: mutation = pending_mutation if persist_account_id is not None: - mutation = _StickyMutation(account_id=persist_account_id) + mutation = _StickyMutation( + account_id=persist_account_id, + refresh_skip_deadline=refresh_skip_deadline, + ) return _StickySelectionOutcome(selection=selection, mutation=mutation) if sticky_existing_account_id is _STICKY_EXISTING_UNSET: @@ -1150,6 +1199,10 @@ def finish_selection( kind=sticky_kind, max_age_seconds=sticky_max_age_seconds, ) + # The skip deadline is only valid for the lookup that produced the + # caller's ``sticky_existing_account_id``; this fresh lookup did not + # observe row freshness, so fall back to write-through refresh. + sticky_refresh_skip_deadline = None else: existing = sticky_existing_account_id if isinstance(sticky_existing_account_id, str) else None # When the pinned account is temporarily unavailable (rate-limited, @@ -1190,6 +1243,18 @@ def finish_selection( if existing: pinned = next((state for state in states if state.account_id == existing), None) if pinned is not None: + # Retaining the pinned owner persists only to advance + # ``updated_at`` on TTL-based kinds. When this request's lookup + # already observed the row inside the repository's refresh-skip + # window, the persist site may skip that write after revalidating + # the observed deadline against the clock: concurrent requests on + # a hot session otherwise serialize on the same row's upsert + # lock. Rebinds and deletes never carry the deadline and always + # write immediately. + pinned_refresh_account_id = pinned.account_id if sticky_max_age_seconds is not None else None + pinned_refresh_skip_deadline = ( + sticky_refresh_skip_deadline if pinned_refresh_account_id is not None else None + ) # Proactively rebind session affinity for any sticky kind # once the pinned account is already above the configured # budget threshold. That preserves continuity below the @@ -1251,7 +1316,8 @@ def finish_selection( if pinned_result.account is not None: return finish_selection( pinned_result, - persist_account_id=pinned.account_id if sticky_max_age_seconds is not None else None, + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) else: # Reallocate only when a burn-first target exists and can @@ -1305,7 +1371,8 @@ def finish_selection( if pinned_result.account is not None: return finish_selection( pinned_result, - persist_account_id=(pinned.account_id if sticky_max_age_seconds is not None else None), + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) reallocate_sticky = True # Grace period: if the pinned account is rate-limited with a @@ -1332,7 +1399,8 @@ def finish_selection( if grace_result.account is not None: return finish_selection( grace_result, - persist_account_id=pinned.account_id if sticky_max_age_seconds is not None else None, + persist_account_id=pinned_refresh_account_id, + refresh_skip_deadline=pinned_refresh_skip_deadline, ) if reallocate_sticky: pending_mutation = _StickyMutation(account_id=None) @@ -1384,6 +1452,27 @@ def finish_selection( return finish_selection(chosen) +def _sticky_refresh_write_skippable( + mutation: _StickyMutation, + *, + initialize_seed_key: str | None, +) -> bool: + """Whether this mutation's write may be omitted at persist time. + + True only for a pure same-owner freshness rewrite whose observed skip + deadline still holds now, at the moment the statement would otherwise be + issued — admission and account-state persistence sit between selection + and this point, so the deadline computed at lookup time must be + revalidated to keep the mapping's effective expiry within the documented + skip-window bound. Deletes and seed-initializing writes are never + skippable. + """ + if mutation.account_id is None or initialize_seed_key is not None: + return False + deadline = mutation.refresh_skip_deadline + return isinstance(deadline, datetime) and utcnow() <= deadline + + async def _persist_sticky_mutation( *, sticky_repo: StickySessionsRepository, diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index 2624fd4cea..d5fb054824 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -1635,6 +1635,7 @@ async def _select_with_stickiness( ignore_standard_quota: bool = False, allow_usage_exhaustion_error: bool = True, usage_exhaustion_states: Iterable[AccountState] | None = None, + sticky_refresh_skip_deadline: datetime | None = None, ) -> _StickySelectionOutcome: return await _run_select_with_stickiness( states=states, @@ -1659,6 +1660,7 @@ async def _select_with_stickiness( ignore_standard_quota=ignore_standard_quota, allow_usage_exhaustion_error=allow_usage_exhaustion_error, usage_exhaustion_states=usage_exhaustion_states, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, ) _persist_sticky_mutation = staticmethod(_persist_sticky_mutation) diff --git a/app/modules/proxy/sticky_repository.py b/app/modules/proxy/sticky_repository.py index 61e09bbff7..becea80803 100644 --- a/app/modules/proxy/sticky_repository.py +++ b/app/modules/proxy/sticky_repository.py @@ -28,6 +28,18 @@ _ContinuitySource = Literal["session_header", "thread_header", "turn_state"] _SESSION_HEADER_ABANDONMENT_SCOPE = "session_header" +# A same-owner TTL refresh upsert only rewrites ``updated_at``. On hot +# (key, kind) rows, concurrent requests serialize on that row lock, so the +# selection path may skip the rewrite while the row is younger than this +# window, revalidating the observed deadline at write time. The window is +# bounded to at most 1% of the mapping TTL (so expiry moves by at most 1% of +# the window it protects) and to a small absolute ceiling; a rebind to a +# different owner, a row carrying any abandonment marker, or a row stamped in +# the future is never skippable because those writes change state beyond +# freshness (or the observation itself is untrustworthy). +_REFRESH_SKIP_TTL_FRACTION = 0.01 +_REFRESH_SKIP_MAX_SECONDS = 15.0 + # Only the Live-call ownership namespace is reserved. Other LF-prefixed keys # (e.g. the pre-existing "\ncodex-lb-affinity-v1" selection affinities) remain # ordinary operator-manageable sessions. @@ -58,6 +70,16 @@ class StickyOwnerLookup: # was retired. Global stale-hard tombstones leave this unset because their # established recovery path may legitimately reselect a recovered owner. abandoned_account_id: str | None = None + # Set only when the row was observed in this lookup with a fresh + # ``updated_at`` (within the refresh-skip window derived from + # ``max_age_seconds``) and no abandonment marker, so a same-owner TTL + # refresh upsert would be a pure ``updated_at`` rewrite. The value is the + # naive-UTC instant (``observed_updated_at`` + skip window) after which + # the skip is no longer valid; consumers must isinstance-check + # ``datetime`` (test doubles may auto-vivify attributes), must revalidate + # the deadline against the clock immediately before omitting the write, + # and must never skip a write that changes the owner account. + refresh_skip_deadline: datetime | None = None def _continuity_is_abandoned_for_source( @@ -90,6 +112,7 @@ def _owner_lookup_from_row( row: StickySession, *, continuity_source: _ContinuitySource | None, + refresh_skip_deadline: datetime | None = None, ) -> StickyOwnerLookup: if _continuity_is_abandoned_for_source( row.continuity_abandoned_at, @@ -105,7 +128,39 @@ def _owner_lookup_from_row( continuity_source, ), ) - return StickyOwnerLookup(account_id=row.account_id, continuity_abandoned=False) + return StickyOwnerLookup( + account_id=row.account_id, + continuity_abandoned=False, + refresh_skip_deadline=refresh_skip_deadline, + ) + + +def _same_owner_refresh_skip_deadline( + row: StickySession, + *, + observed_updated_at: datetime, + now: datetime, + max_age_seconds: int, +) -> datetime | None: + """Deadline until which a same-owner upsert of this row stays skippable. + + Any abandonment marker disqualifies the skip: an upsert re-establishes + ownership by clearing both marker columns, so that write is semantic even + when the owner account is unchanged. A row whose ``updated_at`` sits in + the future (database clock ahead of this process, or a restored row) is + also never skippable: an upper-bound-only age comparison would let such a + row satisfy the window for longer than the documented bound. + """ + + if row.continuity_abandoned_at is not None or row.continuity_abandonment_scope is not None: + return None + age_seconds = (now - observed_updated_at).total_seconds() + if age_seconds < 0: + return None + skip_window_seconds = min(_REFRESH_SKIP_MAX_SECONDS, max_age_seconds * _REFRESH_SKIP_TTL_FRACTION) + if age_seconds > skip_window_seconds: + return None + return observed_updated_at + timedelta(seconds=skip_window_seconds) class StickySessionsRepository: @@ -150,10 +205,20 @@ async def get_account_id_and_abandonment( return StickyOwnerLookup(account_id=None, continuity_abandoned=False) if max_age_seconds is None: return _owner_lookup_from_row(row, continuity_source=continuity_source) - cutoff = utcnow() - timedelta(seconds=max_age_seconds) + now = utcnow() + cutoff = now - timedelta(seconds=max_age_seconds) observed_updated_at = to_utc_naive(row.updated_at) if observed_updated_at >= cutoff: - return _owner_lookup_from_row(row, continuity_source=continuity_source) + return _owner_lookup_from_row( + row, + continuity_source=continuity_source, + refresh_skip_deadline=_same_owner_refresh_skip_deadline( + row, + observed_updated_at=observed_updated_at, + now=now, + max_age_seconds=max_age_seconds, + ), + ) # Release the read snapshot before attempting a SQLite write upgrade. # The DELETE remains safe because every value observed above participates diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md b/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md new file mode 100644 index 0000000000..d043b49d10 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/proposal.md @@ -0,0 +1,65 @@ +# Coalesce same-owner sticky refresh writes + +## Why + +The sticky-session upsert is the single most expensive statement in a production +deployment: over 10 days of `pg_stat_statements` it accounted for 44% of total +database execution time (1,203,934 calls, mean 32.1ms, stddev 259.9ms, max 23.2s, +zero I/O time). The table itself is small (26MB / 27k rows) and healthy; the cost +is row-lock serialization. Every request that retains its pinned owner on a +TTL-based mapping (`prompt_cache`) re-executes +`INSERT ... ON CONFLICT (key, kind) DO UPDATE SET account_id = ..., updated_at = now(), ...` +purely to advance `updated_at`. Concurrent requests of one hot session hit the +same `(key, kind)` row and queue on its row lock through each other's commits, +which produces the heavy tail (stddev 8x the mean) and burns wall-clock time on +the TTFT-critical selection path. + +## What Changes + +- The owner lookup that selection already performs per request now also reports + a refresh-skip deadline when the row was observed fresh: `updated_at` within + `min(15s, 1% of the mapping TTL)`, not stamped in the future, AND no + abandonment marker in either `continuity_abandoned_at` or + `continuity_abandonment_scope`. The deadline is + `observed_updated_at + skip window`. +- When selection retains the same pinned owner, the mutation carries that + deadline to the persist site, which revalidates it against the clock at the + moment the statement would be issued and only then omits the same-owner + refresh upsert — no statement, no row lock. A deadline that lapsed during + admission or account-state persistence writes through, so the mapping's + effective expiry never moves earlier by more than the skip window. The next + request after the window closes performs the normal write-through refresh. +- Every state-changing write is unaffected and still immediate: rebinding to a + different account, deleting a mapping, restoring after failed admission, + clearing an abandonment tombstone, seeding a new mapping (including a thread + retention that must initialize a missing process seed — that write is the + seed-initialization carrier and is never skipped), and the raw legacy owner + paths. A row carrying any abandonment marker is never skippable because the + upsert also clears those marker columns. +- The deadline is DB-observed within the same request (no cross-request cache), + so it is correct with any number of workers or replicas: a replica can only + skip a write whose freshness it just read from the shared database. + +## Freshness window rationale + +`updated_at` on `prompt_cache` mappings is consumed by two TTL clocks, both +driven by `openai_cache_affinity_max_age_seconds` (default 1800s): the read-path +expiry in the owner lookup and the background cleanup loop. Skipping a refresh +while the row is younger than `min(15s, TTL * 0.01)` means a mapping's effective +expiry can move at most that window earlier — at most 1% of the TTL it protects, +and never more than 15 seconds. Sessions with request gaps longer than the +window (the overwhelming majority) still refresh on every request; only bursts +faster than the window coalesce, and those bursts re-refresh within the window +by construction. Durable kinds (`codex_session`, `sticky_thread` without TTL) +never used the refresh-on-retention write and are untouched. + +## Impact + +- Affected specs: `sticky-session-operations` +- Affected code: `app/modules/proxy/sticky_repository.py`, + `app/modules/proxy/_load_balancer/sticky_selection.py`, + `app/modules/proxy/load_balancer.py` +- No new settings, no migration, no dashboard surface. Routing decisions are + byte-identical; only the redundant same-owner freshness write is coalesced. +- Operators see `updated_at` in the dashboard sticky-session list advance in + steps of up to the skip window on hot sessions instead of per request. diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md b/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..828b742e49 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/specs/sticky-session-operations/spec.md @@ -0,0 +1,82 @@ +## ADDED Requirements + +### Requirement: Same-owner sticky refresh writes are coalesced + +When selection retains the existing pinned owner of a TTL-based sticky mapping, the +mapping write exists only to advance the mapping's freshness timestamp. The system +MUST skip that write when the same request's owner lookup already observed the row +with a freshness timestamp younger than a bounded skip window, so concurrent requests +of one hot session do not serialize on the same row's lock. + +The skip window MUST NOT exceed 1% of the mapping's configured TTL and MUST NOT +exceed 15 seconds, so a mapping's effective expiry — on both the read-path TTL check +and the background cleanup loop — moves at most that window earlier than today's +write-per-request behavior. + +The skip decision MUST be derived from row state observed in the current request's +database lookup, not from cross-request in-process state, so any number of workers or +replicas remain correct. The lookup MUST report the skip as a deadline (the observed +freshness timestamp plus the skip window), and the write path MUST revalidate that +deadline against the clock at the moment the write would otherwise be issued — a +deadline that lapsed while the request was being admitted no longer authorizes a +skip. A row whose observed freshness timestamp lies in the future (clock skew or a +restored row) MUST NOT be skippable at all. + +A skip MUST apply only to a pure freshness rewrite. The following writes MUST remain +immediate and unconditional: rebinding the mapping to a different account, deleting +the mapping, restoring a provisional owner after failed admission, initializing a +seed mapping, and any upsert against a row carrying an abandonment marker (whose +write also clears the marker columns). In particular, a retention write that would +initialize a missing seed mapping MUST NOT be skipped even when the retained row +itself was observed fresh, because the seed initialization piggybacks on that write. +A raw legacy owner that shadows the namespaced row MUST NOT inherit the namespaced +row's freshness observation. + +#### Scenario: Hot same-owner retention skips the redundant refresh write + +- **GIVEN** a `prompt_cache` mapping pinned to an eligible account +- **AND** the request's owner lookup observed the row fresher than the skip window + with no abandonment marker +- **WHEN** selection retains the pinned account +- **THEN** the request routes to the pinned account +- **AND** no sticky-session write is issued for the retention + +#### Scenario: Retention outside the skip window refreshes write-through + +- **GIVEN** a `prompt_cache` mapping pinned to an eligible account +- **AND** the row's freshness timestamp is older than the skip window but inside the TTL +- **WHEN** selection retains the pinned account +- **THEN** the mapping's freshness timestamp is advanced by a write + +#### Scenario: Rebind is never coalesced + +- **GIVEN** a soft mapping whose row was observed fresher than the skip window +- **WHEN** selection rebinds the mapping to a different account +- **THEN** the rebind is persisted immediately + +#### Scenario: A skipped refresh does not clobber a concurrent rebind + +- **GIVEN** a request that observed a fresh same-owner row and skipped its refresh write +- **AND** a concurrent request rebinds the same mapping to another account +- **WHEN** both requests complete +- **THEN** the mapping's owner is the rebind target + +#### Scenario: A retention that must initialize a missing seed is never skipped + +- **GIVEN** a thread mapping observed fresher than the skip window +- **AND** the corresponding process seed mapping does not exist +- **WHEN** selection retains the thread mapping's pinned account +- **THEN** the retention write is issued and the seed mapping is initialized + +#### Scenario: A deadline that lapsed during admission writes through + +- **GIVEN** a request whose lookup observed the row inside the skip window +- **AND** admission latency carried the request past the observed skip deadline +- **WHEN** the retention write would be issued +- **THEN** the deadline is revalidated and the freshness write is performed + +#### Scenario: A future freshness timestamp is never skippable + +- **GIVEN** a mapping whose freshness timestamp lies ahead of the current clock +- **WHEN** the owner lookup evaluates the skip window +- **THEN** no skip deadline is reported and retention writes through diff --git a/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md b/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md new file mode 100644 index 0000000000..5ff96ff431 --- /dev/null +++ b/openspec/changes/coalesce-sticky-same-owner-refresh/tasks.md @@ -0,0 +1,47 @@ +# Tasks + +## 1. Repository freshness observation + +- [x] 1.1 Extend `StickyOwnerLookup` with `refresh_skip_deadline` + (`observed_updated_at + skip window`), computed only on the fresh-row TTL + lookup path: `updated_at` within `min(15s, 1% of TTL)`, not in the future, + and both abandonment marker columns NULL +- [x] 1.2 Keep the deadline unset on the stale-delete recovery path, on lookups + without a TTL, on rows carrying any abandonment marker, and on rows whose + `updated_at` is ahead of the clock + +## 2. Selection wiring + +- [x] 2.1 Thread the deadline from `run_sticky_selection_path`'s per-attempt owner + lookup through `_select_with_stickiness` onto the retention mutation; reset it + when the raw legacy owner shadows the namespaced row, when the inner helper + re-resolves the owner itself, and when the process seed is still missing + (seed initialization piggybacks on the retention write) +- [x] 2.2 Revalidate the deadline at the persist site (`_sticky_refresh_write_skippable`) + and only then omit the same-owner refresh statement — on both the non-probe + persist path and the recovery-probe admission path (whose compensating + restores are skipped symmetrically when nothing was written); rebinds, + deletes, restores of actually-written rows, and seed-initializing writes + keep writing immediately + +## 3. Verification + +- [x] 3.1 Unit tests: skip on fresh same-owner retention, write-through when the + deadline is unset or lapsed at persist time, rebind/departed-owner writes never + suppressed, grace-period retention honors the window, internal re-resolution + resets the deadline, persist-time gate guards deletes/seed writes/non-datetime + deadlines +- [x] 3.2 Balancer-level tests: fresh same-owner retention issues no write when the + seed exists, a fresh thread row with a missing seed still writes and initializes + the seed, an expired deadline writes through, and a fresh retention of a + due-probing pinned owner skips the write on the probe admission path while + the probe reservation still commits +- [x] 3.3 Integration tests: deadline conditions against the real repository (fresh + row, TTL-scaled window, marker disqualification, future timestamp, no-TTL + lookup), concurrent upserts on one `(key, kind)` keep RETURNING/self-write and + single-row semantics, a skipped refresh never clobbers a concurrent rebind +- [x] 3.4 `uv run pytest tests/unit/test_select_with_stickiness.py + tests/unit/test_load_balancer_concurrency.py + tests/integration/test_proxy_sticky_sessions.py`, `uv run ruff check`, + `uv run ruff format --check`, `make typecheck` +- [x] 3.5 `openspec validate coalesce-sticky-same-owner-refresh --strict` diff --git a/tests/integration/test_proxy_sticky_sessions.py b/tests/integration/test_proxy_sticky_sessions.py index 0302976c05..95e49fe25c 100644 --- a/tests/integration/test_proxy_sticky_sessions.py +++ b/tests/integration/test_proxy_sticky_sessions.py @@ -3,7 +3,7 @@ import asyncio import base64 import json -from datetime import timedelta, timezone +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import cast @@ -2926,3 +2926,198 @@ async def test_seed_hard_sticky_outage_grace_on_startup_refreshes_only_unavailab entry = await repo.get_entry(f"turn_{account_id}", kind=StickySessionKind.CODEX_SESSION) assert entry is not None assert entry.updated_at == long_ago + + +async def _create_account(account_id: str) -> None: + encryptor = TokenEncryptor() + async with SessionLocal() as session: + await AccountsRepository(session).upsert( + Account( + id=account_id, + email=f"{account_id}@example.com", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + + +async def _backdate_sticky_row(key: str, kind: StickySessionKind, *, age_seconds: float) -> None: + from app.db.models import StickySession + + async with SessionLocal() as session: + await session.execute( + update(StickySession) + .where(StickySession.key == key, StickySession.kind == kind) + .values(updated_at=utcnow() - timedelta(seconds=age_seconds)) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_sticky_lookup_refresh_skippable_only_for_fresh_unmarked_rows(db_setup): + """refresh_skip_deadline is set only when a same-owner upsert would be a + pure updated_at rewrite: fresh within min(15s, 1% of TTL), not stamped in + the future, and free of any abandonment marker.""" + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_refresh_skip") + key = "key_refresh_skip" + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(key, "acc_refresh_skip", kind=StickySessionKind.PROMPT_CACHE) + + fresh = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert fresh.account_id == "acc_refresh_skip" + assert isinstance(fresh.refresh_skip_deadline, datetime) + # The deadline is observed_updated_at + window: never further out + # than the full window from now. + assert fresh.refresh_skip_deadline <= utcnow() + timedelta(seconds=15.0) + + # Without a TTL there is no refresh write to skip. + durable = await repo.get_account_id_and_abandonment(key, kind=StickySessionKind.PROMPT_CACHE) + assert durable.account_id == "acc_refresh_skip" + assert durable.refresh_skip_deadline is None + + # 10s old: inside the 15s cap for an 1800s TTL, but outside 1% of a 600s + # TTL (6s) — the window scales with the TTL it protects. + await _backdate_sticky_row(key, StickySessionKind.PROMPT_CACHE, age_seconds=10.0) + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + within_cap = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert within_cap.account_id == "acc_refresh_skip" + assert isinstance(within_cap.refresh_skip_deadline, datetime) + beyond_fraction = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=600, + ) + assert beyond_fraction.account_id == "acc_refresh_skip" + assert beyond_fraction.refresh_skip_deadline is None + + # A future updated_at (database clock ahead of the application, or a + # restored row) is never skippable: an upper-bound-only age check would + # otherwise satisfy the window for longer than the documented bound. + await _backdate_sticky_row(key, StickySessionKind.PROMPT_CACHE, age_seconds=-30.0) + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + future_stamped = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert future_stamped.account_id == "acc_refresh_skip" + assert future_stamped.refresh_skip_deadline is None + + # An abandonment marker disqualifies the skip even on a fresh row: the + # upsert that would be skipped also clears the marker columns. + async with SessionLocal() as session: + await session.execute( + update(StickySession) + .where(StickySession.key == key, StickySession.kind == StickySessionKind.PROMPT_CACHE) + .values(updated_at=utcnow(), continuity_abandonment_scope="session_header") + ) + await session.commit() + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + marked = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + continuity_source="turn_state", + ) + # Non-matching source keeps the owner, but the marker still makes a + # same-owner upsert semantic (it would clear the scope). + assert marked.account_id == "acc_refresh_skip" + assert marked.refresh_skip_deadline is None + + +@pytest.mark.asyncio +async def test_sticky_upsert_concurrent_same_key_semantics(db_setup): + """Concurrent upserts on one (key, kind) must each observe their own + write in RETURNING, keep exactly one row, and settle on one of the + written owners.""" + from sqlalchemy import func as sa_func + from sqlalchemy import select + + from app.db.models import StickySession + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_conc_a") + await _create_account("acc_conc_b") + key = "key_concurrent_upsert" + started_at = utcnow() + + async def _one_upsert(index: int) -> str: + account_id = "acc_conc_a" if index % 2 == 0 else "acc_conc_b" + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + row = await repo.upsert(key, account_id, kind=StickySessionKind.PROMPT_CACHE) + assert row.key == key + # RETURNING must reflect this statement's own write, not a + # concurrent winner's row. + assert row.account_id == account_id + assert row.continuity_abandoned_at is None + return row.account_id + + results = await asyncio.gather(*(_one_upsert(index) for index in range(12))) + assert set(results) == {"acc_conc_a", "acc_conc_b"} + + async with SessionLocal() as session: + row_count = await session.scalar( + select(sa_func.count()) + .select_from(StickySession) + .where(StickySession.key == key, StickySession.kind == StickySessionKind.PROMPT_CACHE) + ) + assert row_count == 1 + final = await StickySessionsRepository(session).get_entry(key, kind=StickySessionKind.PROMPT_CACHE) + assert final is not None + assert final.account_id in {"acc_conc_a", "acc_conc_b"} + # Backend timestamps may carry second precision only. + assert final.updated_at >= started_at.replace(microsecond=0) + + +@pytest.mark.asyncio +async def test_sticky_refresh_skip_never_clobbers_concurrent_rebind(db_setup): + """A request that observed a fresh same-owner row and skipped its refresh + write must leave a concurrent rebind to another account intact.""" + from app.modules.proxy.sticky_repository import StickySessionsRepository + + await _create_account("acc_skip_old") + await _create_account("acc_skip_new") + key = "key_skip_vs_rebind" + + async with SessionLocal() as session: + repo = StickySessionsRepository(session) + await repo.upsert(key, "acc_skip_old", kind=StickySessionKind.PROMPT_CACHE) + lookup = await repo.get_account_id_and_abandonment( + key, + kind=StickySessionKind.PROMPT_CACHE, + max_age_seconds=1800, + ) + assert lookup.account_id == "acc_skip_old" + # The selection layer would skip its same-owner refresh here. + assert isinstance(lookup.refresh_skip_deadline, datetime) + + # Concurrent request rebinds the mapping while the first request is still + # in flight; the first request performs no compensating write. + async with SessionLocal() as session: + await StickySessionsRepository(session).upsert(key, "acc_skip_new", kind=StickySessionKind.PROMPT_CACHE) + + async with SessionLocal() as session: + final = await StickySessionsRepository(session).get_account_id(key, kind=StickySessionKind.PROMPT_CACHE) + assert final == "acc_skip_new" diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index 5bf62dcbc7..ab433a77dd 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -5,7 +5,7 @@ import time from collections.abc import AsyncIterator, Collection from contextlib import asynccontextmanager -from datetime import datetime, timezone +from datetime import datetime, timedelta, timezone from types import SimpleNamespace from typing import Any, Literal, cast from unittest.mock import AsyncMock @@ -227,6 +227,9 @@ def __init__(self) -> None: # ambiguous-owner check for them. self.abandoned_keys: set[str] = set() self.scoped_abandoned_account_ids_by_key: dict[str, str] = {} + # Refresh-skip deadlines reported alongside the owner lookup, keyed by + # sticky key (see StickyOwnerLookup.refresh_skip_deadline). + self.refresh_skip_deadlines_by_key: dict[str, datetime] = {} self.deleted: list[tuple[str, StickySessionKind | None]] = [] self.upserts: list[tuple[str, str, StickySessionKind | None]] = [] self.insert_if_absent_calls: list[tuple[str, str, StickySessionKind]] = [] @@ -248,9 +251,17 @@ async def get_account_id_and_abandonment(self, *args: Any, **kwargs: Any) -> Sti if key in self.abandoned_keys: return StickyOwnerLookup(account_id=None, continuity_abandoned=True) if self.account_ids_by_key is not None: - return StickyOwnerLookup(account_id=self.account_ids_by_key.get(key), continuity_abandoned=False) + return StickyOwnerLookup( + account_id=self.account_ids_by_key.get(key), + continuity_abandoned=False, + refresh_skip_deadline=self.refresh_skip_deadlines_by_key.get(key), + ) del kwargs - return StickyOwnerLookup(account_id=self.account_id, continuity_abandoned=False) + return StickyOwnerLookup( + account_id=self.account_id, + continuity_abandoned=False, + refresh_skip_deadline=self.refresh_skip_deadlines_by_key.get(key), + ) async def upsert(self, *args: Any, **kwargs: Any) -> Any: sticky_key = cast(str, args[0]) @@ -3027,6 +3038,116 @@ async def test_first_codex_thread_initializes_process_preference_once_for_later_ ] +@pytest.mark.asyncio +async def test_fresh_same_owner_retention_skips_refresh_write_when_seed_exists() -> None: + """A hot same-owner retention whose lookup observed the row inside the + refresh-skip window issues no sticky write at all.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-refresh") + assert alternate is not None + process_session = "process-skip-refresh" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-skip"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {process_key: owner.id, thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [] + assert sticky_repo.seeded_upserts == [] + assert sticky_repo.deleted == [] + + +@pytest.mark.asyncio +async def test_fresh_thread_row_with_missing_seed_still_writes_and_initializes_seed() -> None: + """Seed initialization piggybacks on the thread retention write; a fresh + thread row must not suppress it while the process seed is absent.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-seedless") + assert alternate is not None + process_session = "process-skip-seedless" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-seedless"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.account_ids_by_key[process_key] == owner.id + assert sticky_repo.seeded_upserts == [ + ( + thread_key, + owner.id, + StickySessionKind.PROMPT_CACHE, + process_key, + StickySessionKind.CODEX_SESSION, + ) + ] + + +@pytest.mark.asyncio +async def test_expired_refresh_skip_deadline_still_writes_through() -> None: + """A deadline that lapsed between lookup and persist must not suppress the + refresh: the skip window is revalidated at write time.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-expired") + assert alternate is not None + process_session = "process-skip-expired" + process_key = _codex_session_selection_key(process_session) + thread_key = _codex_backend_identity( + {"session-id": process_session, "thread-id": "thread-expired"} + ).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {process_key: owner.id, thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) - timedelta(seconds=1) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + legacy_sticky_key=process_session, + sticky_seed_key=process_key, + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [(thread_key, owner.id, StickySessionKind.PROMPT_CACHE)] + + @pytest.mark.asyncio async def test_required_file_owner_does_not_rewrite_existing_thread_row() -> None: balancer, thread_owner, file_owner, sticky_repo = _make_cap_spillover_balancer("file-pin-thread") @@ -4439,3 +4560,110 @@ async def test_api_key_fair_share_concurrent_sticky_selections_cannot_overshoot_ # The commit re-check kept heavy at exactly its share across both paths. heavy_total = sum((runtime.stream_key_inflight or {}).get("heavy", 0) for runtime in balancer._runtime.values()) assert heavy_total == 2 + + +@pytest.mark.asyncio +async def test_fresh_same_owner_retention_skips_refresh_write_on_probe_admission() -> None: + """The recovery-probe admission path honors the refresh-skip deadline the + same way the non-probe persist site does: a fresh same-owner retention of + a due-probing pinned owner issues no sticky write.""" + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + healthy = _make_account("acc-probe-skip-healthy") + probing = _make_account("acc-probe-skip-probing") + key = "probe-skip-session" + + def _build(sticky_repo: _StubStickySessionsRepository) -> LoadBalancer: + accounts_repo = _StubAccountsRepository([healthy, probing]) + usage_repo = _StubUsageRepository( + primary={ + healthy.id: _usage_row_with_percent( + 150, + healthy.id, + used_percent=30.0, + reset_at=now_epoch + 300, + ), + probing.id: _usage_row_with_percent( + 151, + probing.id, + used_percent=10.0, + reset_at=now_epoch + 300, + ), + }, + secondary={}, + ) + balancer = LoadBalancer(lambda: _repo_factory(accounts_repo, usage_repo, sticky_repo)) + balancer._runtime[probing.id] = RuntimeState( + health_tier=HEALTH_TIER_PROBING, + last_selected_at=0.0, + version=17, + ) + return balancer + + # Control: without a freshness observation the probe admission persists + # the retention write, proving this scenario exercises the probe branch. + control_repo = _StubStickySessionsRepository() + control_repo.account_ids_by_key = {key: probing.id} + control_balancer = _build(control_repo) + control = await control_balancer.select_account( + sticky_key=key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert control.account is not None + assert control.account.id == probing.id + assert control_repo.upserts == [(key, probing.id, StickySessionKind.PROMPT_CACHE)] + await control_balancer.release_account_lease(control.lease) + + skip_repo = _StubStickySessionsRepository() + skip_repo.account_ids_by_key = {key: probing.id} + skip_repo.refresh_skip_deadlines_by_key[key] = datetime.now(tz=timezone.utc).replace(tzinfo=None) + timedelta( + seconds=10 + ) + skip_balancer = _build(skip_repo) + selected = await skip_balancer.select_account( + sticky_key=key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + lease_kind="stream", + ) + assert selected.account is not None + assert selected.account.id == probing.id + assert skip_repo.upserts == [] + assert skip_repo.deleted == [] + # The probe reservation itself still committed: runtime advanced. + probing_runtime = skip_balancer._runtime[probing.id] + assert probing_runtime.version > 17 + assert probing_runtime.last_selected_at is not None + assert probing_runtime.last_selected_at > 0.0 + await skip_balancer.release_account_lease(selected.lease) + + +@pytest.mark.asyncio +async def test_fresh_thread_only_retention_without_seed_key_skips_refresh_write() -> None: + """Thread-only affinity (no process seed key at all) has nothing to + initialize, so a fresh same-owner retention skips its refresh write.""" + balancer, owner, alternate, sticky_repo = _make_cap_spillover_balancer("thread-skip-no-seedkey") + assert alternate is not None + thread_key = _codex_backend_identity({"thread-id": "thread-only-skip"}).thread_selection_key + assert thread_key is not None + sticky_repo.account_ids_by_key = {thread_key: owner.id} + sticky_repo.refresh_skip_deadlines_by_key[thread_key] = datetime.now(tz=timezone.utc).replace( + tzinfo=None + ) + timedelta(seconds=10) + + selected = await balancer.select_account( + sticky_key=thread_key, + sticky_kind=StickySessionKind.PROMPT_CACHE, + sticky_source="thread_header", + sticky_max_age_seconds=300, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + assert sticky_repo.upserts == [] + assert sticky_repo.seeded_upserts == [] + assert sticky_repo.deleted == [] diff --git a/tests/unit/test_select_with_stickiness.py b/tests/unit/test_select_with_stickiness.py index e4d9436a39..fd4f6fd1f7 100644 --- a/tests/unit/test_select_with_stickiness.py +++ b/tests/unit/test_select_with_stickiness.py @@ -9,6 +9,7 @@ import time from contextlib import asynccontextmanager +from datetime import datetime, timedelta, timezone from typing import cast from unittest.mock import AsyncMock @@ -16,6 +17,10 @@ from app.core.balancer import AccountState, RoutingCost, RoutingCostsByAccount, RoutingStrategy from app.db.models import Account, AccountStatus, StickySessionKind +from app.modules.proxy._load_balancer.sticky_selection import ( + _STICKY_EXISTING_UNSET, + _sticky_refresh_write_skippable, +) from app.modules.proxy.load_balancer import LoadBalancer pytestmark = pytest.mark.unit @@ -77,6 +82,8 @@ async def _invoke_stickiness( relative_availability_power: float = 2.0, relative_availability_top_k: int = 5, routing_costs_by_account_id: RoutingCostsByAccount | None = None, + sticky_refresh_skip_deadline: datetime | None = None, + sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET, ): """Wrapper that calls production LoadBalancer._select_with_stickiness. @@ -107,8 +114,16 @@ async def mock_repo_factory(): relative_availability_top_k=relative_availability_top_k, sticky_repo=sticky_repo, routing_costs_by_account_id=routing_costs_by_account_id, + sticky_refresh_skip_deadline=sticky_refresh_skip_deadline, + sticky_existing_account_id=sticky_existing_account_id, ) - if outcome.mutation is not None: + # Mirror the production persist site (run_sticky_selection_path): a pure + # same-owner freshness rewrite is omitted only after revalidating its + # observed skip deadline at write time. + if outcome.mutation is not None and not _sticky_refresh_write_skippable( + outcome.mutation, + initialize_seed_key=None, + ): await lb._persist_sticky_mutation( sticky_repo=sticky_repo, sticky_key=sticky_key, @@ -118,6 +133,10 @@ async def mock_repo_factory(): return outcome.selection +def _future_deadline(seconds: float = 10.0) -> datetime: + return datetime.now(tz=timezone.utc).replace(tzinfo=None) + timedelta(seconds=seconds) + + # --------------------------------------------------------------------------- # Fix 1+3: sticky session is preserved when pinned account is temporarily down # --------------------------------------------------------------------------- @@ -1085,3 +1104,174 @@ async def test_burn_first_reallocation_only_when_burn_first_is_selectable(): assert result.account.account_id == "a" repo.delete.assert_not_called() repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +# --------------------------------------------------------------------------- +# Same-owner refresh skip: hot (key, kind) rows must not be rewritten on every +# request when the lookup already observed a fresh row. +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_refresh_skippable_pinned_retention_skips_upsert(): + """A healthy pinned owner within the refresh-skip window routes to the + pinned account without any sticky write.""" + acc_a = _active("a", used_percent=10.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_not_called() + repo.delete.assert_not_called() + + +@pytest.mark.asyncio +async def test_refresh_skippable_false_pinned_retention_still_refreshes(): + """Without the freshness observation the pinned retention keeps its + write-through updated_at refresh.""" + acc_a = _active("a", used_percent=10.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=None, + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_never_suppresses_reallocation_write(): + """Budget-pressure rebind to a different owner must persist immediately + even when the old row was observed fresh.""" + acc_a = _active("a", used_percent=96.0) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "b" + repo.upsert.assert_called_once_with("key1", "b", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_never_suppresses_departed_owner_rebind(): + """A pinned owner that left the pool is still rebound with an immediate + write even when the old row was observed fresh.""" + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "b" + repo.upsert.assert_called_once_with("key1", "b", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skippable_grace_period_retention_skips_upsert(): + """The grace-period pinned retention also honors the skip window.""" + now = time.time() + pinned = _rate_limited("a", reset_at=now + 10) + acc_b = _active("b", used_percent=50.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [pinned, acc_b], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_not_called() + + +@pytest.mark.asyncio +async def test_refresh_skippable_reset_when_existing_owner_not_prefetched(): + """The freshness observation belongs to the caller-provided lookup; an + internal owner lookup must fall back to write-through refresh.""" + acc_a = _active("a", used_percent=10.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(), + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +@pytest.mark.asyncio +async def test_refresh_skip_deadline_expired_at_persist_time_still_refreshes(): + """The skip deadline is revalidated at write time: a deadline that lapsed + between lookup and persist must not suppress the refresh, keeping the + mapping's effective expiry within the documented skip-window bound.""" + acc_a = _active("a", used_percent=10.0) + repo = _make_sticky_repo(existing_account_id="a") + + result = await _invoke_stickiness( + [acc_a], + "key1", + repo, + sticky_refresh_skip_deadline=_future_deadline(-0.5), + sticky_existing_account_id="a", + ) + + assert result.account is not None + assert result.account.account_id == "a" + repo.upsert.assert_called_once_with("key1", "a", kind=StickySessionKind.PROMPT_CACHE) + + +def test_refresh_write_skippable_guards_seed_and_delete_and_deadline(): + """The persist-time gate never skips deletes, seed-initializing writes, + non-datetime deadlines (auto-vivified test doubles), or lapsed deadlines.""" + from app.modules.proxy._load_balancer.sticky_selection import _StickyMutation + + refresh = _StickyMutation(account_id="a", refresh_skip_deadline=_future_deadline()) + assert _sticky_refresh_write_skippable(refresh, initialize_seed_key=None) is True + # Seed initialization piggybacks on this write and must never be skipped. + assert _sticky_refresh_write_skippable(refresh, initialize_seed_key="seed-key") is False + # Deletes are never skippable. + delete = _StickyMutation(account_id=None, refresh_skip_deadline=_future_deadline()) + assert _sticky_refresh_write_skippable(delete, initialize_seed_key=None) is False + # A lapsed deadline fails revalidation. + expired = _StickyMutation(account_id="a", refresh_skip_deadline=_future_deadline(-0.5)) + assert _sticky_refresh_write_skippable(expired, initialize_seed_key=None) is False + # Mutations without an observed deadline always write through. + plain = _StickyMutation(account_id="a") + assert _sticky_refresh_write_skippable(plain, initialize_seed_key=None) is False From 6ff22e0e528fb7bdd6c69c059178681254b143af Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:34:36 +0900 Subject: [PATCH 054/117] docs(dashboard): clarify routing, sticky affinity, quota thresholds, warm-up, and eligibility copy (#1781) * docs(dashboard): clarify routing, sticky affinity, quota thresholds, warm-up, and eligibility copy Implements the frontend copy/help/docs slice of #1708 (items 1-6): - Sticky threads is described as a soft preference, with a note that hard Codex continuation affinity (turn state, previous responses, uploaded files) is not disabled by the toggle. - New "Primary vs secondary quota" explainer names the 5-hour and weekly windows and states the used-vs-remaining unit split between account pages and settings thresholds. - Sticky threshold descriptions name the window and unit (percent used) and render a live "X% used - equivalent to Y% remaining" hint. - Prefer earlier reset describes the actual selection behavior (earliest reset bucket of the selected window, weekly compared by day, not applied under relative availability). - Limit warm-up states that a probe is one small real request using the configured model/prompt and consumes a small amount of quota. - The Active status badge on the accounts list carries a hint that configured status is not per-request eligibility. - docs/routing.md gains a routing/quotas/eligibility explainer section. - en/ko/zh-CN i18n keys added/updated. The per-account "why not selected" inspector and the actionable "No available accounts" breakdown are deferred to a selector-plumbing follow-up. Refs #1708 Co-Authored-By: Claude Fable 5 * fix(dashboard): address codex review round 1 on routing help copy - Name the strategies that honor prefer-earlier-reset (capacity weighted, usage weighted, fill first) instead of implying only relative availability ignores it. - Qualify the secondary window as weekly-or-monthly since monthly-only plans normalize the monthly window into the secondary slot. - Describe the Active badge as the displayed (derived) status rather than the configured status. - Derive the remaining percent from the rounded used percent so the threshold hint values always sum to exactly 100. Co-Authored-By: Claude Fable 5 * fix(dashboard): address codex review round 2 on routing help copy - Warm-up copy describes a confirmed newly reset quota window without asserting prior exhaustion or traffic gating (eligibility does not require the previous sample to be exhausted, and a probe failure only records an attempt). - Docs scope the warm-up cooldown to staggered idle probes; ordinary reset-confirmed probes fire once per confirmed reset. - The Active eligibility hint is also exposed as the focusable account row's native title, giving keyboard and screen-reader users an accessible description instead of a hover-only badge tooltip. - docs/routing.md footer links the frontend-architecture and usage-refresh-policy specs alongside account-routing. Co-Authored-By: Claude Fable 5 * fix(dashboard): address codex review round 3 on routing help copy - Qualify the threshold hint as quota-only arithmetic and note in the quota-window explainer that routing counts in-flight work as temporary extra usage, so thresholds can trigger before raw account-page numbers reach the value. - Add a visible status-vs-eligibility note under the accounts list so the hint is discoverable without pointer hover; keep the row/badge titles for hover and screen-reader description. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- docs/routing.md | 32 +++++++- frontend/src/components/status-badge.tsx | 5 +- .../components/account-list-item.test.tsx | 24 ++++++ .../accounts/components/account-list-item.tsx | 7 +- .../accounts/components/account-list.test.tsx | 29 +++++++ .../accounts/components/account-list.tsx | 6 ++ .../components/routing-settings.test.tsx | 59 ++++++++++++++ .../settings/components/routing-settings.tsx | 54 ++++++++++--- frontend/src/i18n/locales/en.json | 16 ++-- frontend/src/i18n/locales/ko.json | 16 ++-- frontend/src/i18n/locales/zh-CN.json | 16 ++-- .../proposal.md | 52 +++++++++++++ .../specs/frontend-architecture/spec.md | 76 +++++++++++++++++++ .../clarify-routing-quota-help-copy/tasks.md | 17 +++++ 14 files changed, 380 insertions(+), 29 deletions(-) create mode 100644 openspec/changes/clarify-routing-quota-help-copy/proposal.md create mode 100644 openspec/changes/clarify-routing-quota-help-copy/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/clarify-routing-quota-help-copy/tasks.md diff --git a/docs/routing.md b/docs/routing.md index ece8042279..ad68f1a405 100644 --- a/docs/routing.md +++ b/docs/routing.md @@ -17,6 +17,36 @@ For low-volume, policy-compliant personal use, start with **Capacity weighted** Change the strategy live in the dashboard under **Settings → Routing** — no restart required. +## Routing, quotas, and eligibility explainer + +### Account eligibility vs displayed status + +An account's badge (`Active`, `Paused`, `Limited`, …) is its **displayed status**, derived from the durable account state plus current usage. Eligibility is decided **per request**: the selector can skip an `Active` account because of a cooldown, error backoff, a quota threshold or exhaustion, model/plan incompatibility, or because a thread's continuation state is owned by a different account. `Active` therefore does not mean "will serve the next request". + +### Soft sticky routing vs hard Codex continuation affinity + +These are two different mechanisms: + +- **Soft sticky routing** (the `Sticky threads` toggle and session/thread locality) is a *preference*: keep requests for the same session on the same account when possible, mostly to preserve warm upstream prompt caches. When the preferred account is unavailable or over the sticky thresholds, traffic can move. +- **Hard Codex continuation affinity** binds a request to the account that owns its continuation state — an explicit Codex turn state, a stored `previous_response_id`/conversation, or uploaded file ids. This binding is **not controlled by `Sticky threads`**: turning the toggle off does not make owner-bound requests portable. codex-lb releases the binding only when it can prove the request is a safe, account-neutral replay (or the continuation is migrated). + +If a thread's owner account becomes unavailable, requests that still require that owner can fail with `No available accounts` even though the rest of the pool is healthy. Starting a fresh thread (no continuation state) routes normally. + +### Primary vs secondary quota, used vs remaining + +- **Primary quota** is the short **5-hour** usage window. +- **Secondary quota** is the longer window: **weekly** on most plans, or **monthly** on plans that report only a monthly window (the monthly window is normalized into the secondary slot for routing). + +Account pages display each window as **percent remaining**; the sticky reallocation thresholds in Settings are **percent used**. A `Sticky secondary threshold` of `70` means "move sticky sessions off an account once more than 70% of its secondary (weekly or monthly) window has been used" — in quota terms, once less than 30% remains. Note that routing evaluates thresholds against reported usage **plus temporary in-flight pressure** (concurrent requests and leased tokens), so reallocation can begin slightly before the raw account-page numbers reach the threshold. + +### Prefer earlier reset + +When enabled and several accounts are otherwise eligible, selection is restricted to the accounts whose selected quota window (5h or weekly) resets soonest. Weekly resets are compared in whole-day buckets; when the selected window has no known reset time, the other window is used as a fallback. The preference applies to the `Capacity weighted`, `Usage weighted`, and `Fill first` strategies; the fixed-order and draw-based strategies (`Round robin`, `Relative availability`, `Sequential drain`, `Reset drain`, `Single account`) ignore it. + +### Limit warm-up + +Limit warm-up sends **one small real request** (using the configured warm-up model and prompt) to an opted-in account when one of its quota windows is confirmed to have newly reset, verifying that the account responds. It consumes a small amount of quota. The optional staggered idle mode additionally pre-starts the 5h window of idle opted-in accounts before traffic arrives; the configured cooldown applies to these staggered idle probes, while ordinary reset-confirmed probes fire once per confirmed reset. Accounts opt in individually (`Enable warm-up` in account actions); the last attempt's result, model, and time are shown on the account list entry. + --- -*Spec: [account-routing](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/account-routing)* +*Specs: [account-routing](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/account-routing) · [frontend-architecture](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/frontend-architecture) · [usage-refresh-policy](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/usage-refresh-policy)* diff --git a/frontend/src/components/status-badge.tsx b/frontend/src/components/status-badge.tsx index 7936c5c1fa..2d2b3e1372 100644 --- a/frontend/src/components/status-badge.tsx +++ b/frontend/src/components/status-badge.tsx @@ -16,15 +16,16 @@ const statusClassMap: Record = { export type StatusBadgeProps = { status: StatusValue; + title?: string; }; -export function StatusBadge({ status }: StatusBadgeProps) { +export function StatusBadge({ status, title }: StatusBadgeProps) { const { t } = useTranslation(); const className = statusClassMap[status] ?? statusClassMap.deactivated; const label = t(`common.status.${status}`, { defaultValue: status }); return ( - + {label} diff --git a/frontend/src/features/accounts/components/account-list-item.test.tsx b/frontend/src/features/accounts/components/account-list-item.test.tsx index 91f9cc0a68..ded9bbfd11 100644 --- a/frontend/src/features/accounts/components/account-list-item.test.tsx +++ b/frontend/src/features/accounts/components/account-list-item.test.tsx @@ -248,6 +248,30 @@ describe("AccountListItem", () => { expect(screen.queryByText("99+")).not.toBeInTheDocument(); }); + it("explains that Active is the displayed status, not per-request eligibility", () => { + const account = createAccountSummary({ status: "active" }); + + render(); + + // The hint lives on the focusable row (accessible description for + // keyboard/screen-reader users) and on the badge for pointer hover. + const hints = screen.getAllByTitle(/Active is the account's displayed status/i); + expect(hints.length).toBeGreaterThan(0); + expect(screen.getByRole("button")).toHaveAttribute( + "title", + expect.stringMatching(/Active is the account's displayed status/i), + ); + }); + + it("omits the eligibility hint for non-active statuses", () => { + const account = createAccountSummary({ status: "paused" }); + + render(); + + expect(screen.queryByTitle(/Active is the account's displayed status/i)).not.toBeInTheDocument(); + expect(screen.getByRole("button")).not.toHaveAttribute("title"); + }); + it("hides the reset-credit badge when badge display is disabled", () => { const account = createAccountSummary({ availableResetCredits: 3 }); diff --git a/frontend/src/features/accounts/components/account-list-item.tsx b/frontend/src/features/accounts/components/account-list-item.tsx index 7a794d7b98..0aff88162e 100644 --- a/frontend/src/features/accounts/components/account-list-item.tsx +++ b/frontend/src/features/accounts/components/account-list-item.tsx @@ -80,10 +80,15 @@ export function AccountListItem({ : t("accounts.listItem.noAttempts"); const availableResetCredits = account.availableResetCredits ?? 0; const resetBadgeLabel = availableResetCredits > 99 ? "99+" : String(availableResetCredits); + const statusEligibilityHint = status === "active" ? t("accounts.listItem.statusActiveHint") : undefined; return (
{ expect(screen.getByText("No accounts yet")).toBeInTheDocument(); expect(screen.getByText("Add an account to start routing.")).toBeInTheDocument(); expect(screen.queryByText("Adjust filters")).not.toBeInTheDocument(); + expect( + screen.queryByText(/Individual requests can still skip an Active account/i), + ).not.toBeInTheDocument(); + }); + + it("shows a visible status-vs-eligibility note when accounts exist", () => { + render( + {}} + onOpenImport={() => {}} + onOpenOauth={() => {}} + />, + ); + + expect( + screen.getByText(/Individual requests can still skip an Active account/i), + ).toBeInTheDocument(); }); it("keeps the add account action outside the scrollable account list", () => { diff --git a/frontend/src/features/accounts/components/account-list.tsx b/frontend/src/features/accounts/components/account-list.tsx index ad49806de8..3e2d24ae19 100644 --- a/frontend/src/features/accounts/components/account-list.tsx +++ b/frontend/src/features/accounts/components/account-list.tsx @@ -177,6 +177,12 @@ export function AccountList({ )}
+ {accounts.length > 0 ? ( +

+ {t("accounts.list.statusEligibilityNote")} +

+ ) : null} + { expect(screen.getByText(/No strategy can guarantee account-safety outcomes/i)).toBeInTheDocument(); }); + it("explains soft sticky routing versus hard Codex continuation affinity", () => { + render(); + + expect( + screen.getByText(/does not disable hard Codex continuation affinity/i), + ).toBeInTheDocument(); + expect(screen.getByText(/soft preference, not a guarantee/i)).toBeInTheDocument(); + }); + + it("explains primary versus secondary quota windows and threshold units", () => { + render(); + + expect(screen.getByText("Primary vs secondary quota")).toBeInTheDocument(); + expect( + screen.getByText(/Primary quota is the short 5-hour usage window/i), + ).toBeInTheDocument(); + expect( + screen.getByText(/5-hour \(primary\) window has been used/i), + ).toBeInTheDocument(); + expect( + screen.getByText(/secondary window \(weekly, or monthly on monthly-only plans\) has been used/i), + ).toBeInTheDocument(); + }); + + it("shows the remaining-percent equivalent for sticky thresholds", async () => { + const user = userEvent.setup(); + render(); + + // Defaults: primary 95% used, secondary 100% used. + expect(screen.getByText("95% used · 5% remaining in quota terms")).toBeInTheDocument(); + expect(screen.getByText("100% used · 0% remaining in quota terms")).toBeInTheDocument(); + + const secondary = screen.getByRole("spinbutton", { name: "Sticky secondary threshold" }); + await user.clear(secondary); + await user.type(secondary, "70"); + + expect(screen.getByText("70% used · 30% remaining in quota terms")).toBeInTheDocument(); + + // Decimal thresholds keep the two displayed values complementary. + await user.clear(secondary); + await user.type(secondary, "12.5"); + + expect(screen.getByText("12.5% used · 87.5% remaining in quota terms")).toBeInTheDocument(); + }); + + it("describes prefer-earlier-reset selection behavior", () => { + render(); + + expect( + screen.getByText(/prefer those whose selected quota window resets sooner/i), + ).toBeInTheDocument(); + }); + + it("describes what limit warm-up sends and that probes consume quota", () => { + render(); + + expect(screen.getByText(/consume a small amount of quota/i)).toBeInTheDocument(); + }); + it("saves staggered idle warm-up when limit warm-up is enabled", async () => { const user = userEvent.setup(); const onSave = vi.fn().mockResolvedValue(undefined); diff --git a/frontend/src/features/settings/components/routing-settings.tsx b/frontend/src/features/settings/components/routing-settings.tsx index 06f9e96083..5582c2adcb 100644 --- a/frontend/src/features/settings/components/routing-settings.tsx +++ b/frontend/src/features/settings/components/routing-settings.tsx @@ -144,6 +144,14 @@ function parseNonnegativeInteger(value: string): number | null { return Number.isSafeInteger(parsed) && parsed >= 0 ? parsed : null; } +function thresholdHintValues(value: number): { used: string; remaining: string } { + // Derive the remaining percent from the rounded used percent so the two + // displayed values always sum to exactly 100. + const used = Number(value.toFixed(1)); + const remaining = Number((100 - used).toFixed(1)); + return { used: String(used), remaining: String(remaining) }; +} + export function RoutingSettings({ settings, accounts = EMPTY_ACCOUNTS, @@ -796,23 +804,41 @@ export function RoutingSettings({ -
-
-

{t("settings.routing.stickyThreads.label")}

-

{t("settings.routing.stickyThreads.description")}

+
+
+
+

{t("settings.routing.stickyThreads.label")}

+

{t("settings.routing.stickyThreads.description")}

+
+ save({ stickyThreadsEnabled: checked })} + />
- save({ stickyThreadsEnabled: checked })} - /> +

+ {t("settings.routing.stickyThreads.hardAffinityNote")} +

+
+ +
+

{t("settings.routing.quotaWindows.title")}

+

{t("settings.routing.quotaWindows.explainer")}

{t("settings.routing.stickyThresholds.primaryLabel")}

{t("settings.routing.stickyThresholds.primaryDescription")}

+ {stickyPrimaryThresholdValid ? ( +

+ {t( + "settings.routing.stickyThresholds.usedRemainingHint", + thresholdHintValues(parsedStickyPrimaryThreshold), + )} +

+ ) : null}

{t("settings.routing.stickyThresholds.secondaryLabel")}

{t("settings.routing.stickyThresholds.secondaryDescription")}

+ {stickySecondaryThresholdValid ? ( +

+ {t( + "settings.routing.stickyThresholds.usedRemainingHint", + thresholdHintValues(parsedStickySecondaryThreshold), + )} +

+ ) : null}
Date: Mon, 17 Aug 2026 19:35:05 +0900 Subject: [PATCH 055/117] fix(server): serve h2c upgrade offers as plain HTTP/1.1 instead of rejecting them (#1782) * fix(server): serve non-WebSocket h2c upgrade offers as plain HTTP/1.1 JetBrains/Ktor clients attach opportunistic cleartext HTTP/2 upgrade headers (Connection: Upgrade + Upgrade: h2c + HTTP2-Settings) to normal HTTP/1.1 Responses API POSTs. Uvicorn's httptools protocol (picked by http="auto" because httptools ships transitively) treats any such request as a protocol switch and wedges the parser: a body coalesced with the headers is silently dropped (empty ASGI body, 422), and a body written as a separate segment - Ktor's write pattern - is answered with "400 Invalid HTTP request received." before auth ever runs. Subclass the httptools protocol to decline non-WebSocket upgrade offers per RFC 9110 section 7.8: defer the parser callbacks that would otherwise start the ASGI cycle with an empty body, then replay the request head through a fresh parser with the declined offer's hop-by-hop headers (Upgrade, HTTP2-Settings, and their Connection tokens) removed, serving the request as plain HTTP/1.1. WebSocket upgrades keep the stock fast path, and the bootstrap falls back to uvicorn's auto selection when httptools is unavailable (h11 already behaves correctly). Fixes #1757 Co-Authored-By: Claude Fable 5 * fix(server): harden upgrade-offer classification and h11 fallback Codex review round 1: - combine repeated Connection fields (RFC 9110 s5.3) when classifying an upgrade offer; uvicorn's _get_upgrade keeps only the last field's tokens, so a trailing "Connection: keep-alive" hid the offer and reproduced the original body loss - the httptools-less fallback now uses an h11 subclass that keeps the stock body delivery but strips the declined offer's hop-by-hop headers from the ASGI scope, matching the new spec requirement Co-Authored-By: Claude Fable 5 * fix(server): combined-field websocket handoff and /v1/responses regression Codex review round 2: - classify the WebSocket handoff with combined Connection fields too, so a handshake carrying a second "Connection: keep-alive" field still switches protocols instead of being served as HTTP - add a live-server regression at the externally failing product path: a split-written h2c POST to /v1/responses must traverse the parser into the application (deterministic no_accounts 503 on an empty pool proves body delivery) instead of the transport-layer 400 Co-Authored-By: Claude Fable 5 * fix(server): replay declined upgrade offers iteratively and fix typecheck The declined-offer replay in UpgradeTolerantHttpToolsProtocol recursed through data_received once per pipelined upgrade-offering request in the same TCP segment. Recursion depth was therefore attacker-controlled: a single ~66KB segment of minimal h2c GETs (well under asyncio's 256KiB per-read buffer) exceeded Python's default 1000-frame limit, the RecursionError escaped the protocol callback into the event loop ("Fatal error: protocol.data_received() call failed"), and each frame pinned its own head+remainder byte copy (O(depth x segment) transient memory). Replay in a loop instead: each iteration strips at least one declined offer, so the loop terminates with O(1) stack depth and one live byte copy. Regression test pipelines 2000 offers in one segment (raised RecursionError before the fix). Also make the branch ty-clean (required typecheck CI job): - _FakeTransport.write now accepts bytes | bytearray | memoryview to be a valid override of asyncio.Transport.write; - the __import__ wrapper in test_cli forwards *args/**kwargs as Any so the delegation to builtins.__import__ typechecks. Co-Authored-By: Claude Fable 5 * fix(server): classify list-valued Upgrade fields per RFC 9110 combined_upgrade_offer kept only the last Upgrade field's raw value, so a WebSocket handshake offering multiple protocols (Upgrade: websocket, h2c, or websocket split across repeated Upgrade fields) compared unequal to b"websocket", was misclassified as an ignorable offer, and was answered by the application over plain HTTP/1.1 instead of reaching the WebSocket stack. Tokenize all Upgrade fields (RFC 9110 sections 5.3 and 7.8) and pick websocket whenever it is offered; otherwise keep the client's first preference. The handshake verdict stays with the WebSocket stack (uvicorn's websockets implementation rejects multi-token values with 426; other implementations may complete the 101). Found by codex review round 1 (P2). Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/cli.py | 11 + app/core/http_protocol.py | 117 +++++ app/core/http_protocol_httptools.py | 146 ++++++ .../proposal.md | 43 ++ .../specs/http-ingress-limits/spec.md | 57 +++ .../tasks.md | 31 ++ .../test_http_upgrade_tolerance.py | 459 ++++++++++++++++++ tests/unit/test_cli.py | 22 + 8 files changed, 886 insertions(+) create mode 100644 app/core/http_protocol.py create mode 100644 app/core/http_protocol_httptools.py create mode 100644 openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md create mode 100644 openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md create mode 100644 openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md create mode 100644 tests/integration/test_http_upgrade_tolerance.py diff --git a/app/cli.py b/app/cli.py index f5d19146c5..93dae15a68 100644 --- a/app/cli.py +++ b/app/cli.py @@ -125,6 +125,12 @@ def _load_graceful_drain_server(): return GracefulDrainServer +def _load_http_protocol_class() -> Any: + from app.core.http_protocol import load_http_protocol_class + + return load_http_protocol_class() + + def _load_shutdown_drain_timeout_seconds() -> int: from app.core.config.settings import get_settings @@ -140,6 +146,11 @@ def _run_server(app: str, **kwargs: Any) -> None: # this explicitly prevents Uvicorn from treating ambient # WEB_CONCURRENCY as an unsupported multiprocess launch. workers=1, + # Serve valid HTTP/1.1 requests that opportunistically offer an h2c + # upgrade (JetBrains/Ktor clients) instead of rejecting them; the + # stock httptools protocol drops the body or answers 400. See + # app/core/http_protocol.py and issue #1757. + http=_load_http_protocol_class(), timeout_graceful_shutdown=drain_timeout_seconds, **kwargs, ) diff --git a/app/core/http_protocol.py b/app/core/http_protocol.py new file mode 100644 index 0000000000..93fcfa1d3b --- /dev/null +++ b/app/core/http_protocol.py @@ -0,0 +1,117 @@ +"""Uvicorn HTTP protocol selection tolerant of opportunistic upgrade offers. + +JetBrains/Ktor clients attach cleartext HTTP/2 upgrade headers +(``Connection: Upgrade, HTTP2-Settings`` + ``Upgrade: h2c`` + +``HTTP2-Settings``) to ordinary HTTP/1.1 Responses API POSTs. RFC 9110 +section 7.8 lets a server ignore such an offer and answer over HTTP/1.1 — +upstream OpenAI endpoints do exactly that — but uvicorn's stock protocol +implementations either wedge on the offer (httptools) or leak the declined +offer's hop-by-hop headers into the ASGI scope (h11). See +https://github.com/Soju06/codex-lb/issues/1757 and the module docstring of +``app.core.http_protocol_httptools`` for the full failure analysis. + +This module exposes :func:`load_http_protocol_class`, which returns the +tolerant httptools subclass when httptools is importable (matching uvicorn's +``auto`` preference) and an h11 subclass with the same header hygiene +otherwise. +""" + +from __future__ import annotations + +import asyncio + +from uvicorn.protocols.http.h11_impl import H11Protocol + +# Hop-by-hop headers that only exist to carry the declined protocol switch. +# ``HTTP2-Settings`` is defined exclusively for the h2c upgrade (RFC 9113 +# section 3.1) and MUST NOT be forwarded once the offer is declined. +UPGRADE_HOP_BY_HOP_HEADERS = frozenset({b"upgrade", b"http2-settings"}) + + +def combined_upgrade_offer(headers: list[tuple[bytes, bytes]]) -> bytes | None: + """Return the accepted ``Upgrade`` token, honoring repeated/list-valued fields. + + Unlike uvicorn's ``_get_upgrade`` — which keeps only the tokens of the + *last* ``Connection`` field (so ``Connection: Upgrade`` followed by + ``Connection: keep-alive`` hides the offer) and the last ``Upgrade`` + field's raw value (so ``Upgrade: websocket, h2c`` matches nothing) — + repeated fields are combined per RFC 9110 section 5.3 and the ``Upgrade`` + protocol list is tokenized per section 7.8. ``websocket`` is returned + whenever it is among the offered protocols (the server may pick any + offered protocol it supports); otherwise the client's first preference is + returned. Header names must already be lowercased (both uvicorn + implementations store them that way). + """ + connection_tokens: list[bytes] = [] + upgrade_tokens: list[bytes] = [] + for name, value in headers: + if name == b"connection": + connection_tokens.extend(token.lower().strip() for token in value.split(b",")) + elif name == b"upgrade": + upgrade_tokens.extend(token for token in (token.lower().strip() for token in value.split(b",")) if token) + if b"upgrade" not in connection_tokens or not upgrade_tokens: + return None + if b"websocket" in upgrade_tokens: + return b"websocket" + return upgrade_tokens[0] + + +def offers_ignorable_upgrade(headers: list[tuple[bytes, bytes]]) -> bool: + """True when the request offers a non-WebSocket protocol switch (e.g. h2c).""" + upgrade = combined_upgrade_offer(headers) + return upgrade is not None and upgrade != b"websocket" + + +def without_upgrade_headers(headers: list[tuple[bytes, bytes]]) -> list[tuple[bytes, bytes]]: + """Drop the declined offer's hop-by-hop headers and ``Connection`` tokens.""" + sanitized: list[tuple[bytes, bytes]] = [] + for name, value in headers: + if name in UPGRADE_HOP_BY_HOP_HEADERS: + continue + if name == b"connection": + tokens = [token.strip() for token in value.split(b",")] + kept = [token for token in tokens if token and token.lower() not in UPGRADE_HOP_BY_HOP_HEADERS] + if not kept: + continue + value = b", ".join(kept) + sanitized.append((name, value)) + return sanitized + + +class UpgradeTolerantH11Protocol(H11Protocol): + """h11 protocol that hides declined non-WebSocket upgrade offers from the app. + + The stock h11 implementation already serves such requests as plain + HTTP/1.1 with the full body, but it exposes the declined offer's + hop-by-hop headers in the ASGI scope and logs a spurious + "Unsupported upgrade request." warning. ``_should_upgrade`` is the seam: + it runs right after ``self.headers`` (the same list object referenced by + ``scope["headers"]``) is populated, so sanitizing in place here is enough. + """ + + def _should_upgrade(self) -> bool: + # Reimplements the stock decision on top of combined Connection fields + # (RFC 9110 section 5.3): the stock ``_get_upgrade`` keeps only the + # last field's tokens, so ``Connection: Upgrade`` followed by + # ``Connection: keep-alive`` would hide the offer entirely. + upgrade = combined_upgrade_offer(self.headers) + if upgrade is None: + return False + if upgrade == b"websocket": + if self._should_upgrade_to_ws(): + return True + self._unsupported_upgrade_warning() + return False + self.headers[:] = without_upgrade_headers(self.headers) + return False + + +def load_http_protocol_class() -> type[asyncio.Protocol]: + """Return the HTTP protocol implementation for ``uvicorn.Config(http=...)``.""" + try: + from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + except ImportError: + # httptools is an optional (transitive) dependency; uvicorn's "auto" + # selection would fall back to h11 as well. + return UpgradeTolerantH11Protocol + return UpgradeTolerantHttpToolsProtocol diff --git a/app/core/http_protocol_httptools.py b/app/core/http_protocol_httptools.py new file mode 100644 index 0000000000..7c1bfc82c3 --- /dev/null +++ b/app/core/http_protocol_httptools.py @@ -0,0 +1,146 @@ +"""Uvicorn httptools protocol that tolerates non-WebSocket upgrade offers. + +Uvicorn's ``auto`` HTTP implementation picks the ``httptools`` parser whenever +the ``httptools`` package is importable (it is, transitively via +``fastapi[standard]``). That parser treats *any* HTTP/1.1 request carrying +``Connection: Upgrade`` as a protocol switch: httptools raises +``HttpParserUpgrade`` at the end of the headers, never delivers the body, and +uvicorn only handles the WebSocket case. For every other ``Upgrade`` offer — +most notably the cleartext HTTP/2 (``h2c``) offer JetBrains/Ktor clients attach +to ordinary Responses API POSTs — uvicorn logs "Unsupported upgrade request." +and stops feeding the parser. Two failure shapes follow: + +- body coalesced with the headers: the body is silently dropped, so the + application sees an empty body (422 from request validation); +- headers and body written as separate segments (Ktor's write pattern): the + next bytes hit the wedged parser, ``HttpParserError`` follows, and the client + receives ``400 Bad Request / Invalid HTTP request received.`` + +RFC 9110 section 7.8 lets a server ignore an upgrade offer and answer over +HTTP/1.1 — upstream OpenAI endpoints do exactly that. This subclass neutralizes +non-WebSocket upgrade offers: the request head is replayed through a fresh +parser with the declined offer's hop-by-hop headers removed, and the request is +served as plain HTTP/1.1. Legitimate WebSocket upgrades keep the stock path. + +See https://github.com/Soju06/codex-lb/issues/1757. +""" + +from __future__ import annotations + +import httptools +from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol + +from app.core.http_protocol import combined_upgrade_offer, offers_ignorable_upgrade, without_upgrade_headers + + +class UpgradeTolerantHttpToolsProtocol(HttpToolsProtocol): + """httptools protocol that serves non-WebSocket upgrade offers as HTTP/1.1.""" + + def _active_parser(self) -> httptools.HttpRequestParser: + # The base class only clears ``self.parser`` in connection_lost, after + # which no parser callback or data_received can run. + parser = self.parser + assert parser is not None + return parser + + def _should_upgrade(self) -> bool: + # Combine repeated Connection fields (RFC 9110 section 5.3) so a + # trailing ``Connection: keep-alive`` field cannot hide a WebSocket + # handshake from the protocol switch (the stock ``_get_upgrade`` keeps + # only the last field's tokens). Also used by the stock parser + # callbacks to defer body handling until the handoff. + return combined_upgrade_offer(self.headers) == b"websocket" and self._should_upgrade_to_ws() + + def _paused_on_ignorable_upgrade(self) -> bool: + return self._active_parser().should_upgrade() and offers_ignorable_upgrade(self.headers) + + # -- Parser callbacks -------------------------------------------------- + # For an upgrade-offering request httptools fires on_headers_complete and + # on_message_complete *before* feed_data raises HttpParserUpgrade, and it + # never delivers the body. The stock callbacks would therefore start the + # ASGI cycle with an empty-but-complete body. Defer instead: data_received + # replays the sanitized request through a fresh parser, and these callbacks + # then run with ``should_upgrade()`` false. + + def on_headers_complete(self) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_headers_complete() + + def on_body(self, body: bytes) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_body(body) + + def on_message_complete(self) -> None: + if self._paused_on_ignorable_upgrade(): + return + super().on_message_complete() + + def data_received(self, data: bytes) -> None: + # Mirrors HttpToolsProtocol.data_received; the upgrade branch cannot be + # intercepted from outside because the stock method swallows the + # HttpParserUpgrade exception itself. + self._unset_keepalive_if_required() + + # Replay declined offers iteratively, not recursively: a single + # segment can pipeline many upgrade-offering requests (one replay + # each), so recursion depth would be attacker-controlled — ~66KB of + # minimal h2c GETs already exceeds Python's default 1000-frame limit, + # and the RecursionError would escape into the event loop and abort + # the connection. Each replay strips at least one declined offer from + # ``data``, so the loop terminates. + while True: + try: + self._active_parser().feed_data(data) + except httptools.HttpParserError: + msg = "Invalid HTTP request received." + self.logger.warning(msg) + self.send_400_response(msg) + except httptools.HttpParserUpgrade as exc: + if self._should_upgrade(): + self.handle_websocket_upgrade() + elif offers_ignorable_upgrade(self.headers): + data = self._continue_as_plain_http(data, exc) + continue + else: + self._unsupported_upgrade_warning() + return + + def _continue_as_plain_http(self, data: bytes, exc: httptools.HttpParserUpgrade) -> bytes: + """Decline the offered protocol switch; return the bytes to re-feed as HTTP/1.1.""" + self.logger.debug( + "Ignoring unsupported upgrade offer; serving the request as plain HTTP/1.1.", + ) + # httptools pauses at the end of the headers; the exception argument is + # the offset of the first unparsed byte in this segment (the body when + # it arrived coalesced with the headers). + offset = exc.args[0] if exc.args else len(data) + head = self._sanitized_request_head() + self.parser = httptools.HttpRequestParser(self) + try: + self.parser.set_dangerous_leniencies(lenient_data_after_close=True) + except AttributeError: # pragma: no cover - httptools < 0.6.3 + pass + # The sanitized head no longer carries upgrade headers, so re-feeding + # it cannot pause the fresh parser on the same offer (a *pipelined* + # follow-up offer pauses again and takes another loop iteration in + # data_received); malformed leftover bytes keep the stock 400 + # handling. Later segments of a split request feed the fresh parser + # through the normal data_received path. + return head + data[offset:] + + def _sanitized_request_head(self) -> bytes: + """Rebuild the parsed request head without the declined upgrade offer. + + ``self.url`` and ``self.headers`` were accumulated by the parser + callbacks of the aborted parse (header names already lowercased), so + the head is complete even when the client split it across segments. + """ + parser = self._active_parser() + method = parser.get_method() + http_version = parser.get_http_version().encode("ascii") + lines = [b"%s %s HTTP/%s\r\n" % (method, self.url, http_version)] + lines.extend(b"%s: %s\r\n" % (name, value) for name, value in without_upgrade_headers(self.headers)) + lines.append(b"\r\n") + return b"".join(lines) diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md new file mode 100644 index 0000000000..bc779b69fc --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/proposal.md @@ -0,0 +1,43 @@ +# Serve h2c Upgrade Offers as Plain HTTP/1.1 + +## Why + +JetBrains/Ktor clients attach opportunistic cleartext HTTP/2 upgrade headers +(`Connection: Upgrade, HTTP2-Settings` + `Upgrade: h2c` + `HTTP2-Settings`) to +ordinary HTTP/1.1 Responses API POSTs. The server's httptools-based HTTP parser +treats any such request as a protocol switch and wedges: a body coalesced with +the headers is silently dropped (the application validates an empty body and +returns 422), and a body written as a separate segment — Ktor's write pattern — +is answered with `400 Invalid HTTP request received.` before authentication +ever runs (issue #1757). RFC 9110 §7.8 allows a server to ignore an upgrade +offer and answer over HTTP/1.1, which is what upstream OpenAI endpoints do. + +## What Changes + +- Serve valid HTTP/1.1 requests that offer a non-WebSocket protocol switch + (such as `h2c`) as normal HTTP/1.1 requests, with the full body delivered to + the application for both client segmentations. +- Strip the declined offer's hop-by-hop headers (`Upgrade`, `HTTP2-Settings`, + and their `Connection` tokens) before the request reaches the application. +- Keep genuine WebSocket upgrades switching protocols exactly as today. +- No new settings, budgets, or defaults; no API or schema change. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `http-ingress-limits`: raw HTTP ingress MUST serve non-WebSocket HTTP/1.1 + upgrade offers as plain HTTP/1.1 instead of dropping the body or rejecting + the request. + +## Impact + +`app/cli.py` server bootstrap and new `app/core/http_protocol.py` / +`app/core/http_protocol_httptools.py` uvicorn protocol subclasses (the h11 +variant covers the httptools-less fallback with the same header hygiene), plus +transport-level regression coverage. No dashboard, API, schema, or +configuration change. diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md new file mode 100644 index 0000000000..ec39a2123e --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/specs/http-ingress-limits/spec.md @@ -0,0 +1,57 @@ +# http-ingress-limits Delta + +## ADDED Requirements + +### Requirement: Non-WebSocket upgrade offers are served as plain HTTP/1.1 + +The server MUST serve a valid HTTP/1.1 request that offers a non-WebSocket +protocol switch (`Connection: Upgrade` with an `Upgrade` token other than +`websocket`, such as `h2c`) as a normal HTTP/1.1 request. The complete request +body MUST reach the application whether it arrives coalesced with the headers +or in later TCP segments, and the offer MUST NOT cause the request or the +connection to be rejected. The declined offer's hop-by-hop headers (`Upgrade`, +`HTTP2-Settings`, and their `Connection` tokens) MUST NOT be exposed to the +application. Genuine WebSocket upgrade requests MUST keep completing the +protocol switch. + +#### Scenario: h2c offer with the body coalesced with the headers + +- **WHEN** a client sends an HTTP/1.1 POST carrying `Connection: Upgrade, + HTTP2-Settings`, `Upgrade: h2c`, and `HTTP2-Settings` headers with the body + in the same TCP segment as the headers +- **THEN** the application receives the complete request body +- **AND** the application does not observe the `Upgrade`, `HTTP2-Settings`, or + `Connection: Upgrade` headers + +#### Scenario: h2c offer with the body in a separate segment + +- **WHEN** the same request arrives with the headers and the body written as + separate TCP segments +- **THEN** the application receives the complete request body +- **AND** the server does not answer `400 Bad Request` at the protocol layer + +#### Scenario: Repeated Connection fields do not hide the offer + +- **WHEN** the h2c offer arrives with `Connection: Upgrade, HTTP2-Settings` + followed by a second `Connection: keep-alive` field +- **THEN** the application receives the complete request body +- **AND** the surviving `Connection` tokens (such as `keep-alive`) are + preserved while the upgrade tokens are removed + +#### Scenario: Connection stays usable after a declined offer + +- **WHEN** a request with a declined h2c offer completes on a keep-alive + connection +- **THEN** a subsequent plain HTTP/1.1 request on the same connection is + served normally + +#### Scenario: Pipelined offers in one segment do not exhaust the server + +- **WHEN** a single TCP segment pipelines many upgrade-offering requests +- **THEN** every request is served as plain HTTP/1.1 without the per-offer + replay growing the call stack or aborting the connection + +#### Scenario: WebSocket upgrades still switch protocols + +- **WHEN** a client requests a WebSocket upgrade (`Upgrade: websocket`) +- **THEN** the protocol switch completes and WebSocket messages flow diff --git a/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md b/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md new file mode 100644 index 0000000000..f7b94a500f --- /dev/null +++ b/openspec/changes/serve-h2c-upgrade-offers-as-http11/tasks.md @@ -0,0 +1,31 @@ +## 1. Implementation + +- [x] 1.1 Neutralize non-WebSocket upgrade offers in the httptools HTTP + protocol: replay the request head without the declined offer's + hop-by-hop headers and serve the request as plain HTTP/1.1. +- [x] 1.2 Wire the tolerant protocol into the server bootstrap, falling back + to an h11 subclass with the same header hygiene when httptools is + unavailable (stock h11 already delivers the body but exposes the + declined offer's headers). +- [x] 1.3 Classify upgrade offers by combining repeated `Connection` fields + (RFC 9110 §5.3) so a second `Connection: keep-alive` field cannot hide + the offer and reproduce the body loss. +- [x] 1.4 Replay declined offers iteratively (loop in `data_received`) rather + than recursively, so a segment pipelining many upgrade-offering requests + cannot drive attacker-controlled recursion depth (RecursionError + escaping into the event loop) or pin per-frame byte copies. + +## 2. Validation + +- [x] 2.1 Add transport-level regressions: h2c offer with coalesced + header/body and with split segments both reach the application with the + full body and succeed; the declined offer's headers are not exposed; the + connection stays reusable. +- [x] 2.2 Add a live-server regression over real sockets using the production + protocol wiring, including a real WebSocket upgrade that must keep + completing, plus a canary pinning the stock uvicorn defect. +- [x] 2.3 Add a regression pipelining 2000 h2c offers in one segment: all are + served and the connection survives (raised RecursionError when the + replay was recursive). +- [x] 2.4 Run focused tests, lint, type checks, and strict OpenSpec + validation. diff --git a/tests/integration/test_http_upgrade_tolerance.py b/tests/integration/test_http_upgrade_tolerance.py new file mode 100644 index 0000000000..0e397467fe --- /dev/null +++ b/tests/integration/test_http_upgrade_tolerance.py @@ -0,0 +1,459 @@ +"""Regression tests for issue #1757: h2c upgrade offers must not break requests. + +JetBrains/Ktor clients opportunistically attach ``Connection: Upgrade`` + +``Upgrade: h2c`` + ``HTTP2-Settings`` to plain HTTP/1.1 POSTs. The stock +uvicorn httptools protocol treats any such request as a protocol switch and +wedges the parser: a body coalesced with the headers is silently dropped, and +a body written as a separate segment (Ktor's pattern) turns into +``400 Invalid HTTP request received.``. + +The suite covers three layers: + +- protocol-level tests driving ``UpgradeTolerantHttpToolsProtocol`` through a + fake transport with both client segmentations; +- canary tests pinning the stock behavior these fixes exist for (if a uvicorn + upgrade makes them fail, the subclass can likely be retired); +- a live-server test using the production protocol wiring over real sockets, + including a real WebSocket upgrade that must keep completing. +""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import pytest +import uvicorn +from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol +from uvicorn.server import ServerState + +from app.cli import _load_http_protocol_class +from app.core.http_protocol import UpgradeTolerantH11Protocol +from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + +pytestmark = pytest.mark.integration + +_BODY = json.dumps({"model": "gpt-5.6-luna", "input": "hello", "stream": False}).encode() +_H2C_HEAD = ( + b"POST /echo HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(_BODY)).encode() + b"\r\n" + b"\r\n" +) + + +async def _echo_app(scope: dict[str, Any], receive: Any, send: Any) -> None: + """Echo the request body and the header names the application observed.""" + if scope["type"] == "websocket": + await receive() + await send({"type": "websocket.accept"}) + message = await receive() + await send({"type": "websocket.send", "text": message.get("text", "")}) + await send({"type": "websocket.close"}) + return + + assert scope["type"] == "http" + body = b"" + while True: + message = await receive() + body += message.get("body", b"") + if not message.get("more_body", False): + break + payload = json.dumps( + { + "echo": body.decode(), + "header_names": sorted({name.decode() for name, _ in scope["headers"]}), + } + ).encode() + await send( + { + "type": "http.response.start", + "status": 200, + "headers": [(b"content-type", b"application/json"), (b"content-length", str(len(payload)).encode())], + } + ) + await send({"type": "http.response.body", "body": payload}) + + +class _FakeTransport(asyncio.Transport): + def __init__(self) -> None: + super().__init__() + self.buffer = bytearray() + self.closed = False + self.protocol: asyncio.BaseProtocol | None = None + + def write(self, data: bytes | bytearray | memoryview) -> None: + self.buffer.extend(data) + + def is_closing(self) -> bool: + return self.closed + + def close(self) -> None: + self.closed = True + + def abort(self) -> None: + self.closed = True + + def pause_reading(self) -> None: + pass + + def resume_reading(self) -> None: + pass + + def set_protocol(self, protocol: asyncio.BaseProtocol) -> None: + self.protocol = protocol + + def get_extra_info(self, name: str, default: Any = None) -> Any: + if name == "sockname": + return ("127.0.0.1", 2455) + if name == "peername": + return ("127.0.0.1", 54321) + return default + + +def _make_protocol(protocol_class: type[Any]) -> tuple[Any, _FakeTransport]: + config = uvicorn.Config(app=_echo_app, lifespan="off") + config.load() + protocol = protocol_class(config=config, server_state=ServerState(), app_state={}) + transport = _FakeTransport() + protocol.connection_made(transport) + return protocol, transport + + +async def _wait_for_response(transport: _FakeTransport, timeout: float = 5.0) -> bytes: + async with asyncio.timeout(timeout): + while b"\r\n\r\n" not in transport.buffer or not bytes(transport.buffer).split(b"\r\n\r\n", 1)[1]: + await asyncio.sleep(0.01) + return bytes(transport.buffer) + + +def _parse_json_body(raw_response: bytes) -> dict[str, Any]: + _, _, body = raw_response.partition(b"\r\n\r\n") + return json.loads(body) + + +async def test_h2c_offer_with_coalesced_body_is_served_as_http11() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD + _BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + # The declined offer's hop-by-hop headers must not reach the application. + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + assert "connection" not in payload["header_names"] + assert not transport.closed + + +async def test_h2c_offer_with_split_head_and_body_is_served_as_http11() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + assert _parse_json_body(raw_response)["echo"] == _BODY.decode() + assert not transport.closed + + +async def test_h2c_offer_keeps_connection_reusable_for_next_request() -> None: + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(_H2C_HEAD + _BODY) + await _wait_for_response(transport) + transport.buffer.clear() + + follow_up = b"POST /echo HTTP/1.1\r\nHost: 127.0.0.1\r\nContent-Length: 2\r\n\r\nhi" + protocol.data_received(follow_up) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + assert _parse_json_body(raw_response)["echo"] == "hi" + + +async def test_h2c_offer_with_repeated_connection_fields_is_served_as_http11() -> None: + """Repeated ``Connection`` fields must be combined when classifying the offer. + + uvicorn's ``_get_upgrade`` keeps only the tokens of the last ``Connection`` + field, so ``Connection: Upgrade`` followed by ``Connection: keep-alive`` + would hide the offer and reproduce the original body loss. + """ + head = ( + b"POST /echo HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Connection: keep-alive\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(_BODY)).encode() + b"\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(head) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + # The unrelated keep-alive token survives the sanitization. + assert "connection" in payload["header_names"] + + +async def test_pipelined_h2c_offers_in_one_segment_do_not_exhaust_the_stack() -> None: + """Many pipelined upgrade offers in one segment must not recurse per offer. + + Each declined offer replays the remaining bytes through a fresh parser. + Done recursively that was one stack frame per pipelined request, so a + single ~66KB segment of minimal h2c GETs (well under asyncio's 256KiB + per-read buffer) raised RecursionError out of ``data_received``, aborting + the connection — and pinned an O(depth x segment) pile of byte copies + while unwinding. The replay must be iterative. + """ + request = b"GET /echo HTTP/1.1\r\nHost: 127.0.0.1\r\nConnection: Upgrade\r\nUpgrade: h2c\r\n\r\n" + count = 2000 # ~148KB, double the depth that overflows the default 1000-frame stack + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(request * count) # raised RecursionError when replay was recursive + + async with asyncio.timeout(30.0): + while transport.buffer.count(b"HTTP/1.1 200 OK") < count: + await asyncio.sleep(0.05) + assert not transport.closed + + +async def test_h11_fallback_serves_h2c_offer_and_hides_upgrade_headers() -> None: + """The httptools-less fallback keeps the body and the header hygiene.""" + protocol, transport = _make_protocol(UpgradeTolerantH11Protocol) + + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK"), raw_response + payload = _parse_json_body(raw_response) + assert payload["echo"] == _BODY.decode() + assert "upgrade" not in payload["header_names"] + assert "http2-settings" not in payload["header_names"] + assert "connection" not in payload["header_names"] + + +async def test_websocket_handshake_with_repeated_connection_fields_switches_protocols() -> None: + """Combined-field classification must not regress (or hide) WebSocket handoffs.""" + handshake = ( + b"GET /ws HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade\r\n" + b"Upgrade: websocket\r\n" + b"Connection: keep-alive\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(handshake) + + async with asyncio.timeout(5.0): + while b"\r\n\r\n" not in transport.buffer: + await asyncio.sleep(0.01) + assert bytes(transport.buffer).startswith(b"HTTP/1.1 101 Switching Protocols"), bytes(transport.buffer) + # The connection was handed off to the WebSocket protocol. + assert transport.protocol is not None + + +async def test_websocket_handshake_with_multiple_upgrade_tokens_reaches_the_websocket_stack() -> None: + """``Upgrade: websocket, h2c`` must be classified as a WebSocket handshake. + + The ``Upgrade`` field is a comma-separated protocol list (RFC 9110 + section 7.8) and the server may pick any offered protocol it supports. + Matching the raw field value against ``websocket`` would misclassify the + handshake as an ignorable offer and answer it *as the application* over + plain HTTP/1.1 (with the WebSocket headers stripped from the scope). The + handshake verdict belongs to the WebSocket stack: uvicorn's default + ``websockets`` implementation currently rejects multi-token ``Upgrade`` + values with 426, other implementations may complete the 101. + """ + handshake = ( + b"GET /ws HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Connection: Upgrade\r\n" + b"Upgrade: websocket, h2c\r\n" + b"Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n" + b"Sec-WebSocket-Version: 13\r\n" + b"\r\n" + ) + protocol, transport = _make_protocol(UpgradeTolerantHttpToolsProtocol) + + protocol.data_received(handshake) + + async with asyncio.timeout(5.0): + while b"\r\n\r\n" not in transport.buffer: + await asyncio.sleep(0.01) + status_line = bytes(transport.buffer).split(b"\r\n", 1)[0] + assert status_line in (b"HTTP/1.1 101 Switching Protocols", b"HTTP/1.1 426 Upgrade Required"), status_line + # The connection was handed off to the WebSocket protocol, not the app. + assert transport.protocol is not None + + +async def test_live_v1_responses_route_serves_split_h2c_offer(db_setup, monkeypatch: pytest.MonkeyPatch) -> None: + """Issue #1757's exact product path: split-written h2c POST to /v1/responses. + + The request must traverse the parser into the application instead of the + stock transport-layer ``400 Invalid HTTP request received.``. With an + empty account pool the route deterministically answers a JSON + ``no_accounts`` 503 — reaching that error proves the request body was + delivered and parsed (the balancer logs the requested model), which is + exactly what the wedged stock parser prevented. + """ + import app.main as main_module + + async def _noop_init_db() -> None: + return None + + monkeypatch.setattr(main_module, "init_db", _noop_init_db) + config = uvicorn.Config( + app=main_module.create_app(), + host="127.0.0.1", + port=0, + http=_load_http_protocol_class(), + log_level="warning", + ) + server = uvicorn.Server(config) + serve_task = asyncio.create_task(server.serve()) + try: + async with asyncio.timeout(10.0): + while not server.started: + await asyncio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + + body = json.dumps({"model": "gpt-5.2", "input": "hello", "stream": False}).encode() + head = ( + b"POST /v1/responses HTTP/1.1\r\n" + b"Host: 127.0.0.1\r\n" + b"Authorization: Bearer sk-bogus\r\n" + b"Connection: Upgrade, HTTP2-Settings\r\n" + b"Upgrade: h2c\r\n" + b"HTTP2-Settings: AAMAAABkAARAAAAAAAIAAAAA\r\n" + b"Content-Type: application/json\r\n" + b"Content-Length: " + str(len(body)).encode() + b"\r\n" + b"\r\n" + ) + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(head) + await writer.drain() + await asyncio.sleep(0.05) + writer.write(body) + await writer.drain() + async with asyncio.timeout(10.0): + status_line = await reader.readline() + raw_headers = await reader.readuntil(b"\r\n\r\n") + content_length = next( + int(line.split(b":", 1)[1]) + for line in raw_headers.lower().splitlines() + if line.startswith(b"content-length:") + ) + payload = json.loads(await reader.readexactly(content_length)) + assert status_line == b"HTTP/1.1 503 Service Unavailable\r\n", status_line + assert payload["error"]["code"] == "no_accounts", payload + writer.close() + await writer.wait_closed() + finally: + server.should_exit = True + async with asyncio.timeout(15.0): + await serve_task + + +async def test_stock_httptools_protocol_still_breaks_on_h2c_offers() -> None: + """Canary pinning the upstream defect this module works around. + + The stock parser drops a coalesced body (the application observes an empty + body) and answers 400 when the body arrives as a separate segment. If a + uvicorn/httptools upgrade makes this test fail, upstream has fixed + https://github.com/Soju06/codex-lb/issues/1757 and + ``UpgradeTolerantHttpToolsProtocol`` can likely be retired. + """ + protocol, transport = _make_protocol(HttpToolsProtocol) + protocol.data_received(_H2C_HEAD + _BODY) + raw_response = await _wait_for_response(transport) + assert raw_response.startswith(b"HTTP/1.1 200 OK") + assert _parse_json_body(raw_response)["echo"] == "" # body silently dropped + + protocol, transport = _make_protocol(HttpToolsProtocol) + protocol.data_received(_H2C_HEAD) + await asyncio.sleep(0.01) + protocol.data_received(_BODY) + async with asyncio.timeout(5.0): + while b"Invalid HTTP request received." not in transport.buffer: + await asyncio.sleep(0.01) + # The body segment hits the wedged parser: the application never sees the + # payload and the client's request ends in a 400. + assert b'"echo": ""' in transport.buffer + assert b"HTTP/1.1 400 Bad Request" in transport.buffer + + +async def test_live_server_serves_h2c_offers_and_websocket_upgrades() -> None: + """End-to-end proof over real sockets with the production protocol wiring.""" + config = uvicorn.Config( + app=_echo_app, + host="127.0.0.1", + port=0, + http=_load_http_protocol_class(), + lifespan="off", + log_level="warning", + ) + server = uvicorn.Server(config) + serve_task = asyncio.create_task(server.serve()) + try: + async with asyncio.timeout(10.0): + while not server.started: + await asyncio.sleep(0.01) + port = server.servers[0].sockets[0].getsockname()[1] + + # Ktor's write pattern: headers first, body as a separate segment. + reader, writer = await asyncio.open_connection("127.0.0.1", port) + writer.write(_H2C_HEAD) + await writer.drain() + await asyncio.sleep(0.05) + writer.write(_BODY) + await writer.drain() + async with asyncio.timeout(10.0): + status_line = await reader.readline() + assert status_line == b"HTTP/1.1 200 OK\r\n" + raw_headers = await reader.readuntil(b"\r\n\r\n") + content_length = next( + int(line.split(b":", 1)[1]) + for line in raw_headers.lower().splitlines() + if line.startswith(b"content-length:") + ) + payload = json.loads(await reader.readexactly(content_length)) + assert payload["echo"] == _BODY.decode() + writer.close() + await writer.wait_closed() + + # A real WebSocket upgrade must keep switching protocols. + from websockets.asyncio.client import connect + + async with connect(f"ws://127.0.0.1:{port}/ws") as websocket: + await websocket.send("ping") + assert await websocket.recv() == "ping" + finally: + server.should_exit = True + async with asyncio.timeout(10.0): + await serve_task diff --git a/tests/unit/test_cli.py b/tests/unit/test_cli.py index 2415d82f4e..d70119a867 100644 --- a/tests/unit/test_cli.py +++ b/tests/unit/test_cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +import builtins import json import logging import sqlite3 @@ -252,11 +253,14 @@ def run(self) -> None: cli._run_server("app.main:app", host="127.0.0.1", port=2455) + from app.core.http_protocol_httptools import UpgradeTolerantHttpToolsProtocol + assert captured["config_args"] == ("app.main:app",) assert captured["config_kwargs"] == { "host": "127.0.0.1", "port": 2455, "workers": 1, + "http": UpgradeTolerantHttpToolsProtocol, "timeout_graceful_shutdown": 17, } assert captured["drain_timeout_seconds"] == 17 @@ -264,6 +268,24 @@ def run(self) -> None: assert captured["ran"] is True +def test_load_http_protocol_class_falls_back_to_h11_without_httptools( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from app.core.http_protocol import UpgradeTolerantH11Protocol + + real_import = builtins.__import__ + + def fail_httptools_import(name: str, *args: Any, **kwargs: Any) -> object: + if name in {"httptools", "app.core.http_protocol_httptools"}: + raise ImportError(name) + return real_import(name, *args, **kwargs) + + monkeypatch.delitem(sys.modules, "app.core.http_protocol_httptools", raising=False) + monkeypatch.setattr(builtins, "__import__", fail_httptools_import) + + assert cli._load_http_protocol_class() is UpgradeTolerantH11Protocol + + def test_run_server_pins_one_worker_despite_ambient_web_concurrency( monkeypatch: pytest.MonkeyPatch, ) -> None: From 539cf934ea654efbc915bad17ea84a87c07afa4d Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:35:13 +0900 Subject: [PATCH 056/117] fix(db): add postgres shm_size and raise default pool headroom (#1791) Production single-replica PostgreSQL hit two capacity ceilings: - Docker's default 64MB /dev/shm makes parallel hash joins fail with "could not resize shared memory segment ... No space left on device" (asyncpg DiskFullError). Pin shm_size: 1gb on the Compose postgres service. The Helm bundled Bitnami sub-chart already mounts a memory-backed /dev/shm by default. - Default pool 15/10 with the fixed 30s checkout timeout exhausts under slow-query pile-ups ("QueuePool limit of size 15 overflow 10 reached"). Raise defaults to 25/15 so one replica's two pooled engines cap at (25+15)*2 = 80 connections, preserving the documented 20 raw-slot reserve on PostgreSQL's default max_connections=100. Helm deployments are unaffected: the chart injects its own pool values. Co-authored-by: Claude Fable 5 --- app/core/config/settings.py | 9 ++-- docker-compose.yml | 7 +++ docs/reference/settings.md | 4 +- .../proposal.md | 48 +++++++++++++++++++ .../specs/database-backends/spec.md | 27 +++++++++++ .../specs/deployment-installation/spec.md | 22 +++++++++ .../tasks.md | 17 +++++++ tests/unit/test_db_session.py | 2 +- tests/unit/test_docker_compose_postgres.py | 9 ++++ tests/unit/test_settings_trace_and_removed.py | 2 +- 10 files changed, 140 insertions(+), 7 deletions(-) create mode 100644 openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md create mode 100644 openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md create mode 100644 openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md create mode 100644 openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md diff --git a/app/core/config/settings.py b/app/core/config/settings.py index 9b21dd0b6c..c8bdac6209 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -242,9 +242,12 @@ class Settings(BaseSettings): database_url: str = DEFAULT_DATABASE_URL # Pool timeout and recycle are fixed constants in ``app/db/session.py``; # the background-task engine always derives its pool sizing from the two - # settings below. - database_pool_size: int = Field(default=15, gt=0) - database_max_overflow: int = Field(default=10, ge=0) + # settings below. Defaults are sized so one replica's two pooled engines + # cap at (25 + 15) * 2 = 80 PostgreSQL connections, preserving >= 20 raw + # server slots on PostgreSQL's default max_connections=100 for reserved + # connections, the migration path's two-connection peak, and operations. + database_pool_size: int = Field(default=25, gt=0) + database_max_overflow: int = Field(default=15, ge=0) database_migrate_on_startup: bool = True database_sqlite_pre_migrate_backup_enabled: bool = True database_sqlite_pre_migrate_backup_max_files: int = Field(default=5, ge=1) diff --git a/docker-compose.yml b/docker-compose.yml index 3c1d518035..b9c730521c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,6 +92,13 @@ services: fi exec docker-entrypoint.sh "$@" - codex-lb-postgres-entrypoint-guard + # Docker's default /dev/shm is 64MB. PostgreSQL parallel workers exchange + # tuples through dynamic shared memory under /dev/shm, so hash joins that + # spill past 64MB abort with "could not resize shared memory segment ... + # No space left on device". 1GB gives parallel query realistic headroom. + # (The Helm chart needs no equivalent: the bundled Bitnami PostgreSQL + # sub-chart mounts a memory-backed /dev/shm by default via shmVolume.) + shm_size: 1gb environment: POSTGRES_USER: codex_lb POSTGRES_PASSWORD: codex_lb diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 229934236e..cb350e1607 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -32,11 +32,11 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | | `CODEX_LB_DATABASE_ALEMBIC_AUTO_REMAP_ENABLED` | `bool` | `True` | -| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `10` | +| `CODEX_LB_DATABASE_MAX_OVERFLOW` | `int` | `15` | | `CODEX_LB_DATABASE_MIGRATE_ON_STARTUP` | `bool` | `True` | | `CODEX_LB_DATABASE_MIGRATION_LOCK_TIMEOUT_SECONDS` | `float` | `300.0` | | `CODEX_LB_DATABASE_MIGRATIONS_FAIL_FAST` | `bool` | `True` | -| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `15` | +| `CODEX_LB_DATABASE_POOL_SIZE` | `int` | `25` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_ENABLED` | `bool` | `True` | | `CODEX_LB_DATABASE_SQLITE_PRE_MIGRATE_BACKUP_MAX_FILES` | `int` | `5` | | `CODEX_LB_DATABASE_SQLITE_STARTUP_CHECK_MODE` | `'quick' \| 'full' \| 'off'` | `'quick'` | diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md new file mode 100644 index 0000000000..efc23843ad --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/proposal.md @@ -0,0 +1,48 @@ +# Expand PostgreSQL /dev/shm and default pool headroom + +## Why + +Two independent capacity ceilings surfaced on a production single-replica +PostgreSQL deployment: + +- The Compose `postgres` service runs with Docker's default 64MB `/dev/shm`. + PostgreSQL parallel workers (`work_mem=32MB`, + `max_parallel_workers_per_gather=2`) exchange spill files through dynamic + shared memory under `/dev/shm`, so parallel hash joins abort with + `could not resize shared memory segment ... No space left on device`, + which asyncpg surfaces as `DiskFullError` on the request path. +- The default SQLAlchemy pool (`database_pool_size=15`, + `database_max_overflow=10`, fixed 30s checkout timeout) exhausts under + slow-query pile-ups: once 25 request-path checkouts are held, every further + request waits 30 seconds and fails with + `QueuePool limit of size 15 overflow 10 reached, connection timed out`. + +## What Changes + +- The Compose `postgres` service sets `shm_size: 1gb`. +- Default `database_pool_size` rises 15 → 25 and `database_max_overflow` + 10 → 15, keeping the per-replica two-engine cap at + `(25 + 15) * 2 = 80` application connections — inside PostgreSQL's default + `max_connections=100` with at least 20 raw server slots reserved (same + reserve rule the Helm capacity guidance already mandates). +- Helm deployments are unaffected: the chart always injects its own + `CODEX_LB_DATABASE_POOL_SIZE` / `CODEX_LB_DATABASE_MAX_OVERFLOW` values, + and the bundled Bitnami PostgreSQL sub-chart already mounts a + memory-backed `/dev/shm` (`shmVolume.enabled=true` by default). + +## Capabilities + +### Modified Capabilities + +- `deployment-installation`: the Compose Postgres profile provisions a + `/dev/shm` large enough for parallel query. +- `database-backends`: default pool sizing preserves the raw-slot reserve on + PostgreSQL's default `max_connections`. + +## Impact + +- SQLite deployments: none (pool sizing applies to pooled backends only). +- Helm deployments: none (chart values override both settings). +- Compose/manual PostgreSQL deployments: applying `shm_size` requires the + postgres container to be recreated (seconds of downtime); per-replica + worst-case application connections rise from 50 to 80. diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md new file mode 100644 index 0000000000..a5231ef1e9 --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/database-backends/spec.md @@ -0,0 +1,27 @@ +## ADDED Requirements + +### Requirement: Default pool sizing preserves raw-slot reserve on default max_connections + +The default values of `database_pool_size` and `database_max_overflow` MUST +keep one replica's aggregate application connection capacity — +`(database_pool_size + database_max_overflow) * 2 pooled engines * 1 +supported worker` — at or below 80, so a single replica on PostgreSQL's +default `max_connections=100` retains at least 20 raw server slots for +PostgreSQL-reserved connections, the migration path's two-connection peak, +administration, and transient non-application clients. + +#### Scenario: Default single replica fits default max_connections + +- **WHEN** one replica runs with the default `database_pool_size` and + `database_max_overflow` +- **THEN** both pooled engines together cap at no more than 80 PostgreSQL + connections +- **AND** at least 20 raw server slots remain on a default + `max_connections=100` server + +#### Scenario: Operators can still tune the pool + +- **WHEN** `CODEX_LB_DATABASE_POOL_SIZE` or `CODEX_LB_DATABASE_MAX_OVERFLOW` + is set in the environment +- **THEN** the configured values override the defaults for both pooled + engines diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md new file mode 100644 index 0000000000..05e880b54a --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/specs/deployment-installation/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Compose Postgres service sizes /dev/shm for parallel query + +The Docker Compose `postgres` service MUST set an explicit `shm_size` of at +least 1GB. Docker's default 64MB `/dev/shm` causes PostgreSQL parallel +workers to fail with `could not resize shared memory segment ... No space +left on device` once a parallel hash join spills past the segment. + +#### Scenario: Compose postgres service pins shm_size + +- **WHEN** `docker-compose.yml` is inspected +- **THEN** the `postgres` service declares `shm_size` of at least 1GB + +#### Scenario: Parallel hash join spills past 64MB + +- **GIVEN** the Compose `postgres` service is running with the declared + `shm_size` +- **WHEN** a parallel hash join spills more than 64MB of build tuples into + dynamic shared memory +- **THEN** the query does not fail with `could not resize shared memory + segment` diff --git a/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md new file mode 100644 index 0000000000..558d9c804e --- /dev/null +++ b/openspec/changes/expand-postgres-shm-and-pool-headroom/tasks.md @@ -0,0 +1,17 @@ +## 1. Implementation + +- [x] 1.1 Add `shm_size: 1gb` to the Compose `postgres` service. +- [x] 1.2 Raise default `database_pool_size` to 25 and + `database_max_overflow` to 15, documenting the 80-connection / + 20-raw-slot budget at the setting definition. +- [x] 1.3 Regenerate `docs/reference/settings.md`. + +## 2. Regression coverage + +- [x] 2.1 Policy-test that the Compose `postgres` service pins `shm_size`. +- [x] 2.2 Update the settings default assertion to the new pool size. + +## 3. Validation + +- [x] 3.1 Run the compose, db-session, settings, and Helm artifact suites. +- [x] 3.2 Run strict OpenSpec validation for this change. diff --git a/tests/unit/test_db_session.py b/tests/unit/test_db_session.py index cd3382a024..5af6d147e6 100644 --- a/tests/unit/test_db_session.py +++ b/tests/unit/test_db_session.py @@ -787,7 +787,7 @@ async def test_init_background_db_derives_postgres_pool_size_from_main_pool() -> if os.environ.get("CODEX_LB_TEST_DATABASE_URL"): assert isinstance(pool, NullPool) else: - assert cast(Any, pool).size() == 15 + assert cast(Any, pool).size() == 25 if session_module._background_engine is not None: await session_module._background_engine.dispose() diff --git a/tests/unit/test_docker_compose_postgres.py b/tests/unit/test_docker_compose_postgres.py index 72b9591cc1..1accaa0150 100644 --- a/tests/unit/test_docker_compose_postgres.py +++ b/tests/unit/test_docker_compose_postgres.py @@ -11,6 +11,15 @@ def _compose() -> dict[str, Any]: return yaml.safe_load((repo_root / "docker-compose.yml").read_text(encoding="utf-8")) +def test_postgres_compose_service_sizes_dev_shm_for_parallel_query() -> None: + postgres = _compose()["services"]["postgres"] + + # Docker's default 64MB /dev/shm makes PostgreSQL parallel hash joins fail + # with "could not resize shared memory segment ... No space left on + # device" once they spill past the segment. Keep an explicit >= 1GB size. + assert postgres["shm_size"] == "1gb" + + def test_postgres18_compose_upgrade_helper_is_digest_pinned() -> None: services = _compose()["services"] postgres = services["postgres"] diff --git a/tests/unit/test_settings_trace_and_removed.py b/tests/unit/test_settings_trace_and_removed.py index dd197e890d..fcf9c16a92 100644 --- a/tests/unit/test_settings_trace_and_removed.py +++ b/tests/unit/test_settings_trace_and_removed.py @@ -153,7 +153,7 @@ def test_phase_3_removed_settings_are_listed_and_ignored(monkeypatch): settings = Settings() assert not hasattr(settings, "database_pool_recycle_seconds") assert not hasattr(settings, "drain_primary_threshold_pct") - assert settings.database_pool_size == 15 + assert settings.database_pool_size == 25 assert settings.soft_drain_enabled is True found = warn_removed_settings( { From d4c43ef88f8d4a548fb952a82901ea482995a633 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:35:39 +0900 Subject: [PATCH 057/117] perf(dashboard): cap projections bulk usage-history read per account (#1779) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(dashboard): cap projections bulk usage-history read per account The projections bulk usage-history read returned every in-window row to Python (~307k rows/call, avg 2.54s, max 56s on the reference PostgreSQL deployment) while its consumers — EWMA depletion (alpha 0.4), the 6-hour weekly-pace burn window, and the <=240-minute pace smoothing mean — only read each account's recent tail. Live snapshot ingestion appends rows per proxied request, so one busy account's 7-day window holds tens of thousands of rows the consumers cannot observe. Bound the PostgreSQL fetch to each account's newest rows: one lateral top-N probe per (account, cutoff) pair that descends the existing covering indexes backward and stops at the cap or the account's cutoff. The dashboard caller supplies the cap (4320 rows: the 6-hour recent-burn window at a 5-second cadence, 12x headroom over the default 60-second refresh), so no consumer-visible value changes; sparse accounts stay under the cap entirely. SQLite keeps its shared-floor snapshot cache and ignores the cap, the same way it ignores per-account cutoffs. No schema change: the covering indexes from 20260806_020000 already serve the capped probes index-only (verified: 408ms/22k buffers/53k rows vs 1355ms/38k buffers/299k rows for the uncapped secondary shape). Co-Authored-By: Claude Fable 5 * perf(dashboard): exempt pace-smoothing window from projection row cap Live snapshot ingestion writes a usage-history row per proxied request whenever the usage fingerprint changes, so the 5-second write throttle is a floor only for unchanged snapshots: a busy account can out-write any fixed row cap inside the weekly-pace smoothing window, and the smoothing mean weighs every in-window sample equally, so truncation would shift the dashboard's smoothed schedule gap and status. Split the capped lateral probe into two disjoint branches over the same covering index: rows at or after the configured smoothing-window start (plumbed from dashboard settings as uncapped_recent_floor) return in full, and the newest-first cap bounds only the older remainder that the tail-weighted EWMA consumers read. Prod EXPLAIN ANALYZE (read-only) keeps both branches index-only: 370ms/24k buffers/65k rows at the 240-minute worst-case floor vs 408ms/22k/52.7k capped-only. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude --- Makefile | 5 + app/modules/dashboard/repository.py | 11 +- app/modules/dashboard/service.py | 44 +++- app/modules/usage/repository.py | 144 ++++++++++- .../cap-projection-history-rows/proposal.md | 58 +++++ .../specs/query-caching/spec.md | 54 ++++ .../cap-projection-history-rows/tasks.md | 27 ++ tests/integration/test_usage_repository.py | 239 ++++++++++++++++++ .../test_dashboard_projection_history_cap.py | 91 +++++++ 9 files changed, 668 insertions(+), 5 deletions(-) create mode 100644 openspec/changes/cap-projection-history-rows/proposal.md create mode 100644 openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md create mode 100644 openspec/changes/cap-projection-history-rows/tasks.md create mode 100644 tests/unit/test_dashboard_projection_history_cap.py diff --git a/Makefile b/Makefile index efee90715e..1bb8dd2901 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,11 @@ POSTGRES_PYTEST_TARGETS := \ tests/integration/test_usage_repository.py::test_bulk_history_since_cutoff_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_secondary_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_covered_read_matches_non_covered_read_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_per_account_row_cap_keeps_newest_rows \ + tests/integration/test_usage_repository.py::test_bulk_history_since_row_cap_respects_per_account_cutoffs_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_row_cap_exempts_uncapped_recent_floor_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_capped_query_plan_is_index_only_postgresql \ + tests/integration/test_usage_repository.py::test_bulk_history_since_capped_floor_query_plan_is_index_only_postgresql \ tests/integration/test_migrations.py::test_usage_history_bulk_covering_indexes_migration_upgrade_and_downgrade \ tests/integration/test_migrations.py::test_usage_history_covering_index_migration_repairs_invalid_leftover_postgresql \ tests/integration/test_migrations.py::test_usage_history_autovacuum_tuning_migration_sets_and_resets_reloptions_postgresql diff --git a/app/modules/dashboard/repository.py b/app/modules/dashboard/repository.py index 3fe42f6172..9d4ca0d7e4 100644 --- a/app/modules/dashboard/repository.py +++ b/app/modules/dashboard/repository.py @@ -51,8 +51,17 @@ async def bulk_usage_history_since( since: datetime, *, cutoffs: dict[str, datetime] | None = None, + per_account_row_cap: int | None = None, + uncapped_recent_floor: datetime | None = None, ) -> dict[str, list[UsageHistorySnapshot]]: - return await self._usage_repo.bulk_history_since(account_ids, window, since, cutoffs=cutoffs) + return await self._usage_repo.bulk_history_since( + account_ids, + window, + since, + cutoffs=cutoffs, + per_account_row_cap=per_account_row_cap, + uncapped_recent_floor=uncapped_recent_floor, + ) async def latest_window_minutes(self, window: str) -> int | None: return await self._usage_repo.latest_window_minutes(window) diff --git a/app/modules/dashboard/service.py b/app/modules/dashboard/service.py index eb6026769e..606aebb7ca 100644 --- a/app/modules/dashboard/service.py +++ b/app/modules/dashboard/service.py @@ -39,6 +39,22 @@ ) from app.modules.usage.mappers import usage_history_to_window_row +# Newest-first per-account row bound for the projections history fetch +# (PostgreSQL; the SQLite snapshot cache keeps the shared floor). Live +# snapshot ingestion appends usage rows per proxied request, so one busy +# account's 7-day secondary window can hold tens of thousands of rows while +# the consumers only read the recent tail. The cap alone covers the +# tail-weighted consumers: the EWMA depletion/burn rates (alpha 0.4 — a +# sample's contribution decays by 0.6^n within a few dozen newer samples) +# are insensitive to samples this deep regardless of write cadence. The one +# equal-weight consumer — the weekly-pace smoothing mean over the configured +# window (<= 240 minutes) — is protected by ``uncapped_recent_floor`` +# instead, because ingestion writes on every fingerprint change and a burst +# could out-write any fixed cap inside the smoothing window. 4320 rows cover +# the 6-hour recent-burn window at the ingestor's 5-second per-account write +# throttle floor; sparse accounts stay under the cap entirely. +_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP = 4320 + def _parse_weekly_pace_working_days(value: str) -> set[int]: try: @@ -181,15 +197,16 @@ async def get_projections(self) -> DashboardProjectionsResponse: encryptor=self._encryptor, include_auth=False, ) + dashboard_settings = await self._repo.get_settings() primary_history, secondary_history = await _load_projection_histories( self._repo, primary_usage, secondary_usage, now, + smoothing_window_minutes=dashboard_settings.weekly_pace_smoothing_minutes, ) pri_depletion, sec_depletion = _build_depletion_by_window(primary_history, secondary_history, now) settings = get_settings() - dashboard_settings = await self._repo.get_settings() weekly_credit_pace = build_weekly_credit_pace( accounts=accounts, account_summaries=account_summaries, @@ -211,6 +228,8 @@ async def _load_projection_histories( primary_usage: dict[str, UsageHistory], secondary_usage: dict[str, UsageHistory], now: datetime, + *, + smoothing_window_minutes: int, ) -> tuple[dict[str, list[UsageHistory]], dict[str, list[UsageHistory]]]: # Compute depletion separately for primary-window and secondary-window # accounts so the aggregate is not skewed by mixing different window durations. @@ -279,13 +298,32 @@ async def _load_projection_histories( if acct_since < sec_since: sec_since = acct_since + # The weekly-pace smoothing mean weighs every sample in its window + # equally, so rows inside the configured smoothing window are exempt from + # the row cap (ingestion writes per fingerprint change; a burst could + # otherwise out-write the cap and silently shift the smoothed values). + smoothing_floor = now - timedelta(minutes=smoothing_window_minutes) all_pri_rows = ( - await repo.bulk_usage_history_since(pri_fetch_ids, "primary", pri_since, cutoffs=pri_cutoffs) + await repo.bulk_usage_history_since( + pri_fetch_ids, + "primary", + pri_since, + cutoffs=pri_cutoffs, + per_account_row_cap=_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + uncapped_recent_floor=smoothing_floor, + ) if pri_fetch_ids else {} ) all_sec_rows = ( - await repo.bulk_usage_history_since(sec_fetch_ids, "secondary", sec_since, cutoffs=sec_cutoffs) + await repo.bulk_usage_history_since( + sec_fetch_ids, + "secondary", + sec_since, + cutoffs=sec_cutoffs, + per_account_row_cap=_PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + uncapped_recent_floor=smoothing_floor, + ) if sec_fetch_ids else {} ) diff --git a/app/modules/usage/repository.py b/app/modules/usage/repository.py index 2965c3efdd..65727688ca 100644 --- a/app/modules/usage/repository.py +++ b/app/modules/usage/repository.py @@ -10,7 +10,22 @@ from typing import Any, Callable, cast from anyio import to_thread -from sqlalchemy import Integer, and_, delete, func, literal_column, or_, select, text, true, tuple_ +from sqlalchemy import ( + Integer, + String, + and_, + column, + delete, + func, + literal_column, + or_, + select, + text, + true, + tuple_, + union_all, + values, +) from sqlalchemy import cast as sqlalchemy_cast from sqlalchemy.ext.asyncio import AsyncSession @@ -932,6 +947,8 @@ async def bulk_history_since( since: datetime, *, cutoffs: dict[str, datetime] | None = None, + per_account_row_cap: int | None = None, + uncapped_recent_floor: datetime | None = None, ) -> dict[str, list[UsageHistorySnapshot]]: """Fetch minimal usage history fields for multiple accounts in a single query. @@ -942,6 +959,24 @@ async def bulk_history_since( ignores ``cutoffs`` (its snapshot cache is keyed on the shared floor); callers keep their own per-account trimming, so honoring the bound here only changes how many rows are read, never the result. + + ``per_account_row_cap`` additionally bounds each account's slice to + its newest rows inside the cutoff (PostgreSQL only). Live snapshot + ingestion appends usage rows per proxied request, so a busy account's + 7-day window can hold tens of thousands of rows while the projection + consumers (EWMA depletion, weekly-pace burn/smoothing) only read the + recent tail. Each capped slice keeps oldest-first ordering. The + SQLite snapshot-cache path ignores the cap the same way it ignores + ``cutoffs``. + + ``uncapped_recent_floor`` exempts rows at or after the given time + from the row cap: every in-cutoff row newer than the floor is always + returned, and the cap bounds only the older remainder. Consumers + whose math weighs every sample in a fixed time window equally (the + weekly-pace smoothing mean) pass their window start here so a + write-rate burst can never silently truncate that window, while + tail-weighted consumers (EWMA) stay covered by the cap alone. + Ignored unless ``per_account_row_cap`` is set on PostgreSQL. """ if not account_ids: return {} @@ -957,6 +992,16 @@ async def bulk_history_since( since, ) + if per_account_row_cap is not None and dialect == "postgresql": + return await self._bulk_history_since_capped_postgresql( + account_ids, + window, + since, + cutoffs=cutoffs, + per_account_row_cap=per_account_row_cap, + uncapped_recent_floor=uncapped_recent_floor, + ) + if cutoffs: recency_clause = or_( *( @@ -1001,6 +1046,103 @@ async def bulk_history_since( grouped.setdefault(snapshot.account_id, []).append(snapshot) return grouped + async def _bulk_history_since_capped_postgresql( + self, + account_ids: list[str], + window: str, + since: datetime, + *, + cutoffs: dict[str, datetime] | None, + per_account_row_cap: int, + uncapped_recent_floor: datetime | None, + ) -> dict[str, list[UsageHistorySnapshot]]: + """Per-account newest-first capped fetch (PostgreSQL). + + One lateral top-N probe per account instead of one shared range scan: + the probe descends idx_usage_window_account_time_covering (or its + raw-window twin) backward and stops at the cap or the account's + cutoff, whichever comes first, so the read never touches the bulk of + a dense account's window. The OR-of-cutoffs shape this replaces + returned every in-window row (hundreds of thousands on dense + deployments) to Python only for the projection consumers to use the + recent tail. + + With ``uncapped_recent_floor`` the probe splits into two disjoint + branches over the same covering index: rows at or after the floor are + returned in full (time-bounded, so still cheap), and the top-N cap + applies only to rows between the cutoff and the floor. Snapshot + ingestion writes per proxied request whenever the usage fingerprint + moves, so a fixed row cap alone cannot guarantee it out-lasts a + burst inside an equal-weight consumer window. + """ + value_columns = [ + column("account_id", String()), + column("cutoff", UsageHistory.recorded_at.type), + ] + if uncapped_recent_floor is not None: + value_columns.append(column("uncapped_floor", UsageHistory.recorded_at.type)) + value_rows: list[tuple] = [] + for account_id in account_ids: + cutoff = max(cutoffs.get(account_id, since), since) if cutoffs else since + if uncapped_recent_floor is None: + value_rows.append((account_id, cutoff)) + else: + value_rows.append((account_id, cutoff, max(cutoff, uncapped_recent_floor))) + account_cutoffs = values(*value_columns, name="account_cutoffs").data(value_rows) + snapshot_columns = ( + UsageHistory.id, + UsageHistory.account_id, + UsageHistory.used_percent, + UsageHistory.recorded_at, + UsageHistory.reset_at, + UsageHistory.window_minutes, + ) + capped_tail = ( + select(*snapshot_columns) + .where( + UsageHistory.account_id == account_cutoffs.c.account_id, + UsageHistory.recorded_at >= account_cutoffs.c.cutoff, + *( + (UsageHistory.recorded_at < account_cutoffs.c.uncapped_floor,) + if uncapped_recent_floor is not None + else () + ), + _window_clause(window), + ) + .order_by(UsageHistory.recorded_at.desc(), UsageHistory.id.desc()) + .limit(per_account_row_cap) + .correlate(account_cutoffs) + ) + if uncapped_recent_floor is not None: + uncapped_recent = ( + select(*snapshot_columns) + .where( + UsageHistory.account_id == account_cutoffs.c.account_id, + UsageHistory.recorded_at >= account_cutoffs.c.uncapped_floor, + _window_clause(window), + ) + .correlate(account_cutoffs) + ) + recent = union_all(uncapped_recent, capped_tail).lateral("recent") + else: + recent = capped_tail.lateral("recent") + stmt = select(recent).select_from(account_cutoffs.join(recent, true())) + result = await self._session.execute(stmt) + grouped: dict[str, list[UsageHistorySnapshot]] = {} + for row in result.all(): + snapshot = UsageHistorySnapshot( + id=int(row.id), + account_id=row.account_id, + used_percent=float(row.used_percent), + recorded_at=row.recorded_at, + reset_at=float(row.reset_at) if row.reset_at is not None else None, + window_minutes=int(row.window_minutes) if row.window_minutes is not None else None, + ) + grouped.setdefault(snapshot.account_id, []).append(snapshot) + for snapshots in grouped.values(): + snapshots.sort(key=lambda snapshot: (snapshot.recorded_at, snapshot.id)) + return grouped + async def trends_by_bucket( self, since: datetime, diff --git a/openspec/changes/cap-projection-history-rows/proposal.md b/openspec/changes/cap-projection-history-rows/proposal.md new file mode 100644 index 0000000000..7ef6c21f52 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/proposal.md @@ -0,0 +1,58 @@ +## Why + +On the reference PostgreSQL deployment the dashboard projections bulk +usage-history read returns ~307k rows per call (540 calls over 10 days, +avg 2.54 s, max 56 s, `usage_history` at ~3M rows / 2.8 GB). Live snapshot +ingestion appends usage rows per proxied request, so one busy account's +7-day secondary window holds tens of thousands of rows — yet the projection +consumers only read the recent tail: EWMA depletion/burn rates decay a +sample's contribution by 0.6^n within a few dozen newer samples, the +weekly-pace recent-burn window is 6 hours, and the pace smoothing mean is at +most 240 minutes. Fetching every in-window row burns database reads, row +transfer, and Python row-building for values no consumer can observe. + +## What Changes + +- Bound the PostgreSQL projections bulk usage-history read to each + account's newest rows (newest-first per-account row cap) inside the + existing per-account cutoffs, via one lateral top-N probe per account + over the existing covering indexes (backward index-only scan that stops + at the cap or cutoff). +- Exempt rows inside the configured pace-smoothing window from the cap + (uncapped recent floor supplied by the projections caller): ingestion + writes per proxied request whenever the usage fingerprint changes, so a + write burst could out-write any fixed cap inside the smoothing window, + and the smoothing mean weighs every in-window sample equally. The probe + splits into two disjoint branches over the same covering index — the + time-bounded floor branch returns in full, the cap bounds the older + remainder. +- The dashboard projections caller supplies the cap, sized so the + tail-weighted consumers are unchanged (covers the 6-hour recent-burn + EWMA window at the ingestor's 5-second per-account write throttle + floor; EWMA contributions decay by 0.6^n well inside the cap). +- SQLite keeps its shared-floor snapshot cache and ignores the cap, the + same way it ignores per-account cutoffs. +- No schema change: the existing covering indexes already serve the capped + probes index-only. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `query-caching`: the projections history bulk read MUST additionally + bound each account's slice to its newest in-cutoff rows on PostgreSQL, + MUST exempt rows inside the configured pace-smoothing window from the + cap, and under-cap accounts MUST return slices equal to the shared-floor + fetch after per-account trimming. + +## Impact + +`app/modules/usage/repository.py` (capped PostgreSQL fetch shape), +`app/modules/dashboard/repository.py` / `app/modules/dashboard/service.py` +(cap plumbed from the projections caller), repository/plan/unit regression +coverage. No API, response-schema, setting, migration, or dashboard UI +change. diff --git a/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md b/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md new file mode 100644 index 0000000000..b3d0767008 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/specs/query-caching/spec.md @@ -0,0 +1,54 @@ +# query-caching Delta + +## MODIFIED Requirements + +### Requirement: Projection history reads are bounded per account +The dashboard projections history fetch MUST NOT widen every account's +lookback to the widest account window. On PostgreSQL the bulk usage-history +read MUST bound rows per account by that account's own window cutoff, and +MUST additionally bound each account's slice to a newest-first per-account +row cap supplied by the projections caller. Because live snapshot ingestion +writes a row per proxied request whenever the usage fingerprint changes, no +fixed row cap alone can guarantee coverage of a fixed time window; the +fetch MUST therefore exempt rows inside the configured pace-smoothing +window (the projections caller supplies its start as an uncapped recent +floor) so every row the equal-weight smoothing mean consumes is returned +regardless of write density, while the cap MUST still bound the rows older +than that floor. The cap MUST be sized to cover the remaining tail-weighted +consumers' lookback (the recent-burn EWMA window) at the ingestor's minimum +per-account write interval. Returned slices MUST keep the newest in-cutoff +rows and MUST remain ordered oldest-first. For accounts whose in-cutoff +rows do not exceed the cap, the returned histories MUST equal the previous +shared-floor fetch after the existing per-account trimming; for accounts +over the cap, the returned history MUST be exactly the union of every +in-cutoff row at or after the uncapped recent floor and the newest cap-many +in-cutoff rows older than the floor. + +#### Scenario: One weekly account does not widen the fetch for short-window accounts +- **GIVEN** one account with a 7-day window and several accounts with 5-hour windows +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** rows for the 5-hour accounts MUST be bounded by their own cutoff in SQL +- **AND** each account's resulting history slice MUST equal the slice the shared-floor fetch produced after per-account trimming + +#### Scenario: A dense account returns only its newest rows +- **GIVEN** an account whose in-cutoff usage-history rows exceed the per-account row cap +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** the account's slice MUST be exactly the in-cutoff rows at or after the uncapped recent floor plus the newest cap-many in-cutoff rows older than the floor, ordered oldest-first +- **AND** accounts whose in-cutoff rows do not exceed the cap MUST return their full trimmed slice unchanged + +#### Scenario: A write burst inside the smoothing window is never truncated +- **GIVEN** an account that wrote more usage-history rows inside the configured pace-smoothing window than the per-account row cap +- **WHEN** the projections history fetch runs on PostgreSQL +- **THEN** every in-cutoff row at or after the smoothing-window start MUST be returned +- **AND** the weekly-pace smoothed values MUST equal the values the uncapped fetch would produce + +#### Scenario: Capped probes stay index-only +- **GIVEN** usage history rows for multiple accounts and a populated visibility map +- **WHEN** the capped per-account probe shape is EXPLAINed on PostgreSQL with sequential and bitmap scans disabled +- **THEN** the plan MUST serve each probe as an Index Only Scan over the covering indexes with no sequential scan of `usage_history` + +#### Scenario: SQLite snapshot cache keeps the shared floor +- **GIVEN** the SQLite backend serves the projections history fetch through its snapshot cache +- **WHEN** per-account cutoffs and a per-account row cap are supplied +- **THEN** the SQLite read MAY keep the shared floor and MAY ignore the row cap +- **AND** per-account trimming in the caller MUST still bound each account's slice diff --git a/openspec/changes/cap-projection-history-rows/tasks.md b/openspec/changes/cap-projection-history-rows/tasks.md new file mode 100644 index 0000000000..ba91d9aa84 --- /dev/null +++ b/openspec/changes/cap-projection-history-rows/tasks.md @@ -0,0 +1,27 @@ +## 1. Implementation + +- [x] 1.1 Add a newest-first per-account row cap to the PostgreSQL bulk + usage-history read (lateral top-N probe per account, composed with the + existing per-account cutoffs; oldest-first slices preserved). +- [x] 1.2 Pass the cap from the dashboard projections history fetch, sized + so the tail-weighted EWMA consumers (depletion, weekly-pace burn) see + identical inputs. +- [x] 1.3 Exempt the configured pace-smoothing window from the cap + (uncapped recent floor plumbed from the projections caller; disjoint + floor + capped-tail branches in the lateral probe) so a per-request + write burst can never truncate the equal-weight smoothing mean. +- [x] 1.4 Keep the SQLite snapshot-cache path on the shared floor (cap + ignored, like cutoffs). + +## 2. Validation + +- [x] 2.1 Regression: capped slices equal the newest rows of the uncapped + fetch, compose with per-account cutoffs, leave under-cap accounts + untouched, and never drop rows at or after the uncapped recent floor; + SQLite ignores the cap. +- [x] 2.2 PostgreSQL plan tests: the capped lateral probes (with and + without the floor branch) stay index-only on the covering indexes. +- [x] 2.3 Unit test: the projections fetch supplies the cap and the + smoothing-window floor. +- [x] 2.4 Run lint, type checks, sqlite + PostgreSQL test slices, and strict + OpenSpec validation. diff --git a/tests/integration/test_usage_repository.py b/tests/integration/test_usage_repository.py index a0b2be0db9..6ac9455378 100644 --- a/tests/integration/test_usage_repository.py +++ b/tests/integration/test_usage_repository.py @@ -1298,6 +1298,245 @@ async def test_bulk_history_since_per_account_cutoffs_parity(db_setup): assert [snapshot.used_percent for snapshot in trimmed] == [20.0] +@pytest.mark.asyncio +async def test_bulk_history_since_per_account_row_cap_keeps_newest_rows(db_setup): + """The PostgreSQL row cap keeps each account's newest in-cutoff rows in + oldest-first order; under-cap accounts are unaffected and SQLite ignores + the cap entirely (snapshot-cache path, like ``cutoffs``).""" + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-dense")) + await accounts_repo.upsert(_make_account("acc-sparse")) + + for offset in range(8): + await repo.add_entry( + "acc-dense", + 10.0 + offset, + window="secondary", + recorded_at=now - timedelta(minutes=8 - offset), + ) + await repo.add_entry("acc-sparse", 90.0, window="secondary", recorded_at=now - timedelta(hours=2)) + await repo.add_entry("acc-sparse", 95.0, window="secondary", recorded_at=now - timedelta(hours=1)) + + since = now - timedelta(days=7) + capped = await repo.bulk_history_since( + ["acc-dense", "acc-sparse"], + "secondary", + since, + per_account_row_cap=3, + ) + uncapped = await repo.bulk_history_since(["acc-dense", "acc-sparse"], "secondary", since) + + dialect = "postgresql" if str(engine.url).startswith("postgresql") else "sqlite" + if dialect == "postgresql": + # Newest three rows, still oldest-first. + assert [snapshot.used_percent for snapshot in capped["acc-dense"]] == [15.0, 16.0, 17.0] + assert capped["acc-dense"] == uncapped["acc-dense"][-3:] + else: + # SQLite serves the shared-floor snapshot cache; the cap is ignored. + assert [snapshot.used_percent for snapshot in capped["acc-dense"]] == [ + snapshot.used_percent for snapshot in uncapped["acc-dense"] + ] + # Under-cap accounts return their full in-cutoff slice on every backend. + assert [snapshot.used_percent for snapshot in capped["acc-sparse"]] == [90.0, 95.0] + + +@pytest.mark.asyncio +async def test_bulk_history_since_row_cap_respects_per_account_cutoffs_postgresql(db_setup): + """The cap composes with per-account cutoffs: the cutoff bounds the + lookback first, then the cap keeps the newest rows inside it.""" + now = utcnow() + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only row-cap test") + + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-short")) + await accounts_repo.upsert(_make_account("acc-wide")) + + await repo.add_entry("acc-short", 10.0, window="primary", recorded_at=now - timedelta(hours=20)) + await repo.add_entry("acc-short", 20.0, window="primary", recorded_at=now - timedelta(hours=1)) + for offset in range(4): + await repo.add_entry( + "acc-wide", + 30.0 + offset, + window="primary", + recorded_at=now - timedelta(hours=20 - offset), + ) + + grouped = await repo.bulk_history_since( + ["acc-short", "acc-wide"], + "primary", + now - timedelta(days=7), + cutoffs={ + "acc-short": now - timedelta(hours=5), + "acc-wide": now - timedelta(days=7), + }, + per_account_row_cap=3, + ) + + # acc-short's 20h-old row falls outside its cutoff even though the cap + # alone would have kept it. + assert [snapshot.used_percent for snapshot in grouped["acc-short"]] == [20.0] + # acc-wide keeps only the newest three of its four in-cutoff rows. + assert [snapshot.used_percent for snapshot in grouped["acc-wide"]] == [31.0, 32.0, 33.0] + + +@pytest.mark.asyncio +async def test_bulk_history_since_row_cap_exempts_uncapped_recent_floor_postgresql(db_setup): + """Rows at or after ``uncapped_recent_floor`` bypass the row cap. + + Live ingestion writes per proxied request whenever the usage fingerprint + moves, so a burst can put more rows inside the pace-smoothing window than + any fixed cap; the smoothing mean weighs those samples equally, so they + must all come back. The cap still bounds the older remainder. + """ + now = utcnow() + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only row-cap test") + + accounts_repo = AccountsRepository(session) + repo = UsageRepository(session) + await accounts_repo.upsert(_make_account("acc-burst")) + + # Six rows inside the floor window (a burst denser than the cap) and + # four older rows between the cutoff and the floor. + for offset in range(6): + await repo.add_entry( + "acc-burst", + 50.0 + offset, + window="secondary", + recorded_at=now - timedelta(minutes=30 - offset), + ) + for offset in range(4): + await repo.add_entry( + "acc-burst", + 10.0 + offset, + window="secondary", + recorded_at=now - timedelta(hours=10 - offset), + ) + + grouped = await repo.bulk_history_since( + ["acc-burst"], + "secondary", + now - timedelta(days=7), + per_account_row_cap=3, + uncapped_recent_floor=now - timedelta(minutes=60), + ) + + # All six in-floor rows survive despite cap=3; the older tail keeps only + # its newest three rows; the slice stays oldest-first. + assert [snapshot.used_percent for snapshot in grouped["acc-burst"]] == [ + 11.0, + 12.0, + 13.0, + 50.0, + 51.0, + 52.0, + 53.0, + 54.0, + 55.0, + ] + + +@pytest.mark.asyncio +async def test_bulk_history_since_capped_query_plan_is_index_only_postgresql(db_setup): + """The capped lateral probes must stay heap-free on the covering indexes. + + Each per-account probe descends the covering index backward and stops at + the cap or cutoff; a plain Index Scan here would mean the probe shape + lost the covering payload and fetches the heap per row. + """ + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + await _seed_bulk_history_plan_fixture(session) + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + plan = ( + await session.execute( + text( + """ + EXPLAIN (FORMAT JSON) + SELECT recent.* + FROM (VALUES ('acc1', now() - interval '5 hours'), + ('acc2', now() - interval '7 days')) + AS account_cutoffs (account_id, cutoff) + JOIN LATERAL ( + SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.cutoff + AND "window" = 'secondary' + ORDER BY recorded_at DESC, id DESC + LIMIT 100 + ) AS recent ON true + """ + ) + ) + ).scalar_one() + + plan_json = json.dumps(plan) + assert "Index Only Scan" in plan_json + assert "idx_usage_window_raw_account_time_covering" in plan_json + assert "Seq Scan on usage_history" not in plan_json + + +@pytest.mark.asyncio +async def test_bulk_history_since_capped_floor_query_plan_is_index_only_postgresql(db_setup): + """The floor-exempt probe shape (uncapped recent branch UNION ALL capped + older branch) must keep both branches heap-free on the covering index.""" + async with SessionLocal() as session: + if _dialect_name(session) != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + await _seed_bulk_history_plan_fixture(session) + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + plan = ( + await session.execute( + text( + """ + EXPLAIN (FORMAT JSON) + SELECT recent.* + FROM (VALUES ('acc1', now() - interval '7 days', now() - interval '4 hours'), + ('acc2', now() - interval '7 days', now() - interval '4 hours')) + AS account_cutoffs (account_id, cutoff, uncapped_floor) + JOIN LATERAL ( + (SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.uncapped_floor + AND "window" = 'secondary') + UNION ALL + (SELECT id, account_id, used_percent, recorded_at, reset_at, window_minutes + FROM usage_history + WHERE account_id = account_cutoffs.account_id + AND recorded_at >= account_cutoffs.cutoff + AND recorded_at < account_cutoffs.uncapped_floor + AND "window" = 'secondary' + ORDER BY recorded_at DESC, id DESC + LIMIT 100) + ) AS recent ON true + """ + ) + ) + ).scalar_one() + + plan_json = json.dumps(plan) + assert "Index Only Scan" in plan_json + assert "idx_usage_window_raw_account_time_covering" in plan_json + assert "Seq Scan on usage_history" not in plan_json + assert "Index Scan using" not in plan_json + + def _legacy_additional_entry( account_id: str, *, diff --git a/tests/unit/test_dashboard_projection_history_cap.py b/tests/unit/test_dashboard_projection_history_cap.py new file mode 100644 index 0000000000..3719a04bd9 --- /dev/null +++ b/tests/unit/test_dashboard_projection_history_cap.py @@ -0,0 +1,91 @@ +"""The projections history fetch must request the per-account row cap. + +The cap is what keeps the PostgreSQL bulk read bounded on deployments where +live snapshot ingestion densifies ``usage_history``; losing the kwarg would +silently regress the read back to full-window row counts. +""" + +from __future__ import annotations + +from datetime import datetime, timedelta +from typing import cast + +import pytest + +from app.db.models import UsageHistory +from app.modules.dashboard.repository import DashboardRepository +from app.modules.dashboard.service import ( + _PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP, + _load_projection_histories, +) + +pytestmark = pytest.mark.unit + + +class _RecordingRepo: + def __init__(self) -> None: + self.calls: list[dict] = [] + + async def bulk_usage_history_since( + self, + account_ids, + window, + since, + *, + cutoffs=None, + per_account_row_cap=None, + uncapped_recent_floor=None, + ): + self.calls.append( + { + "account_ids": list(account_ids), + "window": window, + "since": since, + "cutoffs": cutoffs, + "per_account_row_cap": per_account_row_cap, + "uncapped_recent_floor": uncapped_recent_floor, + } + ) + return {} + + +def _usage_entry(account_id: str, window: str, window_minutes: int, recorded_at: datetime) -> UsageHistory: + return UsageHistory( + id=1, + account_id=account_id, + used_percent=10.0, + window=window, + window_minutes=window_minutes, + recorded_at=recorded_at, + ) + + +@pytest.mark.asyncio +async def test_projection_history_fetch_passes_per_account_row_cap(): + now = datetime(2026, 8, 16, 12, 0, 0) + repo = _RecordingRepo() + primary_usage = { + "acc1": _usage_entry("acc1", "primary", 300, now - timedelta(minutes=1)), + } + secondary_usage = { + "acc1": _usage_entry("acc1", "secondary", 10080, now - timedelta(minutes=1)), + } + + await _load_projection_histories( + cast(DashboardRepository, repo), + primary_usage, + secondary_usage, + now, + smoothing_window_minutes=240, + ) + + assert len(repo.calls) == 2 + assert {call["window"] for call in repo.calls} == {"primary", "secondary"} + for call in repo.calls: + assert call["per_account_row_cap"] == _PROJECTION_HISTORY_PER_ACCOUNT_ROW_CAP + assert call["cutoffs"] is not None + # The weekly-pace smoothing mean weighs every in-window sample + # equally, so the fetch must exempt the configured smoothing window + # from the row cap; a write burst may otherwise out-write the cap and + # shift the smoothed schedule gap. + assert call["uncapped_recent_floor"] == now - timedelta(minutes=240) From c1caa4468cdf2f94f9abe675b63e63a13dbdd383 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:36:05 +0900 Subject: [PATCH 058/117] perf(accounts): bound the account-listing live tail with a 2h fold lag and a 30s summary cache (#1792) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(accounts): bound the summary live tail with a 2h fold lag and a 30s summary cache The account listing dedupes and re-aggregates every raw request_logs row above the lifetime fold watermark on each accounts load. The 24h fold lag kept that tail at ~660k rows (11% of the table) on the reference deployment: 1,459 calls averaging 2.1s, max 131s, with the cold plan degrading to a full seq-scan hash join over the whole table (measured 59.9s vs 63ms for the identical query bounded to 2h). The 24h sizing guarded against a threat the write path never had: it assumed rows are dated at request start but inserted at stream end. requested_at is stamped inside add_log at write time (requested_at or utcnow(); no live caller passes a value), so the lag only needs to cover insert-visibility skew — measured worst case over one full production history (6.0M rows): 7.9s below the requested_at frontier, p99.9 30ms. Post-insert mutators are fenced independently: update_model_for_request skips rows at/below the watermarks, and consolidation/deletion reassign logs under the fold-state lock while mirroring folded sums. FOLD_LAG drops to 2h (~900x margin); the one-time 22h watermark jump on upgrade is absorbed by the ordinary bounded backfill slices. Account request-usage summaries additionally get a process-local 30s TTL cache keyed by the account-id signature, mirroring the request-log COUNT cache (#1340): the listing tolerates short staleness and polls on the same cadence. Account deletion and duplicate-identity consolidation clear it; a non-positive TTL bypasses it (the test suite runs with 0). The rollup/retention parity corpus pins CORPUS_FOLD_LAG = 24h: its 10-day geometry (TARGET_W, prune floor, unaligned windows) was authored against the old lag and the parity semantics it proves are lag-independent. OpenSpec: openspec/changes/bound-account-summary-live-tail Co-Authored-By: Claude Fable 5 * fix(accounts): fence summary-cache fills against concurrent invalidation A fill already computing when account deletion or duplicate-identity consolidation cleared the cache could store its pre-clear result afterwards, serving stale attribution for a full TTL despite the invalidation. Fills now capture an invalidation generation before their first await and stores are discarded on mismatch; the lifecycle clear runs synchronously right after its commit, so on the single event loop every store either precedes the commit (wiped by the clear) or observes the bumped generation. Also documents the operational clock bound the shortened fold lag relies on (writer clocks within one fold lag of the fold leader; pre-existing requirement, only the margin changed) in the change context, and adds the in-flight invalidation scenario to the spec delta. Found by local codex review of this branch. OpenSpec: openspec/changes/bound-account-summary-live-tail Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude --- app/modules/accounts/repository.py | 75 ++++++++ app/modules/accounts/usage_rollup.py | 35 +++- .../context.md | 107 +++++++++++ .../proposal.md | 68 +++++++ .../specs/query-caching/spec.md | 91 ++++++++++ .../bound-account-summary-live-tail/tasks.md | 42 +++++ tests/conftest.py | 11 ++ .../integration/test_account_usage_rollup.py | 166 +++++++++++++++++- .../test_request_usage_rollup_parity.py | 31 +++- 9 files changed, 609 insertions(+), 17 deletions(-) create mode 100644 openspec/changes/bound-account-summary-live-tail/context.md create mode 100644 openspec/changes/bound-account-summary-live-tail/proposal.md create mode 100644 openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md create mode 100644 openspec/changes/bound-account-summary-live-tail/tasks.md diff --git a/app/modules/accounts/repository.py b/app/modules/accounts/repository.py index 3d1b62fd24..64dbff5140 100644 --- a/app/modules/accounts/repository.py +++ b/app/modules/accounts/repository.py @@ -1,6 +1,7 @@ from __future__ import annotations import json +import time import uuid from dataclasses import dataclass from datetime import datetime @@ -65,6 +66,62 @@ class AccountRequestUsageSummary: total_cost_usd: float +# The account-listing request-usage summary dedupes and re-aggregates the +# un-folded raw tail on every dashboard accounts load, and the displayed +# lifetime totals tolerate short staleness. Cache the merged summaries per +# account-id signature for a small fixed TTL, mirroring the request-log +# COUNT cache (issue #1340 / PRINCIPLES.md P2); the test suite patches the +# TTL to 0 so summaries stay exact within a test. Account deletion and +# duplicate-identity consolidation clear the cache because they re-attribute +# usage rather than merely append to it. +_SUMMARY_CACHE_TTL_SECONDS = 30.0 +_SUMMARY_CACHE_MAX_ENTRIES = 64 +_request_usage_summary_cache: dict[tuple[str, ...] | None, tuple[dict[str, AccountRequestUsageSummary], float]] = {} +# Invalidation generation: a fill that was already computing when a clear +# happened must not re-populate the cache with its pre-clear result. Fills +# capture the generation before their first await and stores are discarded +# on mismatch. Deletion/consolidation clear synchronously right after their +# commit (no await in between), so every store either precedes the commit +# (its stale data is wiped by the clear) or observes the bumped generation. +_summary_cache_generation = 0 + + +def _clear_request_usage_summary_cache() -> None: + global _summary_cache_generation + _summary_cache_generation += 1 + _request_usage_summary_cache.clear() + + +def _cached_request_usage_summaries( + key: tuple[str, ...] | None, +) -> dict[str, AccountRequestUsageSummary] | None: + entry = _request_usage_summary_cache.get(key) + if entry is None: + return None + summaries, expires_at = entry + if time.monotonic() >= expires_at: + _request_usage_summary_cache.pop(key, None) + return None + return summaries + + +def _store_request_usage_summaries( + key: tuple[str, ...] | None, + summaries: dict[str, AccountRequestUsageSummary], + ttl_seconds: float, + generation: int, +) -> None: + if generation != _summary_cache_generation: + return + if len(_request_usage_summary_cache) >= _SUMMARY_CACHE_MAX_ENTRIES: + oldest = min( + _request_usage_summary_cache, + key=lambda existing: _request_usage_summary_cache[existing][1], + ) + _request_usage_summary_cache.pop(oldest, None) + _request_usage_summary_cache[key] = (summaries, time.monotonic() + ttl_seconds) + + class AccountIdentityConflictError(Exception): def __init__(self, email: str) -> None: self.email = email @@ -122,6 +179,13 @@ async def list_request_usage_summary_by_account( self, account_ids: list[str] | None = None, ) -> dict[str, AccountRequestUsageSummary]: + ttl_seconds = _SUMMARY_CACHE_TTL_SECONDS + cache_key = tuple(sorted(account_ids)) if account_ids is not None else None + generation = _summary_cache_generation + if ttl_seconds > 0: + cached = _cached_request_usage_summaries(cache_key) + if cached is not None: + return dict(cached) rollup_repo = AccountUsageRollupRepository(self._session) folded, watermark = await rollup_repo.read_state(account_ids) @@ -165,6 +229,9 @@ async def list_request_usage_summary_by_account( cached_input_tokens=cached_total, total_cost_usd=round(float(total_cost_usd), 6), ) + if ttl_seconds > 0: + _store_request_usage_summaries(cache_key, summaries, ttl_seconds, generation) + return dict(summaries) return summaries async def exists_active_chatgpt_account_id(self, chatgpt_account_id: str) -> bool: @@ -274,6 +341,10 @@ async def _upsert_unlocked( await self._session.commit() if usage_cache_dirty: _clear_bulk_history_since_sqlite_cache() + # Consolidation re-attributes request logs and rollup sums + # to the canonical account; cached listing summaries would + # keep reporting the deleted duplicates until TTL expiry. + _clear_request_usage_summary_cache() # Duplicate reconciliation deletes Account rows, cascading # any account_proxy_bindings they owned. The route cache is # keyed by deterministic account id, so stale duplicate-id @@ -852,6 +923,10 @@ async def delete(self, account_id: str, *, delete_history: bool = False) -> bool await self._session.commit() if deleted_id is not None: _clear_bulk_history_since_sqlite_cache() + # Deletion drops the account's rollup row and detaches or + # deletes its request logs; cached listing summaries would + # keep reporting the account until TTL expiry. + _clear_request_usage_summary_cache() return deleted_id is not None async def rotate_tokens( diff --git a/app/modules/accounts/usage_rollup.py b/app/modules/accounts/usage_rollup.py index 9a4cd95912..e8c7da70c5 100644 --- a/app/modules/accounts/usage_rollup.py +++ b/app/modules/accounts/usage_rollup.py @@ -18,13 +18,34 @@ # Rows younger than the lag stay on the live side of the fold boundary. # The lag MUST exceed the maximum possible distance between a log row's -# requested_at and its actual insertion time: requested_at is the request -# START, but the row is written at stream END, so a long-running stream -# inserts a row dated its full duration in the past — if that lands below an -# already-advanced watermark it is neither folded nor in the live tail and -# vanishes from totals. 24h dwarfs any survivable stream duration and the -# post-stream duplicate/model/cost rewrite paths (which settle in seconds). -FOLD_LAG = timedelta(hours=24) +# requested_at and the moment its insert becomes visible: a row landing below +# an already-advanced watermark is neither folded nor in the live tail and +# vanishes from totals. Every insert path stamps requested_at inside +# ``RequestLogsRepository.add_log`` at write time (``requested_at or +# utcnow()``; no live caller passes an explicit value — the parameter exists +# for tests), so the distance is only clock skew between replicas plus the +# single-row insert transaction's commit latency, NOT the request duration: +# a stream-end log write is dated at the write, not at the request start. +# Measured over one full production history (6.0M rows) the worst insert +# landed 7.9s below the requested_at frontier (p99.9 = 30ms). Post-insert +# mutators need no allowance here either: ``update_model_for_request`` +# skips rows at/below the watermarks, and consolidation/deletion reassign +# logs under the fold-state lock while mirroring the folded sums. +# +# 2h keeps a ~900x margin over the observed worst case (covering scheduler +# stalls, NTP drift, paused VMs) while bounding the raw tail every account +# and API-key summary read must dedupe and re-aggregate; the previous 24h +# lag — sized for a stream-start dating scheme this codebase never actually +# had — left an always-rescanned tail of ~660k rows (11% of the table) on +# the reference deployment and degraded the listing aggregate to a full +# seq-scan hash join (measured 60s cold / 2.1s warm vs 63ms at 2h). +# +# This lag is a shared visibility contract: the hourly/conversation fold +# targets and the retention min-gate (``now - 2 * FOLD_LAG`` freshness, +# ``watermark - FOLD_LAG`` prune floor) derive from the same constant. +# A fold pass after an upgrade from the 24h lag absorbs the watermark jump +# as one ordinary backfill slice (FOLD_SLICE bounds it). +FOLD_LAG = timedelta(hours=2) # Historical backfill folds at most this much history per transaction. FOLD_SLICE = timedelta(days=7) diff --git a/openspec/changes/bound-account-summary-live-tail/context.md b/openspec/changes/bound-account-summary-live-tail/context.md new file mode 100644 index 0000000000..48589fc819 --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/context.md @@ -0,0 +1,107 @@ +# Context: measurements behind the 2h fold lag + +All numbers from the reference production deployment (PostgreSQL 18, +2 vCPU), 2026-08-16, read-only. + +## Insert-visibility skew (what the lag must actually cover) + +`requested_at` is assigned inside `RequestLogsRepository.add_log` as +`requested_at or utcnow()`; a repo-wide audit found **no live caller passing +an explicit value** (the parameter exists for tests; the +`limit_warmup` Protocol declares it but its call site does not use it). The +log write happens at stream end and is dated at the write, so the +stream-start-dating threat the original 24h comment guarded against does not +exist — and has not existed since the initial commit. + +Frontier-lag measurement (how far below the running `max(requested_at)` by +insert order — `id` — a row lands at insert): + +| window | rows | p99 | p99.9 | max | +|---|---|---|---|---| +| last 2M rows (~3.6 days) | 1,999,999 | 4.6ms | 46ms | 5.36s | +| full history | 5,992,511 | — | 30ms | **7.89s** | + +This bound covers every insert path (normal, warmup, duplicates), because it +is computed over every row. 2h ≈ 900x the all-history worst case, with room +for replica clock skew, paused VMs, and event-loop stalls far beyond +anything observed. + +Operational bound made explicit by this change: writer-replica wall clocks +(the `utcnow()` used by `add_log`) must stay within one fold lag of the fold +leader's clock, and no insert transaction may stay open that long. This +requirement is not new — it existed at 24h and only its margin changed. All +replicas share one database and one NTP discipline; a clock trailing by two +hours implies TLS/OAuth breakage long before rollup drift, and the DB pool +recycles connections far below the lag. The residual exposure (a 2–24h +trailing clock that the old lag absorbed) is accepted; deployments that +cannot bound clock skew should not shorten further. A hard fence (stamping +`requested_at` from the database clock) would cost the write path's +no-refresh optimization and is left as a follow-up if ever needed. + +Post-insert mutators are fenced independently of the lag: + +- `update_model_for_request` selects only rows strictly above the lifetime + watermark and at/above the hourly watermark, under the fold-state lock. +- Account deletion and duplicate-identity consolidation reassign request + logs under the fold-state lock and mirror folded sums + (`merge_rollups_into`, time-rollup mirrors). + +## Cache invalidation race (generation fence) + +A summary fill that is between its two statements when deletion or +consolidation commits could otherwise store its pre-commit result *after* +the lifecycle clear, serving stale attribution for a full TTL. Fills capture +a generation counter before their first await; `_clear_...` bumps it and +stores are discarded on mismatch. The clear runs synchronously right after +the lifecycle commit (no await between them), so on the single event loop +every store either precedes the commit (wiped by the clear) or observes the +bumped generation. Regression: +`test_summary_cache_fill_discarded_when_invalidated_mid_flight`. + +## Read-path cost of the 24h tail + +- Tail at measurement time: 655,804 of 5,992,447 rows (11%). +- Listing aggregate (`deduped_usage_aggregate_stmt` above the watermark), + production `EXPLAIN (ANALYZE, BUFFERS)`: + - **24h watermark (actual)**: 59.96s cold — the planner abandons the + covering index at ~656k tail rows and degrades to a parallel seq scan + (external-merge sort, 28MB spill) hash-joined against a hash of the + entire 5.99M-row table (46,474kB hash memory, 16 batches, 75k temp + pages). Warm-cache production average for the same call family: 2.1s + over 1,459 calls, max 131s. + - **2h parameter, same query shape**: 63ms — index scan on + `idx_logs_dash_usage_covering` (9,124 rows), hash-agg dedupe, nested + loop over `request_logs_pkey`. + - **1h parameter**: 18ms. +- No index or query-shape change is needed once the tail is bounded; the + existing covering index already serves the bounded range. The residual + per-render cost is then amortized by the 30s summary cache. + +## Upgrade path + +`run_fold_pass` folds toward `now - FOLD_LAG` in bounded `FOLD_SLICE` (7d) +transactions, so the one-time 22h watermark jump after this change lands in +a single ordinary slice; `test_fold_absorbs_widened_watermark_gap` pins the +totals across the jump. The watermark only advances, so a rollback to the +24h constant simply pauses folding until `now - 24h` catches up with the +already-advanced watermark — reads stay correct throughout (rows above the +watermark are always live-tail-served). + +## Retention interaction + +The retention job's raw-prune floor (`watermark - FOLD_LAG`) and freshness +gate (`watermark` within `2 * FOLD_LAG` of now) tighten with the constant. +The floor exists so no rollup is robbed of raw it has not folded and so +concurrent readers holding a slightly older watermark lose nothing; both +need seconds. Consequence for retention-enabled deployments: raw becomes +physically prunable at ~4h age instead of ~48h when the configured retention +period is that short; sub-hour partial-window reads (non-hour-aligned +`since`/`until`) over pruned history hit their documented raw-degrade path +sooner. Hour-aligned reads are rollup-served and unaffected. + +The rollup/retention parity corpus +(`tests/integration/test_request_usage_rollup_parity.py`) was authored +against the 24h lag (TARGET_W at BASE+9d, prune floor at BASE+8d, unaligned +windows and boundary rows placed between them); it now pins +`CORPUS_FOLD_LAG = 24h` explicitly because the parity semantics it proves +are lag-independent. diff --git a/openspec/changes/bound-account-summary-live-tail/proposal.md b/openspec/changes/bound-account-summary-live-tail/proposal.md new file mode 100644 index 0000000000..b164f9cdda --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/proposal.md @@ -0,0 +1,68 @@ +# Bound the account-summary live tail: 2h fold lag + short summary TTL cache + +## Why + +The account listing recomputes its request-usage summaries by deduping and +re-aggregating every raw `request_logs` row above the lifetime fold watermark +on each dashboard accounts load. The watermark trails `now` by the fold lag, +so the lag directly sizes that always-rescanned tail. + +The lag has been 24h since the rollup shipped, justified by a premise the +write path never had: the sizing comment (and the fold spec scenario) claim a +log row is *dated at request start but inserted at stream end*, so the lag +must exceed the maximum request duration. In this codebase `requested_at` is +stamped **inside `RequestLogsRepository.add_log` at write time** +(`requested_at or utcnow()`, and no live caller passes an explicit value — +the parameter exists for tests). A row can land below the `requested_at` +frontier only through replica clock skew, the single-row insert transaction's +commit latency, or a process stall. Measured over one full production history +(6.0M rows), the worst insert landed 7.9s below the frontier (p99.9 = 30ms). +Post-insert mutators need no lag allowance either: `update_model_for_request` +skips rows at/below the watermarks, and account consolidation/deletion +reassign logs under the fold-state lock while mirroring the folded sums. + +The oversized lag is expensive: on the reference deployment the 24h tail is +~660k rows (11% of the table), the listing aggregate was measured at 1,459 +calls averaging 2.1s (max 131s), and the cold plan degrades to a full +seq-scan hash join over the whole table (measured 60s). With a 2h bound the +identical query runs in 63ms via the covering-index nested-loop plan. + +Independently, the listing recomputes the summaries on every accounts load +even though the displayed lifetime totals tolerate short staleness — the same +shape the request-log COUNT cache already addresses (issue #1340). + +## What Changes + +- `FOLD_LAG` drops from 24h to 2h. 2h keeps a ~900x margin over the worst + insert-visibility skew ever observed while bounding the raw tail every + account and API-key summary read must re-aggregate. The hourly and + conversation fold targets and the retention min-gate derive from the same + constant and tighten with it; the retention floor's purpose (protect raw + the folds have not consumed and concurrent readers holding a slightly + older watermark) needs seconds, not hours. +- On upgrade, the first fold pass absorbs the 22h watermark jump as ordinary + bounded backfill slices; totals are unchanged (fold moves rows from the + live tail into the persisted sums). +- The fold-lag spec scenario is corrected to state the real invariant + (insert-visibility skew), replacing the false stream-start-dating premise. +- Account request-usage summaries gain a process-local fixed-TTL (30s) cache + keyed by the account-id signature, mirroring the request-log COUNT cache. + Account deletion and duplicate-identity consolidation clear it because they + re-attribute usage rather than merely append; a non-positive TTL bypasses + the cache (the test suite runs with TTL 0). + +No schema change, no new settings, no API change. + +## Impact + +- Affected specs: `query-caching` (fold safety-lag requirement, account + summary read requirement). +- Affected code: `app/modules/accounts/usage_rollup.py` (constant + sizing + rationale), `app/modules/accounts/repository.py` (summary TTL cache + + invalidation hooks), `tests/`. +- Behavior visible to operators: account/API-key summary reads get a 2h raw + tail instead of 24h; listing summaries may be up to 30s stale (matching the + dashboard's 30s poll cadence); with retention enabled, raw rows become + physically prunable once they are one fold lag (now 2h) below the + watermark, so sub-hour partial-window raw reads over pruned history reach + their documented degrade path sooner. diff --git a/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md b/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md new file mode 100644 index 0000000000..8a2555c4fc --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/specs/query-caching/spec.md @@ -0,0 +1,91 @@ +# query-caching (delta) + +## MODIFIED Requirements + +### Requirement: Account request usage summaries combine a persistent rollup with a bounded live tail + +Account request-usage summaries MUST NOT aggregate the full `request_logs` history per read. The read MUST combine persisted per-account rollup sums with a live aggregate constrained to rows newer than the rollup watermark, while preserving existing dedupe semantics (latest row id per `(account_id, request_id, requested_at)`) and existing filters (warmup kinds and soft-deleted rows excluded) on the live portion. + +The merged summaries MAY be served from a process-local cache keyed by the requested account-id signature for a small fixed TTL, because the displayed lifetime totals tolerate short staleness. Account deletion and duplicate-identity consolidation MUST clear the cache in the process that performed them (they re-attribute or remove usage rather than append to it). A non-positive TTL MUST bypass the cache entirely so tests and precision-sensitive callers observe exact totals. + +#### Scenario: Summary read does not scan folded history + +- **GIVEN** rollup rows exist with watermark `folded_through = T` +- **WHEN** account request-usage summaries are loaded +- **THEN** the live request-log aggregate MUST constrain to `requested_at > T` +- **AND** the returned totals MUST equal the persisted rollup sums plus the live-tail aggregate per account +- **AND** the cached-input clamp (`cached_input_tokens ≤ input_tokens`) MUST apply to the merged totals + +#### Scenario: Summary before the first fold matches legacy behavior + +- **GIVEN** no rollup rows exist yet +- **WHEN** account request-usage summaries are loaded +- **THEN** the live aggregate MUST cover all non-deleted, non-warmup request-log history +- **AND** the returned totals MUST equal the pre-rollup query results + +#### Scenario: Folding does not change reported totals + +- **GIVEN** a set of request-log rows including duplicate rows sharing `(account_id, request_id, requested_at)` +- **WHEN** a fold pass folds part of that history and summaries are read afterwards +- **THEN** the totals MUST equal the totals the legacy full-history dedupe aggregate would report for the same rows + +#### Scenario: Summary read is snapshot-consistent with a concurrent fold commit + +- **GIVEN** a fold slice may commit at any point during a summary read +- **WHEN** the read fetches rollup sums and the watermark +- **THEN** both MUST come from a single database snapshot (one statement) +- **AND** no qualifying request-log row's contribution may be absent from both the rollup sums and the live-tail aggregate of that read + +#### Scenario: Cached summaries are served within the TTL per signature + +- **GIVEN** a positive summary cache TTL +- **AND** summaries were computed for one account-id signature +- **WHEN** the same signature is requested again within the TTL +- **THEN** the cached summaries MAY be returned without touching the database +- **AND** a different account-id signature MUST NOT be served from that entry + +#### Scenario: Account deletion invalidates cached summaries + +- **GIVEN** cached summaries that include an account +- **WHEN** that account is deleted, or a duplicate-identity consolidation removes it +- **THEN** the cache MUST be cleared so the next read reflects the new attribution +- **AND** a summary computation already in flight when the invalidation happens MUST NOT re-populate the cache with its pre-invalidation result + +### Requirement: A background fold job advances the account usage rollup safely + +A periodic background job MUST fold request-log rows into `account_usage_rollups` and advance the watermark. Folding MUST be restricted to rows older than a safety lag, MUST apply the dedupe and filtering semantics of the summary query within the folded window, MUST run on at most one instance at a time, and MUST be idempotent under repeated or concurrent invocation. + +#### Scenario: Fold boundary respects the safety lag + +- **WHEN** a fold pass runs at time `now` +- **THEN** it MUST NOT fold any row with `requested_at > now − lag` +- **AND** rows younger than the lag remain covered by the live-tail aggregate +- **AND** the lag MUST exceed the maximum possible distance between a row's `requested_at` and the moment its insert becomes visible — `requested_at` is stamped at write time inside the log insert path, so this distance is bounded by replica clock skew, insert-commit latency, and process stalls, not by request duration — because a row landing below the watermark would otherwise vanish from totals +- **AND** post-insert mutations of folded rows MUST NOT rely on the lag: they are fenced by the watermark (skipped below it) or run under the fold-state lock while mirroring the folded sums + +#### Scenario: Widening the lag gap is absorbed as ordinary backfill + +- **GIVEN** a deployment whose persisted watermark trails `now − lag` by more than one fold cadence (for example after the lag constant is shortened) +- **WHEN** the next fold passes run +- **THEN** the gap MUST be folded in the bounded backfill slices with reported totals unchanged + +#### Scenario: Duplicate rows never split across the fold boundary + +- **GIVEN** duplicate request-log rows sharing the same `(account_id, request_id, requested_at)` +- **WHEN** a fold pass selects its window by `requested_at` +- **THEN** all rows of the duplicate group MUST land on the same side of the boundary +- **AND** only the latest row id of the group MUST contribute to the folded sums + +#### Scenario: Fold is idempotent and single-writer + +- **GIVEN** a fold pass has committed sums through watermark `T` +- **WHEN** another fold pass runs for the same window (repeat invocation or a second instance) +- **THEN** it MUST observe watermark `T` inside its transaction and fold no row at or before `T` +- **AND** no request-log row's contribution appears twice in the rollup + +#### Scenario: Historical backfill is sliced and non-blocking + +- **GIVEN** a deployment with existing request-log history and no rollup rows +- **WHEN** the first fold passes run +- **THEN** history MUST be folded in bounded time slices, each committed in its own transaction +- **AND** summary reads issued during backfill MUST return correct totals (rollup so far plus remaining live tail) diff --git a/openspec/changes/bound-account-summary-live-tail/tasks.md b/openspec/changes/bound-account-summary-live-tail/tasks.md new file mode 100644 index 0000000000..e27a34be4b --- /dev/null +++ b/openspec/changes/bound-account-summary-live-tail/tasks.md @@ -0,0 +1,42 @@ +# Tasks + +## 1. Fold lag + +- [x] 1.1 Shorten `FOLD_LAG` to 2h and rewrite the sizing comment around the + actual invariant: `requested_at` is stamped at insert time inside + `add_log`, so the lag bounds insert-visibility skew (clock skew + + commit latency + stalls), not request duration; note the measured + production worst case (7.9s over 6.0M rows) and the fenced post-insert + mutators +- [x] 1.2 Verify the watermark jump after the lag change is absorbed by the + ordinary backfill path with totals unchanged (regression test + `test_fold_absorbs_widened_watermark_gap`) + +## 2. Summary TTL cache + +- [x] 2.1 Cache `list_request_usage_summary_by_account` results per + account-id signature for a fixed 30s TTL with a bounded entry count, + mirroring the request-log COUNT cache; non-positive TTL bypasses +- [x] 2.2 Clear the cache on account deletion and on duplicate-identity + consolidation (both re-attribute usage), alongside the existing + `_clear_bulk_history_since_sqlite_cache()` call sites +- [x] 2.3 Zero the TTL for the test suite via an autouse conftest fixture so + summaries stay exact within a test (same pattern as the COUNT cache) + +## 3. Tests + +- [x] 3.1 Cache behavior: staleness within TTL, per-signature keying, + invalidation on delete and on consolidation +- [x] 3.2 Re-anchor the rollup/retention parity corpus on an explicit + `CORPUS_FOLD_LAG = 24h` pin — its 10-day geometry (TARGET_W, prune + floor, unaligned windows) was authored against the old lag and the + parity semantics are lag-independent +- [x] 3.3 Fix the lag-coupled backdated insert in + `test_account_delete_removes_rollup_row` (`now - FOLD_LAG / 2`) +- [x] 3.4 `uv run pytest`, `uv run ruff check`, `uv run ruff format --check` + +## 4. Spec + +- [x] 4.1 Correct the fold safety-lag scenario in `query-caching` and add the + summary-cache allowance to the account summary requirement +- [x] 4.2 `openspec validate bound-account-summary-live-tail --strict` diff --git a/tests/conftest.py b/tests/conftest.py index fd23f99058..48c9f8120c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -142,6 +142,17 @@ def _disable_request_log_count_cache(monkeypatch): monkeypatch.setattr(logs_repository_module, "_COUNT_CACHE_TTL_SECONDS", 0.0) +@pytest.fixture(autouse=True) +def _disable_account_usage_summary_cache(monkeypatch): + """Zero the account request-usage summary cache TTL so listing summaries + stay exact within a test. The TTL is a fixed constant in production; + cache-behavior tests patch it back to a positive value.""" + import app.modules.accounts.repository as accounts_repository_module + + accounts_repository_module._clear_request_usage_summary_cache() + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 0.0) + + @pytest.fixture(autouse=True) def _disable_rate_limit_reset_credits_scheduler_startup(monkeypatch): import app.main as main_module diff --git a/tests/integration/test_account_usage_rollup.py b/tests/integration/test_account_usage_rollup.py index 5273cd24d6..06864e1dcf 100644 --- a/tests/integration/test_account_usage_rollup.py +++ b/tests/integration/test_account_usage_rollup.py @@ -396,7 +396,7 @@ async def test_account_delete_removes_rollup_row(db_setup): logs_repo, account_id="acc_del2", request_id="req_2", - requested_at=now - timedelta(hours=12), + requested_at=now - FOLD_LAG / 2, ) await run_fold_pass(now=now + timedelta(days=1)) assert len(await _rollup_rows()) == 1 @@ -444,3 +444,167 @@ async def test_backfill_start_skips_excluded_prefix(db_setup): summaries = await _summaries() assert summaries["acc_prefix"].request_count == 1 + + +@pytest.mark.asyncio +async def test_fold_absorbs_widened_watermark_gap(db_setup): + """Upgrading from the previous 24h fold lag leaves the watermark far + behind the new ``now - FOLD_LAG`` target; the next pass must absorb the + gap as ordinary backfill slices with reported totals unchanged.""" + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_gap", "gap@example.com")) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_old", requested_at=now - timedelta(hours=30) + ) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_mid", requested_at=now - timedelta(hours=12) + ) + await _add_log( + logs_repo, account_id="acc_gap", request_id="req_gap_young", requested_at=now - timedelta(minutes=10) + ) + + # Simulate the pre-upgrade deployment: a fold pass whose target lands a + # day back, leaving rows between the old and new targets unfolded. + await run_fold_pass(now=now - timedelta(hours=24) + FOLD_LAG) + assert await _watermark() == now - timedelta(hours=24) + before = await _summaries() + assert before["acc_gap"].request_count == 3 + + await run_fold_pass(now=now) + assert await _watermark() == now - FOLD_LAG + rows = await _rollup_rows() + assert len(rows) == 1 + assert rows[0].request_count == 2 # old + mid folded, young stays live + + assert await _summaries() == before + + +@pytest.mark.asyncio +async def test_summary_cache_serves_within_ttl_per_signature(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_ttl", "ttl@example.com")) + await _add_log(logs_repo, account_id="acc_ttl", request_id="req_ttl_1", requested_at=now - timedelta(minutes=5)) + + first = await _summaries() + assert first["acc_ttl"].request_count == 1 + + async with SessionLocal() as session: + await _add_log( + RequestLogsRepository(session), + account_id="acc_ttl", + request_id="req_ttl_2", + requested_at=now - timedelta(minutes=1), + ) + + # Same signature within the TTL: served from cache, staleness tolerated. + assert (await _summaries())["acc_ttl"].request_count == 1 + # A different account-id signature is a different cache entry. + scoped = await _summaries(["acc_ttl"]) + assert scoped["acc_ttl"].request_count == 2 + + accounts_repository_module._clear_request_usage_summary_cache() + assert (await _summaries())["acc_ttl"].request_count == 2 + + +@pytest.mark.asyncio +async def test_summary_cache_cleared_on_account_delete(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_keep", "keep@example.com")) + await accounts_repo.upsert(_make_account("acc_gone", "gone@example.com")) + await _add_log(logs_repo, account_id="acc_keep", request_id="req_keep", requested_at=now - timedelta(minutes=5)) + await _add_log(logs_repo, account_id="acc_gone", request_id="req_gone", requested_at=now - timedelta(minutes=5)) + + first = await _summaries() + assert "acc_gone" in first + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_gone") + + after = await _summaries() + assert "acc_gone" not in after + assert after["acc_keep"].request_count == 1 + + +@pytest.mark.asyncio +async def test_summary_cache_cleared_on_identity_consolidation(db_setup, monkeypatch): + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + canonical = _make_account("acc_cc", "cc@example.com", chatgpt_account_id="chatgpt_cc") + duplicate = _make_account("acc_cc__copy", "cc@example.com", chatgpt_account_id="chatgpt_cc") + await accounts_repo.upsert(canonical, merge_by_email=False) + await accounts_repo.upsert(duplicate, merge_by_email=False) + await _add_log(logs_repo, account_id="acc_cc", request_id="req_cc_1", requested_at=now - timedelta(minutes=5)) + await _add_log( + logs_repo, account_id="acc_cc__copy", request_id="req_cc_2", requested_at=now - timedelta(minutes=5) + ) + + first = await _summaries() + assert first["acc_cc"].request_count == 1 + assert first["acc_cc__copy"].request_count == 1 + + async with SessionLocal() as session: + reauth = _make_account("acc_cc", "cc@example.com", chatgpt_account_id="chatgpt_cc") + saved = await AccountsRepository(session).upsert(reauth, merge_by_email=False, merge_by_chatgpt_identity=True) + assert saved.id == "acc_cc" + + after = await _summaries() + assert "acc_cc__copy" not in after + assert after["acc_cc"].request_count == 2 + + +@pytest.mark.asyncio +async def test_summary_cache_fill_discarded_when_invalidated_mid_flight(db_setup, monkeypatch): + """A fill already computing when deletion/consolidation clears the cache + must not re-populate it with its pre-clear result (generation fence).""" + import app.modules.accounts.repository as accounts_repository_module + + monkeypatch.setattr(accounts_repository_module, "_SUMMARY_CACHE_TTL_SECONDS", 30.0) + accounts_repository_module._clear_request_usage_summary_cache() + now = utcnow() + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + await accounts_repo.upsert(_make_account("acc_racefill", "racefill@example.com")) + await _add_log( + logs_repo, account_id="acc_racefill", request_id="req_rf", requested_at=now - timedelta(minutes=5) + ) + + real_read_state = accounts_repository_module.AccountUsageRollupRepository.read_state + + async def _read_state_with_racing_clear(self, account_ids=None): + result = await real_read_state(self, account_ids) + # Simulate a consolidation/deletion committing and clearing while + # this fill is still between its two statements. + accounts_repository_module._clear_request_usage_summary_cache() + return result + + monkeypatch.setattr( + accounts_repository_module.AccountUsageRollupRepository, "read_state", _read_state_with_racing_clear + ) + first = await _summaries() + assert first["acc_racefill"].request_count == 1 + # The interleaved clear must have won: nothing was cached. + assert accounts_repository_module._request_usage_summary_cache == {} diff --git a/tests/integration/test_request_usage_rollup_parity.py b/tests/integration/test_request_usage_rollup_parity.py index ae749dd597..42e5dd75f4 100644 --- a/tests/integration/test_request_usage_rollup_parity.py +++ b/tests/integration/test_request_usage_rollup_parity.py @@ -35,7 +35,6 @@ ) from app.db.session import SessionLocal from app.modules.accounts.repository import AccountsRepository -from app.modules.accounts.usage_rollup import FOLD_LAG from app.modules.accounts.usage_time_rollup import ( floor_to_hour, run_conversation_fold_pass, @@ -50,10 +49,24 @@ _EPOCH = datetime(1970, 1, 1) +# The 10-day corpus geometry below (TARGET_W at BASE + 9d, prune floor at +# BASE + 8d, unaligned windows and boundary rows placed between them) was +# authored against a 24h fold lag. The parity semantics under test are +# lag-independent, so the lag is pinned here to keep the corpus exercising +# every boundary it was designed around; the production FOLD_LAG's own +# tail/absorption behavior is covered in test_account_usage_rollup.py. +CORPUS_FOLD_LAG = timedelta(hours=24) + + +@pytest.fixture(autouse=True) +def _pin_corpus_fold_lag(monkeypatch): + monkeypatch.setattr("app.modules.accounts.usage_time_rollup.FOLD_LAG", CORPUS_FOLD_LAG) + + # Fixed 10-day corpus timeline (all naive UTC, matching requested_at). BASE = datetime(2025, 7, 1) NOW = BASE + timedelta(days=10, minutes=37) -TARGET_W = floor_to_hour(NOW - FOLD_LAG) # BASE + 9d +TARGET_W = floor_to_hour(NOW - CORPUS_FOLD_LAG) # BASE + 9d MID_W = BASE + timedelta(days=5, hours=3) # whole hour mid-history SINCE_ALIGNED = BASE + timedelta(days=2) @@ -462,10 +475,10 @@ async def test_switched_readers_match_legacy_across_watermark_states(db_setup): # Watermark state 2 — mid-history whole hour. The hourly and conversation # folds advance separately (mixed-watermark states in between must hold # parity too: each satellite degrades on its own watermark). - await run_hourly_fold_pass(now=MID_W + FOLD_LAG) + await run_hourly_fold_pass(now=MID_W + CORPUS_FOLD_LAG) assert await _watermark() == MID_W _assert_snapshots_equal(await _snapshot(), reference) - await run_conversation_fold_pass(now=MID_W + FOLD_LAG) + await run_conversation_fold_pass(now=MID_W + CORPUS_FOLD_LAG) assert await _conversation_watermark() == MID_W _assert_snapshots_equal(await _snapshot(), reference) @@ -486,8 +499,8 @@ async def test_reader_is_consistent_under_concurrent_fold_commit(db_setup, monke came from, and folding never deletes raw rows.""" await _seed_corpus() reference = await _snapshot() - await run_hourly_fold_pass(now=MID_W + FOLD_LAG) - await run_conversation_fold_pass(now=MID_W + FOLD_LAG) + await run_hourly_fold_pass(now=MID_W + CORPUS_FOLD_LAG) + await run_conversation_fold_pass(now=MID_W + CORPUS_FOLD_LAG) real_read_hourly_window = request_logs_repository_module.read_hourly_window fold_injections = {"count": 0} @@ -568,7 +581,7 @@ async def test_escape_hatch_reset_degrades_to_legacy_then_rebackfills(db_setup): @pytest.mark.asyncio async def test_statistics_survive_retention_pruning_folded_raw(db_setup, monkeypatch): """The headline guarantee: after raw rows below the retention gate - (watermark - FOLD_LAG) are physically deleted, every rollup-served + (watermark - fold lag) are physically deleted, every rollup-served statistic is unchanged — INCLUDING the distinct-conversation metrics, which the conversation presence satellite now serves for folded history (they used to be raw-bound and shrink here). earliest_activity_at falls @@ -584,7 +597,7 @@ async def test_statistics_survive_retention_pruning_folded_raw(db_setup, monkeyp lead_ceil = floor_to_hour(SINCE_UNALIGNED) + timedelta(hours=1) leadless = await _snapshot(lead_since=lead_ceil) - prune_cutoff = TARGET_W - FOLD_LAG + prune_cutoff = TARGET_W - CORPUS_FOLD_LAG async with SessionLocal() as session: await session.execute(delete(RequestLog).where(RequestLog.requested_at < prune_cutoff)) await session.commit() @@ -693,7 +706,7 @@ async def test_dashboard_overview_json_is_identical_before_and_after_fold(async_ before = await async_client.get("/api/dashboard/overview?timeframe=7d") assert before.status_code == 200 - folded_slices = await run_hourly_fold_pass() # real now: rows are > FOLD_LAG old + folded_slices = await run_hourly_fold_pass() # real now: rows are > fold lag old assert folded_slices > 0 assert await run_conversation_fold_pass() > 0 async with SessionLocal() as session: From d4f9e23cd623d67beee2df153723839e391d44d1 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:36:30 +0900 Subject: [PATCH 059/117] perf(accounts): make account deletion a fast mark + background batch drain (#1795) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(accounts): move account deletion bulk row work to a background batch worker DELETE /api/accounts/{id} previously detached (or deleted) the account's entire raw history in one transaction while holding the fold-state lock: measured on production, ~313s to soft-detach 133k request_logs rows plus 2x11.6s for usage_history, blocking every fold pass, pinning a pool connection past the QueuePool timeout, and timing out the HTTP client. The API now stamps a durable pending-deletion marker in a millisecond transaction (terminal DEACTIVATED status, immediate listing/serving exclusion, sticky/bridge cleanup, frozen delete_history choice) and a new leader-gated worker drains usage_history, additional_usage_history, and request_logs in 5k-row transactions without the fold-state lock, then finalizes residual rows, the folded-bucket lifecycle mirrors, and the sticky/rollup/account rows in one fold-state-locked transaction with the historical shape. Fold slices interleaving between chunks converge at finalization because every slice holds the fold-state row lock from raw read to commit: it lands either before the mirrors (moved/removed) or after (sees no attributed raw rows), so folded rows never resurrect. Deletion is restart-safe (all progress in the database), idempotent (repeat requests do not escalate the frozen variant), supports both delete_history variants, and is superseded by a credential replacement that clears the marker; the worker abandons superseded accounts before finalizing. Response contract ({"status": "deleted"}) and frontend are unchanged. Co-Authored-By: Claude Fable 5 * fix(accounts): harden background deletion supersede, exports, fairness, and queue index Address local codex review round 1: - Chunk transactions now re-read the pending marker under the account row lock (PostgreSQL FOR NO KEY UPDATE) so a credential replacement cannot clear the marker between the read and the chunk's row mutations; no chunk can commit row work after a replacement has successfully returned. - Credential-export endpoints (export, auth export, opencode auth export) treat marked accounts as not found: a successful DELETE no longer leaves decrypted tokens retrievable during the background drain window. - Deletion passes round-robin one chunk per pending account and re-scan the pending set between rounds, so one account's multi-minute drain cannot starve another marked account or a delete request landing mid-pass. - Partial index idx_accounts_delete_requested_at ((delete_requested_at, id) WHERE delete_requested_at IS NOT NULL) backs the per-interval pending probe and the queue-order scan; empty in the steady state. OpenSpec delta/design/tasks updated; new integration coverage for export 404s, round-robin interleave, and mid-pass pickup. Co-Authored-By: Claude Fable 5 * fix(accounts): wipe tokens, drop key assignments, fence status writes, tighten round-robin Address local codex review round 2: - begin_delete overwrites the stored access/refresh/id token ciphertext with empty-credential ciphertext: the row outlives the DELETE response by the drain duration, and readers that do not know the marker (pre-upgrade replicas' export endpoints during a rolling deploy) must not be able to produce usable credentials from it. Rotation is CAS-guarded on the pre-wipe refresh ciphertext; every supersede path writes fresh material. - begin_delete deletes the account's ApiKeyAccountAssignment rows (the projection the synchronous delete's FK cascade produced): GET /api/api-keys and pooled-usage reads exclude the account immediately, while the key's persisted assignment-scope flag keeps it scoped. Fixes test_deleted_assigned_accounts_do_not_fall_back_to_other_accounts. - update_status / update_status_if_current gain a delete_requested_at IS NULL fence so stale in-flight settlements (e.g. a late 429) cannot replace the terminal DEACTIVATED state and make a marked account selectable mid-drain; credential replacement bypasses these writers and still supersedes. - A drain round now yields after the first NONEMPTY chunk (not the first batch-size-full one), so an account with small tables cannot stack three row-touching transactions into one round. OpenSpec delta/design/tasks updated; new integration coverage for token wipe and the status-write fence. Co-Authored-By: Claude Fable 5 * fix(accounts): preserve seat identity before the pending-deletion token wipe Address local codex review round 3: targeted OAuth reauthentication (a promised supersede path) verifies the seat against chatgpt_user_id or, on legacy rows where it was never backfilled, the stored id-token claims. Wiping id_token_encrypted destroyed the only saved seat identity on such rows, so _save_oauth_account derived an empty intended-seat set and raised ReauthSeatMismatchError before replace_reauthorized could clear the marker — the account would finalize despite fresh credentials arriving. begin_delete now backfills chatgpt_user_id from the id-token claims (via resolve_seat_identity, non-secret identity only) in the same transaction, before overwriting the ciphertext; on PostgreSQL the row is held FOR NO KEY UPDATE across the derive-then-write. OpenSpec delta/design/tasks updated; regression test covers the legacy-row backfill through begin_delete. Co-Authored-By: Claude Fable 5 * fix(accounts): legacy-replica supersede via wipe sentinel, reject marked accounts in key assignment Address local codex review round 4: - A credential replacement handled by a pre-upgrade replica during a rolling deploy writes fresh ciphertext but cannot clear marker columns its ORM does not know, so the worker would have drained and finalized a freshly reauthorized account. Every marker re-check (chunk and finalization) now also inspects the refresh ciphertext: non-wiped (or undecryptable) material on a marked row is itself the supersede signal — the worker clears the marker under the account row lock and abandons the deletion instead. - ApiKeysRepository.list_accounts_by_ids rejects marked accounts, so an API-key create/update racing (or following) the DELETE cannot recreate an assignment that would re-surface the deleted account in key listings or pooled-usage projections before finalization. OpenSpec delta/design/tasks updated; regression coverage for mid-drain and pre-finalize legacy supersede and for post-DELETE key assignment rejection. Co-Authored-By: Claude Fable 5 * fix(accounts): serialize finalization against in-flight log inserts, atomic assignment marker re-check Address local codex review round 5: - Finalization upgrades the account row to FOR UPDATE (PostgreSQL) after the fold lock, before the residual sweeps. FOR UPDATE conflicts with the KEY SHARE a request-log FK insert takes, so an in-flight stream's log row either commits before the sweep (and is swept) or blocks until the transaction commits and then fails its FK against the deleted row. This closes the window where an insert landing between the sweep and the account-row delete was ON DELETE SET NULL'ed into a live orphan (soft) or survived outright (delete_history). Lock order (identity -> fold -> row exclusive) matches the historical transaction. - replace_account_assignments now inserts through a conditional INSERT..SELECT WHERE delete_requested_at IS NULL (FOR SHARE on PostgreSQL), re-checking the pending-deletion marker atomically with the write: a key create/update whose validation raced the DELETE either commits first (begin_delete's cleanup removes the assignment) or sees the marker and skips the account. OpenSpec delta/design/tasks updated; pg interleaving regression test for the in-flight insert and an atomic re-check test for assignments. Co-Authored-By: Claude Fable 5 * fix(accounts): widen wipe sentinel to all token fields, fix assignment lock order Address local codex review round 6: - credentials_replaced_since_wipe now inspects all three token ciphertexts (access/refresh/id): a legal legacy replacement may carry an empty refresh token while providing fresh access/id material, and a refresh-only check would mistake it for the original wipe and finalize a freshly replaced account. - replace_account_assignments acquires the account FOR SHARE locks BEFORE deleting the key's assignment rows, matching begin_delete's account-then-assignment lock order; the previous order (assignment-row delete first, account lock second) formed a cycle with a concurrent begin_delete and deadlocked on PostgreSQL instead of serializing. OpenSpec delta/design updated; regression test for the empty-refresh legacy replacement. Co-Authored-By: Claude Fable 5 * docs(accounts): make supersede-after-partial-drain folded end state an explicit contract Address local codex review round 7: the design claimed the mid-drain folded/raw divergence was 'bounded by drain duration', which is false when a supersede lands after a partial drain — finalization's mirrors never run and rows drained before the replacement keep folded attribution under the revived account permanently. That end state is historically correct (the folded numbers pre-existed the delete; nothing is added or inflated) and cannot double- or under-count a read: below-watermark reads are folded-only, drained below-watermark rows are never re-folded, and drained above-watermark rows fold exactly once under the orphaned dimension. Reconciling at supersede time was rejected — it would drag the fold lock and per-row delta mirroring into every credential-replacement path to 'fix' attribution that is already correct. Design D3 + Risks and the spec supersede requirement now state this contract explicitly, and a regression test pins it (folded counts unchanged by later folds after a partial-drain supersede; raw rows stay detached). Co-Authored-By: Claude Fable 5 * fix(accounts): self-heal unfenced-replica drift per chunk, fast repeat-delete short-circuit, deterministic tests Address local codex review round 8: - Every drain chunk transaction now self-heals drift written by pre-upgrade replicas during a rolling deploy (their writers carry no marker fence): when the marked row's status was replaced (e.g. by a late 429 settlement) or an API-key assignment was recreated, and the token ciphertext is still wiped (i.e. no credential replacement), the chunk re-asserts DEACTIVATED/pending_deletion and re-removes the assignments under the row lock it already holds — bounding any mixed-version drift to one chunk transaction. A DB trigger was rejected as disproportionate. - Repeat DELETE requests short-circuit on an unlocked marker+wipe read before the writer section / row lock, keeping the millisecond fast-path contract while a drain chunk holds the account row for seconds. The short-circuit falls through to the full (re-wipe, re-arm) path when credentials were replaced without clearing the marker. - Deletion tests neutralize the scheduler's startup/interval tick (the async_client lifespan starts the real worker with an inline leader election), removing the race between a tick and mark-state assertions. OpenSpec delta/design/tasks updated; regression tests for the per-chunk self-heal and the non-blocking repeat delete (pg row-lock interleaving). Co-Authored-By: Claude Fable 5 * fix(accounts): 404 every ID-based route for marked accounts, filter unscoped API-key pool Address local codex review round 9: - Every ID-based account route now treats a marked account as absent, as the synchronous delete did once the row was gone: reads (trends, reset-credit views) and action routes (pause, probe, reset-credit consume) go through a marker-aware fetch; mutations (account update, alias, limit-warmup, routing policy) gain an atomic delete_requested_at IS NULL write predicate. Only credential-replacement paths may address the marked row. - ApiKeysRepository.list_all_accounts (unscoped pooled-usage projections and /v1/usage) filters the marker as well: status alone is not enough while unfenced pre-upgrade replicas can briefly replace the terminal status during a rolling deploy. OpenSpec delta/tasks updated; route sweep added to the immediate-mark integration test. Co-Authored-By: Claude Fable 5 * fix(accounts): fence remaining account-ID surfaces, propagate drift-repair invalidation, add migration round-trip test Address local codex review round 10: - Remaining ID-based account surfaces now treat marked rows as absent: dashboard rate-limit reset-credit read/consume routes 404, the settings upstream-proxy binding route reports not-found instead of mutating a deleted account's binding, and /v1 reset-credit redemption treats the marked account exactly like one outside the API-key pool (its credentials are wiped anyway). - After a drain chunk repairs pre-upgrade-replica drift (status resurrection / recreated assignment), the worker now propagates the same cache invalidation as the delete request itself, so replicas that cached the drift stop selecting the wiped account or honoring the stale assignment. - Alembic round-trip coverage for 20260816_000000_add_account_pending_deletion: parent -> revision -> downgrade -> guarded upgrade (pre-existing column) -> head, asserting the marker columns and the partial queue index at each step; wired into the PostgreSQL CI target list. OpenSpec delta/tasks updated. Co-Authored-By: Claude Fable 5 * docs(accounts): scope the first-request-wins variant invariant to upgraded replicas Address local codex review round 11: the spec stated first-request-wins unqualified, contradicting the design's own rolling-upgrade rule that a delete handled by a pre-upgrade replica is simply the legacy synchronous delete — whose caller-provided delete_history variant can differ from the frozen first-request choice during the mixed deploy window. The invariant is now explicitly scoped to replicas running this revision, with the mixed-window caveat and the rejection rationale documented: new code cannot retrofit a fence into binaries that predate the marker columns, a DB trigger is disproportionate for a window bounded by the deploy, and a feature gate would add permanent configuration for a transient condition (the production single-replica topology has no mixed window at all). The legacy delete remains a complete, fold-locked, mirror-correct deletion — only the history-policy choice can diverge, and only under contradictory operator repeats inside the window. Co-Authored-By: Claude Fable 5 * fix(db): refuse pending-deletion migration downgrade while deletions are queued Address local codex review round 12: the marker columns are the deletion queue's only durable state — downgrading while a background deletion is pending would silently abandon an acknowledged deletion and hand the parent build unusable (credential-wiped, partially drained) account rows it lists again. The downgrade now raises with the pending count and instructions (let the worker finish, or supersede via credential re-import) instead of dropping the columns; the round-trip migration test covers the refusal and the subsequent clean downgrade. Co-Authored-By: Claude Fable 5 * perf(accounts): pin deletion chunk scans to account-leading indexes, bound chunk transactions The chunk batch subquery (account_id = :id LIMIT n, no ORDER BY) planned as a LIMIT-terminated Seq Scan on the production planner for exactly the large accounts the background drain targets (verified: 606,970-row account -> Seq Scan on all three drain tables): equality folds account_id into a constant, so no pathkey forces the account index and the planner bets on uniformly interleaved matches. That bet loses mid-drain (detached rows form a growing dead prefix the scan must skip) and catastrophically once a table is drained but statistics are stale — every empty probe became a full heap scan, and usage tables were re-probed EVERY round. - Select each batch with an account_id >= :id AND account_id <= :id range (same rows, but account_id survives as the leading sort pathkey) ordered by the target index's exact column order, making the account-leading index the only sort-free plan: idx_usage_account_time, ix_additional_usage_distinct_labels, and the covering idx_logs_account_kind_deleted_latest (index-only scan). Verified against the production planner: all three chunks and the drained-table probes now run as index (or index-only) scans with the account range as Index Cond. - Stop re-probing tables already observed empty within the same pass (rows settling mid-drain are converged by finalization's residual sweep); each probe was a full account-row-locking transaction per table per round. - DELETE_BATCH_SIZE 5k -> 1k: each chunk holds the account row FOR NO KEY UPDATE for its full duration, and the design's own measured detach rate (~23s/10k request_logs) put 5k chunks at ~11.5s, not "a few seconds"; 1k bounds the worst table at ~2.3s so supersedes and fenced settlements wait at most that long. - Pause between row-touching rounds proportionally to round duration (capped) so a multi-hundred-chunk drain leaves the 2-vCPU database headroom instead of running chunk transactions back-to-back. Regression coverage: a structural test pins the range predicate + index-order ORDER BY of every batch builder against the model indexes, a PostgreSQL plan test asserts the batch shape is served by the pinned indexes without a sort or heap scan (including the drained-probe case), and a pass-level test asserts drained tables are probed exactly once per pass. Co-Authored-By: Claude Fable 5 * test: fix ty diagnostics in batch pinning helpers Precise Callable/Table annotations plus index.columns (typed) instead of index.expressions (str union) so the type gate passes. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- Makefile | 2 + ...816_000000_add_account_pending_deletion.py | 81 ++ app/db/models.py | 25 + app/main.py | 4 + app/modules/accounts/deletion.py | 539 +++++++++ app/modules/accounts/repository.py | 282 ++++- app/modules/accounts/service.py | 47 +- app/modules/api_keys/repository.py | 40 +- app/modules/proxy/api.py | 9 +- app/modules/rate_limit_reset_credits/api.py | 7 +- app/modules/settings/api.py | 5 +- .../background-account-deletion/design.md | 334 ++++++ .../background-account-deletion/proposal.md | 56 + .../specs/account-deletion/spec.md | 321 +++++ .../background-account-deletion/tasks.md | 105 ++ .../test_account_deletion_background.py | 1033 +++++++++++++++++ .../integration/test_accounts_api_extended.py | 33 +- tests/integration/test_migrations.py | 83 ++ .../unit/test_accounts_service_transitions.py | 1 + 19 files changed, 2979 insertions(+), 28 deletions(-) create mode 100644 app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py create mode 100644 app/modules/accounts/deletion.py create mode 100644 openspec/changes/background-account-deletion/design.md create mode 100644 openspec/changes/background-account-deletion/proposal.md create mode 100644 openspec/changes/background-account-deletion/specs/account-deletion/spec.md create mode 100644 openspec/changes/background-account-deletion/tasks.md create mode 100644 tests/integration/test_account_deletion_background.py diff --git a/Makefile b/Makefile index 1bb8dd2901..6e11247a63 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,7 @@ POSTGRES_PYTEST_TARGETS := \ tests/integration/test_db_commit_durability.py \ tests/test_request_logs_options_api.py \ tests/integration/test_account_usage_rollup.py \ + tests/integration/test_account_deletion_background.py \ tests/integration/test_request_usage_time_rollup.py \ tests/integration/test_request_usage_rollup_parity.py \ tests/integration/test_migrations.py::test_request_usage_time_rollups_migration_upgrade_and_downgrade \ @@ -33,6 +34,7 @@ POSTGRES_PYTEST_TARGETS := \ tests/integration/test_repositories.py::test_replace_reauthorized_discards_pending_downgrade_evidence \ tests/integration/test_repositories.py::test_upsert_account_slot_discards_pending_downgrade_evidence_on_reimport \ tests/integration/test_migrations.py::test_account_plan_downgrade_observations_migration_upgrade_and_downgrade \ + tests/integration/test_migrations.py::test_account_pending_deletion_migration_upgrade_and_downgrade \ tests/integration/test_usage_repository.py::test_bulk_history_since_primary_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_cutoff_query_plan_is_index_only_postgresql \ tests/integration/test_usage_repository.py::test_bulk_history_since_secondary_query_plan_is_index_only_postgresql \ diff --git a/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py b/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py new file mode 100644 index 0000000000..8b9938181c --- /dev/null +++ b/app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py @@ -0,0 +1,81 @@ +"""add account pending-deletion marker columns + +Revision ID: 20260816_000000_add_account_pending_deletion +Revises: 20260812_120000_add_sticky_abandonment_scope +Create Date: 2026-08-16 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op + +revision = "20260816_000000_add_account_pending_deletion" +down_revision = "20260812_120000_add_sticky_abandonment_scope" +branch_labels = None +depends_on = None + +_TABLE = "accounts" +_INDEX = "idx_accounts_delete_requested_at" + + +def _columns(bind) -> set[str]: + return {column["name"] for column in sa.inspect(bind).get_columns(_TABLE)} + + +def _indexes(bind) -> set[str]: + return {index["name"] for index in sa.inspect(bind).get_indexes(_TABLE)} + + +def upgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if "delete_requested_at" not in columns: + op.add_column(_TABLE, sa.Column("delete_requested_at", sa.DateTime(), nullable=True)) + if "delete_history_requested" not in columns: + op.add_column( + _TABLE, + sa.Column( + "delete_history_requested", + sa.Boolean(), + nullable=False, + server_default=sa.false(), + ), + ) + if _INDEX not in _indexes(bind): + # Pending-deletion queue probe/order support; partial so it is empty + # (and free) in the steady state with no pending deletions. + op.create_index( + _INDEX, + _TABLE, + ["delete_requested_at", "id"], + postgresql_where=sa.text("delete_requested_at IS NOT NULL"), + sqlite_where=sa.text("delete_requested_at IS NOT NULL"), + ) + + +def downgrade() -> None: + bind = op.get_bind() + columns = _columns(bind) + if "delete_requested_at" in columns: + # The marker columns are the deletion queue's only durable state: + # dropping them while deletions are queued would silently abandon + # acknowledged deletions and hand the parent build unusable + # (credential-wiped, partially drained) account rows it would list + # again. Refuse instead — let the worker finish (or supersede the + # deletions via re-import/reauth) before downgrading. + pending = bind.execute( + sa.text(f"SELECT COUNT(*) FROM {_TABLE} WHERE delete_requested_at IS NOT NULL") # noqa: S608 + ).scalar() + if pending: + raise RuntimeError( + f"cannot downgrade {revision}: {pending} account(s) are still queued for " + "background deletion; wait for the deletion worker to finish (or supersede " + "the deletions with a credential re-import) before downgrading" + ) + if _INDEX in _indexes(bind): + op.drop_index(_INDEX, table_name=_TABLE) + if "delete_history_requested" in columns: + op.drop_column(_TABLE, "delete_history_requested") + if "delete_requested_at" in columns: + op.drop_column(_TABLE, "delete_requested_at") diff --git a/app/db/models.py b/app/db/models.py index d16ab1a982..2197aaa048 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -137,6 +137,20 @@ class Account(Base): server_default=false(), nullable=False, ) + # Pending-deletion marker: set by the fast DELETE path, consumed by the + # background deletion worker, cleared only by a credential replacement + # (re-import/reauth) that supersedes the deletion. Non-NULL rows are + # hidden from account listings and are already unroutable (the fast path + # also sets status=DEACTIVATED). + delete_requested_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + # Frozen at the first delete request (repeat requests do not escalate): + # True selects the history-deleting variant in the background worker. + delete_history_requested: Mapped[bool] = mapped_column( + Boolean, + default=False, + server_default=false(), + nullable=False, + ) api_key_assignments: Mapped[list["ApiKeyAccountAssignment"]] = relationship( "ApiKeyAccountAssignment", @@ -2121,6 +2135,17 @@ class HttpBridgeRetryCircuit(Base): postgresql_include=["used_percent", "reset_at", "window_minutes", "id"], ) Index("idx_accounts_email", Account.email) +# Pending-deletion queue: every replica probes ``delete_requested_at IS NOT +# NULL LIMIT 1`` each worker interval and the leader orders the queue by +# (delete_requested_at, id); the partial index keeps both reads off the full +# accounts table and is empty in the steady state (no pending deletions). +Index( + "idx_accounts_delete_requested_at", + Account.delete_requested_at, + Account.id, + postgresql_where=text("delete_requested_at IS NOT NULL"), + sqlite_where=text("delete_requested_at IS NOT NULL"), +) Index("idx_api_keys_name", ApiKey.name) Index("idx_logs_account_time", RequestLog.account_id, RequestLog.requested_at) Index("idx_logs_model_source_time", RequestLog.model_source_id, RequestLog.requested_at) diff --git a/app/main.py b/app/main.py index 1b73c6c78b..6b44c8290a 100644 --- a/app/main.py +++ b/app/main.py @@ -63,6 +63,7 @@ from app.core.utils.time import utcnow from app.db.session import SessionLocal, close_db, close_session, init_background_db, init_db from app.modules.accounts import api as accounts_api +from app.modules.accounts.deletion import build_account_deletion_scheduler from app.modules.accounts.repository import AccountsRepository from app.modules.accounts.usage_rollup_scheduler import build_account_usage_rollup_scheduler from app.modules.api_keys import api as api_keys_api @@ -488,6 +489,7 @@ async def lifespan(app: FastAPI): automations_scheduler = build_automations_scheduler() rate_limit_reset_credits_scheduler = build_rate_limit_reset_credits_scheduler() account_usage_rollup_scheduler = build_account_usage_rollup_scheduler() + account_deletion_scheduler = build_account_deletion_scheduler() data_retention_scheduler = build_data_retention_scheduler() telemetry_scheduler = build_telemetry_scheduler() start_live_usage_ingestor() @@ -501,6 +503,7 @@ async def lifespan(app: FastAPI): await automations_scheduler.start() await rate_limit_reset_credits_scheduler.start() await account_usage_rollup_scheduler.start() + await account_deletion_scheduler.start() await data_retention_scheduler.start() await telemetry_scheduler.start() if settings.metrics_enabled and PROMETHEUS_AVAILABLE: @@ -718,6 +721,7 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N await stop_live_usage_ingestor() await rate_limit_reset_credits_scheduler.stop() await account_usage_rollup_scheduler.stop() + await account_deletion_scheduler.stop() await data_retention_scheduler.stop() await telemetry_scheduler.stop() # Release the scheduler leader lease only after every leader-gated diff --git a/app/modules/accounts/deletion.py b/app/modules/accounts/deletion.py new file mode 100644 index 0000000000..70bd766b46 --- /dev/null +++ b/app/modules/accounts/deletion.py @@ -0,0 +1,539 @@ +"""Background account deletion: fast-marked accounts drained in bounded chunks. + +``DELETE /api/accounts/{id}`` used to detach (or delete) the account's entire +raw history in ONE transaction while holding the fold-state lock: for a +long-lived account that is hundreds of thousands of rows across ``request_logs`` +(18 indexes) and ``usage_history``, minutes of fold blockage, one pinned pool +connection, and an HTTP client timeout. The API now only stamps the +pending-deletion marker (``AccountsRepository.begin_delete``); this module +drains the bulk rows afterwards, ``DELETE_BATCH_SIZE`` rows per transaction, +and finalizes with the exact transaction shape the synchronous path used. + +Fold-safety argument — why the chunk transactions do NOT take the fold-state +lock, yet a deleted account's folded rows can never resurrect: + +1. Chunk transactions touch only raw rows (``usage_history``, + ``additional_usage_history``, ``request_logs``); they never write a rollup + table and never move a watermark. ``usage_history`` tables are not + fold-governed at all. +2. A fold slice that interleaves between chunks may aggregate rows still + attributed to the account (adding folded rows under the account dimension) + or rows a chunk already detached (adding them under the orphaned-deleted + dimension — exactly the soft-path end state). Both are converged by the + finalization transaction: it takes ``lock_fold_state()`` and only then + detaches/deletes the residual raw rows and runs the lifecycle mirrors, + which move or remove EVERY folded row carrying the account dimension, + including rows folded mid-drain. +3. Every fold slice holds the fold-state row lock from before it reads raw + rows until its commit. A slice therefore commits either before + finalization (its account-attributed output exists when the mirrors run + and is moved/removed by them) or after (it observes the post-finalization + raw state, which carries no attribution to the account). No slice can + commit pre-deletion attribution after the mirrors ran — the exact + resurrection the single-transaction path guarded against. + +Restart safety and idempotency: all progress lives in the database (the +marker columns plus the shrinking predicate ``WHERE account_id = :id``), so a +leader restart resumes mid-drain, and re-running any chunk is a no-op. +A credential replacement (re-import/reauth) clears the marker and supersedes +the deletion: every chunk transaction re-reads the marker under the account +row lock (PostgreSQL ``FOR NO KEY UPDATE``; the SQLite writer section +serializes writers) before touching rows, and finalization re-checks it the +same way, so no chunk can commit row work after a replacement committed and +a superseded account is never finalized. + +Fairness: a deletion pass round-robins one chunk per pending account and +re-scans for newly marked accounts between rounds, so one account's +multi-minute drain can neither starve another marked account nor delay a +delete request that arrives mid-pass. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import importlib +import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from typing import Protocol, TypeVar, cast + +from sqlalchemy import Select, delete, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from app.core.auth.api_key_cache import get_api_key_cache +from app.core.cache.invalidation import NAMESPACE_API_KEY, get_cache_invalidation_poller +from app.core.upstream_proxy.cache import get_upstream_route_cache +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + AccountStatus, + AdditionalUsageHistory, + ApiKeyAccountAssignment, + RequestLog, + UsageHistory, +) +from app.db.session import get_background_session, sqlite_writer_section +from app.modules.accounts.repository import ( + ACCOUNT_PENDING_DELETION_REASON, + AccountsRepository, + credentials_replaced_since_wipe, +) +from app.modules.proxy.account_cache import ( + get_account_selection_cache, + mark_account_routing_unavailable, + propagate_account_routing_change, +) +from app.modules.usage.repository import _clear_bulk_history_since_sqlite_cache + +logger = logging.getLogger(__name__) + +# Worker tick; the fast delete path additionally wakes the local worker, so a +# single-replica deployment (or a delete that lands on the leader) starts +# draining immediately and the tick only covers follower-received requests +# and restart resume. +DELETION_INTERVAL_SECONDS = 30 +# Rows per chunk transaction. Every chunk holds the account row lock +# (``FOR NO KEY UPDATE``) for its full duration, so a supersede +# (``replace_reauthorized``) or a fenced settlement write can wait for at most +# one chunk. Measured production rates: ~1.2s/10k usage_history deletes and +# ~23s/10k request_logs detaches (18 indexes, non-HOT updates) — 1k bounds the +# worst table at ~2.3s per transaction (5k would have been ~11.5s there). +DELETE_BATCH_SIZE = 1_000 +# Between consecutive row-touching rounds the pass sleeps a fraction of the +# round's own duration (capped), so a multi-hundred-chunk drain leaves the +# 2-vCPU database headroom for foreground traffic instead of running chunk +# transactions back-to-back. +INTER_ROUND_PAUSE_RATIO = 0.25 +INTER_ROUND_PAUSE_CAP_SECONDS = 2.0 + +_T = TypeVar("_T") + + +class _LeaderElectionLike(Protocol): + async def run_if_leader(self, fn: Callable[[], Awaitable[_T]]) -> _T | None: ... + + +def _get_leader_election() -> _LeaderElectionLike: + module = importlib.import_module("app.core.scheduling.leader_election") + return cast(_LeaderElectionLike, module.get_leader_election()) + + +async def run_account_deletion_pass(*, batch_size: int = DELETE_BATCH_SIZE) -> dict[str, str]: + """Drain and finalize every account marked for deletion. + + Round-robin fairness: each round advances every pending account by at + most one nonempty chunk, and the pending set is re-scanned between + rounds, so a newly marked account starts draining within one chunk + transaction of its request even while another account's long drain is in + progress. + + Returns an outcome per account id: ``finalized`` (rows drained, account + row removed), ``superseded`` (marker cleared mid-drain by a credential + replacement — deletion abandoned), or ``error`` (logged; retried on the + next tick). + """ + outcomes: dict[str, str] = {} + # Tables observed empty for an account earlier in THIS pass are not + # re-probed on later rounds: a drained table stays drained for the rest of + # the drain (rows that land afterwards — e.g. a stream settling a log row + # mid-drain — are swept by finalization's residual pass), and re-probing + # would cost one account-row-locking transaction per table per round. + drained: dict[str, set[str]] = {} + loop = asyncio.get_running_loop() + while True: + runnable = [ + account_id for account_id in await _pending_deletion_ids() if outcomes.get(account_id) in (None, "draining") + ] + if not runnable: + break + round_started = loop.time() + for account_id in runnable: + try: + outcomes[account_id] = await _advance_account( + account_id, batch_size=batch_size, drained=drained.setdefault(account_id, set()) + ) + except Exception: + logger.exception("Background account deletion failed account_id=%s", account_id) + outcomes[account_id] = "error" + if any(outcomes.get(account_id) == "draining" for account_id in runnable): + elapsed = loop.time() - round_started + await asyncio.sleep(min(INTER_ROUND_PAUSE_CAP_SECONDS, elapsed * INTER_ROUND_PAUSE_RATIO)) + # ``draining`` cannot survive the loop: an account leaves the runnable + # set only through a terminal outcome or by vanishing from the pending + # scan (its marker was cleared — a supersede that raced the scan). + for account_id, outcome in outcomes.items(): + if outcome == "draining": + outcomes[account_id] = "superseded" + if outcomes: + logger.info("Account deletion pass outcomes=%s", outcomes) + return outcomes + + +async def _pending_deletion_ids() -> list[str]: + async with get_background_session() as session: + rows = await session.execute( + select(Account.id) + .where(Account.delete_requested_at.is_not(None)) + .order_by(Account.delete_requested_at.asc(), Account.id.asc()) + ) + return list(rows.scalars().all()) + + +async def _advance_account(account_id: str, *, batch_size: int, drained: set[str] | None = None) -> str: + """One bounded round of work for one account. + + Runs the drain tables in order (usage snapshots first — not + fold-governed — then the raw request logs) but stops after the first + NONEMPTY chunk so the caller can round-robin other pending accounts — + each round commits at most one row-touching transaction per account: + ``draining`` means more work may remain. Only tables whose chunk came up + empty are known drained; when every table is, finalize. ``drained`` + (caller-owned, per pass) records those tables so later rounds skip their + probes instead of re-running one locking transaction per table per round. + """ + if drained is None: + drained = set() + for label, chunk_fn in ( + ("usage_history", _usage_history_chunk), + ("additional_usage_history", _additional_usage_history_chunk), + ("request_logs", _request_logs_chunk), + ): + if label in drained: + continue + affected = await _run_chunk(chunk_fn, account_id, batch_size=batch_size) + if affected is None: + return "superseded" + if not affected: + drained.add(label) + continue + if label == "usage_history": + # Same hygiene as retention pruning: bulk usage-history reads are + # cached on SQLite and must not serve the drained account. + _clear_bulk_history_since_sqlite_cache() + return "draining" + # Finalization: residual rows (streams that settled a log row mid-drain), + # folded-bucket mirrors, sticky/rollup rows, and the account row itself — + # one fold-state-locked transaction, identical in shape to the historical + # synchronous delete but over a residual row set instead of full history. + async with get_background_session() as session: + finalized = await AccountsRepository(session).delete(account_id, only_pending=True) + if not finalized: + return "superseded" + # Invalidate immediately (not at end of pass): the account row is gone + # and ids are deterministic, so cached routing/API-key snapshots must not + # outlive it while the pass keeps draining other accounts. + await _invalidate_account_caches() + return "finalized" + + +_ChunkFn = Callable[..., Awaitable[int]] + + +async def _run_chunk(chunk_fn: _ChunkFn, account_id: str, *, batch_size: int) -> int | None: + """Run one chunk transaction; None when the pending marker disappeared + (deletion superseded).""" + drift_repaired = False + async with get_background_session() as session: + async with sqlite_writer_section(): + state = await _pending_state(session, account_id) + if state is None: + await session.rollback() + return None + delete_history, drift_repaired = state + affected = await chunk_fn(session, account_id, delete_history=delete_history, batch_size=batch_size) + await session.commit() + if drift_repaired: + # The drift a pre-upgrade replica wrote may already be cached in + # selection/API-key snapshots on this or peer replicas; repairing the + # database alone would leave those caches selecting the wiped account + # (or honoring a stale assignment) until expiry. Same invalidation + # fan-out as the delete request itself. + mark_account_routing_unavailable(account_id) + await _invalidate_account_caches() + return affected + + +async def _pending_state(session: AsyncSession, account_id: str) -> tuple[bool, bool] | None: + """``(delete_history, drift_repaired)``, or None when no longer pending. + + On PostgreSQL the read locks the account row (``FOR NO KEY UPDATE``) for + the rest of the chunk transaction, so a credential replacement cannot + clear the marker between this read and the chunk's row mutations — the + replacement blocks until the chunk commits, then the next chunk observes + the cleared marker and stops. On SQLite the writer section already + serializes this transaction against every other writer. + + A replacement handled by a pre-upgrade replica (rolling deploy) writes + fresh credentials but cannot clear marker columns its ORM does not know; + fresh non-wiped ciphertext on a marked row is therefore itself the + supersede signal — the marker is cleared here, under the same lock. + """ + stmt = select( + Account.delete_requested_at, + Account.delete_history_requested, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + Account.status, + Account.deactivation_reason, + ).where(Account.id == account_id) + if session.get_bind().dialect.name == "postgresql": + stmt = stmt.with_for_update(key_share=True) + row = (await session.execute(stmt)).first() + if row is None or row[0] is None: + return None + if credentials_replaced_since_wipe(row[2], row[3], row[4]): + await session.execute( + update(Account) + .where(Account.id == account_id) + .values(delete_requested_at=None, delete_history_requested=False) + ) + await session.commit() + return None + # Self-heal drift written by pre-upgrade replicas during a rolling + # deploy (their writers are unfenced): a late settlement may have + # replaced the terminal status — making the wiped account selectable + # again — and an unconditional assignment insert may have recreated an + # API-key assignment begin_delete removed. Re-fence both under the row + # lock held above; any drift is bounded by one chunk transaction. The + # credentials check above already excluded genuine replacements, so a + # non-DEACTIVATED status here can only be such drift. The caller + # propagates cache invalidation after commit when drift was repaired. + drift_repaired = False + if row[5] is not AccountStatus.DEACTIVATED or row[6] != ACCOUNT_PENDING_DELETION_REASON: + drift_repaired = True + await session.execute( + update(Account) + .where(Account.id == account_id) + .values( + status=AccountStatus.DEACTIVATED, + deactivation_reason=ACCOUNT_PENDING_DELETION_REASON, + reset_at=None, + blocked_at=None, + ) + ) + assignment_rows = await session.execute( + delete(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.account_id == account_id) + .returning(ApiKeyAccountAssignment.account_id) + ) + if assignment_rows.scalars().first() is not None: + drift_repaired = True + return bool(row[1]), drift_repaired + + +# Chunk batch shape — why a RANGE predicate plus an index-matching ORDER BY +# instead of plain ``account_id = :id LIMIT n``: +# +# With an equality predicate the PostgreSQL planner folds ``account_id`` into +# a constant, drops it from the sort pathkeys, and — whenever the per-account +# row estimate is large (exactly the accounts this drain exists for) — plans +# the LIMIT subquery as an early-terminating Seq Scan (or a scan of an +# unrelated time index with a filter), betting on uniformly interleaved +# matches. That bet loses precisely mid-drain: detached/deleted rows no +# longer match, so each chunk re-scans a growing dead prefix, and once the +# table is drained but statistics are stale, every empty probe is a FULL heap +# scan (0 matches → no early termination). Verified against the production +# planner (606,970-row account): Seq Scans on all three tables. +# +# ``account_id >= :id AND account_id <= :id`` selects the same rows but keeps +# ``account_id`` out of the constant-equivalence class, so it survives as the +# leading ORDER BY pathkey; the ORDER BY then lists the target index's exact +# column order, making that account-leading index the only sort-free plan: +# +# usage_history → idx_usage_account_time (account_id, recorded_at) +# additional_usage_history → ix_additional_usage_distinct_labels +# (account_id, quota_key, limit_name, metered_feature) +# request_logs → idx_logs_account_kind_deleted_latest +# (account_id, request_kind, deleted_at, +# requested_at, id) — covering: Index Only Scan +# +# Verified on the production planner: all three plan as index (or index-only) +# scans with the account range as the Index Cond, and a drained-table probe +# costs one index descent (~8 cost units) instead of a heap scan. Chunk scan +# work is therefore bounded by the account's own remaining rows regardless of +# statistics staleness, and detached rows leave the scanned key range +# immediately. Row order is irrelevant to correctness (every row is drained); +# the ORDER BY exists purely to pin the plan. + + +def _usage_history_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(UsageHistory.id) + .where(UsageHistory.account_id >= account_id, UsageHistory.account_id <= account_id) + .order_by(UsageHistory.account_id, UsageHistory.recorded_at) + .limit(batch_size) + ) + + +def _additional_usage_history_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(AdditionalUsageHistory.id) + .where( + AdditionalUsageHistory.account_id >= account_id, + AdditionalUsageHistory.account_id <= account_id, + ) + .order_by( + AdditionalUsageHistory.account_id, + AdditionalUsageHistory.quota_key, + AdditionalUsageHistory.limit_name, + AdditionalUsageHistory.metered_feature, + ) + .limit(batch_size) + ) + + +def _request_logs_batch(account_id: str, batch_size: int) -> Select[tuple[int]]: + return ( + select(RequestLog.id) + .where(RequestLog.account_id >= account_id, RequestLog.account_id <= account_id) + .order_by( + RequestLog.account_id, + RequestLog.request_kind, + RequestLog.deleted_at, + RequestLog.requested_at, + RequestLog.id, + ) + .limit(batch_size) + ) + + +async def _usage_history_chunk(session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int) -> int: + batch = _usage_history_batch(account_id, batch_size).scalar_subquery() + result = await session.execute(delete(UsageHistory).where(UsageHistory.id.in_(batch)).returning(UsageHistory.id)) + return len(result.scalars().all()) + + +async def _additional_usage_history_chunk( + session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int +) -> int: + batch = _additional_usage_history_batch(account_id, batch_size).scalar_subquery() + result = await session.execute( + delete(AdditionalUsageHistory).where(AdditionalUsageHistory.id.in_(batch)).returning(AdditionalUsageHistory.id) + ) + return len(result.scalars().all()) + + +async def _request_logs_chunk(session: AsyncSession, account_id: str, *, delete_history: bool, batch_size: int) -> int: + """Detach (soft) or delete (hard) one chunk of the account's raw logs. + + Deliberately NOT fold-state-locked and NOT mirrored: see the module + docstring for why interleaved fold slices converge at finalization. + """ + batch = _request_logs_batch(account_id, batch_size).scalar_subquery() + if delete_history: + result = await session.execute(delete(RequestLog).where(RequestLog.id.in_(batch)).returning(RequestLog.id)) + else: + result = await session.execute( + update(RequestLog) + .where(RequestLog.id.in_(batch)) + .values(account_id=None, deleted_at=utcnow()) + .returning(RequestLog.id) + ) + return len(result.scalars().all()) + + +async def _invalidate_account_caches() -> None: + """Post-finalization invalidation, mirroring the synchronous delete path. + + Account ids are deterministic (delete-then-re-import regenerates the same + id), so cached route outcomes and API-key assignment snapshots must not + survive the account row's removal. + """ + get_account_selection_cache().invalidate() + get_api_key_cache().clear() + await get_upstream_route_cache().invalidate() + await propagate_account_routing_change() + poller = get_cache_invalidation_poller() + if poller is not None: + await poller.bump(NAMESPACE_API_KEY) + + +@dataclass(slots=True) +class AccountDeletionScheduler: + """Leader-gated worker tick with a local wake signal. + + Each tick first checks — with one cheap indexed-table read, before any + leader-election work — whether any account is pending deletion, so the + steady state (no pending deletions) costs one SELECT per interval. + """ + + interval_seconds: int + _task: asyncio.Task[None] | None = None + _stop: asyncio.Event = field(default_factory=asyncio.Event) + _wake: asyncio.Event = field(default_factory=asyncio.Event) + _lock: asyncio.Lock = field(default_factory=asyncio.Lock) + + async def start(self) -> None: + if self._task and not self._task.done(): + return + self._stop.clear() + self._task = asyncio.create_task(self._run_loop()) + + async def stop(self) -> None: + if not self._task: + return + self._stop.set() + self._task.cancel() + with contextlib.suppress(asyncio.CancelledError): + await self._task + self._task = None + + def wake(self) -> None: + """Start the next pass immediately (fast delete path just committed).""" + self._wake.set() + + async def _run_loop(self) -> None: + while not self._stop.is_set(): + # Clear BEFORE running so a wake that lands mid-pass (a second + # delete request) schedules another pass instead of being lost. + self._wake.clear() + await self._run_once() + try: + await asyncio.wait_for(self._wake.wait(), timeout=self.interval_seconds) + except asyncio.TimeoutError: + continue + + async def _run_once(self) -> None: + try: + if not await _any_pending_deletion(): + return + except Exception: + logger.exception("Failed to check for pending account deletions") + return + await _get_leader_election().run_if_leader(self._run_as_leader) + + async def _run_as_leader(self) -> None: + async with self._lock: + try: + await run_account_deletion_pass() + except Exception: + logger.exception("Account deletion pass failed") + + +async def _any_pending_deletion() -> bool: + async with get_background_session() as session: + row = await session.execute(select(Account.id).where(Account.delete_requested_at.is_not(None)).limit(1)) + return row.scalar_one_or_none() is not None + + +_scheduler: AccountDeletionScheduler | None = None + + +def build_account_deletion_scheduler() -> AccountDeletionScheduler: + global _scheduler + _scheduler = AccountDeletionScheduler(interval_seconds=DELETION_INTERVAL_SECONDS) + return _scheduler + + +def request_account_deletion_run() -> None: + """Nudge the local worker after a delete request commits. + + A follower's nudge is a no-op (``run_if_leader`` declines) and the + leader's periodic tick picks the request up within the interval; when the + receiving replica IS the leader — the common single-replica case — the + drain starts immediately. + """ + if _scheduler is not None: + _scheduler.wake() diff --git a/app/modules/accounts/repository.py b/app/modules/accounts/repository.py index 64dbff5140..58d1094461 100644 --- a/app/modules/accounts/repository.py +++ b/app/modules/accounts/repository.py @@ -7,12 +7,14 @@ from datetime import datetime from typing import Any -from sqlalchemy import delete, or_, select, text, update +from sqlalchemy import case, delete, func, or_, select, text, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.dialects.sqlite import insert as sqlite_insert from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession +from app.core.auth import extract_id_token_claims, resolve_seat_identity +from app.core.crypto import TokenEncryptor from app.core.upstream_proxy.cache import get_upstream_route_cache from app.core.utils.time import utcnow from app.db.account_identity_lock import advisory_lock_key, lock_postgresql_account_identities @@ -51,6 +53,42 @@ _SETTINGS_ROW_ID = 1 _DUPLICATE_ACCOUNT_SUFFIX = "__copy" +# deactivation_reason stamped by the fast DELETE path while the background +# worker drains the account's rows. The authoritative pending marker is +# accounts.delete_requested_at; the reason string is operator-facing only. +ACCOUNT_PENDING_DELETION_REASON = "pending_deletion" + + +def credentials_replaced_since_wipe( + access_token_encrypted: bytes, + refresh_token_encrypted: bytes, + id_token_encrypted: bytes, +) -> bool: + """True when a marked account's token ciphertext is no longer the + empty-credential wipe stamped by :meth:`AccountsRepository.begin_delete`. + + New-code credential replacements clear the pending-deletion marker in the + same transaction, but a replacement handled by a PRE-UPGRADE replica + during a rolling deploy writes fresh ciphertext without knowing the + marker columns. Fresh (non-wiped) credentials on a still-marked row are + therefore themselves the supersede signal; the caller must clear the + marker and abandon the deletion. ALL THREE token fields are inspected: a + legal replacement may carry an empty refresh token while providing fresh + access/id material, and mistaking it for the wipe would finalize a + freshly replaced account. Undecryptable material also counts as + replaced — never finalize a row whose credentials we cannot attribute to + our own wipe. + """ + encryptor = TokenEncryptor() + for ciphertext in (access_token_encrypted, refresh_token_encrypted, id_token_encrypted): + try: + if encryptor.decrypt(ciphertext) != "": + return True + except Exception: + return True + return False + + _UNSET = object() _HARD_STICKY_UNAVAILABLE_STATUSES = frozenset( (AccountStatus.PAUSED, AccountStatus.RATE_LIMITED, AccountStatus.QUOTA_EXCEEDED) @@ -160,7 +198,11 @@ async def get_by_id_fresh(self, account_id: str) -> Account | None: return result.scalar_one_or_none() async def list_accounts(self, *, refresh_existing: bool = False) -> list[Account]: - stmt = select(Account).order_by(Account.email) + # Accounts marked for background deletion are already deleted from the + # operator's point of view: they never appear in listings (dashboard, + # usage refresh, automations) even though their rows survive until the + # deletion worker finishes draining them. + stmt = select(Account).where(Account.delete_requested_at.is_(None)).order_by(Account.email) if refresh_existing: stmt = stmt.execution_options(populate_existing=True) result = await self._session.execute(stmt) @@ -169,7 +211,12 @@ async def list_accounts(self, *, refresh_existing: bool = False) -> list[Account async def list_accounts_by_ids(self, account_ids: list[str], *, refresh_existing: bool = False) -> list[Account]: if not account_ids: return [] - stmt = select(Account).where(Account.id.in_(account_ids)).order_by(Account.email) + stmt = ( + select(Account) + .where(Account.id.in_(account_ids)) + .where(Account.delete_requested_at.is_(None)) + .order_by(Account.email) + ) if refresh_existing: stmt = stmt.execution_options(populate_existing=True) result = await self._session.execute(stmt) @@ -678,7 +725,17 @@ async def update_status( if blocked_at is not _UNSET: values["blocked_at"] = blocked_at result = await self._session.execute( - update(Account).where(Account.id == account_id).values(**values).returning(Account.id) + update(Account) + .where(Account.id == account_id) + # An account marked for background deletion is terminal: a + # stale in-flight settlement (e.g. a 429 from a request that + # was selected before the DELETE) must not replace the + # DEACTIVATED/pending_deletion state and make the account + # selectable again mid-drain. Only a credential replacement + # (which clears the marker) may resurrect the row. + .where(Account.delete_requested_at.is_(None)) + .values(**values) + .returning(Account.id) ) updated_id = result.scalar_one_or_none() if updated_id is not None and self._hard_sticky_outage_started(previous_status, status): @@ -694,6 +751,10 @@ async def update_security_work_authorized(self, account_id: str, enabled: bool) result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(security_work_authorized=enabled) .returning(Account.id) ) @@ -726,6 +787,9 @@ async def update_status_if_current( update(Account) .where(Account.id == account_id) .where(Account.status == expected_status) + # Same pending-deletion fence as ``update_status``: marked + # rows are terminal for ordinary status writers. + .where(Account.delete_requested_at.is_(None)) .values(**values) .returning(Account.id) ) @@ -860,7 +924,14 @@ async def _close_http_bridge_sessions_for_account(self, account_id: str) -> None async def update_alias(self, account_id: str, alias: str | None) -> bool: async with sqlite_writer_section(): result = await self._session.execute( - update(Account).where(Account.id == account_id).values(alias=alias).returning(Account.id) + update(Account) + .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) + .values(alias=alias) + .returning(Account.id) ) await self._session.commit() return result.scalar_one_or_none() is not None @@ -870,6 +941,10 @@ async def update_limit_warmup_enabled(self, account_id: str, enabled: bool) -> b result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(limit_warmup_enabled=enabled) .returning(Account.id) ) @@ -881,24 +956,211 @@ async def update_routing_policy(self, account_id: str, routing_policy: str) -> b result = await self._session.execute( update(Account) .where(Account.id == account_id) + # Marked-for-deletion rows are gone from the operator's + # perspective: ID-based mutations must report not-found, as the + # synchronous delete did once the row was removed. + .where(Account.delete_requested_at.is_(None)) .values(routing_policy=routing_policy) .returning(Account.id) ) await self._session.commit() return result.scalar_one_or_none() is not None - async def delete(self, account_id: str, *, delete_history: bool = False) -> bool: + async def begin_delete(self, account_id: str, *, delete_history: bool = False) -> bool: + """Mark an account for background deletion; commits in milliseconds. + + Fast path of ``DELETE /api/accounts/{id}``: the account becomes + terminal (``DEACTIVATED`` — every serving path already excludes it) + and carries the pending-deletion marker that hides it from listings + and enqueues it for the deletion worker, which drains its bulk rows + in chunks and finalizes via :meth:`delete` with ``only_pending=True``. + + The stored token ciphertext is overwritten with empty-credential + ciphertext in the same transaction: the row outlives the DELETE + response by the drain duration, and no reader — including a + pre-upgrade replica during a rolling deploy, whose export endpoints + do not know the marker — may still be able to produce usable + credentials from it. A credential replacement (the only supersede + path) writes fresh ciphertext, and token rotation is CAS-guarded on + the pre-wipe refresh ciphertext, so a stale in-flight rotation + misses rather than resurrecting the old material. Before the wipe, + the non-secret seat identity is preserved: legacy rows whose + ``chatgpt_user_id`` was never backfilled carry it only inside the + id-token claims, and targeted reauthentication — the promised + supersede path — verifies the seat against exactly those two + sources, so ``chatgpt_user_id`` is backfilled from the claims when + absent. + + API-key account assignments are removed here as well (the FK cascade + used to do this when the synchronous delete removed the row), so key + listings and pooled-usage projections exclude the account + immediately; the key's ``account_assignment_scope_enabled`` flag is + persisted separately and keeps the key scoped. + + Idempotent: a repeat request on an already-marked account succeeds + without changing the frozen ``delete_history`` choice (first request + wins — matching the synchronous behavior, where a second DELETE after + the first completed found nothing left to escalate). + """ + # Repeat requests short-circuit BEFORE the writer section / row lock: + # a drain chunk holds the account row (and, on SQLite, the writer + # section) for up to a few seconds, and the fast-path contract is a + # millisecond-scale response. The unlocked read is safe because the + # repeat changes nothing — the first request froze the variant and + # the wipe/cleanup already ran — and a replacement racing this read + # supersedes the deletion exactly as if it landed after this + # response. When credentials were replaced WITHOUT clearing the + # marker (a pre-upgrade replica's replacement), fall through to the + # full path so an explicit re-delete re-wipes and re-arms. + marked_row = ( + await self._session.execute( + select( + Account.delete_requested_at, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + ).where(Account.id == account_id) + ) + ).first() + if ( + marked_row is not None + and marked_row[0] is not None + and not credentials_replaced_since_wipe(marked_row[1], marked_row[2], marked_row[3]) + ): + return True + encryptor = TokenEncryptor() + wiped_token = encryptor.encrypt("") + async with sqlite_writer_section(): + seat_stmt = select(Account.chatgpt_user_id, Account.id_token_encrypted).where(Account.id == account_id) + if self._dialect_name() == "postgresql": + # Hold the row through the mark so the derived seat identity + # cannot go stale between this read and the update below. + seat_stmt = seat_stmt.with_for_update(key_share=True) + seat_row = (await self._session.execute(seat_stmt)).first() + if seat_row is None: + await self._session.rollback() + return False + seat_user_id: str | None = seat_row[0] + if seat_user_id is None: + try: + claims = extract_id_token_claims(encryptor.decrypt(seat_row[1])) + seat_user_id = resolve_seat_identity(claims, claims.auth) + except Exception: + seat_user_id = None + values: dict[str, Any] = { + "status": AccountStatus.DEACTIVATED, + "deactivation_reason": ACCOUNT_PENDING_DELETION_REASON, + "reset_at": None, + "blocked_at": None, + "access_token_encrypted": wiped_token, + "refresh_token_encrypted": wiped_token, + "id_token_encrypted": wiped_token, + "delete_requested_at": func.coalesce(Account.delete_requested_at, utcnow()), + "delete_history_requested": case( + (Account.delete_requested_at.is_(None), delete_history), + else_=Account.delete_history_requested, + ), + } + if seat_user_id is not None: + values["chatgpt_user_id"] = seat_user_id + result = await self._session.execute( + update(Account).where(Account.id == account_id).values(**values).returning(Account.id) + ) + updated_id = result.scalar_one_or_none() + if updated_id is not None: + # Same immediate cleanup the DEACTIVATED transition performs: + # sticky mappings and bridge sessions must not outlive the + # account's routability. + await self._session.execute(delete(StickySession).where(StickySession.account_id == account_id)) + await self._close_http_bridge_sessions_for_account(account_id) + await self._session.execute( + delete(ApiKeyAccountAssignment).where(ApiKeyAccountAssignment.account_id == account_id) + ) + await self._session.commit() + return updated_id is not None + + async def delete( + self, + account_id: str, + *, + delete_history: bool = False, + only_pending: bool = False, + ) -> bool: async with sqlite_writer_section(): if self._dialect_name() == "postgresql": # Identity membership precedes the fold-state lock so live # settlement and deletion cannot form an identity/fold cycle. - await self._lock_postgresql_account_identity_membership(account_id, None) + locked_account = await self._lock_postgresql_account_identity_membership(account_id, None) + pending_state = ( + None + if locked_account is None + else ( + locked_account.delete_requested_at, + locked_account.delete_history_requested, + locked_account.access_token_encrypted, + locked_account.refresh_token_encrypted, + locked_account.id_token_encrypted, + ) + ) + else: + pending_state = ( + await self._session.execute( + select( + Account.delete_requested_at, + Account.delete_history_requested, + Account.access_token_encrypted, + Account.refresh_token_encrypted, + Account.id_token_encrypted, + ).where(Account.id == account_id) + ) + ).first() + if only_pending: + # Background finalization: a credential replacement + # (re-import/reauth) that cleared the marker supersedes the + # deletion, so touch nothing. The variant comes from the + # persisted flag frozen at request time, never the caller. + # On PostgreSQL the identity-membership row lock held above + # keeps the marker stable through this transaction; on SQLite + # the writer section serializes all writers. + if pending_state is None or pending_state[0] is None: + await self._session.rollback() + return False + if credentials_replaced_since_wipe(pending_state[2], pending_state[3], pending_state[4]): + # A pre-upgrade replica replaced the credentials without + # being able to clear marker columns its ORM does not + # know. That replacement supersedes the deletion: clear + # the marker (we hold the row lock) and abandon. + await self._session.execute( + update(Account) + .where(Account.id == account_id) + .values(delete_requested_at=None, delete_history_requested=False) + ) + await self._session.commit() + return False + delete_history = bool(pending_state[1]) # Serialize against fold passes before touching the account's # request logs: without the fold-state lock an in-flight hourly # slice could aggregate the pre-delete attribution but commit # after this transaction, resurrecting the account's folded rows # the mirrors below just moved or removed. await lock_fold_state(self._session) + if self._dialect_name() == "postgresql": + # Upgrade the account row to a full FOR UPDATE lock BEFORE the + # raw sweeps. FOR UPDATE conflicts with the KEY SHARE taken by + # concurrent request-log FK inserts, so every in-flight + # stream's log row either commits before this point (and the + # sweeps below see it) or its insert blocks until this + # transaction commits and then fails its FK against the + # deleted row — the same outcome a post-delete insert always + # had. Without the upgrade, an insert could commit between + # the sweep and the account-row delete: the FK's ON DELETE + # SET NULL would leave a live (deleted_at IS NULL) orphan on + # the soft path, or surviving raw history under + # delete_history. Lock order (identity -> fold -> row + # exclusive) matches the historical transaction, where the + # final DELETE acquired this same exclusive lock after the + # fold lock. + await self._session.execute(select(Account.id).where(Account.id == account_id).with_for_update()) await self._session.execute(delete(UsageHistory).where(UsageHistory.account_id == account_id)) if delete_history: await self._session.execute(delete(RequestLog).where(RequestLog.account_id == account_id)) @@ -1288,6 +1550,12 @@ def _apply_account_updates(target: Account, source: Account) -> None: target.deactivation_reason = source.deactivation_reason target.reset_at = source.reset_at target.blocked_at = source.blocked_at + # A credential replacement (re-import/reauth) supersedes a pending + # background deletion: clearing the marker makes the deletion worker + # abandon the account before finalizing (rows already drained stay + # detached — history loss was requested by the earlier delete). + target.delete_requested_at = None + target.delete_history_requested = False def _slot_lock_key(account: Account, *, preserve_unknown_workspace_duplicates: bool = True) -> str: diff --git a/app/modules/accounts/service.py b/app/modules/accounts/service.py index f6c53d7309..71f0b19de7 100644 --- a/app/modules/accounts/service.py +++ b/app/modules/accounts/service.py @@ -40,6 +40,7 @@ from app.db.models import Account, AccountStatus, DashboardSettings from app.db.session import get_background_session from app.modules.accounts.auth_manager import AuthManager +from app.modules.accounts.deletion import request_account_deletion_run from app.modules.accounts.mappers import build_account_summaries, build_account_usage_trends from app.modules.accounts.repository import AccountsRepository from app.modules.accounts.schemas import ( @@ -233,7 +234,7 @@ async def list_accounts(self, *, account_ids: list[str] | None = None) -> list[A ) async def get_account_trends(self, account_id: str) -> AccountTrendsResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if not account or not self._usage_repo: return None now = utcnow() @@ -255,7 +256,7 @@ async def get_account_trends(self, account_id: str) -> AccountTrendsResponse | N ) async def get_usage_reset_credits(self, account_id: str) -> AccountUsageResetCreditsResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -318,7 +319,7 @@ async def consume_usage_reset_credit( *, redeem_request_id: str | None = None, ) -> AccountUsageResetConsumeResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -423,8 +424,19 @@ async def _resolve_usage_reset_credit_route( encryptor=self._encryptor, ) - async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExportResponse | None: + async def _get_visible_account(self, account_id: str) -> Account | None: + """Account fetch for ID-based operator routes; marked-for-deletion + rows are gone from the operator's perspective (the synchronous delete + returned 404 on every one of these routes once the row was removed) + and MUST NOT keep serving reads, mutations, or decrypted tokens + during the background drain window.""" account = await self._repo.get_by_id(account_id) + if account is None or account.delete_requested_at is not None: + return None + return account + + async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExportResponse | None: + account = await self._get_visible_account(account_id) if account is None: return None @@ -449,7 +461,7 @@ async def export_opencode_auth(self, account_id: str) -> AccountOpenCodeAuthExpo ) async def export_auth(self, account_id: str) -> AccountAuthExportResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None @@ -592,6 +604,11 @@ async def reactivate_account(self, account_id: str) -> bool: account = await self._repo.get_by_id(account_id) if account is None: return False + if account.delete_requested_at is not None: + # Marked for background deletion: already invisible in listings + # and about to be removed — report it as gone rather than racing + # the deletion worker back to ACTIVE. + return False if account.status == AccountStatus.REAUTH_REQUIRED: raise AccountStateTransitionError("Account requires re-authentication and cannot be reactivated directly") result = await self._repo.update_status_if_current( @@ -614,7 +631,7 @@ async def reactivate_account(self, account_id: str) -> bool: return result async def pause_account(self, account_id: str) -> bool: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return False if account.status in (AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): @@ -659,19 +676,25 @@ async def set_routing_policy(self, account_id: str, routing_policy: str) -> bool return result async def delete_account(self, account_id: str, *, delete_history: bool = False) -> bool: - result = await self._repo.delete(account_id, delete_history=delete_history) + # Fast path: stamp the pending-deletion marker (terminal status, hidden + # from listings, sticky/bridge cleanup) and return in milliseconds; the + # background deletion worker drains the bulk rows and removes the + # account row afterwards (see app.modules.accounts.deletion). + result = await self._repo.begin_delete(account_id, delete_history=delete_history) if result: mark_account_routing_unavailable(account_id) get_account_selection_cache().invalidate() get_api_key_cache().clear() - # Deletion cascades the account_proxy_bindings row away, and account - # ids are deterministic (delete-then-re-import regenerates the same - # id), so the cached route outcome must not survive the deletion. + # Finalization cascades the account_proxy_bindings row away, and + # account ids are deterministic (delete-then-re-import regenerates + # the same id), so the cached route outcome must not survive the + # delete request; the worker invalidates again after finalizing. await get_upstream_route_cache().invalidate() await propagate_account_routing_change() poller = get_cache_invalidation_poller() if poller is not None: await poller.bump(NAMESPACE_API_KEY) + request_account_deletion_run() return result async def set_account_alias(self, account_id: str, alias: str | None) -> bool: @@ -681,7 +704,7 @@ async def set_account_alias(self, account_id: str, alias: str | None) -> bool: return await self._repo.update_alias(account_id, normalized) async def export_account(self, account_id: str) -> AccountExportResponse | None: - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if not account: return None access_token = self._encryptor.decrypt(account.access_token_encrypted) @@ -722,7 +745,7 @@ async def probe_account( before/after snapshot so the operator can see whether the upstream state changed. """ - account = await self._repo.get_by_id(account_id) + account = await self._get_visible_account(account_id) if account is None: return None if account.status in (AccountStatus.PAUSED, AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED): diff --git a/app/modules/api_keys/repository.py b/app/modules/api_keys/repository.py index 63bbdec207..2cc3fb4f8c 100644 --- a/app/modules/api_keys/repository.py +++ b/app/modules/api_keys/repository.py @@ -6,7 +6,7 @@ from enum import Enum from typing import Any -from sqlalchemy import BigInteger, Integer, cast, delete, func, or_, select, true, update +from sqlalchemy import BigInteger, Integer, cast, delete, func, insert, literal, or_, select, true, update from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import load_only, selectinload @@ -183,6 +183,11 @@ async def list_accounts_by_ids(self, account_ids: list[str]) -> list[Account]: select(Account) .options(load_only(Account.id, Account.plan_type, Account.status)) .where(Account.id.in_(account_ids)) + # An account marked for background deletion is already deleted + # from the operator's point of view: assignment validation must + # reject it (the synchronous delete removed the row outright) and + # pooled-usage projections must not count it while its rows drain. + .where(Account.delete_requested_at.is_(None)) ) return list(result.scalars().all()) @@ -199,6 +204,11 @@ async def list_all_accounts(self) -> list[Account]: .where( ~Account.status.in_((AccountStatus.REAUTH_REQUIRED, AccountStatus.DEACTIVATED, AccountStatus.PAUSED)) ) + # Status alone is not enough: an unfenced pre-upgrade replica can + # briefly replace a marked account's terminal status during a + # rolling deploy, and a deleted account must never re-enter the + # unscoped pooled-usage projections. + .where(Account.delete_requested_at.is_(None)) ) return list(result.scalars().all()) @@ -436,9 +446,33 @@ async def upsert_limits(self, key_id: str, limits: list[ApiKeyLimit], *, commit: return await self.get_limits_by_key(key_id) async def replace_account_assignments(self, key_id: str, account_ids: list[str], *, commit: bool = True) -> None: + # Re-check the pending-deletion marker atomically with the write: + # validation ran in an earlier transaction, and an account DELETE can + # commit in between — the marked row still exists (background drain), + # so a plain FK insert would succeed and resurrect an assignment + # begin_delete just removed. The FOR SHARE lock (PostgreSQL) + # conflicts with begin_delete's row update, so either this + # transaction commits first (and begin_delete's assignment cleanup + # removes its rows) or the marker is visible below and the account is + # skipped. The account locks are taken BEFORE the assignment-row + # delete to match begin_delete's order (account row, then assignment + # rows) — taking them after would form a lock cycle with a + # concurrent begin_delete and deadlock. SQLite serializes writers, + # so the marker predicate alone is race-free there. + if account_ids and self._session.get_bind().dialect.name == "postgresql": + await self._session.execute( + select(Account.id).where(Account.id.in_(account_ids)).with_for_update(read=True) + ) await self._session.execute(delete(ApiKeyAccountAssignment).where(ApiKeyAccountAssignment.api_key_id == key_id)) - for account_id in account_ids: - self._session.add(ApiKeyAccountAssignment(api_key_id=key_id, account_id=account_id)) + if account_ids: + assignment_source = ( + select(literal(key_id), Account.id) + .where(Account.id.in_(account_ids)) + .where(Account.delete_requested_at.is_(None)) + ) + await self._session.execute( + insert(ApiKeyAccountAssignment).from_select(["api_key_id", "account_id"], assignment_source) + ) if commit: await self._session.commit() parent = await self._session.get(ApiKey, key_id) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index c2a361720b..abefd6d389 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -1714,7 +1714,9 @@ async def _ensure_v1_reset_credit_account_fresh(account_id: str) -> _V1ResetCred async with get_background_session() as session: repo = AccountsRepository(session) account = await repo.get_by_id(account_id) - if account is None: + # An account marked for background deletion is already deleted from + # every consumer's point of view (its credentials are wiped). + if account is None or account.delete_requested_at is not None: raise HTTPException(status_code=404, detail="Account not found") auth_manager = AuthManager( repo, @@ -1755,6 +1757,11 @@ async def v1_redeem_reset_credit( return capability_transport_denial async with get_background_session() as session: account = await AccountsRepository(session).get_by_id(payload.account_id) + # A pending-deletion account is gone (credentials wiped): treat it + # exactly like an account outside the pool. ``getattr`` because pool + # membership tests stub the account with plain namespaces. + if account is not None and getattr(account, "delete_requested_at", None) is not None: + account = None if not _is_reset_credit_account_in_api_key_pool(account, api_key): raise HTTPException(status_code=403, detail="Account is outside the API key pool") if account is None: diff --git a/app/modules/rate_limit_reset_credits/api.py b/app/modules/rate_limit_reset_credits/api.py index 034c83e2a2..01bb4d431f 100644 --- a/app/modules/rate_limit_reset_credits/api.py +++ b/app/modules/rate_limit_reset_credits/api.py @@ -132,7 +132,8 @@ async def get_rate_limit_reset_credits( ) -> RateLimitResetCreditsSnapshotResponse | None: store = get_rate_limit_reset_credits_store() account = await context.repository.get_by_id(account_id) - if account is None: + # A pending-deletion account is gone from the operator's point of view. + if account is None or account.delete_requested_at is not None: await store.invalidate(account_id) return None if account.status in _NON_REDEEMABLE_STATUSES or not account.chatgpt_account_id: @@ -158,7 +159,9 @@ async def consume_rate_limit_reset_credit( context: AccountsContext = Depends(get_accounts_context), ) -> ConsumeResetCreditResponseSchema: account = await context.repository.get_by_id(account_id) - if account is None: + # A pending-deletion account is gone from the operator's point of view: + # the synchronous delete 404'd here once the row was removed. + if account is None or account.delete_requested_at is not None: raise DashboardNotFoundError("Account not found", code="account_not_found") store = get_rate_limit_reset_credits_store() diff --git a/app/modules/settings/api.py b/app/modules/settings/api.py index c132267538..813af868a3 100644 --- a/app/modules/settings/api.py +++ b/app/modules/settings/api.py @@ -413,7 +413,10 @@ async def _validate_proxy_pool_id(context: SettingsContext, pool_id: str | None) async def _get_account_or_error(context: SettingsContext, account_id: str) -> Account: account = await context.session.get(Account, account_id) - if account is None: + # An account marked for background deletion is already deleted from the + # operator's point of view: binding mutations must report not-found, as + # the synchronous delete did once the row was removed. + if account is None or account.delete_requested_at is not None: raise DashboardBadRequestError("Account not found", code="account_not_found") return account diff --git a/openspec/changes/background-account-deletion/design.md b/openspec/changes/background-account-deletion/design.md new file mode 100644 index 0000000000..2de8e239db --- /dev/null +++ b/openspec/changes/background-account-deletion/design.md @@ -0,0 +1,334 @@ +## Context + +Reference commit: `origin/main` 0c8d9219. Verified surfaces: +`AccountsRepository.delete()` and its fold-state lock comment, +`app/modules/accounts/usage_rollup.py` (lifetime fold, `lock_fold_state`), +`app/modules/accounts/usage_time_rollup.py` (hourly/demand/error/conversation +folds, lifecycle mirrors, history-rewrite discipline), `app/core/retention/` +(chunked prune precedent, BATCH_SIZE=10k), duplicate-account consolidation +(`_reconcile_chatgpt_identity_duplicates`, same fold lock), scheduler + leader +election pattern, Alembic single head `20260812_120000_add_sticky_abandonment_scope`. + +Production measurements (10.0.0.113): account deletion = single transaction +holding the fold-state lock for the whole drain; ~93k `usage_history` rows ≈ +11.6 s each for two accounts; ~133k `request_logs` soft-detach = 313 s (18 +indexes ≈ 8.3 GB, every row non-HOT); fold blocked; 73 pool timeouts in 36 h; +HTTP client timeout on the DELETE call. + +## Goals / Non-Goals + +**Goals:** + +- DELETE API returns in milliseconds; the account is immediately invisible to + listings and unroutable. +- Bulk row work proceeds in bounded background transactions (1k rows each, + batch selection pinned to account-leading indexes) so the fold, the pool, + and vacuum are never blocked for minutes. +- Same end state as the synchronous delete for both `delete_history` + variants, including the folded-bucket lifecycle mirrors and the + "rollup row deleted with the account row" invariant. +- Restart-safe resume, idempotent repeat requests, explicit supersede path. + +**Non-Goals:** + +- No change to duplicate-account consolidation (still synchronous under the + fold lock; its row volume is bounded by the duplicate's history and it must + stay atomic with the identity swap). +- No change to retention, fold cadence, or watermark semantics. +- No new settings; no dashboard UI for drain progress (the account is simply + gone; the worker logs outcomes). + +## Decisions + +### D1: Terminal mark = existing `DEACTIVATED` status + marker columns, not a new enum value + +Serving-path exclusion lists are written as denylists +(`status not in (REAUTH_REQUIRED, DEACTIVATED, PAUSED)` in +`proxy/helpers.py`, `load_balancer.py`, `account_cache.py`, `proxy/api.py`, +`realtime_live.py`): a new `DELETING` enum value would be *routable* until +every denylist was found and extended, and would require a PostgreSQL enum +migration. Reusing `DEACTIVATED` inherits every existing exclusion (sticky +purge, bridge close, selection caches) with zero new status handling. The +pending-deletion state itself lives in `accounts.delete_requested_at` +(authoritative marker + queue ordering) and `delete_history_requested` +(variant, frozen at request time); `deactivation_reason="pending_deletion"` +is operator-facing only. + +The marker also fences ordinary status writers (`update_status`, +`update_status_if_current` gain `delete_requested_at IS NULL`): a stale +in-flight settlement — e.g. a 429 for a request selected before the DELETE — +would otherwise replace `DEACTIVATED` with `RATE_LIMITED` and make the +account selectable again for direct-by-id paths mid-drain. Credential +replacement does not go through these writers (it writes fields directly and +clears the marker in the same transaction), so the supersede path is +unaffected. Pre-upgrade replicas' writers are unfenced during a rolling +deploy, so every drain chunk additionally self-heals: when the marked row +drifted (non-terminal status or a recreated API-key assignment) without a +credential replacement, the chunk re-asserts the terminal status and +re-removes the assignments under the row lock it already holds — a DB +trigger was rejected as disproportionate machinery for a drift window that +is already bounded to one chunk transaction (seconds). + +Repeat DELETE requests short-circuit on an unlocked marker+wipe read before +entering the writer section / row lock: a drain chunk holds the account row +(and, on SQLite, the writer section) for seconds at a time, and the +fast-path contract must hold throughout the drain. The short-circuit falls +through to the full path when the credentials were replaced without +clearing the marker, so an explicit re-delete after a legacy replacement +re-wipes and re-arms the deletion. + +`begin_delete` additionally produces the two projections the synchronous +delete's row removal produced instantly: it deletes the account's +`ApiKeyAccountAssignment` rows (key listings and pooled-usage reads exclude +the account immediately; the key's persisted `account_assignment_scope_enabled` +flag keeps the key scoped, exactly as after the FK cascade) and overwrites +the access/refresh/id token ciphertext with empty-credential ciphertext. +The wipe is what keeps the rolling upgrade honest: a pre-upgrade replica's +export endpoints read the row without knowing the marker, and must not be +able to hand out usable credentials during the drain window. Token rotation +is CAS-guarded on the pre-wipe refresh ciphertext (a stale rotation misses), +and every supersede path writes complete fresh ciphertext. The wipe must +not break the reauth supersede path itself: targeted reauthentication +verifies the seat against `chatgpt_user_id` or — on legacy rows where it +was never backfilled — the stored id-token claims, so `begin_delete` +backfills `chatgpt_user_id` from those claims (non-secret identity) in the +same transaction before destroying them. + +The wipe doubles as the supersede signal for pre-upgrade replicas: a +replacement handled by old code writes fresh ciphertext but cannot clear +marker columns its ORM does not know. Every marker re-check (chunk and +finalization) therefore also inspects ALL THREE token ciphertexts — +non-wiped or undecryptable material in any field of a marked row means a +replacement happened (a legal replacement may carry an empty refresh token +while providing fresh access/id material, so a refresh-only check would +finalize a freshly replaced account), and the worker clears the marker +itself (under the row lock) instead of draining further or finalizing. API-key assignment validation +(`ApiKeysRepository.list_accounts_by_ids`) likewise rejects marked +accounts, and `replace_account_assignments` locks the target account rows (`FOR SHARE` +on PostgreSQL, which conflicts with `begin_delete`'s row update) BEFORE +touching any assignment row — the same account-then-assignment order +`begin_delete` uses, so the race serializes instead of deadlocking — and +then inserts through a conditional `INSERT … SELECT … WHERE +delete_requested_at IS NULL`. A key create/update whose validation raced +the DELETE therefore cannot recreate an assignment that would re-surface +the account in key listings: either it commits first and `begin_delete`'s +assignment cleanup removes its rows, or the marker is visible to the +insert and the account is skipped. + +### D2: The account row is the queue (no new table) + +The marker columns make the `accounts` row its own durable work item: the +worker scans `delete_requested_at IS NOT NULL`, progress is the shrinking +`WHERE account_id = :id` predicates, and finalization's row delete is the +dequeue. Restart resume and idempotency need no extra state machine; a +crash between any two chunk transactions loses nothing. + +### D3: Chunks do NOT take the fold-state lock; only finalization does + +The single-transaction delete held the fold lock to prevent an in-flight +fold slice from committing pre-delete attribution after the mirrors ran +(resurrecting folded rows). The chunked drain preserves that invariant with +lock-free chunks because: + +1. Chunk transactions touch only raw rows; they never write a rollup table + or move a watermark (`usage_history` tables are not fold-governed at all). +2. An interleaved fold slice aggregates either still-attached rows (folded + under the account dimension) or already-detached rows (folded under the + orphaned-deleted dimension — the soft-path end state). Both converge at + finalization: it takes `lock_fold_state()`, detaches/deletes residual raw + rows, and runs the lifecycle mirrors, which move or remove EVERY folded + row carrying the account dimension — including rows folded mid-drain. +3. Every fold slice holds the fold-state row lock (`FOR UPDATE` on the + `account_usage_rollup_state` row) from before it reads raw rows until its + commit. A slice therefore commits strictly before finalization (its + output is mirrored) or strictly after (it sees no attributed raw rows). + Post-finalization resurrection is impossible. + +Per-chunk fold-lock acquisition was considered and rejected: it adds fold +stalls proportional to drain length while providing nothing the finalization +lock does not already guarantee (the mirrors are a pure dimension move over +whatever is folded at mirror time). + +The letter of the history-rewrite discipline in `usage_time_rollup.py` +("mutations of folded dimensions below the watermark take the fold lock and +mirror or skip **in the same transaction**") is relaxed for this one path: +mid-drain, folded buckets may still attribute to an account whose raw rows a +chunk already detached. That intermediate state never double- or +under-counts a read (folded side serves below-watermark, raw tail above; +the watermark folds each raw row exactly once, and already-folded rows are +never re-read), and on the deletion path it is bounded by drain duration: +the end state is byte-identical to the synchronous path, with the module +docstring of `deletion.py` documenting this as the single sanctioned +exception, converged by finalization. + +When a supersede lands after a partial drain, finalization never runs and +the divergence for rows drained before the supersede is the PERMANENT, +intended end state: folded buckets keep attributing that traffic to the +revived account (it is the account's true pre-delete history — nothing is +added or inflated), while the raw rows stay detached (soft) or deleted +(`delete_history`), exactly as the "rows already drained stay detached" +trade-off promises. Reads stay consistent for the same reason as mid-drain: +below-watermark reads are folded-only, drained rows below the watermark are +never re-folded, and drained rows above the watermark fold once under the +orphaned dimension. Reconciling instead (running the lifecycle mirrors at +supersede time) was rejected: it would drag the fold lock and per-row delta +mirroring into every credential-replacement path to "fix" attribution that +is already historically correct. + +### D4: Finalization reuses `AccountsRepository.delete()` with a marker guard + +`delete(only_pending=True)` is the historical transaction verbatim — +identity-membership lock (PostgreSQL), fold-state lock, residual +usage-history delete, residual detach/delete + mirrors, sticky + rollup + +account row — plus: it aborts (touching nothing) unless the marker is still +set, and it reads the `delete_history` variant from the persisted flag +rather than the caller. The identity-membership `FOR NO KEY UPDATE` row lock +(PostgreSQL) keeps the marker stable through the transaction; on SQLite the +writer section serializes writers. Lock order (identity → fold) matches +consolidation, so no new deadlock ordering is introduced. Residual rows also +cover stragglers: a stream that started before the mark settles its +request-log row at stream end, possibly after every chunk ran. + +After the fold lock, finalization upgrades the account row to a full +`FOR UPDATE` lock (PostgreSQL) before the residual sweeps. `FOR UPDATE` +conflicts with the `KEY SHARE` a request-log FK insert takes, so an +in-flight stream's insert either commits before the sweep (and is swept) or +blocks until the transaction commits and then fails its FK against the +deleted row — the same outcome a post-delete insert always had. Without the +upgrade, an insert could commit between the sweep and the account-row +delete, where `ON DELETE SET NULL` would leave a live (`deleted_at IS +NULL`) orphan on the soft path or surviving raw history under +`delete_history`. The lock order (identity → fold → row exclusive) matches +the historical transaction, whose final `DELETE` acquired the same +exclusive lock after the fold lock. + +### D5: Supersede-by-replacement, first-request-wins idempotency + +`_apply_account_updates` (every credential replacement: re-import, reauth, +slot reuse) clears the marker: account ids are deterministic, so +delete-then-reimport lands on the marked row, and letting the worker delete +a just-reimported account would be data loss. Every chunk transaction and +finalization re-read the marker under the account row lock (PostgreSQL +`FOR NO KEY UPDATE`, compatible with the `KEY SHARE` taken by concurrent +rollup FK inserts; on SQLite the writer section serializes writers), so a +replacement either commits before the marker read (the chunk sees the +cleared marker and stops) or blocks until the chunk commits — no chunk can +mutate rows after a replacement has successfully returned, and a superseded +account is never finalized (rows already drained stay detached — history +loss was requested by the earlier delete). The chunk takes only the account +row lock and touches only that account's child rows, so no new lock ordering +is introduced. Marked accounts are also absent from the credential-export +endpoints: the synchronous delete made exports 404 immediately, and the +asynchronous drain window must not keep decrypted tokens retrievable after +a successful DELETE. Repeat DELETE requests return +success without escalating `delete_history` (first request wins), matching +the synchronous world where a second DELETE arrived after the account was +already gone. `reactivate_account` treats a marked account as not found +rather than racing the worker back to ACTIVE. + +### D6: Worker = leader-gated 30 s tick + local wake, cheap pre-check, round-robin pass + +Same scheduler shape as retention. Each tick runs one `LIMIT 1` existence +probe *before* leader election — served by the partial index +`idx_accounts_delete_requested_at` (`WHERE delete_requested_at IS NOT +NULL`), which is empty in the steady state — so a tick with nothing to do +costs one tiny index probe. `delete_account` wakes the local worker after +commit: on the leader (the single-replica common case) draining starts +immediately; a follower's wake is a no-op and the leader's tick picks the +request up within 30 s. Batch size 1k: measured ~1.2 s/10k `usage_history` +deletes and ~23 s/10k `request_logs` detaches (18 indexes, non-HOT updates) +bound the worst table at ~2.3 s per transaction — and every chunk holds the +account row lock (`FOR NO KEY UPDATE`) for its full duration, so a supersede +or fenced settlement waits for at most one chunk. Between row-touching +rounds the pass sleeps a fraction of the round's own duration (capped), so +a multi-hundred-chunk drain leaves the 2-vCPU database headroom instead of +running chunk transactions back-to-back. + +Chunk batch selection is planner-pinned to the account-leading indexes +(`idx_usage_account_time`, `ix_additional_usage_distinct_labels`, +`idx_logs_account_kind_deleted_latest` — the last one covering, so the +request-log batch is an index-only scan): the batch subquery selects with an +`account_id >= :id AND account_id <= :id` range (equivalent rows, but the +range keeps `account_id` out of the constant-equivalence class and thus in +the sort pathkeys) ordered by the target index's exact column order, making +that index the only sort-free plan. A plain `account_id = :id LIMIT n` shape +was verified on the production planner to run as a LIMIT-terminated Seq +Scan for exactly the large accounts this change targets — with an unbounded +dead-prefix re-scan mid-drain and a guaranteed full heap scan for every +empty probe once per-account statistics go stale. With the pinned shape, +chunk scan work is bounded by the account's own remaining rows and a +drained-table probe is a single index descent. Within one pass, a table +observed empty is not re-probed on later rounds (rows settling mid-drain +are converged by finalization's residual sweep anyway). + +A deletion pass round-robins: each round advances every pending account by +at most one NONEMPTY chunk transaction (a round stops at the first chunk +that touched rows, not the first batch-size-full one, so small tables cannot +stack several row-touching transactions into one round) and the pending set +is re-scanned between rounds. A multi-minute drain (the measured 133k-row +account is ~133 chunks) therefore cannot starve another marked account, and +a DELETE that lands mid-pass is picked up by the next round's re-scan rather +than waiting for the whole pass to finish. + +### D7: API contract unchanged (`{"status": "deleted"}`) + +The dashboard's delete mutation only toasts and refetches the listing, which +already excludes the marked account — the operator-visible contract ("after +DELETE, the account is gone from the list") holds exactly. Returning a new +`"deleting"` status would break any consumer comparing against "deleted" +while conveying nothing actionable: the deletion is irrevocable (modulo +re-import) once the API returns. The spec states row purge is asynchronous. + +## Risks / Trade-offs + +- **Mid-drain visibility**: statistics pages may briefly attribute folded + history to the (invisible) account while raw rows are already detached. + Bounded by drain duration; strictly better than the previous minutes-long + fold outage. +- **Interleaved folds vs hard delete**: a fold slice between hard-delete + chunks may fold rows (account and API-key aggregates) that the + single-transaction path would have deleted first. This is inherent fold + timing (a fold 1 s before the DELETE captured them under the old code + too); the account side is removed by the mirrors, and API-key folded sums + keeping settled traffic is the documented behavior for folded history. +- **Supersede after partial drain**: a re-import that lands mid-drain keeps + the account but its already-detached rows stay detached (and + already-deleted rows stay deleted), while folded buckets keep attributing + the pre-supersede-drained traffic to the revived account — permanently, + since finalization's mirrors never run. This is historically correct + attribution (the folded numbers pre-existed the delete), never double- or + under-counts a read (see D3), and is the documented consequence of the + operator asking for deletion first. +- **Alembic head races**: the revision sits on the current single head; + parallel PRs adding revisions require the usual head merge. + +## Migration + +`20260816_000000_add_account_pending_deletion`: adds +`accounts.delete_requested_at` (nullable DateTime), +`accounts.delete_history_requested` (Boolean, `server_default false`), and +the partial queue index `idx_accounts_delete_requested_at` +(`(delete_requested_at, id) WHERE delete_requested_at IS NOT NULL`), with +existence guards and a symmetric downgrade. Existing rows are +untouched (no pending deletions can predate the feature). Rolling upgrade: an +old replica neither sets nor reads the marker; a delete handled by an old +replica is simply the old synchronous delete, and a delete handled by a new +replica leaves old replicas nothing exploitable — the fast path wipes the +token ciphertext, so old export/read paths that do not know the marker can +only produce empty credentials until finalization removes the row (old +replicas may transiently show the account in listings during the mixed +window; it is unroutable via the terminal status either way). + +One mixed-window caveat follows directly from "old replica = old delete": a +repeat DELETE routed to a pre-upgrade replica while the row is still marked +runs the legacy synchronous delete with its caller-provided +`delete_history` variant, which can differ from the frozen first-request +choice (either direction). The legacy delete is still a complete, +fold-locked, mirror-correct deletion — only the history-policy choice +diverges, only inside the deploy window, and only when the operator issues +contradictory repeat requests inside it. Fencing was rejected: new code +cannot retrofit a fence into binaries that predate the marker columns, a +database trigger is disproportionate machinery for the window, and a +"defer background marks until the fleet is upgraded" gate would add a +permanent setting for a transient condition (single-replica deployments — +the production topology — have no mixed window at all). diff --git a/openspec/changes/background-account-deletion/proposal.md b/openspec/changes/background-account-deletion/proposal.md new file mode 100644 index 0000000000..4c1292cabf --- /dev/null +++ b/openspec/changes/background-account-deletion/proposal.md @@ -0,0 +1,56 @@ +## Why + +`DELETE /api/accounts/{id}` detaches (or deletes) the account's entire raw +history in one transaction while holding the fold-state lock. Measured on +production: ~11.6 s to delete ~93k `usage_history` rows and **313 s** to +soft-detach ~133k `request_logs` rows (18 indexes, every row non-HOT). For +those minutes the fold is blocked, one pool connection is pinned (contributing +to `QueuePool` timeouts), the HTTP client times out, and the long transaction +delays vacuum. + +## What Changes + +- `DELETE /api/accounts/{id}` becomes a fast mark: the account turns terminal + (`DEACTIVATED` + a pending-deletion marker), disappears from listings and + serving immediately, and the API returns within milliseconds with the + existing `{"status": "deleted"}` contract. +- A new leader-gated background worker drains the account's bulk rows + (`usage_history`, `additional_usage_history`, `request_logs`) in bounded + chunks (5k rows per transaction, no fold-state lock), then finalizes in one + fold-state-locked transaction with the exact shape of the old synchronous + delete: residual rows, folded-bucket lifecycle mirrors, sticky/rollup rows, + account row. +- Deletion is restart-safe (all progress in the database), idempotent + (repeat DELETE requests succeed without escalating the frozen + `delete_history` choice), supports both `delete_history` variants, and is + superseded by a credential replacement (re-import/reauth) that clears the + marker. +- Schema: two new nullable-safe columns on `accounts` + (`delete_requested_at`, `delete_history_requested`), Alembic revision + `20260816_000000_add_account_pending_deletion` on the current single head. + +## Capabilities + +### New Capabilities + +- `account-deletion`: asynchronous account deletion lifecycle — fast terminal + mark, immediate listing/serving exclusion, chunked background drain with + fold-interleave safety, restart-safe finalization, idempotency, and + supersede-by-replacement semantics. + +### Modified Capabilities + +(none — the query-caching lifecycle requirement "rollup row deleted in the +same transaction as the account deletion" continues to hold: the finalization +transaction removes both together.) + +## Impact + +- `app/modules/accounts/repository.py` (`begin_delete`, marker-guarded + `delete(only_pending=True)`, listing filters, replacement clears marker), + `app/modules/accounts/deletion.py` (new worker + scheduler), + `app/modules/accounts/service.py`, `app/main.py` (scheduler wiring), + `app/db/models.py`, one Alembic revision. +- No API schema change (`AccountDeleteResponse` unchanged), no frontend + change (the listing refetch after delete already sees the account gone), + no new settings. diff --git a/openspec/changes/background-account-deletion/specs/account-deletion/spec.md b/openspec/changes/background-account-deletion/specs/account-deletion/spec.md new file mode 100644 index 0000000000..ed6dd89fad --- /dev/null +++ b/openspec/changes/background-account-deletion/specs/account-deletion/spec.md @@ -0,0 +1,321 @@ +# account-deletion Delta + +## ADDED Requirements + +### Requirement: Account deletion requests return fast and hide the account immediately + +`DELETE /api/accounts/{account_id}` MUST NOT perform the account's bulk row +work (raw request-log detach/delete, usage-history removal) on the request +path. The request MUST only stamp a durable pending-deletion marker in a +short transaction: terminal `DEACTIVATED` status, the pending-deletion +marker (`delete_requested_at`), the frozen `delete_history` choice +(`delete_history_requested`), sticky-session removal, bridge-session +closure, API-key account-assignment removal (the projection the synchronous +delete's FK cascade produced — key listings and pooled-usage reads exclude +the account immediately while the key's persisted assignment-scope flag is +untouched), and an overwrite of the stored access/refresh/id token +ciphertext with empty-credential ciphertext so that NO reader of the +surviving row — including a pre-upgrade replica during a rolling deploy, +whose export endpoints do not know the marker — can produce usable +credentials during the drain window. Repeat DELETE requests MUST +short-circuit before taking the account row lock or the SQLite writer +section, so the millisecond contract holds even while a drain chunk +transaction is holding the row. Because targeted reauthentication (a +supersede path) verifies the seat against `chatgpt_user_id` or, on legacy +rows where it was never backfilled, the stored id-token claims, the fast +path MUST preserve the non-secret seat identity before the wipe by +backfilling `chatgpt_user_id` from those claims when it is absent. The +response contract remains `{"status": "deleted"}` with 200 for an existing +account and 404 otherwise; row purge is asynchronous. + +Accounts carrying the pending-deletion marker MUST be excluded from account +listings (`GET /api/accounts` and every listing-derived read) and MUST be +excluded from proxy serving via the terminal status. EVERY ID-based account +surface MUST report a marked account as not found (or absent) — reads +(trends, reset-credit views), mutations (account update, alias, +limit-warmup, routing policy, upstream-proxy binding), action routes +(pause, probe, reset-credit consumption on both the dashboard and +rate-limit route families, `/v1` reset-credit redemption, reactivation), +and the credential-export endpoints (account export, auth export, opencode +auth export) — because the synchronous delete returned 404 on all of them +once the row was removed, and a successful DELETE MUST NOT leave decrypted +tokens retrievable during the background drain window. Only +credential-replacement paths (re-import, reauthentication) may address the +marked row. + +Ordinary status writes MUST NOT modify a marked account: a stale in-flight +settlement (for example a 429 landing after the DELETE for a request +selected before it) must not replace the terminal `DEACTIVATED` state and +make the account selectable mid-drain. Only a credential replacement — +which clears the marker — may change a marked account's state. Because +pre-upgrade replicas' status writers are unfenced during a rolling deploy, +every drain chunk transaction MUST re-assert the terminal status (and +re-remove any recreated API-key assignments) under the account row lock +when the marked row has drifted without a credential replacement, bounding +such drift to one chunk transaction; after the repairing chunk commits, the +worker MUST propagate the same cache invalidation as the delete request +(routing unavailability, selection/API-key snapshots, routing-change bump) +so replicas that cached the drift stop serving it. + +Marked accounts MUST be rejected by API-key account-assignment validation +and excluded from API-key pooled-usage projections, and assignment +insertion MUST re-check the marker atomically with the write (a conditional +insert; on PostgreSQL additionally serialized against the delete mark by a +`FOR SHARE` lock on the account rows, acquired BEFORE any assignment-row +mutation so the lock order matches the delete path's account-then-assignment +order and the race serializes instead of deadlocking), so an assignment +created or updated after — or racing — the DELETE cannot re-surface the +account in key listings before finalization. + +#### Scenario: Delete responds without draining rows + +- **GIVEN** an account with raw request-log and usage-history rows +- **WHEN** `DELETE /api/accounts/{id}` returns 200 `{"status": "deleted"}` +- **THEN** the account no longer appears in `GET /api/accounts` +- **AND** the account row still exists, terminal and marked, with its raw + rows untouched until the background worker drains them + +#### Scenario: Marked account cannot be reactivated + +- **GIVEN** an account marked for background deletion +- **WHEN** `POST /api/accounts/{id}/reactivate` is called +- **THEN** the response is 404 `account_not_found` + +#### Scenario: All ID-based routes report the marked account as gone + +- **GIVEN** an account marked for background deletion whose rows are not yet + drained +- **WHEN** any ID-based account route (trends, reset-credit read/consume, + probe, pause, update, alias, limit-warmup, routing policy) is called +- **THEN** the response is 404 `account_not_found` + +#### Scenario: Marked account no longer serves credential exports + +- **GIVEN** an account marked for background deletion whose rows are not yet + drained +- **WHEN** any credential-export endpoint is called for the account +- **THEN** the response is 404 and no token material is returned +- **AND** the row's stored token ciphertext decrypts to empty credentials + (nothing usable remains for readers that do not know the marker) + +#### Scenario: Seat identity survives the token wipe for reauth supersede + +- **GIVEN** a legacy account whose `chatgpt_user_id` is unset (seat identity + lives only in the stored id-token claims) +- **WHEN** `DELETE /api/accounts/{id}` marks the account and wipes the token + ciphertext +- **THEN** `chatgpt_user_id` is backfilled from the id-token claims in the + same transaction, so a targeted reauthentication can still verify the + seat and supersede the deletion + +#### Scenario: Deleted account leaves API-key listings immediately + +- **GIVEN** an account assigned to an API key +- **WHEN** `DELETE /api/accounts/{id}` returns +- **THEN** the key's listed assigned-account ids no longer contain the + account and its pooled-usage projection excludes it +- **AND** the key's assignment-scope flag remains enabled + +#### Scenario: Marked account cannot be assigned to an API key + +- **GIVEN** an account marked for background deletion +- **WHEN** an API-key create or update names the account in its assigned + account ids +- **THEN** the request is rejected as referencing an unknown account + +#### Scenario: Stale settlement cannot resurrect a marked account + +- **GIVEN** an account marked for background deletion +- **WHEN** an ordinary status write (e.g. a late rate-limit settlement) + targets the account +- **THEN** the write is rejected and the account stays terminal and marked + +#### Scenario: Drift written by an unfenced pre-upgrade replica is re-fenced + +- **GIVEN** a marked account whose status was replaced (or whose API-key + assignment was recreated) by a pre-upgrade replica's unfenced writer, + with the token ciphertext still wiped +- **WHEN** the next drain chunk transaction runs +- **THEN** the terminal status and reason are re-asserted and the recreated + assignment is removed, in the same chunk transaction + +#### Scenario: Repeat delete stays fast during an active drain + +- **GIVEN** a marked account whose drain chunk transaction currently holds + the account row lock +- **WHEN** a repeat `DELETE /api/accounts/{id}` arrives +- **THEN** it returns success without waiting for the chunk transaction + +### Requirement: Background worker drains marked accounts in bounded chunks + +A leader-gated background worker MUST drain each marked account's +`usage_history`, `additional_usage_history`, and `request_logs` rows in +bounded per-transaction chunks (at most `DELETE_BATCH_SIZE` rows per +transaction) without holding the fold-state lock, and MUST then finalize in +ONE fold-state-locked transaction that detaches or deletes residual raw rows +(including request-log rows settled mid-drain by in-flight streams), runs +the folded-bucket lifecycle mirrors, and removes the sticky, lifetime-rollup, +and account rows together. Finalization MUST serialize against in-flight +raw-row inserts (on PostgreSQL by upgrading the account row to a full lock +that conflicts with the FK's `KEY SHARE` before the residual sweep), so a +log row committed by an in-flight stream is either swept by finalization or +its insert fails against the already-deleted account — finalization may +leave behind neither a live orphan row (soft variant) nor surviving raw +history (`delete_history` variant). The soft variant MUST detach raw rows +(`account_id=NULL, deleted_at` set); the `delete_history` variant MUST +delete them. The worker MUST start a drain promptly after a delete request +on the leader replica and within one worker interval otherwise. A deletion +pass MUST round-robin across pending accounts — at most one nonempty chunk +transaction per account per round — and re-scan for newly marked accounts +between rounds, so one account's long drain cannot delay another marked +account's drain start by more than one chunk transaction per pending +account. Chunk batch selection MUST be served by an account-leading index +on every drain table (on PostgreSQL: `idx_usage_account_time`, +`ix_additional_usage_distinct_labels`, and the covering +`idx_logs_account_kind_deleted_latest`), so per-chunk scan work is bounded +by the account's own remaining rows: it MUST NOT degrade to sequential +scans when per-account statistics are large or stale mid-drain, and a probe +of an already-drained table MUST terminate on the index without scanning +the heap. Within one pass, the worker MUST NOT re-probe a drain table it +already observed empty for an account (rows that land after that +observation are swept by finalization's residual pass), and it MUST pause +between consecutive row-touching rounds in proportion to the round's +duration so a long drain does not run chunk transactions back-to-back. + +#### Scenario: Chunked drain reaches the synchronous end state (soft) + +- **GIVEN** a marked account whose raw rows exceed one chunk +- **WHEN** the worker completes the drain and finalization +- **THEN** every raw request-log row is detached and soft-deleted, usage + snapshots are removed, and the sticky, lifetime-rollup, and account rows + are deleted in the finalization transaction + +#### Scenario: Chunked drain reaches the synchronous end state (delete_history) + +- **GIVEN** an account marked with the `delete_history` variant +- **WHEN** the worker completes the drain and finalization +- **THEN** the account's raw request-log rows are deleted and its folded + time-axis buckets are removed + +#### Scenario: In-flight log insert cannot escape finalization + +- **GIVEN** a marked account whose drain is complete and an in-flight stream + holding an uncommitted request-log insert for it +- **WHEN** finalization runs +- **THEN** finalization waits for the insert to commit and sweeps the late + row (or the insert fails against the deleted account), leaving no live + orphan and no surviving history + +#### Scenario: A long drain does not starve other marked accounts + +- **GIVEN** one marked account whose drain spans many chunks +- **WHEN** another account is marked for deletion (before or during the pass) +- **THEN** the second account's drain starts within one chunk round and both + accounts finalize + +#### Scenario: Chunk selection stays on the account index + +- **GIVEN** a marked account whose per-account row estimate is large (or + stale after a partial drain) +- **WHEN** a drain chunk selects its batch on PostgreSQL +- **THEN** the batch subquery is planned as a scan of the account-leading + index on each drain table (index-only for `request_logs`), not a + sequential scan, and an empty probe terminates on the index + +#### Scenario: Drained tables are not re-probed within a pass + +- **GIVEN** a marked account whose usage tables drained while its request + logs still span further chunks +- **WHEN** subsequent rounds of the same pass advance the account +- **THEN** the drained tables' chunk transactions do not run again and + finalization still sweeps any rows that landed after the empty + observation + +### Requirement: Interleaved fold slices never resurrect a deleted account's folded rows + +Fold passes MUST remain able to run between drain chunks. Because every fold +slice holds the fold-state row lock from before reading raw rows until its +commit, and finalization takes the same lock before running the lifecycle +mirrors over whatever is folded at that moment, a fold slice MUST either +commit before finalization (its account-attributed output is moved or +removed by the mirrors) or after (it observes no raw rows attributed to the +account). After finalization commits, no folded row in any rollup table may +carry the deleted account's dimension, and under the soft variant the +orphaned-deleted dimension MUST preserve the account's full folded history. + +#### Scenario: Fold between chunks is converged by finalization + +- **GIVEN** a marked account with part of its raw history already detached + by drain chunks and part still attached +- **WHEN** a fold pass commits between chunks (attributing the still-attached + rows to the account) and the worker then completes finalization +- **THEN** no rollup table contains rows under the account's dimension +- **AND** (soft variant) the orphaned-deleted dimension carries the account's + complete folded history +- **AND** fold passes run after finalization add nothing under the account's + dimension + +### Requirement: Deletion is restart-safe, idempotent, and superseded by credential replacement + +All drain progress MUST live in the database so a worker restart resumes an +interrupted deletion with no separate recovery step. Repeat DELETE requests +for a marked account MUST succeed idempotently and MUST NOT change the +frozen `delete_history` choice (first request wins). The first-request-wins +invariant is scoped to replicas running this revision: during a rolling +deploy, a repeat DELETE routed to a pre-upgrade replica performs the legacy +synchronous delete with its caller-provided variant (exactly the pre-change +behavior — a complete, mirror-correct deletion whose variant choice may +differ from the frozen one). This window is bounded by the deploy itself, +requires the operator to issue contradictory repeat requests inside it, and +is accepted: new code cannot fence binaries that predate the marker, and a +deployment gate would add permanent configuration for a transient window. A credential +replacement (re-import or reauthentication landing on the marked row) MUST +clear the marker and supersede the deletion: every drain chunk and the +finalization transaction MUST re-check the marker under the account row lock +(PostgreSQL `FOR NO KEY UPDATE`; the SQLite writer section) before mutating +rows, so no chunk commits row work after a replacement committed and a +superseded account is never finalized (rows already drained stay detached). + +After a supersede that followed a partial drain, rows drained before the +replacement keep their drained end state (detached under the soft variant, +deleted under `delete_history`), and folded rollups keep attributing the +pre-supersede-drained traffic to the revived account: it is the account's +true pre-delete history, and reads MUST NOT double- or under-count as a +result (below-watermark reads are folded-only; drained rows above the +watermark fold exactly once, under the orphaned dimension). + +A credential replacement handled by a pre-upgrade replica during a rolling +deploy writes fresh credentials but cannot clear marker columns unknown to +its ORM. The worker MUST therefore treat non-wiped (or undecryptable) +ciphertext in ANY of the access/refresh/id token fields of a marked row as +a credential replacement (a legal replacement may carry an empty refresh +token while providing fresh access/id material): it MUST clear the marker +itself under the account row lock and abandon the deletion without mutating +any further rows. + +#### Scenario: Restart resumes a partial drain + +- **GIVEN** a marked account whose drain was interrupted after some chunks +- **WHEN** a fresh worker pass runs +- **THEN** the drain resumes from the database state and finalizes normally + +#### Scenario: Repeat delete does not escalate the variant + +- **GIVEN** an account marked by a request without `delete_history` +- **WHEN** a second `DELETE` request arrives with `delete_history=true` +- **THEN** the request succeeds and the frozen choice remains the soft variant + +#### Scenario: Re-import supersedes a pending deletion + +- **GIVEN** a marked account mid-drain +- **WHEN** a credential replacement lands on the row and clears the marker +- **THEN** the worker abandons the deletion without removing the account row +- **AND** rows detached before the replacement remain detached + +#### Scenario: Legacy-replica replacement supersedes without clearing the marker + +- **GIVEN** a marked account mid-drain whose credentials were replaced by a + pre-upgrade replica (fresh ciphertext, marker still set) +- **WHEN** the worker's next chunk or finalization re-checks the row +- **THEN** the worker clears the marker, abandons the deletion, and the + fresh credentials survive diff --git a/openspec/changes/background-account-deletion/tasks.md b/openspec/changes/background-account-deletion/tasks.md new file mode 100644 index 0000000000..246ec37486 --- /dev/null +++ b/openspec/changes/background-account-deletion/tasks.md @@ -0,0 +1,105 @@ +## 1. Schema + +- [x] 1.1 Add `accounts.delete_requested_at` and + `accounts.delete_history_requested` columns (model + Alembic revision + `20260816_000000_add_account_pending_deletion` on the current head, + guarded upgrade/downgrade). +- [x] 1.2 Partial queue index `idx_accounts_delete_requested_at` + (`(delete_requested_at, id) WHERE delete_requested_at IS NOT NULL`) so + the per-interval pending probe and the queue-order scan never touch the + full accounts table. + +## 2. Fast delete path + +- [x] 2.1 `AccountsRepository.begin_delete`: terminal `DEACTIVATED` mark + + pending marker + sticky/bridge cleanup in one short transaction; + idempotent, first request freezes the `delete_history` choice. +- [x] 2.2 Hide marked accounts from `list_accounts` / `list_accounts_by_ids`; + block reactivation of marked accounts; keep the DELETE response + contract (`{"status": "deleted"}`). +- [x] 2.3 Clear the marker in `_apply_account_updates` so credential + replacement supersedes a pending deletion. +- [x] 2.4 Hide marked accounts from the credential-export endpoints (account + export, auth export, opencode auth export): 404 during the drain + window, matching the synchronous delete's contract. +- [x] 2.5 Wipe the stored token ciphertext in `begin_delete` so readers that + do not know the marker (pre-upgrade replicas during a rolling deploy) + cannot export usable credentials mid-drain; backfill the non-secret + seat identity (`chatgpt_user_id`) from the id-token claims first so + targeted reauthentication can still verify and supersede. +- [x] 2.6 Remove the account's API-key assignments in `begin_delete` (the + projection the synchronous FK cascade produced): key listings and + pooled-usage reads exclude the account immediately, scope flag intact. +- [x] 2.7 Fence ordinary status writers (`update_status`, + `update_status_if_current`) on `delete_requested_at IS NULL` so stale + in-flight settlements cannot resurrect a marked account. +- [x] 2.8 Treat non-wiped credential ciphertext on a marked row as a + supersede (replacement by a pre-upgrade replica that cannot clear the + marker): chunks and finalization clear the marker and abandon. +- [x] 2.9 Reject marked accounts in API-key assignment validation + (`ApiKeysRepository.list_accounts_by_ids`) so post-DELETE key updates + cannot recreate assignments for the account. +- [x] 2.10 Atomic marker re-check in `replace_account_assignments` + (conditional INSERT…SELECT, `FOR SHARE` on PostgreSQL) so validation + that raced the DELETE cannot recreate an assignment. +- [x] 2.11 Finalization upgrades the account row to `FOR UPDATE` before the + residual sweeps (PostgreSQL) so in-flight FK inserts are either swept + or fail post-delete — no live orphans, no surviving history. +- [x] 2.12 Per-chunk self-heal of drift written by unfenced pre-upgrade + replicas: re-assert terminal status and re-remove recreated API-key + assignments under the chunk's row lock. +- [x] 2.13 Repeat DELETE short-circuits on an unlocked marker+wipe read so + the millisecond contract holds while a drain chunk holds the account + row lock / SQLite writer section. +- [x] 2.14 Treat marked accounts as absent on every ID-based account surface + (trends, reset-credit read/consume on both route families, probe, + pause, update, alias, limit-warmup, routing policy, upstream-proxy + binding, `/v1` reset-credit redemption) via a marker-aware fetch or an + atomic write predicate; filter the marker in the unscoped API-key pool + query (`list_all_accounts`) as well. +- [x] 2.15 Propagate the delete-request cache invalidation after a chunk + repairs pre-upgrade-replica drift, so cached drift stops being served. + +## 3. Background worker + +- [x] 3.1 `app/modules/accounts/deletion.py`: chunked drain + (usage_history, additional_usage_history, request_logs; 1k rows per + transaction, marker re-check under the account row lock per chunk, no + fold-state lock) for both variants; round-robin at most one nonempty + chunk per pending account per round with a pending re-scan between + rounds. +- [x] 3.1a Pin chunk batch selection to the account-leading indexes + (`account_id` range predicate + index-order ORDER BY → + `idx_usage_account_time`, `ix_additional_usage_distinct_labels`, + covering `idx_logs_account_kind_deleted_latest`), verified against the + production planner; skip re-probing tables observed empty within a + pass; pause between row-touching rounds proportionally to round + duration. +- [x] 3.2 Finalization via `AccountsRepository.delete(only_pending=True)`: + historical transaction shape (identity lock → fold-state lock → + residual rows → mirrors → sticky/rollup/account) plus marker guard and + persisted-variant read. +- [x] 3.3 Leader-gated scheduler (30 s tick, cheap pending pre-check before + leader election, local wake from the delete path), wired into the app + lifespan; post-finalization cache invalidation mirroring the old + synchronous path. + +## 4. Validation + +- [x] 4.1 Integration coverage: chunk-boundary drain (both variants), fold + pass interleaved between chunks (no folded-row resurrection, orphaned + dimension preserves history), restart resume, straggler row settled + mid-drain, repeat-request idempotency without variant escalation, + supersede by replacement (including the drain/finalize race), fast-path + API contract (immediate hide, 404 reactivate, 404 credential exports), + round-robin interleave and mid-pass pickup of newly marked accounts. +- [x] 4.2 Update the existing delete API tests to drive the worker pass; + keep the direct synchronous `AccountsRepository.delete` coverage. +- [x] 4.3 `ruff check` + `ruff format` + architecture checks + focused + account/rollup/migration test suites + strict OpenSpec validation. +- [x] 4.4 Alembic round-trip coverage for + `20260816_000000_add_account_pending_deletion` (parent -> revision -> + downgrade -> guarded upgrade -> head), wired into the PostgreSQL CI + target list; downgrade REFUSES while any deletion is queued (the + marker columns are the queue's only durable state) and the refusal is + covered by the round-trip test. diff --git a/tests/integration/test_account_deletion_background.py b/tests/integration/test_account_deletion_background.py new file mode 100644 index 0000000000..aa08fb2f76 --- /dev/null +++ b/tests/integration/test_account_deletion_background.py @@ -0,0 +1,1033 @@ +"""Background (chunked) account deletion: drain, fold interleave, restart, +idempotency, and supersede semantics for both delete_history variants.""" + +from __future__ import annotations + +import base64 +import json +from collections.abc import Callable +from datetime import timedelta +from typing import cast + +import pytest +from sqlalchemy import Table, func, select, text, update +from sqlalchemy.sql import Select + +from app.core.crypto import TokenEncryptor +from app.core.utils.time import utcnow +from app.db.models import ( + Account, + AccountStatus, + AccountUsageRollup, + RequestDemandQuarterRollup, + RequestLog, + RequestUsageHourlyRollup, + StickySession, + StickySessionKind, + UsageHistory, +) +from app.db.session import SessionLocal, get_background_session, sqlite_writer_section +from app.modules.accounts.deletion import _request_logs_chunk, run_account_deletion_pass +from app.modules.accounts.repository import ACCOUNT_PENDING_DELETION_REASON, AccountsRepository +from app.modules.accounts.usage_rollup import run_fold_pass +from app.modules.accounts.usage_time_rollup import run_hourly_fold_pass, to_dimension +from app.modules.request_logs.repository import RequestLogsRepository +from app.modules.usage.repository import UsageRepository + +pytestmark = pytest.mark.integration + +_ORPHAN_DIMENSION = to_dimension(None) + + +@pytest.fixture(autouse=True) +def _no_background_wake(monkeypatch): + """Keep the drain under explicit test control. + + The suite's stand-in leader election runs scheduler bodies inline, so the + delete API's worker wake would drain accounts concurrently with (and race) + the passes these tests drive step by step. The scheduler's own tick (one + pass at startup plus every interval) is neutralized as well: a tick firing + between a DELETE and the assertions would drain the account these tests + expect to still be marked. + """ + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) + + +def _make_account(account_id: str, email: str) -> Account: + encryptor = TokenEncryptor() + return Account( + id=account_id, + email=email, + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + + +async def _add_log(logs_repo: RequestLogsRepository, *, account_id: str, request_id: str, requested_at) -> None: + await logs_repo.add_log( + account_id=account_id, + request_id=request_id, + model="gpt-5.1-codex", + input_tokens=100, + output_tokens=50, + latency_ms=100, + status="success", + error_code=None, + requested_at=requested_at, + cost_usd=0.01, + ) + + +async def _seed_account(account_id: str, *, log_count: int, usage_count: int = 0, requested_at=None) -> None: + requested_at = requested_at or (utcnow() - timedelta(days=2)) + async with SessionLocal() as session: + accounts_repo = AccountsRepository(session) + logs_repo = RequestLogsRepository(session) + usage_repo = UsageRepository(session) + await accounts_repo.upsert(_make_account(account_id, f"{account_id}@example.com")) + for index in range(log_count): + await _add_log( + logs_repo, + account_id=account_id, + request_id=f"req_{account_id}_{index}", + requested_at=requested_at, + ) + for index in range(usage_count): + await usage_repo.add_entry(account_id, float(index), window="primary") + + +async def _account_row(account_id: str) -> Account | None: + async with SessionLocal() as session: + return await session.get(Account, account_id) + + +async def _attached_log_count(account_id: str) -> int: + async with SessionLocal() as session: + return ( + await session.execute(select(func.count(RequestLog.id)).where(RequestLog.account_id == account_id)) + ).scalar_one() + + +async def _log_rows(prefix: str) -> list[RequestLog]: + async with SessionLocal() as session: + return list( + (await session.execute(select(RequestLog).where(RequestLog.request_id.like(f"req_{prefix}%")))) + .scalars() + .all() + ) + + +async def _hourly_rows_for_dimension(dimension: str) -> list[RequestUsageHourlyRollup]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(RequestUsageHourlyRollup).where(RequestUsageHourlyRollup.account_id == dimension) + ) + ) + .scalars() + .all() + ) + + +async def _demand_rows_for_dimension(dimension: str) -> list[RequestDemandQuarterRollup]: + async with SessionLocal() as session: + return list( + ( + await session.execute( + select(RequestDemandQuarterRollup).where(RequestDemandQuarterRollup.account_id == dimension) + ) + ) + .scalars() + .all() + ) + + +async def _lifetime_rollup(account_id: str) -> AccountUsageRollup | None: + async with SessionLocal() as session: + return await session.get(AccountUsageRollup, account_id) + + +async def _run_one_detach_chunk(account_id: str, *, batch_size: int, delete_history: bool = False) -> int: + async with get_background_session() as session: + async with sqlite_writer_section(): + affected = await _request_logs_chunk( + session, account_id, delete_history=delete_history, batch_size=batch_size + ) + await session.commit() + return affected + + +@pytest.mark.asyncio +async def test_delete_api_marks_and_hides_immediately(async_client, db_setup): + await _seed_account("acc_bg_mark", log_count=2, usage_count=2) + + delete = await async_client.delete("/api/accounts/acc_bg_mark") + assert delete.status_code == 200 + assert delete.json()["status"] == "deleted" + + # Hidden from the listing immediately, before any background work ran. + accounts = await async_client.get("/api/accounts") + assert accounts.status_code == 200 + assert all(entry["accountId"] != "acc_bg_mark" for entry in accounts.json()["accounts"]) + + # The row itself survives, terminal and marked, until the worker drains it. + row = await _account_row("acc_bg_mark") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + assert await _attached_log_count("acc_bg_mark") == 2 + + # Repeat request is idempotent and does not escalate the frozen variant. + repeat = await async_client.delete("/api/accounts/acc_bg_mark?delete_history=true") + assert repeat.status_code == 200 + row = await _account_row("acc_bg_mark") + assert row is not None + assert row.delete_history_requested is False + + # A marked account is gone from the operator's perspective: reactivation + # reports not-found instead of racing the deletion worker. + reactivate = await async_client.post("/api/accounts/acc_bg_mark/reactivate") + assert reactivate.status_code == 404 + + # Credential exports must not keep serving decrypted tokens during the + # drain window: the synchronous delete 404'd here immediately. + for export_path in ("export", "export/auth", "export/opencode-auth"): + export = await async_client.post(f"/api/accounts/acc_bg_mark/{export_path}") + assert export.status_code == 404, export_path + + # Every other ID-based account route treats the marked row as gone too — + # the synchronous delete returned 404 on all of them once the row was + # removed. + base = "/api/accounts/acc_bg_mark" + assert (await async_client.get(f"{base}/trends")).status_code == 404 + assert (await async_client.get(f"{base}/usage-reset-credits")).status_code == 404 + assert (await async_client.post(f"{base}/usage-reset-credits/consume")).status_code == 404 + assert (await async_client.post(f"{base}/probe")).status_code == 404 + assert (await async_client.post(f"{base}/pause")).status_code == 404 + assert (await async_client.patch(base, json={"securityWorkAuthorized": True})).status_code == 404 + assert (await async_client.put(f"{base}/alias", json={"alias": "ghost"})).status_code == 404 + assert (await async_client.put(f"{base}/limit-warmup", json={"enabled": True})).status_code == 404 + assert (await async_client.put(f"{base}/routing-policy", json={"routingPolicy": "preserve"})).status_code == 404 + + +def _fake_id_token(payload: dict) -> str: + encoded = base64.urlsafe_b64encode(json.dumps(payload).encode()).decode().rstrip("=") + return f"header.{encoded}.signature" + + +@pytest.mark.asyncio +async def test_begin_delete_preserves_seat_identity_before_token_wipe(db_setup): + """Legacy rows carry their seat identity only inside the id-token claims; + targeted reauthentication (a supersede path) verifies the seat against + chatgpt_user_id or those claims, so the wipe must backfill the non-secret + identity first.""" + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = _make_account("acc_bg_seat", "acc_bg_seat@example.com") + assert account.chatgpt_user_id is None + account.id_token_encrypted = encryptor.encrypt(_fake_id_token({"sub": "user-legacy-seat"})) + await AccountsRepository(session).upsert(account) + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_seat") + + row = await _account_row("acc_bg_seat") + assert row is not None + # The ciphertext is wiped (no usable credentials remain on the row)... + assert encryptor.decrypt(row.id_token_encrypted) == "" + # ...but the seat identity survives in the non-secret column. + assert row.chatgpt_user_id == "user-legacy-seat" + + +@pytest.mark.asyncio +async def test_marked_account_wipes_tokens_and_rejects_stale_status_writes(db_setup): + await _seed_account("acc_bg_fence", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fence") + + # The surviving row must not carry usable credentials: readers that do + # not know the marker (pre-upgrade replicas during a rolling deploy) can + # only produce empty credentials from it. + row = await _account_row("acc_bg_fence") + assert row is not None + encryptor = TokenEncryptor() + assert encryptor.decrypt(row.access_token_encrypted) == "" + assert encryptor.decrypt(row.refresh_token_encrypted) == "" + assert encryptor.decrypt(row.id_token_encrypted) == "" + + # Stale in-flight settlements (e.g. a late 429 for a request selected + # before the DELETE) must not replace the terminal state and make the + # account selectable again mid-drain. + async with SessionLocal() as session: + repo = AccountsRepository(session) + assert await repo.update_status("acc_bg_fence", AccountStatus.RATE_LIMITED, "rate_limited") is False + assert ( + await repo.update_status_if_current( + "acc_bg_fence", + AccountStatus.RATE_LIMITED, + "rate_limited", + expected_status=AccountStatus.DEACTIVATED, + expected_deactivation_reason=ACCOUNT_PENDING_DELETION_REASON, + ) + is False + ) + row = await _account_row("acc_bg_fence") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + + +@pytest.mark.asyncio +async def test_chunked_soft_delete_drains_across_chunk_boundaries(db_setup): + await _seed_account("acc_bg_soft", log_count=7, usage_count=5) + async with SessionLocal() as session: + session.add( + StickySession( + key="sticky_bg_soft", + kind=StickySessionKind.CODEX_SESSION, + account_id="acc_bg_soft", + ) + ) + await session.commit() + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_soft") + + outcomes = await run_account_deletion_pass(batch_size=3) + assert outcomes == {"acc_bg_soft": "finalized"} + + assert await _account_row("acc_bg_soft") is None + logs = await _log_rows("acc_bg_soft") + assert len(logs) == 7 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + async with SessionLocal() as session: + usage_left = ( + await session.execute(select(func.count(UsageHistory.id)).where(UsageHistory.account_id == "acc_bg_soft")) + ).scalar_one() + sticky_left = ( + await session.execute( + select(func.count(StickySession.key)).where(StickySession.account_id == "acc_bg_soft") + ) + ).scalar_one() + assert usage_left == 0 + assert sticky_left == 0 + assert await _lifetime_rollup("acc_bg_soft") is None + + # Idempotent: a second pass finds nothing to do. + assert await run_account_deletion_pass(batch_size=3) == {} + + +@pytest.mark.asyncio +async def test_chunked_hard_delete_removes_history(db_setup): + await _seed_account("acc_bg_hard", log_count=5, usage_count=2) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_hard", delete_history=True) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_hard": "finalized"} + + assert await _account_row("acc_bg_hard") is None + assert await _log_rows("acc_bg_hard") == [] + + +@pytest.mark.asyncio +async def test_fold_interleaved_between_chunks_does_not_resurrect_soft(db_setup): + """A fold slice committing between detach chunks re-attributes still- + attached rows to the account; finalization's fold-locked mirrors must + move ALL of it to the orphaned-deleted dimension.""" + now = utcnow() + account_dimension = to_dimension("acc_bg_fold") + # Group A (2 rows) old enough for the first fold; group B (2 rows) folded + # only by the interleaved fold below. + await _seed_account("acc_bg_fold", log_count=2, requested_at=now - timedelta(days=5)) + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + for index in range(2): + await _add_log( + logs_repo, + account_id="acc_bg_fold", + request_id=f"req_acc_bg_fold_b{index}", + requested_at=now - timedelta(days=2), + ) + + # First fold covers only group A (target = now-3d - FOLD_LAG). + await run_fold_pass(now=now - timedelta(days=3)) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + assert await _hourly_rows_for_dimension(account_dimension) != [] + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fold") + + # One chunk detaches the two oldest (group A) rows; group B stays attached. + assert await _run_one_detach_chunk("acc_bg_fold", batch_size=2) == 2 + assert await _attached_log_count("acc_bg_fold") == 2 + + # Interleaved folds aggregate group B while it is still attributed. + await run_fold_pass(now=now) + await run_hourly_fold_pass(now=now) + assert await _lifetime_rollup("acc_bg_fold") is not None + interleaved_hourly = await _hourly_rows_for_dimension(account_dimension) + assert sum(row.request_count for row in interleaved_hourly if not row.is_deleted) >= 2 + + # Resume and finish the deletion. + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_fold": "finalized"} + + # No folded row anywhere still carries the account dimension... + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _demand_rows_for_dimension(account_dimension) == [] + assert await _lifetime_rollup("acc_bg_fold") is None + # ...and the orphaned-deleted dimension preserves the full folded history. + orphan_hourly = await _hourly_rows_for_dimension(_ORPHAN_DIMENSION) + assert sum(row.request_count for row in orphan_hourly if row.is_deleted) == 4 + logs = await _log_rows("acc_bg_fold") + assert len(logs) == 4 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + # Folds after finalization see only detached raw rows: nothing new may + # appear under the account dimension. + await run_hourly_fold_pass(now=now + timedelta(days=1)) + await run_fold_pass(now=now + timedelta(days=1)) + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _lifetime_rollup("acc_bg_fold") is None + + +@pytest.mark.asyncio +async def test_fold_interleaved_between_chunks_does_not_resurrect_hard(db_setup): + now = utcnow() + account_dimension = to_dimension("acc_bg_fhard") + await _seed_account("acc_bg_fhard", log_count=2, requested_at=now - timedelta(days=5)) + async with SessionLocal() as session: + logs_repo = RequestLogsRepository(session) + for index in range(2): + await _add_log( + logs_repo, + account_id="acc_bg_fhard", + request_id=f"req_acc_bg_fhard_b{index}", + requested_at=now - timedelta(days=2), + ) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_fhard", delete_history=True) + + assert await _run_one_detach_chunk("acc_bg_fhard", batch_size=2, delete_history=True) == 2 + await run_hourly_fold_pass(now=now) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_fhard": "finalized"} + + assert await _hourly_rows_for_dimension(account_dimension) == [] + assert await _demand_rows_for_dimension(account_dimension) == [] + assert await _log_rows("acc_bg_fhard") == [] + + +@pytest.mark.asyncio +async def test_pass_round_robins_chunks_across_pending_accounts(db_setup, monkeypatch): + """One account's long drain must not starve another: each round advances + every pending account by at most one full chunk.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_rr_a", log_count=3) + await _seed_account("acc_bg_rr_b", log_count=3) + async with SessionLocal() as session: + repo = AccountsRepository(session) + assert await repo.begin_delete("acc_bg_rr_a") + assert await repo.begin_delete("acc_bg_rr_b") + + chunk_calls: list[str] = [] + original_chunk = deletion._request_logs_chunk + + async def spy_chunk(session, account_id, *, delete_history, batch_size): + chunk_calls.append(account_id) + return await original_chunk(session, account_id, delete_history=delete_history, batch_size=batch_size) + + monkeypatch.setattr(deletion, "_request_logs_chunk", spy_chunk) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_rr_a": "finalized", "acc_bg_rr_b": "finalized"} + # Full chunks alternate between the two accounts instead of draining one + # account to completion first. + assert chunk_calls[:6] == [ + "acc_bg_rr_a", + "acc_bg_rr_b", + "acc_bg_rr_a", + "acc_bg_rr_b", + "acc_bg_rr_a", + "acc_bg_rr_b", + ] + assert await _account_row("acc_bg_rr_a") is None + assert await _account_row("acc_bg_rr_b") is None + + +@pytest.mark.asyncio +async def test_pass_picks_up_account_marked_mid_pass(db_setup, monkeypatch): + """A DELETE that lands while a pass is draining another account is picked + up by the between-rounds re-scan, not deferred to the next tick.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_mid_a", log_count=2) + await _seed_account("acc_bg_mid_b", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_mid_a") + + original_advance = deletion._advance_account + marked_second = False + + async def advance_and_mark(account_id, *, batch_size, drained=None): + nonlocal marked_second + if not marked_second: + marked_second = True + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_mid_b") + return await original_advance(account_id, batch_size=batch_size, drained=drained) + + monkeypatch.setattr(deletion, "_advance_account", advance_and_mark) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_mid_a": "finalized", "acc_bg_mid_b": "finalized"} + assert await _account_row("acc_bg_mid_a") is None + assert await _account_row("acc_bg_mid_b") is None + + +@pytest.mark.asyncio +async def test_restart_resumes_partial_drain(db_setup): + await _seed_account("acc_bg_resume", log_count=5, usage_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_resume") + + # Simulate a crash after one detach chunk: progress lives in the rows. + assert await _run_one_detach_chunk("acc_bg_resume", batch_size=2) == 2 + assert await _attached_log_count("acc_bg_resume") == 3 + + # A fresh pass (restarted leader) resumes from the database state. + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_resume": "finalized"} + assert await _account_row("acc_bg_resume") is None + logs = await _log_rows("acc_bg_resume") + assert len(logs) == 5 + assert all(row.account_id is None for row in logs) + + +@pytest.mark.asyncio +async def test_straggler_row_settled_mid_drain_is_finalized(db_setup): + """A stream that settles its request-log row after the drain chunks ran + is caught by finalization's residual sweep.""" + await _seed_account("acc_bg_late", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_late") + assert await _run_one_detach_chunk("acc_bg_late", batch_size=10) == 3 + + async with SessionLocal() as session: + await _add_log( + RequestLogsRepository(session), + account_id="acc_bg_late", + request_id="req_acc_bg_late_straggler", + requested_at=utcnow() - timedelta(hours=1), + ) + + outcomes = await run_account_deletion_pass(batch_size=10) + assert outcomes == {"acc_bg_late": "finalized"} + logs = await _log_rows("acc_bg_late") + assert len(logs) == 4 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_credential_replacement_supersedes_pending_deletion(db_setup): + await _seed_account("acc_bg_super", log_count=4) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_super") + assert await _run_one_detach_chunk("acc_bg_super", batch_size=2) == 2 + + # Re-import lands on the same row via the slot-identity path and clears + # the marker (credential replacement supersedes the deletion). + async with SessionLocal() as session: + replacement = _make_account("acc_bg_super", "acc_bg_super@example.com") + saved = await AccountsRepository(session).upsert(replacement, merge_by_email=True) + assert saved.id == "acc_bg_super" + + row = await _account_row("acc_bg_super") + assert row is not None + assert row.delete_requested_at is None + assert row.status is AccountStatus.ACTIVE + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {} + assert await _account_row("acc_bg_super") is not None + # Rows detached before the supersede stay detached; the rest survive. + assert await _attached_log_count("acc_bg_super") == 2 + + +async def _legacy_replace_credentials(account_id: str, encryptor: TokenEncryptor) -> None: + """Mimic a credential replacement by a pre-upgrade replica: fresh + ciphertext and status, but the marker columns its ORM does not know stay + untouched.""" + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == account_id) + .values( + access_token_encrypted=encryptor.encrypt("fresh-access"), + refresh_token_encrypted=encryptor.encrypt("fresh-refresh"), + id_token_encrypted=encryptor.encrypt("fresh-id"), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_legacy_replica_replacement_supersedes_mid_drain(db_setup): + """A replacement handled by a pre-upgrade replica cannot clear the marker; + fresh (non-wiped) ciphertext on a marked row must itself supersede.""" + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy") + assert await _run_one_detach_chunk("acc_bg_legacy", batch_size=2) == 2 + + await _legacy_replace_credentials("acc_bg_legacy", encryptor) + + outcomes = await run_account_deletion_pass(batch_size=2) + assert outcomes == {"acc_bg_legacy": "superseded"} + row = await _account_row("acc_bg_legacy") + assert row is not None + # The worker cleared the marker itself and preserved the fresh material. + assert row.delete_requested_at is None + assert encryptor.decrypt(row.refresh_token_encrypted) == "fresh-refresh" + # Rows detached before the replacement stay detached; the rest survive. + assert await _attached_log_count("acc_bg_legacy") == 1 + # The account is no longer rescanned on later passes. + assert await run_account_deletion_pass(batch_size=2) == {} + + +@pytest.mark.asyncio +async def test_legacy_replacement_with_empty_refresh_token_supersedes(db_setup): + """A legal replacement may carry an empty refresh token while providing + fresh access/id material; a refresh-only wipe check would mistake it for + the original wipe and finalize the freshly replaced account.""" + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy_er", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy_er") + assert await _run_one_detach_chunk("acc_bg_legacy_er", batch_size=10) == 1 + + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_legacy_er") + .values( + access_token_encrypted=encryptor.encrypt("fresh-access"), + refresh_token_encrypted=encryptor.encrypt(""), + id_token_encrypted=encryptor.encrypt("fresh-id"), + status=AccountStatus.ACTIVE, + deactivation_reason=None, + ) + ) + await session.commit() + + outcomes = await run_account_deletion_pass(batch_size=10) + assert outcomes == {"acc_bg_legacy_er": "superseded"} + row = await _account_row("acc_bg_legacy_er") + assert row is not None + assert row.delete_requested_at is None + assert encryptor.decrypt(row.access_token_encrypted) == "fresh-access" + + +@pytest.mark.asyncio +async def test_legacy_replica_replacement_before_finalize_is_abandoned(db_setup): + encryptor = TokenEncryptor() + await _seed_account("acc_bg_legacy_fin", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_legacy_fin") + assert await _run_one_detach_chunk("acc_bg_legacy_fin", batch_size=10) == 1 + + await _legacy_replace_credentials("acc_bg_legacy_fin", encryptor) + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_bg_legacy_fin", only_pending=True) is False + row = await _account_row("acc_bg_legacy_fin") + assert row is not None + assert row.delete_requested_at is None + assert encryptor.decrypt(row.refresh_token_encrypted) == "fresh-refresh" + + +@pytest.mark.asyncio +async def test_repeat_delete_short_circuits_without_waiting_on_chunk_lock(db_setup): + """A repeat DELETE must keep the millisecond contract even while a drain + chunk transaction holds the account row lock.""" + import asyncio + + async with SessionLocal() as probe: + if probe.get_bind().dialect.name != "postgresql": + pytest.skip("row-lock wait behavior is PostgreSQL-specific") + + await _seed_account("acc_bg_repeat", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_repeat") + + async with SessionLocal() as locker: + # Hold the same lock a drain chunk holds for its whole transaction. + await locker.execute(select(Account.id).where(Account.id == "acc_bg_repeat").with_for_update(key_share=True)) + async with SessionLocal() as session: + repeat = await asyncio.wait_for(AccountsRepository(session).begin_delete("acc_bg_repeat"), timeout=2.0) + assert repeat is True + await locker.rollback() + + +@pytest.mark.asyncio +async def test_chunk_self_heals_drift_from_unfenced_replicas(db_setup): + """During a rolling deploy, pre-upgrade replicas' unfenced writers can + replace the terminal status or recreate API-key assignments on a marked + row; the next chunk transaction must re-fence both.""" + from app.db.models import ApiKey, ApiKeyAccountAssignment + from app.modules.accounts import deletion + + await _seed_account("acc_bg_heal", log_count=2) + async with SessionLocal() as session: + session.add( + ApiKey( + id="key_bg_heal", + name="heal-key", + key_hash="hash_bg_heal", + key_prefix="sk-heal", + account_assignment_scope_enabled=True, + ) + ) + await session.commit() + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_heal") + + # Old-replica drift: unfenced status write + unconditional assignment + # insert (tokens stay wiped, so this is NOT a credential replacement). + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_heal") + .values(status=AccountStatus.RATE_LIMITED, deactivation_reason="rate_limited") + ) + session.add(ApiKeyAccountAssignment(api_key_id="key_bg_heal", account_id="acc_bg_heal")) + await session.commit() + + affected = await deletion._run_chunk(deletion._usage_history_chunk, "acc_bg_heal", batch_size=10) + assert affected is not None + + row = await _account_row("acc_bg_heal") + assert row is not None + assert row.status is AccountStatus.DEACTIVATED + assert row.deactivation_reason == ACCOUNT_PENDING_DELETION_REASON + assert row.delete_requested_at is not None + async with SessionLocal() as session: + assigned = ( + await session.execute( + select(func.count()) + .select_from(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.account_id == "acc_bg_heal") + ) + ).scalar_one() + assert assigned == 0 + + +@pytest.mark.asyncio +async def test_finalization_serializes_against_inflight_log_insert(db_setup): + """PostgreSQL: an in-flight stream's request-log insert holds the FK KEY + SHARE on the account row; finalization's FOR UPDATE row upgrade must wait + for it, so the late row is swept instead of surviving as a live orphan + via ON DELETE SET NULL.""" + import asyncio + + async with SessionLocal() as probe: + if probe.get_bind().dialect.name != "postgresql": + pytest.skip("FK KEY SHARE / FOR UPDATE interleaving is PostgreSQL-specific") + + await _seed_account("acc_bg_inflight", log_count=1) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_inflight") + assert await _run_one_detach_chunk("acc_bg_inflight", batch_size=10) == 1 + + async with SessionLocal() as inflight: + # In-flight stream: the insert takes (and holds) FK KEY SHARE on the + # account row until commit. + inflight.add( + RequestLog( + account_id="acc_bg_inflight", + request_id="req_acc_bg_inflight_late", + requested_at=utcnow(), + model="gpt-5.1-codex", + status="success", + input_tokens=1, + output_tokens=1, + cost_usd=0.0, + ) + ) + await inflight.flush() + + pass_task = asyncio.create_task(run_account_deletion_pass(batch_size=10)) + # Finalization must block on the row upgrade while the insert is open. + done, _ = await asyncio.wait({pass_task}, timeout=1.0) + commit_first = not done + await inflight.commit() + outcomes = await pass_task + + assert commit_first, "finalization finished while an uncommitted FK insert held KEY SHARE" + assert outcomes == {"acc_bg_inflight": "finalized"} + assert await _account_row("acc_bg_inflight") is None + logs = await _log_rows("acc_bg_inflight") + assert len(logs) == 2 + # The late row was swept by the residual sweep, not orphaned live. + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_assignment_insert_rechecks_marker_atomically(db_setup): + """replace_account_assignments must skip marked accounts even when an + earlier validation (different transaction) still believed they existed.""" + from app.db.models import ApiKey, ApiKeyAccountAssignment + from app.modules.api_keys.repository import ApiKeysRepository + + await _seed_account("acc_bg_atomic", log_count=0) + async with SessionLocal() as session: + session.add( + ApiKey( + id="key_bg_atomic", + name="atomic-recheck-key", + key_hash="hash_bg_atomic", + key_prefix="sk-atomic", + account_assignment_scope_enabled=True, + ) + ) + await session.commit() + + # DELETE lands after validation would have passed. + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_atomic") + + async with SessionLocal() as session: + await ApiKeysRepository(session).replace_account_assignments("key_bg_atomic", ["acc_bg_atomic"]) + + async with SessionLocal() as session: + assigned = ( + await session.execute( + select(func.count()) + .select_from(ApiKeyAccountAssignment) + .where(ApiKeyAccountAssignment.api_key_id == "key_bg_atomic") + ) + ).scalar_one() + assert assigned == 0 + + +@pytest.mark.asyncio +async def test_marked_account_cannot_be_assigned_to_api_key(async_client, db_setup): + await _seed_account("acc_bg_assign", log_count=1) + + delete = await async_client.delete("/api/accounts/acc_bg_assign") + assert delete.status_code == 200 + + # A key update racing (or following) the DELETE must not recreate an + # assignment that would re-surface the deleted account in key listings. + create = await async_client.post("/api/api-keys/", json={"name": "post-delete-key"}) + assert create.status_code == 200 + key_id = create.json()["id"] + update_resp = await async_client.patch( + f"/api/api-keys/{key_id}", + json={"assignedAccountIds": ["acc_bg_assign"]}, + ) + assert update_resp.status_code == 400 + assert update_resp.json()["error"]["code"] == "invalid_api_key_payload" + + +@pytest.mark.asyncio +async def test_supersede_after_partial_drain_preserves_folded_attribution(db_setup): + """Rows drained before a supersede stay drained, and folded rollups keep + attributing that traffic to the revived account — with no double count + from later folds (drained below-watermark rows are never re-folded).""" + now = utcnow() + account_dimension = to_dimension("acc_bg_sfold") + await _seed_account("acc_bg_sfold", log_count=2, requested_at=now - timedelta(days=5)) + + # Fold the two rows under the account dimension first. + await run_fold_pass(now=now - timedelta(days=3)) + await run_hourly_fold_pass(now=now - timedelta(days=3)) + folded_before = sum(row.request_count for row in await _hourly_rows_for_dimension(account_dimension)) + assert folded_before == 2 + + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_sfold") + # One chunk detaches both already-folded rows. + assert await _run_one_detach_chunk("acc_bg_sfold", batch_size=10) == 2 + + # Re-import supersedes before finalization ever runs. + async with SessionLocal() as session: + saved = await AccountsRepository(session).upsert( + _make_account("acc_bg_sfold", "acc_bg_sfold@example.com"), merge_by_email=True + ) + assert saved.id == "acc_bg_sfold" + assert await run_account_deletion_pass(batch_size=10) == {} + + # Folded attribution is the permanent end state: unchanged by later + # folds (no loss, no double count), while raw rows stay detached. + await run_fold_pass(now=now) + await run_hourly_fold_pass(now=now) + folded_after = sum(row.request_count for row in await _hourly_rows_for_dimension(account_dimension)) + assert folded_after == folded_before + assert await _lifetime_rollup("acc_bg_sfold") is not None + logs = await _log_rows("acc_bg_sfold") + assert len(logs) == 2 + assert all(row.account_id is None and row.deleted_at is not None for row in logs) + + +@pytest.mark.asyncio +async def test_supersede_between_drain_and_finalize_is_abandoned(db_setup): + await _seed_account("acc_bg_race", log_count=2) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_race") + assert await _run_one_detach_chunk("acc_bg_race", batch_size=10) == 2 + + # Marker cleared right before finalization (replacement won the race). + async with SessionLocal() as session: + await session.execute( + update(Account) + .where(Account.id == "acc_bg_race") + .values(delete_requested_at=None, delete_history_requested=False) + ) + await session.commit() + + async with SessionLocal() as session: + assert await AccountsRepository(session).delete("acc_bg_race", only_pending=True) is False + assert await _account_row("acc_bg_race") is not None + + +def _batch_pinning_cases() -> tuple[tuple[Callable[[str, int], Select[tuple[int]]], Table, str], ...]: + from app.db.models import AdditionalUsageHistory + from app.modules.accounts import deletion + + return ( + (deletion._usage_history_batch, cast("Table", UsageHistory.__table__), "idx_usage_account_time"), + ( + deletion._additional_usage_history_batch, + cast("Table", AdditionalUsageHistory.__table__), + "ix_additional_usage_distinct_labels", + ), + (deletion._request_logs_batch, cast("Table", RequestLog.__table__), "idx_logs_account_kind_deleted_latest"), + ) + + +def test_chunk_batch_statements_pin_account_leading_index_order(): + """The chunk batch shape is what keeps the planner off sequential scans. + + An ``account_id = :id LIMIT n`` subquery plans as a LIMIT-terminated Seq + Scan on the production planner for exactly the large accounts the drain + targets (equality folds account_id out of the sort pathkeys). The batch + builders must keep (a) the range predicate pair (never plain equality) + and (b) an ORDER BY that lists the target account-leading index's exact + column order, so that index is the only sort-free plan. + """ + from sqlalchemy.dialects import postgresql as postgresql_dialect + + for batch_fn, table, index_name in _batch_pinning_cases(): + index = next(idx for idx in table.indexes if idx.name == index_name) + sql = str(batch_fn("acc_bg_pin", 50).compile(dialect=postgresql_dialect.dialect())) + expected_order = ", ".join(f"{table.name}.{column.name}" for column in index.columns) + assert f"ORDER BY {expected_order}" in sql, sql + assert f"{table.name}.account_id >= " in sql, sql + assert f"{table.name}.account_id <= " in sql, sql + assert f"{table.name}.account_id = " not in sql, sql + + +@pytest.mark.asyncio +async def test_chunk_batch_query_plan_uses_account_leading_indexes_postgresql(db_setup): + """The batch ORDER BY must be served by the pinned index, not a sort. + + Sequential/bitmap scans and (incremental) sorts are disabled so the + planner has to surface an ordered index path for the batch shape; the + only index that can provide the ORDER BY after the leading account_id + range is the pinned account-leading index. A drained (or missing) + account's probe must terminate on the same index instead of falling + back to a heap scan. + """ + await _seed_account("acc_bg_plan", log_count=8, usage_count=8) + async with SessionLocal() as session: + if session.get_bind().dialect.name != "postgresql": + pytest.skip("PostgreSQL-only query plan test") + + from app.db.models import AdditionalUsageHistory + + session.add_all( + AdditionalUsageHistory( + account_id="acc_bg_plan", + quota_key="codex_spark", + limit_name="GPT-5.3-Codex-Spark", + metered_feature="codex_bengalfox", + window="primary", + used_percent=float(index), + ) + for index in range(8) + ) + await session.commit() + + await session.execute(text("SET enable_seqscan = off")) + await session.execute(text("SET enable_bitmapscan = off")) + await session.execute(text("SET enable_sort = off")) + await session.execute(text("SET enable_incremental_sort = off")) + for batch_fn, _table, index_name in _batch_pinning_cases(): + for account_id in ("acc_bg_plan", "acc_bg_plan_drained_probe"): + compiled = batch_fn(account_id, 5).compile( + dialect=session.get_bind().dialect, compile_kwargs={"literal_binds": True} + ) + plan = (await session.execute(text(f"EXPLAIN (FORMAT JSON) {compiled}"))).scalar_one() + plan_json = json.dumps(plan) + assert index_name in plan_json, (account_id, plan_json) + assert "Seq Scan" not in plan_json, (account_id, plan_json) + assert "Sort Key" not in plan_json, (account_id, plan_json) + + +@pytest.mark.asyncio +async def test_pass_probes_drained_tables_once_per_pass(db_setup, monkeypatch): + """A table observed empty for an account is not re-probed on later rounds + of the same pass: each probe is a full account-row-locking transaction, + and per-account statistics can go stale exactly during the churn window.""" + from app.modules.accounts import deletion + + await _seed_account("acc_bg_memo", log_count=3) + async with SessionLocal() as session: + assert await AccountsRepository(session).begin_delete("acc_bg_memo") + + calls = {"usage_history": 0, "additional_usage_history": 0, "request_logs": 0} + for attr, key in ( + ("_usage_history_chunk", "usage_history"), + ("_additional_usage_history_chunk", "additional_usage_history"), + ("_request_logs_chunk", "request_logs"), + ): + original = getattr(deletion, attr) + + def _make_spy(original=original, key=key): + async def spy(session, account_id, *, delete_history, batch_size): + calls[key] += 1 + return await original(session, account_id, delete_history=delete_history, batch_size=batch_size) + + return spy + + monkeypatch.setattr(deletion, attr, _make_spy()) + + outcomes = await run_account_deletion_pass(batch_size=1) + assert outcomes == {"acc_bg_memo": "finalized"} + # usage tables: exactly one (empty) probe in round 1, then skipped while + # rounds 2-4 drain the logs; request_logs: three one-row chunks plus the + # final empty probe that lets the pass finalize. + assert calls == {"usage_history": 1, "additional_usage_history": 1, "request_logs": 4} + assert await _account_row("acc_bg_memo") is None diff --git a/tests/integration/test_accounts_api_extended.py b/tests/integration/test_accounts_api_extended.py index 144bcc9a0d..3e39c84d43 100644 --- a/tests/integration/test_accounts_api_extended.py +++ b/tests/integration/test_accounts_api_extended.py @@ -13,6 +13,7 @@ from app.core.utils.time import naive_utc_to_epoch, utcnow from app.db.models import Account, AccountStatus, RequestLog from app.db.session import SessionLocal +from app.modules.accounts.deletion import run_account_deletion_pass from app.modules.accounts.repository import AccountsRepository from app.modules.proxy.account_cache import clear_account_routing_unavailable, is_account_routing_unavailable from app.modules.request_logs.repository import RequestLogsRepository @@ -592,7 +593,17 @@ async def test_delete_account_removes_from_list(async_client): @pytest.mark.asyncio -async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): +async def test_delete_account_soft_deletes_request_logs(async_client, db_setup, monkeypatch): + # The suite's inline leader election would let the API's worker wake race + # the explicit pass below; keep the drain under test control. The + # scheduler's own startup/interval tick is neutralized for the same + # reason. + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) async with SessionLocal() as session: accounts_repo = AccountsRepository(session) logs_repo = RequestLogsRepository(session) @@ -612,6 +623,10 @@ async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): delete = await async_client.delete("/api/accounts/acc_delete_logs") assert delete.status_code == 200 + # The API only marks the account; the background worker drains the rows. + outcomes = await run_account_deletion_pass() + assert outcomes["acc_delete_logs"] == "finalized" + async with SessionLocal() as session: row = ( await session.execute(select(RequestLog).where(RequestLog.request_id == "req_delete_logs_1")) @@ -629,7 +644,17 @@ async def test_delete_account_soft_deletes_request_logs(async_client, db_setup): @pytest.mark.asyncio -async def test_delete_account_with_delete_history_hard_deletes_request_logs(async_client, db_setup): +async def test_delete_account_with_delete_history_hard_deletes_request_logs(async_client, db_setup, monkeypatch): + # The suite's inline leader election would let the API's worker wake race + # the explicit pass below; keep the drain under test control. The + # scheduler's own startup/interval tick is neutralized for the same + # reason. + monkeypatch.setattr("app.modules.accounts.service.request_account_deletion_run", lambda: None) + + async def _no_tick(self) -> None: + return None + + monkeypatch.setattr("app.modules.accounts.deletion.AccountDeletionScheduler._run_once", _no_tick) async with SessionLocal() as session: accounts_repo = AccountsRepository(session) logs_repo = RequestLogsRepository(session) @@ -650,6 +675,10 @@ async def test_delete_account_with_delete_history_hard_deletes_request_logs(asyn assert delete.status_code == 200 assert delete.json()["status"] == "deleted" + # The API only marks the account; the background worker drains the rows. + outcomes = await run_account_deletion_pass() + assert outcomes["acc_hard_delete"] == "finalized" + async with SessionLocal() as session: result = await session.execute(select(RequestLog).where(RequestLog.request_id == "req_hard_delete_1")) assert result.scalar_one_or_none() is None diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index 253525b7fc..df70df1b78 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1352,6 +1352,89 @@ async def _usage_history_reloptions() -> set[str]: assert expected_options <= await _usage_history_reloptions() +@pytest.mark.asyncio +async def test_account_pending_deletion_migration_upgrade_and_downgrade(tmp_path): + """Round-trip the pending-deletion marker migration through Alembic: + parent -> revision adds the two guarded marker columns and the partial + queue index, downgrade removes all three, the guarded upgrade tolerates + pre-existing columns, and an upgrade to head proves the revision sits on + the single-head path.""" + from alembic import command + from sqlalchemy import inspect as sa_inspect + + from app.db.migrate import _build_alembic_config + + db_url = f"sqlite+aiosqlite:///{tmp_path / 'account-pending-deletion.sqlite'}" + parent_revision = "20260812_120000_add_sticky_abandonment_scope" + pending_deletion_revision = "20260816_000000_add_account_pending_deletion" + marker_columns = {"delete_requested_at", "delete_history_requested"} + index_name = "idx_accounts_delete_requested_at" + + def _schema_state(sync_conn): + inspector = sa_inspect(sync_conn) + columns = {column["name"] for column in inspector.get_columns("accounts")} + indexes = {index["name"] for index in inspector.get_indexes("accounts")} + return {"columns": columns & marker_columns, "index_present": index_name in indexes} + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent_revision, bootstrap_legacy=False)) + engine = create_async_engine(db_url, future=True) + try: + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": set(), "index_present": False} + + await to_thread.run_sync(lambda: run_upgrade(db_url, pending_deletion_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + + # Downgrade refuses while a deletion is queued: the marker columns are + # the queue's only durable state, and dropping them would silently + # abandon an acknowledged deletion. + async with engine.begin() as conn: + await conn.execute( + text( + "INSERT INTO accounts (id, codex_installation_id, email, plan_type, " + "access_token_encrypted, refresh_token_encrypted, id_token_encrypted, " + "last_refresh, status, delete_requested_at, delete_history_requested) " + "VALUES ('acc_mig_pending', 'install-mig-pending', 'mig@example.com', 'plus', " + "X'00', X'00', X'00', '2026-08-16 00:00:00', 'deactivated', " + "'2026-08-16 00:00:00', 0)" + ) + ) + with pytest.raises(Exception, match="queued for"): + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent_revision)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + async with engine.begin() as conn: + await conn.execute(text("DELETE FROM accounts WHERE id = 'acc_mig_pending'")) + + await to_thread.run_sync(lambda: command.downgrade(_build_alembic_config(db_url), parent_revision)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": set(), "index_present": False} + + # Guarded upgrade: a database where the columns already exist (e.g. a + # pre-merge build of this revision) must upgrade cleanly and still + # create the missing index. + async with engine.begin() as conn: + await conn.execute(text("ALTER TABLE accounts ADD COLUMN delete_requested_at DATETIME")) + await to_thread.run_sync(lambda: run_upgrade(db_url, pending_deletion_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + + # Single-head path: upgrading to head from here must succeed and keep + # the marker schema in place. + await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state == {"columns": marker_columns, "index_present": True} + finally: + await engine.dispose() + + @pytest.mark.asyncio async def test_account_plan_downgrade_observations_migration_upgrade_and_downgrade(tmp_path): """Round-trip the plan-downgrade evidence migration through Alembic itself. diff --git a/tests/unit/test_accounts_service_transitions.py b/tests/unit/test_accounts_service_transitions.py index db7f318863..10888c1eeb 100644 --- a/tests/unit/test_accounts_service_transitions.py +++ b/tests/unit/test_accounts_service_transitions.py @@ -26,6 +26,7 @@ def _account( deactivation_reason=deactivation_reason, reset_at=reset_at, blocked_at=blocked_at, + delete_requested_at=None, ) From 66fd1033165133e943a5818d75c40bc86e4a6b49 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:38:15 +0900 Subject: [PATCH 060/117] fix(usage): fence leaked live-usage-ingestor tasks and settle their failures deterministically (#1783) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: fence leaked live-usage-ingestor tasks at every test boundary The unit suite runs on a session-scoped asyncio loop (asyncio_default_test_loop_scope = "session"), so a background task leaked by one test survives into every later test. Tests that enter the real app.main lifespan start the module-global live-usage ingestor (app/modules/usage/live_ingest._ingestor) whose "live-usage-ingestor" consumer task lands on that shared loop; if the lifespan is cancelled before its shutdown path reaches stop_live_usage_ingestor() — e.g. a wait_for-bounded assertion times out mid-drain — the consumer outlives the test. The zombie then poisons unrelated tests: it eats into the otel lifespan-drain test's 5s budget and surfaces as an unobserved-task exception inside test_proxy_utils' startup-probe loop-exception assertions, the exact failing pairing from #1755. Add an autouse teardown fence in tests/conftest.py that awaits stop_live_usage_ingestor() after every test (resetting the singleton and unregistering the live-usage publisher) and then reaps any "live-usage-ingestor" task the stop path no longer tracks (a stop that was itself cancelled between clearing the global and awaiting the task). Add an order-dependent regression pair pinning the fence: the first test starts the ingestor exactly like the lifespan does and deliberately never stops it; the second asserts no consumer task, singleton, or publisher crossed the boundary. Without the fence the second test fails with a pending live-usage-ingestor task. Fixes #1755 Co-Authored-By: Claude Fable 5 * test: detect leaks synchronously and consume dead-task exceptions in the fence Round-2 hardening of the live-usage ingestor fence: - The fence is now a sync fixture that only enters the shared loop when a leak is actually present. An async fixture's teardown spins the session loop after every test, and the loop clock calls time.monotonic(), which several tests monkeypatch globally with finite fakes still active during function-scoped teardown (test_conversation_archive's exhausting iterator fails with StopIteration under the async variant). - The reap consumes exceptions from tasks that already died instead of re-raising mid-cleanup, reports them via pytest.fail at the leaking test, and sweeps orphaned live-usage-ingestor tasks no singleton tracks (a stop cancelled between clearing the global and awaiting the task). - A session-scoped capture fixture exposes the session loop to the sync teardown, since pytest-asyncio has no public accessor. Adds direct regression tests for the failed-consumer consumption and the orphan sweep. Co-Authored-By: Claude Fable 5 * test: sweep orphaned ingestor tasks even when singleton globals are clear Codex review round-2 findings: - The fence's fast path treated cleared module globals as clean, skipping the name-based sweep for the orphan-consumer state (a stop cancelled between clearing the global and awaiting the task). Leak detection now also enumerates pending ingestor-owned tasks on the idle session loop (asyncio.all_tasks(loop) is loop-passive) before deciding to skip. - The trailing cache-invalidation task was created unnamed, so the sweep could not reclaim it once orphaned and it could mutate shared account/header caches in a later test. Name it "live-usage-trailing-invalidation" at creation and reap both ingestor-owned task names. Co-Authored-By: Claude Fable 5 * test: consume exceptions from dead detached ingestor tasks via weak registry Codex review round-3 finding: asyncio.all_tasks() returns only unfinished tasks, so an ingestor-owned task that dies with an exception after the singleton and its task fields are cleared was invisible to the fence's pending sweep — its unretrieved exception would fire the loop exception handler when the task object is garbage-collected inside a later test, which is the reported #1755 poisoning shape. Give live_ingest a weak ownership registry (_owned_tasks) enrolling every task any ingestor creates; weak references never extend task lifetime. The fence walks it loop-free after every test: Task.exception() both reports the failure at the test that leaked it and marks it retrieved so no unobserved-task warning can fire later. The async reap now defers all failure reporting to that single pass, so a task reaped while pending and a task found dead are reported exactly once through the same path. Co-Authored-By: Claude Fable 5 * fix(usage): settle ingestor task failures deterministically at completion Codex review round-4 finding: the weak ownership registry alone cannot guarantee a fully detached dead task survives until the fence inspects it, leaving the settlement dependent on GC timing. Attach a done callback to every ingestor-owned task at creation that retrieves the task's exception the moment it completes — so the loop's unobserved-task warning can never fire at garbage-collection time in an unrelated test — logs it, and records it into a bounded strong failure handoff (live_ingest._owned_task_failures). The test fence drains that handoff loop-free after every test and still sweeps the weak registry for tasks whose callback is queued because they finished in the loop's final iteration. A settled-task weak set gates recording so the callback and the sweep settle each task exactly once. This is also a small production hardening: an unexpected consumer or trailing-invalidation death is now logged immediately with its traceback instead of surfacing as a nondeterministic "Task exception was never retrieved" at GC. Co-Authored-By: Claude Fable 5 * docs(openspec): add settle-live-ingest-task-failures change Codex review round-5 finding: the done-callback settlement changes production task-failure behavior, which the repository's OpenSpec hard gate requires an openspec/changes artifact for. Adds proposal, tasks, and the live-usage-ingestion spec delta; validated with openspec validate --strict. Co-Authored-By: Claude Fable 5 * fix(usage): record task failures as traceback-free metadata Codex review round-6 finding: storing raw exception objects preserves tracebacks whose frames retain the failed ingestor's object graph (task, queue, cached state) for the process lifetime, since production never drains the failure record. Store (task name, exception repr) instead; the full traceback is still logged at settlement time. Co-Authored-By: Claude Fable 5 * fix(usage): scope ingestor lifecycle to the lifespan instance The integration-core failures ("cannot reuse already awaited coroutine", teardown-attributed by the new leak fence) came from nested app lifespans: tests that open a portal-loop TestClient inside an app already running on the suite's session loop. start_live_usage_ingestor() unconditionally overwrote the module-global singleton — the outer ingestor's only strong root — leaving it an unreferenced cycle (consumer task -> coroutine frame -> ingestor -> queue -> getter future -> task). The cyclic GC then finalized the cycle mid-await, and the session loop stepping the half-collected task raised RuntimeError('cannot reuse already awaited coroutine') — the exact unobserved-exception flake behind issue #1755. The nested shutdown also cleared the global, so the outer shutdown stopped nothing and the consumer leaked across the test boundary. Fix at the root: - The lifespan holds the instance start returned (keeping it strongly rooted) and passes it to stop_live_usage_ingestor(instance). - stop only clears the module global and publisher registration when the stopped instance still owns them, so nested lifespans cannot orphan or unhook each other's instances. - The suite fence's reap now only cancels/awaits tasks bound to the loop it runs on; tasks bound to a foreign (e.g. closed portal) loop are enrolled for exception accounting and left inert instead of raising 'Event loop is closed' / cross-loop errors mid-cleanup. - Regression test pins the nested-lifespan contract; openspec change docs extended accordingly. Gates: make test-integration-core green (1875 passed), make test-unit green (6120 passed), ruff check/format clean, openspec validate --strict clean. Co-Authored-By: Claude Fable 5 * fix(usage): restore the displaced outer ingestor registration after nested shutdown Instance-scoped stop cleared the registration and publisher whenever the stopped instance owned them, so a nested lifespan's shutdown left a still-running outer ingestor registered-less: publications silently stopped flowing to it. Startup now remembers a displaced still-running registration on a LIFO stack (lifespans can nest more than one level deep, and a single prior slot would forget everything below the newest displacement); stopping the current instance restores the most recent displaced instance that still runs and its publisher wiring, never a stopped or dead one, and a stopped instance is removed from the stack so it can never be restored later. The test leak fence drains the stack too, and the nested-lifespan regression now proves a post-nested-exit publication is ingested end to end by the outer instance. Co-Authored-By: Claude Fable 5 * docs(openspec): define restoration eligibility predicate and edge scenarios Address CodeRabbit review: state the done()-false consumer-task predicate, the removed-before-shutdown tracking rule, and add failed/stopping displaced ingestor scenarios. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/main.py | 7 +- app/modules/usage/live_ingest.py | 115 +++++++++++- .../proposal.md | 58 ++++++ .../specs/live-usage-ingestion/spec.md | 91 +++++++++ .../settle-live-ingest-task-failures/tasks.md | 34 ++++ tests/conftest.py | 160 ++++++++++++++++ tests/integration/test_live_usage_ingest.py | 49 +++++ tests/unit/test_live_ingest_leak_fence.py | 174 ++++++++++++++++++ 8 files changed, 681 insertions(+), 7 deletions(-) create mode 100644 openspec/changes/settle-live-ingest-task-failures/proposal.md create mode 100644 openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md create mode 100644 openspec/changes/settle-live-ingest-task-failures/tasks.md create mode 100644 tests/unit/test_live_ingest_leak_fence.py diff --git a/app/main.py b/app/main.py index 6b44c8290a..2baa10a9f7 100644 --- a/app/main.py +++ b/app/main.py @@ -492,7 +492,10 @@ async def lifespan(app: FastAPI): account_deletion_scheduler = build_account_deletion_scheduler() data_retention_scheduler = build_data_retention_scheduler() telemetry_scheduler = build_telemetry_scheduler() - start_live_usage_ingestor() + # Hold the instance: this lifespan owns it (and keeps it strongly rooted) + # even if a nested lifespan on another loop replaces the module-global + # singleton in the meantime; shutdown below stops exactly this instance. + live_usage_ingestor = start_live_usage_ingestor() await usage_scheduler.start() await api_key_limit_reset_scheduler.start() await api_key_last_used_flush_scheduler.start() @@ -718,7 +721,7 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N # touch is never parked in a pending map with no remaining flusher. await api_key_last_used_flush_scheduler.stop() await usage_scheduler.stop() - await stop_live_usage_ingestor() + await stop_live_usage_ingestor(live_usage_ingestor) await rate_limit_reset_credits_scheduler.stop() await account_usage_rollup_scheduler.stop() await account_deletion_scheduler.stop() diff --git a/app/modules/usage/live_ingest.py b/app/modules/usage/live_ingest.py index b36a5ea525..af4ecba115 100644 --- a/app/modules/usage/live_ingest.py +++ b/app/modules/usage/live_ingest.py @@ -3,6 +3,7 @@ import asyncio import logging import time +import weakref from dataclasses import dataclass from app.core import usage as usage_core @@ -23,6 +24,50 @@ _WRITE_MIN_INTERVAL_SECONDS = 5.0 _CACHE_INVALIDATION_MIN_INTERVAL_SECONDS = 5.0 +# Ownership accounting for every task any ingestor instance creates (consumer +# and trailing cache invalidation), so a task an owner lost track of (a stop +# cancelled mid-await) can never end in a silently dropped exception: +# +# - `_owned_tasks` holds weak references (they never extend task lifetime) so +# the test suite's leak fence can cancel pending tasks and settle completed +# ones that some reference chain kept alive across a test boundary. +# - `_record_owned_task_result` runs as each task's done callback: it +# retrieves the exception (so the loop's unobserved-task warning can never +# fire at garbage-collection time), logs it, and records it in the bounded +# `_owned_task_failures` strong handoff for the fence to drain (#1755). +# +# `_settled_owned_tasks` (also weak) marks tasks whose result was already +# recorded, so the done callback and the fence's sweep of completed tasks +# settle each task exactly once even when both observe it. +_owned_tasks: weakref.WeakSet[asyncio.Task[None]] = weakref.WeakSet() +_settled_owned_tasks: weakref.WeakSet[asyncio.Task[None]] = weakref.WeakSet() +# (task name, exception repr) pairs. Reprs, not exception objects: a stored +# exception's traceback would keep the failed ingestor's whole object graph +# (task, queue, cached state) alive for the process lifetime, since production +# never drains this record. +_owned_task_failures: list[tuple[str, str]] = [] +_MAX_OWNED_TASK_FAILURES = 16 + + +def _record_owned_task_result(task: asyncio.Task[None]) -> None: + if task in _settled_owned_tasks: + return + _settled_owned_tasks.add(task) + _owned_tasks.discard(task) + if task.cancelled(): + return + exc = task.exception() + if exc is None: + return + logger.error("Live usage ingestor task %r died unexpectedly", task.get_name(), exc_info=exc) + if len(_owned_task_failures) < _MAX_OWNED_TASK_FAILURES: + _owned_task_failures.append((task.get_name(), repr(exc))) + + +def _enroll_owned_task(task: asyncio.Task[None]) -> None: + _owned_tasks.add(task) + task.add_done_callback(_record_owned_task_result) + @dataclass(frozen=True, slots=True) class _QueuedSnapshot: @@ -96,6 +141,11 @@ def publish( def start(self) -> None: if self._consumer is None or self._consumer.done(): self._consumer = asyncio.create_task(self._run(), name="live-usage-ingestor") + _enroll_owned_task(self._consumer) + + def is_running(self) -> bool: + consumer = self._consumer + return consumer is not None and not consumer.done() async def stop(self) -> None: consumer = self._consumer @@ -213,7 +263,11 @@ async def _invalidate_caches_throttled(self) -> None: await self._invalidate_caches_now() return if self._trailing_invalidation is None or self._trailing_invalidation.done(): - self._trailing_invalidation = asyncio.create_task(self._trailing_invalidate(remaining)) + self._trailing_invalidation = asyncio.create_task( + self._trailing_invalidate(remaining), + name="live-usage-trailing-invalidation", + ) + _enroll_owned_task(self._trailing_invalidation) async def _trailing_invalidate(self, delay_seconds: float) -> None: await asyncio.sleep(delay_seconds) @@ -229,9 +283,28 @@ async def _invalidate_caches_now(self) -> None: _ingestor: LiveUsageIngestor | None = None +# Registrations a nested startup displaced, innermost-last. A stack rather +# than a single prior slot: lifespans can nest more than one level deep (each +# portal-loop ``TestClient`` adds one), and a stack restores each displaced +# outer lifespan in LIFO order while an out-of-order stop simply removes its +# instance from wherever it sits — a single slot would forget everything below +# the most recent displacement. +_displaced_ingestors: list[LiveUsageIngestor] = [] def start_live_usage_ingestor() -> LiveUsageIngestor | None: + """Create, start, and register a fresh ingestor as the current singleton. + + The caller (the app lifespan) MUST hold the returned instance and pass it + back to ``stop_live_usage_ingestor`` at shutdown. Two lifespans can be + live in one process (the test suite nests a portal-loop ``TestClient`` + inside an app already running on the session loop); each owns its own + instance, and the module global only tracks whichever registered last. A + started ingestor whose only strong root is the module global would become + an unreferenced reference cycle (task -> coroutine frame -> ingestor -> + queue -> getter future -> task) the moment a nested startup overwrites the + global, and the cyclic GC would then destroy its consumer task mid-await. + """ global _ingestor settings = get_settings() if not getattr(settings, "live_usage_ingestion_enabled", True): @@ -242,15 +315,47 @@ def start_live_usage_ingestor() -> LiveUsageIngestor | None: write_min_interval_seconds=_WRITE_MIN_INTERVAL_SECONDS, ) ingestor.start() + if _ingestor is not None and _ingestor.is_running(): + # A nested startup displaces a still-running outer registration; + # remember it so the nested shutdown can restore it (a dead instance + # is never worth remembering). + _displaced_ingestors.append(_ingestor) register_live_usage_publisher(ingestor.publish) _ingestor = ingestor return ingestor -async def stop_live_usage_ingestor() -> None: +async def stop_live_usage_ingestor(ingestor: LiveUsageIngestor | None = None) -> None: + """Stop ``ingestor``, or the current singleton when omitted. + + The module global and the publisher registration are touched only when + the stopped instance still owns them, so a lifespan shutting down cannot + orphan or unregister a nested lifespan's newer instance — and a nested + lifespan's shutdown cannot leave the outer instance dangling with no + stop path (the leak behind issue #1755's cross-test poisoning). When the + stopped instance is the current registration, the most recent displaced + ingestor that is still running is restored (registration and publisher + wiring), so a still-live outer lifespan resumes receiving publications + instead of going silently deaf after a nested shutdown. + """ global _ingestor - ingestor = _ingestor - _ingestor = None - register_live_usage_publisher(None) + if ingestor is None: + ingestor = _ingestor + if ingestor is not None: + # Whatever happens next, a stopped instance must never be restorable. + try: + _displaced_ingestors.remove(ingestor) + except ValueError: + pass + if ingestor is None or _ingestor is ingestor: + restored: LiveUsageIngestor | None = None + while _displaced_ingestors: + candidate = _displaced_ingestors.pop() + if candidate.is_running(): + restored = candidate + break + # Stopped or dead in the meantime — never restore a dead instance. + _ingestor = restored + register_live_usage_publisher(restored.publish if restored is not None else None) if ingestor is not None: await ingestor.stop() diff --git a/openspec/changes/settle-live-ingest-task-failures/proposal.md b/openspec/changes/settle-live-ingest-task-failures/proposal.md new file mode 100644 index 0000000000..a61149b2e0 --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/proposal.md @@ -0,0 +1,58 @@ +## Why + +An ingestor-owned background task (the `live-usage-ingestor` consumer or the +trailing cache-invalidation sleeper) that dies with an exception after its +owner lost track of it — for example a shutdown cancelled between clearing the +singleton and awaiting the task — surfaces only as a nondeterministic +"Task exception was never retrieved" loop warning at garbage-collection time. +In production that hides the failure until an arbitrary later moment; in the +test suite's shared session loop it poisons unrelated tests (issue #1755: +the otel lifespan-drain test and test_proxy_utils' startup-probe assertions +fail together). + +## What Changes + +- Enroll every task the live-usage ingestor creates in a weak ownership + registry and attach a done callback that settles the task at completion: + retrieve its exception, log it immediately with its traceback, and record it + in a bounded in-process failure handoff. +- Settle each task exactly once (a settled-task registry gates recording) so + the callback and any external sweep cannot double-report. +- Make the ingestor lifecycle instance-scoped: the app lifespan holds the + instance `start_live_usage_ingestor()` returned and passes it to + `stop_live_usage_ingestor(instance)`; stop only clears the module global + and publisher registration when that instance still owns them. Two live + lifespans in one process (a portal-loop `TestClient` nested inside an app + already running on the suite's session loop) previously orphaned the outer + ingestor: the nested startup overwrote the module global — the orphan's + only strong root — leaving an unreferenced cycle whose consumer task the + cyclic GC destroyed mid-await (`cannot reuse already awaited coroutine`), + and the nested shutdown cleared the global so the outer shutdown stopped + nothing. +- Keep ingestion behavior unchanged: enqueueing, coalescing, throttling, + shutdown ordering, and the fire-and-forget contract are untouched. +- Test infrastructure (out of spec scope): an autouse fence stops leaked + ingestor singletons after every test and drains the failure handoff so no + ingestor task or unretrieved exception crosses a test boundary. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `live-usage-ingestion`: unexpected ingestor-owned task deaths MUST be + settled at completion — exception retrieved, logged, and recorded in a + bounded handoff — instead of surfacing as garbage-collection-time + unobserved-task warnings. + +## Impact + +`app/modules/usage/live_ingest.py` (task enrollment, done-callback +settlement, bounded failure record, instance-scoped stop), `app/main.py` +(lifespan holds and stops its own ingestor instance), `tests/conftest.py` +(leak fence), `tests/unit/test_live_ingest_leak_fence.py` and +`tests/integration/test_live_usage_ingest.py` (regression coverage). No API, +schema, setting, or dashboard change. diff --git a/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md b/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md new file mode 100644 index 0000000000..92d4c4707c --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/specs/live-usage-ingestion/spec.md @@ -0,0 +1,91 @@ +# live-usage-ingestion Delta + +## ADDED Requirements + +### Requirement: Ingestor-owned task failures are settled at completion + +Every background task the live usage ingestor creates MUST be settled when it +completes: if the task ends with an exception other than cancellation, the +exception MUST be retrieved at completion time, logged immediately with its +traceback, and recorded in a bounded in-process failure record as +traceback-free metadata (task name and exception representation) so the +record cannot retain the failed task's object graph. An +ingestor-owned task failure MUST NOT surface as a garbage-collection-time +unobserved-task warning. Each task MUST be settled exactly once, including +when an external supervisor (such as test infrastructure) also observes the +task. Settlement MUST NOT extend task lifetime, change ingestion behavior, or +affect the serving path. + +#### Scenario: Detached consumer death is logged deterministically + +- **GIVEN** a consumer task whose owner lost track of it (for example a stop + cancelled between clearing the singleton and awaiting the task) +- **WHEN** the task dies with an exception +- **THEN** the exception is retrieved and logged at completion time +- **AND** it is recorded in the bounded failure record +- **AND** no unobserved-task warning fires at garbage collection + +#### Scenario: Cancelled tasks settle silently + +- **WHEN** an ingestor-owned task ends by cancellation +- **THEN** settlement records no failure and logs no error + +#### Scenario: Failure record stays bounded + +- **WHEN** ingestor-owned tasks fail repeatedly without the record being + drained +- **THEN** the failure record retains at most its fixed capacity of entries +- **AND** every failure is still logged + +### Requirement: Ingestor lifecycle is instance-scoped + +Each application lifespan MUST hold the ingestor instance its startup created +and stop exactly that instance at shutdown. Stopping an instance MUST touch +the process-wide singleton registration and the publisher hook only when the +stopped instance still owns them; when it does own them, the most recently +displaced ingestor that is still running MUST be restored as the registration +and publisher. Restoration eligibility is defined as: the candidate holds an +existing consumer task whose `done()` is false — this excludes consumers that +failed or completed, and ingestors whose `stop()` already cleared their +consumer. An instance MUST be removed from restoration tracking before its own +shutdown begins, so a stopping or stopped instance can never be restored +later. When several +lifespans are live in one process, no lifespan's startup or shutdown may +orphan another lifespan's running ingestor, leave it without a stop path, or +leave it registered-less while it still runs. + +#### Scenario: Nested lifespan cannot orphan the outer ingestor + +- **GIVEN** an app whose lifespan started ingestor A +- **WHEN** a nested lifespan starts ingestor B (taking over the singleton and + publisher) and later stops it +- **THEN** ingestor A keeps running, strongly rooted by its own lifespan +- **AND** the outer lifespan's shutdown stops ingestor A and its tasks + +#### Scenario: Nested shutdown restores the outer registration + +- **GIVEN** an app whose lifespan started ingestor A +- **AND** a nested lifespan whose startup displaced A by registering + ingestor B +- **WHEN** the nested lifespan stops ingestor B +- **THEN** ingestor A is restored as the singleton registration and publisher +- **AND** publications after the nested exit flow to ingestor A and are + ingested +- **AND** a displaced ingestor that already stopped is not restored (the + registration falls through to the next still-running displaced instance, + or is cleared) + +#### Scenario: A failed displaced ingestor is not restored + +- **GIVEN** displaced ingestor A whose consumer task has settled with an + exception (its task `done()` is true) +- **WHEN** the current registration stops +- **THEN** A is skipped by restoration (fall through to the next eligible + displaced instance, or clear the registration) + +#### Scenario: A stopping displaced ingestor is not restored + +- **GIVEN** displaced ingestor A whose `stop()` has begun (A was removed from + restoration tracking before its shutdown started) +- **WHEN** the current registration stops concurrently +- **THEN** A is never restored, even if its consumer task has not yet finished diff --git a/openspec/changes/settle-live-ingest-task-failures/tasks.md b/openspec/changes/settle-live-ingest-task-failures/tasks.md new file mode 100644 index 0000000000..01b1cdb4c6 --- /dev/null +++ b/openspec/changes/settle-live-ingest-task-failures/tasks.md @@ -0,0 +1,34 @@ +## 1. Implementation + +- [x] 1.1 Enroll ingestor-created tasks (consumer, trailing invalidation) in a + weak ownership registry with named tasks. +- [x] 1.2 Attach a done callback that settles each task exactly once: + retrieve the exception, log it with its traceback, and record it in the + bounded failure handoff. +- [x] 1.3 Add the autouse test fence that stops leaked ingestor singletons, + reaps pending ingestor-owned tasks (only those bound to the loop the + reap runs on; foreign-loop tasks are enrolled and left inert), and + drains the failure handoff after every test. +- [x] 1.4 Make the lifecycle instance-scoped: the lifespan holds the started + instance and stops exactly that instance; stop touches the module global + and publisher only when the stopped instance still owns them, so nested + lifespans cannot orphan the outer ingestor into a GC-collectable cycle. +- [x] 1.5 Restore the displaced registration after nested shutdown: a startup + that displaces a still-running instance remembers it (LIFO stack), and + stopping the current instance restores the most recent displaced + instance that still runs — never a stopped or dead one — so the outer + lifespan's ingestion resumes instead of going deaf. + +## 2. Validation + +- [x] 2.1 Order-dependent regression pair proving a leaked consumer no longer + crosses a test boundary (fails on main without the fence). +- [x] 2.2 Regressions for dead-consumer settlement, orphaned-task sweep, + detached-death recording, queued-callback settlement, and exactly-once + reporting. +- [x] 2.3 Regression for nested lifespans: a nested start/stop pair must not + orphan, kill, or unhook the outer lifespan's ingestor; the nested stop + restores the outer registration (a post-exit publication is ingested + end to end), and the outer stop reaps its own consumer. +- [x] 2.4 Run the full unit suite, live-usage integration tests, lint, type + checks, and strict OpenSpec validation. diff --git a/tests/conftest.py b/tests/conftest.py index 48c9f8120c..7d96d60c1e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import os import tempfile from pathlib import Path @@ -403,3 +404,162 @@ def _reset_shutdown_task_admission(): shutdown_state.reset() yield shutdown_state.reset() + + +_SESSION_LOOP: asyncio.AbstractEventLoop | None = None + +# Both task names the live-usage ingestor owns (consumer and throttled +# trailing cache invalidation); the fence below reclaims them by name when the +# singleton no longer tracks them. +_LIVE_INGEST_TASK_NAMES = ("live-usage-ingestor", "live-usage-trailing-invalidation") + + +def _pending_live_ingest_tasks(loop: asyncio.AbstractEventLoop) -> list[asyncio.Task]: + return [task for task in asyncio.all_tasks(loop) if not task.done() and task.get_name() in _LIVE_INGEST_TASK_NAMES] + + +@pytest_asyncio.fixture(scope="session", autouse=True) +async def _capture_session_loop(): + """Expose the shared session loop to sync fixture teardowns. + + The live-usage ingestor fence below must run coroutine cleanup from a + synchronous teardown (see its docstring for why it cannot be an async + fixture), and pytest-asyncio has no public API to reach the session loop + from sync code. + """ + global _SESSION_LOOP + _SESSION_LOOP = asyncio.get_running_loop() + yield + _SESSION_LOOP = None + + +async def _reap_leaked_live_usage_ingestor() -> None: + """Stop and reset the live-usage ingestor singleton. + + Mirrors ``stop_live_usage_ingestor()`` but never re-raises: every awaited + task ends up done, and ``_consume_dead_live_ingest_task_failures`` then + retrieves and reports its exception exactly once. Also sweeps by name for + ingestor-owned tasks (consumer and trailing invalidation) the stop path no + longer tracks — a stop that was itself cancelled between clearing the + global and awaiting the tasks. + + Only tasks bound to the loop this coroutine runs on are cancelled and + awaited. A leaked singleton can hold tasks that belong to a different + loop entirely — integration tests run ``TestClient`` portals whose loop + is a private per-portal loop that is already closed by teardown time. + Cancelling such a task raises ``RuntimeError('Event loop is closed')`` + from ``call_soon`` and awaiting it raises the cross-loop RuntimeError; + neither can ever reap it. Those tasks are inert (a closed loop never + steps again), so they are enrolled for exception accounting and left + alone. + """ + from app.core.usage.live_hub import register_live_usage_publisher + from app.modules.usage import live_ingest + + ingestors: list[live_ingest.LiveUsageIngestor] = [] + if live_ingest._ingestor is not None: + ingestors.append(live_ingest._ingestor) + live_ingest._ingestor = None + # Displaced (nested-over) registrations hold live tasks too, and a stale + # stack entry must never be restored into a later test. + ingestors.extend(live_ingest._displaced_ingestors) + live_ingest._displaced_ingestors.clear() + register_live_usage_publisher(None) + leaked: list[asyncio.Task[None]] = [] + for ingestor in ingestors: + for task in (ingestor._consumer, ingestor._trailing_invalidation): + if task is not None and task not in leaked: + leaked.append(task) + ingestor._consumer = None + ingestor._trailing_invalidation = None + loop = asyncio.get_running_loop() + for task in _pending_live_ingest_tasks(loop): + if task not in leaked: + leaked.append(task) + reapable: list[asyncio.Task[None]] = [] + for task in leaked: + live_ingest._owned_tasks.add(task) + if task.get_loop() is loop: + reapable.append(task) + for task in reapable: + task.cancel() + for task in reapable: + try: + await task + except (Exception, asyncio.CancelledError): + # Settled and reported by _drain_live_ingest_task_failures. + continue + + +def _drain_live_ingest_task_failures() -> list[str]: + """Collect failures from dead ingestor-owned tasks, loop-free. + + ``asyncio.all_tasks`` only returns unfinished tasks, so a leaked task that + already died with an exception is invisible to the pending sweep; its + unretrieved exception would otherwise fire the loop exception handler when + the task object is garbage-collected inside a LATER test (test_proxy_utils' + startup-probe assertions capture exactly that). live_ingest's done + callback normally settles each task the moment it completes (retrieving + the exception into the strong ``_owned_task_failures`` handoff); the sweep + over the weak registry here additionally settles tasks whose callback is + still queued because the task finished in the loop's final iteration. + Settlement is gated by live_ingest's settled-task registry, so each task + is reported exactly once even when both paths observe it. + """ + from app.modules.usage import live_ingest + + for task in list(live_ingest._owned_tasks): + if task.done(): + live_ingest._record_owned_task_result(task) + failures = [f"{name!r} died with {exc_repr}" for name, exc_repr in live_ingest._owned_task_failures] + live_ingest._owned_task_failures.clear() + return failures + + +@pytest.fixture(autouse=True) +def _stop_leaked_live_usage_ingestor(): + """Fence the module-global live-usage ingestor per test (issue #1755). + + The suite runs on a session-scoped asyncio loop, so a task leaked by one + test survives into every later test. Any test that enters the real app + lifespan starts the live-usage ingestor singleton + (``app.modules.usage.live_ingest._ingestor``) whose ``live-usage-ingestor`` + consumer task lands on that shared loop; if the lifespan is cancelled + before its shutdown path reaches ``stop_live_usage_ingestor()`` (e.g. a + ``wait_for``-bounded assertion times out mid-drain), the consumer outlives + the test. The zombie then poisons unrelated tests: it eats into the otel + lifespan test's drain budget and surfaces as an unobserved-task exception + inside test_proxy_utils' startup-probe loop-exception assertions — the + exact failing pairing from #1755. Stop and reset the singleton after every + test so no ingestor task ever crosses a test boundary. + + Deliberately a sync fixture that only enters the event loop when a leak is + actually present: an async fixture's teardown would spin the shared loop + after EVERY test, and the loop's clock calls ``time.monotonic()`` — which + several tests monkeypatch globally with finite or call-count-sensitive + fakes that are still active while function-scoped teardowns run (e.g. + test_conversation_archive's exhausting iterator). Leak detection itself is + loop-passive: reading the module globals, enumerating + ``asyncio.all_tasks(loop)`` on the idle session loop, and retrieving + exceptions from already-dead owned tasks never runs the loop. + """ + yield + from app.core.usage import live_hub + from app.modules.usage import live_ingest + + loop = _SESSION_LOOP + loop_usable = loop is not None and not loop.is_closed() and not loop.is_running() + needs_reap = ( + live_ingest._ingestor is not None + or bool(live_ingest._displaced_ingestors) + or live_hub._publisher is not None + or (loop_usable and loop is not None and _pending_live_ingest_tasks(loop)) + ) + if needs_reap and loop_usable and loop is not None: + loop.run_until_complete(_reap_leaked_live_usage_ingestor()) + failures = _drain_live_ingest_task_failures() + if failures: + pytest.fail( + "test leaked a live-usage ingestor whose task(s) already failed: " + "; ".join(failures), + pytrace=False, + ) diff --git a/tests/integration/test_live_usage_ingest.py b/tests/integration/test_live_usage_ingest.py index dd4d4d66c7..9e16704921 100644 --- a/tests/integration/test_live_usage_ingest.py +++ b/tests/integration/test_live_usage_ingest.py @@ -856,3 +856,52 @@ async def test_live_ingestion_kill_switch_disables_publishing(monkeypatch, db_se finally: await live_ingest.stop_live_usage_ingestor() get_settings.cache_clear() + + +@pytest.mark.asyncio +async def test_nested_lifespan_stop_does_not_orphan_or_kill_the_outer_ingestor(db_setup) -> None: + del db_setup + + # Two app lifespans can be live in one process: the suite's async_client + # runs one on the session loop while a test opens a TestClient whose + # portal runs another. Each lifespan owns the instance start returned and + # stops exactly that instance. Before instance-scoped stop, the nested + # startup overwrote the module global, orphaned the outer ingestor as an + # unreferenced cycle, and the cyclic GC destroyed its consumer mid-await + # ("cannot reuse already awaited coroutine" — issue #1755's integration + # signature); the nested shutdown then cleared the global so the outer + # shutdown stopped nothing. And a nested shutdown that merely cleared the + # registration would leave the still-running outer ingestor deaf: it must + # instead restore the outer instance as the current registration. + def _pending_consumers() -> list[asyncio.Task[object]]: + return [t for t in asyncio.all_tasks() if not t.done() and t.get_name() == "live-usage-ingestor"] + + async with SessionLocal() as session: + await AccountsRepository(session).upsert(_make_account("acc_live_nested", "live-nested@example.com")) + + outer = live_ingest.start_live_usage_ingestor() + assert outer is not None + inner = live_ingest.start_live_usage_ingestor() + assert inner is not None and inner is not outer + assert live_ingest._ingestor is inner + + # Nested lifespan shutdown: releases the registration it owns and + # restores the still-running outer instance in its place. + await live_ingest.stop_live_usage_ingestor(inner) + assert live_ingest._ingestor is outer + assert live_ingest._displaced_ingestors == [] + assert outer._consumer is not None and not outer._consumer.done() + + # Outer ingestion RESUMES: a hub publication after the nested exit must + # flow to the outer instance and be ingested end to end. + live_hub.publish_live_usage(_snapshot(), account_id="acc_live_nested") + primary, secondary = await _wait_for_rows("acc_live_nested") + assert primary is not None and secondary is not None + assert primary.used_percent == pytest.approx(33.0) + + # Outer lifespan shutdown: stops its own instance, clears the restored + # registration, and no consumer survives for the suite fence. + await live_ingest.stop_live_usage_ingestor(outer) + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert _pending_consumers() == [] diff --git a/tests/unit/test_live_ingest_leak_fence.py b/tests/unit/test_live_ingest_leak_fence.py new file mode 100644 index 0000000000..63d3b585c9 --- /dev/null +++ b/tests/unit/test_live_ingest_leak_fence.py @@ -0,0 +1,174 @@ +"""Regression tests for issue #1755: cross-test live-usage-ingestor leakage. + +The unit suite runs on a session-scoped asyncio loop, so a background task +leaked by one test survives into every later test. Tests that enter the real +app lifespan start the module-global live-usage ingestor; when the lifespan is +cancelled before its shutdown path reaches ``stop_live_usage_ingestor()`` +(e.g. a ``wait_for``-bounded assertion times out mid-drain), the +``live-usage-ingestor`` consumer task outlives the test and later poisons the +otel lifespan-drain test and test_proxy_utils' startup-probe loop-exception +assertions. + +The first two tests are ORDER-DEPENDENT by design (pytest runs them in +definition order): the first reproduces the leak by starting the ingestor +singleton exactly like the app lifespan does and deliberately never stopping +it; the second asserts the autouse ``_stop_leaked_live_usage_ingestor`` fence +in tests/conftest.py reclaimed the consumer at the previous test's boundary. +Without the fence the second test fails with a pending +``live-usage-ingestor`` task — the coupling observed in #1755. +""" + +from __future__ import annotations + +import asyncio +import time + +import pytest + +from app.core.usage import live_hub +from app.modules.usage import live_ingest + + +def _pending_ingestor_tasks() -> list[asyncio.Task[object]]: + return [task for task in asyncio.all_tasks() if not task.done() and task.get_name() == "live-usage-ingestor"] + + +async def test_abandoned_ingestor_simulates_lifespan_cancelled_before_stop() -> None: + # This is exactly what app.main's lifespan startup does; a lifespan + # cancelled mid-shutdown-drain never reaches stop_live_usage_ingestor(), + # so nothing in this test stops the singleton either. The autouse fence + # in tests/conftest.py must reclaim it at this test's boundary. + ingestor = live_ingest.start_live_usage_ingestor() + + assert ingestor is not None + assert live_ingest._ingestor is ingestor + assert live_hub._publisher is not None + assert len(_pending_ingestor_tasks()) == 1 + + +async def test_fence_reclaims_leaked_consumer_at_test_boundary() -> None: + assert _pending_ingestor_tasks() == [] + assert live_ingest._ingestor is None + assert live_hub._publisher is None + + +async def test_reap_settles_and_reports_already_failed_consumer(monkeypatch: pytest.MonkeyPatch) -> None: + # A leaked consumer can already be dead with an exception by the time the + # fence runs (#1755 observed RuntimeError('cannot reuse already awaited + # coroutine')). The fence must retrieve that exception — so it neither + # crashes mid-cleanup nor resurfaces later as an unobserved-task loop + # exception in an unrelated test — and report it exactly once even though + # both the done callback and the fence sweep observe the dead task. + from tests import conftest as suite_conftest + + async def _boom(self: live_ingest.LiveUsageIngestor) -> None: + raise RuntimeError("cannot reuse already awaited coroutine") + + monkeypatch.setattr(live_ingest.LiveUsageIngestor, "_run", _boom) + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + live_ingest._ingestor = ingestor + live_hub.register_live_usage_publisher(ingestor.publish) + await asyncio.sleep(0) + assert ingestor._consumer is not None and ingestor._consumer.done() + + await suite_conftest._reap_leaked_live_usage_ingestor() + failures = suite_conftest._drain_live_ingest_task_failures() + + assert failures == ["'live-usage-ingestor' died with RuntimeError('cannot reuse already awaited coroutine')"] + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert _pending_ingestor_tasks() == [] + # Settlement is exactly-once: a second pass reports nothing. + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_reap_sweeps_orphaned_tasks_not_tracked_by_singleton() -> None: + # A stop that is itself cancelled between clearing the module global and + # awaiting the ingestor's tasks leaves pending tasks no singleton tracks; + # the reap's name-based sweep must still cancel and await both the + # consumer and the trailing cache-invalidation sleeper. + from tests import conftest as suite_conftest + + async def _pending_forever() -> None: + await asyncio.Event().wait() + + consumer = asyncio.create_task(_pending_forever(), name="live-usage-ingestor") + trailing = asyncio.create_task(_pending_forever(), name="live-usage-trailing-invalidation") + await asyncio.sleep(0) + assert live_ingest._ingestor is None + + await suite_conftest._reap_leaked_live_usage_ingestor() + + assert consumer.cancelled() + assert trailing.cancelled() + assert suite_conftest._drain_live_ingest_task_failures() == [] + assert suite_conftest._pending_live_ingest_tasks(asyncio.get_running_loop()) == [] + + +async def test_dead_detached_owned_task_failure_is_recorded_and_drained_loop_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # An ingestor-owned task can die with an exception after the singleton and + # its task fields are already cleared. asyncio.all_tasks() only returns + # unfinished tasks, so the pending sweep cannot see it — the done callback + # installed at task creation must have already retrieved the exception + # into the strong failure handoff, which the fence drains without running + # the event loop. + from tests import conftest as suite_conftest + + async def _boom(self: live_ingest.LiveUsageIngestor) -> None: + raise RuntimeError("late detached failure") + + monkeypatch.setattr(live_ingest.LiveUsageIngestor, "_run", _boom) + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + await asyncio.sleep(0) + await asyncio.sleep(0) # let the done callback run + ingestor._consumer = None # fully detach: no owner, no pending task + del ingestor + assert live_ingest._ingestor is None + assert live_hub._publisher is None + assert suite_conftest._pending_live_ingest_tasks(asyncio.get_running_loop()) == [] + + failures = suite_conftest._drain_live_ingest_task_failures() + + assert failures == ["'live-usage-ingestor' died with RuntimeError('late detached failure')"] + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_drain_settles_dead_task_whose_done_callback_has_not_run() -> None: + # A task that finishes in the loop's final iteration can still have its + # done callback queued when the sync fence runs; the drain's sweep over + # the weak ownership registry must settle it directly, and the callback + # running later must not report it a second time. + from tests import conftest as suite_conftest + + async def _boom() -> None: + raise RuntimeError("callback still queued") + + task = asyncio.create_task(_boom(), name="live-usage-trailing-invalidation") + live_ingest._owned_tasks.add(task) # enrolled, but callback never attached + await asyncio.sleep(0) + assert task.done() + + failures = suite_conftest._drain_live_ingest_task_failures() + assert failures == ["'live-usage-trailing-invalidation' died with RuntimeError('callback still queued')"] + + # The (simulated late) callback observes an already-settled task. + live_ingest._record_owned_task_result(task) + assert suite_conftest._drain_live_ingest_task_failures() == [] + + +async def test_ingestor_enrolls_both_task_types_in_the_ownership_registry() -> None: + # The registry only protects tests if production task creation actually + # enrolls both task types. + ingestor = live_ingest.LiveUsageIngestor(queue_size=1, write_min_interval_seconds=0.0) + ingestor.start() + ingestor._last_cache_invalidation = time.monotonic() + await ingestor._invalidate_caches_throttled() + + assert ingestor._consumer in live_ingest._owned_tasks + assert ingestor._trailing_invalidation in live_ingest._owned_tasks + + await ingestor.stop() From 0a4c0a1071cfe2e91357d8dc34b433cb511b1aec Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 19:57:14 +0900 Subject: [PATCH 061/117] fix(docker): upgrade util-linux family in runtime image for CVE-2026-53615 (#1796) Trivy's high-severity gate went red on main overnight: CVE-2026-53615 (HIGH) against bsdutils/util-linux in the python:3.14-slim base, fixed upstream in 2.41.5-0+deb13u1. Add the util-linux binary family to the existing --only-upgrade remediation list so the runtime stage picks up the patched debs; --only-upgrade skips any member not present in slim. Co-authored-by: Claude Fable 5 --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ef2cd5aaa0..b5edd041c7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -40,7 +40,8 @@ WORKDIR /app RUN apt-get update \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends --only-upgrade \ - libc-bin libc6 libcap2 libssl3t64 libsystemd0 libudev1 openssl sed \ + bsdutils libblkid1 libc-bin libc6 libcap2 libmount1 libsmartcols1 libssl3t64 \ + libsystemd0 libudev1 libuuid1 openssl sed util-linux \ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ openssl-provider-legacy \ && rm -rf /var/lib/apt/lists/* From 9d9f099197326f37eea4042e27c5b09606f99043 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 21:33:35 +0900 Subject: [PATCH 062/117] perf(proxy): validate stream payloads only for lifecycle events (#1784) * perf(proxy): validate stream payloads only for lifecycle events Pydantic validation of streamed SSE/websocket payloads now runs only for the lifecycle frames whose validated fields are actually consumed (response.created/completed/incomplete/failed and error); all other frames are classified from the already-parsed payload dict via the new shared classify_event_type + _LIFECYCLE_EVENT_TYPES in app/core/openai/parsing.py, which _event_type_from_payload and tool_call_dedupe.event_type_from_payload delegate to. - core client: delete the per-chunk parse_sse_event terminal checks that duplicated the normalized_event_type dict branch computed one line earlier; websocket receive loops detect terminal frames from parse_sse_data_json + the payload type string - websocket relay: single json.loads per frame (no synthetic-block re-parse), lifecycle-only validation, pass event= into rewrite_parallel_tool_call_text, stop building the discarded format_sse_event argument, and skip the per-frame json.dumps when the downstream response-id rewrite returned the payload unchanged - tool-call rewrite helpers: no re-validation on the unchanged path (callers own lifecycle-gated validation) - streaming mixin and HTTP-bridge upstream reader: same lifecycle gating SSE output bytes, usage settlement, error normalization, and terminal detection are unchanged; existing streaming/websocket suites pass unmodified. OpenSpec change: validate-stream-lifecycle-events-only. Co-Authored-By: Claude Fable 5 * perf(proxy): decode websocket frames once and gate error-envelope validation Close the two remaining hot-path gaps against the validate-stream-lifecycle-events-only requirement: - _stream_websocket_events / _stream_codex_websocket_events now yield (sse_block, event_type) pairs, and _stream_responses_via_websocket plus the outer _stream_responses_with_session loop reuse the threaded type instead of re-running parse_sse_data_json per frame, so every websocket frame is json-decoded exactly once instead of three times - _normalize_stream_event_payload classifies the parsed payload first (classify_event_type) and runs parse_error_payload's pydantic error-envelope validation only for error-shaped frames (type == "error" or a top-level error dict), so HTTP SSE and websocket delta frames skip schema validation entirely; error frames are validated and rewritten exactly as before Regression tests pin one json.loads per websocket frame with zero parse_sse_data_json re-parses, zero error-adapter runs on delta frames, and unchanged error-frame rewrites. Co-Authored-By: Claude Fable 5 * fix(tests): satisfy ty on error-envelope gating regressions Co-Authored-By: Claude Fable 5 * fix(proxy): close lifecycle-gating gaps from review - Websocket relay: decode each direct upstream text frame exactly once in _process_and_forward_upstream_websocket_text and share the parsed frame with archive attribution and relay processing; _websocket_archive_request_id_for_message no longer re-parses or pydantic-validates non-lifecycle deltas. - Streaming mixin: apply the malformed-error SDK-contract fallback to the first upstream frame too (extracted into _rewrite_malformed_stream_error_event, shared with the later-frame loop), so a schema-less {"type":"error","message":...} first frame is rewritten to a terminal response.failed and settles as an error. - Websocket response-id rewrite: return the original payload object when no identifier value actually changes, keeping the identity fast-path so already-aligned frames are forwarded without re-encoding. Regression tests cover the product paths: single decode + lifecycle-only validation through _process_and_forward_upstream_websocket_text, malformed first-frame error rewrite with non-success settlement, and byte-identical forwarding when the frame already carries the assigned response id. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/clients/proxy.py | 134 +++-- app/core/openai/parsing.py | 31 ++ .../_service/http_bridge/upstream_events.py | 11 +- .../proxy/_service/streaming/helpers.py | 30 + app/modules/proxy/_service/streaming/mixin.py | 51 +- app/modules/proxy/_service/support.py | 10 +- .../proxy/_service/websocket/helpers.py | 16 +- app/modules/proxy/_service/websocket/mixin.py | 91 +-- app/modules/proxy/tool_call_dedupe.py | 23 +- .../design.md | 91 +++ .../proposal.md | 76 +++ .../specs/responses-api-compat/spec.md | 36 ++ .../tasks.md | 29 + tests/unit/test_proxy_tool_call_dedupe.py | 56 ++ tests/unit/test_proxy_utils.py | 523 +++++++++++++++++- tests/unit/test_sse.py | 29 +- .../test_websocket_terminal_cancellation.py | 2 + 17 files changed, 1080 insertions(+), 159 deletions(-) create mode 100644 openspec/changes/validate-stream-lifecycle-events-only/design.md create mode 100644 openspec/changes/validate-stream-lifecycle-events-only/proposal.md create mode 100644 openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/validate-stream-lifecycle-events-only/tasks.md diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index fe50a4085e..86904f9d0c 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -60,9 +60,9 @@ from app.core.openai.model_registry import get_model_registry from app.core.openai.models import CompactResponsePayload, OpenAIError from app.core.openai.parsing import ( + classify_event_type, parse_compact_response_payload, parse_error_payload, - parse_sse_event, ) from app.core.openai.requests import ( ResponsesCompactRequest, @@ -1402,37 +1402,42 @@ def _normalize_stream_event_payload(payload: dict[str, JsonValue]) -> dict[str, normalized = dict(payload) normalized["type"] = _SSE_EVENT_TYPE_ALIASES[event_type] return normalized - error = parse_error_payload(payload) - if error is not None: - detail = error.model_dump(exclude_none=True) - event = response_failed_event( - _normalize_error_code(detail.get("code"), detail.get("type")), - detail.get("message", "Upstream websocket error"), - error_type=detail.get("type") or "server_error", - response_id=get_request_id(), - error_param=detail.get("param"), - ) - _copy_quota_error_metadata(event["response"]["error"], detail) - return cast(dict[str, JsonValue], event) - if event_type == "error": - message = _extract_upstream_message(payload) or "Upstream websocket error" - code = payload.get("code") - error_type = payload.get("error_type") or payload.get("type") - normalized_code = _normalize_error_code( - code if isinstance(code, str) else None, - error_type if isinstance(error_type, str) else None, - ) - if not isinstance(code, str) and normalized_code == "error": - normalized_code = "upstream_error" - return cast( - dict[str, JsonValue], - response_failed_event( - normalized_code, - message, - error_type=error_type if isinstance(error_type, str) and error_type != "error" else "server_error", + # Error-envelope schema validation is the only pydantic work on this hot + # path: classify from the parsed dict first and validate only error-shaped + # frames (``type == "error"`` or a top-level ``error`` envelope) so delta + # frames never reach the pydantic adapter. + if classify_event_type(payload) == "error" or isinstance(payload.get("error"), dict): + error = parse_error_payload(payload) + if error is not None: + detail = error.model_dump(exclude_none=True) + event = response_failed_event( + _normalize_error_code(detail.get("code"), detail.get("type")), + detail.get("message", "Upstream websocket error"), + error_type=detail.get("type") or "server_error", response_id=get_request_id(), - ), - ) + error_param=detail.get("param"), + ) + _copy_quota_error_metadata(event["response"]["error"], detail) + return cast(dict[str, JsonValue], event) + if event_type == "error": + message = _extract_upstream_message(payload) or "Upstream websocket error" + code = payload.get("code") + error_type = payload.get("error_type") or payload.get("type") + normalized_code = _normalize_error_code( + code if isinstance(code, str) else None, + error_type if isinstance(error_type, str) else None, + ) + if not isinstance(code, str) and normalized_code == "error": + normalized_code = "upstream_error" + return cast( + dict[str, JsonValue], + response_failed_event( + normalized_code, + message, + error_type=error_type if isinstance(error_type, str) and error_type != "error" else "server_error", + response_id=get_request_id(), + ), + ) return payload @@ -1808,7 +1813,12 @@ async def _stream_websocket_events( total_timeout_seconds: float | None, max_event_bytes: int, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs. + + The event type is extracted from the payload parsed once here so that + downstream consumers never re-decode the formatted block. + """ deadline = None if total_timeout_seconds is None else time.monotonic() + total_timeout_seconds while True: @@ -1851,9 +1861,10 @@ async def _stream_websocket_events( if not isinstance(payload, dict): continue normalized = payload if not enforce_openai_sdk_contract else _normalize_stream_event_payload(payload) - event_type = normalized.get("type") - yield format_sse_event(normalized) - if isinstance(event_type, str) and _is_response_stream_terminal_event_type( + raw_event_type = normalized.get("type") + event_type = raw_event_type if isinstance(raw_event_type, str) else None + yield format_sse_event(normalized), event_type + if event_type is not None and _is_response_stream_terminal_event_type( event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): @@ -1867,7 +1878,8 @@ async def _stream_codex_websocket_events( total_timeout_seconds: float | None, max_event_bytes: int, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs; see ``_stream_websocket_events``.""" deadline = None if total_timeout_seconds is None else time.monotonic() + total_timeout_seconds while True: @@ -1917,9 +1929,10 @@ async def _stream_codex_websocket_events( if not isinstance(payload, dict): continue normalized = payload if not enforce_openai_sdk_contract else _normalize_stream_event_payload(payload) - event_type = normalized.get("type") - yield format_sse_event(normalized) - if isinstance(event_type, str) and _is_response_stream_terminal_event_type( + raw_event_type = normalized.get("type") + event_type = raw_event_type if isinstance(raw_event_type, str) else None + yield format_sse_event(normalized), event_type + if event_type is not None and _is_response_stream_terminal_event_type( event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): @@ -1954,7 +1967,8 @@ async def _stream_responses_via_websocket( route_trace: UpstreamProxyRouteTrace | None = None, allow_direct_egress: bool = True, enforce_openai_sdk_contract: bool = True, -) -> AsyncIterator[str]: +) -> AsyncIterator[tuple[str, str | None]]: + """Yield ``(sse_block, event_type)`` pairs from the upstream websocket.""" websocket_url = _to_websocket_upstream_url(url) request_started_at = time.monotonic() request_payload = _prepare_websocket_response_create_payload(payload_dict) @@ -2133,7 +2147,7 @@ async def _record_lifecycle_failure(exc: Exception) -> None: enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) ) - async for event in event_iter: + async for event, event_type in event_iter: archive_text( direction="server_to_codex", kind="responses", @@ -2145,14 +2159,13 @@ async def _record_lifecycle_failure(exc: Exception) -> None: headers=headers, extra={"event_format": "sse"}, ) - parsed_event = parse_sse_event(event) - if parsed_event and _is_response_stream_terminal_event_type( - parsed_event.type, + if event_type is not None and _is_response_stream_terminal_event_type( + event_type, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ): seen_terminal = True await _record_lifecycle_success() - yield event + yield event, event_type if not seen_terminal: await _record_lifecycle_failure(aiohttp.ClientError("Upstream websocket closed without terminal event")) except Exception as exc: @@ -2908,13 +2921,7 @@ async def _stream_via_http_attempt( event_block, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) - event = parse_sse_event(event_block) - if event: - if event.type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or ( - event.type == "error" and not enforce_openai_sdk_contract - ): - seen_terminal = True - elif isinstance(normalized_event_type, str) and ( + if isinstance(normalized_event_type, str) and ( normalized_event_type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or (normalized_event_type == "error" and not enforce_openai_sdk_contract) ): @@ -3002,13 +3009,7 @@ async def _stream_via_http_attempt( event_block, enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) - event = parse_sse_event(event_block) - if event: - if event.type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or ( - event.type == "error" and not enforce_openai_sdk_contract - ): - seen_terminal = True - elif isinstance(normalized_event_type, str) and ( + if isinstance(normalized_event_type, str) and ( normalized_event_type in _RESPONSE_STREAM_TERMINAL_EVENT_TYPES or (normalized_event_type == "error" and not enforce_openai_sdk_contract) ): @@ -3116,7 +3117,7 @@ async def _stream_via_http_after_websocket_rejection( try: if transport == "websocket": try: - async for event_block in _stream_responses_via_websocket( + async for event_block, event_type in _stream_responses_via_websocket( payload_dict=payload_dict, url=url, headers=upstream_headers, @@ -3134,14 +3135,11 @@ async def _stream_via_http_after_websocket_rejection( ): if status_code is None: status_code = 101 - event = parse_sse_event(event_block) - if event: - event_type = event.type - if _is_response_stream_terminal_event_type( - event_type, - enforce_openai_sdk_contract=enforce_openai_sdk_contract, - ): - seen_terminal = True + if event_type is not None and _is_response_stream_terminal_event_type( + event_type, + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + ): + seen_terminal = True yield event_block except aiohttp.WSServerHandshakeError as exc: if not _should_fallback_to_http_after_websocket_handshake_error(transport_mode, exc): diff --git a/app/core/openai/parsing.py b/app/core/openai/parsing.py index 9b6f614352..295ffd71be 100644 --- a/app/core/openai/parsing.py +++ b/app/core/openai/parsing.py @@ -17,6 +17,37 @@ _RESPONSE_ADAPTER = TypeAdapter(OpenAIResponsePayload) _COMPACT_RESPONSE_ADAPTER = TypeAdapter(CompactResponsePayload) +# Stream lifecycle frames are the only events whose validated model fields the +# proxy consumes (usage settlement, error normalization, response-id capture). +# Hot streaming paths validate only these frames and classify everything else +# from the already-parsed payload dict via ``classify_event_type``. +_LIFECYCLE_EVENT_TYPES = frozenset( + { + "response.created", + "response.completed", + "response.incomplete", + "response.failed", + "error", + } +) + + +def classify_event_type(payload: JsonValue | None) -> str | None: + """Classify an SSE event type from an already-parsed payload dict. + + Mirrors the dict branch of the proxy's ``_event_type_from_payload``: + a string ``type`` field wins; a typeless payload carrying a dict + ``error`` classifies as ``"error"``. No pydantic validation is run. + """ + if not isinstance(payload, dict): + return None + payload_type = payload.get("type") + if isinstance(payload_type, str): + return payload_type + if isinstance(payload.get("error"), dict): + return "error" + return None + def parse_sse_event(line: str) -> OpenAIEvent | None: return parse_sse_event_payload(parse_sse_data_json(line)) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 1f7e55b67c..2bd052dd76 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -39,7 +39,11 @@ ) from app.core.errors import response_failed_event from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event_payload +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, +) from app.core.types import JsonValue from app.core.usage.live_hub import publish_live_usage from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text @@ -138,7 +142,6 @@ _clear_websocket_deferred_reasoning_downstream_texts, _clear_websocket_precreated_replay_fallback, _clear_websocket_request_error_overrides, - _event_type_from_payload, _HTTPBridgeCompletedDeliveryScope, _HTTPBridgeRetryCircuitAttemptSelection, _HTTPBridgeSession, @@ -1652,8 +1655,8 @@ async def _process_http_bridge_upstream_text( ) -> None: event_block = f"data: {text}\n\n" payload = parse_sse_data_json(event_block) - event = parse_sse_event_payload(payload) - event_type = _event_type_from_payload(event, payload) + event_type = classify_event_type(payload) + event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None completed_delivery_scope = _HTTPBridgeCompletedDeliveryScope() if event_type == "response.completed" else None claimed_terminal_request_states: list[_WebSocketRequestState] = [] try: diff --git a/app/modules/proxy/_service/streaming/helpers.py b/app/modules/proxy/_service/streaming/helpers.py index 0d720916e4..c37a1d1498 100644 --- a/app/modules/proxy/_service/streaming/helpers.py +++ b/app/modules/proxy/_service/streaming/helpers.py @@ -647,6 +647,36 @@ def _mark_downstream_stream_cancelled( ) +def _rewrite_malformed_stream_error_event( + *, + enforce_openai_sdk_contract: bool, + event: OpenAIEvent | None, + event_type: str | None, + event_payload: dict[str, JsonValue] | None, + response_id: str, +) -> tuple[str, OpenAIEvent | None, dict[str, JsonValue] | None, str | None] | None: + """Rewrite a schema-less upstream ``error`` frame under the SDK contract. + + A malformed frame like ``{"type":"error","message":"..."}`` classifies as + ``error`` but carries no error envelope (``event`` is None or has no + ``error``), so it must become a terminal ``response.failed`` instead of + leaking the raw frame with a success settlement. Returns None when the + frame is not a malformed error (well-formed errors keep their + envelope-driven handling). + """ + if not enforce_openai_sdk_contract or event_type != "error": + return None + if (event is not None and event.error is not None) or not isinstance(event_payload, dict): + return None + message_value = event_payload.get("message") + message = message_value.strip() if isinstance(message_value, str) and message_value.strip() else "Upstream error" + return _build_rewritten_stream_response_failed_event( + response_id=response_id, + error_code="upstream_error", + error_message=message, + ) + + def _build_rewritten_stream_response_failed_event( *, response_id: str, diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 92a96f7fc5..39d4cdad16 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -41,7 +41,11 @@ from app.core.errors import ( response_failed_event, ) -from app.core.openai.parsing import parse_sse_event_payload +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, +) from app.core.openai.requests import ( ResponsesRequest, ) @@ -274,6 +278,7 @@ _mark_downstream_stream_cancelled, _mark_upstream_stream_incomplete, _raw_stream_error_code_or_upstream, + _rewrite_malformed_stream_error_event, ) from app.modules.proxy._service.streaming.helpers import ( _raw_stream_error_fields as _raw_error_fields, @@ -292,7 +297,6 @@ _WEBSOCKET_FULL_REPLAY_WAIT_MIN_ITEMS, # noqa: F401 _WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401 _ApiKeyReservationTouchState, - _event_type_from_payload, _finalize_ttft_latency_ms, _RequestLogFailureMetadata, _RetryableStreamError, @@ -609,10 +613,19 @@ async def _stream_once( await proxy._load_balancer.release_account_lease(account_response_create_lease) account_response_create_lease = None first_payload = parse_sse_data_json(first) - event = parse_sse_event_payload(first_payload) - event_type = _event_type_from_payload(event, first_payload) + event_type = classify_event_type(first_payload) + event = parse_sse_event_payload(first_payload) if event_type in _LIFECYCLE_EVENT_TYPES else None terminal_event_seen = False preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" + malformed_error_rewrite = _rewrite_malformed_stream_error_event( + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + event=event, + event_type=event_type, + event_payload=first_payload, + response_id=response_id, + ) + if malformed_error_rewrite is not None: + first, event, first_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( api_key=api_key, @@ -765,26 +778,18 @@ async def _stream_once( raise terminal_stream_error async for line in iterator: event_payload = parse_sse_data_json(line) - event = parse_sse_event_payload(event_payload) - event_type = _event_type_from_payload(event, event_payload) + event_type = classify_event_type(event_payload) + event = parse_sse_event_payload(event_payload) if event_type in _LIFECYCLE_EVENT_TYPES else None preserve_raw_sse_line = not enforce_openai_sdk_contract and event_type == "error" - if ( - enforce_openai_sdk_contract - and event_type == "error" - and (event is None or event.error is None) - and isinstance(event_payload, dict) - ): - message_value = event_payload.get("message") - message = ( - message_value.strip() - if isinstance(message_value, str) and message_value.strip() - else "Upstream error" - ) - line, event, event_payload, event_type = _facade()._build_rewritten_stream_response_failed_event( - response_id=response_id, - error_code="upstream_error", - error_message=message, - ) + malformed_error_rewrite = _rewrite_malformed_stream_error_event( + enforce_openai_sdk_contract=enforce_openai_sdk_contract, + event=event, + event_type=event_type, + event_payload=event_payload, + response_id=response_id, + ) + if malformed_error_rewrite is not None: + line, event, event_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( api_key=api_key, diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 404c63487e..441b3e29e1 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -22,6 +22,7 @@ from app.core.errors import OpenAIErrorEnvelope, openai_error from app.core.openai.model_registry import get_model_registry from app.core.openai.models import OpenAIEvent +from app.core.openai.parsing import classify_event_type from app.core.plan_types import account_plan_matches_allowed from app.core.resilience.network_recovery import PROCESS_NETWORK_UNAVAILABLE_CODE from app.core.resilience.overload import is_local_overload_error_code @@ -1553,14 +1554,7 @@ class _WebSocketReceiveTimeout: def _event_type_from_payload(event: OpenAIEvent | None, payload: dict[str, JsonValue] | None) -> str | None: if event is not None: return event.type - if payload is None: - return None - payload_type = payload.get("type") - if isinstance(payload_type, str): - return payload_type - if isinstance(payload.get("error"), dict): - return "error" - return None + return classify_event_type(payload) async def _wait_for_websocket_continuity_gap( diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 1d4eec183f..ee600a9db6 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -676,11 +676,21 @@ def _rewrite_websocket_downstream_response_id( if downstream_response_id is None: return payload + direct_response_id = payload.get("response_id") + rewrite_direct = isinstance(direct_response_id, str) and direct_response_id != downstream_response_id + response = payload.get("response") + nested_response_id = response.get("id") if isinstance(response, dict) else None + rewrite_nested = isinstance(nested_response_id, str) and nested_response_id != downstream_response_id + if not rewrite_direct and not rewrite_nested: + # Identity fast-path contract: callers skip re-serialization when the + # original payload object comes back, so an already-aligned frame must + # not be copied into an equal-but-new dict. + return payload + rewritten = dict(payload) - if isinstance(rewritten.get("response_id"), str): + if rewrite_direct: rewritten["response_id"] = downstream_response_id - response = rewritten.get("response") - if isinstance(response, dict) and isinstance(response.get("id"), str): + if rewrite_nested and isinstance(response, dict): rewritten["response"] = {**response, "id": downstream_response_id} return rewritten diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index bdd122a2e0..d11cbdc3e8 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -7,7 +7,7 @@ import time from collections import deque from contextlib import contextmanager -from dataclasses import replace +from dataclasses import dataclass, replace from datetime import datetime from typing import Any, Iterator, Mapping, NoReturn, cast @@ -71,7 +71,11 @@ from app.core.exceptions import AppError, ProxyAuthError from app.core.openai.exceptions import ClientPayloadError from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event +from app.core.openai.parsing import ( + _LIFECYCLE_EVENT_TYPES, + classify_event_type, + parse_sse_event_payload, +) from app.core.openai.requests import ( ResponsesRequest, ) @@ -84,7 +88,7 @@ from app.core.upstream_proxy import UpstreamProxyRouteError from app.core.utils.request_id import get_request_id, reset_request_id, set_request_id from app.core.utils.sse import CODEX_KEEPALIVE_FRAME as CODEX_KEEPALIVE_FRAME # noqa: F401 -from app.core.utils.sse import format_sse_event, parse_sse_data_json +from app.core.utils.sse import format_sse_event from app.core.utils.time import utcnow as utcnow from app.db.models import ( Account, @@ -323,7 +327,6 @@ _clear_websocket_precreated_replay_fallback, _clear_websocket_request_error_overrides, _DownstreamWebSocketActivity, - _event_type_from_payload, _finalize_ttft_reasoning_deltas, _PreparedWebSocketRequest, _record_response_event, @@ -693,35 +696,52 @@ def _websocket_archive_request_state_for_payload( ) +@dataclass(frozen=True, slots=True) +class _ParsedUpstreamWebSocketFrame: + payload: dict[str, JsonValue] | None + event_type: str | None + event: OpenAIEvent | None + + +def _parse_upstream_websocket_text_frame(text: str) -> _ParsedUpstreamWebSocketFrame: + """Decode an upstream websocket text frame exactly once. + + The payload is json-decoded a single time, the event type is classified + from the parsed dict, and pydantic validation runs only for lifecycle + frames (the only events whose validated model fields the proxy consumes). + """ + try: + raw_payload = json.loads(text) + except json.JSONDecodeError: + raw_payload = None + payload = cast(dict[str, JsonValue], raw_payload) if isinstance(raw_payload, dict) else None + event_type = classify_event_type(payload) + event = parse_sse_event_payload(payload) if event_type in _LIFECYCLE_EVENT_TYPES else None + return _ParsedUpstreamWebSocketFrame(payload=payload, event_type=event_type, event=event) + + async def _websocket_archive_request_id_for_message( message: Any, *, pending_requests: deque[_WebSocketRequestState], pending_lock: anyio.Lock, + parsed_frame: _ParsedUpstreamWebSocketFrame | None = None, ) -> str | None: if message.kind != "text" or message.text is None: async with pending_lock: if len(pending_requests) == 1: return pending_requests[0].archive_request_id return None - event_block = f"data: {message.text}\n\n" - payload = parse_sse_data_json(event_block) - if payload is None: - try: - raw_payload = json.loads(message.text) - except json.JSONDecodeError: - raw_payload = None - if isinstance(raw_payload, dict): - payload = cast(dict[str, JsonValue], raw_payload) - event_block = format_sse_event(payload) - event = parse_sse_event(event_block) - event_type = _event_type_from_payload(event, payload) + # Archive attribution only needs the payload dict (response ids and error + # fields are read from it directly), so reuse the caller's parsed frame + # when provided and never re-validate non-lifecycle deltas. + frame = parsed_frame if parsed_frame is not None else _parse_upstream_websocket_text_frame(message.text) async with pending_lock: request_state = _websocket_archive_request_state_for_payload( pending_requests, - event=event, - payload=payload, - event_type=event_type, + event=frame.event, + payload=frame.payload, + event_type=frame.event_type, ) return None if request_state is None else request_state.archive_request_id @@ -939,10 +959,12 @@ async def _process_and_forward_upstream_websocket_text( continuity_state: _WebSocketContinuityState | None, codex_session_affinity: bool, ) -> bool: + parsed_frame = _parse_upstream_websocket_text_frame(text) archive_request_id = await _websocket_archive_request_id_for_message( message, pending_requests=pending_requests, pending_lock=pending_lock, + parsed_frame=parsed_frame, ) _archive_received_websocket_message( upstream, @@ -951,6 +973,7 @@ async def _process_and_forward_upstream_websocket_text( ) downstream_text = await proxy._process_upstream_websocket_text( text, + parsed_frame=parsed_frame, account=account, account_id_value=account_id_value, pending_requests=pending_requests, @@ -5040,21 +5063,15 @@ async def _process_upstream_websocket_text( response_create_gate: asyncio.Semaphore, continuity_state: "_WebSocketContinuityState | None" = None, codex_session_affinity: bool = False, + parsed_frame: _ParsedUpstreamWebSocketFrame | None = None, ) -> str: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - event_block = f"data: {text}\n\n" - payload = parse_sse_data_json(event_block) - if payload is None: - try: - raw_payload = json.loads(text) - except json.JSONDecodeError: - raw_payload = None - if isinstance(raw_payload, dict): - payload = cast(dict[str, JsonValue], raw_payload) - event_block = format_sse_event(payload) - event = parse_sse_event(event_block) - event_type = _event_type_from_payload(event, payload) + if parsed_frame is None: + parsed_frame = _parse_upstream_websocket_text_frame(text) + payload = parsed_frame.payload + event_type = parsed_frame.event_type + event = parsed_frame.event response_id = _websocket_response_id(event, payload) error_message = _websocket_event_error_message(event_type, payload) is_typeless_error_event = ( @@ -5079,10 +5096,14 @@ async def _process_upstream_websocket_text( message=error_message, ) previous_response_id_hint = _facade()._previous_response_id_from_not_found_message(error_message) + # The returned event block is unused here; the rewrite helper rebuilds + # its own canonical block on the (rare) changed path, so avoid the + # per-frame ``format_sse_event`` re-encode and pass the raw framing. text, payload, event, event_type, _event_block = rewrite_parallel_tool_call_text( text, payload, - event_block=format_sse_event(payload) if payload is not None else f"data: {text}\n\n", + event_block=f"data: {text}\n\n", + event=event, ) async with pending_lock: @@ -5166,8 +5187,10 @@ async def _process_upstream_websocket_text( request_state.suppress_next_created_downstream = False upstream_control.suppress_downstream_event = True if payload is not None: - payload = _rewrite_websocket_downstream_response_id(payload, request_state) - text = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) + rewritten_payload = _rewrite_websocket_downstream_response_id(payload, request_state) + if rewritten_payload is not payload: + payload = rewritten_payload + text = json.dumps(payload, ensure_ascii=True, separators=(",", ":")) sequence_number = payload.get("sequence_number") if isinstance(sequence_number, int) and not isinstance(sequence_number, bool): upstream_control.downstream_sequence_request_state = request_state diff --git a/app/modules/proxy/tool_call_dedupe.py b/app/modules/proxy/tool_call_dedupe.py index 6c1c676799..6c2aa756d4 100644 --- a/app/modules/proxy/tool_call_dedupe.py +++ b/app/modules/proxy/tool_call_dedupe.py @@ -7,7 +7,7 @@ from app.core.openai import tool_call_safety from app.core.openai.models import OpenAIEvent -from app.core.openai.parsing import parse_sse_event_payload +from app.core.openai.parsing import classify_event_type, parse_sse_event_payload from app.core.types import JsonValue from app.core.utils.sse import format_sse_event @@ -36,14 +36,7 @@ def is_downstream_side_effect_tool_call(item: Mapping[str, JsonValue]) -> bool: def event_type_from_payload(event: OpenAIEvent | None, payload: dict[str, JsonValue] | None) -> str | None: if event is not None: return event.type - if payload is None: - return None - payload_type = payload.get("type") - if isinstance(payload_type, str): - return payload_type - if isinstance(payload.get("error"), dict): - return "error" - return None + return classify_event_type(payload) def response_id_from_payload(payload: dict[str, JsonValue] | None) -> str | None: @@ -785,10 +778,10 @@ def rewrite_parallel_tool_call_text( ) -> tuple[str, dict[str, JsonValue] | None, OpenAIEvent | None, str | None, str]: rewritten_payload, changed, _removed_count = rewrite_parallel_tool_call_payload(payload) if not changed: - # Reuse the caller's parsed event; validating the payload directly - # avoids re-parsing the raw block when a caller has neither. - if event is None: - event = parse_sse_event_payload(payload) + # Reuse the caller's parsed event as-is. Hot streaming paths validate + # only lifecycle frames, so ``event`` is intentionally None for the + # delta bulk; the event type is classified from the payload dict + # instead of re-validating per frame. return text, payload, event, event_type_from_payload(event, payload), event_block assert rewritten_payload is not None rewritten_text = json.dumps(rewritten_payload, ensure_ascii=True, separators=(",", ":")) @@ -811,8 +804,8 @@ def rewrite_parallel_tool_call_sse_line( ) -> tuple[str, dict[str, JsonValue] | None, OpenAIEvent | None, str | None]: rewritten_payload, changed, _removed_count = rewrite_parallel_tool_call_payload(payload) if not changed: - if event is None: - event = parse_sse_event_payload(payload) + # See rewrite_parallel_tool_call_text: no per-frame re-validation on + # the unchanged path; callers own lifecycle-gated validation. return line, payload, event, event_type_from_payload(event, payload) assert rewritten_payload is not None rewritten_line = format_sse_event(rewritten_payload) diff --git a/openspec/changes/validate-stream-lifecycle-events-only/design.md b/openspec/changes/validate-stream-lifecycle-events-only/design.md new file mode 100644 index 0000000000..05c8e5b006 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/design.md @@ -0,0 +1,91 @@ +# Design — validate-stream-lifecycle-events-only + +## Context + +py-spy GIL profiles attribute ~3% of proxy CPU to `validate_python` on the +stream hot path, plus a second full validation per websocket frame inside the +parallel-tool-call rewrite and redundant JSON extract+parse in the core client +and websocket relay. Every consumer of the validated `OpenAIEvent` model reads +it only on lifecycle frames; delta frames need only the `type` string. + +## Goals / Non-Goals + +**Goals:** pydantic validation only on lifecycle frames; one `json.loads` per +frame per owning layer; byte-identical SSE output; unchanged settlement, +error-classification, and terminal-detection semantics. + +**Non-Goals:** verbatim relay of unmodified SSE delta frames (the +`format_sse_event` canonical re-encode in the streaming mixin stays — that is +the separate `relay-unmodified-sse-frames-verbatim` follow-up); touching the +chat/completions bridge (genuine cross-dialect translation keeps full +parsing); the `/v1` public normalizer (independent per-chunk consumer); the +cold rewrite/prewarm paths (`limit_warmup`, `quota_planner`, +`http_bridge/helpers`, `websocket/helpers` rewrite builders), which run once +per stream or per rewrite. + +## Decisions + +- **Lifecycle set = {response.created, response.completed, + response.incomplete, response.failed, error}.** Terminal frames carry the + usage and error fields settlement needs; `response.created` is included + because the websocket path assigns the upstream response id from the + validated model there. +- **Classify-then-validate.** `classify_event_type(payload)` mirrors the dict + branch of `_event_type_from_payload` exactly (string `type` wins; a typeless + dict `error` classifies as `"error"`). Ordering classification before + validation is equivalence-preserving: when validation succeeds, + `event.type == payload["type"]`; when it fails (e.g. typeless error + payloads, or a lifecycle `type` with a non-dict `response` — `OpenAIEvent` + has no before-validator on `response`), today's code already fell back to + the same dict branch with `event=None`. +- **`event=None` for non-lifecycle frames compiles against existing + structure.** Every downstream read of `event.response`/`event.error` in the + streaming mixin, websocket finalization, and bridge settlement is gated on a + terminal `event_type`, so non-lifecycle `None` never reaches them. +- **Core-client deletion is a pure no-op.** At the two SSE loops the + `elif normalized_event_type` branch performed the identical terminal check + from the same payload the deleted `parse_sse_event` re-parsed; whenever the + model validated, its `type` equalled `normalized_event_type`. The websocket + receive loops now use the payload `type` string directly, matching the + dict semantics the SSE loops already had (a malformed lifecycle frame that + failed whole-event validation now counts for terminal detection there, as + it already did on the SSE loops). +- **Rewrite helpers no longer validate on the unchanged path.** The + `event is None` fallback in `rewrite_parallel_tool_call_text/_sse_line` was + the second per-frame validation on the websocket path; callers now own + lifecycle-gated validation and the helpers classify from the dict. The + changed path (actual dedupe rewrite of `response.output_item.done`) still + validates the rewritten payload as before. +- **Websocket identity skip.** `_rewrite_websocket_downstream_response_id` + returns the same object when no replay rewrite applies; only then is the + per-frame `json.dumps` skipped and the upstream text relayed unchanged. + Frames without a matched request state were already relayed verbatim, so + downstream clients already accept upstream-encoded frames on this surface. + +## Risks / Trade-offs + +- [Whitespace-padded response ids] Non-lifecycle frames that carry a + `response` object (`response.in_progress`) now resolve their response id via + the dict fallback, which strips whitespace, where the model path did not. + Upstream ids are never whitespace-padded; `response.created` (the id + assignment point) keeps the model path. +- [Usage on delta frames] If upstream ever emitted `usage` on non-lifecycle + frames it would no longer be validated — accepted limitation; today usage + appears only on `response.completed`/`response.incomplete`, and no consumer + reads usage outside terminal branches. +- [Websocket byte relaxation] Identity frames relay upstream JSON text + (raw UTF-8, upstream spacing) instead of the canonical + `ensure_ascii` re-encode. JSON-semantically identical, and consistent with + the existing unmatched-frame behavior on the same socket. +- [First-frame response-id capture] A stream whose first frame is a + non-lifecycle `response`-carrying frame (never observed; upstream always + opens with `response.created`) no longer captures `settlement.response_id` + from that frame; it is still captured at the terminal frame. + +## Migration Plan + +Code-only; rollback = revert. + +## Open Questions + +None. diff --git a/openspec/changes/validate-stream-lifecycle-events-only/proposal.md b/openspec/changes/validate-stream-lifecycle-events-only/proposal.md new file mode 100644 index 0000000000..3fe9be28dd --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/proposal.md @@ -0,0 +1,76 @@ +# Validate Stream Lifecycle Events Only + +## Why + +Every streamed SSE/websocket event still pays a full pydantic `OpenAIEvent` +validation per layer, even though every consumer of the validated model reads +it only on stream lifecycle frames: usage/token settlement +(`response.completed`/`response.incomplete`), error normalization and +retry/health classification (`response.failed`/`error`), and websocket +response-id assignment (`response.created`). Delta frames — the dominant +traffic by two to three orders of magnitude — only ever need their `type` +string, which the already-parsed payload dict provides. On top of that, the +core client validated each chunk a second time solely to duplicate a terminal +check it had already computed from the dict, the websocket relay extracted and +re-parsed each frame's JSON twice, built a canonical SSE re-encode per frame +just to populate a discarded argument, re-validated each frame inside the +parallel-tool-call rewrite, and re-encoded `json.dumps` on every matched frame +even when the response-id rewrite changed nothing. This is the follow-up the +`2026-07-13-optimize-sse-single-parse` design doc deferred ("threading a +parsed-event struct across layer boundaries"). + +## What Changes + +- `app/core/openai/parsing.py` gains the shared `_LIFECYCLE_EVENT_TYPES` + frozenset (`response.created`, `response.completed`, `response.incomplete`, + `response.failed`, `error`) and `classify_event_type(payload)`, the dict-only + classifier that `_event_type_from_payload` and + `tool_call_dedupe.event_type_from_payload` now delegate to. +- Streaming mixin, websocket relay, and HTTP-bridge upstream reader classify + each frame from the parsed dict first and run `parse_sse_event_payload` + only for lifecycle frames; all other frames flow with `event=None`, which + every downstream branch already guards for. +- Core client: the redundant per-chunk `parse_sse_event` terminal checks are + deleted (the `normalized_event_type` dict branch is the same check from the + same payload); the websocket receive loops detect terminal frames from + `parse_sse_data_json` + the payload `type` string. +- Websocket relay: single `json.loads` per frame (no synthetic-block re-parse), + the caller's `event` is passed into `rewrite_parallel_tool_call_text` and the + rewrite helpers no longer re-validate on the unchanged path, the discarded + `format_sse_event` argument is no longer built, and the downstream + response-id re-encode is skipped when the rewrite returned the payload + unchanged (identity), relaying the upstream frame bytes as-is. +- No output-byte change on the SSE paths: canonical `format_sse_event` + serialization, usage settlement, error rewriting, and terminal detection are + unchanged. Existing streaming/websocket suites pass unmodified. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: the single-parse streaming requirement is extended — + schema validation of parsed stream payloads MUST run only for lifecycle + frames, with all other frames classified from the parsed payload dict and + all downstream semantics (framing, settlement, error normalization) + unchanged; identity websocket relay frames are forwarded without a canonical + re-encode. + +## Impact + +- **Code**: `app/core/openai/parsing.py`, `app/core/clients/proxy.py`, + `app/modules/proxy/tool_call_dedupe.py`, + `app/modules/proxy/_service/support.py`, + `app/modules/proxy/_service/streaming/mixin.py`, + `app/modules/proxy/_service/websocket/mixin.py`, + `app/modules/proxy/_service/http_bridge/upstream_events.py`. +- **Behavior**: none on SSE surfaces. Websocket relay frames whose matched + response-id rewrite is an identity are now forwarded with the upstream JSON + text instead of a canonical `json.dumps` re-encode (JSON-equivalent; frames + without a matched request state were already forwarded verbatim). +- **Performance**: removes 1–2 pydantic validations and 1–2 redundant JSON + parses per delta frame per layer, plus one wasted `format_sse_event` and one + wasted `json.dumps` per websocket frame. diff --git a/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md b/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..f9fe0a7b95 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/specs/responses-api-compat/spec.md @@ -0,0 +1,36 @@ +# responses-api-compat Delta + +## MODIFIED Requirements + +### Requirement: Streaming events are parsed once and re-serialized only when modified + +Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, websocket relay, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Schema validation of the parsed payload MUST run only for stream lifecycle frames (`response.created`, `response.completed`, `response.incomplete`, `response.failed`, `error`); all other frames MUST be classified from the parsed payload's `type` field (with a typeless payload carrying an `error` object classifying as `error`). Event framing, payload contents, dedupe/rewrite semantics, usage settlement, and error normalization MUST be unchanged. + +#### Scenario: Unmodified events pass through the /v1 normalizer verbatim + +- **GIVEN** a canonical stream event that no normalizer branch rewrites +- **WHEN** the /v1 response normalizer processes it +- **THEN** the original block is yielded byte-identically without re-serialization + +#### Scenario: Tool-call rewrite reuses the parsed event on the no-change path + +- **GIVEN** an event without duplicate parallel tool calls +- **WHEN** the rewrite step runs with the caller's parsed event +- **THEN** it returns the original line, payload, and event without re-parsing or re-validating + +#### Scenario: Rewritten events stay consistent + +- **WHEN** the rewrite step removes duplicate tool calls +- **THEN** the returned line, payload, and validated event all reflect the rewritten content + +#### Scenario: Delta frames skip schema validation + +- **GIVEN** a stream of `response.output_text.delta` frames between `response.created` and `response.completed` +- **WHEN** the streaming mixin, websocket relay, or bridge upstream reader processes the stream +- **THEN** only the lifecycle frames are schema-validated, the delta frames are classified from the parsed payload dict, and downstream output, usage settlement, and error normalization are unchanged + +#### Scenario: Identity websocket relay frames are forwarded without re-encoding + +- **GIVEN** a websocket frame matched to a request whose downstream response-id rewrite does not apply +- **WHEN** the relay forwards the frame downstream +- **THEN** the upstream frame text is forwarded as-is instead of a canonical JSON re-encode diff --git a/openspec/changes/validate-stream-lifecycle-events-only/tasks.md b/openspec/changes/validate-stream-lifecycle-events-only/tasks.md new file mode 100644 index 0000000000..7fddb00bc0 --- /dev/null +++ b/openspec/changes/validate-stream-lifecycle-events-only/tasks.md @@ -0,0 +1,29 @@ +# Tasks — validate-stream-lifecycle-events-only + +## 1. Implementation + +- [x] 1.1 `_LIFECYCLE_EVENT_TYPES` + `classify_event_type` in + `app/core/openai/parsing.py`; `_event_type_from_payload` and + `tool_call_dedupe.event_type_from_payload` delegate to it +- [x] 1.2 Streaming mixin (first-event block + loop): classify from dict, + validate lifecycle frames only +- [x] 1.3 Core client: delete redundant per-chunk `parse_sse_event` terminal + checks (SSE loops keep the `normalized_event_type` branch); websocket + receive loops detect terminal via `parse_sse_data_json` + `type` +- [x] 1.4 Websocket relay: single `json.loads`, lifecycle-only validation, + pass `event=` into `rewrite_parallel_tool_call_text`, stop building the + discarded `format_sse_event` argument, skip `json.dumps` on identity + response-id rewrite +- [x] 1.5 `tool_call_dedupe` rewrite helpers: no re-validation on the + unchanged path +- [x] 1.6 HTTP-bridge upstream reader: same lifecycle gating + +## 2. Validation + +- [x] 2.1 Existing streaming/websocket/bridge unit + integration suites pass + unmodified (byte-level SSE assertions act as the parity oracle) +- [x] 2.2 Regression tests: lifecycle-only validation counts with interleaved + `event=None` deltas on the SSE mixin and websocket relay (usage + settlement, error rewrite, response-id assignment), dedupe helpers do + not re-validate unchanged frames, `classify_event_type` unit coverage +- [x] 2.3 `uvx ruff format --check`, `uv run ruff check` on changed files diff --git a/tests/unit/test_proxy_tool_call_dedupe.py b/tests/unit/test_proxy_tool_call_dedupe.py index 71e48b21a0..f9ad9f0749 100644 --- a/tests/unit/test_proxy_tool_call_dedupe.py +++ b/tests/unit/test_proxy_tool_call_dedupe.py @@ -2178,3 +2178,59 @@ def test_rewrite_parallel_tool_call_payload_removes_duplicate_goal_side_effects( "functions.update_plan", "functions.request_user_input", ] + + +def test_rewrite_parallel_tool_call_text_does_not_revalidate_unchanged_frames( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Hot streaming paths validate only lifecycle frames; the unchanged path + # must not re-run pydantic validation when the caller passed event=None. + calls: list[JsonValue | None] = [] + + def counting_validate(payload: JsonValue | None) -> None: + calls.append(payload) + return None + + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", counting_validate) + payload: dict[str, JsonValue] = {"type": "response.output_text.delta", "delta": "hello"} + text = json.dumps(payload, separators=(",", ":")) + + rewritten_text, rewritten_payload, event, event_type, event_block = ( + tool_call_dedupe.rewrite_parallel_tool_call_text( + text, + payload, + event_block=f"data: {text}\n\n", + ) + ) + + assert calls == [] + assert event is None + assert event_type == "response.output_text.delta" + assert rewritten_text == text + assert rewritten_payload is payload + assert event_block == f"data: {text}\n\n" + + +def test_rewrite_parallel_tool_call_sse_line_does_not_revalidate_unchanged_frames( + monkeypatch: pytest.MonkeyPatch, +) -> None: + calls: list[JsonValue | None] = [] + + def counting_validate(payload: JsonValue | None) -> None: + calls.append(payload) + return None + + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", counting_validate) + payload: dict[str, JsonValue] = {"type": "response.reasoning_text.delta", "delta": "r"} + line = format_sse_event(payload) + + rewritten_line, rewritten_payload, event, event_type = tool_call_dedupe.rewrite_parallel_tool_call_sse_line( + line, + payload, + ) + + assert calls == [] + assert event is None + assert event_type == "response.reasoning_text.delta" + assert rewritten_line == line + assert rewritten_payload is payload diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 70f8db1e34..d495ced37b 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -8798,7 +8798,7 @@ async def fake_stream_websocket_events( del enforce_openai_sdk_contract recorded["total_timeout_seconds"] = total_timeout_seconds if False: - yield "" + yield "", None monkeypatch.setattr(proxy_module, "_open_upstream_websocket", fake_open_upstream_websocket) monkeypatch.setattr(proxy_module, "_stream_websocket_events", fake_stream_websocket_events) @@ -8877,7 +8877,9 @@ async def fake_open_upstream_websocket( ] assert len(events) == 1 - assert parse_sse_data_json(events[0]) == raw_payload + event_block, event_type = events[0] + assert parse_sse_data_json(event_block) == raw_payload + assert event_type == "error" assert successes == 1 assert failures == [] @@ -8907,10 +8909,111 @@ async def test_stream_codex_websocket_events_treats_raw_error_as_terminal_when_s ] assert len(events) == 1 - assert parse_sse_data_json(events[0]) == raw_payload + event_block, event_type = events[0] + assert parse_sse_data_json(event_block) == raw_payload + assert event_type == "error" assert websocket._index == 1 +@pytest.mark.asyncio +async def test_stream_responses_websocket_decodes_each_frame_once_and_skips_error_validation(monkeypatch): + # Regression for the parse-once requirement on the websocket hot path: + # each upstream frame is json-decoded exactly once in the receive loop, + # the relay and outer stream loops reuse the threaded event type instead + # of re-parsing the formatted block, and no error-envelope validation + # runs for non-error-shaped frames. + class Settings: + upstream_base_url = "https://chatgpt.com/backend-api" + upstream_stream_transport = "websocket" + upstream_connect_timeout_seconds = 8.0 + stream_idle_timeout_seconds = 45.0 + max_sse_event_bytes = 1024 + image_inline_fetch_enabled = False + trace_channels = frozenset() + proxy_request_budget_seconds = 75.0 + + monkeypatch.setattr(proxy_module, "get_settings", lambda: Settings()) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_start", lambda **kwargs: None) + monkeypatch.setattr(proxy_module, "_maybe_log_upstream_request_complete", lambda **kwargs: None) + + frame_payloads = [ + {"type": "response.created", "response": {"id": "resp_ws_once"}}, + {"type": "response.output_text.delta", "delta": "a"}, + {"type": "response.output_text.delta", "delta": "b"}, + {"type": "response.completed", "response": {"id": "resp_ws_once"}}, + ] + frame_texts = [json.dumps(payload, ensure_ascii=True, separators=(",", ":")) for payload in frame_payloads] + websocket = _WsResponse([_WsMessage(proxy_module.aiohttp.WSMsgType.TEXT, text) for text in frame_texts]) + session = _WsSession(websocket) + + loads_spy = MagicMock(wraps=json.loads) + monkeypatch.setattr(proxy_module.json, "loads", loads_spy) + parse_spy = MagicMock(wraps=proxy_module.parse_sse_data_json) + monkeypatch.setattr(proxy_module, "parse_sse_data_json", parse_spy) + error_validate_spy = MagicMock(wraps=proxy_module.parse_error_payload) + monkeypatch.setattr(proxy_module, "parse_error_payload", error_validate_spy) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "hi", + "input": [{"role": "user", "content": "hi"}], + "stream": True, + } + ) + events = [ + event + async for event in proxy_module.stream_responses( + payload, + headers={}, + access_token="token", + account_id="acc_ws_parse_once", + session=cast(proxy_module.aiohttp.ClientSession, session), + ) + ] + + assert events == [proxy_module.format_sse_event(payload) for payload in frame_payloads] + for text in frame_texts: + decode_calls = [c for c in loads_spy.call_args_list if c.args and c.args[0] == text] + assert len(decode_calls) == 1 + assert parse_spy.call_count == 0 + assert error_validate_spy.call_count == 0 + + +def test_normalize_stream_event_payload_validates_error_envelope_only_for_error_shaped_frames(monkeypatch): + # Delta frames never reach the pydantic error-envelope adapter; error + # frames are still validated and rewritten exactly as before. + from app.core.openai import parsing as parsing_module + + adapter_spy = MagicMock(wraps=parsing_module._ERROR_ADAPTER) + monkeypatch.setattr(parsing_module, "_ERROR_ADAPTER", adapter_spy) + + delta_payload: dict[str, Any] = {"type": "response.output_text.delta", "delta": "a"} + assert proxy_module._normalize_stream_event_payload(delta_payload) is delta_payload + in_progress_payload: dict[str, Any] = {"type": "response.in_progress", "response": {"id": "resp_np"}} + assert proxy_module._normalize_stream_event_payload(in_progress_payload) is in_progress_payload + assert adapter_spy.validate_python.call_count == 0 + + envelope_payload: dict[str, Any] = { + "error": {"message": "quota exhausted", "type": "server_error", "code": "insufficient_quota"} + } + rewritten: Any = proxy_module._normalize_stream_event_payload(envelope_payload) + assert adapter_spy.validate_python.call_count == 1 + assert rewritten["type"] == "response.failed" + envelope_error = rewritten["response"]["error"] + assert envelope_error["message"] == "quota exhausted" + assert envelope_error["code"] == proxy_module._normalize_error_code("insufficient_quota", "server_error") + assert envelope_error["type"] == "server_error" + + bare_error_payload: dict[str, Any] = {"type": "error", "code": "rate_limit_exceeded", "message": "slow down"} + rewritten_bare: Any = proxy_module._normalize_stream_event_payload(bare_error_payload) + assert adapter_spy.validate_python.call_count == 2 + assert rewritten_bare["type"] == "response.failed" + bare_error = rewritten_bare["response"]["error"] + assert bare_error["code"] == proxy_module._normalize_error_code("rate_limit_exceeded", "error") + assert bare_error["message"] == "slow down" + + @pytest.mark.asyncio async def test_stream_responses_websocket_broken_pipe_is_not_replayable_upstream_unavailable(monkeypatch): logged_completions: list[dict[str, object]] = [] @@ -46233,3 +46336,417 @@ def test_compact_account_neutral_replay_payload_accepts_canonical_lite_full_rese replay = proxy_compact_service._compact_account_neutral_replay_payload(payload) assert replay is not None assert getattr(replay, "previous_response_id", None) is None + + +@pytest.mark.asyncio +async def test_stream_with_retry_validates_only_lifecycle_frames_and_settles_usage(monkeypatch): + # Delta frames must skip pydantic validation entirely (event=None) while + # lifecycle frames keep it, with byte-identical SSE output and unchanged + # usage settlement from the validated response.completed frame. + from app.modules.proxy import tool_call_dedupe + from app.modules.proxy._service.streaming import mixin as streaming_mixin_module + + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_lifecycle_only_validation") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + mixin_validate = MagicMock(wraps=streaming_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(streaming_mixin_module, "parse_sse_event_payload", mixin_validate) + dedupe_validate = MagicMock(wraps=tool_call_dedupe.parse_sse_event_payload) + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", dedupe_validate) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"response.created","response":{"id":"resp_lifecycle_only"}}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"b"}\n\n' + yield ( + 'data: {"type":"response.completed","response":{"id":"resp_lifecycle_only",' + '"usage":{"input_tokens":3,"output_tokens":5,' + '"input_tokens_details":{"cached_tokens":2},' + '"output_tokens_details":{"reasoning_tokens":1}}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-lifecycle-only"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + # Only the created + completed lifecycle frames are validated; the two + # deltas are classified from the parsed dict and never re-validated by + # the parallel-tool-call rewrite either. + assert mixin_validate.call_count == 2 + assert dedupe_validate.call_count == 0 + # SSE output stays byte-identical to the canonical re-encode. + assert chunks[1] == 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + assert json.loads(chunks[-1].split("data: ", 1)[1])["type"] == "response.completed" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["request_id"] == "resp_lifecycle_only" + assert request_logs.calls[0]["input_tokens"] == 3 + assert request_logs.calls[0]["output_tokens"] == 5 + assert request_logs.calls[0]["cached_input_tokens"] == 2 + assert request_logs.calls[0]["reasoning_tokens"] == 1 + + +@pytest.mark.asyncio +async def test_stream_with_retry_rewrites_terminal_error_after_unvalidated_deltas(monkeypatch): + # A bare upstream ``error`` frame after unvalidated deltas must still be + # rewritten to a terminal response.failed under the SDK contract and + # settle as an error. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_lifecycle_error_rewrite") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"response.created","response":{"id":"resp_lifecycle_err"}}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"error","message":"upstream exploded"}\n\n' + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-lifecycle-error"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + terminal = json.loads(chunks[-1].split("data: ", 1)[1]) + assert terminal["type"] == "response.failed" + assert terminal["response"]["error"]["message"] == "upstream exploded" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "error" + + +@pytest.mark.asyncio +async def test_process_upstream_websocket_text_validates_only_lifecycle_frames(monkeypatch): + # Websocket relay: response.created keeps validated response-id + # assignment, deltas skip validation and are relayed with their original + # bytes when no response-id rewrite applies, and response.completed still + # hands a validated usage-bearing event to finalization. + from app.core.openai.models import OpenAIEvent + from app.modules.proxy import tool_call_dedupe + from app.modules.proxy._service.websocket import mixin as websocket_mixin_module + + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_lifecycle_validation") + finalize_request_state = AsyncMock() + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request_state) + ws_validate = MagicMock(wraps=websocket_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(websocket_mixin_module, "parse_sse_event_payload", ws_validate) + dedupe_validate = MagicMock(wraps=tool_call_dedupe.parse_sse_event_payload) + monkeypatch.setattr(tool_call_dedupe, "parse_sse_event_payload", dedupe_validate) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_lifecycle_validation", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_create_gate_acquired=True, + ) + pending_requests = deque([request_state]) + pending_lock = anyio.Lock() + upstream_control = proxy_service._WebSocketUpstreamControl() + response_create_gate = asyncio.Semaphore(0) + + async def relay(text: str) -> str: + return await service._process_upstream_websocket_text( + text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=pending_lock, + api_key=None, + upstream_control=upstream_control, + response_create_gate=response_create_gate, + ) + + created_text = json.dumps( + {"type": "response.created", "response": {"id": "resp_ws_lifecycle", "status": "in_progress"}}, + separators=(",", ":"), + ) + await relay(created_text) + assert request_state.response_id == "resp_ws_lifecycle" + assert ws_validate.call_count == 1 + + delta_text = json.dumps( + { + "type": "response.output_text.delta", + "response_id": "resp_ws_lifecycle", + "sequence_number": 3, + "delta": "hello", + }, + separators=(",", ":"), + ) + downstream_delta = await relay(delta_text) + assert ws_validate.call_count == 1 + assert dedupe_validate.call_count == 0 + assert downstream_delta == delta_text + finalize_request_state.assert_not_awaited() + + completed_text = json.dumps( + { + "type": "response.completed", + "response": { + "id": "resp_ws_lifecycle", + "status": "completed", + "usage": {"input_tokens": 11, "output_tokens": 7}, + }, + }, + separators=(",", ":"), + ) + await relay(completed_text) + assert ws_validate.call_count == 2 + finalize_request_state.assert_awaited_once() + finalize_call = finalize_request_state.await_args + assert finalize_call is not None + assert finalize_call.kwargs["event_type"] == "response.completed" + completed_event = finalize_call.kwargs["event"] + assert isinstance(completed_event, OpenAIEvent) + assert completed_event.response is not None + assert completed_event.response.usage is not None + assert completed_event.response.usage.input_tokens == 11 + assert completed_event.response.usage.output_tokens == 7 + + +@pytest.mark.asyncio +async def test_stream_with_retry_rewrites_malformed_error_when_it_is_the_first_frame(monkeypatch): + # Regression: a malformed first upstream frame like + # {"type":"error","message":"..."} classifies as "error" but carries no + # error envelope (event=None). The first-frame path must apply the same + # SDK-contract fallback as the later-frame loop: rewrite to a terminal + # response.failed and settle as an error instead of leaking the raw frame + # with a success settlement. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_first_frame_error_rewrite") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'data: {"type":"error","message":"upstream exploded first"}\n\n' + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-first-frame-error"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + assert chunks + terminal = json.loads(chunks[-1].split("data: ", 1)[1]) + assert terminal["type"] == "response.failed" + assert terminal["response"]["error"]["message"] == "upstream exploded first" + assert terminal["response"]["error"]["code"] == "upstream_error" + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] != "success" + assert request_logs.calls[0]["status"] == "error" + + +@pytest.mark.asyncio +async def test_process_upstream_websocket_text_keeps_frame_bytes_when_response_id_already_matches(): + # Regression for the response-id rewrite identity fast-path: when the + # frame already carries the assigned downstream response id, the rewrite + # helper must hand back the original payload object so the relay forwards + # the upstream text without re-encoding it. + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_response_id_identity") + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_response_id_identity", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_id="resp_ws_identity", + replay_downstream_response_id="resp_ws_identity", + ) + pending_requests = deque([request_state]) + upstream_control = proxy_service._WebSocketUpstreamControl() + + delta_text = json.dumps( + { + "type": "response.output_text.delta", + "response_id": "resp_ws_identity", + "sequence_number": 7, + "delta": "hello", + }, + separators=(",", ":"), + ) + downstream_delta = await service._process_upstream_websocket_text( + delta_text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=asyncio.Semaphore(0), + ) + + assert downstream_delta is delta_text + assert upstream_control.downstream_sequence_number == 7 + assert upstream_control.downstream_sequence_request_state is request_state + + +@pytest.mark.asyncio +async def test_process_and_forward_upstream_websocket_text_decodes_once_and_validates_lifecycle_only(monkeypatch): + # Product-path regression for the parse-once requirement: every direct + # upstream websocket text frame flows through + # _process_and_forward_upstream_websocket_text, which must json-decode the + # frame exactly once (shared by archive attribution and relay processing) + # and run pydantic validation only for lifecycle frames. + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_ws_forward_parse_once") + finalize_request_state = AsyncMock() + monkeypatch.setattr(service, "_finalize_websocket_request_state", finalize_request_state) + sent_downstream: list[str] = [] + + async def record_downstream(_websocket: object, *, text: str, **_kwargs: object) -> None: + sent_downstream.append(text) + + monkeypatch.setattr(service, "_send_downstream_websocket_text", record_downstream) + ws_validate = MagicMock(wraps=websocket_mixin_module.parse_sse_event_payload) + monkeypatch.setattr(websocket_mixin_module, "parse_sse_event_payload", ws_validate) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_forward_parse_once", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_create_gate_acquired=True, + archive_request_id="archive_ws_forward_parse_once", + ) + pending_requests = deque([request_state]) + pending_lock = anyio.Lock() + upstream_control = proxy_service._WebSocketUpstreamControl() + downstream_activity = proxy_service._DownstreamWebSocketActivity() + archived: list[tuple[object, str | None]] = [] + + class _ArchivingUpstream: + def archive_received(self, message: object) -> None: + archived.append((message, get_request_id())) + + upstream = _ArchivingUpstream() + + frame_payloads: list[dict[str, Any]] = [ + {"type": "response.created", "response": {"id": "resp_ws_forward_once", "status": "in_progress"}}, + {"type": "response.output_text.delta", "response_id": "resp_ws_forward_once", "delta": "hello"}, + { + "type": "response.completed", + "response": { + "id": "resp_ws_forward_once", + "status": "completed", + "usage": {"input_tokens": 4, "output_tokens": 2}, + }, + }, + ] + frame_texts = [json.dumps(frame, separators=(",", ":")) for frame in frame_payloads] + + loads_spy = MagicMock(wraps=json.loads) + monkeypatch.setattr(websocket_mixin_module.json, "loads", loads_spy) + + for frame_text in frame_texts: + terminal = await websocket_mixin_module._process_and_forward_upstream_websocket_text( + cast(Any, service), + cast(Any, SimpleNamespace()), + cast(Any, upstream), + message=SimpleNamespace(kind="text", text=frame_text), + text=frame_text, + account=account, + account_id_value=account.id, + pending_requests=pending_requests, + pending_lock=pending_lock, + client_send_lock=anyio.Lock(), + api_key=None, + upstream_control=upstream_control, + response_create_gate=asyncio.Semaphore(0), + downstream_activity=downstream_activity, + continuity_state=None, + codex_session_affinity=False, + ) + assert terminal is False + + # Exactly one json decode per frame across archive attribution and relay. + for frame_text in frame_texts: + decode_calls = [c for c in loads_spy.call_args_list if c.args and c.args[0] == frame_text] + assert len(decode_calls) == 1 + # Only the created + completed lifecycle frames are pydantic-validated; + # the delta is classified from the parsed dict without validation. + assert ws_validate.call_count == 2 + # Archive attribution still resolves the owning request from the shared + # parsed frame for every message. + assert [request_id for _message, request_id in archived] == ["archive_ws_forward_parse_once"] * 3 + # The delta frame is forwarded downstream with its original bytes. + assert sent_downstream[1] is frame_texts[1] + finalize_request_state.assert_awaited_once() + assert finalize_request_state.await_args is not None + assert finalize_request_state.await_args.kwargs["event_type"] == "response.completed" diff --git a/tests/unit/test_sse.py b/tests/unit/test_sse.py index 44983fbdc6..02b912c4d4 100644 --- a/tests/unit/test_sse.py +++ b/tests/unit/test_sse.py @@ -9,7 +9,7 @@ from hypothesis import given, settings from hypothesis import strategies as st -from app.core.openai.parsing import parse_sse_event +from app.core.openai.parsing import _LIFECYCLE_EVENT_TYPES, classify_event_type, parse_sse_event from app.core.utils.sse import ( CODEX_KEEPALIVE_FRAME, SSE_KEEPALIVE_FRAME, @@ -198,3 +198,30 @@ def test_extract_sse_data_joins_crlf_multiline_data(): block = "data: line1\r\ndata: line2\rdata: line3\n\n" assert extract_sse_data(block) == "line1\nline2\nline3" + + +def test_classify_event_type_prefers_string_type_field(): + assert classify_event_type({"type": "response.output_text.delta", "delta": "x"}) == "response.output_text.delta" + + +def test_classify_event_type_maps_typeless_error_payload_to_error(): + assert classify_event_type({"error": {"message": "boom"}, "status": 400}) == "error" + + +def test_classify_event_type_rejects_non_dict_and_typeless_payloads(): + assert classify_event_type(None) is None + assert classify_event_type([1, 2, 3]) is None + assert classify_event_type({"type": 42}) is None + assert classify_event_type({"delta": "x"}) is None + + +def test_lifecycle_event_types_cover_terminal_and_created_frames(): + assert _LIFECYCLE_EVENT_TYPES == frozenset( + { + "response.created", + "response.completed", + "response.incomplete", + "response.failed", + "error", + } + ) diff --git a/tests/unit/test_websocket_terminal_cancellation.py b/tests/unit/test_websocket_terminal_cancellation.py index d285e0edff..b33d5b6fd5 100644 --- a/tests/unit/test_websocket_terminal_cancellation.py +++ b/tests/unit/test_websocket_terminal_cancellation.py @@ -1102,12 +1102,14 @@ async def _observed_archive_attribution( *, pending_requests: deque[proxy_service._WebSocketRequestState], pending_lock: anyio.Lock, + parsed_frame: object | None = None, ) -> str | None: archive_attribution_started.set() return await original_archive_attribution( message, pending_requests=pending_requests, pending_lock=pending_lock, + parsed_frame=cast("websocket_mixin._ParsedUpstreamWebSocketFrame | None", parsed_frame), ) async def _blocking_release_gate( From 7dacb04181390b85bd42b335a73e27ad4d90ec2f Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 21:33:38 +0900 Subject: [PATCH 063/117] perf(api-keys,proxy): shape ORM hot-path queries (#1788) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(api-keys): narrow admission-path ApiKey load to enforcement columns enforce_limits_for_request paid the full get_by_id graph load (base row + 3 selectin round trips) twice per request when a lazy limit reset fired, while the enforcement transaction only reads is_active/expires_at and the limits collection. Add ApiKeysRepository.get_for_limit_enforcement with load_only(is_active, expires_at, raiseload=True) + selectinload(limits) and raiseload on the assignment relationships, and use it for both the admission fetch and the post-lazy-reset refetch: 4 SELECT round trips drop to 2 per request (8 to 4 on lazy reset). populate_existing is kept because the lazy reset commits mid-enforcement and the refetch must re-hydrate identity-mapped rows (sessions run expire_on_commit=False). Column audit of the enforcement path (_ensure_valid_api_key_row, _limit_applies_for_request, _reserve_delta_for_limit): only is_active and expires_at are read off ApiKey; everything else reads ApiKeyLimit rows, so the narrowed load is semantics-identical and fail-loud (raiseload) if future enforcement code touches an unlisted attribute. Co-Authored-By: Claude Fable 5 * perf(proxy): share one session across sticky owner lookups The legacy raw-key lookup, the process-seed lookup, and the sticky selection loop's first owner read each opened their own repo bundle (pool checkout + BEGIN + one SELECT + COMMIT) on every sticky request. Open one shared bundle in select_account spanning all three; the SELECTs stay separate on purpose so the per-source predicate semantics of get_account_id_and_abandonment (tombstone visibility, max_age handling, continuity-source scoping) are byte-for-byte untouched — the saving is the 2 extra session lifecycles per request, not a merged query. The first-iteration owner read is hoisted into the shared bundle and passed to run_sticky_selection_path as StickySelectionRequest.initial_sticky_owner_lookup, consumed exactly once; every retry (including post-reset attempts that wrap the attempt counter back) still re-reads fresh ownership evidence through its own repo bundle, and no sticky owner caching is introduced. Co-Authored-By: Claude Fable 5 * perf(request-logs): insert request logs via SQLAlchemy Core add_log fully builds the row before persisting, so the ORM unit-of-work flush (identity-map registration, relationship cascade scan, per-attribute history snapshots) was pure overhead on every request's log write. Execute a Core insert() built from the transient instance's columns instead; the instance stays detached and is still returned as the typed result with its primary key assigned from the insert. The relaxed-durability commit (SET LOCAL synchronous_commit=off) and the detached persistence task around this write are unchanged. Co-Authored-By: Claude Fable 5 * fix(request-logs): keep core-inserted log session-tracked The Core-insert fast path returned the RequestLog as a transient instance, so callers mutating the returned log and committing through the same session (e.g. soft-delete setting deleted_at) had their update silently discarded. After the insert resolves the primary key, re-attach the instance via make_transient_to_detached + session.add so it becomes persistent-and-clean: no extra SQL on the insert commit, but subsequent mutations are tracked as normal ORM updates. Sessions use expire_on_commit=False, so the attach adds no post-commit loads. Co-Authored-By: Claude Fable 5 * fix(request-logs): type core-insert result as CursorResult for ty Co-Authored-By: Claude Fable 5 * fix(proxy): refresh ownership-lookup snapshots and guard narrowed api-key load The shared owner-lookup session kept one SQLite/WAL read snapshot across the legacy, seed, and first-sticky SELECTs, so an owner committed concurrently between the reads stayed invisible and selection could overwrite the newly established mapping. Each later ownership source now releases the shared read snapshot first (COMMIT under expire_on_commit=False, never rollback, so tracked ORM state survives), keeping the single-session perf win while restoring per-source snapshot freshness; Postgres READ COMMITTED already read per-statement. Document the session-isolation invariant that keeps the narrowed get_for_limit_enforcement load (populate_existing + raiseload) from poisoning a previously fully-loaded ApiKey: every caller runs it in a dedicated short-lived session that never full-loads ApiKey first, which was verified against all call sites. The shared-lookup regression test now stamps each repo-bundle open with a context id and pins the final semantics: one repository context serves all three ownership reads, with a snapshot release before each later source. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/modules/api_keys/repository.py | 42 ++++++- app/modules/api_keys/service.py | 10 +- .../proxy/_load_balancer/sticky_selection.py | 77 +++++++------ app/modules/proxy/load_balancer.py | 106 ++++++++++++------ app/modules/proxy/sticky_repository.py | 16 +++ app/modules/request_logs/repository.py | 38 ++++++- tests/integration/test_api_keys_api.py | 74 +++++++++++- tests/unit/test_api_keys_service.py | 3 + tests/unit/test_load_balancer_concurrency.py | 96 ++++++++++++++++ tests/unit/test_request_logs_repository.py | 7 +- 10 files changed, 391 insertions(+), 78 deletions(-) diff --git a/app/modules/api_keys/repository.py b/app/modules/api_keys/repository.py index 2cc3fb4f8c..4a26949c58 100644 --- a/app/modules/api_keys/repository.py +++ b/app/modules/api_keys/repository.py @@ -8,7 +8,7 @@ from sqlalchemy import BigInteger, Integer, cast, delete, func, insert, literal, or_, select, true, update from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import load_only, selectinload +from sqlalchemy.orm import load_only, raiseload, selectinload from app.core.utils.time import utcnow from app.db.models import ( @@ -168,6 +168,46 @@ async def get_by_id(self, key_id: str) -> ApiKey | None: result = await self._session.execute(self._select_api_key().where(ApiKey.id == key_id)) return result.scalar_one_or_none() + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: + """Admission-path load for ``enforce_limits_for_request``. + + The enforcement transaction reads only ``is_active``/``expires_at`` + plus the ``limits`` collection, so this skips the + ``account_assignments``/``source_assignments`` selectin round trips + that ``get_by_id`` pays on every proxied request. ``raiseload`` keeps + the narrowing fail-loud: any future enforcement code that touches an + unlisted column or relationship raises instead of silently lazy + loading. ``populate_existing`` stays required because the lazy limit + reset commits mid-enforcement and the refetch must re-hydrate rows + already in the identity map (sessions use ``expire_on_commit=False``). + + Session-isolation invariant: ``populate_existing`` + ``raiseload`` + would poison a *fully loaded* ``ApiKey`` already in this session's + identity map — re-populating it flips its unlisted columns and + relationships into raise-on-access state for every other holder of + that instance. That is unreachable today because every caller runs + this query in a dedicated short-lived session that never full-loads + an ``ApiKey`` first (``_enforce_request_limits`` and the websocket + reservation path open fresh background sessions/repo bundles; the + quota-planner warmup session never loads ``ApiKey`` rows), and the + only prior instance this query can re-populate is the one it loaded + itself with these same options. Do not call this on a session that + may already hold a fully loaded ``ApiKey`` (e.g. via ``get_by_id`` / + ``get_by_hash``) without dropping the narrowing first. + """ + result = await self._session.execute( + select(ApiKey) + .execution_options(populate_existing=True) + .options( + load_only(ApiKey.is_active, ApiKey.expires_at, raiseload=True), + selectinload(ApiKey.limits), + raiseload(ApiKey.account_assignments), + raiseload(ApiKey.source_assignments), + ) + .where(ApiKey.id == key_id) + ) + return result.scalar_one_or_none() + async def get_by_hash(self, key_hash: str) -> ApiKey | None: result = await self._session.execute(self._select_api_key().where(ApiKey.key_hash == key_hash)) return result.scalar_one_or_none() diff --git a/app/modules/api_keys/service.py b/app/modules/api_keys/service.py index 096b274841..3f80c7669a 100644 --- a/app/modules/api_keys/service.py +++ b/app/modules/api_keys/service.py @@ -58,6 +58,8 @@ async def create(self, row: ApiKey, *, commit: bool = True) -> ApiKey: ... async def get_by_id(self, key_id: str) -> ApiKey | None: ... + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: ... + async def get_by_hash(self, key_hash: str) -> ApiKey | None: ... async def list_all(self) -> list[ApiKey]: ... @@ -827,11 +829,15 @@ async def _enforce_limits_for_request_once( ) -> ApiKeyUsageReservationData: now = utcnow() async with sqlite_writer_section(): - row = _ensure_valid_api_key_row(await self._repository.get_by_id(key_id)) + row = _ensure_valid_api_key_row(await self._repository.get_for_limit_enforcement(key_id)) if row.expires_at is not None and row.expires_at < now: raise ApiKeyInvalidError("API key has expired") limits_reset = await _lazy_reset_expired_limits(self._repository, row.limits, now=now) - refreshed = _ensure_valid_api_key_row(await self._repository.get_by_id(key_id)) if limits_reset else row + refreshed = ( + _ensure_valid_api_key_row(await self._repository.get_for_limit_enforcement(key_id)) + if limits_reset + else row + ) if refreshed.expires_at is not None and refreshed.expires_at < now: raise ApiKeyInvalidError("API key has expired") diff --git a/app/modules/proxy/_load_balancer/sticky_selection.py b/app/modules/proxy/_load_balancer/sticky_selection.py index fa5235ecea..a4403f88f2 100644 --- a/app/modules/proxy/_load_balancer/sticky_selection.py +++ b/app/modules/proxy/_load_balancer/sticky_selection.py @@ -40,7 +40,7 @@ fair_share_denial_message, ) from app.modules.proxy.repo_bundle import ProxyRepoFactory -from app.modules.proxy.sticky_repository import StickySessionsRepository +from app.modules.proxy.sticky_repository import StickyOwnerLookup, StickySessionsRepository from app.modules.quota_planner.logic import PlannerSettings, build_routing_costs # Preserve the established observability surface while implementation moves to @@ -238,6 +238,10 @@ class StickySelectionRequest(Generic[SelectionInputsT]): allow_usage_exhaustion_error: bool = True api_key_id: str | None = None api_key_stream_fair_share_threshold_pct: int = 0 + # First-iteration owner read performed by the caller inside its shared + # owner-lookup session (see load_balancer.select_account). Consumed exactly + # once; retries re-read fresh ownership evidence through a repo bundle. + initial_sticky_owner_lookup: StickyOwnerLookup | None = None @dataclass(frozen=True, slots=True) @@ -354,43 +358,52 @@ def _direct_error( ) attempt = 0 suppress_recovery_probe_candidates = False + pending_initial_owner_lookup = request.initial_sticky_owner_lookup while True: attempt += 1 sticky_existing_is_legacy = isinstance(legacy_existing_account_id, str) if sticky_kind is not None: async with owner._runtime_lock: pass - async with owner._repo_factory() as repos: - sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( - sticky_key, - kind=sticky_kind, - max_age_seconds=sticky_max_age_seconds, - continuity_source=sticky_source, - ) - sticky_existing_account_id = sticky_owner_lookup.account_id - # `is True` (not a truthy check): an un-configured test double - # for sticky_sessions may return an object whose attribute - # access auto-vivifies to a mock rather than a real bool, and - # that must fail safe as "not abandoned", the same as it - # always has, rather than silently bypassing the ambiguous - # owner check below. - sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True - # ``isinstance`` for the same test-double reason as above. The - # deadline is only ever an optimization hint: None always - # falls back to today's write-on-every-request refresh - # behavior, and seed-needing requests never skip. - observed_refresh_skip_deadline = sticky_owner_lookup.refresh_skip_deadline - sticky_refresh_skip_deadline = ( - observed_refresh_skip_deadline - if isinstance(observed_refresh_skip_deadline, datetime) and not seed_initialization_pending - else None - ) - sticky_abandoned_account_id = sticky_owner_lookup.abandoned_account_id - if sticky_owner_lookup.continuity_abandoned is True and isinstance( - sticky_abandoned_account_id, - str, - ): - retired_legacy_owner_account_ids.add(sticky_abandoned_account_id) + if pending_initial_owner_lookup is not None: + # The caller already read this iteration's owner inside its + # shared owner-lookup session. Consume it exactly once so + # every retry (including post-reset attempts that wrap + # ``attempt`` back to 1) still re-reads fresh evidence. + sticky_owner_lookup = pending_initial_owner_lookup + pending_initial_owner_lookup = None + else: + async with owner._repo_factory() as repos: + sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + sticky_key, + kind=sticky_kind, + max_age_seconds=sticky_max_age_seconds, + continuity_source=sticky_source, + ) + sticky_existing_account_id = sticky_owner_lookup.account_id + # `is True` (not a truthy check): an un-configured test double + # for sticky_sessions may return an object whose attribute + # access auto-vivifies to a mock rather than a real bool, and + # that must fail safe as "not abandoned", the same as it + # always has, rather than silently bypassing the ambiguous + # owner check below. + sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True + # ``isinstance`` for the same test-double reason as above. The + # deadline is only ever an optimization hint: None always + # falls back to today's write-on-every-request refresh + # behavior, and seed-needing requests never skip. + observed_refresh_skip_deadline = sticky_owner_lookup.refresh_skip_deadline + sticky_refresh_skip_deadline = ( + observed_refresh_skip_deadline + if isinstance(observed_refresh_skip_deadline, datetime) and not seed_initialization_pending + else None + ) + sticky_abandoned_account_id = sticky_owner_lookup.abandoned_account_id + if sticky_owner_lookup.continuity_abandoned is True and isinstance( + sticky_abandoned_account_id, + str, + ): + retired_legacy_owner_account_ids.add(sticky_abandoned_account_id) if sticky_existing_is_legacy: # Mixed-version replicas can create both rows on # different accounts. The raw row was loaded before diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index d5fb054824..ce12408944 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -135,7 +135,7 @@ if TYPE_CHECKING: from app.modules.accounts.repository import AccountsRepository - from app.modules.proxy.sticky_repository import StickySessionsRepository + from app.modules.proxy.sticky_repository import StickyOwnerLookup, StickySessionsRepository logger = logging.getLogger(__name__) @@ -724,42 +724,77 @@ async def load_selection_inputs() -> _SelectionInputs: selection_resets_at: int | None = None legacy_existing_account_id: str | None = None legacy_abandoned_account_id: str | None = None - if legacy_sticky_key is not None: - async with self._repo_factory() as repos: - legacy_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( - legacy_sticky_key, - kind=StickySessionKind.CODEX_SESSION, - # Raw rows may be historical turn-state ownership. The - # bounded thread TTL must never age out that hard evidence. - max_age_seconds=None, - # Process-session raw text is session_header even when - # request locality is thread_header. Thread-only raw keys - # keep thread_header so a session_header tombstone cannot - # hide a distinct thread owner. - continuity_source=legacy_continuity_source or "session_header", - ) - legacy_existing_account_id = legacy_owner_lookup.account_id - abandoned_account_id = legacy_owner_lookup.abandoned_account_id - if legacy_owner_lookup.continuity_abandoned is True and isinstance(abandoned_account_id, str): - legacy_abandoned_account_id = abandoned_account_id - if required_account_id is not None and ( - legacy_existing_account_id is not None and legacy_existing_account_id != required_account_id - ): - # The required owner came from a file/response/bridge index, - # while the raw row may be legacy turn-state ownership. Neither - # source can be discarded or rewritten to resolve a conflict. - return AccountSelection( - account=None, - error_message="Account-owned continuity sources conflict; retry the logical turn", - error_code="continuity_owner_conflict", - ) sticky_seed_account_id: str | None = None - if sticky_seed_key is not None and sticky_seed_kind is not None: + initial_sticky_owner_lookup: StickyOwnerLookup | None = None + needs_owner_lookups = ( + legacy_sticky_key is not None + or (sticky_seed_key is not None and sticky_seed_kind is not None) + or (sticky_key is not None and sticky_kind is not None) + ) + if needs_owner_lookups: + # One shared session serves the legacy/seed/first-sticky owner + # lookups. The SELECTs stay separate on purpose so the per-source + # predicate semantics of get_account_id_and_abandonment (tombstone + # visibility, max_age handling) are untouched; the saving is the + # 2-3 extra pool checkouts + session create/teardown lifecycles + # per request. Each later source still starts a fresh read + # transaction (release_read_snapshot): on SQLite/WAL the shared + # session would otherwise pin one snapshot at the first SELECT + # and hide a hard sticky or seed owner committed concurrently + # between the reads, letting selection overwrite that mapping. async with self._repo_factory() as repos: - sticky_seed_account_id = await repos.sticky_sessions.get_account_id( - sticky_seed_key, - kind=sticky_seed_kind, - ) + owner_snapshot_pinned = False + if legacy_sticky_key is not None: + legacy_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + legacy_sticky_key, + kind=StickySessionKind.CODEX_SESSION, + # Raw rows may be historical turn-state ownership. The + # bounded thread TTL must never age out that hard evidence. + max_age_seconds=None, + # Process-session raw text is session_header even when + # request locality is thread_header. Thread-only raw keys + # keep thread_header so a session_header tombstone cannot + # hide a distinct thread owner. + continuity_source=legacy_continuity_source or "session_header", + ) + legacy_existing_account_id = legacy_owner_lookup.account_id + abandoned_account_id = legacy_owner_lookup.abandoned_account_id + if legacy_owner_lookup.continuity_abandoned is True and isinstance(abandoned_account_id, str): + legacy_abandoned_account_id = abandoned_account_id + if required_account_id is not None and ( + legacy_existing_account_id is not None and legacy_existing_account_id != required_account_id + ): + # The required owner came from a file/response/bridge index, + # while the raw row may be legacy turn-state ownership. Neither + # source can be discarded or rewritten to resolve a conflict. + return AccountSelection( + account=None, + error_message="Account-owned continuity sources conflict; retry the logical turn", + error_code="continuity_owner_conflict", + ) + owner_snapshot_pinned = True + if sticky_seed_key is not None and sticky_seed_kind is not None: + if owner_snapshot_pinned: + await repos.sticky_sessions.release_read_snapshot() + sticky_seed_account_id = await repos.sticky_sessions.get_account_id( + sticky_seed_key, + kind=sticky_seed_kind, + ) + owner_snapshot_pinned = True + if sticky_key is not None and sticky_kind is not None: + if owner_snapshot_pinned: + await repos.sticky_sessions.release_read_snapshot() + # First-iteration owner read for run_sticky_selection_path, + # hoisted here so it shares this session. The selection + # loop consumes it exactly once; every retry (including + # post-reset attempts) still re-reads fresh ownership + # evidence through its own repo bundle. + initial_sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment( + sticky_key, + kind=sticky_kind, + max_age_seconds=sticky_max_age_seconds, + continuity_source=sticky_source, + ) # Resolve uniqueness from the model/API-key/security-scoped pool before # runtime health, budget, or cap filtering. Transient pressure cannot # prove that another candidate does not own an upstream conversation. @@ -882,6 +917,7 @@ async def load_selection_inputs() -> _SelectionInputs: reload_inputs=load_selection_inputs, record_account_cap_rejection=_record_account_cap_rejection, allow_usage_exhaustion_error=allow_usage_exhaustion_error, + initial_sticky_owner_lookup=initial_sticky_owner_lookup, ), ) selection_inputs = sticky_outcome.selection_inputs diff --git a/app/modules/proxy/sticky_repository.py b/app/modules/proxy/sticky_repository.py index becea80803..23ff2b6685 100644 --- a/app/modules/proxy/sticky_repository.py +++ b/app/modules/proxy/sticky_repository.py @@ -284,6 +284,22 @@ async def get_account_id_and_abandonment( ) return StickyOwnerLookup(account_id=current_account_id, continuity_abandoned=False) + async def release_read_snapshot(self) -> None: + """End the session's current read transaction. + + On the default SQLite/WAL configuration one transaction pins one read + snapshot at its first SELECT, so a session shared across successive + ownership lookups would leave every later lookup blind to owners + committed concurrently after the first read. Committing ends that + snapshot so the next SELECT begins a fresh transaction; on PostgreSQL + READ COMMITTED each statement already reads fresh committed state, so + this is a near-free no-op. COMMIT (not rollback) on purpose: rollback + expires all tracked ORM state regardless of ``expire_on_commit``, + while commit under the session factory's ``expire_on_commit=False`` + keeps rows loaded by earlier lookups readable. + """ + await self._session.commit() + async def get_entry(self, key: str, *, kind: StickySessionKind) -> StickySession | None: if not key: return None diff --git a/app/modules/request_logs/repository.py b/app/modules/request_logs/repository.py index d31095c0a7..79a65dac72 100644 --- a/app/modules/request_logs/repository.py +++ b/app/modules/request_logs/repository.py @@ -3,13 +3,15 @@ import time from dataclasses import dataclass from datetime import datetime, timedelta, timezone +from typing import Any from typing import cast as typing_cast import anyio -from sqlalchemy import Integer, String, and_, case, cast, func, or_, select +from sqlalchemy import Integer, String, and_, case, cast, func, insert, or_, select from sqlalchemy import exc as sa_exc +from sqlalchemy.engine import CursorResult from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import InstrumentedAttribute +from sqlalchemy.orm import InstrumentedAttribute, make_transient_to_detached from sqlalchemy.sql.elements import ColumnElement from app.core.usage.logs import ( @@ -67,6 +69,14 @@ class _RequestLogFilters: # Earliest representable listing lower bound for the rollup-count window. _ROLLUP_EPOCH = datetime(1970, 1, 1) +# Column keys for the Core request-log insert in ``add_log``. Every non-PK +# column is read off the fully built transient instance; columns ``add_log`` +# never sets are nullable with no Python/server default the flush would have +# applied, so an explicit NULL is identical to the old unit-of-work insert. +_REQUEST_LOG_INSERT_COLUMN_KEYS: tuple[str, ...] = tuple( + column.key for column in RequestLog.__table__.columns if column.key != "id" +) + def _naive_utc(value: datetime) -> datetime: """FastAPI parses ISO `Z` query bounds as offset-aware datetimes; @@ -1108,12 +1118,28 @@ async def add_log( if model_source_id is not None else calculated_cost_from_log(typing_cast(RequestLogLike, log)) ) - self._session.add(log) + # Core insert instead of unit-of-work: the row is fully built + # above, so the ORM flush (relationship cascade scan, + # per-attribute history snapshots) is pure overhead on every + # request's log write. No refresh: every column is set explicitly + # before insert. Once the primary key is known, the instance is + # re-attached as *persistent* (clean, no pending SQL) so callers + # that mutate the returned log and commit through the same + # session get a tracked UPDATE instead of a silent no-op + # (sessions run with ``expire_on_commit=False``, so the attach + # never triggers post-commit lazy loads). + insert_values = {key: getattr(log, key) for key in _REQUEST_LOG_INSERT_COLUMN_KEYS} try: + result = typing_cast( + CursorResult[Any], + await self._session.execute(insert(RequestLog).values(insert_values)), + ) + inserted_primary_key = result.inserted_primary_key + if inserted_primary_key is not None and inserted_primary_key[0] is not None: + log.id = int(inserted_primary_key[0]) + make_transient_to_detached(log) + self._session.add(log) await self._session.commit() - # No refresh: every column is set explicitly before insert and - # expire_on_commit=False, so the round trip was pure overhead - # on every request's log write. return log except sa_exc.ResourceClosedError: return log diff --git a/tests/integration/test_api_keys_api.py b/tests/integration/test_api_keys_api.py index 093eff9b7b..e81c94f6f9 100644 --- a/tests/integration/test_api_keys_api.py +++ b/tests/integration/test_api_keys_api.py @@ -10,6 +10,7 @@ import pytest from fastapi.responses import JSONResponse +from sqlalchemy import exc as sqlalchemy_exc from sqlalchemy import select, update import app.core.clients.proxy as core_proxy_module @@ -26,7 +27,7 @@ from app.db.session import SessionLocal from app.modules.api_keys.last_used_coalescer import get_api_key_last_used_coalescer from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyCreateData, ApiKeysService, LimitRuleInput +from app.modules.api_keys.service import ApiKeyCreateData, ApiKeyInvalidError, ApiKeysService, LimitRuleInput from app.modules.model_sources.forwarding import ( SourceChatCompletion, SourceResponsesStream, @@ -3607,6 +3608,77 @@ async def fake_sqlite_writer_section(): assert limits[0].current_value == fresh_reservation.items[0].reserved_delta +@pytest.mark.asyncio +async def test_enforce_limits_lazy_reset_and_expiry_with_narrowed_admission_load(async_client): + """Regression for the narrowed admission load (``get_for_limit_enforcement``). + + The enforcement path loads only ``is_active``/``expires_at`` plus the + ``limits`` collection. The lazy expired-limit reset (which commits + mid-enforcement and refetches through the same narrowed load) and the + key-expiry rejection must behave exactly as with the full-graph load. + """ + del async_client + now = utcnow() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="narrowed-admission-load", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="daily", max_value=50_000), + ], + ) + ) + limits = await repo.get_limits_by_key(created.id) + assert len(limits) == 1 + # Exhausted AND expired: without the lazy reset the enforcement + # would reject; the reset must zero the counter and advance reset_at. + limits[0].current_value = 50_000 + limits[0].reset_at = now - timedelta(hours=2) + await session.commit() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation.has_applicable_limits is True + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + limits = await repo.get_limits_by_key(created.id) + assert len(limits) == 1 + assert limits[0].reset_at > now + reserved = await repo.get_usage_reservation(reservation.reservation_id) + assert reserved is not None + assert reserved.status == "reserved" + assert limits[0].current_value == reserved.items[0].reserved_delta + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + # Fail-loud contract of the narrowed load: unlisted columns and the + # assignment relationships raise instead of lazy loading. + row = await repo.get_for_limit_enforcement(created.id) + assert row is not None + assert row.is_active is True + assert row.expires_at is None + assert len(row.limits) == 1 + with pytest.raises(sqlalchemy_exc.InvalidRequestError): + _ = row.name + with pytest.raises(sqlalchemy_exc.InvalidRequestError): + _ = row.account_assignments + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + await repo.update(created.id, expires_at=now - timedelta(minutes=1)) + with pytest.raises(ApiKeyInvalidError): + await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + @pytest.mark.asyncio async def test_release_stale_usage_reservations_max_age_ceiling_beats_orphaned_heartbeat(async_client, monkeypatch): """Issue #1594: a leaked heartbeat keeps refreshing ``updated_at`` forever. diff --git a/tests/unit/test_api_keys_service.py b/tests/unit/test_api_keys_service.py index f31476ec0a..a11f0673bc 100644 --- a/tests/unit/test_api_keys_service.py +++ b/tests/unit/test_api_keys_service.py @@ -94,6 +94,9 @@ async def get_by_id(self, key_id: str) -> ApiKey | None: row.source_assignments = self._source_assignments.get(key_id, []) return row + async def get_for_limit_enforcement(self, key_id: str) -> ApiKey | None: + return await self.get_by_id(key_id) + async def get_by_hash(self, key_hash: str) -> ApiKey | None: for row in self.rows.values(): if row.key_hash == key_hash: diff --git a/tests/unit/test_load_balancer_concurrency.py b/tests/unit/test_load_balancer_concurrency.py index ab433a77dd..09de146a6f 100644 --- a/tests/unit/test_load_balancer_concurrency.py +++ b/tests/unit/test_load_balancer_concurrency.py @@ -239,6 +239,11 @@ async def get_account_id(self, *args: Any, **kwargs: Any) -> str | None: lookup = await self.get_account_id_and_abandonment(*args, **kwargs) return lookup.account_id + async def release_read_snapshot(self) -> None: + # The shared owner-lookup session releases its read snapshot between + # ownership sources; the stub has no transaction to end. + return None + async def get_account_id_and_abandonment(self, *args: Any, **kwargs: Any) -> StickyOwnerLookup: key = cast(str, args[0]) scoped_abandoned_account_id = self.scoped_abandoned_account_ids_by_key.get(key) @@ -4667,3 +4672,94 @@ async def test_fresh_thread_only_retention_without_seed_key_skips_refresh_write( assert sticky_repo.upserts == [] assert sticky_repo.seeded_upserts == [] assert sticky_repo.deleted == [] + + +class _LookupCountingStickyRepo(_StubStickySessionsRepository): + """Records owner-lookup and snapshot-release events, each stamped with the + repository context that issued it, so tests can pin lookup count/order and + the one-session/fresh-transaction-per-source contract.""" + + def __init__(self) -> None: + super().__init__() + # Set by the test's repo factory each time a repo bundle opens. + self.current_context_id: int | None = None + self.owner_lookup_events: list[tuple[str, int | None, str | None]] = [] + + async def get_account_id_and_abandonment(self, *args: Any, **kwargs: Any) -> StickyOwnerLookup: + self.owner_lookup_events.append(("lookup", self.current_context_id, cast(str, args[0]))) + return await super().get_account_id_and_abandonment(*args, **kwargs) + + async def release_read_snapshot(self) -> None: + self.owner_lookup_events.append(("release_snapshot", self.current_context_id, None)) + await super().release_read_snapshot() + + +@pytest.mark.asyncio +async def test_shared_owner_lookup_session_reads_each_owner_key_exactly_once() -> None: + """Regression for the shared owner-lookup session. + + The legacy/seed/first-sticky owner reads moved into one repo bundle in + ``select_account``; the sticky selection loop consumes the hoisted first + read exactly once instead of re-reading. Each owner key must be looked up + exactly once, in the legacy -> seed -> sticky order, all three reads must + share one repository context (one session), each later ownership source + must first release the shared read snapshot so it starts a fresh + transaction, and the resolved hard owner must still win selection. + """ + now_epoch = int(datetime.now(tz=timezone.utc).timestamp()) + owner = _make_account("acc-shared-owner-lookup") + other = _make_account("acc-shared-owner-other") + accounts_repo = _StubAccountsRepository([owner, other]) + usage_repo = _StubUsageRepository( + primary={ + owner.id: _usage_row(70, owner.id, window="primary", reset_at=now_epoch + 300), + other.id: _usage_row(71, other.id, window="primary", reset_at=now_epoch + 300), + }, + secondary={}, + ) + sticky_repo = _LookupCountingStickyRepo() + sticky_repo.account_ids_by_key = {"shared-lookup-sticky": owner.id} + opened_context_count = 0 + + @asynccontextmanager + async def context_stamping_repo_factory() -> AsyncIterator[ProxyRepositories]: + # Stamp every bundle open with a distinct identifier so the events + # recorded by the sticky repo prove which context issued each read; + # the old per-lookup-session flow would record three distinct ids. + nonlocal opened_context_count + opened_context_count += 1 + sticky_repo.current_context_id = opened_context_count + async with _repo_factory(accounts_repo, usage_repo, sticky_repo) as repos: + yield repos + + balancer = LoadBalancer(context_stamping_repo_factory) + + selected = await balancer.select_account( + sticky_key="shared-lookup-sticky", + sticky_kind=StickySessionKind.CODEX_SESSION, + sticky_source="turn_state", + legacy_sticky_key="shared-lookup-legacy", + sticky_seed_key="shared-lookup-seed", + sticky_seed_kind=StickySessionKind.CODEX_SESSION, + routing_strategy="usage_weighted", + ) + + assert selected.account is not None + assert selected.account.id == owner.id + # get_account_id (seed) delegates to get_account_id_and_abandonment in the + # stub, so this also proves the seed lookup ran exactly once. The + # release_snapshot events pin the fix semantics: one shared session, but a + # fresh read transaction before each later ownership source so a + # concurrently committed owner stays visible on SQLite/WAL. + assert [(event, key) for event, _, key in sticky_repo.owner_lookup_events] == [ + ("lookup", "shared-lookup-legacy"), + ("release_snapshot", None), + ("lookup", "shared-lookup-seed"), + ("release_snapshot", None), + ("lookup", "shared-lookup-sticky"), + ] + lookup_context_ids = {context_id for _, context_id, _ in sticky_repo.owner_lookup_events} + # One repository context served every ownership source; the old + # session-per-lookup flow would have recorded three distinct ids here. + assert len(lookup_context_ids) == 1 + assert None not in lookup_context_ids diff --git a/tests/unit/test_request_logs_repository.py b/tests/unit/test_request_logs_repository.py index 784705922d..ff56ef296a 100644 --- a/tests/unit/test_request_logs_repository.py +++ b/tests/unit/test_request_logs_repository.py @@ -22,7 +22,12 @@ def _clear_recent_count_cache_between_tests(): @pytest.mark.asyncio -async def test_add_log_ignores_closed_transaction(monkeypatch) -> None: +async def test_add_log_ignores_closed_transaction(monkeypatch, db_setup) -> None: + # The insert now executes eagerly (Core insert instead of a unit-of-work + # flush inside commit), so the schema must exist; the contract under test + # is unchanged: a ResourceClosedError commit is swallowed and the built + # log row is still returned. + del db_setup async with SessionLocal() as session: repo = RequestLogsRepository(session) From 980572eb8ee5ed70acb9edfc5a7a2acc35f46216 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 22:21:02 +0900 Subject: [PATCH 064/117] perf(proxy): relay unmodified SSE frames verbatim (#1785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(proxy): relay unmodified SSE frames verbatim Canonically framed stream frames that no per-event consumer needs parsed now relay upstream bytes verbatim through the streaming mixin instead of paying a JSON parse plus the ensure_ascii format_sse_event re-encode per delta. A new sse_event_type_from_block helper matches only the exact canonical block shape format_sse_event emits (leading `event: ` line, single JSON-object `data:` line, LF framing); anything else — data-only blocks, multi-line data, CR/CRLF framing, or an `event:` field after `data:` — falls back to the full parse + re-serialization path, preserving the EventSource framing guarantee from 5ee532cb. - streaming mixin: verbatim branch gated on the must-parse set (lifecycle/terminal, response.output_item.*, output_text.done, content_part.done), the TTFT first-token window (including the pending reasoning-delta window), and a raw-line `"service_tier"` marker; reservation touches and text-visibility settlement accounting are preserved; the first-event block stays fully parsed - core client: _normalize_stream_payload_for_http_block returns the cheap `event:`-line type without parsing for canonical non-error, non-alias blocks without an `"error"` substring; error frames and top-level error envelopes keep the SDK-contract rewrite - core client: _normalize_sse_event_block narrows its gate from '"type":' (matched every event) to the three legacy alias substrings, and alias rewrites now cover the stale `event:` framing line alongside the data payload — previously masked by the mixin re-encode, a live bug under verbatim relay - shared touch-reservation closure dedupes the three per-frame touch call sites (keeps streaming/mixin.py within its architecture budget) Byte-visible but JSON-equivalent: unmodified delta frames now carry raw UTF-8 and upstream key order/spacing instead of the canonical re-encode. The /v1 identity pass-through gate (parsed-payload object identity plus the `event:` framing prefix) accepts verbatim blocks unchanged. OpenSpec change: relay-unmodified-sse-frames-verbatim (stacked on validate-stream-lifecycle-events-only; the spec delta carries the union text of both changes). Co-Authored-By: Claude Fable 5 * fix(proxy): normalize multi-line alias payloads before rewriting event framing When a legacy alias payload is split across multiple legal SSE data: lines, the per-line normalizer cannot decode the fragments, yet the event: framing line was still rewritten — emitting a frame whose framing and payload disagree, with raw mode ultimately relaying the legacy alias payload downstream. Decode the combined payload (data-line values joined with \n per the SSE spec) before touching the framing line: when the alias is decodable both surfaces are rewritten together (fragments collapse into one canonical data: line), and when the combined payload cannot be decoded both surfaces are left untouched instead of partially rewritten. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/clients/proxy.py | 95 +++++++- app/core/utils/sse.py | 23 ++ app/modules/proxy/_service/streaming/mixin.py | 40 ++-- app/modules/proxy/_service/support.py | 53 +++++ .../design.md | 115 ++++++++++ .../proposal.md | 78 +++++++ .../specs/responses-api-compat/spec.md | 68 ++++++ .../tasks.md | 34 +++ .../unit/test_proxy_api_responses_contract.py | 32 +++ tests/unit/test_proxy_utils.py | 211 ++++++++++++++++++ tests/unit/test_sse.py | 38 ++++ 11 files changed, 766 insertions(+), 21 deletions(-) create mode 100644 openspec/changes/relay-unmodified-sse-frames-verbatim/design.md create mode 100644 openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md create mode 100644 openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index 86904f9d0c..f823e61bfc 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -88,7 +88,7 @@ from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text, parse_rate_limit_headers from app.core.utils.json_guards import is_json_mapping from app.core.utils.request_id import get_request_id -from app.core.utils.sse import format_sse_event, parse_sse_data_json +from app.core.utils.sse import format_sse_event, parse_sse_data_json, sse_event_type_from_block CODEX_INSTALLATION_ID_HEADER = "x-codex-installation-id" CODEX_TURN_METADATA_HEADER = "x-codex-turn-metadata" @@ -125,6 +125,11 @@ "response.audio.delta": "response.output_audio.delta", "response.audio_transcript.delta": "response.output_audio_transcript.delta", } +# Bare (unquoted) alias names gate the block-level alias normalizer: they +# match both the JSON `"type":""` in a data line and a stale +# `event: ` framing line. False positives (an alias name inside delta +# text) just take the full-parse path. +_SSE_EVENT_TYPE_ALIAS_MARKERS = tuple(_SSE_EVENT_TYPE_ALIASES) _SSE_LINE_BOUNDARY_RE = re.compile(r"\r\n|\r|\n") _RESPONSE_STREAM_TERMINAL_EVENT_TYPES = frozenset( { @@ -1352,11 +1357,73 @@ def _normalize_sse_data_line(line: str) -> str: return line +def _normalize_sse_event_type_line(line: str) -> str: + if not line.startswith("event:"): + return line + value = line[6:] + if value.startswith(" "): + value = value[1:] + normalized_type = _SSE_EVENT_TYPE_ALIASES.get(value) + if normalized_type is None: + return line + return f"event: {normalized_type}" + + +def _normalize_multi_data_sse_block( + event_block: str, + lines: list[str], + line_separator: str, + terminator: str, +) -> str: + # Fragments of a payload split across multiple `data:` lines are not + # individually decodable, so alias detection must run on the combined + # payload (the SSE spec joins data-line values with "\n"). Decode it + # before touching the `event:` framing line so both surfaces are + # rewritten together; if the combined payload cannot be decoded, leave + # the whole block — framing line included — untouched rather than + # emitting a partially rewritten frame. + payload = parse_sse_data_json(event_block) + if payload is None: + return event_block + + data_replacement: str | None = None + event_type = payload.get("type") + if isinstance(event_type, str) and event_type in _SSE_EVENT_TYPE_ALIASES: + payload["type"] = _SSE_EVENT_TYPE_ALIASES[event_type] + data_replacement = f"data: {json.dumps(payload, ensure_ascii=True, separators=(',', ':'))}" + + normalized_lines: list[str] = [] + changed = False + data_line_emitted = False + for line in lines: + if line.startswith("data:"): + if data_replacement is None: + normalized_lines.append(line) + elif not data_line_emitted: + # The rewritten payload re-serializes compactly, so the + # fragments collapse into one canonical `data:` line. + normalized_lines.append(data_replacement) + data_line_emitted = True + changed = True + continue + normalized_line = _normalize_sse_event_type_line(line) + if normalized_line != line: + changed = True + normalized_lines.append(normalized_line) + if not changed: + return event_block + + normalized = line_separator.join(normalized_lines) + if terminator: + return normalized + terminator + return normalized + + def _normalize_sse_event_block(event_block: str) -> str: if not event_block: return event_block - if '"type":' not in event_block: + if not any(marker in event_block for marker in _SSE_EVENT_TYPE_ALIAS_MARKERS): return event_block if event_block.endswith("\r\n\r\n"): @@ -1380,10 +1447,17 @@ def _normalize_sse_event_block(event_block: str) -> str: if not lines: return event_block + if sum(1 for line in lines if line.startswith("data:")) > 1: + return _normalize_multi_data_sse_block(event_block, lines, line_separator, terminator) + normalized_lines: list[str] = [] changed = False for line in lines: - normalized_line = _normalize_sse_data_line(line) + # Rewrite both surfaces of a legacy alias: the JSON payload's `type` + # and the SSE `event:` framing line. Rewriting only the data line + # would emit mismatched framing when the block is relayed verbatim + # downstream instead of being re-serialized. + normalized_line = _normalize_sse_event_type_line(_normalize_sse_data_line(line)) if normalized_line != line: changed = True normalized_lines.append(normalized_line) @@ -1446,6 +1520,21 @@ def _normalize_stream_payload_for_http_block( *, enforce_openai_sdk_contract: bool = True, ) -> tuple[str, str | None]: + # Cheap path for the dominant delta traffic: a canonically framed block + # exposes its event type on the `event:` line, so no JSON parse is needed. + # Full parsing remains for `error` frames and any block carrying an + # `"error"` substring (the SDK-contract rewrite in + # `_normalize_stream_event_payload` keys off a top-level error envelope), + # legacy alias types (rewritten payloads), and non-canonical or data-only + # framing (the event type then comes from the payload itself). + cheap_event_type = sse_event_type_from_block(event_block) + if ( + cheap_event_type is not None + and cheap_event_type != "error" + and cheap_event_type not in _SSE_EVENT_TYPE_ALIASES + and '"error"' not in event_block + ): + return event_block, cheap_event_type if not enforce_openai_sdk_contract: payload = parse_sse_data_json(event_block) if payload is None: diff --git a/app/core/utils/sse.py b/app/core/utils/sse.py index b25d185d7f..f757856c83 100644 --- a/app/core/utils/sse.py +++ b/app/core/utils/sse.py @@ -20,6 +20,29 @@ SSE_KEEPALIVE_FRAME = ": keepalive\n\n" CODEX_KEEPALIVE_FRAME = 'event: codex.keepalive\ndata: {"type":"codex.keepalive"}\n\n' +# The exact single-event shape ``format_sse_event`` emits (and the upstream +# Codex backend sends): a leading ``event: `` line, one JSON-object +# ``data:`` line, LF-only framing, and a blank-line terminator. Blocks that +# match can expose their event type without a JSON parse and are safe to +# relay downstream byte-for-byte. +_CANONICAL_SSE_BLOCK = re.compile(r"\Aevent: ([^\r\n]+)\ndata: \{[^\r\n]*\n\n\Z") + + +def sse_event_type_from_block(event_block: str) -> str | None: + """Cheaply extract the event type from a canonically framed SSE block. + + Returns the ``event:`` line's value only when the block matches the exact + shape ``format_sse_event`` produces (see ``_CANONICAL_SSE_BLOCK``). + Anything else — data-only blocks, multi-line data, CR/CRLF framing, + comment or ``id:`` lines, non-object data payloads, or an ``event:`` field + that appears after ``data:`` (legal SSE, but not canonical here) — returns + ``None`` so callers fall back to a full parse. + """ + match = _CANONICAL_SSE_BLOCK.match(event_block) + if match is None: + return None + return match.group(1) + async def inject_sse_keepalives( source: AsyncIterator[str], diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 39d4cdad16..5bab3d94b4 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -303,6 +303,7 @@ _StreamSettlement, _TerminalStreamError, _ttft_event_latency_ms, + _verbatim_relay_event_type, _WebSocketUpstreamControl, ) from app.modules.proxy._service.support import ( @@ -525,6 +526,16 @@ async def _stream_once( response_create_lease = AdmissionLease(None, stage="response_create", request_id=request_id) account_response_create_lease: AccountLease | None = None api_key_reservation_touch_state = _ApiKeyReservationTouchState(last_touch_at=start) + + async def _touch_api_key_reservation() -> None: + api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( + api_key=api_key, + reservation=api_key_reservation, + last_touch_at=api_key_reservation_touch_state.last_touch_at, + request_id=request_id, + surface="stream", + ) + api_key_reservation_heartbeat_stop = asyncio.Event() api_key_reservation_heartbeat_task: asyncio.Task[None] | None = None if api_key_reservation is not None: @@ -579,9 +590,8 @@ async def _stream_once( error_code = "stream_incomplete" error_message = "Upstream websocket closed before response.completed" settlement.record_success = False - settlement.account_health_error = True + terminal_event_seen = settlement.account_health_error = True settlement.error = {"message": error_message} - terminal_event_seen = True yield format_sse_event( response_failed_event( error_code, @@ -598,9 +608,8 @@ async def _stream_once( error_code = "upstream_unavailable" error_message = str(exc) or "Request to upstream timed out" settlement.record_success = False - settlement.account_health_error = True + terminal_event_seen = settlement.account_health_error = True settlement.error = {"message": error_message} - terminal_event_seen = True yield format_sse_event( response_failed_event( error_code, @@ -627,13 +636,7 @@ async def _stream_once( if malformed_error_rewrite is not None: first, event, first_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: - api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( - api_key=api_key, - reservation=api_key_reservation, - last_touch_at=api_key_reservation_touch_state.last_touch_at, - request_id=request_id, - surface="stream", - ) + await _touch_api_key_reservation() event_service_tier = _facade()._service_tier_from_event_payload(first_payload) if event_service_tier is not None: actual_service_tier = event_service_tier @@ -777,6 +780,13 @@ async def _stream_once( if terminal_stream_error is not None: raise terminal_stream_error async for line in iterator: + if verbatim_type := _verbatim_relay_event_type(line, latency_first_token_ms, ttft_reasoning_deltas): + await _touch_api_key_reservation() + if verbatim_type in _facade()._TEXT_DELTA_EVENT_TYPES: + saw_text_delta = settlement.downstream_text_visible = True + settlement.downstream_visible = True + yield line + continue event_payload = parse_sse_data_json(line) event_type = classify_event_type(event_payload) event = parse_sse_event_payload(event_payload) if event_type in _LIFECYCLE_EVENT_TYPES else None @@ -791,13 +801,7 @@ async def _stream_once( if malformed_error_rewrite is not None: line, event, event_payload, event_type = malformed_error_rewrite if event_type not in {"response.completed", "response.failed", "response.incomplete", "error"}: - api_key_reservation_touch_state.last_touch_at = await proxy._maybe_touch_api_key_reservation( - api_key=api_key, - reservation=api_key_reservation, - last_touch_at=api_key_reservation_touch_state.last_touch_at, - request_id=request_id, - surface="stream", - ) + await _touch_api_key_reservation() event_service_tier = _facade()._service_tier_from_event_payload(event_payload) if event_service_tier is not None: actual_service_tier = event_service_tier diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 441b3e29e1..fd11e00f20 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -28,6 +28,7 @@ from app.core.resilience.overload import is_local_overload_error_code from app.core.types import JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute +from app.core.utils.sse import sse_event_type_from_block from app.db.models import Account from app.modules.api_keys.service import ( ApiKeyData, @@ -267,6 +268,58 @@ def _finalize_ttft_latency_ms( return _ttft_latency_ms_from_visible_at(_finalize_ttft_reasoning_deltas(pending_reasoning_deltas), started_at) +# Stream frames whose parsed payload feeds a real per-event consumer: +# lifecycle/terminal handling and usage settlement (created/in_progress/ +# completed/failed/incomplete/error), parallel tool-call rewrite + duplicate +# side-effect suppression (response.output_item.*), and text-done suppression +# (response.output_text.done / response.content_part.done). Canonically framed +# frames of any other type can relay upstream bytes verbatim once the TTFT +# window is settled and no service-tier snapshot is present. +_MUST_PARSE_STREAM_EVENT_TYPES = frozenset( + { + "response.created", + "response.in_progress", + "response.completed", + "response.failed", + "response.incomplete", + "error", + "response.output_item.added", + "response.output_item.done", + "response.output_text.done", + "response.content_part.done", + } +) +# Raw-line gate for service-tier attribution: response snapshots carry +# `"service_tier"` in their JSON, and a false positive (the substring inside +# delta text) merely takes the full-parse path. +_SERVICE_TIER_MARKER = '"service_tier"' + + +def _verbatim_relay_event_type( + line: str, + latency_first_token_ms: int | None, + pending_reasoning_deltas: dict[tuple[str | None, int | None, int | None], _TTFTReasoningDeltaState], +) -> str | None: + """Return the cheap event type when a stream frame can relay verbatim. + + Eligible frames are canonically framed (leading ``event: `` line, + single JSON-object ``data:`` line — see ``sse_event_type_from_block``), + outside the must-parse set, past the TTFT first-token window (including a + pending reasoning-delta window), and free of the service-tier marker. + Everything else returns ``None`` and takes the full parse + + ``format_sse_event`` path, preserving the EventSource framing guarantee + for data-only blocks. + """ + if latency_first_token_ms is None or pending_reasoning_deltas: + return None + if _SERVICE_TIER_MARKER in line: + return None + event_type = sse_event_type_from_block(line) + if event_type is None or event_type in _MUST_PARSE_STREAM_EVENT_TYPES: + return None + return event_type + + def _bind_propagated_capacity_startup_wait(event: asyncio.Event) -> Token[asyncio.Event | None]: return _PROPAGATED_CAPACITY_STARTUP_WAIT.set(event) diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md new file mode 100644 index 0000000000..4cffc70cd3 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/design.md @@ -0,0 +1,115 @@ +# Design — relay-unmodified-sse-frames-verbatim + +## Context + +Follow-up to `validate-stream-lifecycle-events-only` (stacked on it) and the +second half of the deferral in `2026-07-13-optimize-sse-single-parse`: after +lifecycle-only validation, every frame still pays one `json.loads` per owning +layer and an unconditional `format_sse_event` re-encode in the streaming +mixin. Delta frames after the first visible token have no per-event consumer, +so their bytes can relay verbatim. + +## Goals / Non-Goals + +**Goals:** zero JSON parse and zero re-encode for canonically framed frames no +consumer needs parsed; preserve every consumer's trigger surface (terminal +settlement, error rewrite, tool-call rewrite/dedupe, text-done suppression, +service-tier attribution, TTFT, reservation touches); keep the +`5ee532cb` framing guarantee (data-only blocks are re-framed with +`event: ` for EventSource clients); fix the stale-`event:`-line alias +bug that verbatim relay would otherwise expose. + +**Non-Goals:** the chat/completions bridge (cross-dialect translation keeps +full parsing); the `/v1` public normalizer (independent per-chunk consumer — +verbatim blocks satisfy its identity gate unchanged, see below); the websocket +relay and bridge upstream reader (every ws frame is parsed for response-id +multiplexing; covered by the stacked lifecycle change); the first-event block +in the streaming mixin (once per stream; retry classification lives there). + +## Decisions + +- **Cheap type = strict canonical shape.** `sse_event_type_from_block` matches + only `event: \ndata: {…}\n\n` (single data line, LF framing, + JSON-object data). SSE legally allows the `event:` field after `data:`, + CR/CRLF framing, comments, and multi-line data — all of those return `None` + and take the existing full-parse path, so only blocks byte-shaped like + `format_sse_event` output (which is what the upstream Codex backend and our + own re-encodes emit) are eligible for verbatim relay. This resolves the + field-ordering checklist item. +- **Must-parse set** = lifecycle/terminal frames (`response.created`, + `response.in_progress`, `response.completed`, `response.failed`, + `response.incomplete`, `error`) + tool-call item frames + (`response.output_item.added`, `response.output_item.done` — rewrite, + duplicate suppression, and TTFT item inspection) + text-done frames + (`response.output_text.done`, `response.content_part.done` — suppression + reads the payload's `part`). Everything else has no payload consumer outside + the gated windows below. +- **TTFT window trigger.** The full parse also runs while + `latency_first_token_ms is None` **or** `ttft_reasoning_deltas` is + non-empty. Verified by reading `support.py`: `_ttft_event_latency_ms` (and + therefore all mutation of the pending reasoning-delta state) is invoked only + under the `latency_first_token_ms is None` guard (mixin loop and first-event + block), and the stream-end `_finalize_ttft_latency_ms` is gated on the same + condition — so the first clause alone already covers the pending window; the + explicit non-empty check is a defensive belt (pending entries can outlive + TTFT settlement, e.g. a second reasoning summary stream, but are never read + after it). +- **Service-tier gate** stays on the raw line (`'"service_tier"'` substring), + not the event type, so a moved snapshot field still full-parses; false + positives (the substring inside delta text) just take the parse path. +- **Verbatim branch bookkeeping.** The reservation touch (non-terminal frames + keep reservations alive), `saw_text_delta`, `settlement.downstream_visible`, + and `settlement.downstream_text_visible` are preserved; text flags derive + from the cheap type, which for canonical frames equals the payload type. +- **Client normalizer laziness.** `_normalize_stream_payload_for_http_block` + returns the cheap type without parsing only when the block is canonical, the + type is not `error` and not a legacy alias, and the block has no `"error"` + substring. The substring guard is load-bearing: `parse_error_payload` + rewrites any payload carrying a top-level `error` envelope regardless of its + `type` (`OpenAIErrorEnvelope.error` is optional, so only an actual `error` + key triggers it), and response snapshots legitimately carry `"error":null` — + both stay on the full-parse path. +- **Alias gate + stale `event:` line.** `_normalize_sse_event_block` now gates + on the three bare alias names (matching both `"type":""` in data + lines and `event: ` framing lines) instead of `'"type":'`, and + rewrites the `event:` line too. Previously only the data line was rewritten + and the mixin's unconditional re-encode masked the mismatch; under verbatim + relay the stale line would reach clients, so the fix lands in the same + change. +- **/v1 identity gate verified for raw UTF-8.** `api.py` pass-through compares + parsed-payload *object identity* (`normalized_payload is parsed_payload`) + plus `_has_canonical_event_framing`, which checks only the + `event: \n` prefix — no comparison against a re-serialization — so + verbatim raw-UTF-8 blocks pass through byte-identically (regression test + added). + +## Accepted limitations (documented drift) + +- **Byte-visible output change.** Unmodified delta frames now carry upstream + bytes (raw UTF-8, upstream key order/spacing) instead of the `ensure_ascii` + canonical re-encode. JSON-equivalent and SSE-valid; codified in the spec + delta. +- **The `event:` framing line is trusted for non-parsed frames.** A + hypothetical upstream frame whose `event:` line disagrees with its payload + `type` (never emitted by upstream; our own re-encodes are consistent by + construction) would be classified by the framing line: a terminal payload + disguised under a delta `event:` line would relay verbatim and settle as + `stream_incomplete` at EOF instead of a terminal settlement. Today's + behavior for such frames differs only in which side wins; the `/v1` + normalizer still parses independently and enforces its own contract. +- **Malformed-JSON canonical frames.** A canonical-looking block whose data is + not valid JSON relays verbatim (today it is also yielded unchanged — the + parse failure path skips the re-encode) but now sets `saw_text_delta` / + text-visibility from the framing line, which the parse path would not. + Only reachable from a misbehaving upstream. +- **Usage on delta frames** would relay verbatim without settlement capture, + as under the stacked lifecycle change (upstream emits usage only on + terminal frames) — unchanged hedge, inherited. + +## Stacking note (delta-merge hazard) + +This change is stacked on `validate-stream-lifecycle-events-only` and MODIFIES +the same `responses-api-compat` requirement. To avoid the concurrent-MODIFIED +last-writer-wins loss (#1772), this change's delta contains the **union** text: +the lifecycle-only validation clauses from the stacked change plus the +verbatim-relay condition, so syncing/archiving in either order preserves both. diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md new file mode 100644 index 0000000000..0ba2cc2b08 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/proposal.md @@ -0,0 +1,78 @@ +# Relay Unmodified SSE Frames Verbatim + +## Why + +Even after lifecycle-only validation (`validate-stream-lifecycle-events-only`), +every streamed SSE frame still pays one `json.loads` per owning layer plus an +unconditional `format_sse_event` re-encode in the streaming mixin, and the +core client parses every frame's payload just to read its `type` for terminal +detection. The dominant traffic — text/reasoning/tool-argument delta frames +after the first visible token — has no per-event consumer at all: tool-call +rewrite and duplicate suppression act only on `response.output_item.*`, +text-done suppression only on `response.output_text.done` / +`response.content_part.done`, service-tier attribution only on response +snapshots carrying `"service_tier"`, TTFT only while the first-token window is +open, and settlement/error handling only on lifecycle frames. This is the +remaining half of the follow-up the `2026-07-13-optimize-sse-single-parse` +design doc deferred, and py-spy attributes the `format_sse_event` `json.dumps` +leaf plus 2–3 redundant `json.loads` per chunk to it. + +## What Changes + +- `app/core/utils/sse.py` gains `sse_event_type_from_block`: cheap event-type + extraction that matches only the exact canonical block shape + `format_sse_event` emits (leading `event: ` line, single JSON-object + `data:` line, LF framing). Data-only blocks, multi-line data, CR/CRLF + framing, and `event:` fields appearing after `data:` (legal SSE, but not + canonical here) return `None` so callers fall back to a full parse. +- Streaming mixin hot loop: compute the cheap type first; run the full + parse only when the type is unavailable, is in the must-parse set + (lifecycle/terminal frames, `response.output_item.added`/`done`, + `response.output_text.done`, `response.content_part.done`), the TTFT + first-token window is open (including a pending reasoning-delta window), or + the raw line carries the `"service_tier"` marker. Otherwise the upstream + block is yielded verbatim — raw UTF-8 and upstream key order/spacing instead + of the `ensure_ascii` canonical re-encode — with text-visibility accounting + set from the cheap type. The first-event block stays fully parsed. +- Core client `_normalize_stream_payload_for_http_block` becomes lazy: a + canonical non-error block without an `"error"` substring returns its cheap + type with no JSON parse; error frames, error-envelope payloads, alias types, + and non-canonical framing keep the full parse + rewrite path. +- Core client `_normalize_sse_event_block` narrows its gate from `'"type":'` + (matches every event, gates nothing) to the three legacy alias substrings, + and — in the same change, because verbatim relay would otherwise expose the + latent bug — rewrites the stale `event:` framing line alongside the `data:` + payload when an alias fires. +- The chat/completions bridge and the `/v1` public normalizer are untouched; + the `/v1` identity pass-through gate (parsed-payload object identity + + canonical framing prefix) accepts verbatim upstream blocks unchanged. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `responses-api-compat`: the single-parse streaming requirement gains the + verbatim-relay condition — a canonically framed frame that no per-event + consumer needs parsed MAY be relayed with upstream bytes verbatim + (JSON-equivalent, SSE-valid); all other frames keep the parse + + canonical-re-serialization path, and legacy alias rewrites MUST cover both + the `data:` payload and the `event:` framing line. + +## Impact + +- **Code**: `app/core/utils/sse.py`, `app/core/clients/proxy.py`, + `app/modules/proxy/_service/streaming/mixin.py`. +- **Behavior**: byte-visible but JSON-equivalent — unmodified delta frames now + carry upstream bytes (raw UTF-8, upstream key order/spacing) instead of the + `ensure_ascii` canonical re-encode. Framing stays SSE-valid and named-event + clients keep seeing `event:` lines (non-canonical blocks still get + re-framed). Legacy `response.text.delta`-style upstreams now get a correct + `event:` line after alias rewrite (previously stale, masked by the mixin + re-encode). +- **Performance**: removes the per-delta `json.loads` in the core client and + the mixin plus the per-delta `format_sse_event` re-encode for the dominant + post-first-token delta traffic. diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..83dddff9fc --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/specs/responses-api-compat/spec.md @@ -0,0 +1,68 @@ +# responses-api-compat Delta + +## MODIFIED Requirements + +### Requirement: Streaming events are parsed once and re-serialized only when modified + +Within each streaming layer (core client consumer, streaming mixin, bridge upstream reader, websocket relay, /v1 normalizers), an SSE event's JSON payload MUST be parsed at most once and reused by that layer's consumers, and an event that no consumer modified MUST NOT be re-serialized by the /v1 normalizers. Schema validation of the parsed payload MUST run only for stream lifecycle frames (`response.created`, `response.completed`, `response.incomplete`, `response.failed`, `error`); all other frames MUST be classified from the parsed payload's `type` field (with a typeless payload carrying an `error` object classifying as `error`). + +A canonically framed SSE block — a leading `event: ` line followed by a single JSON-object `data:` line with LF framing — whose type requires no per-event consumer MAY skip payload parsing entirely and be relayed downstream with the upstream bytes verbatim (raw UTF-8 and upstream key order/spacing preserved; JSON-equivalent to the canonical re-encode). A frame MUST take the parse path when any consumer needs it: lifecycle/terminal frames, tool-call item frames (`response.output_item.added`, `response.output_item.done`), text-done frames (`response.output_text.done`, `response.content_part.done`), frames arriving while the TTFT first-token window is open (including a pending reasoning-delta window), frames carrying a `"service_tier"` marker, and any block without canonical framing (data-only blocks, multi-line data, or an `event:` field that does not lead the block). A parsed frame MUST be re-serialized with canonical `event: ` + `data:` framing when modified or when its source block lacked canonical framing. Legacy event-type alias rewrites MUST cover both the `data:` payload type and the `event:` framing line. Event framing, payload contents, dedupe/rewrite semantics, usage settlement, and error normalization MUST be unchanged. + +#### Scenario: Unmodified events pass through the /v1 normalizer verbatim + +- **GIVEN** a canonical stream event that no normalizer branch rewrites +- **WHEN** the /v1 response normalizer processes it +- **THEN** the original block is yielded byte-identically without re-serialization + +#### Scenario: Tool-call rewrite reuses the parsed event on the no-change path + +- **GIVEN** an event without duplicate parallel tool calls +- **WHEN** the rewrite step runs with the caller's parsed event +- **THEN** it returns the original line, payload, and event without re-parsing or re-validating + +#### Scenario: Rewritten events stay consistent + +- **WHEN** the rewrite step removes duplicate tool calls +- **THEN** the returned line, payload, and validated event all reflect the rewritten content + +#### Scenario: Delta frames skip schema validation + +- **GIVEN** a stream of `response.output_text.delta` frames between `response.created` and `response.completed` +- **WHEN** the streaming mixin, websocket relay, or bridge upstream reader processes the stream +- **THEN** only the lifecycle frames are schema-validated, the delta frames are classified from the parsed payload dict, and downstream output, usage settlement, and error normalization are unchanged + +#### Scenario: Identity websocket relay frames are forwarded without re-encoding + +- **GIVEN** a websocket frame matched to a request whose downstream response-id rewrite does not apply +- **WHEN** the relay forwards the frame downstream +- **THEN** the upstream frame text is forwarded as-is instead of a canonical JSON re-encode + +#### Scenario: Unmodified canonical delta frames relay upstream bytes verbatim + +- **GIVEN** a canonically framed `response.output_text.delta` frame containing raw UTF-8, arriving after the first visible token settled the TTFT window +- **WHEN** the streaming mixin processes it +- **THEN** the upstream block is yielded byte-identically without a JSON parse or `ensure_ascii` re-encode, and downstream text-visibility accounting still updates + +#### Scenario: Data-only frames regain canonical framing + +- **GIVEN** a delta frame without a leading `event:` line +- **WHEN** the streaming mixin processes it after the TTFT window settles +- **THEN** the frame is parsed and re-serialized with the canonical `event: ` line so named-event (EventSource) clients keep seeing the event name + +#### Scenario: Legacy alias frames are rewritten on both lines + +- **GIVEN** an upstream block whose `event:` line and `data:` payload both carry the legacy `response.text.delta` type +- **WHEN** the core client normalizes the block +- **THEN** both the `event:` framing line and the payload `type` read `response.output_text.delta` + +#### Scenario: Error frames keep the full parse and rewrite path + +- **GIVEN** a canonically framed `error` frame, or a frame whose payload carries a top-level `error` envelope +- **WHEN** the core client normalizes the stream for the SDK contract +- **THEN** the frame is parsed and rewritten to a terminal `response.failed` event exactly as before verbatim relay + +#### Scenario: /v1 identity pass-through accepts verbatim raw-UTF-8 blocks + +- **GIVEN** an upstream-verbatim canonical delta block containing raw UTF-8 +- **WHEN** the /v1 normalizer leaves the parsed payload unmodified +- **THEN** the block passes through byte-identically (the identity gate compares parsed-payload object identity and the `event:` framing prefix, not re-serialized bytes) diff --git a/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md b/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md new file mode 100644 index 0000000000..3c0b3945b8 --- /dev/null +++ b/openspec/changes/relay-unmodified-sse-frames-verbatim/tasks.md @@ -0,0 +1,34 @@ +# Tasks — relay-unmodified-sse-frames-verbatim + +## 1. Implementation + +- [x] 1.1 `sse_event_type_from_block` in `app/core/utils/sse.py`: strict + canonical-shape matcher (leading `event:` line, single JSON-object + `data:` line, LF framing); `None` otherwise +- [x] 1.2 Streaming mixin hot loop: verbatim relay branch gated on cheap type + ∉ must-parse set, TTFT window settled (`latency_first_token_ms` set and + no pending reasoning deltas), and no `"service_tier"` marker; keeps + reservation touch + text-visibility accounting; first-event block stays + fully parsed +- [x] 1.3 `_normalize_stream_payload_for_http_block`: lazy cheap-type return + for canonical non-error, non-alias blocks without an `"error"` + substring +- [x] 1.4 `_normalize_sse_event_block`: gate narrowed from `'"type":'` to the + three alias substrings; alias rewrite covers the `event:` framing line + in addition to the `data:` payload + +## 2. Validation + +- [x] 2.1 Unit coverage for `sse_event_type_from_block` (canonical, raw + UTF-8, data-only, trailing `event:` ordering, CRLF/multi-line, + non-object data) +- [x] 2.2 Mixin regressions: raw-UTF-8 delta relayed byte-identically with no + JSON parse after TTFT settles; data-only delta re-framed with + `event: ` (5ee532cb regression class); usage settlement unchanged +- [x] 2.3 Client normalizer regressions: canonical frames skip `json.loads`; + `error` frames and top-level error envelopes still rewritten; alias + rewrite covers both lines; non-alias blocks skip the alias parse +- [x] 2.4 `/v1` identity pass-through accepts verbatim raw-UTF-8 blocks + byte-identically +- [x] 2.5 Existing streaming/contract/dedupe/TTFT suites pass; `uvx ruff + format --check`, `uv run ruff check` on changed files diff --git a/tests/unit/test_proxy_api_responses_contract.py b/tests/unit/test_proxy_api_responses_contract.py index 6525334689..7ba651d6f5 100644 --- a/tests/unit/test_proxy_api_responses_contract.py +++ b/tests/unit/test_proxy_api_responses_contract.py @@ -1806,6 +1806,38 @@ async def test_normalize_public_stream_passes_canonical_unmutated_blocks_verbati assert delta in blocks +@pytest.mark.asyncio +async def test_normalize_public_stream_passes_raw_utf8_verbatim_blocks_byte_identically() -> None: + """Upstream-verbatim delta blocks (raw UTF-8, upstream key spacing — not + the ensure_ascii canonical re-encode) still satisfy the identity + pass-through gate: it compares parsed-payload object identity plus the + `event:` framing prefix, never re-serialized bytes.""" + created = proxy_api_module.format_sse_event( + {"type": "response.created", "response": {"id": "resp_utf8", "output": []}} + ) + verbatim_delta = ( + "event: response.output_text.delta\n" + 'data: {"type": "response.output_text.delta", "item_id": "msg_1", "output_index": 0, "delta": "안녕"}\n\n' + ) + completed_payload: dict[str, Any] = { + "type": "response.completed", + "response": { + "id": "resp_utf8", + "output": [{"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "안녕"}]}], + }, + } + completed = proxy_api_module.format_sse_event(completed_payload) + + blocks = [ + block + async for block in proxy_api_module._normalize_public_responses_stream( + _iter_blocks(created, verbatim_delta, completed) + ) + ] + + assert verbatim_delta in blocks + + @pytest.mark.asyncio async def test_normalize_public_stream_reframes_data_only_blocks_with_event_name() -> None: """A data-only block (e.g. bridge-rewritten terminal event) must regain diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index d495ced37b..1403d6dbd8 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -46750,3 +46750,214 @@ def archive_received(self, message: object) -> None: finalize_request_state.assert_awaited_once() assert finalize_request_state.await_args is not None assert finalize_request_state.await_args.kwargs["event_type"] == "response.completed" + + +def test_normalize_sse_event_block_rewrites_alias_on_both_event_and_data_lines(): + # A legacy alias must be rewritten on the SSE `event:` framing line too, + # not just inside the JSON payload — under verbatim relay a stale + # `event:` line would otherwise reach clients with mismatched framing. + block = 'event: response.text.delta\ndata: {"type":"response.text.delta","delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == ( + 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + ) + + +def test_normalize_sse_event_block_rewrites_alias_split_across_data_lines(): + # A legal SSE payload split across multiple `data:` lines is only + # decodable as the combined value (fragments joined with "\n"); the alias + # rewrite must still land on both the payload and the `event:` line. + block = 'event: response.text.delta\ndata: {"type":"response.text.delta",\ndata: "delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == ( + 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + ) + + +def test_normalize_sse_event_block_leaves_undecodable_multi_line_data_untouched(): + # When the combined multi-line payload cannot be decoded, neither surface + # may be rewritten: rewriting only the `event:` framing line would emit a + # frame whose framing and payload disagree about the event type. + block = 'event: response.text.delta\ndata: {"type":"response.te\ndata: xt.delta","delta":"hi"}\n\n' + + normalized = proxy_module._normalize_sse_event_block(block) + + assert normalized == block + + +def test_normalize_sse_event_block_skips_json_parsing_for_non_alias_types(monkeypatch) -> None: + # The alias normalizer only inspects blocks carrying one of the legacy + # alias names; canonical event types pass through without a JSON parse. + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + def fail_json_parse(_: str) -> object: + raise AssertionError("json.loads should not run for blocks without an alias marker") + + monkeypatch.setattr(proxy_module.json, "loads", fail_json_parse) + + assert proxy_module._normalize_sse_event_block(block) == block + + +def test_normalize_stream_payload_for_http_block_skips_parse_for_canonical_frames(monkeypatch) -> None: + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + def fail_json_parse(_: str) -> object: + raise AssertionError("json.loads should not run for canonical non-error frames") + + monkeypatch.setattr(proxy_module.json, "loads", fail_json_parse) + + assert proxy_module._normalize_stream_payload_for_http_block(block) == (block, "response.output_text.delta") + assert proxy_module._normalize_stream_payload_for_http_block(block, enforce_openai_sdk_contract=False) == ( + block, + "response.output_text.delta", + ) + + +def test_normalize_stream_payload_for_http_block_still_rewrites_error_frames(): + block = 'event: error\ndata: {"type":"error","message":"boom"}\n\n' + + normalized_block, normalized_type = proxy_module._normalize_stream_payload_for_http_block(block) + + assert normalized_type == "response.failed" + assert '"boom"' in normalized_block + + +def test_normalize_stream_payload_for_http_block_still_rewrites_error_envelopes_on_non_error_types(): + # A payload carrying a top-level error envelope is rewritten regardless of + # its event type; the `"error"` substring guard keeps it on the full-parse + # path. + block = ( + "event: response.output_text.delta\n" + 'data: {"type":"response.output_text.delta","error":{"message":"broken"},"delta":"hi"}\n\n' + ) + + normalized_block, normalized_type = proxy_module._normalize_stream_payload_for_http_block(block) + + assert normalized_type == "response.failed" + assert '"broken"' in normalized_block + + +@pytest.mark.asyncio +async def test_stream_with_retry_relays_unmodified_canonical_delta_frames_verbatim(monkeypatch): + # After the TTFT window settles, canonically framed delta frames are + # relayed with upstream bytes (raw UTF-8, upstream spacing) and are never + # JSON-parsed; usage settlement from the parsed terminal frame is + # unchanged. + from app.modules.proxy._service.streaming import mixin as streaming_mixin_module + + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_verbatim_relay") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + mixin_parse = MagicMock(wraps=streaming_mixin_module.parse_sse_data_json) + monkeypatch.setattr(streaming_mixin_module, "parse_sse_data_json", mixin_parse) + + verbatim_delta = ( + 'event: response.output_text.delta\ndata: {"type": "response.output_text.delta", "delta": "안녕 upstream"}\n\n' + ) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_verbatim"}}\n\n' + yield 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield verbatim_delta + yield ( + 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_verbatim",' + '"usage":{"input_tokens":3,"output_tokens":5}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-verbatim-relay"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + # created (lifecycle), the first delta (TTFT window still open), and + # completed (lifecycle) are parsed; the settled second delta is relayed + # without any JSON parse. + assert mixin_parse.call_count == 3 + # Upstream bytes are preserved exactly: raw UTF-8 and upstream key + # spacing, not the ensure_ascii canonical re-encode. + assert chunks[2] == verbatim_delta + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" + assert request_logs.calls[0]["input_tokens"] == 3 + assert request_logs.calls[0]["output_tokens"] == 5 + + +@pytest.mark.asyncio +async def test_stream_with_retry_reframes_data_only_delta_frames_after_ttft(monkeypatch): + # Data-only frames (no `event:` line, e.g. bridge rewrite leftovers) never + # take the verbatim path: they are parsed and re-serialized with canonical + # `event: ` framing so named-event (EventSource) clients keep seeing + # the event name — the 5ee532cb regression class. + settings = _make_proxy_settings() + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account = _make_account("acc_verbatim_data_only") + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr( + service, + "_select_account_with_budget_compatible", + AsyncMock(return_value=AccountSelection(account=account, error_message=None)), + ) + monkeypatch.setattr(service, "_ensure_fresh_with_budget", AsyncMock(return_value=account)) + + async def fake_core_stream_responses(*_args: object, **_kwargs: object): + yield 'event: response.created\ndata: {"type":"response.created","response":{"id":"resp_data_only"}}\n\n' + yield 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"a"}\n\n' + yield 'data: {"type":"response.output_text.delta","delta":"b"}\n\n' + yield ( + 'event: response.completed\ndata: {"type":"response.completed","response":{"id":"resp_data_only",' + '"usage":{"input_tokens":1,"output_tokens":1}}}\n\n' + ) + + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_core_stream_responses) + + payload = ResponsesRequest.model_validate({"model": "gpt-5.1", "instructions": "hi", "input": [], "stream": True}) + chunks = [ + chunk + async for chunk in service._stream_with_retry( + payload, + {"session_id": "sid-verbatim-data-only"}, + codex_session_affinity=False, + propagate_http_errors=False, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + request_transport="http", + upstream_stream_transport_override="http", + ) + ] + + assert chunks[2] == 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"b"}\n\n' + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[0]["status"] == "success" diff --git a/tests/unit/test_sse.py b/tests/unit/test_sse.py index 02b912c4d4..5151f48025 100644 --- a/tests/unit/test_sse.py +++ b/tests/unit/test_sse.py @@ -18,6 +18,7 @@ format_sse_event, inject_sse_keepalives, parse_sse_data_json, + sse_event_type_from_block, ) from tests.unit.hypothesis_strategies import json_objects, json_values @@ -225,3 +226,40 @@ def test_lifecycle_event_types_cover_terminal_and_created_frames(): "error", } ) + + +def test_sse_event_type_from_block_extracts_type_from_canonical_block(): + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + assert sse_event_type_from_block(block) == "response.output_text.delta" + + +def test_sse_event_type_from_block_accepts_raw_utf8_payloads(): + block = 'event: response.output_text.delta\ndata: {"type":"response.output_text.delta","delta":"안녕"}\n\n' + + assert sse_event_type_from_block(block) == "response.output_text.delta" + + +def test_sse_event_type_from_block_rejects_data_only_blocks(): + assert sse_event_type_from_block('data: {"type":"response.output_text.delta","delta":"hi"}\n\n') is None + + +def test_sse_event_type_from_block_rejects_trailing_event_field_ordering(): + # `event:` after `data:` is legal SSE but not the canonical framing this + # proxy relays verbatim; callers must fall back to a full parse. + block = 'data: {"type":"response.output_text.delta","delta":"hi"}\nevent: response.output_text.delta\n\n' + + assert sse_event_type_from_block(block) is None + + +def test_sse_event_type_from_block_rejects_non_lf_framing_and_multiline_data(): + crlf = 'event: response.output_text.delta\r\ndata: {"type":"response.output_text.delta"}\r\n\r\n' + multiline = 'event: response.output_text.delta\ndata: {"type":\ndata: "response.output_text.delta"}\n\n' + + assert sse_event_type_from_block(crlf) is None + assert sse_event_type_from_block(multiline) is None + + +def test_sse_event_type_from_block_rejects_non_object_data_payloads(): + assert sse_event_type_from_block("event: done\ndata: [DONE]\n\n") is None + assert sse_event_type_from_block("event: ping\ndata: \n\n") is None From 8a2d0660e2b216a93593757f2fec242e69f92724 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 22:21:06 +0900 Subject: [PATCH 065/117] perf(api-keys): skip usage reservations when no limit applies (#1789) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * perf(api-keys): skip usage reservations when no limit applies Admission previously INSERTed an empty reservation row plus a full-durability commit for every request on a key without applicable limits, and stream end still ran the whole settlement transaction just to flip that empty row. Return None from enforcement when no configured limit applies (no limits, or none matching the request model): the reservation INSERT and its commit are skipped and every downstream consumer (stream/compact settlement, release paths, heartbeat touch, bridge forwarding, quota-planner warmup) already no-ops on a missing reservation. The quota-planner warmup executor is adapted to probe without finalizing when admission returns no reservation. Settlement is also the production writer of the key's last-used touch, so the limit-free path records the write-behind coalescer touch at admission itself (in-memory, no extra commit) — last_used_at keeps advancing for limit-free keys. The early return also rolls back to close the implicit read transaction opened by the admission SELECTs, so long-lived sessions (quota-planner warmup) do not idle in transaction across the probe's upstream round-trip. Keys with an applicable limit are unchanged: reservation items (including zero-delta ones), full commit durability (#1665), and exactly-once settlement are untouched. Deferred account backoff lifecycles with a None reservation follow the pre-existing keyless (api_key=None) semantics — backoffs carry across bridge submit retries; documented in the change design. OpenSpec: openspec/changes/skip-empty-usage-reservations (api-keys reservation-ledger delta). Co-Authored-By: Claude Fable 5 * fix(api-keys): close limit-free admission transaction without expiring shared state The limit-free early return closed the implicit admission read transaction with rollback(), but AsyncSession.rollback() expires every tracked ORM object regardless of expire_on_commit=False. The quota-planner warmup service shares its long-lived session with this repository and already tracks the target account and decision; after admission the probe's account.access_token_encrypted access (and the error path's decision.id) raised MissingGreenlet, so limit-free warmups never executed. Switch the transaction close to commit(): with expire_on_commit=False tracked state stays loaded, and the commit is semantically equivalent here because the transaction holds only the admission SELECTs — no reservation write ran, proxy call sites dedicate a session to admission, and quota-planner repositories commit every prior write inside their own methods, so nothing unrelated can be flushed. Add a regression that drives the REAL ApiKeysService through the warmup service's shared session with a real limit-free key and asserts the probe's attribute access survives admission (verified red under rollback: MissingGreenlet). Update the limit-free unit tests to the new contract (one read-only transaction-close commit, no rollback) and sync the openspec requirement, scenarios, tasks, and design rationale. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/modules/api_keys/service.py | 58 +++++-- app/modules/quota_planner/warmup.py | 4 +- .../skip-empty-usage-reservations/design.md | 80 +++++++++ .../skip-empty-usage-reservations/proposal.md | 51 ++++++ .../specs/api-keys/spec.md | 51 ++++++ .../skip-empty-usage-reservations/tasks.md | 50 ++++++ tests/integration/test_api_keys_api.py | 56 ++++++ .../integration/test_detached_persistence.py | 71 ++++++++ tests/integration/test_proxy_compact.py | 2 + .../test_proxy_websocket_responses.py | 15 +- tests/integration/test_quota_planner_api.py | 160 +++++++++++++++++- tests/unit/test_api_keys_service.py | 114 ++++++++++++- 12 files changed, 697 insertions(+), 15 deletions(-) create mode 100644 openspec/changes/skip-empty-usage-reservations/design.md create mode 100644 openspec/changes/skip-empty-usage-reservations/proposal.md create mode 100644 openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md create mode 100644 openspec/changes/skip-empty-usage-reservations/tasks.md diff --git a/app/modules/api_keys/service.py b/app/modules/api_keys/service.py index 3f80c7669a..e3675fa29b 100644 --- a/app/modules/api_keys/service.py +++ b/app/modules/api_keys/service.py @@ -802,7 +802,7 @@ async def enforce_limits_for_request( request_model: str | None, request_service_tier: str | None = None, request_usage_budget: ApiKeyRequestUsageBudget | None = None, - ) -> ApiKeyUsageReservationData: + ) -> ApiKeyUsageReservationData | None: for attempt in range(_SQLITE_BUSY_RETRY_ATTEMPTS): try: return await self._enforce_limits_for_request_once( @@ -826,7 +826,7 @@ async def _enforce_limits_for_request_once( request_model: str | None, request_service_tier: str | None, request_usage_budget: ApiKeyRequestUsageBudget | None, - ) -> ApiKeyUsageReservationData: + ) -> ApiKeyUsageReservationData | None: now = utcnow() async with sqlite_writer_section(): row = _ensure_valid_api_key_row(await self._repository.get_for_limit_enforcement(key_id)) @@ -842,6 +842,7 @@ async def _enforce_limits_for_request_once( raise ApiKeyInvalidError("API key has expired") reservation_items: list[UsageReservationItemData] = [] + reservation_id: str | None = None normalized_usage_budget = _normalize_request_usage_budget(request_usage_budget) try: for limit in refreshed.limits: @@ -872,18 +873,55 @@ async def _enforce_limits_for_request_once( ) ) - reservation_id = _next_usage_reservation_id() - await self._repository.create_usage_reservation( - reservation_id, - key_id=key_id, - model=request_model or "", - items=reservation_items, - ) - await self._repository.commit() + if not reservation_items: + # No configured limit applies to this request, so there is + # nothing to reserve and nothing to settle. Skip the empty + # reservation INSERT and its full-durability commit (the + # lazy expired-limit reset above commits inside + # ``reset_limit`` itself, so no write is pending here). + # Every downstream consumer treats a missing reservation + # as "nothing to settle". Commit to close the implicit + # transaction opened by the admission SELECTs: on sessions + # that outlive this call (e.g. the quota-planner warmup + # service) an open transaction would otherwise idle across + # the upstream round-trip until the next commit. This must + # be ``commit()`` rather than ``rollback()``: + # ``AsyncSession.rollback()`` expires every tracked ORM + # object regardless of ``expire_on_commit``, and the warmup + # service shares this session with already-loaded + # ``account``/``decision`` rows whose attribute access + # after expiry raises ``MissingGreenlet``. ``commit()`` + # with ``expire_on_commit=False`` (app/db/session.py) + # leaves tracked state loaded, and is semantically + # equivalent here because the transaction holds only the + # admission SELECTs — no reservation write ran, and every + # production caller either dedicates a session to + # admission (proxy paths) or commits each prior write + # inside its repository methods (quota-planner), so no + # unrelated dirty state can be flushed by this commit. + await self._repository.commit() + else: + reservation_id = _next_usage_reservation_id() + await self._repository.create_usage_reservation( + reservation_id, + key_id=key_id, + model=request_model or "", + items=reservation_items, + ) + await self._repository.commit() except Exception: await self._repository.rollback() raise + if reservation_id is None: + # Settlement is the only other production writer of + # ``last_used_at``; without a reservation it never runs, so record + # the last-used touch at admission instead. Recorded outside + # sqlite_writer_section() for the same reason as settlement: the + # shutdown write-through flush takes the writer section itself. + await self._last_used_coalescer.record(key_id, utcnow()) + return None + return ApiKeyUsageReservationData( reservation_id=reservation_id, key_id=key_id, diff --git a/app/modules/quota_planner/warmup.py b/app/modules/quota_planner/warmup.py index 4b57e6debb..b1e9022c02 100644 --- a/app/modules/quota_planner/warmup.py +++ b/app/modules/quota_planner/warmup.py @@ -151,7 +151,9 @@ async def warm_now( output_tokens=WARMUP_DEFAULT_OUTPUT_BUDGET, ), ) - reservation_id = reservation.reservation_id + # ``None`` means no configured limit applies to the warmup + # probe; there is nothing to finalize afterwards. + reservation_id = reservation.reservation_id if reservation is not None else None except ApiKeyNotFoundError: row = await self._planner.update_decision_status( decision.id, diff --git a/openspec/changes/skip-empty-usage-reservations/design.md b/openspec/changes/skip-empty-usage-reservations/design.md new file mode 100644 index 0000000000..14f0aba9bc --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/design.md @@ -0,0 +1,80 @@ +# Design + +## Where the skip lives + +`ApiKeysService._enforce_limits_for_request_once` builds +`reservation_items` by iterating the key's limits and appending one item +per **applicable** limit (including zero-delta items — the +"Zero-reservation limits still settle actual usage" requirement depends on +those items existing). `reservation_items` is therefore empty **iff** no +limit applies to the request. In that case the function returns `None` +before `create_usage_reservation` + `commit`. + +Safety of skipping the commit: with zero applicable limits no +`try_reserve_usage` CAS ran (and a zero-delta call is read-only), and the +lazy expired-limit reset commits inside `reset_limit` itself, so there is +no pending write to lose when admission returns early. The early return +still issues a `commit()` to close the implicit transaction opened by +the admission SELECTs: proxy call sites use short-lived background +sessions, but the quota-planner warmup service holds one long-lived +session, and leaving the read transaction open would pin an +idle-in-transaction window across the warmup probe's upstream round-trip. + +Why `commit()` and not `rollback()`: `AsyncSession.rollback()` expires +every tracked ORM instance **regardless of** `expire_on_commit=False`. +The warmup service shares its long-lived session with this repository and +already tracks `account` and `decision` rows; expiring them makes the +subsequent `_send_warmup_probe` access to `account.access_token_encrypted` +(and the error path's `decision.id`) raise `MissingGreenlet` — limit-free +warmups would never execute. `commit()` with `expire_on_commit=False` +(both session factories in `app/db/session.py`) leaves tracked state +loaded. It is semantically equivalent to a rollback at this point because +the open transaction holds only the admission SELECTs, and no unrelated +dirty state can be flushed by it: the proxy call sites dedicate a +fresh/scoped session to admission (`get_background_session`, +`_repo_factory`), and the quota-planner repositories commit every prior +write inside their own methods (`log_decision`, `update_decision_status`, +`claim_warmup_decision` — the latter even commits at the start of its own +transaction). Regression coverage drives the real `ApiKeysService` through +a shared session and asserts the probe's attributes stay readable. + +Settlement is not a pure no-op for the ledger only: `_settle_usage_reservation` +is also the production writer of the key's last-used touch (write-behind +coalescer → `api_keys.last_used_at`). Without a reservation settlement never +runs, so the limit-free admission path records the coalescer touch itself +before returning `None` — `last_used_at` keeps advancing for limit-free keys +exactly once per admitted request, at admission time instead of stream end. +The record is in-memory (no extra commit) and sits outside +`sqlite_writer_section()` for the same reason as settlement's: the shutdown +write-through flush takes the writer section itself. + +## Consumer audit (verified in code before implementation) + +| Consumer | Behavior on missing reservation | +| --- | --- | +| `_settle_stream_api_key_usage` (`api_key_usage.py`) | `api_key_reservation is None` → returns `True` (settled no-op) | +| `_settle_compact_api_key_usage` | `api_key_reservation is None` → returns | +| `_release_reservation` / `_release_reservation_best_effort` / `_finalize_image_reservation` / `_settle_source_reservation` (`proxy/api.py`) | `reservation is None` → return / `True` | +| `_release_websocket_reservation` / heartbeat `_maybe_touch_api_key_reservation` / heartbeat task start | `None` → no-op | +| HTTP bridge forwarding | reservation headers only added when non-`None`; `_reservation_from_headers` returns `None` when absent | +| Bridge retry re-reservation (`http_bridge/streaming.py`) | guarded by `api_key_reservation is not None`; `begin_bridge_lifecycle` accepts `None`. With a `None` reservation, `same_reservation` (`previous is reservation`) is `True` across submit retries, so deferred account error backoffs carry over instead of resetting per re-reservation. This is the **pre-existing** lifecycle semantic for every keyless request (`api_key=None` account-direct traffic exercises `begin_bridge_lifecycle(None)` on each retry today); limit-free keyed requests now intentionally join that class. Drain-once is preserved (the dict is carried by reference and popped on drain), and the wrapper-finally early-release branch not firing for both-`None` loses nothing: releasing a `None` reservation is a no-op and non-empty `pending_backoffs` still triggers the branch via the `or`. | +| Stale-release scheduler | operates on reservation rows; limit-free admissions simply produce none | +| Quota-planner warmup | **adapted**: `reservation_id` becomes `None` when admission returns no reservation; finalize/fail calls already guard on `reservation_id is not None` | + +## Interaction with `has_applicable_limits` + +`ApiKeyUsageReservationData.has_applicable_limits` stays (the bridge +header round-trip and `_reservation_requires_usage` read it), but a +returned reservation now always has it `True`; `None` replaces the former +"reservation exists but has no applicable limits" state. The +`_reservation_requires_usage(reservation)` predicate is unchanged and +degenerates to `reservation is not None`. + +## Rejected alternatives + +- Keeping the empty INSERT with relaxed durability: still pays the + round trips and the stream-end settlement transaction; #1665 pinned + reservation-ledger writes to full durability, so relaxing is off-limits. +- Returning a sentinel reservation without persisting it: every consumer + would need to learn the sentinel; `None` already has a fully audited + no-op path (the `api_key is None` case exercises it today). diff --git a/openspec/changes/skip-empty-usage-reservations/proposal.md b/openspec/changes/skip-empty-usage-reservations/proposal.md new file mode 100644 index 0000000000..f466969082 --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/proposal.md @@ -0,0 +1,51 @@ +## Why + +Every keyed request pays the reservation ledger even when the key has no +applicable limits: admission INSERTs an empty `api_key_usage_reservations` +row (zero items) and runs a full-durability commit, and stream end runs the +whole settlement transaction just to flip that empty row to `finalized`. +For unlimited keys this is pure hot-path CPU and write amplification with +no enforcement value — there is nothing to reserve and nothing to settle. + +## What Changes + +- API-key admission returns no reservation when no configured limit applies + to the request (key has no limits, or none match the request model). The + reservation INSERT and its full-durability commit are skipped entirely. +- Downstream reservation consumers (stream/compact settlement, release + paths, heartbeat touch, quota-planner warmup finalize) already no-op on a + missing reservation; the quota-planner warmup executor is adapted to + tolerate admission returning no reservation. +- Because settlement is also the production writer of the key's last-used + touch, the limit-free admission path records the write-behind coalescer + touch itself, so dashboard-visible `last_used_at` keeps advancing for + limit-free keys (per admitted request, at admission time instead of + stream end). +- Keys with at least one applicable limit are unaffected: reservation + creation, full commit durability (#1665), exactly-once settlement, and + stale-reservation reclamation are unchanged for them. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: Add a reservation-ledger requirement that limit-free + admissions skip reservation creation and that downstream settlement, + release, and heartbeat paths no-op without a reservation. + +## Impact + +- `app/modules/api_keys/service.py`: `enforce_limits_for_request` (and the + single-attempt worker) return `None` when no reservation items exist. +- `app/modules/quota_planner/warmup.py`: warmup executor handles a `None` + reservation (probes without finalizing). +- Per-request effect for unlimited keys: one INSERT + one synchronous + full-durability commit removed from admission, and the entire stream-end + settlement transaction removed. No API, setting, dependency, migration, + or dashboard change. Operator-visible effect: keys without applicable + limits no longer produce reservation rows, so stale-reservation + reclamation counts no longer include them. diff --git a/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md b/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md new file mode 100644 index 0000000000..eb63781dcc --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/specs/api-keys/spec.md @@ -0,0 +1,51 @@ +## ADDED Requirements + +### Requirement: Limit-free admissions skip the reservation ledger + +When API-key admission finds no applicable limit for a request (the key has no configured limits, or none of its limits apply to the request model), the system MUST NOT create a usage reservation row and MUST NOT run the reservation commit for that request. Admission MUST report that no reservation exists, and every downstream reservation consumer (stream and compact settlement, release paths, heartbeat touch, quota-planner warmup finalization) MUST treat the missing reservation as "nothing to settle" and no-op without error. Admission-time validity checks (key active, key not expired, lazy expired-limit reset) MUST still run unchanged. Because settlement — which records the key's last-used touch for reserved requests — never runs without a reservation, admission MUST record the last-used touch itself on the limit-free path so `last_used_at` continues to advance for these keys. Admission MUST also close the read transaction it opened before returning without a reservation, and MUST do so without expiring ORM state tracked by a caller-shared session (callers such as the quota-planner warmup service hold already-loaded rows on the same session and access them after admission). Keys with at least one applicable limit MUST continue to create reservations with per-limit items (including zero-delta items) and full commit durability. + +#### Scenario: Key without limits creates no reservation + +- **WHEN** admission runs for an API key with no configured limits +- **THEN** no usage reservation row is inserted and no reservation write is committed (the only commit issued closes the read-only admission transaction) +- **AND** the request is admitted without a reservation + +#### Scenario: Key whose limits do not apply to the request model creates no reservation + +- **WHEN** admission runs for a key whose limits all carry a `model_filter` that does not match the request model +- **THEN** no usage reservation row is inserted +- **AND** the non-matching limits' `current_value` values are unchanged + +#### Scenario: Limit-free admissions still advance last-used + +- **WHEN** admission runs for a key with no applicable limits +- **THEN** the key's last-used touch is recorded at admission via the write-behind coalescer +- **AND** the dashboard-visible `last_used_at` continues to advance for the key + +#### Scenario: Settlement, release, and heartbeat no-op without a reservation + +- **WHEN** a request admitted without a reservation finishes (success or failure) +- **THEN** settlement, release, and heartbeat-touch paths skip without error +- **AND** no settlement transaction runs for that request + +#### Scenario: Quota-planner warmup probes without a reservation + +- **WHEN** the quota-planner warmup executor admits its probe with a key that has no applicable limits +- **THEN** the warmup probe executes +- **AND** no reservation finalization is attempted + +#### Scenario: Limit-free admission preserves shared-session ORM state + +- **WHEN** a caller that holds already-loaded ORM rows on the same session (the quota-planner warmup service tracks the target account and decision) admits a request with a limit-free key +- **THEN** the admission read transaction is closed before admission returns +- **AND** the caller's tracked rows remain readable afterwards without reload errors, so the warmup probe executes + +#### Scenario: Stale-reservation reclamation sees no rows for limit-free admissions + +- **WHEN** stale usage-reservation reclamation runs after admissions for keys without applicable limits +- **THEN** those admissions contribute no reservations to reclaim + +#### Scenario: Limited keys are unaffected + +- **WHEN** admission runs for a key with an applicable limit +- **THEN** a reservation with per-limit items is created and committed exactly as before admission returned reservations unconditionally diff --git a/openspec/changes/skip-empty-usage-reservations/tasks.md b/openspec/changes/skip-empty-usage-reservations/tasks.md new file mode 100644 index 0000000000..242f323bb7 --- /dev/null +++ b/openspec/changes/skip-empty-usage-reservations/tasks.md @@ -0,0 +1,50 @@ +# Tasks + +## 1. Admission skip + +- [x] 1.1 Return `None` from `ApiKeysService._enforce_limits_for_request_once` + when `reservation_items` is empty (no applicable limits), skipping the + reservation INSERT and its commit; widen `enforce_limits_for_request` + return type to `ApiKeyUsageReservationData | None`. + +## 2. Consumer audit / adaptation + +- [x] 2.1 Verify settlement (`_settle_stream_api_key_usage`, + `_settle_compact_api_key_usage`), release paths + (`_release_reservation*`, `_release_websocket_reservation`), + heartbeat (`_maybe_touch_api_key_reservation`), bridge forwarding + (`_reservation_from_headers`), and retry re-reservation guards all + no-op on a `None` reservation. +- [x] 2.2 Adapt the quota-planner warmup executor to a `None` reservation + (probe without finalize). +- [x] 2.3 Record the key's last-used coalescer touch at admission on the + limit-free path (settlement, the production `last_used_at` writer, + never runs without a reservation). +- [x] 2.4 Roll back the admission read transaction before the limit-free + early return (long-lived sessions must not idle in transaction + across upstream round-trips). + +## 3. Tests + +- [x] 3.1 Unit: key without limits → admission returns `None`, no + reservation INSERT; the only commit is the read-only transaction close. +- [x] 3.2 Unit: key whose limits do not match the request model → `None`, + limits untouched. +- [x] 3.2b Unit: limit-free admission records the last-used coalescer touch + and closes the read transaction via commit — never rollback, which + would expire shared-session ORM state (quota-planner warmup). +- [x] 3.3 Unit: settlement/release/heartbeat with `reservation=None` no-op. +- [x] 3.4 Integration: quota-planner warmup executes with a limit-free key + (no finalize call, probe succeeds). +- [x] 3.4b Integration (regression): warmup through the REAL ApiKeysService + on the shared session — the probe's `account.access_token_encrypted` + access stays readable after the limit-free early return (no + MissingGreenlet from expired shared state). +- [x] 3.5 Integration: stale-release reclamation finds no rows after + limit-free admissions; limited keys keep creating reservations + (regression). + +## 4. Spec + +- [x] 4.1 Delta to `openspec/specs/api-keys/spec.md` reservation-ledger + requirements; validate with `openspec validate --specs`. diff --git a/tests/integration/test_api_keys_api.py b/tests/integration/test_api_keys_api.py index e81c94f6f9..78a2309cd6 100644 --- a/tests/integration/test_api_keys_api.py +++ b/tests/integration/test_api_keys_api.py @@ -3559,6 +3559,10 @@ async def fake_sqlite_writer_section(): stale_second = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") abandoned_before_first_heartbeat = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") fresh = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert stale is not None + assert stale_second is not None + assert abandoned_before_first_heartbeat is not None + assert fresh is not None await session.execute( update(ApiKeyUsageReservation) .where(ApiKeyUsageReservation.id.in_([stale.reservation_id, stale_second.reservation_id])) @@ -3608,6 +3612,55 @@ async def fake_sqlite_writer_section(): assert limits[0].current_value == fresh_reservation.items[0].reserved_delta +@pytest.mark.asyncio +async def test_limit_free_admission_creates_no_reservation_rows(async_client): + """Limit-free keys skip the reservation ledger: admission returns no + reservation, writes no rows, and stale-reservation reclamation has + nothing to release; limited keys keep creating reservations.""" + del async_client + now = utcnow() + + async with SessionLocal() as session: + repo = ApiKeysRepository(session) + service = ApiKeysService(repo) + unlimited = await service.create_key( + ApiKeyCreateData(name="limit-free-admission", allowed_models=None, expires_at=None) + ) + limited = await service.create_key( + ApiKeyCreateData( + name="limited-admission-regression", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=50_000), + ], + ) + ) + unlimited_reservation = await service.enforce_limits_for_request(unlimited.id, request_model="gpt-5.1") + limited_reservation = await service.enforce_limits_for_request(limited.id, request_model="gpt-5.1") + + assert unlimited_reservation is None + assert limited_reservation is not None + assert limited_reservation.has_applicable_limits is True + + async with SessionLocal() as session: + rows = await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.api_key_id == unlimited.id) + ) + assert rows.scalars().all() == [] + repo = ApiKeysRepository(session) + limited_row = await repo.get_usage_reservation(limited_reservation.reservation_id) + assert limited_row is not None + assert limited_row.status == "reserved" + # A future cutoff would reclaim any reserved row; the limit-free key + # contributed none, so only the limited key's reservation is released. + released_count = await repo.release_stale_usage_reservations( + cutoff=now + timedelta(hours=1), + max_age_cutoff=now + timedelta(hours=1), + ) + assert released_count == 1 + + @pytest.mark.asyncio async def test_enforce_limits_lazy_reset_and_expiry_with_narrowed_admission_load(async_client): """Regression for the narrowed admission load (``get_for_limit_enforcement``). @@ -3645,6 +3698,7 @@ async def test_enforce_limits_lazy_reset_and_expiry_with_narrowed_admission_load repo = ApiKeysRepository(session) service = ApiKeysService(repo) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert reservation.has_applicable_limits is True async with SessionLocal() as session: @@ -3708,6 +3762,8 @@ async def fake_sqlite_writer_section(): ) heartbeat_kept = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") fresh = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert heartbeat_kept is not None + assert fresh is not None # An orphaned heartbeat keeps the reservation's updated_at current # even though it was created past the hard age ceiling. await session.execute( diff --git a/tests/integration/test_detached_persistence.py b/tests/integration/test_detached_persistence.py index cb18ef800b..732f0c66d4 100644 --- a/tests/integration/test_detached_persistence.py +++ b/tests/integration/test_detached_persistence.py @@ -144,6 +144,7 @@ async def test_failed_detached_settlement_retries_failed_release_until_persisted output_tokens=6, ), ) + assert reservation is not None original_get_reservation = ApiKeysRepository.get_usage_reservation reservation_read_attempts = 0 @@ -212,6 +213,76 @@ async def fail_first_two_reservation_reads( assert retry_was_tracked is True +@pytest.mark.asyncio +async def test_settlement_release_and_heartbeat_noop_without_reservation(): + """A limit-free admission yields no reservation; settlement, release, and + heartbeat must no-op without opening a repository session.""" + from contextlib import asynccontextmanager + from typing import cast + + from app.core.utils.time import utcnow + from app.modules.api_keys.service import ApiKeyData + + factory_uses = 0 + + @asynccontextmanager + async def repo_factory(): + nonlocal factory_uses + factory_uses += 1 + yield object() + + service = proxy_service_module.ProxyService(cast(proxy_service_module.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_unlimited", + name="unlimited", + key_prefix="sk-clb-test", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + settlement = proxy_service_module._StreamSettlement( + status="success", + model="gpt-5.5", + input_tokens=4, + output_tokens=6, + ) + + assert ( + await service._settle_stream_api_key_usage( + api_key, + None, + settlement, + request_id="req_no_reservation", + ) + is True + ) + assert settlement.usage_settlement_transferred is False + await service._settle_compact_api_key_usage( + api_key=api_key, + api_key_reservation=None, + response=None, + request_service_tier=None, + ) + await service._release_websocket_reservation(None) + assert ( + await service._maybe_touch_api_key_reservation( + api_key=api_key, + reservation=None, + last_touch_at=123.0, + request_id="req_no_reservation", + surface="stream", + ) + == 123.0 + ) + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert factory_uses == 0 + + @pytest.mark.asyncio async def test_drain_ignores_stuck_non_persistence_cleanup_tasks(): """A stuck bridge-close cleanup in _background_cleanup_tasks must not diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 5810721243..2bd9a215a5 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -86,6 +86,7 @@ async def test_proxy_compact_forwarded_bridge_settlement_failure_surfaces_code_a request_service_tier=None, request_usage_budget=estimate_api_key_request_usage(compact_model), ) + assert reservation is not None async with SessionLocal() as session: row = await session.get(ApiKeyUsageReservation, reservation.reservation_id) assert row is not None @@ -1459,6 +1460,7 @@ async def test_proxy_compact_forwarded_bridge_preflight_budget_exhausted_settles request_service_tier=None, request_usage_budget=estimate_api_key_request_usage(compact_model), ) + assert reservation is not None async with SessionLocal() as session: row = await session.get(ApiKeyUsageReservation, reservation.reservation_id) assert row is not None diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index f03254d995..38234ab208 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -37,7 +37,13 @@ from app.db.models import Account, AccountStatus, ApiKeyUsageReservation, RequestLog from app.db.session import SessionLocal from app.modules.api_keys.repository import ApiKeysRepository -from app.modules.api_keys.service import ApiKeyCreateData, ApiKeyData, ApiKeysService, ApiKeyUsageReservationData +from app.modules.api_keys.service import ( + ApiKeyCreateData, + ApiKeyData, + ApiKeysService, + ApiKeyUsageReservationData, + LimitRuleInput, +) from app.modules.proxy._service.websocket import mixin as websocket_mixin_module from app.modules.proxy.affinity import _codex_session_selection_key from app.modules.proxy.capability_routing import ( @@ -419,12 +425,19 @@ async def prepare_persistence_rows() -> tuple[ApiKeyData, ApiKeyUsageReservation ApiKeyCreateData( name="route drain", allowed_models=None, + # Limit-free keys skip the reservation ledger entirely, and + # this test asserts drain-time settlement ownership of a + # real reservation, so give the key an applicable limit. + limits=[ + LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000), + ], ) ) usage_reservation = await service.enforce_limits_for_request( created_key.id, request_model="gpt-5.6-sol", ) + assert usage_reservation is not None return created_key, usage_reservation async def read_persisted_results( diff --git a/tests/integration/test_quota_planner_api.py b/tests/integration/test_quota_planner_api.py index 587bc62b37..23e65793ec 100644 --- a/tests/integration/test_quota_planner_api.py +++ b/tests/integration/test_quota_planner_api.py @@ -12,7 +12,15 @@ from app.core.crypto import TokenEncryptor from app.core.utils.time import utcnow -from app.db.models import Account, AccountStatus, QuotaPlannerDecision, QuotaWindowObservation, RequestLog, UsageHistory +from app.db.models import ( + Account, + AccountStatus, + ApiKey, + QuotaPlannerDecision, + QuotaWindowObservation, + RequestLog, + UsageHistory, +) from app.db.session import SessionLocal from app.modules.api_keys.service import ApiKeyInvalidError, ApiKeyNotFoundError, ApiKeyRateLimitExceededError from app.modules.quota_planner.logic import PlannerSettings @@ -933,6 +941,156 @@ async def cancel_probe(self, *, account, model, request_id): assert failed_reservations == [("reservation-cancelled", "gpt-5.4-mini", 0, 0, 0)] +@pytest.mark.asyncio +async def test_quota_planner_warm_now_limit_free_key_probes_without_reservation(monkeypatch, db_setup): + """A key with no applicable limits admits without a reservation; the + warmup probe must execute and never attempt reservation settlement.""" + del db_setup + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = Account( + id="acc-warm-unlimited-key", + email="warm-unlimited-key@example.test", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + ) + session.add(account) + repo = QuotaPlannerRepository(session) + await repo.upsert_settings( + PlannerSettings( + mode="auto", + allow_synthetic_traffic=True, + dry_run=False, + max_warmup_credits_per_day=1.0, + warmup_model_preference="gpt-5.4-mini", + ) + ) + await repo.add_window_observation( + account_id=account.id, + model="gpt-5.4-mini", + source="warmup_probe", + confidence="observed", + ) + service = QuotaWarmupService(session) + + class FakeApiKeys: + async def enforce_limits_for_request(self, *args, **kwargs): + del args, kwargs + return None + + async def finalize_usage_reservation(self, *args, **kwargs): + del args, kwargs + raise AssertionError("limit-free warmup must not finalize a reservation") + + async def fail_usage_reservation(self, *args, **kwargs): + del args, kwargs + raise AssertionError("limit-free warmup must not fail a reservation") + + async def fake_send(self, *, account, model, request_id): + del self, account, model, request_id + return WarmupUsage(input_tokens=3, output_tokens=1, cached_input_tokens=0, reasoning_tokens=None) + + async def noop_record_effect(self, account, model, *, source, confidence): + del self, account, model, source, confidence + + monkeypatch.setattr(service, "_api_keys", FakeApiKeys()) + monkeypatch.setattr(QuotaWarmupService, "_send_warmup_probe", fake_send) + monkeypatch.setattr(QuotaWarmupService, "_record_warmup_effect", noop_record_effect) + + result = await service.warm_now( + account_id=account.id, + model="gpt-5.4-mini", + api_key_id="api-key-unlimited", + force_probe=True, + ) + + assert result.status == "executed" + assert result.reason == "warmup_executed" + + +@pytest.mark.asyncio +async def test_quota_planner_warm_now_limit_free_admission_keeps_shared_session_state(monkeypatch, db_setup): + """Regression: the limit-free early return closes the admission + transaction on the warmup service's shared session. It must do so + without expiring tracked ORM state (``rollback()`` expires everything + even with ``expire_on_commit=False``): the probe reads + ``account.access_token_encrypted`` and the error path reads + ``decision.id`` after admission, which raised ``MissingGreenlet`` when + the shared ``account``/``decision`` objects were expired. Exercises the + REAL ``ApiKeysService`` with a real limit-free key row — a fake api-key + service returning ``None`` never runs the transaction-closing path.""" + del db_setup + encryptor = TokenEncryptor() + async with SessionLocal() as session: + account = Account( + id="acc-warm-limit-free-real", + email="warm-limit-free-real@example.test", + plan_type="plus", + access_token_encrypted=encryptor.encrypt("access"), + refresh_token_encrypted=encryptor.encrypt("refresh"), + id_token_encrypted=encryptor.encrypt("id"), + last_refresh=utcnow(), + status=AccountStatus.ACTIVE, + ) + session.add(account) + # Real API key with no configured limits: admission takes the + # limit-free early return inside the real ApiKeysService. + session.add( + ApiKey( + id="api-key-limit-free-real", + name="limit-free warmup key", + key_hash="limit-free-warmup-hash", + key_prefix="sk-lfw", + ) + ) + repo = QuotaPlannerRepository(session) + await repo.upsert_settings( + PlannerSettings( + mode="auto", + allow_synthetic_traffic=True, + dry_run=False, + max_warmup_credits_per_day=1.0, + warmup_model_preference="gpt-5.4-mini", + ) + ) + service = QuotaWarmupService(session) + + probed_tokens: list[str] = [] + + async def fake_send(self, *, account, model, request_id): + del model, request_id + # Mirror the real probe's first attribute access on the shared + # session's tracked account: raises MissingGreenlet if admission + # expired it. + probed_tokens.append(self._encryptor.decrypt(account.access_token_encrypted)) + return WarmupUsage(input_tokens=3, output_tokens=1, cached_input_tokens=0, reasoning_tokens=None) + + async def noop_record_effect(self, account, model, *, source, confidence): + del self, account, model, source, confidence + + monkeypatch.setattr(QuotaWarmupService, "_send_warmup_probe", fake_send) + monkeypatch.setattr(QuotaWarmupService, "_record_warmup_effect", noop_record_effect) + + result = await service.warm_now( + account_id=account.id, + model="gpt-5.4-mini", + api_key_id="api-key-limit-free-real", + force_probe=True, + ) + + # The tracked objects must remain readable after admission (the + # failure-handling path reads ``decision.id``-style attributes too). + assert account.access_token_encrypted is not None + + assert probed_tokens == ["access"] + assert result.status == "executed" + assert result.reason == "warmup_executed" + + @pytest.mark.asyncio async def test_quota_planner_warm_now_api_key_not_found_is_skipped(monkeypatch, db_setup): del db_setup diff --git a/tests/unit/test_api_keys_service.py b/tests/unit/test_api_keys_service.py index a11f0673bc..8255bf73b1 100644 --- a/tests/unit/test_api_keys_service.py +++ b/tests/unit/test_api_keys_service.py @@ -1395,6 +1395,7 @@ async def test_enforce_limits_reserves_tier_aware_cost_budget() -> None: request_service_tier="priority", request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8192, output_tokens=8192), ) + assert priority_reservation is not None assert priority_reservation.key_id == priority_created.id priority_limits = await repo.get_limits_by_key(priority_created.id) @@ -1417,6 +1418,7 @@ async def test_enforce_limits_reserves_tier_aware_cost_budget() -> None: request_service_tier=None, request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8192, output_tokens=8192), ) + assert standard_reservation is not None assert standard_reservation.key_id == standard_created.id standard_limits = await repo.get_limits_by_key(standard_created.id) @@ -1456,7 +1458,9 @@ async def test_enforce_limits_default_budget_allows_eight_priority_lanes_under_f ) assert len(reservations) == 8 - assert {reservation.key_id for reservation in reservations} == {created.id} + granted = [reservation for reservation in reservations if reservation is not None] + assert len(granted) == 8 + assert {reservation.key_id for reservation in granted} == {created.id} limits = await repo.get_limits_by_key(created.id) cost_limit = next(lim for lim in limits if lim.limit_type == LimitType.COST_USD) assert 0 < cost_limit.current_value < 5_000_000 @@ -1513,6 +1517,7 @@ async def test_finalize_usage_reservation_accounts_for_zero_reserved_limit_item( request_model="gpt-5.5", request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=0, output_tokens=0), ) + assert reservation is not None limits = await repo.get_limits_by_key(created.id) output_limit = next(limit for limit in limits if limit.limit_type == LimitType.OUTPUT_TOKENS) @@ -1556,16 +1561,110 @@ async def create_usage_reservation( repo = _BusyRepo() service = ApiKeysService(repo) monkeypatch.setattr("app.modules.api_keys.service.asyncio.sleep", _async_noop) - created = await service.create_key(ApiKeyCreateData(name="busy-retry-key", allowed_models=None, expires_at=None)) + created = await service.create_key( + ApiKeyCreateData( + name="busy-retry-key", + allowed_models=None, + expires_at=None, + limits=[LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000)], + ) + ) initial_commit_count = repo.commit_count reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert reservation.key_id == created.id assert repo.create_usage_reservation_calls == 3 assert repo.commit_count == initial_commit_count + 1 +@pytest.mark.asyncio +async def test_enforce_limits_without_limits_skips_reservation_and_commit() -> None: + repo = _FakeApiKeysRepository() + coalescer = ApiKeyLastUsedCoalescer() + service = ApiKeysService(repo, last_used_coalescer=coalescer) + created = await service.create_key(ApiKeyCreateData(name="unlimited-key", allowed_models=None, expires_at=None)) + initial_commit_count = repo.commit_count + initial_rollback_calls = repo.rollback_calls + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + assert reservation is None + assert repo._reservations == {} + # The implicit transaction opened by the admission SELECTs must be closed + # on the limit-free path (long-lived sessions would otherwise idle in + # transaction across the upstream round-trip) — via commit(), never + # rollback(): rollback expires every ORM object tracked by a shared + # session even with expire_on_commit=False, which broke quota-planner + # warmup probes. The commit is read-only (no reservation was inserted). + assert repo.commit_count == initial_commit_count + 1 + assert repo.rollback_calls == initial_rollback_calls + # Settlement never runs without a reservation, so admission itself must + # record the last-used touch for limit-free keys. + pending = coalescer.pending_snapshot() + assert set(pending) == {created.id} + assert pending[created.id] <= utcnow() + + +@pytest.mark.asyncio +async def test_enforce_limits_with_no_applicable_limits_skips_reservation_and_leaves_limits_untouched() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="filtered-limits-key", + allowed_models=None, + expires_at=None, + limits=[ + LimitRuleInput( + limit_type="total_tokens", + limit_window="weekly", + max_value=10_000, + model_filter="gpt-5.1", + ), + ], + ) + ) + initial_commit_count = repo.commit_count + initial_rollback_calls = repo.rollback_calls + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.5") + + assert reservation is None + assert repo._reservations == {} + # One read-only commit closes the admission transaction; no rollback + # (rollback would expire shared-session ORM state — see the limit-free + # transaction-close comment in _enforce_limits_for_request_once). + assert repo.commit_count == initial_commit_count + 1 + assert repo.rollback_calls == initial_rollback_calls + limits = await repo.get_limits_by_key(created.id) + assert limits[0].current_value == 0 + + +@pytest.mark.asyncio +async def test_enforce_limits_with_applicable_limit_still_creates_reservation() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + created = await service.create_key( + ApiKeyCreateData( + name="limited-regression-key", + allowed_models=None, + expires_at=None, + limits=[LimitRuleInput(limit_type="total_tokens", limit_window="weekly", max_value=1_000_000)], + ) + ) + initial_commit_count = repo.commit_count + + reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + + assert reservation is not None + assert reservation.key_id == created.id + assert reservation.has_applicable_limits is True + assert reservation.reservation_id in repo._reservations + assert repo.commit_count == initial_commit_count + 1 + + @pytest.mark.asyncio async def test_enforce_limits_retries_sqlite_busy_during_lazy_reset_rolls_back(monkeypatch: pytest.MonkeyPatch) -> None: class _BusyRepo(_FakeApiKeysRepository): @@ -1611,6 +1710,7 @@ async def rollback(self) -> None: reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5") + assert reservation is not None assert reservation.key_id == created.id assert repo.reset_limit_calls == 3 assert repo.rollback_calls >= 2 @@ -1857,6 +1957,7 @@ async def test_usage_reservation_uses_gpt_5_6_personality_pricing( request_model=model, request_usage_budget=ApiKeyRequestUsageBudget(input_tokens=8_192, output_tokens=8_192), ) + assert reservation is not None limits = await repo.get_limits_by_key(created.id) cost_limit = next(lim for lim in limits if lim.limit_type == LimitType.COST_USD) @@ -1888,6 +1989,7 @@ async def test_release_usage_reservation_restores_reserved_counter() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None limits = await repo.get_limits_by_key(created.id) assert limits[0].current_value == 100 @@ -1912,6 +2014,7 @@ async def test_touch_usage_reservation_only_updates_reserved_reservation() -> No ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert await service.touch_usage_reservation(reservation.reservation_id) is True await service.release_usage_reservation(reservation.reservation_id) @@ -1935,6 +2038,7 @@ async def test_finalize_usage_reservation_is_idempotent() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.finalize_usage_reservation( reservation.reservation_id, model="gpt-5.1", @@ -1972,6 +2076,7 @@ async def test_finalize_usage_reservation_records_last_used_in_coalescer() -> No initial_commit_count = repo.commit_count reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None assert repo.commit_count == initial_commit_count + 1 await service.finalize_usage_reservation( @@ -2022,6 +2127,7 @@ async def get_usage_reservation(self, reservation_id: str) -> UsageReservationDa ) ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.finalize_usage_reservation( reservation.reservation_id, @@ -2069,6 +2175,7 @@ async def get_usage_reservation(self, reservation_id: str) -> UsageReservationDa ) ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.release_usage_reservation(reservation.reservation_id) @@ -2097,6 +2204,7 @@ async def test_fail_usage_reservation_preserves_failed_request_record() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.fail_usage_reservation( reservation.reservation_id, model="gpt-5.1", @@ -2129,6 +2237,7 @@ async def test_release_after_finalize_is_noop() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None limits = await repo.get_limits_by_key(created.id) assert limits[0].current_value == 100 # reserved @@ -2167,6 +2276,7 @@ async def test_finalize_after_release_is_noop() -> None: ) reservation = await service.enforce_limits_for_request(created.id, request_model="gpt-5.1") + assert reservation is not None await service.release_usage_reservation(reservation.reservation_id) From b26df4176463b51d310b56a41b05b70163e4d30f Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 17 Aug 2026 23:31:25 +0900 Subject: [PATCH 066/117] chore: release v1.24.0-beta.1 (#1704) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 ++-- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index a0e8b6b5b5..b6ae45e81c 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.23.0" # x-release-please-version +__version__ = "1.24.0-beta.1" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index 1931c51a37..7a7ccf4169 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.23.0 -appVersion: 1.23.0 +version: 1.24.0-beta.1 +appVersion: 1.24.0-beta.1 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index ae9bc40d0f..8692371cac 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.23.0", + "version": "1.24.0-beta.1", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 5057644eed..43d7fad762 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.23.0" +version = "1.24.0-beta.1" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 60b221b908..2c246a951e 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.23.0" +version = "1.24.0-beta.1" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From 4a3832a564f571c7bfde665ddb237399ecb7d0cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:23:15 +0900 Subject: [PATCH 067/117] chore(ci): bump astral-sh/setup-uv from 9.0.0 to 10.0.1 (#1802) Bumps [astral-sh/setup-uv](https://github.com/astral-sh/setup-uv) from 9.0.0 to 10.0.1. - [Release notes](https://github.com/astral-sh/setup-uv/releases) - [Commits](https://github.com/astral-sh/setup-uv/compare/c771a70e6277c0a99b617c7a806ffedaca235ff9...20cfd1bf945f4377ade1205e4dbc17946fc9a30d) --- updated-dependencies: - dependency-name: astral-sh/setup-uv dependency-version: 10.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 18 +++++++++--------- .github/workflows/docs.yml | 2 +- .github/workflows/release-please.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/windows-startup.yml | 2 +- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0f88b4c6b9..2a560860d6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -293,7 +293,7 @@ jobs: key: playwright-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('frontend/bun.lock') }} - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -321,7 +321,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -343,7 +343,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -397,7 +397,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -452,7 +452,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -550,7 +550,7 @@ jobs: - name: Set up uv if: needs.changes.outputs.backend == 'true' - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -573,7 +573,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -609,7 +609,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true @@ -646,7 +646,7 @@ jobs: bun-1.3.14-${{ runner.os }}-${{ runner.arch }}- - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 639c580268..f6556c25b8 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -39,7 +39,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 80fc06b8cf..87bf575033 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -53,7 +53,7 @@ jobs: - name: Set up uv if: ${{ steps.release-branch.outputs.branch != '' }} - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f36b06ac1b..c9cff32d01 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -71,7 +71,7 @@ jobs: run: cd frontend && bun run build - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: false diff --git a/.github/workflows/windows-startup.yml b/.github/workflows/windows-startup.yml index d6110e57cf..f4bc0f1d4b 100644 --- a/.github/workflows/windows-startup.yml +++ b/.github/workflows/windows-startup.yml @@ -18,7 +18,7 @@ jobs: persist-credentials: false - name: Set up uv - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d with: python-version: "3.13" enable-cache: true From a4fa1233b6fc17c9d570848643dfa5c715d8a665 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:23:19 +0900 Subject: [PATCH 068/117] chore(deps): bump the python-minor-patch group with 10 updates (#1806) Bumps the python-minor-patch group with 10 updates: | Package | From | To | | --- | --- | --- | | [alembic](https://github.com/sqlalchemy/alembic) | `1.19.0` | `1.19.1` | | [greenlet](https://github.com/python-greenlet/greenlet) | `3.5.4` | `3.5.5` | | [sqlalchemy](https://github.com/sqlalchemy/sqlalchemy) | `2.0.51` | `2.0.52` | | [uvicorn](https://github.com/Kludex/uvicorn) | `0.52.1` | `0.52.3` | | [aiohttp-socks](https://github.com/romis2012/aiohttp-socks) | `0.11.0` | `0.12.0` | | [pre-commit](https://github.com/pre-commit/pre-commit) | `4.6.1` | `4.6.2` | | [ruff](https://github.com/astral-sh/ruff) | `0.16.2` | `0.16.3` | | [ty](https://github.com/astral-sh/ty) | `0.0.69` | `0.0.72` | | [hypothesis](https://github.com/HypothesisWorks/hypothesis) | `6.165.3` | `6.165.8` | | [hatchling](https://github.com/pypa/hatch) | `1.31.0` | `1.32.0` | Updates `alembic` from 1.19.0 to 1.19.1 - [Release notes](https://github.com/sqlalchemy/alembic/releases) - [Changelog](https://github.com/sqlalchemy/alembic/blob/main/CHANGES) - [Commits](https://github.com/sqlalchemy/alembic/commits) Updates `greenlet` from 3.5.4 to 3.5.5 - [Changelog](https://github.com/python-greenlet/greenlet/blob/master/CHANGES.rst) - [Commits](https://github.com/python-greenlet/greenlet/compare/3.5.4...3.5.5) Updates `sqlalchemy` from 2.0.51 to 2.0.52 - [Release notes](https://github.com/sqlalchemy/sqlalchemy/releases) - [Changelog](https://github.com/sqlalchemy/sqlalchemy/blob/main/CHANGES.rst) - [Commits](https://github.com/sqlalchemy/sqlalchemy/commits) Updates `uvicorn` from 0.52.1 to 0.52.3 - [Release notes](https://github.com/Kludex/uvicorn/releases) - [Changelog](https://github.com/Kludex/uvicorn/blob/main/docs/release-notes.md) - [Commits](https://github.com/Kludex/uvicorn/compare/0.52.1...0.52.3) Updates `aiohttp-socks` from 0.11.0 to 0.12.0 - [Release notes](https://github.com/romis2012/aiohttp-socks/releases) - [Commits](https://github.com/romis2012/aiohttp-socks/compare/v0.11.0...v0.12.0) Updates `pre-commit` from 4.6.1 to 4.6.2 - [Release notes](https://github.com/pre-commit/pre-commit/releases) - [Changelog](https://github.com/pre-commit/pre-commit/blob/main/CHANGELOG.md) - [Commits](https://github.com/pre-commit/pre-commit/compare/v4.6.1...v4.6.2) Updates `ruff` from 0.16.2 to 0.16.3 - [Release notes](https://github.com/astral-sh/ruff/releases) - [Changelog](https://github.com/astral-sh/ruff/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ruff/compare/0.16.2...0.16.3) Updates `ty` from 0.0.69 to 0.0.72 - [Release notes](https://github.com/astral-sh/ty/releases) - [Changelog](https://github.com/astral-sh/ty/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/ty/compare/0.0.69...0.0.72) Updates `hypothesis` from 6.165.3 to 6.165.8 - [Release notes](https://github.com/HypothesisWorks/hypothesis/releases) - [Commits](https://github.com/HypothesisWorks/hypothesis/compare/v6.165.3...v6.165.8) Updates `hatchling` from 1.31.0 to 1.32.0 - [Release notes](https://github.com/pypa/hatch/releases) - [Commits](https://github.com/pypa/hatch/compare/hatchling-v1.31.0...hatchling-v1.32.0) --- updated-dependencies: - dependency-name: alembic dependency-version: 1.19.1 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: greenlet dependency-version: 3.5.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: sqlalchemy dependency-version: 2.0.52 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: uvicorn dependency-version: 0.52.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: aiohttp-socks dependency-version: 0.12.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: python-minor-patch - dependency-name: pre-commit dependency-version: 4.6.2 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: ruff dependency-version: 0.16.3 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: ty dependency-version: 0.0.72 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: hypothesis dependency-version: 6.165.8 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: python-minor-patch - dependency-name: hatchling dependency-version: 1.32.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: python-minor-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- pyproject.toml | 4 +- uv.lock | 357 +++++++++++++++++++++++++------------------------ 2 files changed, 187 insertions(+), 174 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 43d7fad762..2ddaa60269 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ dev = [ "pytest-timeout>=2.4.0", "httpx>=0.28.1", "ruff>=0.14.13", - "ty==0.0.69", + "ty==0.0.72", "openai>=2.16.0", "pytest-xdist>=3.8.0", "pytest-cov>=7.1.0", @@ -128,7 +128,7 @@ codex-lb = "app.cli:main" codex-lb-db = "app.db.migrate:main" [build-system] -requires = ["hatchling==1.31.0"] +requires = ["hatchling==1.32.0"] build-backend = "hatchling.build" [tool.hatch.build] diff --git a/uv.lock b/uv.lock index 2c246a951e..e44000d9c3 100644 --- a/uv.lock +++ b/uv.lock @@ -110,15 +110,15 @@ wheels = [ [[package]] name = "aiohttp-socks" -version = "0.11.0" +version = "0.12.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, { name = "python-socks" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1f/cc/e5bbd54f76bd56291522251e47267b645dac76327b2657ade9545e30522c/aiohttp_socks-0.11.0.tar.gz", hash = "sha256:0afe51638527c79077e4bd6e57052c87c4824233d6e20bb061c53766421b10f0", size = 11196, upload-time = "2025-12-09T13:35:52.564Z" } +sdist = { url = "https://files.pythonhosted.org/packages/18/1d/a306e0111222180e60f17131a3f5d9bc694dd999a8115959a7dd76c2238e/aiohttp_socks-0.12.0.tar.gz", hash = "sha256:3caf9f5a4164611122d412bc11b2f9114fd29c85e1ba27bb38060d3c236bdc8d", size = 12061, upload-time = "2026-08-12T04:43:15.791Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/7d/4b633d709b8901d59444d2e512b93e72fe62d2b492a040097c3f7ba017bb/aiohttp_socks-0.11.0-py3-none-any.whl", hash = "sha256:9aacce57c931b8fbf8f6d333cf3cafe4c35b971b35430309e167a35a8aab9ec1", size = 10556, upload-time = "2025-12-09T13:35:50.18Z" }, + { url = "https://files.pythonhosted.org/packages/86/64/ca6289632020523ea1841f01363f83ada10eeb0f21dd931fc1118dd85668/aiohttp_socks-0.12.0-py3-none-any.whl", hash = "sha256:ba6f95ec775c761d87f8578ab48f137d0457c676da104984202bf75e747d5ee6", size = 10693, upload-time = "2026-08-12T04:43:14.524Z" }, ] [[package]] @@ -144,16 +144,16 @@ wheels = [ [[package]] name = "alembic" -version = "1.19.0" +version = "1.19.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "mako" }, { name = "sqlalchemy" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fb/01/a48dab7827ac4421272399f7ed9a2ec17edd12c8bcde4417bd7b6821b71a/alembic-1.19.0.tar.gz", hash = "sha256:6487c612fc719dcfa22b17d2dd5b2b458929641e6aa2f0b65b135727f5e6d501", size = 2069906, upload-time = "2026-08-04T18:57:04.599Z" } +sdist = { url = "https://files.pythonhosted.org/packages/16/2b/e4153978368de59918115c9e01d3ebf58a558a7285efa7e960c383c4b59a/alembic-1.19.1.tar.gz", hash = "sha256:e0fca0518118c78acc493e31bcb5402f190057aaf6df8b5b95ce94c4789cf648", size = 2070816, upload-time = "2026-08-08T16:32:01.565Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/34/79/59ab6f1fe72eee229de91aa393225389a85df12a0fbbbfa264efcf6d7872/alembic-1.19.0-py3-none-any.whl", hash = "sha256:cf839d3849116aab3cc047e09c6968b9bd6b2fde61b6bb7c1e97352fe5503580", size = 265738, upload-time = "2026-08-04T18:57:06.219Z" }, + { url = "https://files.pythonhosted.org/packages/20/89/e62cc37b69ad357cc8ecd6e7367f5245f523d3cbb338a66197212bdf6749/alembic-1.19.1-py3-none-any.whl", hash = "sha256:b39018cb3d9413a19cbd54cf3c02ad33998641f0538eb77413a488a21c3e14be", size = 265946, upload-time = "2026-08-08T16:32:03.153Z" }, ] [[package]] @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0-beta.1" +version = "1.24.0b1" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -600,7 +600,7 @@ dev = [ { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.14.13" }, - { name = "ty", specifier = "==0.0.69" }, + { name = "ty", specifier = "==0.0.72" }, ] docs = [{ name = "mkdocs-material", specifier = ">=9.6" }] @@ -1013,59 +1013,59 @@ wheels = [ [[package]] name = "greenlet" -version = "3.5.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a3/74/b13368064b09053253555d3f2839cc2684d22d5aed0d2ccffbf7a6736558/greenlet-3.5.4.tar.gz", hash = "sha256:0232ae1de90a8e07867bb127d7a6ba2301e859145489f25cda8a6096dabe1d20", size = 206538, upload-time = "2026-07-22T12:47:14.468Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/9a/e51225dcd58713f16ccbdcc501a8da21098ea14515b7870f1f94459e5ff5/greenlet-3.5.4-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:24e61b88cb7e1b1d794b32a10cc346ac779681d6d74ff137a3e0a444d2bf1f02", size = 294831, upload-time = "2026-07-22T11:38:53.389Z" }, - { url = "https://files.pythonhosted.org/packages/9f/ea/de50a50fadf979713ab18b46f22ad5ff5f2dcfc637a3ebdecf669801e1a5/greenlet-3.5.4-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:870d730fec833f5a06906a32596cc099b9161594642a92a520b7a88911c95356", size = 614619, upload-time = "2026-07-22T12:26:42.282Z" }, - { url = "https://files.pythonhosted.org/packages/db/c7/2aae27fea41205b8650294c301f042a2a4bb6155eea48c995b890a92f2c1/greenlet-3.5.4-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ec5ff0d1878df6af3bf9b638a5a92a7d5693291de77c91bff10fa48519c604ef", size = 627021, upload-time = "2026-07-22T12:29:03.445Z" }, - { url = "https://files.pythonhosted.org/packages/1b/80/fb4d4788bbc8e54761f1fc88533af9523a6e86299fa113d6e8a8503ed9fc/greenlet-3.5.4-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:07bd44616608d873d06735b63ef1a88191d6ca57c8d291d6559c71bc14c0893c", size = 632845, upload-time = "2026-07-22T12:43:45.19Z" }, - { url = "https://files.pythonhosted.org/packages/eb/56/79fd826f9ccaae0b84e1b4ef68dabba5e105bb044ffcd448a0b782fcba9a/greenlet-3.5.4-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d84d993f6e575c950d91a23c1345d18fe1a4310d447bf630849d7809196b52f0", size = 624002, upload-time = "2026-07-22T11:51:11.391Z" }, - { url = "https://files.pythonhosted.org/packages/42/e3/6086fa578ebb72772722cdc4bcd628459814b42e0c2db1e3cbd6552b3271/greenlet-3.5.4-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:3529a8a933582ad19e224792cac7372489526576b75b4c124e8e4f29948f4861", size = 435053, upload-time = "2026-07-22T12:39:52.715Z" }, - { url = "https://files.pythonhosted.org/packages/0a/1a/27319f97e731298513dcba1a2e91b63e9d8811d9de22130f960b129b1bf1/greenlet-3.5.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:58023945f421093de5e6fa108c0985a8659d43f49e0216da25099369a121bcbd", size = 1581533, upload-time = "2026-07-22T12:25:05.322Z" }, - { url = "https://files.pythonhosted.org/packages/b1/6d/24240bf562e9786dd2799ee0a4a4dadb4ded22510f41b20245099159ac8c/greenlet-3.5.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:bae2728e1897aa8df8cb1af38cd48b3a743aefe29372de7b8b7a9f532501e69f", size = 1645781, upload-time = "2026-07-22T11:51:14.805Z" }, - { url = "https://files.pythonhosted.org/packages/c1/5a/442ab1a9ef7ca6bf7210e5397a95972206a91a31033a03c8900866a10039/greenlet-3.5.4-cp313-cp313-win_amd64.whl", hash = "sha256:ca5726c0b08ca35ae873557266a78b2c3f3b2b7d7401aa5ff886c2045dd0111c", size = 247133, upload-time = "2026-07-22T11:39:20.661Z" }, - { url = "https://files.pythonhosted.org/packages/3e/e6/9160210222386b1a378ff94db846b9508ca24a121cf684991561fdb69280/greenlet-3.5.4-cp313-cp313-win_arm64.whl", hash = "sha256:7c1303791d603080cac6fc3b34df51c3b75b723739c282c8029e48a0d241672f", size = 245500, upload-time = "2026-07-22T11:40:22.185Z" }, - { url = "https://files.pythonhosted.org/packages/a5/a7/6ab1d4f9cd548d15ab90da29947f2076100130bb179b0bde59f795a459e3/greenlet-3.5.4-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:7e8afa5eac028f8140ceafe5ceec66e6aa127ddcb21452d2a564dcd2900b5f22", size = 295410, upload-time = "2026-07-22T11:40:35.747Z" }, - { url = "https://files.pythonhosted.org/packages/cd/7a/422f63b4715cbc0b24385305407adf38b48f6bb68b3e6b04090e994d0f5a/greenlet-3.5.4-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:73b37afe369021423ea53dd3123e04bffa7e93ac64429b9f50835b2e4fcae7cf", size = 661286, upload-time = "2026-07-22T12:26:43.8Z" }, - { url = "https://files.pythonhosted.org/packages/d0/31/5a1cac663bf5582190c5a714ef81364f03cde232227f39748f8ae4c11da5/greenlet-3.5.4-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:3ef964f56dfcb6f9bbef2a190d9126795eac408716aeae47b5e7c73c32aafca9", size = 673517, upload-time = "2026-07-22T12:29:04.815Z" }, - { url = "https://files.pythonhosted.org/packages/9c/bf/250c2921c7b585dde12f5239e313ca2dcbc464d161ecca36e4e6ef21762d/greenlet-3.5.4-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:cef589bc65fae02d10bca2ac341191c5b33acc2967892ebf4fcbd10eabb7a74c", size = 677968, upload-time = "2026-07-22T12:43:46.788Z" }, - { url = "https://files.pythonhosted.org/packages/15/4a/2a82a1e3f8aaca020853ac8d12211280ca2b231aa08ea39f636f1060c319/greenlet-3.5.4-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c53ff01a5c53a40f2c16820ebc56d7c61a77f5fbe009dadd96292d5682f80f8", size = 670917, upload-time = "2026-07-22T11:51:13.589Z" }, - { url = "https://files.pythonhosted.org/packages/18/40/10bfcf6513558d82f7b95dd728001c63bd388259fe27d3e30ae01f103430/greenlet-3.5.4-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:dfc41ae893d9ceaf22c824f2153a88b30651b20e8758c2cd9ac143f23640563c", size = 480643, upload-time = "2026-07-22T12:39:54.149Z" }, - { url = "https://files.pythonhosted.org/packages/68/b0/e379a152b17bfdfa95795af4049e37c0fd1b4d81f020d426db104ed07c77/greenlet-3.5.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ecca4d80d55a01ad6b23b33262662956149fbb7b2c6be2910f1705921958cbf3", size = 1628478, upload-time = "2026-07-22T12:25:06.678Z" }, - { url = "https://files.pythonhosted.org/packages/5e/43/bffdfa64f7317f954c5c1230b5dd5922676ce198689a68c1ac1ed4b1b1a5/greenlet-3.5.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2ffbc533e0eaf8e80d8471411646ab88fe58f641d508c0b02b24494479f4d9ec", size = 1692021, upload-time = "2026-07-22T11:51:17.008Z" }, - { url = "https://files.pythonhosted.org/packages/d0/11/f799f9637e2c6e9b0b716015e339040598b058cf7654dfc0d67468b177ed/greenlet-3.5.4-cp314-cp314-win_amd64.whl", hash = "sha256:305f69e6c4523d7f6979ed001cff4e5853c063e5da04880296603aa0227e544c", size = 248031, upload-time = "2026-07-22T11:40:11.007Z" }, - { url = "https://files.pythonhosted.org/packages/05/75/625bcdd74d5e6b2dca1ecba3c3ac77bcf8a026c21a649a46cef23e421f97/greenlet-3.5.4-cp314-cp314-win_arm64.whl", hash = "sha256:f260930bbbbcf9caee661211235a5111c86dfe5832fdf6ae4570da1e0995320f", size = 246892, upload-time = "2026-07-22T11:40:27.357Z" }, - { url = "https://files.pythonhosted.org/packages/ec/69/35c62ed49c320cb4d98e14698ccca5467d3bfe683984172be9cb564d9ce3/greenlet-3.5.4-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:41ddab54e4b238f4a6c323f39b4e59e176affd5a94d461a9fb7583dac74240a3", size = 305571, upload-time = "2026-07-22T11:40:31.659Z" }, - { url = "https://files.pythonhosted.org/packages/5b/6c/64d60216b3640dcb0b62d913dd9e0d80030c09115bb2e4ba70c95d10ca45/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b3dabe3e2809013052c68bdf0b7fa5f5f2859c43a80803131ad61af9cabd7867", size = 672568, upload-time = "2026-07-22T12:26:45.298Z" }, - { url = "https://files.pythonhosted.org/packages/5c/de/ba3ab0a96292e53039530333b0d2ae18d9e508f3a325cd7bf15f8172944c/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:27d3f00718634d4520a3a150154ac5da36f257869d41321953375b90bfbbc72c", size = 680076, upload-time = "2026-07-22T12:29:06.125Z" }, - { url = "https://files.pythonhosted.org/packages/ae/db/24a10af12bf8e639cec46c38b9ce1a282543ba42ff4fb0b31a970f1ab603/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:39169a11d87a6a263afda3e9a27d1df16d0f919d40a4837cc73986c9884c0dd8", size = 681690, upload-time = "2026-07-22T12:43:48.109Z" }, - { url = "https://files.pythonhosted.org/packages/a5/be/aeada79083c6f1c15f45d77a332f9c441af263ee298e3eb17522cd337d22/greenlet-3.5.4-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cbd60b5763c6543c1827e48faaf14ea9bfbad245f52b1a4d76a2a2d8884c6c66", size = 676733, upload-time = "2026-07-22T11:51:16.027Z" }, - { url = "https://files.pythonhosted.org/packages/f4/60/44a2eca7b9fd71ae0fae7ff184da1cd3169d176652b97aa1cffcbb0ef961/greenlet-3.5.4-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:bd3d1145f603b2db19feb9078c2e6855eb7c67e15580c010ed815cee519b86fd", size = 510263, upload-time = "2026-07-22T12:39:55.678Z" }, - { url = "https://files.pythonhosted.org/packages/e9/10/2392fc3a98948652ef5fd1e7275c04f861dd13f74b78a2b4309f4ee4d090/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:f00f910f0e7b35416c63b23ad78b769aeccfc1775f712b43c4ee525624a2eef7", size = 1637327, upload-time = "2026-07-22T12:25:07.879Z" }, - { url = "https://files.pythonhosted.org/packages/55/c6/e7237a3dfa1f205ed0d9ea1e46d70bd2811b32d516266399fb59d28ab90a/greenlet-3.5.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:91c26423753b92caf41ab3f98fd547d7374d4d9fc2d85be041886c1579d9255e", size = 1697493, upload-time = "2026-07-22T11:51:19.214Z" }, - { url = "https://files.pythonhosted.org/packages/55/e3/4ba8154ba2a3d43729e499f72471b4b5c993f3826d3e24da81d5f06d6572/greenlet-3.5.4-cp314-cp314t-win_amd64.whl", hash = "sha256:ee032b91fd8ec29ec6c4cea2b8c561b178435134bd0752c7334b94e9c736c132", size = 251637, upload-time = "2026-07-22T11:40:37.44Z" }, - { url = "https://files.pythonhosted.org/packages/90/03/e3f96dfc100261a29545ddc8270cafe58f9195b6651466910e820910de77/greenlet-3.5.4-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:178111881dd7a6c946471fda85485ec796e1043c2b939f694b096e2ecf986809", size = 296076, upload-time = "2026-07-22T11:39:38.364Z" }, - { url = "https://files.pythonhosted.org/packages/a4/3d/da52d208e5c977bce8667e784729e584e38b5785f4c1ec0f4c836e9a1c42/greenlet-3.5.4-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d92df08dd65fede97fc37aad36c2e9dcda3b31c467f8e0c2c096456cb818e927", size = 666870, upload-time = "2026-07-22T12:26:46.691Z" }, - { url = "https://files.pythonhosted.org/packages/2d/8a/7e6dee25cb8a8cf9b362c8e597cc269593378bd916f16c736c059a52e85a/greenlet-3.5.4-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:99e8f8c4ebc4fd80aa26c1280ae9ad43a0976e786349703a181cf0bae60413e5", size = 677678, upload-time = "2026-07-22T12:29:07.508Z" }, - { url = "https://files.pythonhosted.org/packages/51/a7/dafc7415d430b0a43a16396eb49ecb3b62fd720877fb259cc4dcfaf5f31e/greenlet-3.5.4-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:1f17e362d78e37559e0506c5a7d066bdd45073c36a0127a543e8a0df27242ff3", size = 681428, upload-time = "2026-07-22T12:43:49.623Z" }, - { url = "https://files.pythonhosted.org/packages/6c/21/5a38699fa45de749e3857d93b8f07e4c20489e77c2d35d915a2e1c456606/greenlet-3.5.4-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:394de08dad5ffcb1f50c2159d93e398d9d2da3ed437645eaa54771fa720db9f0", size = 676067, upload-time = "2026-07-22T11:51:18.163Z" }, - { url = "https://files.pythonhosted.org/packages/2e/d9/6298f3432de301d4718766cf934bd73c418c73f81fbb77247319364b0d96/greenlet-3.5.4-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:cd320d998cbaa032932830448e39abf3c6a12901295e386e8114db926e10cffb", size = 487446, upload-time = "2026-07-22T12:39:57.044Z" }, - { url = "https://files.pythonhosted.org/packages/5e/ba/863116ab8ff1ca7a729e327800268939d182db47aa433db70e216e7d9194/greenlet-3.5.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:c883d61f2282d72c767a14936641b3efcbde9d82f1080712aaea0b1d3126cb88", size = 1633489, upload-time = "2026-07-22T12:25:09.605Z" }, - { url = "https://files.pythonhosted.org/packages/fa/06/7466ced82818d6132462d7f26b3f83c66ea15d2b193a6c0088d558ed7d95/greenlet-3.5.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:2a924f15d17957e252a810acefcb5942f5ca712298e8b6fcaed9a307d357522c", size = 1696584, upload-time = "2026-07-22T11:51:21.304Z" }, - { url = "https://files.pythonhosted.org/packages/f9/4d/55b638489260065de9ffce606c8b5d04507bef705de4b212a0c3d6a1a0df/greenlet-3.5.4-cp315-cp315-win_amd64.whl", hash = "sha256:ed17e5f3420360d5b459de8462efb52060399a5326a613d4cde31cef63ef95da", size = 248297, upload-time = "2026-07-22T11:42:04.055Z" }, - { url = "https://files.pythonhosted.org/packages/bb/08/9dd4ae635da93d41dc268bc34bd62a9d711ed8b8825c5d22ac910c7d6e6d/greenlet-3.5.4-cp315-cp315-win_arm64.whl", hash = "sha256:f908898d6fa484ce4b6f447ce70ea99b52c503fee419e53cf74d60a16bc9e667", size = 247423, upload-time = "2026-07-22T11:44:00.764Z" }, - { url = "https://files.pythonhosted.org/packages/19/66/7c87ed9cdbf1d49c2c6cd1c7b9dd4d16c33b24235ca03972293a1876b30c/greenlet-3.5.4-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:1833637f17d5e7472548a48575c394fe39f1b1890d676d162d86593610f44d8c", size = 306487, upload-time = "2026-07-22T11:41:25.118Z" }, - { url = "https://files.pythonhosted.org/packages/5b/05/0a4201e7c0054866eefc05da234f236dd4c950d0fbf9ca0517141f01b269/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:12cda9122e03341f1cb6b8207a19d7a9d375e52f1b4e9243918375f40fd7b4b9", size = 676479, upload-time = "2026-07-22T12:26:48.129Z" }, - { url = "https://files.pythonhosted.org/packages/3d/c0/4b6b8c5a3aec70f0649cd89662d120fdd6421e2bdc8e3b15c3ab5ec568d8/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:d83ae0e32d14957ab7170785a20f582635c8474deab1bfbb552b17e769a6ce25", size = 684321, upload-time = "2026-07-22T12:29:08.925Z" }, - { url = "https://files.pythonhosted.org/packages/88/15/0b167aeea95285b0e654ddce651922f666c089363c2ec528ca8b9a9ba74f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:123aa379c962ed5fe90a880327e0c3066124ac64ec99e12a238be9fd8eb3db3d", size = 685995, upload-time = "2026-07-22T12:43:50.993Z" }, - { url = "https://files.pythonhosted.org/packages/24/c9/b49c31c9a972eee91e260445770e922244a7efc542697f51a012ec046d0f/greenlet-3.5.4-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9f1467de1bb767f75db0aa34c195e3a496d8d1278c796e70c24ce205d3e99cde", size = 681293, upload-time = "2026-07-22T11:51:20.43Z" }, - { url = "https://files.pythonhosted.org/packages/de/90/c023ec337f32ff505be7db759c80d98f0532bb94d0c6fa13645efe9bee2e/greenlet-3.5.4-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:adf2244d7f69409925a8f22ed22cc5f93cdfe5c9dc87ff3476be2c2aaae61a05", size = 516928, upload-time = "2026-07-22T12:39:58.359Z" }, - { url = "https://files.pythonhosted.org/packages/95/6f/7f2d4653770500eee667866016d42d7a68e3d3462f80df6b8e3fcd48a0eb/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:0fa53040b78b578120eecdc0265e3f1051487cc425d11a2b7c761daadf4feaa8", size = 1642474, upload-time = "2026-07-22T12:25:10.819Z" }, - { url = "https://files.pythonhosted.org/packages/5a/d8/8cba31036a4caae448087ba5d150660ab03b4a0f54d9150f6495a3be7262/greenlet-3.5.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:60e0bc961d367df506660e9ac0177a76bc6d81305300704b0977d1634f76efe2", size = 1701012, upload-time = "2026-07-22T11:51:23.17Z" }, - { url = "https://files.pythonhosted.org/packages/e3/cd/3f77a4cce3bae631b08eb52f53a82a976669600337e21dfdba811cb50267/greenlet-3.5.4-cp315-cp315t-win_amd64.whl", hash = "sha256:f680e549edb3eaf21eea4e7fe101e15ec180c74b7879ab46adc080f22d4015d2", size = 251977, upload-time = "2026-07-22T11:41:38.125Z" }, - { url = "https://files.pythonhosted.org/packages/93/e8/65e8707d00fe2a49bf12f609a9b2b39ba6dd23c2810eacad877c4fc94bfe/greenlet-3.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:08fc36de8442d5c3e95b044550dbea9bf144d31ec0cc58e36fb241cb6ef6a994", size = 250538, upload-time = "2026-07-22T11:40:17.985Z" }, +version = "3.5.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0b/d8/7cc97c142388aef03f622e001c572c4f84e9252a439549d483f555771970/greenlet-3.5.5.tar.gz", hash = "sha256:adb4bae02e91a8e863e48b177e4014bdcac8a6b5e047ea1df687a61534b85e6c", size = 207585, upload-time = "2026-08-10T15:09:36.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fb/3d/8cef5f724ec0d4add2af8961d504535ec60c3cca9e464f6d03bdba29d85b/greenlet-3.5.5-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:b79fd2a5bc099b5e744f34c4c9a58954a5f4cb7529fb4b6e8446057d61b6edaa", size = 294730, upload-time = "2026-08-10T13:27:51.206Z" }, + { url = "https://files.pythonhosted.org/packages/88/4b/8e7aa3f514273aecff30a16ab1bac09ff54cfc7e6860fdd8058c37ff2499/greenlet-3.5.5-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:634cf15a233a949136879dd388e25d3296e16f3f1e217d2456797b8579ebc6ed", size = 614536, upload-time = "2026-08-10T14:14:36.589Z" }, + { url = "https://files.pythonhosted.org/packages/85/48/4e95e9dd5a8a397dc6a6345dd7f1935113d0fca4f85e89d3976da9cd988d/greenlet-3.5.5-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:499adea519f748407fc6806d20eedabac2884fd73b9f38d81236e190ba20dfef", size = 626924, upload-time = "2026-08-10T14:27:27.048Z" }, + { url = "https://files.pythonhosted.org/packages/0e/84/eaa476d6bf3816828d0d70e80dcc36bf30a058233bd889e707e693f6e860/greenlet-3.5.5-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f7278591501941bb2456af102bb9cd59aab48c6cfd6e2dd68fa1290bb0c49a42", size = 632726, upload-time = "2026-08-10T14:30:09.874Z" }, + { url = "https://files.pythonhosted.org/packages/89/5d/398a1c71fa7a277deeb376c999979de6786f08fc2d5747a0b9d6e11738dd/greenlet-3.5.5-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2eabb980975cba5b93a95f6f69287d05fc05ac955bfd6a320a7c083eeb52c0b0", size = 623906, upload-time = "2026-08-10T13:40:50.501Z" }, + { url = "https://files.pythonhosted.org/packages/d0/f2/0cc2849ede68579291e9c59b3ab6ec1958f98681cca5b14d8fc75bf674a4/greenlet-3.5.5-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:4dfc7c4470354e7b09184d1a3a985761053a2fd694ddb5b5c80242afc2c8c90b", size = 434966, upload-time = "2026-08-10T14:30:03.729Z" }, + { url = "https://files.pythonhosted.org/packages/04/1b/745450fc5ea9e0cb17d840d248f284db3363de736d362c7d2d883e3eadba/greenlet-3.5.5-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:03115c2e0a371999bf8ae616aa8d653f96641d4705c457aebaa187276e9f7537", size = 1581430, upload-time = "2026-08-10T14:15:06.853Z" }, + { url = "https://files.pythonhosted.org/packages/d4/29/d51b296e3191bb15d3d81ec375af1909e4466c0f395d744ed475801798a9/greenlet-3.5.5-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4441153ffba21b90d3ca89fe3d31f5c093ae6c0bf0cfdfc98f54cde22f95b62e", size = 1645684, upload-time = "2026-08-10T13:40:32.133Z" }, + { url = "https://files.pythonhosted.org/packages/12/63/369f1a1625e64e9e31df3963c6044056e3fdfa3fa3fdba3c54ffefa6e987/greenlet-3.5.5-cp313-cp313-win_amd64.whl", hash = "sha256:95c5b1f4b3a193f8a0c2de4bfdcb48d119f7f1063941f1de1f2168051b3e52dd", size = 324075, upload-time = "2026-08-10T13:26:58.974Z" }, + { url = "https://files.pythonhosted.org/packages/45/78/649cb5c09d4d81f6dd1444e75474a7206784743283a21d24171562ac4899/greenlet-3.5.5-cp313-cp313-win_arm64.whl", hash = "sha256:1af90aa4bc129883b340cdd6957a3bc74f60528a4993bbd1f53aaebe1d9981cc", size = 308260, upload-time = "2026-08-10T13:27:50.795Z" }, + { url = "https://files.pythonhosted.org/packages/7f/8c/080e881fa2be95ff1ddbd6994b2bab3b1a78df3b3fcab39306011764fcc7/greenlet-3.5.5-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:d4a389a852e392a6366058651a20fa5ba40d979865aa81bea2ccbdc44805070d", size = 295309, upload-time = "2026-08-10T13:26:03.032Z" }, + { url = "https://files.pythonhosted.org/packages/25/cc/0ac614e6586c0e42d4cc281a5819150f4f43685744a4c5ff77139286409d/greenlet-3.5.5-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70b157cd319873e8b544ddc2de158f55bbd0a9b0218c8ce9332039801518e328", size = 661185, upload-time = "2026-08-10T14:14:37.867Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b9/6808725354be8ad305dfe5172377664fc9642d4fc043be246b3314cf4482/greenlet-3.5.5-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8bdfd1424abcf26832961e766570cae79efdb9599d709088c9cb6ef82b194926", size = 673419, upload-time = "2026-08-10T14:27:28.652Z" }, + { url = "https://files.pythonhosted.org/packages/eb/52/f005d579acde46c3d1cc3cab1c9f3d5708c8a3006a4120e8cf5da801afe9/greenlet-3.5.5-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d98ef6f92e67c6dbf299dbfd8facc1b0d2d9cedf91e325e73b3d0373fe4309d8", size = 677863, upload-time = "2026-08-10T14:30:11.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/2e/40c509967da7f254680826a2fa0dd22138ec79946c70b97542d74cde8b43/greenlet-3.5.5-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:182de51c6b572a705f2fafaab2e783bcf7d2760940229dfe73086cbae037af3e", size = 670822, upload-time = "2026-08-10T13:40:51.833Z" }, + { url = "https://files.pythonhosted.org/packages/c4/8a/a75f8a2bdcef3c358a3147cdc9db3aa83755f0a038f766ab0bedb66f512c/greenlet-3.5.5-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:159df1942d88e8f784cbb38d6f18bdb365cd11319cfbb3e89623de2b97892d53", size = 480554, upload-time = "2026-08-10T14:30:05.171Z" }, + { url = "https://files.pythonhosted.org/packages/2d/22/c3c2eee4a8fe191d6d1d183086c56133d646024e3d70bfd414829f64560b/greenlet-3.5.5-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8fec3f165dfe332e490c3247c0f6c23b0bfc45f06496ad7f00ddb00e3d35e4dc", size = 1628469, upload-time = "2026-08-10T14:15:08.11Z" }, + { url = "https://files.pythonhosted.org/packages/f7/87/25babd09b94cb1f03e71db815fde463f0262e40cfbd953d58a8d77311351/greenlet-3.5.5-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c6ce25fee6cabc8bf22cb8b52e642cbb821be5b9aec8094d07ff03378141b8e9", size = 1691952, upload-time = "2026-08-10T13:40:33.502Z" }, + { url = "https://files.pythonhosted.org/packages/2e/3d/5cc9701117ea4dc0eb7bf1f4f9b7888a6e2e5277ddfae095805ace50f2b6/greenlet-3.5.5-cp314-cp314-win_amd64.whl", hash = "sha256:7dffc5c859fe6059974df1e37d7923d654a83e2ae18fdd616994270e001115e1", size = 327458, upload-time = "2026-08-10T13:27:02.868Z" }, + { url = "https://files.pythonhosted.org/packages/a7/6b/594fa2de7fae7629168a404a4305d7d7e31a5742c50a801b1839543cb93d/greenlet-3.5.5-cp314-cp314-win_arm64.whl", hash = "sha256:5e2afcfc4d4305dd715809b03da5cbe437c8984f61d8917751eb5fe4aefa3e07", size = 311146, upload-time = "2026-08-10T13:27:25.046Z" }, + { url = "https://files.pythonhosted.org/packages/24/e0/50cd600b469e5734c72709b6b1838b6bc63f307b573c772c3132d6ecfe92/greenlet-3.5.5-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:0e5a7de979d764aea1f5b6e95cf92b5b37741b9823702041f34b126e7f690277", size = 305471, upload-time = "2026-08-10T13:26:20.568Z" }, + { url = "https://files.pythonhosted.org/packages/75/a3/77acd66dfc6387b5219b2080806c0cabb73c10eb1bb44b413c40a62015ba/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fef01bd457f11fc158b130ca0027a3c365693280e8e231b65bdaf57999f39f5b", size = 672470, upload-time = "2026-08-10T14:14:39.058Z" }, + { url = "https://files.pythonhosted.org/packages/b9/71/0d178142dca3ec19f46fb2212ae73d30ad53b9d548dc64804086033a7089/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5173a72310725a74afc82c164f0e52cb8ad0de62f2bb623f24f6c0cc07d80272", size = 679973, upload-time = "2026-08-10T14:27:30.072Z" }, + { url = "https://files.pythonhosted.org/packages/aa/ac/0d7887aa4bbfc9eba075cc428244dfc96f623478454d5ec81180d0d6bd5a/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5e9ec2e7c98e895fcea0c5cc57b2606cf86ece6d0a56578f3eb225e2af4f0387", size = 681587, upload-time = "2026-08-10T14:30:13.519Z" }, + { url = "https://files.pythonhosted.org/packages/6e/31/46eb8567302eaf787abf88d09df014e14ae3baf460af1b8b0efdbd3efcd5/greenlet-3.5.5-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:44f08341873200ba8a60a8bc14ace3d91f1754f7fa7bc66157714a8cd420a476", size = 676634, upload-time = "2026-08-10T13:40:53.004Z" }, + { url = "https://files.pythonhosted.org/packages/4f/18/8d58ba1c429b0383e3219a3d0e0bba241d0444d8ed05b73349953c7d7c7b/greenlet-3.5.5-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:102817506f6090b5176c746a82603341a549b40e5c3d5b72a4c672228a918c41", size = 510175, upload-time = "2026-08-10T14:30:07.047Z" }, + { url = "https://files.pythonhosted.org/packages/a3/e9/b88bbf5b29970cb84172dc2c32aa3e5e579ceb94c808e81c826454138850/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d246c0db9a2513cd45f019ba178ea4d4d4705bd210ee465e2c15d76a1ab13874", size = 1637320, upload-time = "2026-08-10T14:15:09.317Z" }, + { url = "https://files.pythonhosted.org/packages/6d/8c/7631ed29cc6f0392f11830076e172ce4885e70b0bc2c1bce1731176d4b4e/greenlet-3.5.5-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:72507285b5caa1d17904a3f7c322ca780823a54170a0e04ec3f37bcc60d4db71", size = 1697412, upload-time = "2026-08-10T13:40:34.924Z" }, + { url = "https://files.pythonhosted.org/packages/da/0f/f7dd935f9c4cb1be49098770587f54d8a78518e55c89bce86c4fb4109057/greenlet-3.5.5-cp314-cp314t-win_amd64.whl", hash = "sha256:7805655781fb8f28a55d05fe57ed61f5f10f1892fb587673e3bb5264f28041f0", size = 331514, upload-time = "2026-08-10T13:29:20.611Z" }, + { url = "https://files.pythonhosted.org/packages/b7/e5/681b01f8fbc1b55232822f99e8f8afeb78a55a7c76a7bf9dbdc7ccb03a6d/greenlet-3.5.5-cp315-cp315-macosx_11_0_universal2.whl", hash = "sha256:c0db80fcd5b8aece93f66c64f78a786bbb6b96c5fe63ef5a5a4581ecf8bab206", size = 295975, upload-time = "2026-08-10T13:28:45.985Z" }, + { url = "https://files.pythonhosted.org/packages/11/f2/69b488cd9e7267bf4b0fe8cdebf25d8d6df680d21bdf41150d23e23d6652/greenlet-3.5.5-cp315-cp315-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b241c32f912ada659808d68e308c568baf577eebf757d15471472de0c18cfad", size = 666823, upload-time = "2026-08-10T14:14:40.222Z" }, + { url = "https://files.pythonhosted.org/packages/84/d4/d5bc2fdebbdda0c94555925ba79948b8395d75a7f6a36cc85dce5bab9f11/greenlet-3.5.5-cp315-cp315-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ef6a08349401d8eaf3cb12688ac8557de95788556b8631ef17555a4a173022c0", size = 677613, upload-time = "2026-08-10T14:27:31.543Z" }, + { url = "https://files.pythonhosted.org/packages/65/53/4e13642efc4d7ad6554ecb2242a5be42666b2e1a067323e88dfc0124a04b/greenlet-3.5.5-cp315-cp315-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:37faa97daccb6d9f4c2141ce3118d023c3c5506864a7d8bdf726f665018c1f76", size = 681436, upload-time = "2026-08-10T14:30:14.839Z" }, + { url = "https://files.pythonhosted.org/packages/bd/93/542d8a3a90f3b35c6ad8bf7e56a03010287f2cafa289a5b7985b5207db39/greenlet-3.5.5-cp315-cp315-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f2e3d061b8e13aec2f0441689b3c71b244a20e5d274a52cb0f7e31bd1d139552", size = 675930, upload-time = "2026-08-10T13:40:54.205Z" }, + { url = "https://files.pythonhosted.org/packages/cd/32/188447c9a468d6977d2989397226b0c6b65ab6f4cf943f931643328512fc/greenlet-3.5.5-cp315-cp315-manylinux_2_39_riscv64.whl", hash = "sha256:b18007dc2473a7942fd157366b55f01da6fed7ce85318591005b419e0a439474", size = 487404, upload-time = "2026-08-10T14:30:08.903Z" }, + { url = "https://files.pythonhosted.org/packages/52/b5/89c9f2e8460d71101037d47a1feed11928615a5edd42370be290e0657eeb/greenlet-3.5.5-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:9ab5f5b93655e77fe0d6c2dfd22b5eac751bb1f876d8ec21761b7c1fb9266007", size = 1633878, upload-time = "2026-08-10T14:15:10.693Z" }, + { url = "https://files.pythonhosted.org/packages/b8/60/297de93f3b02ac78a5e04d32bb8bbe3080f4a73d8ed95016561463b70618/greenlet-3.5.5-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:f0e5a21bd4452a88cf032fc43c4a5b307ab1380eacb63b5988f9c0317885e773", size = 1696597, upload-time = "2026-08-10T13:40:36.252Z" }, + { url = "https://files.pythonhosted.org/packages/18/25/54c6eaff4f337fb670215e89eb2d00d9499487b658e709d4b477be4a342e/greenlet-3.5.5-cp315-cp315-win_amd64.whl", hash = "sha256:469dbb0a78625642f4a626cfd0c6e8bccc0385b5e49189b6308bbe849ec88a8e", size = 327700, upload-time = "2026-08-10T13:28:06.752Z" }, + { url = "https://files.pythonhosted.org/packages/67/67/857e88a36301caa0e029870132c2478bd55d896630321432afab03a3115f/greenlet-3.5.5-cp315-cp315-win_arm64.whl", hash = "sha256:2d57406c3efd32d7a81e17a674314e8bd00792cdab49ea3228a49aa1bfb2e769", size = 311750, upload-time = "2026-08-10T13:34:08.815Z" }, + { url = "https://files.pythonhosted.org/packages/10/e2/3144c0a116067ac1e30457b0139a94d60d1d36a86e015de68e9ac87cb3bc/greenlet-3.5.5-cp315-cp315t-macosx_11_0_universal2.whl", hash = "sha256:68184dfcf50ccaa8e864770fe0633a7e27250ea9329f8192ef47ee9ecfd78e1c", size = 306387, upload-time = "2026-08-10T13:27:00.897Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a1/cb4223a7e9b9f43b8807e8eb212358bfe2dfaa174a9ea2889eb1714dcba2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9ec0dc0e59dc9c61af5c47348365ccbbd7addfafe0a93b00336ff3da2907bdc6", size = 676472, upload-time = "2026-08-10T14:14:41.417Z" }, + { url = "https://files.pythonhosted.org/packages/9e/cd/a154b4498e5d8f12ada291cfb3b8d596eadde2177f5bf09a9be699d2a446/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e604f58e35833fc46ef20302bcb314dddbfd3fcf33a4f936216d51dd678d63ae", size = 684238, upload-time = "2026-08-10T14:27:32.946Z" }, + { url = "https://files.pythonhosted.org/packages/ce/f4/e450a68a152f819491d8c7df6a8254e761d87e6a78759268961f8c5bd4dd/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2888a3a38bc5ee5bb6c438372197152e815837e4fab7ed7a1f86ef18ffd58ad1", size = 686022, upload-time = "2026-08-10T14:30:15.96Z" }, + { url = "https://files.pythonhosted.org/packages/bf/bb/b0031d260c2968a3c87deebc51d80c64e499377f993aafe06ee3b7488cc2/greenlet-3.5.5-cp315-cp315t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:40239b5384f96da3963585cc6d7eaa9b56f8ae67e8d92cc82dd9e202fc847de3", size = 681246, upload-time = "2026-08-10T13:40:55.402Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/17e63d6bf3b9c9b9dbea981b7f643a71f79603bdfb4f1c3a9cf353e22aed/greenlet-3.5.5-cp315-cp315t-manylinux_2_39_riscv64.whl", hash = "sha256:1e8d9391fe77f15649589a907cef972dbbd6352ef7ff7dc0492f658c0c26495f", size = 516951, upload-time = "2026-08-10T14:30:10.907Z" }, + { url = "https://files.pythonhosted.org/packages/9a/07/da554b71ab88e649da146e1065d86a48a5c5d92e50ab74ef41b504aa7f56/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:a1eaccf5c3a1d3e46dead602c72e6836731e8e245c9de6a27764567b6b62d4c0", size = 1642735, upload-time = "2026-08-10T14:15:11.92Z" }, + { url = "https://files.pythonhosted.org/packages/78/76/26a3782a051677668af9d92beaa47cd87ba9dd5072f762961144a03dd4c6/greenlet-3.5.5-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:19e4e026fe20691f333b8eb1a3bc9625eceba8c3f9d62ec5a6f8581afbc6b5a5", size = 1700925, upload-time = "2026-08-10T13:40:37.656Z" }, + { url = "https://files.pythonhosted.org/packages/28/d9/fe7baf4190c2ae71f267efb9de21b3172bb35bc0ed1ef53dd6027d658e33/greenlet-3.5.5-cp315-cp315t-win_amd64.whl", hash = "sha256:712aee154f648bde84634654bb38bb78c69ac640c37a45c9effed800735049d8", size = 331829, upload-time = "2026-08-10T13:26:48.851Z" }, + { url = "https://files.pythonhosted.org/packages/df/af/419a4e383bd600858a9b67e9b280a60fdc383ee3f2fe5b6c0c1ef04e74d1/greenlet-3.5.5-cp315-cp315t-win_arm64.whl", hash = "sha256:7f049911ee81a16a03c33d5450d8d5867d27f596ca5fb201b86f4524e874468b", size = 315093, upload-time = "2026-08-10T13:29:34.949Z" }, ] [[package]] @@ -1167,48 +1167,67 @@ wheels = [ [[package]] name = "hypothesis" -version = "6.165.3" +version = "6.165.8" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/18/7a/7a277ac07776191be594f74f6425649d529e4876f7d3ff1ee96d393ffdbc/hypothesis-6.165.3.tar.gz", hash = "sha256:687c5abb1a9c11478577c2cf18685c0eb82150d278477d3e14da290a1ef2a098", size = 502263, upload-time = "2026-08-11T01:23:09.1Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/c7/18152acad5f85f91554b2030000319b952a54151509953651ec40f37d50d/hypothesis-6.165.3-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:56af539c811b11ab5475704c300b8f0b46cc6dd0edc267e02a16487e803c77f8", size = 781671, upload-time = "2026-08-11T01:22:09.176Z" }, - { url = "https://files.pythonhosted.org/packages/d3/77/4293ea8a7fdb713956a8bf460b9070115df69f8216a900507633f9cdb225/hypothesis-6.165.3-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:f40c10cfdb1ea2cd75e5d4e6e0cfdcb6198ab8406e8922666480e6dc11eea341", size = 777291, upload-time = "2026-08-11T01:22:15.991Z" }, - { url = "https://files.pythonhosted.org/packages/02/fa/fa2071a6afaefc082dc7a033f41ae61436caf442d5973ba8ca9c29a69460/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a0854b1de4577f7e1beb1d681360285b5d678b65a809787ff4eab5b8b25efca", size = 1106490, upload-time = "2026-08-11T01:22:07.858Z" }, - { url = "https://files.pythonhosted.org/packages/ba/86/de724b7f9cd10e3be4efa21770457172e549d7576b1d8e29d6177eef5e47/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:360991cda8e488924905af48949033b90d4877ac97b9ad5d826d4d0f5a4b8cfb", size = 1135054, upload-time = "2026-08-11T01:22:29.499Z" }, - { url = "https://files.pythonhosted.org/packages/12/6a/96721cf447bd3c64b5e6843dde4444b20f3ddd901ad366cc73d0e7314bf5/hypothesis-6.165.3-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bf502000f4a8ef4c9ab9493ca3b4fe17ae3033c18a8e2a31cdd69515dc7d97be", size = 1155997, upload-time = "2026-08-11T01:21:48.496Z" }, - { url = "https://files.pythonhosted.org/packages/0f/ae/a793cce6497f233b155f97684bf7d0e424c25613dd87b8af8a4e87820232/hypothesis-6.165.3-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:b9fcf47ad18f87f7c15bd36289bd45708bbfd250129d73bf554653e2f9afc931", size = 1111326, upload-time = "2026-08-11T01:22:21.9Z" }, - { url = "https://files.pythonhosted.org/packages/28/8d/dc3cdfd55843d038effa2458a9c9bd73002218a8c0fd58c2c0ab7fa328db/hypothesis-6.165.3-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2fb05529cbcab5a317d03d7bb0e90d382f79ef1643e3568577916d0e24bfe70b", size = 1148079, upload-time = "2026-08-11T01:21:51.071Z" }, - { url = "https://files.pythonhosted.org/packages/db/4b/2f62924ac41f3d3482b29ded4c213f27ff4a103e56e84eeb528d4900cac7/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:57eae10a64340cd621a78eae9cb0459bd68ea99fbaa933c4f00e34d5087b6376", size = 1281862, upload-time = "2026-08-11T01:21:39.274Z" }, - { url = "https://files.pythonhosted.org/packages/7e/d4/01c78b7b7348b6e8cef9b999109dfb93b14c7e1e38bc22170129f8b17181/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:19df0f2239052e9a870634a1d9bcdff95e2a2ab508573e5dd5c3d1ca545f5b3c", size = 1408437, upload-time = "2026-08-11T01:22:13.243Z" }, - { url = "https://files.pythonhosted.org/packages/35/76/e940b5a5aaf75bcd4784f1f3f9bf2b9a642a706bc0a9639077ca84f1325f/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:9781a8026adff4b4516404cf0e5f2cadcb471318c2882a264e1c57c4c092266f", size = 1281168, upload-time = "2026-08-11T01:21:58.964Z" }, - { url = "https://files.pythonhosted.org/packages/fc/84/b153e81a614f45e0902e3b9e8a8b079e64214c50abb6fbe9acc62ccf686d/hypothesis-6.165.3-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1dd7e05f88e3e108a5e4f5f71a3eaf205559e8951e3c1f1ffd04cea82ed3b731", size = 1323263, upload-time = "2026-08-11T01:21:52.374Z" }, - { url = "https://files.pythonhosted.org/packages/fb/5f/e5144d9e91ab7260650cb1ee032ca23208d49ebb1334845baf6407c1a9d9/hypothesis-6.165.3-cp310-abi3-win32.whl", hash = "sha256:d1389bda38cb222acc109aef5b31643ce799a39a76294a50ad8b84e32f92d76d", size = 667499, upload-time = "2026-08-11T01:21:47.36Z" }, - { url = "https://files.pythonhosted.org/packages/a9/18/f008b6f1f1c293d51c2776f8815d95bccb777dcf87df2a0ab56b273b47dc/hypothesis-6.165.3-cp310-abi3-win_amd64.whl", hash = "sha256:10cda6988ca4b1da389548b6fdd71af236b588a601fc1757e56eb8988e4240d8", size = 673643, upload-time = "2026-08-11T01:22:46.975Z" }, - { url = "https://files.pythonhosted.org/packages/03/3e/95cba31dbe775b99a4548cdae192e1ad15cee7b64fbdfe6cd4c9d00031b4/hypothesis-6.165.3-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:447f139d6dd70a5d8b178ef507463fb0430ace9ce42e3b2351d2803a391fe774", size = 783183, upload-time = "2026-08-11T01:22:45.286Z" }, - { url = "https://files.pythonhosted.org/packages/56/0e/51bf125cdf7855b69097b8f59c73ef3cf5f4e3d68a16e808d2d1f08a1ff1/hypothesis-6.165.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6152c718606f1705e673c6b30a6ebd3ff08d340da85291dd3c432c73b28a9b3a", size = 774820, upload-time = "2026-08-11T01:22:24.825Z" }, - { url = "https://files.pythonhosted.org/packages/0a/69/b954f742b97441a5c49f8f8704826ee0637a6cce3a7d06ce85fbefc54ac5/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:27fe7826ad83ccc2e8062f0fab43b137bab34cef1a149a926b34a7b8382ee22c", size = 1105186, upload-time = "2026-08-11T01:21:41.733Z" }, - { url = "https://files.pythonhosted.org/packages/6e/8a/33e41d9cc1be7661e0b4129c225a93c3f12544714300aadb95ae7eedf894/hypothesis-6.165.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1ff92876a324f7b9cdb92cedf103e380b7a12aa7df55ebcb16dd0f495a879e8", size = 1155215, upload-time = "2026-08-11T01:22:06.604Z" }, - { url = "https://files.pythonhosted.org/packages/ee/53/ba09526c9100ace5752908ac7251d2dc3960ce7e0e97a31152aaa26c33ee/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:53f1564c97d27fc109f212404d49cd71d7789777dbe0685628ffe9838df56240", size = 1279245, upload-time = "2026-08-11T01:21:56.182Z" }, - { url = "https://files.pythonhosted.org/packages/c1/93/fc637d355791a65364a3409ff06ade7ba5d3fc6f1d07a729781dec315fa0/hypothesis-6.165.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:788a9b0a7aae719a2b71a1c2f07e51deb1d0fe990164a9090c686833ed4bfbad", size = 1322370, upload-time = "2026-08-11T01:22:48.492Z" }, - { url = "https://files.pythonhosted.org/packages/b6/8d/826053ba0263143fed2b0e8af009dc868d8a932e7246e191deb8ca7ce8ff/hypothesis-6.165.3-cp313-cp313-win_amd64.whl", hash = "sha256:37830f0795abfdf738d2a5b6f829a73f3ab498de45a2e61b0bf3bd38d8c9ddb9", size = 670804, upload-time = "2026-08-11T01:22:00.103Z" }, - { url = "https://files.pythonhosted.org/packages/e7/27/3230f8de3d853b2b547731916ae1d1026bd197cd3f2d35dafc0b445da46b/hypothesis-6.165.3-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:38826441dbf528cc156388d0a05526086a12da3e1348353d3fa14de03e57c4b2", size = 783286, upload-time = "2026-08-11T01:22:40.816Z" }, - { url = "https://files.pythonhosted.org/packages/6c/28/9f9ca830d376c50babe55c616f6d99eea886c6ebcd8b512dcd5d56f9e40c/hypothesis-6.165.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:87490115edd34a246a4ba8b1144cbdf571438c46c406ece05caf65908667c9a9", size = 774963, upload-time = "2026-08-11T01:23:05.409Z" }, - { url = "https://files.pythonhosted.org/packages/33/3c/3c81f08ec1edce160da509c5785d78c0e25a7913899c4b9ff724bfd01420/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f51f4346cfa26bca68c68f7bbbd2b1812208bc9f572187c95ecab080ed402153", size = 1105730, upload-time = "2026-08-11T01:22:42.311Z" }, - { url = "https://files.pythonhosted.org/packages/bf/b5/f6f81b9aec9999ec63920d168617cab67a038be05487eff3410ccd072bfe/hypothesis-6.165.3-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4da89eb4b36b3260ff714d2ecc3274b9bd599fd96687d2d9ed53d5e1a801a7a7", size = 1155383, upload-time = "2026-08-11T01:21:57.58Z" }, - { url = "https://files.pythonhosted.org/packages/dd/27/7f3a8c6101675bf95c80cd8c9173d65892ca0b7b640551156dc4537fab1f/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:eb6d31c14d7bdfe03e501d88ee296c149a74cc93e3d01c76ea335e64ee5f33ec", size = 1279606, upload-time = "2026-08-11T01:22:37.56Z" }, - { url = "https://files.pythonhosted.org/packages/d9/16/0c23e06a24e421e532f62a95021fae34f685f3a194c081c6991b4ab202b3/hypothesis-6.165.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:0863e1a9258bc103abe616fa9471cfa66a1535ea404dd8a0bf360e0a29502397", size = 1322697, upload-time = "2026-08-11T01:22:27.857Z" }, - { url = "https://files.pythonhosted.org/packages/dc/56/8356dadf45e5c635b46aa2b57fa74f3210250a8e38b860b6b75f50ed0b42/hypothesis-6.165.3-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:53c56155f2cfbb45ec97fef9ea3b8453b4a34c48c3c5cacee16f97dd2a037994", size = 614859, upload-time = "2026-08-11T01:22:01.304Z" }, - { url = "https://files.pythonhosted.org/packages/e3/79/124d4faf235219acd685c359760a5cb3995609bc50ce465e54c3249841ee/hypothesis-6.165.3-cp314-cp314-win_amd64.whl", hash = "sha256:c48f41e950b5e602e2fdf8f92dcc8ac7bf715a003bf822afb7c9d5cbc41bc344", size = 670600, upload-time = "2026-08-11T01:22:10.356Z" }, - { url = "https://files.pythonhosted.org/packages/01/7a/41ac5e68d9ce079d1b76d4c54126354df61b948c3d519d1289aca877eedc/hypothesis-6.165.3-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:9563d3040178fb1f522665bcec6458cc0d21ab77d7c637058a8be4ea8c01d236", size = 781746, upload-time = "2026-08-11T01:22:34.472Z" }, - { url = "https://files.pythonhosted.org/packages/5d/fb/7ecc21aae63a83dbc8036f9a0544c6b3d798db566b97b5202bdf8e770f80/hypothesis-6.165.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1fe1783543b43ba9808c016950e5e84b3804dc3365ba77c37c427b5896a558a1", size = 773382, upload-time = "2026-08-11T01:22:55.279Z" }, - { url = "https://files.pythonhosted.org/packages/cb/f2/9cc2a4768f9a483b12e307ba585f5eb9c7f5500bd16ff82ddbf62a9a1b88/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9ea34806a4df4e8305a096dcf8e53cdd903c96c1e0d2dd5b001d2283f639c3f1", size = 1103911, upload-time = "2026-08-11T01:23:00.286Z" }, - { url = "https://files.pythonhosted.org/packages/54/9e/b551a494f84976ee5bb9374c197ccc126dea2ec6f22098d5f70705237473/hypothesis-6.165.3-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d188454b95ce46ba991e3c52161255d76af25170ad28591f6b30b045e501216e", size = 1154060, upload-time = "2026-08-11T01:22:20.413Z" }, - { url = "https://files.pythonhosted.org/packages/69/37/8e22a236f1f1e599525549a34672fb0523109f571486fe209b12a84a942e/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:a1c47b15ce97a9b1346bc7d7013c5f215380f78ae01c0f73a1638bd8b98bdd76", size = 1277631, upload-time = "2026-08-11T01:22:30.965Z" }, - { url = "https://files.pythonhosted.org/packages/13/0f/feb33bfc23853b4ba6360ff5e34235cd8bea0d7dd1eb21e17491f581c4e2/hypothesis-6.165.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:996077ef7a3bb332b6638f698ddf7555c82784b58dde80eeba3f07c0a322b40f", size = 1321326, upload-time = "2026-08-11T01:21:45.089Z" }, - { url = "https://files.pythonhosted.org/packages/9f/3b/ad56b56540a0719f493edec0dd442ebb21272147d2482ef505d19760a6d3/hypothesis-6.165.3-cp314-cp314t-win_amd64.whl", hash = "sha256:57a8273bdafe3f450afe66999fd130d4935d775eaf4ef63fcac0bee8015fc512", size = 670613, upload-time = "2026-08-11T01:22:05.294Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/28/1a/8afd0551a13e43513b107298074e8c16dccf1bc03fb2bcf4220e5b01316a/hypothesis-6.165.8.tar.gz", hash = "sha256:8d19b159ca5ff72db9f0c13183ebf3a5a2f07e2310130bcb6ce7eb24ea9ea9d2", size = 503524, upload-time = "2026-08-14T19:08:14.403Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/55/b717931951a64b0d5de2cb822fe07ae0800031ff53637b8ac65d94d6fbcb/hypothesis-6.165.8-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2b2d1cc1c482e53a52fee8bfd7ca605665dcb24e31d0467dfc513714f25c496f", size = 783023, upload-time = "2026-08-14T19:05:48.344Z" }, + { url = "https://files.pythonhosted.org/packages/9a/62/86ad7e0fdeaadeb73da0816cfd820aaa1211302a663845b43d2e2f707bda/hypothesis-6.165.8-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d95d4007ed1b1922ad935477fdff0b5c32f7f124b9011b79046bbc95cd3d8a49", size = 778589, upload-time = "2026-08-14T19:06:24.736Z" }, + { url = "https://files.pythonhosted.org/packages/e8/5b/519ea72ed5c43356699d32db05aa7cf19675105860c6dae7b68c493470c2/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b67b996ae2ae76cda1bea2a7537f1c5d0a9ea346b1043ef12b248fcf7d01205", size = 1107806, upload-time = "2026-08-14T19:08:01.253Z" }, + { url = "https://files.pythonhosted.org/packages/b3/92/d6bb217ac935b2d622400982d5c2ea8aea75cbb58ccb2152cfe5a75907f0/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:35a25f0bb13b96096f803c2237d44a9b44aeaf79a996575cbffc6429b7dd363e", size = 1136423, upload-time = "2026-08-14T19:06:37.918Z" }, + { url = "https://files.pythonhosted.org/packages/24/a2/2f8063c36c25aea4d1f9f585fe803d645d16105e69e5872658f225c932ec/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78f243af26f9166eceb672061fecf7221edebf9aefe81aeb5c7003e29a3c7df", size = 1135048, upload-time = "2026-08-14T19:07:15.195Z" }, + { url = "https://files.pythonhosted.org/packages/ed/64/c8d99086b20c02f1e013f1783deb96febe34ed151169d0a373a82ba485bc/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7567a399ece2643fcc86bf9156186a5958754a9238ab31701ee2cb91cc25dcf", size = 1157305, upload-time = "2026-08-14T19:08:05.672Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2a/995099db937fa105355adbc93af451ff18e0c619ab6e8e07d0a3e3297274/hypothesis-6.165.8-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:9c30099a1fc223108a8c1e5d8ad8c71532904022a887e8d0f318f825ff49a60a", size = 1112633, upload-time = "2026-08-14T19:06:16.192Z" }, + { url = "https://files.pythonhosted.org/packages/1e/12/56ff501135a2e227a6fefbe04856b6b4c374be57148fc8e83b7896cbeee3/hypothesis-6.165.8-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8501178c455bfd9b23c75c5429549a45e7b40fea42d14d6bf78694912f13b92", size = 1149398, upload-time = "2026-08-14T19:06:12.879Z" }, + { url = "https://files.pythonhosted.org/packages/de/99/2f35dd48d61d914a443e8ee3abf278cd08cbe858b99faaec03db04bc708f/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cfb2f628bcaf740a51f6874e9dc1e354a94509bc374266daaee7ae64de5ea2ee", size = 1283253, upload-time = "2026-08-14T19:07:11.261Z" }, + { url = "https://files.pythonhosted.org/packages/61/34/ec9de1e751ffc429234eedb185b13095b5556f2b30dfceb0131674ede01e/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:496d2116b5472bb4691931087a3bcd813c3fa4b9a679490dca8543fd5950e7bf", size = 1409756, upload-time = "2026-08-14T19:07:27.903Z" }, + { url = "https://files.pythonhosted.org/packages/d4/33/f53329f3aaaa4a61e3aa92dfae4e815b13bb74dafef526d0f45b731cabec/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:45bf3bcaa7f01688f4dc77b377745566b195afc2e41074a182ddf7d862690611", size = 1264781, upload-time = "2026-08-14T19:05:57.085Z" }, + { url = "https://files.pythonhosted.org/packages/29/56/7a2fe9de26b136161d19487529ebe75864693bc347f7d5baed8510cebe2d/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7a892413976cecad6d53abc6adf266c6446b64958dc2b5cf405ba8fe0c72a48d", size = 1282528, upload-time = "2026-08-14T19:07:54.893Z" }, + { url = "https://files.pythonhosted.org/packages/1e/73/d4363e6f9740a6c5508a583111793e9406cef8e79e6616e60b988e8ae11e/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ff5d035dd93fc4706974cb84968aaa8f7d4d463d0628ba2dbeb209a2173efb7c", size = 1324582, upload-time = "2026-08-14T19:06:29.579Z" }, + { url = "https://files.pythonhosted.org/packages/d8/b3/c96f16bb6cd5fedff00bb196bec3379205b90be9af3503d59be8ba60dce7/hypothesis-6.165.8-cp310-abi3-win32.whl", hash = "sha256:08ad80fb46118951c797dd10fe8e2b789c17bd5d8d6b37229d542b86808a092e", size = 668817, upload-time = "2026-08-14T19:07:40.316Z" }, + { url = "https://files.pythonhosted.org/packages/cd/e3/f9d54ef4dd8748487cd5f6b6adbf22342792ccdc312f71c77d01fef7812e/hypothesis-6.165.8-cp310-abi3-win_amd64.whl", hash = "sha256:8af82df1e702a27c44957e33e5ec9da52a4fa4fa9dce6ae573e49a7ea76056c6", size = 674969, upload-time = "2026-08-14T19:06:21.194Z" }, + { url = "https://files.pythonhosted.org/packages/b6/db/8be03eed5476497135d2b4c385912200e7f0c42e1acea8c1b6158ee52677/hypothesis-6.165.8-cp310-abi3-win_arm64.whl", hash = "sha256:f82627da51d12f74f3751fb471d65c51af3f546f07977a4c6c8de9353bd16e96", size = 673309, upload-time = "2026-08-14T19:07:52.906Z" }, + { url = "https://files.pythonhosted.org/packages/3b/4d/87e7fda9ed1c80ef741eab5cc3266a5c145af756d3043c6918b588de6965/hypothesis-6.165.8-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:22563b83b52891356c275945cb6c939d5f6f6d29c39cf9a59f3e0c586d53d508", size = 784504, upload-time = "2026-08-14T19:06:05.026Z" }, + { url = "https://files.pythonhosted.org/packages/de/dd/eca7718276a3ef5fb516303cdf17637fa04ed368bfa4eb15eb7b0b5479fb/hypothesis-6.165.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fb6d5135a5dc095c34882ec0005f9475ef26498ab52698c7f0cbf5f8b7ae148f", size = 776133, upload-time = "2026-08-14T19:06:08.131Z" }, + { url = "https://files.pythonhosted.org/packages/03/a0/38d84c32b21a30116c77e854356159d60663b72b028029dfe891ccd8a426/hypothesis-6.165.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:686a9699b59945758d49c17396c9391b2d63c59f618176d0c2d0515a41a6737c", size = 1106544, upload-time = "2026-08-14T19:06:01.491Z" }, + { url = "https://files.pythonhosted.org/packages/85/bc/53ba55d504a617bd438160b305a4b19fbdd73bf8e20f4420ade290c02f06/hypothesis-6.165.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a035d988587327cfa345f953927f3430ce2b4bb609f2c753a72dbc460844af5", size = 1156527, upload-time = "2026-08-14T19:06:06.612Z" }, + { url = "https://files.pythonhosted.org/packages/51/a2/5a50a78e7b98914a29d98a5b8ddc42ed3bb6efd2bf7d5f98f7c0022fcf91/hypothesis-6.165.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:21c32750e77c1ddc7147964d0872679a3f0908f74e9aaa5b8be6f46595ea0125", size = 1280572, upload-time = "2026-08-14T19:05:52.727Z" }, + { url = "https://files.pythonhosted.org/packages/6b/28/2ae1810296d786e0341c4d496b5f3676b9164df1d71fdf246fa4a321211b/hypothesis-6.165.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08c5a7e08f348f01138e084ac5f09c59254394ef9f7a7957fe0ae418c49ac4fe", size = 1323651, upload-time = "2026-08-14T19:07:13.443Z" }, + { url = "https://files.pythonhosted.org/packages/63/1f/ac475606ebc2915091143f8b0cfeb853f021e443f9dfa97c3c0397b026be/hypothesis-6.165.8-cp313-cp313-win_amd64.whl", hash = "sha256:13bc0f4c8f144a222a038a12511c51e85c5171694b836e1df3b66ffa5250b71d", size = 672124, upload-time = "2026-08-14T19:07:50.792Z" }, + { url = "https://files.pythonhosted.org/packages/4e/52/207d717abfb745bfa3832abf336835c16e4150c6aeeefdee8c8a648c12f3/hypothesis-6.165.8-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0c7177e1da92ed23fd73c27ecb4d5b8298f4fff3fe840d3b7072fe0b8ccd8009", size = 784608, upload-time = "2026-08-14T19:07:07.314Z" }, + { url = "https://files.pythonhosted.org/packages/c5/07/efd8d6c16b94c78da020604f4bde3ae320d524f0e3b481bd57466d90e03a/hypothesis-6.165.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:537d1cf3ec45f34cd73f77e89867085465bb66491e709bb2acadc3d32aaba9b6", size = 776280, upload-time = "2026-08-14T19:05:51.414Z" }, + { url = "https://files.pythonhosted.org/packages/65/2f/1e3b5272b2482d01c33d2ad346bc6b1213ceabac194c4f98f1d36f1f3eeb/hypothesis-6.165.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d06eb8b5c56aedb46934b7d285b1bd599505219d33563e20796df823cfca2d94", size = 1107056, upload-time = "2026-08-14T19:06:09.879Z" }, + { url = "https://files.pythonhosted.org/packages/ab/e2/3822efb584f663706a45dee8d23bdeba651853901b71e20512dae6003cd9/hypothesis-6.165.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5de2603cbe246804aaac7a5e17fb0171cb0bf2c902afbab0fc9bccc0f8ff5f", size = 1156667, upload-time = "2026-08-14T19:05:49.756Z" }, + { url = "https://files.pythonhosted.org/packages/27/c3/6bd4eccbdbc4ffe37fbfb4b505b04dc4188f7ea05b384a2f732a7ec82a3d/hypothesis-6.165.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:26e8c164a556cd0324709c70ecb5b095111fe5f1e619e49260eb4f142742fd0b", size = 1280967, upload-time = "2026-08-14T19:06:48.699Z" }, + { url = "https://files.pythonhosted.org/packages/b9/61/facf2b95c10ad141e5dea884f1a2b84b39bc014569c665dc6ae337bc6fff/hypothesis-6.165.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4a8ebaaacd542b5aa07085e8f4d6adf8b05b75c616482d4267cea800a992a94f", size = 1324021, upload-time = "2026-08-14T19:07:18.888Z" }, + { url = "https://files.pythonhosted.org/packages/09/39/49d22db5a207ef1c5f371777ac4e8e4efee7acb8d11931891c0809ca6650/hypothesis-6.165.8-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:2f62bf3b168c7227e361dec31d9cf53f7f4448d49248ff4a79ea2c529689a394", size = 616171, upload-time = "2026-08-14T19:06:03.189Z" }, + { url = "https://files.pythonhosted.org/packages/42/01/03ffa475c46b14f0324b35a4e8e13613e03abc1f374bbf276c26a7b79dee/hypothesis-6.165.8-cp314-cp314-win_amd64.whl", hash = "sha256:3b31f549cebaf42902e031510742d08b61abfa79e43872f8c05d43599fc5dae0", size = 671925, upload-time = "2026-08-14T19:06:50.459Z" }, + { url = "https://files.pythonhosted.org/packages/69/9d/999668ef0c455048d23e394fcc5fd44b682890d35d9701ff7d8f05d5857d/hypothesis-6.165.8-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:2d6ba5947bd7084e062ffbf375e49d2d0f438f80462fd78fdf70170c6c87ff9a", size = 783067, upload-time = "2026-08-14T19:06:17.935Z" }, + { url = "https://files.pythonhosted.org/packages/d6/dc/7284b3e3a1b7e3dcefe1c473d2da8a1f206746a2f84516c671ee95b6fb70/hypothesis-6.165.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c9c77c83d59deb4e52bb0567a906cb8c0e8231669d2e165a90a8f3f2cdc153c6", size = 774693, upload-time = "2026-08-14T19:07:56.962Z" }, + { url = "https://files.pythonhosted.org/packages/b9/0f/e894faff27e3c665075281e78578479d71d16886c8cadec3d9a37d720502/hypothesis-6.165.8-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8626e21861b66bdc0d90bcb3fe831baa6e42fd29aed3f1070a0a84917d941b1c", size = 1105291, upload-time = "2026-08-14T19:08:03.463Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3e/2681fa031dff98e0b30b77aeeff1a5b5e84857fa4d0d03a2136e0e504716/hypothesis-6.165.8-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7155cfd391220855dc523c58968763fbd4143a63f68331f96b73e5284dc96e1", size = 1155413, upload-time = "2026-08-14T19:07:30.085Z" }, + { url = "https://files.pythonhosted.org/packages/18/03/c828173cea01ffaa38e8faf9e79db38ffc0a4600db1ae50f55e893977118/hypothesis-6.165.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:273885939b364e34ea3d4e0c7b5f71649c8dddd686e011326320c3f661650f00", size = 1278962, upload-time = "2026-08-14T19:08:08.066Z" }, + { url = "https://files.pythonhosted.org/packages/3a/bf/817d7c693f28d079714ac37379e686f2b46b131ae1aa568ffd22aff29a2d/hypothesis-6.165.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:13dd1471755d14fbb0a81ff522fc24b4fd84015bff724b5ba1cd1bcae09bd253", size = 1322644, upload-time = "2026-08-14T19:06:39.873Z" }, + { url = "https://files.pythonhosted.org/packages/78/a1/79357a3992b3c9a049f5982b7f2bf17a54c2c888f29f0453e4511e00b81c/hypothesis-6.165.8-cp314-cp314t-win_amd64.whl", hash = "sha256:f604eb3e86ee6eb8f68363079ff3b85e44281ea0af44fd3085345d29528a66ea", size = 671923, upload-time = "2026-08-14T19:07:01.819Z" }, + { url = "https://files.pythonhosted.org/packages/e9/9e/5c521f75c36b4883b11bb3d0f3ecb112081eda3f645df496e0fe8b048101/hypothesis-6.165.8-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:00fa62ff3ed7265ffe9f064b148d2fae9f5bdb8e59212e7bae6d463b97a8622a", size = 782655, upload-time = "2026-08-14T19:06:27.922Z" }, + { url = "https://files.pythonhosted.org/packages/e1/09/63eab3462d42f4a7252437fddfa6d7c6b9ad44d9497546917b8b62c5e752/hypothesis-6.165.8-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:ce7321947298ab4529ee8e5b8e38393bb12f36125d2f1affc4f860510933f7cd", size = 774338, upload-time = "2026-08-14T19:07:25.953Z" }, + { url = "https://files.pythonhosted.org/packages/8a/78/ec97e7f981ad61463bc537d4e8bd46e0b52cdbb501b1e12bd7065531064d/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:486c2b5bdd797667aefe845221be7c8f45ec5af599cdaa6536dc71d445bc81b6", size = 1104676, upload-time = "2026-08-14T19:06:57.807Z" }, + { url = "https://files.pythonhosted.org/packages/88/78/9ca6a51788527fba0dc42e473061d214b02ecc9bebeec1d35e9b26d64627/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bce581a7f2df2d54291f01d8e458ae5eead5d097761411bf0277f9aa6b944efc", size = 1133138, upload-time = "2026-08-14T19:06:46.736Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/7e8025c115fb76c1adad0984b7b0b25b59c746aad01cc7d4b87f92ce33e5/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff023503609e114cc9a36139e1597e36df6d443fb87931e232436402e74d57ac", size = 1132073, upload-time = "2026-08-14T19:05:46.653Z" }, + { url = "https://files.pythonhosted.org/packages/3e/d4/4746b097ff330c240d79e691c4575c8d00f0f4f8a5dd41602247130445d6/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d34b700aae1fee6ec644d0417a67e7710e109ec50f1ad5db5f8a3a42ca1fbb", size = 1154989, upload-time = "2026-08-14T19:07:48.698Z" }, + { url = "https://files.pythonhosted.org/packages/fb/24/4a87adbd95823be301b73a8df7fd669b6468759d6ee6d503f4f7c2f24552/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:258b12f5683c04a1374adde24f40cbf65fcdbb76758ae000cc901a4a26a33c77", size = 1109664, upload-time = "2026-08-14T19:06:26.242Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b4/7b33b233671621bdfcebf7e12e2944aa179e86e92791fbba87b5bfb03790/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7ea1106f25dd191c4e600ae31b61552d6066a0ab7da7fd8ae8fe3b4bfb31ae1", size = 1144746, upload-time = "2026-08-14T19:07:46.456Z" }, + { url = "https://files.pythonhosted.org/packages/75/9c/1546de2abd9e084c409671e721740648a1f88da98a64fd0f314cd15d420c/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:871c32499f86473df83c4670399e8994ee869b0fba12c711885f4c48510cfa1d", size = 1278501, upload-time = "2026-08-14T19:06:32.897Z" }, + { url = "https://files.pythonhosted.org/packages/26/56/ac1d16b9cfc2fdc11ddd425e4576cc8e7cf2bcd525e8c70e65ca8ea152cb/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:e769cc3cbe002134be0eda1ce6872efa61b13146c3f66eea3eaff6c7a1907c4a", size = 1406976, upload-time = "2026-08-14T19:05:55.607Z" }, + { url = "https://files.pythonhosted.org/packages/2c/92/059d33ed4711a81e73d939ee7303ce7792d5678954e913676fe696931c39/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4af4d747f5f1fc81806844dfa8618f20f71ad599d55f17c8dca2a8c7fb3cc16e", size = 1261140, upload-time = "2026-08-14T19:07:36.099Z" }, + { url = "https://files.pythonhosted.org/packages/5f/0e/21483d154bb33181da16aba706d391ca88eabf53c2259fbde1d228f950d9/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:46f16174505dc6d977b9513e23f5cae60382e849861673fff881fdc0ab229946", size = 1279005, upload-time = "2026-08-14T19:06:22.975Z" }, + { url = "https://files.pythonhosted.org/packages/95/0f/2b2dc2688f98f482666b8fd30bfb58ee58c07cdbf99226ea357e71beec9c/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:fb61b9a4cb8a63e16baf37b0f693482e47985adde2498a6b6e9cd0dc6141bca0", size = 1322177, upload-time = "2026-08-14T19:08:12.392Z" }, + { url = "https://files.pythonhosted.org/packages/7d/ec/d53cb547ac4c7c39a21760e9a658b7730cbb6b3b93900dcf69cbadf31dcc/hypothesis-6.165.8-cp315-abi3.abi3t-win32.whl", hash = "sha256:8b32c2eed1a439ecf2217403eb048de2c160a61b909fb031a871af030c507c1f", size = 665817, upload-time = "2026-08-14T19:07:03.881Z" }, + { url = "https://files.pythonhosted.org/packages/69/f1/d581a48906ed56f5a57eb39c54b05d584ed0a771c8f3bf5d5d0532ab8c0d/hypothesis-6.165.8-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7166c761411625770c0fabe475de642102d0f27f910644306337fd208da25924", size = 671723, upload-time = "2026-08-14T19:06:43.24Z" }, + { url = "https://files.pythonhosted.org/packages/76/76/7d905c691331442965e58ddfc267900efeb7e8354b6083455aaa67e41834/hypothesis-6.165.8-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:88e9d1803ed4d7b2580c0e84c1c035f2a1d74327613aab58bffeeb4e3025b6dd", size = 669710, upload-time = "2026-08-14T19:05:59.934Z" }, ] [[package]] @@ -1832,7 +1851,7 @@ wheels = [ [[package]] name = "pre-commit" -version = "4.6.1" +version = "4.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "cfgv" }, @@ -1841,9 +1860,9 @@ dependencies = [ { name = "pyyaml" }, { name = "virtualenv" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/25/3a/ddb78f32a0814e66b18a099377a106a2dcdce92d86a034d69d65df9b256e/pre_commit-4.6.1.tar.gz", hash = "sha256:03e809865c7d178b9979d06c761fcbfe6808fdaded8581a745bb110e52050421", size = 198646, upload-time = "2026-07-21T20:56:58.225Z" } +sdist = { url = "https://files.pythonhosted.org/packages/74/89/1f3e8e1fc3e97de0fa963495832f581f025f29471602a309e48808244292/pre_commit-4.6.2.tar.gz", hash = "sha256:8f5d7bfb021ecdbcd9d49d89847082dd24172ccde534390081a679ad046e2441", size = 198670, upload-time = "2026-08-10T22:07:18.421Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fb/49/bc925106abcdac498074f2cbe6137e94e09f418dd2b7775df5b577dc0313/pre_commit-4.6.1-py2.py3-none-any.whl", hash = "sha256:0e3b2942510d1fb34eec167a3ec57331bf8442122f1153a9fb8b58f5c49b2717", size = 226186, upload-time = "2026-07-21T20:56:57.064Z" }, + { url = "https://files.pythonhosted.org/packages/45/e2/bbb7129c9e7999a6b8ee9cca3b66486c25c423ab5a75f34071798b74ce94/pre_commit-4.6.2-py2.py3-none-any.whl", hash = "sha256:e2dde9a75d3bce11bd3831c26d134df00a2803c1d818be6a0383c3dcda25dc4e", size = 226202, upload-time = "2026-08-10T22:07:16.942Z" }, ] [[package]] @@ -2400,27 +2419,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/73/e1/4508a569211b35599016e84ba65c1a992b7a4004b4b6c4bea02a851cba1b/ruff-0.16.2.tar.gz", hash = "sha256:c3d7828d12e8927a6fc65fe38e2c2541b9e762d360a1786d752cb1b8883b3c9c", size = 4885811, upload-time = "2026-08-07T13:31:01.432Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/14/57/db19951540f98859c956b50bdb4d31089b4d91e9f15e2968e7d5193806d5/ruff-0.16.2-py3-none-linux_armv6l.whl", hash = "sha256:3c8de4cf2181f01d57946d87d777aa52916976fc09942aed89938fab5e013318", size = 10847925, upload-time = "2026-08-07T13:30:14.468Z" }, - { url = "https://files.pythonhosted.org/packages/13/5a/995fe85a8470d3e391ac0f7fa8054bb454eaf33ee138196d6172ed1079c0/ruff-0.16.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:9a48cc05c6fbc811ca81b5d7ba95375affea6582d1b8024e455e41afbbf55344", size = 11072662, upload-time = "2026-08-07T13:30:18.143Z" }, - { url = "https://files.pythonhosted.org/packages/32/53/370d767c61c71a971a4ace36703a7ecd8c393956349a7325d7fab2b56827/ruff-0.16.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:a2c0d14fcbb26c91f0f867a6dc9bd71bbc30b1b6151829c884f23faeab2e5700", size = 10566771, upload-time = "2026-08-07T13:30:20.899Z" }, - { url = "https://files.pythonhosted.org/packages/85/d6/9d96948caf5a632be62d62202d5ec914d6856f204fd79eb036e5915e79ea/ruff-0.16.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:335c621622c4650330be50842561c6586ac6971bb8ab5407fe34dcc9efb16bbe", size = 10975825, upload-time = "2026-08-07T13:30:23.517Z" }, - { url = "https://files.pythonhosted.org/packages/3b/92/ea87129b3414acb0b5770563779c51804d37ac67675c7ba35447ddb14773/ruff-0.16.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:20e66910f2c37cc753f9ef6580c914a621b80c4fa3549d3e3521e29d0f5bfc3f", size = 10649437, upload-time = "2026-08-07T13:30:26.097Z" }, - { url = "https://files.pythonhosted.org/packages/ac/43/f8f291dcd4af5bb7872b74fdfa41a7cd7c856ca1d4069670971cf1b9f5cb/ruff-0.16.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7e36fbfba65510548156902bcf1350a979a958ce0347ce0f90d73894036b39f", size = 11446761, upload-time = "2026-08-07T13:30:28.752Z" }, - { url = "https://files.pythonhosted.org/packages/71/4a/ef991fb2fcf516ab71f0808adcdd8da5e18c8cde447f4ceaf5f47a5132a5/ruff-0.16.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:f0eab35f80df8f134aae5d1630e751901321d317cc8e50dc39e36fa3ed34cd12", size = 12336364, upload-time = "2026-08-07T13:30:31.468Z" }, - { url = "https://files.pythonhosted.org/packages/f3/24/f615e74f307e6ca0e56a482872477b856c70d530aa356abfb6dfe5ca8a80/ruff-0.16.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:40ea8c0594feb894e89c8c61ab9c103d38b0ea72dfde6c594107147ca31b1140", size = 11630720, upload-time = "2026-08-07T13:30:34.426Z" }, - { url = "https://files.pythonhosted.org/packages/c5/d3/8ef50149e8412a77f7ab409efdef0e2b23803707a3863da4fc64cb23d459/ruff-0.16.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ab3d62dde0b19facdd632008cc4827fc28ada7736c6bd35ab6f1050f0bfed53f", size = 11466130, upload-time = "2026-08-07T13:30:36.958Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a7/a19334985c4dea8c381981fa252cd854c7ee52dc4b1686dc16f4a911c702/ruff-0.16.2-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:e43e1f5b8388da9eca1b9e88328d47a5cec794633ccf6f7484ac2dd15eee92c0", size = 11523634, upload-time = "2026-08-07T13:30:39.822Z" }, - { url = "https://files.pythonhosted.org/packages/6e/6c/96d192b0e742412ceda08c0a50f9669b253dde9fd6a60ea1a10c9fa79a63/ruff-0.16.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:c24788a980581e1d7ea3a0cbe4344c4fbeb0a6a9b1f4713aa46bb104f8294690", size = 10949807, upload-time = "2026-08-07T13:30:42.745Z" }, - { url = "https://files.pythonhosted.org/packages/fa/51/e26599ceca11e79ee255c7df515995561edf87e9ca1893284e44d98f5a86/ruff-0.16.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:81806b08329130005dd4a8a8394a0c9da8c6f4cafb16ba438d2a2ee6a18bedf1", size = 10646891, upload-time = "2026-08-07T13:30:45.522Z" }, - { url = "https://files.pythonhosted.org/packages/68/01/800c4b1f97bc8d7c6029e06b1f20473a3cf1e13c4933d8f3342add83fc55/ruff-0.16.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:4ce4e02bad779bef557f541a1b31f20d6abeae1cc05ed1b1ac019d4ffd1044c8", size = 11162063, upload-time = "2026-08-07T13:30:48.131Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d0/1477ea50fc5a0d4b0b71d1d63d50770bdd794d90b43e37a7618e63ec9894/ruff-0.16.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e0422abdf70070255fc4073ce9dfc814cc03db577013761ddd09bc1e4a9a4fbd", size = 11556038, upload-time = "2026-08-07T13:30:50.686Z" }, - { url = "https://files.pythonhosted.org/packages/b8/76/a7776f32048d991e16d4fa8ff91790b877342d3596cc3ed04acdbf1aaedc/ruff-0.16.2-py3-none-win32.whl", hash = "sha256:bf3a63d78fb39f4bf5ac8ae52051c5520505301abe19ba4e204c453b3f09bb0b", size = 10872850, upload-time = "2026-08-07T13:30:53.471Z" }, - { url = "https://files.pythonhosted.org/packages/00/0d/929c800d920e61397d82a01b60bffc68da3052c17d31de59efaad2e4ed75/ruff-0.16.2-py3-none-win_amd64.whl", hash = "sha256:bcabe2f6d0fc7819f1431793005af4e4de7371927d037345bf941252b195b9fa", size = 12023338, upload-time = "2026-08-07T13:30:56.193Z" }, - { url = "https://files.pythonhosted.org/packages/5b/6c/93e26c22c5f78ff87363e07da49c84955affbeb1098bd1936bf3b3f293bf/ruff-0.16.2-py3-none-win_arm64.whl", hash = "sha256:d614e95cedf38a2053fd351c55b103ba30d017d61688fdbfd40ee0412852a99f", size = 11374065, upload-time = "2026-08-07T13:30:58.775Z" }, +version = "0.16.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, + { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, + { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, + { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, + { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, + { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, + { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, + { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, + { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, + { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, + { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, + { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, + { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, + { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, + { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, + { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, ] [[package]] @@ -2483,36 +2502,30 @@ wheels = [ [[package]] name = "sqlalchemy" -version = "2.0.51" +version = "2.0.52" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/f1/a7a892f18d4d224e6b26f706531eafccc41e37594d37d304786969ee13cb/sqlalchemy-2.0.51.tar.gz", hash = "sha256:804dccd8a4a6242c4e30ad961e540e18a588f6527202f2d6791b01845d59fdc9", size = 9912201, upload-time = "2026-06-15T15:41:20.012Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/54/fe/a210d52fd1a90ecfae8a78e9d8b27e18d733d60818a8bf250ff690b75120/sqlalchemy-2.0.51-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:7c2056838b6685b72fdb36c99996cf862753461a62f2e84f4196371d3b2d6a07", size = 2157184, upload-time = "2026-06-15T16:08:50.374Z" }, - { url = "https://files.pythonhosted.org/packages/17/6b/2dce8369b199cb855110e056032f94a9f66dacc2237d3d39c115a86eac56/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:483b11bd46bf35fc14c52faf338b04300c9e6ce554bce9b11be85bfec3bc3195", size = 3284735, upload-time = "2026-06-15T16:19:46.934Z" }, - { url = "https://files.pythonhosted.org/packages/53/ff/dbc495b8a14da840faffb353857a72d4190113cac33727906fb997047f0f/sqlalchemy-2.0.51-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1bed1ee8b01da6088210aa9412023326fb98a599ba502e6118308601dcbef77f", size = 3302756, upload-time = "2026-06-15T16:26:41.336Z" }, - { url = "https://files.pythonhosted.org/packages/cf/d5/fde8f4dddcf518ee15ab35a7c6a28acc32c8ba548d1d2aa451f96e6dbb0b/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:72ca54c952107ba5cd58854b67a5a6268631289d21651a1235396f3b98b47400", size = 3232055, upload-time = "2026-06-15T16:19:49.286Z" }, - { url = "https://files.pythonhosted.org/packages/67/d1/43d3a0ac955a58601c24fa23038b1c55ee3a1ec02c0f96ebb1eae2bcf614/sqlalchemy-2.0.51-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:b3e693d15533a45cd5906f0589f9c35090bef6ef45bf1e8195c424aa0ae06a8d", size = 3269850, upload-time = "2026-06-15T16:26:43.017Z" }, - { url = "https://files.pythonhosted.org/packages/94/df/de669c7054cd47c4439ac34b1b2ee8b804a794791fbb10720e997a2c87c7/sqlalchemy-2.0.51-cp313-cp313-win32.whl", hash = "sha256:b93ab07b5292dbe7e6b8da89475275e7042744283921344b56105f3eeb0f828b", size = 2117721, upload-time = "2026-06-15T16:23:12.36Z" }, - { url = "https://files.pythonhosted.org/packages/d0/8a/403c51d064196bae20a0bc2476577f83a3f8dd299719a97417086b7f2ec5/sqlalchemy-2.0.51-cp313-cp313-win_amd64.whl", hash = "sha256:0f053118c30e53161857a953e4de667d90e274980dccbe5dd3829bbbeece72a5", size = 2143615, upload-time = "2026-06-15T16:23:13.906Z" }, - { url = "https://files.pythonhosted.org/packages/b1/49/a739be2e1d02a96a658eb71ab45d921c874249252358ad24a5bffdd02525/sqlalchemy-2.0.51-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:6ea306caaae6bd5afd0a46050003c88f6bf33227377a49298c498c3cb88ff491", size = 2158999, upload-time = "2026-06-15T16:08:51.759Z" }, - { url = "https://files.pythonhosted.org/packages/23/6b/2e0e38cf75c8780eca78d9b2e78164f8bcfd70125e5caa588ff5cbb9c9f4/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c45a496d6bc05dec41dcd4c3a2b183723f47473255c159cd80b503c8f246424d", size = 3282539, upload-time = "2026-06-15T16:19:51.065Z" }, - { url = "https://files.pythonhosted.org/packages/dd/a1/e77854cb5336fd37dc3c6ae3b71de242c98caac5725120be0b526b31cbd0/sqlalchemy-2.0.51-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4004ada0aafe8ae1991b2cd1d99c6d9146126e123bd6f883c260d974aa012e54", size = 3287545, upload-time = "2026-06-15T16:26:44.735Z" }, - { url = "https://files.pythonhosted.org/packages/f6/ab/9e17272fd4dac8df3b83c4fbe52b998a1c9d89a843c8c35ff29b74ff7364/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:0f6bcad487aee1c638d707235682fc96f741de00663619881ab235400d03289e", size = 3230929, upload-time = "2026-06-15T16:19:52.625Z" }, - { url = "https://files.pythonhosted.org/packages/02/3c/52f408ea701781caee975606beccc48845f2aee8711ac29843d612c0306c/sqlalchemy-2.0.51-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:39a76529db6305693d8d4affa58ad5b5e2e18edd62daea628b29b97930b3513d", size = 3252888, upload-time = "2026-06-15T16:26:46.454Z" }, - { url = "https://files.pythonhosted.org/packages/24/16/3efd2ee6bc4ca4693a30a1dd17a91b606cae15d517d2a4746611d9b73ce8/sqlalchemy-2.0.51-cp314-cp314-win32.whl", hash = "sha256:08a204d8b5638717c26a24df18fcf40af45a6b22e35b70b1d62f0113c2e278e8", size = 2120551, upload-time = "2026-06-15T16:23:15.629Z" }, - { url = "https://files.pythonhosted.org/packages/7b/78/55b12e70f45bccc40d9e483925c065027b3b98ea4cbbdf6f8c2546feaf6c/sqlalchemy-2.0.51-cp314-cp314-win_amd64.whl", hash = "sha256:96747bfbadb055466e5b46d572618170046b45ce5a4879167f50d70a5319a499", size = 2146318, upload-time = "2026-06-15T16:23:17.108Z" }, - { url = "https://files.pythonhosted.org/packages/21/db/a9574ed40fed418924b1b1a3e54f47ee3963053b3d3d325a0d36b41f2c08/sqlalchemy-2.0.51-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:e5ea1a213be1fcd5e49d9904c3b9939211ded90bc2a64e93f4c01963474285de", size = 2178920, upload-time = "2026-06-15T15:59:56.285Z" }, - { url = "https://files.pythonhosted.org/packages/bf/90/a1bb5c7cbba76b7bc1fbd586d0a5479a7bc9c27b4a8298f22ec9423b2bb3/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7c6b36ed71f41942bdcd2ad2522be46bfce09d5705be5640ecf19bbc7660e4b7", size = 3566534, upload-time = "2026-06-15T15:58:35.024Z" }, - { url = "https://files.pythonhosted.org/packages/15/4b/481f1fed30e0e9e8dd24aecbb49f29eb57fe7657ece5cf06ee9b84bb97d8/sqlalchemy-2.0.51-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c2c62877097e1a0db401fba5cb4debee33265e5b2a55c4ccb489c02c53b4f72", size = 3535844, upload-time = "2026-06-15T16:02:43.973Z" }, - { url = "https://files.pythonhosted.org/packages/02/71/0aa64aeda645510af0a43f7d9ee70932f0d1dc4263aed34c50ee891d9df3/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:0378d055e9e8cd6ce4d8dff683bdd3d7d413533c4ee51d67a2b1e0f9eacc0f23", size = 3475355, upload-time = "2026-06-15T15:58:36.592Z" }, - { url = "https://files.pythonhosted.org/packages/05/db/6061db32316446135a3abae5f308d144ab988a34234726042da3e58b1c63/sqlalchemy-2.0.51-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:6e46fc36029eff666391e0531e5387b62ce6c4f1d8e50b3fb3099eaca1b42522", size = 3486591, upload-time = "2026-06-15T16:02:45.346Z" }, - { url = "https://files.pythonhosted.org/packages/0d/c9/f14fdf71bb8957e0c7e39db69bbdf12b5c80f4ef775fdfa127bf4e0d6760/sqlalchemy-2.0.51-cp314-cp314t-win32.whl", hash = "sha256:9161cfc9efce70d1715f47d6ff40f79c6778c00d53be4fbc09d70301e4b83ba7", size = 2151313, upload-time = "2026-06-15T16:03:39.127Z" }, - { url = "https://files.pythonhosted.org/packages/6a/c6/673e618e6f4f297e126d9b56ea2f6478708f6c1af4e3223835c22e2c3697/sqlalchemy-2.0.51-cp314-cp314t-win_amd64.whl", hash = "sha256:159bb6ba32059f57ad7375a8f50d844dd2f19d14954ecf820cd33e20debd46b2", size = 2186280, upload-time = "2026-06-15T16:03:40.569Z" }, - { url = "https://files.pythonhosted.org/packages/e2/22/dbf013a12ec759e54a34a119e9e217435b3f71b2dd5c61a7ade0a25dae87/sqlalchemy-2.0.51-py3-none-any.whl", hash = "sha256:bb024d8b621d0be75f4f44ecc7c950450026e76d66dc8f791bb5331d7fed59d5", size = 1944334, upload-time = "2026-06-15T16:09:22.418Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/3b/21/77b4c147963073040dc3c3a5cb7a8c3001a1893c0209432cb77f9df836aa/sqlalchemy-2.0.52.tar.gz", hash = "sha256:5e2d46356ac2ccb7d268ab6c2319ac6a2b42f1b8d5fd8bd3d46855cd82abee97", size = 9945637, upload-time = "2026-08-11T19:07:09.829Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/18/e30c6fe1eca1bf34a39fbdd6066121cc9974c850faf6f349eac563697a26/sqlalchemy-2.0.52-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2eb3c6a64b1bfe6704777cfd504e7b8ad093a5f3e03ce67663a5e6742f294e43", size = 2167724, upload-time = "2026-08-11T20:58:12.679Z" }, + { url = "https://files.pythonhosted.org/packages/d0/56/2e17d161a4f7ecc1c2ffb93e607b4e1898bb551b451b283235acb8f6ce47/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:923bb183c1dc64fdf7b717965e3d59938ec4f8b8710b419a21ce403e5da9a9e1", size = 3321189, upload-time = "2026-08-11T21:02:41.932Z" }, + { url = "https://files.pythonhosted.org/packages/cf/b8/8490916e893f3f8d74dc9cc54c078619364999dee37047a188e73abbc852/sqlalchemy-2.0.52-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:651d6d8782e80679e6151707c7b490834d46ada526328895abf567f25e63d29c", size = 3338185, upload-time = "2026-08-11T21:17:02.597Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f7/752cc8ee453da222829b3f5c4613614bf750d97429363b70414fa10478e4/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b08cddb8989775e3c88799d86704bdfc3ee6e9846118201aa5997f16f27e3a15", size = 3271698, upload-time = "2026-08-11T21:02:43.963Z" }, + { url = "https://files.pythonhosted.org/packages/51/e6/074ade0c07b9e4c8e8bca46820320ed94df9702afdb6f2af06623068d2e6/sqlalchemy-2.0.52-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ab66fa9618269390d4dfa222f2f2f88f7bc4bf5da13905131b818217db7e8057", size = 3308936, upload-time = "2026-08-11T21:17:04.172Z" }, + { url = "https://files.pythonhosted.org/packages/66/07/557c0d04716705599227945ac14e0a17ad0338e899f37d8c2ddff4dcc663/sqlalchemy-2.0.52-cp313-cp313-win32.whl", hash = "sha256:c63bda077685c85ca513286547a531ba57e7a68cf0a7ed3bafcc2bbd18896f4d", size = 2127308, upload-time = "2026-08-11T21:14:53.879Z" }, + { url = "https://files.pythonhosted.org/packages/96/4e/226eda27654318ce525d043025221f689abef883da2c7126f9065121618c/sqlalchemy-2.0.52-cp313-cp313-win_amd64.whl", hash = "sha256:9876b09b9f1ce7398b0ffece585c0a911244c53191187341f6bcae640e133751", size = 2153876, upload-time = "2026-08-11T21:14:55.527Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f5/71cb30af58c9b80a4e1fac0b73bb48f86d497a774a6a2eb6d2f1e657bb73/sqlalchemy-2.0.52-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:410d52be41d17f1a236d19520fbe776257dc16516ed06bd16d433311842aefd9", size = 2169537, upload-time = "2026-08-11T20:58:13.855Z" }, + { url = "https://files.pythonhosted.org/packages/4c/93/d07ebd645d1b07b6b5ed63450a70f063a346a7e0f2c8810daf2e532400cb/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dfe9ce533dbe4d0a2ae1486546619bd30b76bcd670539a44d910361376175f5e", size = 3319606, upload-time = "2026-08-11T21:02:45.829Z" }, + { url = "https://files.pythonhosted.org/packages/ae/5c/290c84c7c2566ecd3b65baaae0fddec9bc33b033b398a06123bb86fbfc6e/sqlalchemy-2.0.52-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:812bae5138bfc0aa46fb0686da0fc7f581f68e2bbb05bc24c3713bebaedd1437", size = 3323642, upload-time = "2026-08-11T21:17:05.675Z" }, + { url = "https://files.pythonhosted.org/packages/13/f5/2cc160590ca49173359557880b92a0572293ccb899e8f6cedf150c5a3ddf/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:50bff43b632a56fbf5ed9afdd76307e1512b62051bcd5afb341ae67205bbb6c8", size = 3268125, upload-time = "2026-08-11T21:02:47.649Z" }, + { url = "https://files.pythonhosted.org/packages/35/f3/ea8933fc9f7d1353e9c2ff9965eae687c4cef181120574591ed2fa0633e1/sqlalchemy-2.0.52-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:49565daf5af554f538e23aef1fc81a95a4e49658f152285e45c02f5fc44f04cd", size = 3289516, upload-time = "2026-08-11T21:17:07.267Z" }, + { url = "https://files.pythonhosted.org/packages/45/67/05cf86541c1e1716fca1e4a996954a439cd74501707cda607fb7cb02ef50/sqlalchemy-2.0.52-cp314-cp314-win32.whl", hash = "sha256:ab9da41e61b9979b910499d633b241df20c51ee5037e5405b11c2faac3cbe1a2", size = 2130249, upload-time = "2026-08-11T21:14:57.273Z" }, + { url = "https://files.pythonhosted.org/packages/96/d7/8ac6ffa1e36169e762ef65bd835046abb2251b1bc17f8f6708e14ed8d31f/sqlalchemy-2.0.52-cp314-cp314-win_amd64.whl", hash = "sha256:a593db51b3bae75db17a5738ad5f992244b3a03863f83c28117ee482c6a3f76d", size = 2156718, upload-time = "2026-08-11T21:14:58.667Z" }, + { url = "https://files.pythonhosted.org/packages/dc/4b/e01a737eef378e734cc6394a82248a6ce13b167dfa36c731075ce9fc9c64/sqlalchemy-2.0.52-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1e61d08bdf4ee2f41024569e3400de7d6734ba498144766b11260936ccfa582", size = 2190344, upload-time = "2026-08-11T19:53:21.393Z" }, + { url = "https://files.pythonhosted.org/packages/b3/3f/3582293d1e185e71d19d7c731c3e2ee20ba21981c4a1115c0806c1f62120/sqlalchemy-2.0.52-py3-none-any.whl", hash = "sha256:3b81b8363a919ce53453591cdb93702e6bd54ade6c4fa2f468fc053baee5ed89", size = 1950700, upload-time = "2026-08-11T20:47:21.603Z" }, ] [[package]] @@ -2541,27 +2554,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.69" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8e/5b/7a618632dfe9373b7df572ecd7a08c8f799d772fbc317da82dd3aa363207/ty-0.0.69.tar.gz", hash = "sha256:b65106e9ff24fa76e25e1142fb09c85244e815c40450e3021d2bf652c231bb43", size = 6565094, upload-time = "2026-08-06T10:04:25.667Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/60/6534092f4d2c15e2491807edd609c2e50d527c1fed957acf40b9f110b64a/ty-0.0.69-py3-none-linux_armv6l.whl", hash = "sha256:98bfd383b273540829af673e7f98b9c1c4bcc8547d12a1a3806cd0bec7f0e087", size = 12364185, upload-time = "2026-08-06T10:03:47.137Z" }, - { url = "https://files.pythonhosted.org/packages/34/2b/5c29689bd4f74c2e3394d983d85e4011b629f2ce3730c9442553b8554bf8/ty-0.0.69-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:964621ddd05771660017c51b4e74078d861d9fc863c21ef2a500db1ab62c9ccf", size = 12042510, upload-time = "2026-08-06T10:03:49.481Z" }, - { url = "https://files.pythonhosted.org/packages/09/46/fa085bde4d23516d7ef14b24736fc5dd7dc498f60f52b3d077e59ffdea20/ty-0.0.69-py3-none-macosx_11_0_arm64.whl", hash = "sha256:3ffea4048dd0da4c9c97393b4be0901098a9065b06fa81be2477cbde65d8a151", size = 11549397, upload-time = "2026-08-06T10:03:51.747Z" }, - { url = "https://files.pythonhosted.org/packages/25/cc/97b9efb2061dcab6fef1e94a4ad99df0bb45bd2cc15d4f5794c787ee0552/ty-0.0.69-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a8684d4a70aadd1eab0f41bdba835e3288ef49db8402a8e6ca81bab52ed5d610", size = 12115567, upload-time = "2026-08-06T10:03:53.79Z" }, - { url = "https://files.pythonhosted.org/packages/e0/c1/a5e0404965093835f3e62544e661784ec0aa8ef0b006ed50af50b19c107e/ty-0.0.69-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:afaaba240ab4122e2069a796836d10be81b4ddb053ae268b3dff962a0b4ca5c7", size = 12149770, upload-time = "2026-08-06T10:03:55.993Z" }, - { url = "https://files.pythonhosted.org/packages/e2/39/8cad6b205a4abe8a044ca0c84aea71e8ccda29b07a75a5f090e310605580/ty-0.0.69-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11ea63ef07d4e33aeb1a775cf5f2c736b3ed22fa6f8b1b608591612c36795044", size = 12941278, upload-time = "2026-08-06T10:03:58.324Z" }, - { url = "https://files.pythonhosted.org/packages/d6/8b/8766d96b732c2a060d70dc8ccafcc4d6a54109a2a95f1deb0705de88892b/ty-0.0.69-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cb3730b1268e92a2907d7aea3afe8dd1b360ae65862f0557080cf479d481b424", size = 13426509, upload-time = "2026-08-06T10:04:00.621Z" }, - { url = "https://files.pythonhosted.org/packages/02/1f/e991b2cde953ea5b94d6a9a4c45c87937bd916bc09235f764407bf471c0a/ty-0.0.69-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a544ff57a752ef186ed40b5a2f44c17402af4cdefeb74a311ca02ebd57c4fca0", size = 13106582, upload-time = "2026-08-06T10:04:02.818Z" }, - { url = "https://files.pythonhosted.org/packages/ea/bb/73538f1b99e3558fd9db87b98698426f0f60fc8666da0b1efd0e70e275eb/ty-0.0.69-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87ed2cbca20caddfdf8e3e14d213ce91b67e75feed78900f4aaf3ef884954028", size = 12708931, upload-time = "2026-08-06T10:04:05.233Z" }, - { url = "https://files.pythonhosted.org/packages/87/cd/484a5208d74c4ad1155933906295ccdce9aa81a257d8df2ab9e41bd60133/ty-0.0.69-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:2684efcbce5b6fe45045faf610b377b50781b6d2aa7e61ea23ecf5b3d2bce421", size = 12985322, upload-time = "2026-08-06T10:04:07.587Z" }, - { url = "https://files.pythonhosted.org/packages/6e/81/b75003f0d4da9ab3bc8fd4f4802f836cb9921ff7e70f460604f7b769a0b5/ty-0.0.69-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:da9aeb26fdac1d2214937542b59e0d4d1ba94ec7a3f45444f33c846de1eb1d63", size = 12063910, upload-time = "2026-08-06T10:04:09.835Z" }, - { url = "https://files.pythonhosted.org/packages/8a/76/088469f547ef63dceefc4a75826aedee5014f9371dc5171cde931896a82c/ty-0.0.69-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:00e7677cd14ede381f705f71104ea7b8ea0ce217a8634e19a89781953de0e9ad", size = 12166823, upload-time = "2026-08-06T10:04:12.114Z" }, - { url = "https://files.pythonhosted.org/packages/0a/c9/ce88a0bec0d46d8ae180b99c6ec014866fecc4cba1727b5feec8877b2765/ty-0.0.69-py3-none-musllinux_1_2_i686.whl", hash = "sha256:d91965eb799649833d0d6042db09cd03d15289125245337cc46a2606effb7bda", size = 12483136, upload-time = "2026-08-06T10:04:14.33Z" }, - { url = "https://files.pythonhosted.org/packages/63/9e/6fae0ff225a0012642cf72c077e20f8f448c0a80771bc3360e8178fe2f32/ty-0.0.69-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:1f03359cd8e5c412aa0c181118fa9b9061a4dddaedbb61bac0a424fb0814d402", size = 12799025, upload-time = "2026-08-06T10:04:16.445Z" }, - { url = "https://files.pythonhosted.org/packages/e4/43/78a658d18b2a4ccf35b053392f2213bf12e3c63b2abea512d3b6751d1f4c/ty-0.0.69-py3-none-win32.whl", hash = "sha256:ec460e01586b1eb91894c4a8403bee3e045a47e7a4ada943cc27ce8e348e88cf", size = 11787774, upload-time = "2026-08-06T10:04:18.622Z" }, - { url = "https://files.pythonhosted.org/packages/3a/5e/88db1f674403f2b81316a853a44a81ed220621fa96f8f7ae586fb6ca7513/ty-0.0.69-py3-none-win_amd64.whl", hash = "sha256:18976ca26a4e28fc3249477f79a695d5502e670803f2e080d89ac905baef3c6e", size = 12864038, upload-time = "2026-08-06T10:04:20.748Z" }, - { url = "https://files.pythonhosted.org/packages/4d/7b/6fc6efd00c69103d70f2bdbe824343089cd70b17b3079170057d3e5a3ac0/ty-0.0.69-py3-none-win_arm64.whl", hash = "sha256:7d4ca3bb74d91cb9947ba3f3b4cb131ad6a2b3ecc76d34040c4ec6092d2e411d", size = 12196693, upload-time = "2026-08-06T10:04:22.902Z" }, +version = "0.0.72" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, + { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, + { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, + { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, + { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, + { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, + { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, + { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, + { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, + { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, + { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, + { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, + { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, + { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, + { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, ] [[package]] @@ -2620,15 +2633,15 @@ wheels = [ [[package]] name = "uvicorn" -version = "0.52.1" +version = "0.52.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/18/ccce41535dee1be77735592bd19965f3972c82e07ee703d324709496b716/uvicorn-0.52.1.tar.gz", hash = "sha256:112ec661814189acbccd3f7b86460147cc065fc92c0821afa78918780e4354dd", size = 100571, upload-time = "2026-08-01T18:19:30.732Z" } +sdist = { url = "https://files.pythonhosted.org/packages/2e/28/64ca011edf31c715b4fad359c587ea52391aaffa125065695590241ff617/uvicorn-0.52.3.tar.gz", hash = "sha256:18857b9e6579300be55c91c0a1cfd37d9a2cf0cabea33b88275f199eb73b8b58", size = 100621, upload-time = "2026-08-13T16:50:02.899Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/d5/68e6e9bca63c0badf67002890a46d3784c958de45b65e1275ec583ca1f06/uvicorn-0.52.1-py3-none-any.whl", hash = "sha256:e4403f9d93188cf9d1088e9f40e3acd12630e2df8675316704379a7fc20fff6a", size = 79859, upload-time = "2026-08-01T18:19:29.294Z" }, + { url = "https://files.pythonhosted.org/packages/dc/2b/ebd108734a8204c6b4b93c681c9a38c5273b3ccd5d129fee4ffc1d97772c/uvicorn-0.52.3-py3-none-any.whl", hash = "sha256:116af2710dbf47c80f463cd20ee4884b6662f4c9f227d797ddc7279d2fcc2c7c", size = 79859, upload-time = "2026-08-13T16:50:01.323Z" }, ] [package.optional-dependencies] From 9bca5e50258b0e9dd4a80b08d5e955e947d01bae Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 11:51:00 +0900 Subject: [PATCH 069/117] chore(deps): bump openai from 2.53.0 to 3.0.0 (#1807) Bump openai 2.53.0 -> 3.0.0 (dependabot) plus test fixture migration for the HTTPX2 breaking change. - openai 3.0 defaults to HTTPX2; `AsyncOpenAI(http_client=...)` now expects `httpx2.AsyncClient`. SDK-facing test fixtures wrap the ASGI apps in `httpx2.ASGITransport`; admin setup clients keep httpx v1 (unrelated to the SDK contract). - App code does not import the openai SDK (usage is tests/scripts only), so no runtime surface changes. - Gates: CI green at e1ed45f1 (ty red on the original bump head resolved), mergeable CLEAN, zero review threads. Local: `uv run ty check` clean, targeted pytest 25 passed, `-W error::DeprecationWarning` clean. --- tests/e2e/test_openai_sdk_compat.py | 9 ++- tests/e2e/test_v1_responses_openai_sdk.py | 7 ++- .../integration/test_openai_client_compat.py | 13 ++-- uv.lock | 61 ++++++++++++++++--- 4 files changed, 74 insertions(+), 16 deletions(-) diff --git a/tests/e2e/test_openai_sdk_compat.py b/tests/e2e/test_openai_sdk_compat.py index daee81f098..23d1584511 100644 --- a/tests/e2e/test_openai_sdk_compat.py +++ b/tests/e2e/test_openai_sdk_compat.py @@ -311,14 +311,17 @@ async def sdk_client( if hasattr(result, "__await__"): await result - transport = e2e_client._transport # noqa: SLF001 import httpx + import httpx2 + + transport = e2e_client._transport # noqa: SLF001 + assert isinstance(transport, httpx.ASGITransport) client = openai.AsyncOpenAI( api_key=created["key"], base_url="http://testserver/v1", - http_client=httpx.AsyncClient( - transport=transport, + http_client=httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=transport.app), base_url="http://testserver", ), ) diff --git a/tests/e2e/test_v1_responses_openai_sdk.py b/tests/e2e/test_v1_responses_openai_sdk.py index 0a582a65b1..2f54c8d47d 100644 --- a/tests/e2e/test_v1_responses_openai_sdk.py +++ b/tests/e2e/test_v1_responses_openai_sdk.py @@ -327,12 +327,15 @@ async def sdk_client( result = registry.update(snapshot) if hasattr(result, "__await__"): await result - # Reuse the same ASGITransport that e2e_client built. + # Reuse the same ASGI app that e2e_client built, wrapped for httpx2. + httpx = __import__("httpx") + httpx2 = __import__("httpx2") transport = e2e_client._transport # noqa: SLF001 + assert isinstance(transport, httpx.ASGITransport) client = openai.AsyncOpenAI( api_key=created["key"], base_url="http://testserver/v1", - http_client=__import__("httpx").AsyncClient(transport=transport, base_url="http://testserver"), + http_client=httpx2.AsyncClient(transport=httpx2.ASGITransport(app=transport.app), base_url="http://testserver"), ) yield client await client.close() diff --git a/tests/integration/test_openai_client_compat.py b/tests/integration/test_openai_client_compat.py index 984d3bbee0..d63ec851d2 100644 --- a/tests/integration/test_openai_client_compat.py +++ b/tests/integration/test_openai_client_compat.py @@ -4,6 +4,7 @@ import json import httpx +import httpx2 import openai import pytest from httpx import ASGITransport @@ -52,7 +53,9 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient(transport=transport, base_url="http://testserver/v1") as http_client: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/v1" + ) as http_client: client = openai.AsyncOpenAI(api_key="test", base_url="http://testserver/v1", http_client=http_client) result = await client.responses.create(model="gpt-5.1", input="hi") @@ -91,8 +94,8 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient( - transport=transport, + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/backend-api/codex", ) as http_client: client = openai.AsyncOpenAI( @@ -127,7 +130,9 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await admin_client.post("/api/accounts/import", files=files) assert response.status_code == 200 - async with httpx.AsyncClient(transport=transport, base_url="http://testserver/v1") as http_client: + async with httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=app_instance), base_url="http://testserver/v1" + ) as http_client: client = openai.AsyncOpenAI(api_key="test", base_url="http://testserver/v1", http_client=http_client) result = await client.chat.completions.create( model="gpt-5.2", diff --git a/uv.lock b/uv.lock index e44000d9c3..2f40f62031 100644 --- a/uv.lock +++ b/uv.lock @@ -1121,6 +1121,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] +[[package]] +name = "httpcore2" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "h11" }, + { name = "truststore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/83/a896fc59940fc5a6e2aff3a4be1d92fa890112936803b331cae75a993c34/httpcore2-2.10.0.tar.gz", hash = "sha256:13c0cc3d1919d4f28457f60cd2c2abe04113a8af184ccf1142811beba936f9dc", size = 67427, upload-time = "2026-08-09T09:11:32.123Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/4f/d149104195a35e2853a2fc203a8e3477747e58c80e17dda686dace174383/httpcore2-2.10.0-py3-none-any.whl", hash = "sha256:7df06cfb34070cae4f7c89be69dc1095eca138e9704ceffb98d25c1912ab6f01", size = 83000, upload-time = "2026-08-09T09:11:29.555Z" }, +] + [[package]] name = "httptools" version = "0.8.0" @@ -1165,6 +1178,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx2" +version = "2.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio", marker = "sys_platform != 'emscripten'" }, + { name = "httpcore2", marker = "sys_platform != 'emscripten'" }, + { name = "httpx2-jsfetch", marker = "sys_platform == 'emscripten'" }, + { name = "idna" }, + { name = "truststore", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/3d/f9a8c07a3884f3e5b26205e8436a18b3af61c5d53192c3bea235574dbbec/httpx2-2.10.0.tar.gz", hash = "sha256:8741d7329fe2c7885fc9ceb61c8217acfb87a85f75723714b89ebf7ad7196338", size = 98749, upload-time = "2026-08-09T09:11:33.24Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b9/6d/a637d52449d98a6892d9a4dc0262587afdb6a66f201871842dce5a97b1c1/httpx2-2.10.0-py3-none-any.whl", hash = "sha256:5e3194a432701e1cc6f69a8b1b2fa199ef907013fede8d9a09a2c5b7b8141a18", size = 94355, upload-time = "2026-08-09T09:11:30.882Z" }, +] + +[[package]] +name = "httpx2-jsfetch" +version = "1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/c4/0e5636363151a2a1795e0a77617168b9ca438e1748ec05fc9b5687f93d64/httpx2_jsfetch-1.0.tar.gz", hash = "sha256:70a0e3eabfef7cce5ad9c629f7d01ca05e418f586646f4ddf14782e4c1454c60", size = 6872, upload-time = "2026-08-07T00:13:07.492Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9b/43/832f631d32e4f1211caa2ba368317739fe71f0b8530e4c9d15dc454bac2a/httpx2_jsfetch-1.0-py3-none-any.whl", hash = "sha256:cb916b707601e69a07721aabc8f3f6659be3a6893bc1ff5c6f9e02241df2da32", size = 6382, upload-time = "2026-08-07T00:13:06.567Z" }, +] + [[package]] name = "hypothesis" version = "6.165.8" @@ -1241,11 +1279,11 @@ wheels = [ [[package]] name = "idna" -version = "3.15" +version = "3.18" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/77/7b3966d0b9d1d31a36ddf1746926a11dface89a83409bf1483f0237aa758/idna-3.15.tar.gz", hash = "sha256:ca962446ea538f7092a95e057da437618e886f4d349216d2b1e294abfdb65fdc", size = 199245, upload-time = "2026-05-12T22:45:57.011Z" } +sdist = { url = "https://files.pythonhosted.org/packages/cd/63/9496c57188a2ee585e0f1db071d75089a11e98aa86eb99d9d7618fc1edce/idna-3.18.tar.gz", hash = "sha256:ffb385a7e039654cef1ab9ef32c6fafe283c0c0467bba1d9029738ce4a14a848", size = 196711, upload-time = "2026-06-02T14:34:07.794Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/23/408243171aa9aaba178d3e2559159c24c1171a641aa83b67bdd3394ead8e/idna-3.15-py3-none-any.whl", hash = "sha256:048adeaf8c2d788c40fee287673ccaa74c24ffd8dcf09ffa555a2fbb59f10ac8", size = 72340, upload-time = "2026-05-12T22:45:55.733Z" }, + { url = "https://files.pythonhosted.org/packages/1e/5e/d4e9f1a599fb8e573b7b87160658329fbf28d19eac2718f51fc3def3aa5a/idna-3.18-py3-none-any.whl", hash = "sha256:7f952cbe720b688055e3f87de14f5c3e5fdaa8bc3928985c4077ca689de849a2", size = 65455, upload-time = "2026-06-02T14:34:06.319Z" }, ] [[package]] @@ -1587,21 +1625,21 @@ wheels = [ [[package]] name = "openai" -version = "2.53.0" +version = "3.0.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, { name = "distro" }, - { name = "httpx" }, + { name = "httpx2" }, { name = "jiter" }, { name = "pydantic" }, { name = "sniffio" }, { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/cf/36e3e7235fdf6d125c052acc0970924611b17a20a4fe580596faf4566a65/openai-2.53.0.tar.gz", hash = "sha256:baf5802ad08980e1d9d561e1b996e800c8bcd14af5847c6d0e7a5cc59e4d4116", size = 1099435, upload-time = "2026-08-03T21:42:01.664Z" } +sdist = { url = "https://files.pythonhosted.org/packages/54/8c/2f500e8be09d1ae98c530467962535198b02cd4550cd418bbbaedc8b2910/openai-3.0.0.tar.gz", hash = "sha256:ffd00ef1678d70957e1f1ed98d5bfcf1d661f41ea4482f22e7d0144a66435a49", size = 1123740, upload-time = "2026-08-12T01:55:50.849Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/0f/cc6afea3542a5142c5d8fc8211c5e059a8375105d004a41dfa2c7948dbb0/openai-2.53.0-py3-none-any.whl", hash = "sha256:c694ffc747a3c4d1663ef2b07b811315a476164ee5efa3a993967349ebca7618", size = 1659829, upload-time = "2026-08-03T21:41:59.581Z" }, + { url = "https://files.pythonhosted.org/packages/7b/0d/9850e7eddb5e66da4439ed503e78e09ad1fd0195e6df51e4236c75763581/openai-3.0.0-py3-none-any.whl", hash = "sha256:8d32ac3a6647a66910d6cb8a64f0fa5a6c823604b6e82db83d9d055c6709bd51", size = 1665775, upload-time = "2026-08-12T01:55:48.678Z" }, ] [[package]] @@ -2552,6 +2590,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, ] +[[package]] +name = "truststore" +version = "0.10.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/53/a3/1585216310e344e8102c22482f6060c7a6ea0322b63e026372e6dcefcfd6/truststore-0.10.4.tar.gz", hash = "sha256:9d91bd436463ad5e4ee4aba766628dd6cd7010cf3e2461756b3303710eebc301", size = 26169, upload-time = "2025-08-12T18:49:02.73Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/19/97/56608b2249fe206a67cd573bc93cd9896e1efb9e98bce9c163bcdc704b88/truststore-0.10.4-py3-none-any.whl", hash = "sha256:adaeaecf1cbb5f4de3b1959b42d41f6fab57b2b1666adb59e89cb0b53361d981", size = 18660, upload-time = "2025-08-12T18:49:01.46Z" }, +] + [[package]] name = "ty" version = "0.0.72" From de7b3bccfcfe22c4eaeb8877db7c47dc755a4eb4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:13:39 +0900 Subject: [PATCH 070/117] chore(docker): bump astral-sh/uv from 0.12.3 to 0.12.5 (#1803) Bumps [astral-sh/uv](https://github.com/astral-sh/uv) from 0.12.3 to 0.12.5. - [Release notes](https://github.com/astral-sh/uv/releases) - [Changelog](https://github.com/astral-sh/uv/blob/main/CHANGELOG.md) - [Commits](https://github.com/astral-sh/uv/compare/0.12.3...0.12.5) --- updated-dependencies: - dependency-name: astral-sh/uv dependency-version: 0.12.5 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- Dockerfile.distroless | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index b5edd041c7..a7387c6998 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.7 -FROM ghcr.io/astral-sh/uv:0.12.3 AS uv-bin +FROM ghcr.io/astral-sh/uv:0.12.5 AS uv-bin FROM oven/bun:1.3.14-alpine AS frontend-build diff --git a/Dockerfile.distroless b/Dockerfile.distroless index 5b34691f0e..a3e61efad8 100644 --- a/Dockerfile.distroless +++ b/Dockerfile.distroless @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1.7 -FROM ghcr.io/astral-sh/uv:0.12.3 AS uv-bin +FROM ghcr.io/astral-sh/uv:0.12.5 AS uv-bin FROM oven/bun:1.3.14-alpine AS frontend-build From a0406bade3e1ca3833cd371aea5ffdcefc2de7dc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:14:02 +0900 Subject: [PATCH 071/117] chore(deps): bump the frontend-minor-patch group (#1804) Bumps the frontend-minor-patch group in /frontend with 12 updates: | Package | From | To | | --- | --- | --- | | [@hookform/resolvers](https://github.com/react-hook-form/resolvers) | `5.7.1` | `5.8.0` | | [lucide-react](https://github.com/lucide-icons/lucide/tree/HEAD/packages/lucide-react) | `1.30.0` | `1.31.0` | | [react-hook-form](https://github.com/react-hook-form/react-hook-form) | `7.84.0` | `7.85.0` | | [sonner](https://github.com/emilkowalski/sonner) | `2.0.7` | `2.0.8` | | [zustand](https://github.com/pmndrs/zustand) | `5.0.14` | `5.0.15` | | [@testing-library/jest-dom](https://github.com/testing-library/jest-dom) | `7.0.0` | `7.0.1` | | [@testing-library/user-event](https://github.com/testing-library/user-event) | `14.6.3` | `14.6.4` | | [eslint-plugin-react-refresh](https://github.com/ArnaudBarre/eslint-plugin-react-refresh) | `0.5.3` | `0.5.4` | | [globals](https://github.com/sindresorhus/globals) | `17.9.0` | `17.11.0` | | [react-doctor](https://github.com/millionco/react-doctor/tree/HEAD/packages/react-doctor) | `0.9.6` | `0.9.12` | | [shadcn](https://github.com/shadcn-ui/ui/tree/HEAD/packages/shadcn) | `4.16.2` | `4.18.0` | | [typescript-eslint](https://github.com/typescript-eslint/typescript-eslint/tree/HEAD/packages/typescript-eslint) | `8.66.0` | `8.67.0` | Updates `@hookform/resolvers` from 5.7.1 to 5.8.0 - [Release notes](https://github.com/react-hook-form/resolvers/releases) - [Commits](https://github.com/react-hook-form/resolvers/compare/v5.7.1...v5.8.0) Updates `lucide-react` from 1.30.0 to 1.31.0 - [Release notes](https://github.com/lucide-icons/lucide/releases) - [Commits](https://github.com/lucide-icons/lucide/commits/1.31.0/packages/lucide-react) Updates `react-hook-form` from 7.84.0 to 7.85.0 - [Release notes](https://github.com/react-hook-form/react-hook-form/releases) - [Changelog](https://github.com/react-hook-form/react-hook-form/blob/master/CHANGELOG.md) - [Commits](https://github.com/react-hook-form/react-hook-form/compare/v7.84.0...v7.85.0) Updates `sonner` from 2.0.7 to 2.0.8 - [Release notes](https://github.com/emilkowalski/sonner/releases) - [Commits](https://github.com/emilkowalski/sonner/compare/v2.0.7...v2.0.8) Updates `zustand` from 5.0.14 to 5.0.15 - [Release notes](https://github.com/pmndrs/zustand/releases) - [Commits](https://github.com/pmndrs/zustand/compare/v5.0.14...v5.0.15) Updates `@testing-library/jest-dom` from 7.0.0 to 7.0.1 - [Release notes](https://github.com/testing-library/jest-dom/releases) - [Changelog](https://github.com/testing-library/jest-dom/blob/main/CHANGELOG.md) - [Commits](https://github.com/testing-library/jest-dom/compare/v7.0.0...v7.0.1) Updates `@testing-library/user-event` from 14.6.3 to 14.6.4 - [Release notes](https://github.com/testing-library/user-event/releases) - [Changelog](https://github.com/testing-library/user-event/blob/main/CHANGELOG.md) - [Commits](https://github.com/testing-library/user-event/compare/v14.6.3...v14.6.4) Updates `eslint-plugin-react-refresh` from 0.5.3 to 0.5.4 - [Release notes](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/releases) - [Changelog](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/blob/main/CHANGELOG.md) - [Commits](https://github.com/ArnaudBarre/eslint-plugin-react-refresh/compare/v0.5.3...v0.5.4) Updates `globals` from 17.9.0 to 17.11.0 - [Release notes](https://github.com/sindresorhus/globals/releases) - [Commits](https://github.com/sindresorhus/globals/compare/v17.9.0...v17.11.0) Updates `react-doctor` from 0.9.6 to 0.9.12 - [Release notes](https://github.com/millionco/react-doctor/releases) - [Changelog](https://github.com/millionco/react-doctor/blob/main/packages/react-doctor/CHANGELOG.md) - [Commits](https://github.com/millionco/react-doctor/commits/react-doctor@0.9.12/packages/react-doctor) Updates `shadcn` from 4.16.2 to 4.18.0 - [Release notes](https://github.com/shadcn-ui/ui/releases) - [Changelog](https://github.com/shadcn-ui/ui/blob/main/packages/shadcn/CHANGELOG.md) - [Commits](https://github.com/shadcn-ui/ui/commits/shadcn@4.18.0/packages/shadcn) Updates `typescript-eslint` from 8.66.0 to 8.67.0 - [Release notes](https://github.com/typescript-eslint/typescript-eslint/releases) - [Changelog](https://github.com/typescript-eslint/typescript-eslint/blob/main/packages/typescript-eslint/CHANGELOG.md) - [Commits](https://github.com/typescript-eslint/typescript-eslint/commits/v8.67.0/packages/typescript-eslint) --- updated-dependencies: - dependency-name: "@hookform/resolvers" dependency-version: 5.8.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: frontend-minor-patch - dependency-name: lucide-react dependency-version: 1.31.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: frontend-minor-patch - dependency-name: react-hook-form dependency-version: 7.85.0 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: frontend-minor-patch - dependency-name: sonner dependency-version: 2.0.8 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: zustand dependency-version: 5.0.15 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: "@testing-library/jest-dom" dependency-version: 7.0.1 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: "@testing-library/user-event" dependency-version: 14.6.4 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: eslint-plugin-react-refresh dependency-version: 0.5.4 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: globals dependency-version: 17.11.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: frontend-minor-patch - dependency-name: react-doctor dependency-version: 0.9.12 dependency-type: direct:development update-type: version-update:semver-patch dependency-group: frontend-minor-patch - dependency-name: shadcn dependency-version: 4.18.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: frontend-minor-patch - dependency-name: typescript-eslint dependency-version: 8.67.0 dependency-type: direct:development update-type: version-update:semver-minor dependency-group: frontend-minor-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- frontend/bun.lock | 180 ++++++++++++++++++++++-------------------- frontend/package.json | 24 +++--- 2 files changed, 105 insertions(+), 99 deletions(-) diff --git a/frontend/bun.lock b/frontend/bun.lock index f695b9ea44..dd14d41e7c 100644 --- a/frontend/bun.lock +++ b/frontend/bun.lock @@ -5,7 +5,7 @@ "": { "name": "frontend", "dependencies": { - "@hookform/resolvers": "^5.7.1", + "@hookform/resolvers": "^5.8.0", "@tailwindcss/vite": "^4.3.3", "@tanstack/react-query": "^5.101.4", "class-variance-authority": "^0.7.1", @@ -14,27 +14,27 @@ "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", "input-otp": "^1.4.2", - "lucide-react": "^1.30.0", + "lucide-react": "^1.31.0", "radix-ui": "^1.6.7", "react": "^19.2.8", "react-day-picker": "^10.0.1", "react-dom": "^19.2.8", - "react-hook-form": "^7.84.0", + "react-hook-form": "^7.85.0", "react-i18next": "^17.0.11", "react-router-dom": "^7.18.2", "recharts": "^3.10.1", - "sonner": "^2.0.7", + "sonner": "^2.0.8", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.3", "zod": "^4.4.3", - "zustand": "^5.0.14", + "zustand": "^5.0.15", }, "devDependencies": { "@eslint/js": "^10.0.1", "@playwright/test": "^1.62.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.3", + "@testing-library/user-event": "^14.6.4", "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", @@ -43,15 +43,15 @@ "@vitest/coverage-v8": "^4.1.10", "eslint": "^10.8.1", "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.9.0", + "eslint-plugin-react-refresh": "^0.5.4", + "globals": "^17.11.0", "jsdom": "^30.0.1", "msw": "^2.15.0", - "react-doctor": "^0.9.6", - "shadcn": "^4.16.2", + "react-doctor": "^0.9.12", + "shadcn": "^4.18.0", "tw-animate-css": "^1.4.0", "typescript": "npm:typescript@~6.0.3", - "typescript-eslint": "^8.66.0", + "typescript-eslint": "^8.67.0", "vite": "^8.2.1", "vitest": "^4.1.10", }, @@ -180,7 +180,7 @@ "@hono/node-server": ["@hono/node-server@1.19.9", "", { "peerDependencies": { "hono": "^4" } }, "sha512-vHL6w3ecZsky+8P5MD+eFfaGTyCeOHUIFYMGpQGbrBTSmNNoxv0if69rEZ5giu36weC5saFuznL411gRX7bJDw=="], - "@hookform/resolvers": ["@hookform/resolvers@5.7.1", "", { "dependencies": { "@standard-schema/utils": "^0.3.0" }, "peerDependencies": { "@sinclair/typebox": ">=0.25.24", "@standard-schema/spec": "^1.0.0", "@typeschema/main": ">=0.13.7", "@vinejs/vine": "^2.0.0 || ^3.0.0 || ^4.0.0", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", "ajv-formats": "^2.1.1", "arktype": "^2.0.0", "ata-validator": "^1.2.0", "class-transformer": ">=0.4.0", "class-validator": ">=0.12.0", "computed-types": "^1.0.0", "effect": "^3.10.3", "fluentvalidation-ts": "^3.0.0", "fp-ts": "^2.7.0", "io-ts": "^2.0.0", "joi": "^17.0.0", "nope-validator": ">=0.12.0", "react-hook-form": "^7.55.0", "superstruct": ">=0.12.0", "typanion": "^3.3.2", "valibot": ">=0.31.0 || ^1.0.0-beta.4 || ^1.0.0-rc", "vest": ">=3.0.0", "yup": "^1.0.0", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@sinclair/typebox", "@standard-schema/spec", "@typeschema/main", "@vinejs/vine", "ajv", "ajv-errors", "ajv-formats", "arktype", "ata-validator", "class-transformer", "class-validator", "computed-types", "effect", "fluentvalidation-ts", "fp-ts", "io-ts", "joi", "nope-validator", "superstruct", "typanion", "valibot", "vest", "yup", "zod"] }, "sha512-8wS/P4UDr5sQDe4nFaV51TVyfDPrWgNIXweqG0Bs9Z5LSuzKLb+RQNPvkN2oHM5SRrJyWrVH/F+LOUcFjUyvwQ=="], + "@hookform/resolvers": ["@hookform/resolvers@5.8.0", "", { "dependencies": { "@standard-schema/utils": "^0.3.0" }, "peerDependencies": { "@sinclair/typebox": ">=0.25.24", "@standard-schema/spec": "^1.0.0", "@typeschema/main": ">=0.13.7", "@vinejs/vine": "^2.0.0 || ^3.0.0 || ^4.0.0", "ajv": "^8.12.0", "ajv-errors": "^3.0.0", "ajv-formats": "^2.1.1", "arktype": "^2.0.0", "ata-validator": "^1.2.0", "class-transformer": ">=0.4.0", "class-validator": ">=0.12.0", "computed-types": "^1.0.0", "effect": "^3.10.3", "fluentvalidation-ts": "^3.0.0", "fp-ts": "^2.7.0", "io-ts": "^2.0.0", "joi": "^17.0.0", "nope-validator": ">=0.12.0", "react-hook-form": "^7.55.0", "superstruct": ">=0.12.0", "typanion": "^3.3.2", "valibot": ">=0.31.0 || ^1.0.0-beta.4 || ^1.0.0-rc", "vest": ">=6.0.0", "yup": "^1.0.0", "zod": "^3.25.0 || ^4.0.0" }, "optionalPeers": ["@sinclair/typebox", "@standard-schema/spec", "@typeschema/main", "@vinejs/vine", "ajv", "ajv-errors", "ajv-formats", "arktype", "ata-validator", "class-transformer", "class-validator", "computed-types", "effect", "fluentvalidation-ts", "fp-ts", "io-ts", "joi", "nope-validator", "superstruct", "typanion", "valibot", "vest", "yup", "zod"] }, "sha512-2m6GvRLmYYK1Fwt093lGMf7db9l/+8pNuAtwoNkpBntJT4xcA5lNthYGWKViOc3z2SuaPD0HjE81pyXmqc1JyA=="], "@humanfs/core": ["@humanfs/core@0.19.1", "", {}, "sha512-5DyQ4+1JEUzejeK1JGICcideyfUbGixgS9jNgex5nqkW+cY7WZhxBigmieN5Qnw9ZosSNVC9KQKyb+GUaGyKUA=="], @@ -254,47 +254,45 @@ "@opentelemetry/semantic-conventions": ["@opentelemetry/semantic-conventions@1.41.1", "", {}, "sha512-/UhIkaZgPutTFmQ7RnIJGgDXZmtEJ7Dvi86xNTFWcnRxVRNk/aotsqDJYeEvDP+FSMB2SdW+pQzNMcWP0rwuNA=="], - "@oxc-parser/binding-android-arm-eabi": ["@oxc-parser/binding-android-arm-eabi@0.142.0", "", { "os": "android", "cpu": "arm" }, "sha512-ZiRGDutGsv1G6bL/ozy/koC0Sv39T1DqyoC4KD1DOy9ZoACm1O5UWhEK2c02Qdk+4lfLVkvFa/mQ0fm/4h1BtQ=="], + "@oxc-parser/binding-android-arm-eabi": ["@oxc-parser/binding-android-arm-eabi@0.143.0", "", { "os": "android", "cpu": "arm" }, "sha512-n9uozULWflPqBtdmI8lAabLqGKNgLVNN0ZH8HfgCwpKGNtzRzauB76jTiW/3YLkcA7N1zskpi9GdVnZuu1SAvg=="], - "@oxc-parser/binding-android-arm64": ["@oxc-parser/binding-android-arm64@0.142.0", "", { "os": "android", "cpu": "arm64" }, "sha512-WZkvGRLNQTz8lR9zP5nLjUdlroRCopBu3g9zF1p/laE6DzT1UbQo8Rdz5MWhaJUPYg/6gp+jo7HUgsyKaN1FtQ=="], + "@oxc-parser/binding-android-arm64": ["@oxc-parser/binding-android-arm64@0.143.0", "", { "os": "android", "cpu": "arm64" }, "sha512-9BbdjHETk6O3zH/DDid9IgBtF0GlpLabNKN231uraXpRDSfY+iiZxTP5bk1Z63GBownVdhdINFIeddmMz4MzpQ=="], - "@oxc-parser/binding-darwin-arm64": ["@oxc-parser/binding-darwin-arm64@0.142.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-l4khS8LQOOVYsGRVARo1gSaCT/aBSceUVXgtovWc2+drnxVuDr082WA3OCHVdVzIz5JIrP/y9CWsSKxBDNmYGg=="], + "@oxc-parser/binding-darwin-arm64": ["@oxc-parser/binding-darwin-arm64@0.143.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-gh+6ecoHUy4/sUcolBl/1qPXKBbYNxFY0Pk0ujgQvINTMSftJY7o4yb8gOkDJPeZeB8+a+u7xTe6umoP8N5HFA=="], - "@oxc-parser/binding-darwin-x64": ["@oxc-parser/binding-darwin-x64@0.142.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-QBsNF3nqlXmcH2B1YOPqQYmCJoy4HuIjUxGbBO/k5JAJUl68ghU2psRY2zPk+RyBaWqKP/qfL4oaFgEMCdwskA=="], + "@oxc-parser/binding-darwin-x64": ["@oxc-parser/binding-darwin-x64@0.143.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-qd1hl2d+lXgHv/VQ/M9qm8TrMC5T4RqDBwtOnl+1D0QMjwcz+8AaB4JSg8STgeag0GP6a6L74XEGAsrTSJWNzQ=="], - "@oxc-parser/binding-freebsd-x64": ["@oxc-parser/binding-freebsd-x64@0.142.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-b7Q7m4Cqc6XqNhri3R+QhU+GVy646Pn+bkdhrDdWym/Fdi0ZUa+d73H9dm5H91JtbtAQ/z1d8XKMW3oOV8a4tQ=="], + "@oxc-parser/binding-freebsd-x64": ["@oxc-parser/binding-freebsd-x64@0.143.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-M5XXcNa7aOqLPKTR41msfghKu2yQ4xWvCm11/gwU0JzOzHNk5sgW//rVEjJ+LO48+VDAMzXTSzurUVxIDKwozw=="], - "@oxc-parser/binding-linux-arm-gnueabihf": ["@oxc-parser/binding-linux-arm-gnueabihf@0.142.0", "", { "os": "linux", "cpu": "arm" }, "sha512-3riVS5IhdH3uCZj1Y9ftDQlR0dvLsIlw/edrRqk8JhgNd5K0XSs+UBtgh50N13CAlW9/TXj6sVGXaKNBocd0Yg=="], + "@oxc-parser/binding-linux-arm-gnueabihf": ["@oxc-parser/binding-linux-arm-gnueabihf@0.143.0", "", { "os": "linux", "cpu": "arm" }, "sha512-T/GXusuOkPNQhCQCSBbcU/N8j0rAypuDBl1IyFK+lyYT594XsVz80clPC/OtbSSpBGyJxj8uYEfctxVuxVYoww=="], - "@oxc-parser/binding-linux-arm-musleabihf": ["@oxc-parser/binding-linux-arm-musleabihf@0.142.0", "", { "os": "linux", "cpu": "arm" }, "sha512-NmXUOpgpTSkhl795TiXmWppTwmSJ92RC1qvD6e4XOF+slgmo3e6Ah+kEu+6AN8s7NAOEwqGmir58MgSQSWmBSA=="], + "@oxc-parser/binding-linux-arm-musleabihf": ["@oxc-parser/binding-linux-arm-musleabihf@0.143.0", "", { "os": "linux", "cpu": "arm" }, "sha512-oKu4RcBlXSqo3OC62dp6YTnQaZIurNDpCX3BnAM3+bJxt7s8J2TJKMnC0UYer1qhlRaDCg6wkTaTw+2IlsZ12w=="], - "@oxc-parser/binding-linux-arm64-gnu": ["@oxc-parser/binding-linux-arm64-gnu@0.142.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-gc0EXsKtXgerujmU2Bql3u1L1HsSQ2774R83idq/FoNMPVV/RY/1ErFsvnit7KoiP/sLvzQixeUo4Ut0ic0wmw=="], + "@oxc-parser/binding-linux-arm64-gnu": ["@oxc-parser/binding-linux-arm64-gnu@0.143.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-WJBbD186AZmMGaSIhlktC+rPl8L3peCTXAh88Ih9uEvK0en2mPojGyCGYiL6mHtV1RPV3JyfJW5t6n5hh0lXhA=="], - "@oxc-parser/binding-linux-arm64-musl": ["@oxc-parser/binding-linux-arm64-musl@0.142.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-F2XvmWSE0uWpie+jHKKIFgdVOe9ypGhkEZxKx5DuW215K6cbAC274yYaPkcM7EqY4Df3Weyhpcz3lsURyH2LVg=="], + "@oxc-parser/binding-linux-arm64-musl": ["@oxc-parser/binding-linux-arm64-musl@0.143.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-t1AcYOwEzgceadT4v5e+vaCCb0AncCA3v5AyzfBAz/tMq11qzVccXKzNHtkWdjBsgvTKwRkaUF3QvT4kot8vcQ=="], - "@oxc-parser/binding-linux-ppc64-gnu": ["@oxc-parser/binding-linux-ppc64-gnu@0.142.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-wLMbT21U/QxknQsk+VvNF0b9D2/aGWhcaQQQ+VYlE8FwD5+GoWZIPPXNzyHmkYyhm0KB3itL+TBavjMatqNnYA=="], + "@oxc-parser/binding-linux-ppc64-gnu": ["@oxc-parser/binding-linux-ppc64-gnu@0.143.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-RsnO/NoD8376LMJq8JS8TwI0ieNaFRTuNe2GVJntQg6gwZNMENZsEbknHdVwjpOmxdGLGodcwaGSbAeRr5Bgjw=="], - "@oxc-parser/binding-linux-riscv64-gnu": ["@oxc-parser/binding-linux-riscv64-gnu@0.142.0", "", { "os": "linux", "cpu": "none" }, "sha512-+G8F/4ckwT7FCJV4H2bt09xEzJbjNCfuL4Sp1AYNaFtFMVtgIGMuJlteT82U+K0UIZ/DzAR/LDlMFnEuajG7Kw=="], + "@oxc-parser/binding-linux-riscv64-gnu": ["@oxc-parser/binding-linux-riscv64-gnu@0.143.0", "", { "os": "linux", "cpu": "none" }, "sha512-48fSVfR9TZi5CASZFyv0VC6z6BCoeihFsX031mAD/oSH7d9PYsPgIqza7d9mjP7Z2KTEpTFyH6SIu0Ui6R1vdg=="], - "@oxc-parser/binding-linux-riscv64-musl": ["@oxc-parser/binding-linux-riscv64-musl@0.142.0", "", { "os": "linux", "cpu": "none" }, "sha512-hTsHtTLxMAfCo+rpF5K3qZJKW2NpPN/CHd4mYB3y7XlSdspHkd2gehDIofP64AacA9nWQw2tY3O7wR6UY8IVOA=="], + "@oxc-parser/binding-linux-riscv64-musl": ["@oxc-parser/binding-linux-riscv64-musl@0.143.0", "", { "os": "linux", "cpu": "none" }, "sha512-T8CpdD+SfE01DnIOD4HpVxu0ZJOfMJ/VhCvikKfaXAxkZ+9veyLM/D2hpi7Y2hFUyPmVQO3FNZHmYzV/WlVR4g=="], - "@oxc-parser/binding-linux-s390x-gnu": ["@oxc-parser/binding-linux-s390x-gnu@0.142.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-6y7qYY3TCUDYjqswImdTGl92y+KA/80twALegQPN27kfY+bG7Ib1+L3jbmrCZQx6wrVnai9IPsEZp07I0hx7JQ=="], + "@oxc-parser/binding-linux-s390x-gnu": ["@oxc-parser/binding-linux-s390x-gnu@0.143.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-QLdeMsCcacenPEFsfxnBUDF1y6opyz5+fmOz9bfD5Y7fiGCMupUCuB3KTPQhNwshIG1P9fPqar9MHxuBDd4bwQ=="], - "@oxc-parser/binding-linux-x64-gnu": ["@oxc-parser/binding-linux-x64-gnu@0.142.0", "", { "os": "linux", "cpu": "x64" }, "sha512-i69kAWU+2LgoH5bR+zWiiu+UzAw7Oxkwv7COeJTeY19pn4e70nKQcr9Pm6cL2Z0Z54d+gl9qADlK/0yyuCPiBA=="], + "@oxc-parser/binding-linux-x64-gnu": ["@oxc-parser/binding-linux-x64-gnu@0.143.0", "", { "os": "linux", "cpu": "x64" }, "sha512-659ujfqLy6k7cuH3sbzhd8b+ztSq+i6E2E9pG78Q0BmHjAExfGIdgc8cGgMdwAozDXeZFHkJ+LXYJdWsaGdgyw=="], - "@oxc-parser/binding-linux-x64-musl": ["@oxc-parser/binding-linux-x64-musl@0.142.0", "", { "os": "linux", "cpu": "x64" }, "sha512-4SQs678MmjYVrmhAgCWD4o0vpaFszXw9xLX5p2Z9MMFcltxiLkA88wQjh80YHjPrXtpyZ2CWI5m+1yNKM0m2Pw=="], + "@oxc-parser/binding-linux-x64-musl": ["@oxc-parser/binding-linux-x64-musl@0.143.0", "", { "os": "linux", "cpu": "x64" }, "sha512-/Mw/9j4TfZcnKphPrzOE6t4MMknXadcAAuVUlDRTF/ETWB5xOgQvOJV2Mh9We/bWxZdoxaGAdc+hy4GuYwQ2yQ=="], - "@oxc-parser/binding-openharmony-arm64": ["@oxc-parser/binding-openharmony-arm64@0.142.0", "", { "os": "none", "cpu": "arm64" }, "sha512-YHpx9N7Ln3a++Tc8rv+H7mrK1zyJQOAwCFg8LZ3lTs1T5afGWeZrLPhPT9HLnIwSjCyJqPWVMIrMxbjcmBr2oQ=="], + "@oxc-parser/binding-openharmony-arm64": ["@oxc-parser/binding-openharmony-arm64@0.143.0", "", { "os": "none", "cpu": "arm64" }, "sha512-8rIKWR2BFuifbIK/1XB9wTaSdtuJ25dlE7ZQYDnEwj/2xH2vHsxnvIjHT3ZjSVuLLwGGlSslIG/fbOJ8TV8rTw=="], - "@oxc-parser/binding-wasm32-wasi": ["@oxc-parser/binding-wasm32-wasi@0.142.0", "", { "dependencies": { "@emnapi/core": "1.11.2", "@emnapi/runtime": "1.11.2", "@napi-rs/wasm-runtime": "^1.1.6" }, "cpu": "none" }, "sha512-3pLDyY3+oogW73RM5uehNgAiR/Xfb7fvO2Q1Z1gIqZ2+50XDVQmBVlRkHXZTU4gKnQHpwETNsYQVsJ3joVB2iA=="], + "@oxc-parser/binding-win32-arm64-msvc": ["@oxc-parser/binding-win32-arm64-msvc@0.143.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-5U9kQYMfRRI6Zq7KDxgbIP0RMnKrfn3gLepRMgJuRkPSUALTiRCk9d/uyhb4lGDjUdzwK7mBkKqhLgzBPCmLpQ=="], - "@oxc-parser/binding-win32-arm64-msvc": ["@oxc-parser/binding-win32-arm64-msvc@0.142.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Had/VeVY28Oyb0K+Q4FV8KCzoBycIh93oDK6pCbya9lkzdq+ikMHMgBubsdqqlybjJmQRawCQRrnBRHyQwYvcQ=="], + "@oxc-parser/binding-win32-ia32-msvc": ["@oxc-parser/binding-win32-ia32-msvc@0.143.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-25P7AaHk4R88Yv2XH4gToDVmh0cOu+bEURQU10CRrmvgabfRArSGAP5osmwUKeSUHj0VS50upbpbRWWW/m7mHA=="], - "@oxc-parser/binding-win32-ia32-msvc": ["@oxc-parser/binding-win32-ia32-msvc@0.142.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-GGi3+YphVHavvgs6gum2UXoNCqzHAmPt/nXkn8ZQZstV2Q1qZD1Mn8fz/nWrDkefHQtrG/+1/XrbMxsBTo6Svw=="], + "@oxc-parser/binding-win32-x64-msvc": ["@oxc-parser/binding-win32-x64-msvc@0.143.0", "", { "os": "win32", "cpu": "x64" }, "sha512-ORMh3JE1s6V7ySicdRK7vgaDQnn5o+UHg9ct989PlWHbel8O9ARrmWXM6kZjrBMtNucxNayQ8g69G0VfWzhANw=="], - "@oxc-parser/binding-win32-x64-msvc": ["@oxc-parser/binding-win32-x64-msvc@0.142.0", "", { "os": "win32", "cpu": "x64" }, "sha512-Ny/Wv4Us1LGC/ljwNTp+Hx3r/pH15EFfeDF0p+n898gt+TtRd6C9SccHcuUhDiNTb8s5tt7jdeAMDRQZ4Vq6hg=="], - - "@oxc-project/types": ["@oxc-project/types@0.142.0", "", {}, "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ=="], + "@oxc-project/types": ["@oxc-project/types@0.143.0", "", {}, "sha512-u6JZdLBTLotrNC9Vd6vPssINdzcCzleKAH6EJKImQb7GtYvX5keN2dxkoK44stCc4tffE6QQRtZTXVSzsLUlWA=="], "@oxc-resolver/binding-android-arm-eabi": ["@oxc-resolver/binding-android-arm-eabi@11.24.2", "", { "os": "android", "cpu": "arm" }, "sha512-y09e0L0SRI2OA2tUIrjBgoV3eH5hvUKXNkJqXmNo5V2WxIjyC7I7aJfRLMEVpA8yi95f90gFDvO0VMgrDw+vwA=="], @@ -334,43 +332,43 @@ "@oxc-resolver/binding-win32-x64-msvc": ["@oxc-resolver/binding-win32-x64-msvc@11.24.2", "", { "os": "win32", "cpu": "x64" }, "sha512-UqGPmo56KDfLlfXFAFIrNflHT8tFxWGEivWg3Zeyp4Uy2NlKN1FGPr6/BxcLGG3+kZ6Wp14g5Uj+n71boqZfiw=="], - "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.76.0", "", { "os": "android", "cpu": "arm" }, "sha512-ZHIE5Zt9AsPDcY4nOlofXt0YfneEeo+QrKMPcPzLf2Z6Q8VtV2W73d7SFJ920WUwyik783u/doKCs3KXdwG+7w=="], + "@oxlint/binding-android-arm-eabi": ["@oxlint/binding-android-arm-eabi@1.77.0", "", { "os": "android", "cpu": "arm" }, "sha512-E06sKWS6PiI6HRxS1wyQg22HvApt01hI7fV+T3wUk3OSbaaP4a3hYGY/MIQDmASqCiRjBdpRQYkgMkqH82cWmQ=="], - "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.76.0", "", { "os": "android", "cpu": "arm64" }, "sha512-shm/ngQilHK6bs+ElJWa4oHfNj5vL1Gl/iVEJldTQjpr0/67oSgr0KUpbmcnLig5Fo0v/l6j2567A7TOL89ONA=="], + "@oxlint/binding-android-arm64": ["@oxlint/binding-android-arm64@1.77.0", "", { "os": "android", "cpu": "arm64" }, "sha512-NvsKz0KZxTp9cYWPLf+FXaSZwB3oO3peAjtukpOMBgse2vhQSoIIVqeO1yR0lEo/UcdZIDL18uq+kL0LzQ0ytA=="], - "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.76.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-rvJmrAPKSQ9aWJ6wIS6CK2tJjwzfW0ApQH9qokq6sfDvmHwoyIHxHFMq7z7i7GiV6fdE6s8qvBqWKPTu8RmT6Q=="], + "@oxlint/binding-darwin-arm64": ["@oxlint/binding-darwin-arm64@1.77.0", "", { "os": "darwin", "cpu": "arm64" }, "sha512-bgjTn6nW4bQCFBvSvuHCpDD+sONvmpo4lGI4PxzMt1quBA+xYxhczk6RiCn3GZ9gY8uhaBbwhj9MdKGfu6T9DA=="], - "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.76.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-U/zYdb7VYKGY6pA9Vd2rYl9O/HlCylcOlb5PGPvVLtg+oLGsk6H3XGKEMHKyqD3nmmtmlmwb/8SwU2vfSAtvMw=="], + "@oxlint/binding-darwin-x64": ["@oxlint/binding-darwin-x64@1.77.0", "", { "os": "darwin", "cpu": "x64" }, "sha512-aotaIttH1R6j1Rwhx0M0htgeZyGtVQqYNTVEYMN/UcgHPquGA6kmk9OyuDc3a2GKUQBC+3C3GVQCcrRPMYqAFA=="], - "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.76.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-WvKG9CAriuo0XNiFzpXjDngUZcRGFNpaK2kLyMUsnJlShxkT96u+BpJQ3KqdQwGOrvI14L6V8bAwXwAYNNY6Jg=="], + "@oxlint/binding-freebsd-x64": ["@oxlint/binding-freebsd-x64@1.77.0", "", { "os": "freebsd", "cpu": "x64" }, "sha512-nNx/wta7ksRAdYvq+l4AWjXkLxEXHALhENxjj2cYbQAIR4ybaA5L+hCbE63HOmft5czQ6ks+hb8vmEAnn7YGPg=="], - "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.76.0", "", { "os": "linux", "cpu": "arm" }, "sha512-qJ5+RH99TqFRq3UCDxkW0zJJu9c+OAHFY72vGlxZLEpuO+MpKo3POgqb8sYipL9KYm8XY6ofb0HsOuvY6hQNqQ=="], + "@oxlint/binding-linux-arm-gnueabihf": ["@oxlint/binding-linux-arm-gnueabihf@1.77.0", "", { "os": "linux", "cpu": "arm" }, "sha512-tMLLjM7xXtzXisVCzkOTXNCy9bZVId2wteNwjohlFDR/jY6WagpEDA1c1wu4xRc20Hojaxj+V6DSR7gbKxijWA=="], - "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.76.0", "", { "os": "linux", "cpu": "arm" }, "sha512-PvPCVptkgVARsucgIqFQQcSmJ6xc6GtnVB5bRBekRahTc9eObMtjHfMjy5M+C2tHt5UCMttWM9RuSk/H9NqYeg=="], + "@oxlint/binding-linux-arm-musleabihf": ["@oxlint/binding-linux-arm-musleabihf@1.77.0", "", { "os": "linux", "cpu": "arm" }, "sha512-MiAFDFaqR0tmHTAyo0YDcZ5hyLREdYw/RQhc2R3cbT+8O3tB+zqPM2th9TTQ+Uo3jn/embS+DO+HyX9ztCPkOQ=="], - "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.76.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-3KeFDx8Bu4HPAXbuHZOr/oHvN+QT+JQhMw/NYPz7Z071xLSsG27Jfh9PIQVEY7hk1I+jr43ExqRIeJ6VKk2yLw=="], + "@oxlint/binding-linux-arm64-gnu": ["@oxlint/binding-linux-arm64-gnu@1.77.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-/xqQ3B16i1T4cyt/9Mn+4CpzhUXoBXp7kVpIwzOXNFLj5JmK1bIjsbSnX296Gg8A/o7oDtKWikFgBx0SLwztkw=="], - "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.76.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-oPFkkKTgl0K/EIg9fQ8oA3IGcI05/Mq1en04iFa41mmNPT+6KEiByVazTOZZJiHMBBrbsns1YJ2e1Scqwzesjw=="], + "@oxlint/binding-linux-arm64-musl": ["@oxlint/binding-linux-arm64-musl@1.77.0", "", { "os": "linux", "cpu": "arm64" }, "sha512-LSbwuRKiNCenPDcbARqAZ5RfBy7gmj7vOvfJRLeCDU3gFtSxWbhv/+VTlaUqzUhNj1gFLHB8h7ALnxa/Az6z6g=="], - "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.76.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-gN7yZ0eqflA5Fhf1wvHxGUltIV3FsvmB1zhNMDEK9vSHhc7E6qg9CuPeBgPZab66Tjzq6w6kHAtNEvnTHf4cyw=="], + "@oxlint/binding-linux-ppc64-gnu": ["@oxlint/binding-linux-ppc64-gnu@1.77.0", "", { "os": "linux", "cpu": "ppc64" }, "sha512-QWdcH31mXEUe5Nq1s0CfCpceaKjIo9uZtwDjAuL681g1axf+5x8xrg/eXWaw//4NCxYZ4V4e5Hu5tvdR+pTBlg=="], - "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.76.0", "", { "os": "linux", "cpu": "none" }, "sha512-S/HqMbn22mQrjtErUxEoS/a55u8kIeXvreIxiJu5G7Le3UecEd6SQZxrDIpuhtgaFnsY/nVra3ytP+pRljDilA=="], + "@oxlint/binding-linux-riscv64-gnu": ["@oxlint/binding-linux-riscv64-gnu@1.77.0", "", { "os": "linux", "cpu": "none" }, "sha512-GnOfYgJxbcElOiPZaDFDl406ONddwvOWk2jvAAAEjwAl4GofNoHF+/HHUIBYa6bFCArlcGPi0XjC4cU1pkgF/Q=="], - "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.76.0", "", { "os": "linux", "cpu": "none" }, "sha512-ZIga3097VJZolGZk6SrIAUokIGfRkxRlhiHDUznZptGBfwrhD7pNfD1rzEzsCwvk/1DX0A1bLz+liuNh5QKIVQ=="], + "@oxlint/binding-linux-riscv64-musl": ["@oxlint/binding-linux-riscv64-musl@1.77.0", "", { "os": "linux", "cpu": "none" }, "sha512-AyEMTUCf0xY+hHF+IxqXFQIX0yQOIR8ykpY0lJNOw9xYqOzUX8dyZfRvlG0RfXwuQn2eonf/8NrMmDSZJjdqsA=="], - "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.76.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-ZGiiA7pFzMJSyMWYZTVlPgbTsx+Vl8ihLGMIujPwaslUF7kIPPWAbVmAlTc+9lWDV+DCiB8Ikixu+lSHeOIIWQ=="], + "@oxlint/binding-linux-s390x-gnu": ["@oxlint/binding-linux-s390x-gnu@1.77.0", "", { "os": "linux", "cpu": "s390x" }, "sha512-sPLzEcNvxd/oyVQ5oZo92CiHkFkpBeRop13E/P3TPY+hZfXHKCOWKI70TE2RYwMKFJDc20EMjH16L7NZICtKTw=="], - "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.76.0", "", { "os": "linux", "cpu": "x64" }, "sha512-JLiy5WuvEBFTT6ErIFV35SLzi0R7Iri6MKU6dZbTxfIx8pndbbPs3Mj780nMipBFcPkti+okAPOJ9POKkHFEgg=="], + "@oxlint/binding-linux-x64-gnu": ["@oxlint/binding-linux-x64-gnu@1.77.0", "", { "os": "linux", "cpu": "x64" }, "sha512-1Oh2ssH2L7lwyvkdSqaMUfsGfwU2Wfvew+obBUYjRVqhpBcUpwnsPSEr1IzVi9XqkuY10geiLsNKecqaZC34Dw=="], - "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.76.0", "", { "os": "linux", "cpu": "x64" }, "sha512-z7lgKQtbo/I1NIe8G5NHLesxJDv0tRSUWTpXKb9Pm3E9nKFKfO4IOSDtFroKgXtOYb0jQbcdH+0wzTyMXVes+A=="], + "@oxlint/binding-linux-x64-musl": ["@oxlint/binding-linux-x64-musl@1.77.0", "", { "os": "linux", "cpu": "x64" }, "sha512-0j/2wRgNGO+Qj/M1uu/p57h/hFTTWWcfie0ufkbabeus2s5+/QqkCflnMOwLLN5m2GsNeWp4xdl4cPa4n7QCOQ=="], - "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.76.0", "", { "os": "none", "cpu": "arm64" }, "sha512-JOjKymIpb9QcYfEhZsN6h4V9Ivd474W38cNIBRv6bg2TbIvogbMTH0Mg6YWW9TiRDqfcX+/Hyfsbo5vcSE5guQ=="], + "@oxlint/binding-openharmony-arm64": ["@oxlint/binding-openharmony-arm64@1.77.0", "", { "os": "none", "cpu": "arm64" }, "sha512-BJ/j54qS0usEnyDkLYURMj2iiD9h5Cyy+ppzeMSXBGRXaGRNWnj1Mw14NqWMR5E/PzdgB30OOCCzLzbRoduafw=="], - "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.76.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-pqDWZiwcmByWUEm1NFUBNiT6aentCcaoMWJv0HbXEmuYermJ4sg8ppVrshubYP2MZ6SHccJJcpr6x469PuDFIw=="], + "@oxlint/binding-win32-arm64-msvc": ["@oxlint/binding-win32-arm64-msvc@1.77.0", "", { "os": "win32", "cpu": "arm64" }, "sha512-Yh8w+g2Lpx7StrvtYkoz9JJvXjB9wxgFChFNb85nrXm/wj/XTwGWS1hve9+900HL7llrntYB3YP+y32E3tRqzA=="], - "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.76.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-Ba0O659kgMv6pwO3z9PdO+K3aMxQRaw9HnG+e6AtOfgwcKFvYilciQYBoUBmxfQvOCKZe1SwjMkuB542NkuDMQ=="], + "@oxlint/binding-win32-ia32-msvc": ["@oxlint/binding-win32-ia32-msvc@1.77.0", "", { "os": "win32", "cpu": "ia32" }, "sha512-zja5b7+6a7UsRFgAQSrnax5vrzliEyNPLCjfXONu/vTWswaIVZGFajJZptaeRvPE4LghtFdAzVFlexTm7MVTGA=="], - "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.76.0", "", { "os": "win32", "cpu": "x64" }, "sha512-5qcirPHO8nKfkoowEVWtpAoVTcYDy6g0UT0NGic450Qv8J2NrOqg4uQ8QppRP4MDTC7Xx47lbZnmadTH03CGGA=="], + "@oxlint/binding-win32-x64-msvc": ["@oxlint/binding-win32-x64-msvc@1.77.0", "", { "os": "win32", "cpu": "x64" }, "sha512-+teyvPDZ2RjUvo+SuCqS/UhaJl1QtdW5fWT5NJTV61V5MIuIS90Db9LixmtEGvXixyttiK62P96MSu3UlpviBw=="], "@playwright/test": ["@playwright/test@1.62.1", "", { "dependencies": { "playwright": "1.62.1" }, "bin": { "playwright": "cli.js" } }, "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ=="], @@ -614,11 +612,11 @@ "@testing-library/dom": ["@testing-library/dom@10.4.1", "", { "dependencies": { "@babel/code-frame": "^7.10.4", "@babel/runtime": "^7.12.5", "@types/aria-query": "^5.0.1", "aria-query": "5.3.0", "dom-accessibility-api": "^0.5.9", "lz-string": "^1.5.0", "picocolors": "1.1.1", "pretty-format": "^27.0.2" } }, "sha512-o4PXJQidqJl82ckFaXUeoAW+XysPLauYI43Abki5hABd853iMhitooc6znOnczgbTYmEP6U6/y1ZyKAIsvMKGg=="], - "@testing-library/jest-dom": ["@testing-library/jest-dom@7.0.0", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" }, "peerDependencies": { "@testing-library/dom": ">=10 <11" } }, "sha512-HKAH9C6mBo5yBG6yRO5i43L2iisencAo5z+o5P/saHUoY+miC5ivXRxHBJcFyB5ypPNxHJdK3BoF/3O4DIptMg=="], + "@testing-library/jest-dom": ["@testing-library/jest-dom@7.0.1", "", { "dependencies": { "@adobe/css-tools": "^4.4.0", "aria-query": "^5.0.0", "css.escape": "^1.5.1", "dom-accessibility-api": "^0.6.3", "picocolors": "^1.1.1", "redent": "^3.0.0" }, "peerDependencies": { "@testing-library/dom": ">=10 <11", "vitest": ">= 0.32" }, "optionalPeers": ["vitest"] }, "sha512-oMDTC3oA+6CXSO2JZnvOI7CA6oVub6kij5ggk9ohwye5slmkwxYDXcPOVxgMw/RQlticjtO0C1RZkR97HgrWMw=="], "@testing-library/react": ["@testing-library/react@16.3.2", "", { "dependencies": { "@babel/runtime": "^7.12.5" }, "peerDependencies": { "@testing-library/dom": "^10.0.0", "@types/react": "^18.0.0 || ^19.0.0", "@types/react-dom": "^18.0.0 || ^19.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0" } }, "sha512-XU5/SytQM+ykqMnAnvB2umaJNIOsLF3PVv//1Ew4CTcpz0/BRyy/af40qqrt7SjKpDdT1saBMc42CUok5gaw+g=="], - "@testing-library/user-event": ["@testing-library/user-event@14.6.3", "", { "peerDependencies": { "@testing-library/dom": ">=7.21.4" } }, "sha512-6dBq67jT8lE+JTE8Exm02Kt6ze43hz1jdiSpSJwtTZiT1xQQ6b7nZYTTQ9njdArdU8XklOwaDp/AbT/eYSKF4g=="], + "@testing-library/user-event": ["@testing-library/user-event@14.6.4", "", { "peerDependencies": { "@testing-library/dom": ">=7.21.4" } }, "sha512-QCGwP6QrjypBLwyj5cuyfVamkaIEy/XGY+1VDehbtbQqOggYmTFpFOdWR5mPz14vX8vXLMVjDHlRNBcClyO9ew=="], "@ts-morph/common": ["@ts-morph/common@0.27.0", "", { "dependencies": { "fast-glob": "^3.3.3", "minimatch": "^10.0.1", "path-browserify": "^1.0.1" } }, "sha512-Wf29UqxWDpc+i61k3oIOzcUfQt79PIT9y/MWfAGlrkjg6lBC1hwDECLXPVJAhWjiGbfBCxZd65F/LIZF3+jeJQ=="], @@ -668,25 +666,25 @@ "@types/validate-npm-package-name": ["@types/validate-npm-package-name@4.0.2", "", {}, "sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw=="], - "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.66.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/type-utils": "8.66.0", "@typescript-eslint/utils": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.66.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-p088eaGrzYz1s+7cov0aMOCkNGTJlVxF4jgubf28c8L0Cv9Rloj8YBHnv4hXLq6IIEE1AsjNWavO+k+8kP2Y0A=="], + "@typescript-eslint/eslint-plugin": ["@typescript-eslint/eslint-plugin@8.67.0", "", { "dependencies": { "@eslint-community/regexpp": "^4.12.2", "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/type-utils": "8.67.0", "@typescript-eslint/utils": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "ignore": "^7.0.5", "natural-compare": "^1.4.0", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "@typescript-eslint/parser": "^8.67.0", "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-Un7Heoyj65NREbKAyIrFxeM143NZpExWmy1Nep4DLeQOeLlTeumPjoNKnBrU5D5moWXbPJgRa5Uwcdu0faVNGQ=="], - "@typescript-eslint/parser": ["@typescript-eslint/parser@8.66.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-X6ypGChaWYk6PBtUg2BwuTZEFFcHJAtGTVJ9/lCTOufhZ4i9fNolQNnktq+kkMCwMj7V8Svsq7+TxSDslmhE0g=="], + "@typescript-eslint/parser": ["@typescript-eslint/parser@8.67.0", "", { "dependencies": { "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "debug": "^4.4.3" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-fUBfTuuEulWqX6V8+O3PtScV01tzYYRUDTAirHFKoRAt7nOzoGiPt0M/bB47wWNy0coOOcgEwAMUtBpykMxl6w=="], - "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.66.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.66.0", "@typescript-eslint/types": "^8.66.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-7MthGPTt4BP69lSryqpqq8HQqxuzynssckL/jyDyk3+TNMQ3y2jFWkptCrktWvBrP+EH787Nl5N5Qpw7WZg+5g=="], + "@typescript-eslint/project-service": ["@typescript-eslint/project-service@8.67.0", "", { "dependencies": { "@typescript-eslint/tsconfig-utils": "^8.67.0", "@typescript-eslint/types": "^8.67.0", "debug": "^4.4.3" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-cvE8c7ulYeXN9fYuszhCeCsbzyVEXuhrRCybnBre7TUmqb5nRmBfQAwCj0O3WJFDeyAZt4VYv51vMCC9LHSdYw=="], - "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0" } }, "sha512-8TGcH25j9zqJ/IULB/ppyhRvxA8QYfFEZ7nfbg6/BN9spDgb8fPWQXlE5l8TWBL50EtUx007uZ1o9VOwrq2/9g=="], + "@typescript-eslint/scope-manager": ["@typescript-eslint/scope-manager@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0" } }, "sha512-EgvsleTwS4E+WzzSvem8fAUubLwatMNF1B5hHSLQxcvs7q2dtRhGyujHwLJSYlG41niJ7GP24Aha2+0mb1b2kg=="], - "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.66.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-9D5gLYZG4rOjcoag8MQ/fWI8WqA9wcPDyOGyWtWFhvM1lHRbliqUSPIY5J3zqCU1tvSwzXxnnjhQhz5Ne7mJ4g=="], + "@typescript-eslint/tsconfig-utils": ["@typescript-eslint/tsconfig-utils@8.67.0", "", { "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-vV+LUSv5njUWsknE71fqKTlXUva+R76SaeORd6Zojcunk/6DvKFXONU3BrAs2H49mbygUXt6gbYunzwqNwlhdg=="], - "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/utils": "8.66.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-LG2dWfjZQQp0ADtAu/EWJVayefGL2UEZ3CDeI44D9v3rXB/WYUqE/jpO28KrEKul5AySrmI+Zh1v6v+xW2U9+g=="], + "@typescript-eslint/type-utils": ["@typescript-eslint/type-utils@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/utils": "8.67.0", "debug": "^4.4.3", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-aVWDXbRmdXO9siTfX4ditQI1T9+zVcNazT48EJCD0v40/9RIFoUgZ05CmGEq9H2gixRpjUn/iplwvlcvutJW/Q=="], - "@typescript-eslint/types": ["@typescript-eslint/types@8.65.0", "", {}, "sha512-JSSwWNy+H0E/01jJEM+hrX6N0OFDzFzeIhHFSAS01tlVaevpG8cFyYRPhS5yjGOvBUx3sqQHVMjCL1CAZZMxBg=="], + "@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], - "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.66.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.66.0", "@typescript-eslint/tsconfig-utils": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/visitor-keys": "8.66.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-8/x4INiiQb10jGgXYD7116/zQ+OL84ZIFn0za68wwFHCanT/VLbBEroWht8RV8fn0/ZCAoazHLQgwUC0UQcDfg=="], + "@typescript-eslint/typescript-estree": ["@typescript-eslint/typescript-estree@8.67.0", "", { "dependencies": { "@typescript-eslint/project-service": "8.67.0", "@typescript-eslint/tsconfig-utils": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/visitor-keys": "8.67.0", "debug": "^4.4.3", "minimatch": "^10.2.2", "semver": "^7.7.3", "tinyglobby": "^0.2.15", "ts-api-utils": "^2.5.0" }, "peerDependencies": { "typescript": ">=4.8.4 <6.1.0" } }, "sha512-EKQBCE9yNlRJYm7jdTW5AhDacDUmSwQb0FAJAmK2EKYrNXIsa2vxcSZx6PvJ/dEdI6lS+Y9W+EXckLj0iPFGcw=="], - "@typescript-eslint/utils": ["@typescript-eslint/utils@8.66.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.66.0", "@typescript-eslint/types": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-jasearZPolBw5NJNYGMwxzHMF83niVWmMU1VdHzG1CyfI2VS7f7nZltnKtHcg20hW+7Uo5GfK4MeDPoU3qI8EA=="], + "@typescript-eslint/utils": ["@typescript-eslint/utils@8.67.0", "", { "dependencies": { "@eslint-community/eslint-utils": "^4.9.1", "@typescript-eslint/scope-manager": "8.67.0", "@typescript-eslint/types": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-U9D1FdwEWBwok3hxxSdhclMb0twvt9QnjIQ0VfQ1AiX2epnpSgv2ubVDsayOFyY8K6FX+AQ7E0FKWVG3iKsj1A=="], - "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.66.0", "", { "dependencies": { "@typescript-eslint/types": "8.66.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-dkKR8q+lKciskj1Y3vthHktl+3cMLWGyVUP23bRiPZ5O9BRT++4EqDDV+TVeIKBL1VXVEqrJlz8MYbcnvJcAlg=="], + "@typescript-eslint/visitor-keys": ["@typescript-eslint/visitor-keys@8.67.0", "", { "dependencies": { "@typescript-eslint/types": "8.67.0", "eslint-visitor-keys": "^5.0.0" } }, "sha512-fkv8dHRDqfGtTHuJeebdrQ7cX6Ad4WAS00rgHh9UGvMycF1mjBfsxry1XsLIFhWZ6Judlh6UdzK+TYlbpCXgnA=="], "@typescript/native": ["typescript@7.0.2", "", { "optionalDependencies": { "@typescript/typescript-aix-ppc64": "7.0.2", "@typescript/typescript-darwin-arm64": "7.0.2", "@typescript/typescript-darwin-x64": "7.0.2", "@typescript/typescript-freebsd-arm64": "7.0.2", "@typescript/typescript-freebsd-x64": "7.0.2", "@typescript/typescript-linux-arm": "7.0.2", "@typescript/typescript-linux-arm64": "7.0.2", "@typescript/typescript-linux-loong64": "7.0.2", "@typescript/typescript-linux-mips64el": "7.0.2", "@typescript/typescript-linux-ppc64": "7.0.2", "@typescript/typescript-linux-riscv64": "7.0.2", "@typescript/typescript-linux-s390x": "7.0.2", "@typescript/typescript-linux-x64": "7.0.2", "@typescript/typescript-netbsd-arm64": "7.0.2", "@typescript/typescript-netbsd-x64": "7.0.2", "@typescript/typescript-openbsd-arm64": "7.0.2", "@typescript/typescript-openbsd-x64": "7.0.2", "@typescript/typescript-sunos-x64": "7.0.2", "@typescript/typescript-win32-arm64": "7.0.2", "@typescript/typescript-win32-x64": "7.0.2" }, "bin": { "tsc": "bin/tsc" } }, "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA=="], @@ -910,7 +908,7 @@ "dequal": ["dequal@2.0.3", "", {}, "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA=="], - "deslop-js": ["deslop-js@0.9.6", "", { "dependencies": { "@oxc-project/types": "^0.142.0", "fast-glob": "^3.3.3", "minimatch": "^10.2.5", "oxc-parser": "^0.142.0", "oxc-resolver": "^11.24.2", "typescript": ">=5.0.4 <6" } }, "sha512-ADY8/3JsK0epUcdXez54MhY4CGWeppT032Ko1fJkwFfmHog5hBJtMeihUeHs5PF7eG6Y17YbKcNQXSp3B67GRA=="], + "deslop-js": ["deslop-js@0.9.12", "", { "dependencies": { "@oxc-project/types": "^0.143.0", "fast-glob": "^3.3.3", "minimatch": "^10.2.5", "oxc-parser": "^0.143.0", "oxc-resolver": "^11.24.2", "typescript": ">=5.0.4 <6" } }, "sha512-Ku6Zngzmu4EISb58WUkRKZytfgWjJ18Cic7lb4wl+/BF0tHWRvpBOLlA0FP35r82mV45Y72AK3RPC1Nw0GLbIw=="], "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], @@ -964,7 +962,7 @@ "eslint-plugin-react-hooks": ["eslint-plugin-react-hooks@7.1.1", "", { "dependencies": { "@babel/core": "^7.24.4", "@babel/parser": "^7.24.4", "hermes-parser": "^0.25.1", "zod": "^3.25.0 || ^4.0.0", "zod-validation-error": "^3.5.0 || ^4.0.0" }, "peerDependencies": { "eslint": "^3.0.0 || ^4.0.0 || ^5.0.0 || ^6.0.0 || ^7.0.0 || ^8.0.0-0 || ^9.0.0 || ^10.0.0" } }, "sha512-f2I7Gw6JbvCexzIInuSbZpfdQ44D7iqdWX01FKLvrPgqxoE7oMj8clOfto8U6vYiz4yd5oKu39rRSVOe1zRu0g=="], - "eslint-plugin-react-refresh": ["eslint-plugin-react-refresh@0.5.3", "", { "peerDependencies": { "eslint": "^9 || ^10" } }, "sha512-5EMmLCV98Pi4o/f/3DP/v/tNqLHMIc9I8LKClNDWhZ9JTho89/kQcitCXQBMG7sAfVRK0Ie3T2EDOzp1YXYiVA=="], + "eslint-plugin-react-refresh": ["eslint-plugin-react-refresh@0.5.4", "", { "peerDependencies": { "eslint": "^9 || ^10" } }, "sha512-7bqTKz7T0r+HKWFarNXByDE9/5+73wI2ru+M3zuqGbR7s/b/5/pQJXZoufWlrngqGqoZto73ZkGumCdLxk+4rw=="], "eslint-scope": ["eslint-scope@9.1.2", "", { "dependencies": { "@types/esrecurse": "^4.3.1", "@types/estree": "^1.0.8", "esrecurse": "^4.3.0", "estraverse": "^5.2.0" } }, "sha512-xS90H51cKw0jltxmvmHy2Iai1LIqrfbw57b79w/J7MfvDfkIkFZ+kj6zC3BjtUwh150HsSSdxXZcsuv72miDFQ=="], @@ -1064,7 +1062,7 @@ "glob-parent": ["glob-parent@6.0.2", "", { "dependencies": { "is-glob": "^4.0.3" } }, "sha512-XxwI8EOhVQgWp6iDL+3b0r86f4d6AX6zSU55HfB4ydCEuXLXc5FcYeOu+nnGftS4TEju/11rt4KJPTMgbfmv4A=="], - "globals": ["globals@17.9.0", "", {}, "sha512-m/MvAW61QVU5VDNF1Vj8axt016h8w7L5TU1e9zlab7XIttAT2YAlCwl75K1fOqvMM9apmD7lbCIRhpfkhmxhCg=="], + "globals": ["globals@17.11.0", "", {}, "sha512-Z2I8hM+PbJDXQDq3Icgpzv+mPdwr68iZUU9d5WW4FuXfDUQfkZaZuvjMv42/5crNyw154+9+VWXbYrUgDXbxNw=="], "gopd": ["gopd@1.2.0", "", {}, "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg=="], @@ -1120,7 +1118,7 @@ "internmap": ["internmap@2.0.3", "", {}, "sha512-5Hh7Y1wQbvY5ooGgPbDaL5iYLAPzMTUrjMulskHLH6wnv/A+1q5rgEaiuqEjB+oxGXIVZs1FF+R/KPN3ZSQYYg=="], - "ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="], + "ip-address": ["ip-address@10.5.0", "", {}, "sha512-R5SnVLJmgYYvf2F2ZgwSBnelz5G4q5AxIC277GDfUaNbrZKNANcBC7RHqYYePlszf4kBolVkJauG0ZjHHFh55g=="], "ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -1234,7 +1232,7 @@ "lru-cache": ["lru-cache@11.5.2", "", {}, "sha512-4pfM1Ff0x50o0tQwb5ucw/RzNyD0/YJME6IVcStalZuMWxdt3sR3huStTtxz4PUmvZfRguvDejasvQ2kifR11g=="], - "lucide-react": ["lucide-react@1.30.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-tUIr2jXLbWpCkdtH8XP7P7YppM9ueWgTky99lpWDY6z5REs6B+O6ZQ3U5tHkUUY59ANyOv/PBcs8E4Fe3KO3eA=="], + "lucide-react": ["lucide-react@1.31.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-G8u2eEtoHUnUa9f8lbvqDhCiORMnYLdUEo06EEG9MQvHQrInKcX3Pa2TH39MM5qyzRcWETxB0+aOwAPI1g1kEg=="], "lz-string": ["lz-string@1.5.0", "", { "bin": "bin/bin.js" }, "sha512-h5bgJWpxJNswbU7qCrV0tIKQCaS3blPDrqKWx+QxzuzL1zGUzij9XCWLrSLsJPu5t+eWA/ycetzYAO5IOMcWAQ=="], @@ -1312,13 +1310,13 @@ "outvariant": ["outvariant@1.4.3", "", {}, "sha512-+Sl2UErvtsoajRDKCE5/dBz4DIvHXQQnAxtQTF04OJxY0+DyZXSo5P5Bb7XYWOh81syohlYL24hbDwxedPUJCA=="], - "oxc-parser": ["oxc-parser@0.142.0", "", { "dependencies": { "@oxc-project/types": "^0.142.0" }, "optionalDependencies": { "@oxc-parser/binding-android-arm-eabi": "0.142.0", "@oxc-parser/binding-android-arm64": "0.142.0", "@oxc-parser/binding-darwin-arm64": "0.142.0", "@oxc-parser/binding-darwin-x64": "0.142.0", "@oxc-parser/binding-freebsd-x64": "0.142.0", "@oxc-parser/binding-linux-arm-gnueabihf": "0.142.0", "@oxc-parser/binding-linux-arm-musleabihf": "0.142.0", "@oxc-parser/binding-linux-arm64-gnu": "0.142.0", "@oxc-parser/binding-linux-arm64-musl": "0.142.0", "@oxc-parser/binding-linux-ppc64-gnu": "0.142.0", "@oxc-parser/binding-linux-riscv64-gnu": "0.142.0", "@oxc-parser/binding-linux-riscv64-musl": "0.142.0", "@oxc-parser/binding-linux-s390x-gnu": "0.142.0", "@oxc-parser/binding-linux-x64-gnu": "0.142.0", "@oxc-parser/binding-linux-x64-musl": "0.142.0", "@oxc-parser/binding-openharmony-arm64": "0.142.0", "@oxc-parser/binding-wasm32-wasi": "0.142.0", "@oxc-parser/binding-win32-arm64-msvc": "0.142.0", "@oxc-parser/binding-win32-ia32-msvc": "0.142.0", "@oxc-parser/binding-win32-x64-msvc": "0.142.0" } }, "sha512-kKR+jPiRJYJDexVoziIg/FVGvr1fT1FZSSJOk6tVoMKKSlsf1Cso+cgGCJkOEDWOP174vRntCPFKg+AS7InWvw=="], + "oxc-parser": ["oxc-parser@0.143.0", "", { "dependencies": { "@oxc-project/types": "^0.143.0" }, "optionalDependencies": { "@oxc-parser/binding-android-arm-eabi": "0.143.0", "@oxc-parser/binding-android-arm64": "0.143.0", "@oxc-parser/binding-darwin-arm64": "0.143.0", "@oxc-parser/binding-darwin-x64": "0.143.0", "@oxc-parser/binding-freebsd-x64": "0.143.0", "@oxc-parser/binding-linux-arm-gnueabihf": "0.143.0", "@oxc-parser/binding-linux-arm-musleabihf": "0.143.0", "@oxc-parser/binding-linux-arm64-gnu": "0.143.0", "@oxc-parser/binding-linux-arm64-musl": "0.143.0", "@oxc-parser/binding-linux-ppc64-gnu": "0.143.0", "@oxc-parser/binding-linux-riscv64-gnu": "0.143.0", "@oxc-parser/binding-linux-riscv64-musl": "0.143.0", "@oxc-parser/binding-linux-s390x-gnu": "0.143.0", "@oxc-parser/binding-linux-x64-gnu": "0.143.0", "@oxc-parser/binding-linux-x64-musl": "0.143.0", "@oxc-parser/binding-openharmony-arm64": "0.143.0", "@oxc-parser/binding-win32-arm64-msvc": "0.143.0", "@oxc-parser/binding-win32-ia32-msvc": "0.143.0", "@oxc-parser/binding-win32-x64-msvc": "0.143.0" } }, "sha512-ov0NzaDCOInknS7mP1cwKdJERt3utPW8ldjtdUXQ8Ty0GEFD08wk422vCUN0d7pST6kqtV7dxoI9w1Zi0l/9TA=="], "oxc-resolver": ["oxc-resolver@11.24.2", "", { "optionalDependencies": { "@oxc-resolver/binding-android-arm-eabi": "11.24.2", "@oxc-resolver/binding-android-arm64": "11.24.2", "@oxc-resolver/binding-darwin-arm64": "11.24.2", "@oxc-resolver/binding-darwin-x64": "11.24.2", "@oxc-resolver/binding-freebsd-x64": "11.24.2", "@oxc-resolver/binding-linux-arm-gnueabihf": "11.24.2", "@oxc-resolver/binding-linux-arm-musleabihf": "11.24.2", "@oxc-resolver/binding-linux-arm64-gnu": "11.24.2", "@oxc-resolver/binding-linux-arm64-musl": "11.24.2", "@oxc-resolver/binding-linux-ppc64-gnu": "11.24.2", "@oxc-resolver/binding-linux-riscv64-gnu": "11.24.2", "@oxc-resolver/binding-linux-riscv64-musl": "11.24.2", "@oxc-resolver/binding-linux-s390x-gnu": "11.24.2", "@oxc-resolver/binding-linux-x64-gnu": "11.24.2", "@oxc-resolver/binding-linux-x64-musl": "11.24.2", "@oxc-resolver/binding-openharmony-arm64": "11.24.2", "@oxc-resolver/binding-wasm32-wasi": "11.24.2", "@oxc-resolver/binding-win32-arm64-msvc": "11.24.2", "@oxc-resolver/binding-win32-x64-msvc": "11.24.2" } }, "sha512-FY91FiDBj7ls5MsFS9jN3tjz2o0/zsdSsymlakySaBwVJZorHhkWyICLZMKxlu1R9vYo+sd3z1jwb4J8x7bNDw=="], - "oxlint": ["oxlint@1.76.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.76.0", "@oxlint/binding-android-arm64": "1.76.0", "@oxlint/binding-darwin-arm64": "1.76.0", "@oxlint/binding-darwin-x64": "1.76.0", "@oxlint/binding-freebsd-x64": "1.76.0", "@oxlint/binding-linux-arm-gnueabihf": "1.76.0", "@oxlint/binding-linux-arm-musleabihf": "1.76.0", "@oxlint/binding-linux-arm64-gnu": "1.76.0", "@oxlint/binding-linux-arm64-musl": "1.76.0", "@oxlint/binding-linux-ppc64-gnu": "1.76.0", "@oxlint/binding-linux-riscv64-gnu": "1.76.0", "@oxlint/binding-linux-riscv64-musl": "1.76.0", "@oxlint/binding-linux-s390x-gnu": "1.76.0", "@oxlint/binding-linux-x64-gnu": "1.76.0", "@oxlint/binding-linux-x64-musl": "1.76.0", "@oxlint/binding-openharmony-arm64": "1.76.0", "@oxlint/binding-win32-arm64-msvc": "1.76.0", "@oxlint/binding-win32-ia32-msvc": "1.76.0", "@oxlint/binding-win32-x64-msvc": "1.76.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-6QoFioEU4fNdiUx/2Eo6TRd6NG7H7njnRCz8rhB66cZmMHDTqcm1Rjvl8Wry+ZTQMBAmyb4Mlf62Mk5X+eHSOw=="], + "oxlint": ["oxlint@1.77.0", "", { "optionalDependencies": { "@oxlint/binding-android-arm-eabi": "1.77.0", "@oxlint/binding-android-arm64": "1.77.0", "@oxlint/binding-darwin-arm64": "1.77.0", "@oxlint/binding-darwin-x64": "1.77.0", "@oxlint/binding-freebsd-x64": "1.77.0", "@oxlint/binding-linux-arm-gnueabihf": "1.77.0", "@oxlint/binding-linux-arm-musleabihf": "1.77.0", "@oxlint/binding-linux-arm64-gnu": "1.77.0", "@oxlint/binding-linux-arm64-musl": "1.77.0", "@oxlint/binding-linux-ppc64-gnu": "1.77.0", "@oxlint/binding-linux-riscv64-gnu": "1.77.0", "@oxlint/binding-linux-riscv64-musl": "1.77.0", "@oxlint/binding-linux-s390x-gnu": "1.77.0", "@oxlint/binding-linux-x64-gnu": "1.77.0", "@oxlint/binding-linux-x64-musl": "1.77.0", "@oxlint/binding-openharmony-arm64": "1.77.0", "@oxlint/binding-win32-arm64-msvc": "1.77.0", "@oxlint/binding-win32-ia32-msvc": "1.77.0", "@oxlint/binding-win32-x64-msvc": "1.77.0" }, "peerDependencies": { "oxlint-tsgolint": ">=7.0.2001", "vite-plus": "*" }, "optionalPeers": ["oxlint-tsgolint", "vite-plus"], "bin": { "oxlint": "bin/oxlint" } }, "sha512-qnGh8XJHaQ0dprrDXNQZgS0FgjI6v+V3+X8DwmaV++5Aamy6jGKfDdQ1TUvhUxtmKFAbEf4/WeO5QZX+5WSngg=="], - "oxlint-plugin-react-doctor": ["oxlint-plugin-react-doctor@0.9.6", "", { "dependencies": { "@shaderfrog/glsl-parser": "^7.0.1", "@typescript-eslint/types": "^8.59.3", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "lightningcss": "^1.33.0", "oxc-parser": "^0.142.0" } }, "sha512-kk39ffbDFaL0UfAx7ndltBsxal1hRkr+qYYsSv57PsD+9DnjcV0Y0jZ6NYMhBmHuvPC7so2CN6dNml+fOGrpxw=="], + "oxlint-plugin-react-doctor": ["oxlint-plugin-react-doctor@0.9.12", "", { "dependencies": { "@shaderfrog/glsl-parser": "^7.0.1", "@typescript-eslint/types": "^8.59.3", "eslint-scope": "^9.1.2", "eslint-visitor-keys": "^5.0.1", "lightningcss": "^1.33.0", "oxc-parser": "^0.143.0" } }, "sha512-BplcCUU/tGByFGgY1YIax6evUmjk0K8zOGGrdPs3A3dqamrzjUvVuJdYpM1Ftyt/B5fFx6+VwRrWi3YZbIz4VQ=="], "p-limit": ["p-limit@3.1.0", "", { "dependencies": { "yocto-queue": "^0.1.0" } }, "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ=="], @@ -1386,11 +1384,11 @@ "react-day-picker": ["react-day-picker@10.0.1", "", { "dependencies": { "@date-fns/tz": "^1.4.1", "date-fns": "^4.1.0" }, "peerDependencies": { "@types/react": ">=16.8.0", "react": ">=16.8.0" }, "optionalPeers": ["@types/react"] }, "sha512-eNh6BlwcYInWaJtRv18mXQ06Ys/H6rdTZAnTaSdOYJuTpwP1JMCHNd1FDRadA+gbeinq+psdULN5Xnowy9mV8w=="], - "react-doctor": ["react-doctor@0.9.6", "", { "dependencies": { "@astrojs/compiler": "^4.0.0", "@babel/code-frame": "^7.29.0", "@sentry/node": "^10.54.0", "agent-install": "0.0.5", "conf": "^15.1.0", "confbox": "^0.2.4", "deslop-js": "0.9.6", "eslint-plugin-react-hooks": "^7.1.1", "figures": "^6.1.0", "jiti": "^2.7.0", "magicast": "^0.5.3", "oxc-resolver": "^11.24.2", "oxlint": ">=1.76.0 <1.77.0", "oxlint-plugin-react-doctor": "0.9.6", "prompts": "^2.4.2", "typescript": ">=5.0.4 <6", "vscode-languageserver": "^9.0.1", "vscode-languageserver-textdocument": "^1.0.12", "vscode-uri": "^3.1.0", "yaml": "^2.9.0", "yoga-layout": "~3.2.1" }, "bin": { "react-doctor": "bin/react-doctor.js" } }, "sha512-X3ZLL6UQfzqIyY1HDKEgUOnoxKreEfy9KphiV2aYhiotgCsR81LBl92XgyZUbZeRgpQuwbIDOiyxQnRRkcX42A=="], + "react-doctor": ["react-doctor@0.9.12", "", { "dependencies": { "@astrojs/compiler": "^4.0.0", "@sentry/node": "^10.54.0", "agent-install": "0.0.5", "conf": "^15.1.0", "confbox": "^0.2.4", "deslop-js": "0.9.12", "eslint-plugin-react-hooks": "^7.1.1", "jiti": "^2.7.0", "magicast": "^0.5.3", "oxc-resolver": "^11.24.2", "oxlint": ">=1.77.0 <1.78.0", "oxlint-plugin-react-doctor": "0.9.12", "prompts": "^2.4.2", "typescript": ">=5.0.4 <6", "vscode-languageserver": "^9.0.1", "vscode-languageserver-textdocument": "^1.0.12", "vscode-uri": "^3.1.0", "yaml": "^2.9.0", "yoga-layout": "~3.2.1" }, "bin": { "react-doctor": "bin/react-doctor.js" } }, "sha512-H7RNg13RYKwpQvi3+O3IkSj8pAD1pGTxSetxu7aOwXX3wmF+BydCLDiWxkPT9Pq7bIMHjuJ0taSZLsxEjlNjcA=="], "react-dom": ["react-dom@19.2.8", "", { "dependencies": { "scheduler": "^0.27.0" }, "peerDependencies": { "react": "^19.2.8" } }, "sha512-rVprimfGBG3DR+Tq0IQG2DT5PxKth1WIGDmj5yPmlzr4YBe7uyE+Du4oVqTDXZSHGGGXRtTJEGSSePyQCMBglQ=="], - "react-hook-form": ["react-hook-form@7.84.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-+hWvQP6GLco56mDwrbU4XnHix8t1z90ltZsDIrREl+jnQFQxYLX8oAzqe/Xn8nHpmoXTY5M6oEXrAhbP1qevNQ=="], + "react-hook-form": ["react-hook-form@7.85.0", "", { "peerDependencies": { "react": "^16.8.0 || ^17 || ^18 || ^19" } }, "sha512-U2MTriFXnclmV4rOE20p2DcRFv5WEg3FIcBFOKcOLFHDVvGIMPvLTkTWefUsonmlaVy23khVDxDWym6uJVGOzw=="], "react-i18next": ["react-i18next@17.0.11", "", { "dependencies": { "@babel/runtime": "^7.29.2", "html-parse-stringify": "^4.0.1", "use-sync-external-store": "^1.6.0" }, "peerDependencies": { "i18next": ">= 26.2.0", "react": ">= 16.8.0", "typescript": "^5 || ^6 || ^7" }, "optionalPeers": ["typescript"] }, "sha512-cDtkXgxjuFTWUH6V+aQn1Ve5vDiUztCNPWW5GtSHDccsgRXO1nE6QFWCEmc1KAutrb3OUv87wFShJL5RhUwPXg=="], @@ -1458,7 +1456,7 @@ "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], - "shadcn": ["shadcn@4.16.2", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/parser": "^7.28.0", "@babel/plugin-transform-typescript": "^7.28.0", "@babel/preset-typescript": "^7.27.1", "@dotenvx/dotenvx": "^1.48.4", "@modelcontextprotocol/sdk": "^1.26.0", "@types/validate-npm-package-name": "^4.0.2", "browserslist": "^4.26.2", "commander": "^14.0.0", "cosmiconfig": "^9.0.0", "dedent": "^1.6.0", "deepmerge": "^4.3.1", "diff": "^8.0.2", "execa": "^9.6.0", "fast-glob": "^3.3.3", "fs-extra": "^11.3.1", "fuzzysort": "^3.1.0", "kleur": "^4.1.5", "open": "^11.0.0", "ora": "^8.2.0", "postcss": "^8.5.6", "postcss-selector-parser": "^7.1.0", "prompts": "^2.4.2", "recast": "^0.23.11", "stringify-object": "^5.0.0", "tailwind-merge": "^3.0.1", "ts-morph": "^26.0.0", "tsconfig-paths": "^4.2.0", "undici": "^7.27.2", "validate-npm-package-name": "^7.0.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.6" }, "bin": { "shadcn": "dist/index.js" } }, "sha512-M1AvZKFWcCzWRDoyApIqJMSLIpY8Ev4uBGuiPLSFmiTbixXhPmzotSTvLzFmBrfoIxG9aIg2dZOETblEaXGUnQ=="], + "shadcn": ["shadcn@4.18.0", "", { "dependencies": { "@babel/core": "^7.28.0", "@babel/parser": "^7.28.0", "@babel/plugin-transform-typescript": "^7.28.0", "@babel/preset-typescript": "^7.27.1", "@dotenvx/dotenvx": "^1.48.4", "@modelcontextprotocol/sdk": "^1.26.0", "@types/validate-npm-package-name": "^4.0.2", "browserslist": "^4.26.2", "commander": "^14.0.0", "cosmiconfig": "^9.0.0", "dedent": "^1.6.0", "deepmerge": "^4.3.1", "diff": "^8.0.2", "execa": "^9.6.0", "fast-glob": "^3.3.3", "fs-extra": "^11.3.1", "fuzzysort": "^3.1.0", "kleur": "^4.1.5", "open": "^11.0.0", "ora": "^8.2.0", "postcss": "^8.5.6", "postcss-selector-parser": "^7.1.0", "prompts": "^2.4.2", "recast": "^0.23.11", "socks": "^2.8.8", "stringify-object": "^5.0.0", "tailwind-merge": "^3.0.1", "ts-morph": "^26.0.0", "tsconfig-paths": "^4.2.0", "undici": "^7.27.2", "validate-npm-package-name": "^7.0.1", "zod": "^3.24.1", "zod-to-json-schema": "^3.24.6" }, "bin": { "shadcn": "dist/index.js" } }, "sha512-tUFZgkYmfVNQVm3xX7lhSzOvDsp+O14ac5dwgXIr5mIsr79ISueb/Mu+ZtWMz0DH6v77u4eYyvbQ9TTMpSn3aw=="], "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], @@ -1478,7 +1476,11 @@ "sisteransi": ["sisteransi@1.0.5", "", {}, "sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg=="], - "sonner": ["sonner@2.0.7", "", { "peerDependencies": { "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" } }, "sha512-W6ZN4p58k8aDKA4XPcx2hpIQXBRAgyiWVkYhT7CvK6D3iAu7xjvVyhQHg2/iaKJZ1XVJ4r7XuwGL+WGEK37i9w=="], + "smart-buffer": ["smart-buffer@4.2.0", "", {}, "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg=="], + + "socks": ["socks@2.8.9", "", { "dependencies": { "ip-address": "^10.1.1", "smart-buffer": "^4.2.0" } }, "sha512-LJhUYUvItdQ0LkJTmPeaEObWXAqFyfmP85x0tch/ez9cahmhlBBLbIqDFnvBnUJGagb0JbIQrkBs1wJ+yRYpEw=="], + + "sonner": ["sonner@2.0.8", "", { "peerDependencies": { "@types/react": "^18.0.0 || ^19.0.0", "react": "^18.0.0 || ^19.0.0 || ^19.0.0-rc", "react-dom": "^18.0.0 || ^19.0.0 || ^19.0.0-rc" }, "optionalPeers": ["@types/react"] }, "sha512-UM/ByIoFra8yzV75n1o0Puu0bw5U/9UNnDacrJNspekBewIfsQ3D6ez1nvlWpt7aTsO6rujQtifBpycwIivqlg=="], "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], @@ -1562,7 +1564,7 @@ "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], - "typescript-eslint": ["typescript-eslint@8.66.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.66.0", "@typescript-eslint/parser": "8.66.0", "@typescript-eslint/typescript-estree": "8.66.0", "@typescript-eslint/utils": "8.66.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-QlEbBPz/RuJ1XUHj29nm3t0F/O/cSlEnntozqPOYHnnTGAXFamnMBu5i9Vn6vhUPHGAjR+Vl+5J8vPN/BMUrJw=="], + "typescript-eslint": ["typescript-eslint@8.67.0", "", { "dependencies": { "@typescript-eslint/eslint-plugin": "8.67.0", "@typescript-eslint/parser": "8.67.0", "@typescript-eslint/typescript-estree": "8.67.0", "@typescript-eslint/utils": "8.67.0" }, "peerDependencies": { "eslint": "^8.57.0 || ^9.0.0 || ^10.0.0", "typescript": ">=4.8.4 <6.1.0" } }, "sha512-S2udFs8tCKEKffuJ4TB1idGUZiXdCPGi3IPBGWXarbLQ5UPXORV8QEVzJ4gCRduURMb5EkpNCdjbk0eDIuI8Yg=="], "uint8array-extras": ["uint8array-extras@1.5.0", "", {}, "sha512-rvKSBiC5zqCCiDZ9kAOszZcDvdAHwwIKJG33Ykj43OKcWsnmcBRL09YTU4nOeHZ8Y2a7l1MgTd08SBe9A8Qj6A=="], @@ -1660,7 +1662,7 @@ "zod-validation-error": ["zod-validation-error@4.0.2", "", { "peerDependencies": { "zod": "^3.25.0 || ^4.0.0" } }, "sha512-Q6/nZLe6jxuU80qb/4uJ4t5v2VEZ44lzQjPDhYJNztRQ4wyWc6VF3D3Kb/fAuPetZQnhS3hnajCf9CsWesghLQ=="], - "zustand": ["zustand@5.0.14", "", { "peerDependencies": { "@types/react": ">=18.0.0", "immer": ">=9.0.6", "react": ">=18.0.0", "use-sync-external-store": ">=1.2.0" }, "optionalPeers": ["@types/react", "immer", "react", "use-sync-external-store"] }, "sha512-/8tAspM5LMPr28b3fwLYrtdj77ECpfZviaP75CMTnwO8ISyaE4GDIG/9rDDYq/cH9D2Xw2A2RXglLInmVBQB/g=="], + "zustand": ["zustand@5.0.15", "", { "peerDependencies": { "@types/react": ">=18.0.0", "immer": ">=9.0.6", "react": ">=18.0.0", "use-sync-external-store": ">=1.2.0" }, "optionalPeers": ["@types/react", "immer", "react", "use-sync-external-store"] }, "sha512-MpSEjRiBkA9crSYeOUH32rJC7SVqAbm0Fqcqge/bUi2PPoLcBWKOsG+C8mevmpr8TwXHBVkChbbJiyvkE+i/3A=="], "@babel/code-frame/js-tokens": ["js-tokens@4.0.0", "", {}, "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ=="], @@ -1714,21 +1716,21 @@ "@typescript-eslint/eslint-plugin/ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], - "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/parser/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/project-service/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/scope-manager/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/type-utils/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/typescript-estree/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.7.4", "", { "bin": "bin/semver.js" }, "sha512-vFKC2IEtQnVhpT78h1Yp8wzwrf8CM+MzKMHGJZfBtzhZNycRFnXsHk6E5TxIkkMsgNS7mdX3AGB7x2QM2di4lA=="], - "@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/utils/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], - "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.66.0", "", {}, "sha512-H6gcYaSDOyvL3AD/jHUtUFo2jqGgn/F6nuyuZSu0QTesxL+cP4dQoIMrODRofuJC09g64+WgZ6tE19Y1N2YIFQ=="], + "@typescript-eslint/visitor-keys/@typescript-eslint/types": ["@typescript-eslint/types@8.67.0", "", {}, "sha512-sBtgslww8nsMYUjhdPBiSyUqSzT8uR6g93A2QXnQC8+cGdjz0CyaOdqHDRJb1AtORbZCNUJBBeFA/tNR2uQmww=="], "ajv-formats/ajv": ["ajv@8.17.1", "", { "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", "json-schema-traverse": "^1.0.0", "require-from-string": "^2.0.2" } }, "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g=="], @@ -1746,6 +1748,8 @@ "express/cookie": ["cookie@0.7.2", "", {}, "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w=="], + "express-rate-limit/ip-address": ["ip-address@10.0.1", "", {}, "sha512-NWv9YLW4PoW2B7xtzaS3NCot75m6nK7Icdv0o3lfMceJVRfSoQwqD4wEH5rLwoKJwUiZ/rfpiVBhnaF0FK4HoA=="], + "fast-glob/glob-parent": ["glob-parent@5.1.2", "", { "dependencies": { "is-glob": "^4.0.1" } }, "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow=="], "html-encoding-sniffer/@exodus/bytes": ["@exodus/bytes@1.14.0", "", { "peerDependencies": { "@noble/hashes": "^1.8.0 || ^2.0.0" } }, "sha512-YiY1OmY6Qhkvmly8vZiD8wZRpW/npGZNg+0Sk8mstxirRHCg6lolHt5tSODCfuNPE/fBsAqRwDJE417x7jDDHA=="], @@ -1778,6 +1782,8 @@ "restore-cursor/onetime": ["onetime@7.0.0", "", { "dependencies": { "mimic-function": "^5.0.0" } }, "sha512-VXJjc87FScF88uafS3JllDgvAm+c/Slfz06lorj2uAY34rlUu0Nt+v8wreiImcrgAjjIHp1rXpTDlLOGw29WwQ=="], + "rolldown/@oxc-project/types": ["@oxc-project/types@0.142.0", "", {}, "sha512-7W+2q5AKQVU36fkaryontrHn3YDt1RyUYXatw9i5H8ocYe2sPKSFB6eS8WNPeRKiN1qAWWZUPm7gwFzJGrccqQ=="], + "router/path-to-regexp": ["path-to-regexp@8.3.0", "", {}, "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA=="], "shadcn/@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], diff --git a/frontend/package.json b/frontend/package.json index 8692371cac..26e2cba030 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -19,7 +19,7 @@ "test:browser-smoke": "playwright test --config browser-smoke/playwright.config.ts" }, "dependencies": { - "@hookform/resolvers": "^5.7.1", + "@hookform/resolvers": "^5.8.0", "@tailwindcss/vite": "^4.3.3", "@tanstack/react-query": "^5.101.4", "class-variance-authority": "^0.7.1", @@ -28,26 +28,26 @@ "i18next": "^26.3.6", "i18next-browser-languagedetector": "^8.2.1", "input-otp": "^1.4.2", - "lucide-react": "^1.30.0", + "lucide-react": "^1.31.0", "radix-ui": "^1.6.7", "react": "^19.2.8", "react-day-picker": "^10.0.1", "react-dom": "^19.2.8", - "react-hook-form": "^7.84.0", + "react-hook-form": "^7.85.0", "react-i18next": "^17.0.11", "react-router-dom": "^7.18.2", "recharts": "^3.10.1", - "sonner": "^2.0.7", + "sonner": "^2.0.8", "tailwind-merge": "^3.6.0", "tailwindcss": "^4.3.3", "zod": "^4.4.3", - "zustand": "^5.0.14" + "zustand": "^5.0.15" }, "devDependencies": { "@eslint/js": "^10.0.1", - "@testing-library/jest-dom": "^7.0.0", + "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.2", - "@testing-library/user-event": "^14.6.3", + "@testing-library/user-event": "^14.6.4", "@types/node": "^26.2.0", "@types/react": "^19.2.18", "@types/react-dom": "^19.2.4", @@ -55,16 +55,16 @@ "@vitest/coverage-v8": "^4.1.10", "eslint": "^10.8.1", "eslint-plugin-react-hooks": "^7.1.1", - "eslint-plugin-react-refresh": "^0.5.3", - "globals": "^17.9.0", + "eslint-plugin-react-refresh": "^0.5.4", + "globals": "^17.11.0", "jsdom": "^30.0.1", "msw": "^2.15.0", - "react-doctor": "^0.9.6", - "shadcn": "^4.16.2", + "react-doctor": "^0.9.12", + "shadcn": "^4.18.0", "tw-animate-css": "^1.4.0", "@typescript/native": "npm:typescript@~7.0.2", "typescript": "npm:typescript@~6.0.3", - "typescript-eslint": "^8.66.0", + "typescript-eslint": "^8.67.0", "@playwright/test": "^1.62.1", "vite": "^8.2.1", "vitest": "^4.1.10" From d1f24e03500bb5a9a4e75fc6b6ef5f63bdaa8747 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Aug 2026 13:14:24 +0900 Subject: [PATCH 072/117] chore(ci): bump github/codeql-action/upload-sarif from 4.37.6 to 4.37.7 (#1805) Bumps [github/codeql-action/upload-sarif](https://github.com/github/codeql-action) from 4.37.6 to 4.37.7. - [Release notes](https://github.com/github/codeql-action/releases) - [Changelog](https://github.com/github/codeql-action/blob/main/CHANGELOG.md) - [Commits](https://github.com/github/codeql-action/compare/5595ccaf912efad79be6eef63a5619ff05969be3...ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd) --- updated-dependencies: - dependency-name: github/codeql-action/upload-sarif dependency-version: 4.37.7 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2a560860d6..78b39d39d1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -697,7 +697,7 @@ jobs: ignore-unfixed: true - name: Upload Trivy scan results to GitHub Security - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) with: sarif_file: trivy-results.sarif diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c9cff32d01..73bacc9796 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -212,7 +212,7 @@ jobs: ignore-unfixed: true - name: Upload Trivy scan results to GitHub Security - uses: github/codeql-action/upload-sarif@5595ccaf912efad79be6eef63a5619ff05969be3 + uses: github/codeql-action/upload-sarif@ff2f1c621b7f889edc0d3c761ac2e6a3f8cdb0dd if: always() && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository) with: sarif_file: trivy-results.sarif From ed31b7da3d225aac30fc42c7c0d128f10955b58e Mon Sep 17 00:00:00 2001 From: yshishenya Date: Tue, 18 Aug 2026 07:40:59 +0300 Subject: [PATCH 073/117] feat(api-keys): allow per-key reasoning effort policies (#1642) * feat(api-keys): allow per-key reasoning effort policies * fix(proxy): satisfy reasoning policy type checks * fix(api-keys): address reasoning policy review * fix(api-keys): fail closed during reasoning policy rollout * fix(api-keys): tolerate legacy key row shapes * fix(api-keys): gate policy writes during rollout * test(api-keys): harden policy rollout checks * fix(api-keys): address reasoning policy review * fix(proxy): preserve source reasoning controls * fix(proxy): retain model alias reasoning effort * fix(api-keys): close policy compatibility gaps * fix(proxy): harden reasoning alias resolution * fix(proxy): preserve chat reasoning effort alias * fix(proxy): reject masked enabled thinking aliases * fix(proxy): retain enforced model alias effort * fix(proxy): merge enabled thinking metadata * style(proxy): format thinking alias logic * fix(proxy): normalize provider reasoning aliases before fallback * fix(proxy): preserve implicit source thinking controls * fix(proxy): retain effort-less source thinking metadata * fix(proxy): canonicalize authorized reasoning controls * fix(proxy): restore source reasoning effort * fix(proxy): resolve final reasoning policy review findings * fix(frontend): clear copy feedback timer on unmount * fix(frontend): ignore copy completion after unmount * fix(proxy): sanitize conflicting thinking selectors * ci: retry transient docs checkout * fix(db): merge telemetry and reasoning policy heads * ci: retry transient upstream checks --------- Co-authored-by: Yan Shishenya Co-authored-by: Darafei Praliaskouski --- app/core/exceptions.py | 15 +- app/core/handlers/exceptions.py | 10 +- app/core/openai/chat_requests.py | 3 - app/core/openai/requests.py | 49 +- ...0_add_api_key_allowed_reasoning_efforts.py | 65 ++ app/db/models.py | 8 + app/modules/api_keys/api.py | 4 + app/modules/api_keys/repository.py | 4 + app/modules/api_keys/schemas.py | 3 + app/modules/api_keys/service.py | 113 +++- .../proxy/_service/websocket/helpers.py | 5 +- app/modules/proxy/api.py | 78 ++- app/modules/proxy/request_policy.py | 194 +++++- docs/api-keys.md | 13 + .../apis-enforced-reasoning-before.jpg | Bin 0 -> 276748 bytes docs/screenshots/apis-reasoning-efforts.jpg | Bin 0 -> 266960 bytes .../__integration__/apis-page-flow.test.tsx | 31 +- frontend/src/components/copy-button.test.tsx | 53 ++ frontend/src/components/copy-button.tsx | 29 +- .../components/api-key-create-dialog.test.tsx | 8 + .../components/api-key-create-dialog.tsx | 24 +- .../components/api-key-edit-dialog.test.tsx | 43 ++ .../components/api-key-edit-dialog.tsx | 26 +- .../reasoning-efforts-multi-select.test.tsx | 27 + .../reasoning-efforts-multi-select.tsx | 81 +++ .../src/features/api-keys/schemas.test.ts | 18 + frontend/src/features/api-keys/schemas.ts | 3 + .../apis/components/api-key-info.test.tsx | 6 +- .../features/apis/components/api-key-info.tsx | 9 + frontend/src/i18n/locales/en.json | 5 + frontend/src/i18n/locales/ko.json | 5 + frontend/src/i18n/locales/zh-CN.json | 5 + .../design.md | 152 +++++ .../proposal.md | 45 ++ .../specs/api-keys/spec.md | 84 +++ .../specs/chat-completions-compat/spec.md | 109 ++++ .../specs/responses-api-compat/spec.md | 113 ++++ .../tasks.md | 21 + tests/integration/test_api_keys_api.py | 128 ++++ .../integration/test_model_source_routing.py | 560 ++++++++++++++++ .../test_proxy_websocket_responses.py | 44 ++ tests/unit/test_api_keys_service.py | 106 ++- tests/unit/test_chat_request_mapping.py | 17 + tests/unit/test_db_migrate.py | 100 +++ tests/unit/test_openai_requests.py | 37 +- tests/unit/test_proxy_utils.py | 15 + tests/unit/test_request_policy.py | 613 +++++++++++++++++- 47 files changed, 3025 insertions(+), 56 deletions(-) create mode 100644 app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py create mode 100644 docs/screenshots/apis-enforced-reasoning-before.jpg create mode 100644 docs/screenshots/apis-reasoning-efforts.jpg create mode 100644 frontend/src/features/api-keys/components/reasoning-efforts-multi-select.test.tsx create mode 100644 frontend/src/features/api-keys/components/reasoning-efforts-multi-select.tsx create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/design.md create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/proposal.md create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/specs/api-keys/spec.md create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/specs/chat-completions-compat/spec.md create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/add-api-key-allowed-reasoning-efforts/tasks.md diff --git a/app/core/exceptions.py b/app/core/exceptions.py index 45253dc27c..777ec2dd1b 100644 --- a/app/core/exceptions.py +++ b/app/core/exceptions.py @@ -8,8 +8,15 @@ class AppError(Exception): code: str = "internal_error" message: str = "Unexpected error" - def __init__(self, message: str | None = None, *, code: str | None = None) -> None: + def __init__( + self, + message: str | None = None, + *, + code: str | None = None, + param: str | None = None, + ) -> None: self.message = message or self.__class__.message + self.param = param if code is not None: self.code = code super().__init__(self.message) @@ -30,6 +37,12 @@ class ProxyModelNotAllowed(AppError): error_type = "permission_error" +class ProxyReasoningEffortNotAllowed(AppError): + status_code = 403 + code = "reasoning_effort_not_allowed" + error_type = "permission_error" + + class ProxyRateLimitError(AppError): status_code = 429 code = "rate_limit_exceeded" diff --git a/app/core/handlers/exceptions.py b/app/core/handlers/exceptions.py index 4e29b2afb4..7caef49013 100644 --- a/app/core/handlers/exceptions.py +++ b/app/core/handlers/exceptions.py @@ -30,6 +30,7 @@ ProxyAuthError, ProxyModelNotAllowed, ProxyRateLimitError, + ProxyReasoningEffortNotAllowed, ProxyRequiredCapabilityTransportError, ProxyUpstreamError, ) @@ -57,6 +58,7 @@ _OPENAI_EXCEPTION_TYPES: tuple[type[AppError], ...] = ( ProxyAuthError, ProxyModelNotAllowed, + ProxyReasoningEffortNotAllowed, ProxyRateLimitError, ProxyRequiredCapabilityTransportError, ProxyUpstreamError, @@ -242,10 +244,10 @@ async def _openai_domain_handler(request: Request, exc: AppError) -> JSONRespons status=exc.status_code, outcome="invalid_request", ) - return JSONResponse( - status_code=exc.status_code, - content=openai_error(exc.code, exc.message, error_type=error_type), - ) + error = openai_error(exc.code, exc.message, error_type=error_type) + if exc.param is not None: + error["error"]["param"] = exc.param + return JSONResponse(status_code=exc.status_code, content=error) # --- Domain exceptions: Dashboard envelope --- diff --git a/app/core/openai/chat_requests.py b/app/core/openai/chat_requests.py index cc5af7b65a..3485a18c92 100644 --- a/app/core/openai/chat_requests.py +++ b/app/core/openai/chat_requests.py @@ -144,10 +144,7 @@ def to_responses_request(self) -> ResponsesRequest: stream_options = data.pop("stream_options", None) raw_tools = data.pop("tools", []) raw_tool_choice = data.pop("tool_choice", None) - reasoning_effort = data.pop("reasoning_effort", None) preserve_instruction_roles = _is_json_object_response_format(response_format) - if reasoning_effort is not None and "reasoning" not in data: - data["reasoning"] = {"effort": reasoning_effort} normalize_reasoning_aliases(data) if response_format is not None: _apply_response_format(data, response_format) diff --git a/app/core/openai/requests.py b/app/core/openai/requests.py index 2f58bbf329..53f718f640 100644 --- a/app/core/openai/requests.py +++ b/app/core/openai/requests.py @@ -6,7 +6,7 @@ from dataclasses import dataclass from typing import cast -from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator +from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, field_validator, model_validator from app.core.openai.exceptions import ClientPayloadError from app.core.openai.tool_call_safety import is_downstream_side_effect_tool_call_item @@ -612,6 +612,8 @@ class ResponsesTextControls(BaseModel): class ResponsesRequest(BaseModel): model_config = ConfigDict(extra="allow") + _codex_lb_client_reasoning_effort: str | None = PrivateAttr(default=None) + _codex_lb_provider_reasoning_effort_materialized: bool = PrivateAttr(default=False) @model_validator(mode="before") @classmethod @@ -739,6 +741,7 @@ def to_replay_safety_payload(self) -> JsonObject: class ResponsesCompactRequest(BaseModel): model_config = ConfigDict(extra="allow") + _codex_lb_client_reasoning_effort: str | None = PrivateAttr(default=None) @model_validator(mode="before") @classmethod @@ -1769,6 +1772,7 @@ def _sanitize_interleaved_reasoning_input(payload: MutableJsonObject) -> None: def normalize_reasoning_aliases(payload: MutableJsonObject) -> None: reasoning_effort = payload.pop("reasoningEffort", None) + snake_case_reasoning_effort = payload.pop("reasoning_effort", None) reasoning_summary = payload.pop("reasoningSummary", None) provider_thinking = payload.pop("thinking", None) provider_enable_thinking = payload.pop("enable_thinking", None) @@ -1779,8 +1783,20 @@ def normalize_reasoning_aliases(payload: MutableJsonObject) -> None: else: reasoning_map = {} - if isinstance(reasoning_effort, str) and "effort" not in reasoning_map: - reasoning_map["effort"] = reasoning_effort + existing_effort = reasoning_map.get("effort") + if isinstance(existing_effort, str) and not existing_effort.strip(): + reasoning_map.pop("effort") + + alias_effort = next( + ( + candidate.strip() + for candidate in (reasoning_effort, snake_case_reasoning_effort) + if isinstance(candidate, str) and candidate.strip() + ), + None, + ) + if alias_effort is not None and "effort" not in reasoning_map: + reasoning_map["effort"] = alias_effort if isinstance(reasoning_summary, str) and "summary" not in reasoning_map: reasoning_map["summary"] = reasoning_summary @@ -1804,15 +1820,14 @@ def _normalize_thinking_alias( enable_thinking: JsonValue, ) -> MutableJsonObject | None: if isinstance(thinking, bool): - return {"effort": "medium"} if thinking else None + if thinking: + return {"effort": "medium"} if isinstance(thinking, str): normalized = thinking.strip().lower() - if normalized in {"low", "medium", "high", "xhigh", "max", "ultra"}: + if normalized in {"minimal", "low", "medium", "high", "xhigh", "max", "ultra"}: return {"effort": normalized} if normalized in {"enabled", "true", "on"}: return {"effort": "medium"} - if normalized in {"disabled", "false", "off"}: - return None thinking_mapping = _json_mapping_or_none(thinking) if thinking_mapping is not None: normalized: MutableJsonObject = {} @@ -1822,19 +1837,19 @@ def _normalize_thinking_alias( normalized["effort"] = effort.strip().lower() if isinstance(summary, str) and summary.strip(): normalized["summary"] = summary.strip() - if normalized: - return normalized thinking_type = thinking_mapping.get("type") - if isinstance(thinking_type, str): - normalized_type = thinking_type.strip().lower() - if normalized_type == "enabled": - return {"effort": "medium"} - if normalized_type == "disabled": - return None + if "effort" not in normalized and isinstance(thinking_type, str) and thinking_type.strip().lower() == "enabled": + normalized["effort"] = "medium" enabled = thinking_mapping.get("enabled") - if isinstance(enabled, bool): - return {"effort": "medium"} if enabled else None + if "effort" not in normalized and enabled is True: + normalized["effort"] = "medium" + if "effort" not in normalized and enable_thinking is True: + normalized["effort"] = "medium" + if normalized: + return normalized + # Disabled `thinking` spellings are inactive, not authoritative: a + # separate enabled alias must still participate in policy evaluation. if isinstance(enable_thinking, bool): return {"effort": "medium"} if enable_thinking else None return None diff --git a/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py b/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py new file mode 100644 index 0000000000..12e14906ae --- /dev/null +++ b/app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py @@ -0,0 +1,65 @@ +"""add API-key reasoning effort allowlists + +The nullable column preserves the existing unrestricted policy for every +existing API key. New writes serialize a non-empty canonical JSON list. + +Revision ID: 20260806_030000_add_api_key_allowed_reasoning_efforts +Revises: 20260816_000000_add_account_pending_deletion +Create Date: 2026-08-06 03:00:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260806_030000_add_api_key_allowed_reasoning_efforts" +down_revision = "20260816_000000_add_account_pending_deletion" +branch_labels = None +depends_on = None + +_TABLE = "api_keys" +_COLUMN = "allowed_reasoning_efforts" +_POLICY_CHECK = "ck_api_keys_reasoning_policy_exclusive" + + +def _columns(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(column["name"]) for column in inspector.get_columns(_TABLE) if column.get("name") is not None} + + +def _check_constraints(connection: Connection) -> set[str]: + inspector = sa.inspect(connection) + if not inspector.has_table(_TABLE): + return set() + return {str(constraint["name"]) for constraint in inspector.get_check_constraints(_TABLE) if constraint.get("name")} + + +def upgrade() -> None: + connection = op.get_bind() + columns = _columns(connection) + if _COLUMN not in columns: + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.add_column(sa.Column(_COLUMN, sa.Text(), nullable=True)) + + constraints = _check_constraints(connection) + if _POLICY_CHECK not in constraints: + with op.batch_alter_table(_TABLE) as batch_op: + batch_op.create_check_constraint( + _POLICY_CHECK, + f"{_COLUMN} IS NULL OR enforced_reasoning_effort IS NULL", + ) + + +def downgrade() -> None: + connection = op.get_bind() + columns = _columns(connection) + if _COLUMN not in columns: + return + with op.batch_alter_table(_TABLE) as batch_op: + if _POLICY_CHECK in _check_constraints(connection): + batch_op.drop_constraint(_POLICY_CHECK, type_="check") + batch_op.drop_column(_COLUMN) diff --git a/app/db/models.py b/app/db/models.py index 2197aaa048..f5634cdb8a 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -7,6 +7,7 @@ from sqlalchemy import ( BigInteger, Boolean, + CheckConstraint, DateTime, Float, ForeignKey, @@ -1145,6 +1146,12 @@ class ApiFirewallAllowlist(Base): class ApiKey(Base): __tablename__ = "api_keys" + __table_args__ = ( + CheckConstraint( + "allowed_reasoning_efforts IS NULL OR enforced_reasoning_effort IS NULL", + name="ck_api_keys_reasoning_policy_exclusive", + ), + ) id: Mapped[str] = mapped_column(String, primary_key=True) name: Mapped[str] = mapped_column(String, nullable=False) @@ -1159,6 +1166,7 @@ class ApiKey(Base): ) enforced_model: Mapped[str | None] = mapped_column(String, nullable=True) enforced_reasoning_effort: Mapped[str | None] = mapped_column(String, nullable=True) + allowed_reasoning_efforts: Mapped[str | None] = mapped_column(Text, nullable=True) enforced_service_tier: Mapped[str | None] = mapped_column(String, nullable=True) traffic_class: Mapped[str] = mapped_column( String, diff --git a/app/modules/api_keys/api.py b/app/modules/api_keys/api.py index 9f8fc122c6..09323c2874 100644 --- a/app/modules/api_keys/api.py +++ b/app/modules/api_keys/api.py @@ -46,6 +46,7 @@ def _to_response(row: ApiKeyData) -> ApiKeyResponse: apply_to_codex_model=row.apply_to_codex_model, enforced_model=row.enforced_model, enforced_reasoning_effort=row.enforced_reasoning_effort, + allowed_reasoning_efforts=row.allowed_reasoning_efforts, enforced_service_tier=row.enforced_service_tier, traffic_class=row.traffic_class, transport_policy_override=row.transport_policy_override, @@ -135,6 +136,7 @@ async def create_api_key( apply_to_codex_model=payload.apply_to_codex_model, enforced_model=payload.enforced_model, enforced_reasoning_effort=payload.enforced_reasoning_effort, + allowed_reasoning_efforts=payload.allowed_reasoning_efforts, enforced_service_tier=payload.enforced_service_tier, traffic_class=payload.traffic_class or "foreground", transport_policy_override=payload.transport_policy_override, @@ -195,6 +197,8 @@ async def update_api_key( enforced_model_set="enforced_model" in fields, enforced_reasoning_effort=payload.enforced_reasoning_effort, enforced_reasoning_effort_set="enforced_reasoning_effort" in fields, + allowed_reasoning_efforts=payload.allowed_reasoning_efforts, + allowed_reasoning_efforts_set="allowed_reasoning_efforts" in fields, enforced_service_tier=payload.enforced_service_tier, enforced_service_tier_set="enforced_service_tier" in fields, traffic_class=payload.traffic_class, diff --git a/app/modules/api_keys/repository.py b/app/modules/api_keys/repository.py index 4a26949c58..02cdea8de2 100644 --- a/app/modules/api_keys/repository.py +++ b/app/modules/api_keys/repository.py @@ -354,6 +354,7 @@ async def update( apply_to_codex_model: bool | _Unset = _UNSET, enforced_model: str | None | _Unset = _UNSET, enforced_reasoning_effort: str | None | _Unset = _UNSET, + allowed_reasoning_efforts: str | None | _Unset = _UNSET, enforced_service_tier: str | None | _Unset = _UNSET, traffic_class: str | _Unset = _UNSET, transport_policy_override: str | None | _Unset = _UNSET, @@ -384,6 +385,9 @@ async def update( if enforced_reasoning_effort is not _UNSET: assert enforced_reasoning_effort is None or isinstance(enforced_reasoning_effort, str) row.enforced_reasoning_effort = enforced_reasoning_effort + if allowed_reasoning_efforts is not _UNSET: + assert allowed_reasoning_efforts is None or isinstance(allowed_reasoning_efforts, str) + row.allowed_reasoning_efforts = allowed_reasoning_efforts if enforced_service_tier is not _UNSET: assert enforced_service_tier is None or isinstance(enforced_service_tier, str) row.enforced_service_tier = enforced_service_tier diff --git a/app/modules/api_keys/schemas.py b/app/modules/api_keys/schemas.py index a8616aa1bc..6947c3dfc0 100644 --- a/app/modules/api_keys/schemas.py +++ b/app/modules/api_keys/schemas.py @@ -32,6 +32,7 @@ class ApiKeyCreateRequest(DashboardModel): enforced_reasoning_effort: str | None = Field( default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) + allowed_reasoning_efforts: list[str] | None = None enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None @@ -51,6 +52,7 @@ class ApiKeyUpdateRequest(DashboardModel): enforced_reasoning_effort: str | None = Field( default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) + allowed_reasoning_efforts: list[str] | None = None enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None @@ -79,6 +81,7 @@ class ApiKeyResponse(DashboardModel): apply_to_codex_model: bool = False enforced_model: str | None enforced_reasoning_effort: str | None + allowed_reasoning_efforts: list[str] | None enforced_service_tier: str | None traffic_class: str transport_policy_override: str | None = None diff --git a/app/modules/api_keys/service.py b/app/modules/api_keys/service.py index e3675fa29b..b6e3ca8f1a 100644 --- a/app/modules/api_keys/service.py +++ b/app/modules/api_keys/service.py @@ -11,7 +11,7 @@ from math import ceil from typing import Protocol -from sqlalchemy.exc import OperationalError +from sqlalchemy.exc import IntegrityError, OperationalError from app.core.auth.api_key_cache import get_api_key_cache from app.core.cache.invalidation import NAMESPACE_API_KEY, get_cache_invalidation_poller @@ -51,6 +51,7 @@ TRAFFIC_CLASS_OPPORTUNISTIC = "opportunistic" _SUPPORTED_TRAFFIC_CLASSES = frozenset({TRAFFIC_CLASS_FOREGROUND, TRAFFIC_CLASS_OPPORTUNISTIC}) _SUPPORTED_TRANSPORT_POLICY_OVERRIDES = frozenset({"smart", "always_http", "always_websocket"}) +_REASONING_POLICY_EXCLUSIVE_CONSTRAINT = "ck_api_keys_reasoning_policy_exclusive" class ApiKeysRepositoryProtocol(Protocol): @@ -87,6 +88,7 @@ async def update( apply_to_codex_model: bool | _Unset = ..., enforced_model: str | None | _Unset = ..., enforced_reasoning_effort: str | None | _Unset = ..., + allowed_reasoning_efforts: str | None | _Unset = ..., enforced_service_tier: str | None | _Unset = ..., traffic_class: str | _Unset = ..., transport_policy_override: str | None | _Unset = ..., @@ -272,6 +274,7 @@ class ApiKeyCreateData: apply_to_codex_model: bool = False enforced_model: str | None = None enforced_reasoning_effort: str | None = None + allowed_reasoning_efforts: list[str] | None = None enforced_service_tier: str | None = None traffic_class: str = TRAFFIC_CLASS_FOREGROUND transport_policy_override: str | None = None @@ -294,6 +297,8 @@ class ApiKeyUpdateData: enforced_model_set: bool = False enforced_reasoning_effort: str | None = None enforced_reasoning_effort_set: bool = False + allowed_reasoning_efforts: list[str] | None = None + allowed_reasoning_efforts_set: bool = False enforced_service_tier: str | None = None enforced_service_tier_set: bool = False traffic_class: str | None = None @@ -328,6 +333,7 @@ class ApiKeyData: is_active: bool created_at: datetime last_used_at: datetime | None + allowed_reasoning_efforts: list[str] | None = None apply_to_codex_model: bool = False traffic_class: str = TRAFFIC_CLASS_FOREGROUND transport_policy_override: str | None = None @@ -470,11 +476,16 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: assigned_source_ids = await self._resolve_assigned_source_ids(payload.assigned_source_ids) enforced_model = _normalize_model_slug(payload.enforced_model) enforced_reasoning_effort = _normalize_reasoning_effort(payload.enforced_reasoning_effort) + allowed_reasoning_efforts = _normalize_allowed_reasoning_efforts(payload.allowed_reasoning_efforts) enforced_service_tier = _normalize_service_tier(payload.enforced_service_tier) traffic_class = _normalize_traffic_class(payload.traffic_class) transport_policy_override = _normalize_transport_policy_override(payload.transport_policy_override) usage_sections = _normalize_usage_sections(payload.usage_sections) _validate_model_enforcement(enforced_model=enforced_model, allowed_models=normalized_allowed_models) + _validate_reasoning_effort_policy( + enforced_reasoning_effort=enforced_reasoning_effort, + allowed_reasoning_efforts=allowed_reasoning_efforts, + ) row = ApiKey( id=str(__import__("uuid").uuid4()), name=_normalize_name(payload.name), @@ -484,6 +495,7 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: apply_to_codex_model=bool(payload.apply_to_codex_model), enforced_model=enforced_model, enforced_reasoning_effort=enforced_reasoning_effort, + allowed_reasoning_efforts=_serialize_allowed_reasoning_efforts(allowed_reasoning_efforts), enforced_service_tier=enforced_service_tier, account_assignment_scope_enabled=bool(assigned_account_ids), source_assignment_scope_enabled=bool(assigned_source_ids), @@ -508,8 +520,12 @@ async def create_key(self, payload: ApiKeyCreateData) -> ApiKeyCreatedData: await self._repository.upsert_limits(created.id, limit_rows, commit=False) await self._repository.commit() - except Exception: + except Exception as exc: await self._repository.rollback() + if isinstance(exc, IntegrityError) and _is_reasoning_policy_constraint_error(exc): + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) from exc raise created = await self._repository.get_by_id(created.id) @@ -609,6 +625,11 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData else: enforced_reasoning_effort = None + if payload.allowed_reasoning_efforts_set: + allowed_reasoning_efforts = _normalize_allowed_reasoning_efforts(payload.allowed_reasoning_efforts) + else: + allowed_reasoning_efforts = None + if payload.enforced_service_tier_set: enforced_service_tier = _normalize_service_tier(payload.enforced_service_tier) else: @@ -636,6 +657,22 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData allowed_models=effective_allowed_models, ) + if payload.enforced_reasoning_effort_set or payload.allowed_reasoning_efforts_set: + effective_enforced_reasoning_effort = ( + enforced_reasoning_effort + if payload.enforced_reasoning_effort_set + else _normalize_reasoning_effort_lenient(existing.enforced_reasoning_effort) + ) + effective_allowed_reasoning_efforts = ( + allowed_reasoning_efforts + if payload.allowed_reasoning_efforts_set + else _deserialize_allowed_reasoning_efforts(existing.allowed_reasoning_efforts) + ) + _validate_reasoning_effort_policy( + enforced_reasoning_effort=effective_enforced_reasoning_effort, + allowed_reasoning_efforts=effective_allowed_reasoning_efforts, + ) + limit_rows: list[ApiKeyLimit] | None = None if payload.limits_set: now = utcnow() @@ -664,6 +701,11 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData enforced_reasoning_effort=( enforced_reasoning_effort if payload.enforced_reasoning_effort_set else _UNSET ), + allowed_reasoning_efforts=( + _serialize_allowed_reasoning_efforts(allowed_reasoning_efforts) + if payload.allowed_reasoning_efforts_set + else _UNSET + ), enforced_service_tier=(enforced_service_tier if payload.enforced_service_tier_set else _UNSET), traffic_class=traffic_class_update, transport_policy_override=transport_policy_override_update, @@ -688,8 +730,12 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData await self._repository.upsert_limits(key_id, limit_rows, commit=False) await self._repository.commit() - except Exception: + except Exception as exc: await self._repository.rollback() + if isinstance(exc, IntegrityError) and _is_reasoning_policy_constraint_error(exc): + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) from exc raise if ( @@ -701,6 +747,7 @@ async def update_key(self, key_id: str, payload: ApiKeyUpdateData) -> ApiKeyData or payload.apply_to_codex_model_set or payload.enforced_model_set or payload.enforced_reasoning_effort_set + or payload.allowed_reasoning_efforts_set or payload.enforced_service_tier_set or payload.traffic_class_set or payload.transport_policy_override_set @@ -1336,6 +1383,12 @@ def _serialize_allowed_models(allowed_models: list[str] | None) -> str | None: return json.dumps(allowed_models) +def _serialize_allowed_reasoning_efforts(allowed_reasoning_efforts: list[str] | None) -> str | None: + if allowed_reasoning_efforts is None: + return None + return json.dumps(allowed_reasoning_efforts) + + def _deserialize_allowed_models(payload: str | None) -> list[str] | None: if payload is None: return None @@ -1346,6 +1399,22 @@ def _deserialize_allowed_models(payload: str | None) -> list[str] | None: return models +def _deserialize_allowed_reasoning_efforts(payload: str | None) -> list[str] | None: + if payload is None: + return None + try: + parsed = json.loads(payload) + if not isinstance(parsed, list): + return [] + return _normalize_allowed_reasoning_efforts(parsed) + except (ApiKeyValidationError, TypeError, json.JSONDecodeError): + return [] + + +def _is_reasoning_policy_constraint_error(exc: IntegrityError) -> bool: + return _REASONING_POLICY_EXCLUSIVE_CONSTRAINT in str(exc).lower() + + def _normalize_allowed_models(allowed_models: list[str] | None) -> list[str] | None: if allowed_models is None: return None @@ -1389,7 +1458,9 @@ def _normalize_model_slug(value: str | None) -> str | None: return normalized -_SUPPORTED_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"}) +_REASONING_EFFORT_ORDER = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") +_SUPPORTED_REASONING_EFFORTS = frozenset({"none", *_REASONING_EFFORT_ORDER}) +_SUPPORTED_SELECTABLE_REASONING_EFFORTS = frozenset(_REASONING_EFFORT_ORDER) _SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default", "priority", "flex"}) @@ -1422,6 +1493,25 @@ def _normalize_reasoning_effort_lenient(value: str | None) -> str | None: return None +def _normalize_allowed_reasoning_efforts(values: list[str] | None) -> list[str] | None: + if values is None: + return None + + normalized: set[str] = set() + for value in values: + if not isinstance(value, str): + raise ApiKeyValidationError("Allowed reasoning efforts must be strings") + effort = value.strip().lower() + if effort not in _SUPPORTED_SELECTABLE_REASONING_EFFORTS: + options = ", ".join(_REASONING_EFFORT_ORDER) + raise ApiKeyValidationError(f"Unsupported allowed reasoning effort '{effort}'. Expected one of: {options}") + normalized.add(effort) + + if not normalized: + raise ApiKeyValidationError("Allowed reasoning efforts must not be empty") + return [effort for effort in _REASONING_EFFORT_ORDER if effort in normalized] + + def _normalize_service_tier(value: str | None) -> str | None: if value is None: return None @@ -1492,6 +1582,17 @@ def _validate_model_enforcement(*, enforced_model: str | None, allowed_models: l ) +def _validate_reasoning_effort_policy( + *, + enforced_reasoning_effort: str | None, + allowed_reasoning_efforts: list[str] | None, +) -> None: + if enforced_reasoning_effort is not None and allowed_reasoning_efforts is not None: + raise ApiKeyValidationError( + "enforced_reasoning_effort and allowed_reasoning_efforts cannot be configured together" + ) + + def _to_limit_rule_data(limit: ApiKeyLimit) -> LimitRuleData: return LimitRuleData( id=limit.id, @@ -1683,6 +1784,7 @@ def _to_created_data(data: ApiKeyData, key: str) -> ApiKeyCreatedData: apply_to_codex_model=data.apply_to_codex_model, enforced_model=data.enforced_model, enforced_reasoning_effort=data.enforced_reasoning_effort, + allowed_reasoning_efforts=data.allowed_reasoning_efforts, enforced_service_tier=data.enforced_service_tier, traffic_class=data.traffic_class, transport_policy_override=data.transport_policy_override, @@ -1718,6 +1820,9 @@ def _to_api_key_data( apply_to_codex_model=getattr(row, "apply_to_codex_model", False), enforced_model=_normalize_model_slug(row.enforced_model), enforced_reasoning_effort=_normalize_reasoning_effort_lenient(row.enforced_reasoning_effort), + allowed_reasoning_efforts=_deserialize_allowed_reasoning_efforts( + getattr(row, "allowed_reasoning_efforts", None) + ), enforced_service_tier=_normalize_service_tier_lenient(row.enforced_service_tier), traffic_class=_normalize_traffic_class_lenient(getattr(row, "traffic_class", TRAFFIC_CLASS_FOREGROUND)), transport_policy_override=_normalize_transport_policy_override_lenient( diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index ee600a9db6..0cfc86b406 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -1864,9 +1864,12 @@ def _is_websocket_response_create(payload: dict[str, JsonValue]) -> bool: def _app_error_to_websocket_event(exc: AppError) -> dict[str, JsonValue]: + payload = openai_error(exc.code, exc.message, error_type=getattr(exc, "error_type", "server_error")) + if exc.param is not None: + payload["error"]["param"] = exc.param return _wrapped_websocket_error_event( exc.status_code, - openai_error(exc.code, exc.message, error_type=getattr(exc, "error_type", "server_error")), + payload, ) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index abefd6d389..e09266f9dd 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -246,9 +246,11 @@ enforce_strict_text_format, model_alias_requests_fast_mode, normalize_responses_request_payload, + normalize_source_reasoning_aliases, openai_client_payload_error, openai_validation_error, resolve_model_alias, + resolve_wire_reasoning_effort, responses_source_route_excluded, restore_source_reasoning_effort, sanitize_source_chat_payload, @@ -1127,6 +1129,7 @@ async def responses( codex_session_affinity=True, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, # The Codex CLI consumes codex.* vendor events and the upstream's # native event ordering, while OpenAI SDK clients pointed at this @@ -1273,6 +1276,7 @@ async def v1_responses( codex_session_affinity=False, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, ) else: @@ -1284,6 +1288,7 @@ async def v1_responses( codex_session_affinity=False, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, ) return _mark_subscription_prompt_cache_fallback(response, responses_payload) @@ -1346,6 +1351,7 @@ async def internal_bridge_responses( codex_session_affinity=forwarded_request_context.context.codex_session_affinity, openai_cache_affinity=True, prefer_http_bridge=True, + api_key_policy_already_applied=True, skip_limit_enforcement=skip_limit_enforcement, api_key_reservation_override=forwarded_request_context.context.reservation, include_rate_limit_headers=False, @@ -4129,6 +4135,11 @@ async def v1_chat_completions( source=source, model=request_model, api_key=api_key, + allowed_reasoning_effort=( + responses_payload._codex_lb_client_reasoning_effort + if api_key is not None and api_key.allowed_reasoning_efforts is not None + else None + ), reservation=reservation, rate_limit_headers=rate_limit_headers, ) @@ -4462,6 +4473,37 @@ async def _source_responses_response( request_usage_budget=estimate_api_key_request_usage(payload), ) source_payload = payload.model_dump_for_forwarding() + preserve_materialized_provider_alias = payload._codex_lb_provider_reasoning_effort_materialized and ( + api_key is None or (api_key.enforced_reasoning_effort is None and api_key.allowed_reasoning_efforts is None) + ) + if preserve_materialized_provider_alias: + reasoning = source_payload.get("reasoning") + if isinstance(reasoning, dict): + reasoning = {key: value for key, value in reasoning.items() if key != "effort"} + if reasoning: + source_payload["reasoning"] = reasoning + else: + source_payload.pop("reasoning") + if api_key is not None and ( + api_key.enforced_reasoning_effort is not None + or (api_key.allowed_reasoning_efforts is not None and payload._codex_lb_client_reasoning_effort is not None) + ): + normalize_source_reasoning_aliases(source_payload) + source_reasoning_effort = ( + api_key.enforced_reasoning_effort + if api_key is not None and api_key.enforced_reasoning_effort is not None + else payload._codex_lb_client_reasoning_effort + ) + if source_reasoning_effort is not None and not preserve_materialized_provider_alias: + source_reasoning_effort = resolve_wire_reasoning_effort(source_reasoning_effort) + reasoning = source_payload.get("reasoning") + if isinstance(reasoning, dict): + source_payload["reasoning"] = { + **reasoning, + "effort": source_reasoning_effort, + } + else: + source_payload["reasoning"] = {"effort": source_reasoning_effort} strip_replayed_tool_call_namespaces_from_payload(source_payload) source_payload["stream"] = bool(payload.stream) _apply_source_response_request_overrides(source_payload, source_model_request_overrides(source, payload.model)) @@ -4764,13 +4806,19 @@ async def _source_chat_completion_response( source: ModelSource, model: str, api_key: ApiKeyData | None, + allowed_reasoning_effort: str | None = None, reservation: ApiKeyUsageReservationData | None, rate_limit_headers: Mapping[str, str], ) -> Response: source_payload = payload.model_dump(mode="json", exclude_none=True) source_payload["model"] = model source_payload["stream"] = bool(payload.stream) - apply_api_key_enforcement_to_chat_payload(source_payload, api_key) + apply_api_key_enforcement_to_chat_payload( + source_payload, + api_key, + allowed_reasoning_effort=allowed_reasoning_effort, + materialize_allowed_reasoning_effort=allowed_reasoning_effort is not None, + ) sanitize_source_chat_payload( source_payload, allow_reasoning=source_model_supports_reasoning(source, model), @@ -5276,6 +5324,7 @@ async def _stream_responses( forwarded_client_ip: str | None = None, enforce_openai_sdk_contract: bool = True, native_codex_heartbeat: bool = False, + api_key_policy_already_applied: bool = False, prohibit_fast_mode: bool = False, ) -> Response: # Owner-forwarded payloads have already passed API-key enforcement, @@ -5284,11 +5333,13 @@ async def _stream_responses( # signed effective tier: an owner with an older/staler model snapshot must # not re-add a tier that the origin authoritatively removed. forwarded_effective_service_tier = payload.service_tier if forwarded_request else None - service_tier_was_enforced = apply_api_key_enforcement( - payload, - api_key, - prohibit_fast_mode=prohibit_fast_mode, - ).service_tier_was_enforced + service_tier_was_enforced = False + if not api_key_policy_already_applied: + service_tier_was_enforced = apply_api_key_enforcement( + payload, + api_key, + prohibit_fast_mode=prohibit_fast_mode, + ).service_tier_was_enforced if forwarded_request: payload.service_tier = forwarded_effective_service_tier else: @@ -5737,15 +5788,16 @@ async def _collect_responses( openai_cache_affinity: bool = False, suppress_text_done_events: bool = False, prefer_http_bridge: bool = False, + api_key_policy_already_applied: bool = False, prohibit_fast_mode: bool = False, ) -> Response: - # The replaced effort is discarded: this path is subscription-only, so the - # rewrite that works around the backend hang must stick. - service_tier_was_enforced = apply_api_key_enforcement( - payload, - api_key, - prohibit_fast_mode=prohibit_fast_mode, - ).service_tier_was_enforced + service_tier_was_enforced = False + if not api_key_policy_already_applied: + service_tier_was_enforced = apply_api_key_enforcement( + payload, + api_key, + prohibit_fast_mode=prohibit_fast_mode, + ).service_tier_was_enforced apply_enforced_service_tier_model_fallback( payload, service_tier_was_enforced=service_tier_was_enforced, diff --git a/app/modules/proxy/request_policy.py b/app/modules/proxy/request_policy.py index 1c5547bf75..cc3d8c5d5f 100644 --- a/app/modules/proxy/request_policy.py +++ b/app/modules/proxy/request_policy.py @@ -6,7 +6,7 @@ from pydantic import ValidationError from app.core.errors import OpenAIErrorEnvelope, openai_error -from app.core.exceptions import ProxyModelNotAllowed +from app.core.exceptions import ProxyModelNotAllowed, ProxyReasoningEffortNotAllowed from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import ModelRegistry, get_model_registry from app.core.openai.requests import ( @@ -14,6 +14,7 @@ ResponsesReasoning, ResponsesRequest, extract_input_file_ids, + normalize_reasoning_aliases, responses_input_uses_lite_tools, ) from app.core.openai.strict_schema import ( @@ -129,6 +130,125 @@ def validate_model_access(api_key: ApiKeyData | None, model: str | None) -> None raise ProxyModelNotAllowed(f"This API key does not have access to model '{model}'") +def validate_reasoning_effort_access(api_key: ApiKeyData | None, effort: str | None) -> None: + if api_key is None: + return + allowed_reasoning_efforts = getattr(api_key, "allowed_reasoning_efforts", None) + if allowed_reasoning_efforts is None or effort is None: + return + normalized_effort = effort.strip().lower() + if normalized_effort in allowed_reasoning_efforts: + return + logger.info( + "api_key_reasoning_effort_not_allowed request_id=%s key_id=%s reasoning_effort=%s", + get_request_id(), + api_key.id, + normalized_effort, + ) + raise ProxyReasoningEffortNotAllowed( + f"This API key does not have access to reasoning effort '{normalized_effort}'", + param="reasoning.effort", + ) + + +def _client_reasoning_effort(payload: ResponsesRequest | ResponsesCompactRequest) -> str | None: + """Return the effort selected by the client before wire normalization. + + Cursor encodes its effort in an accepted model alias, where ``xhigh`` is + later lowered to the upstream's ``high`` value. API-key policies are an + operator-facing client-plane control, so they must compare against the + original selection rather than that wire representation. + """ + model_effort = _client_reasoning_effort_from_model(payload.model) + if model_effort is not None: + return model_effort + + reasoning = payload.reasoning.model_dump(mode="json", exclude_none=True) if payload.reasoning is not None else None + if is_json_mapping(reasoning): + effort = reasoning.get("effort") + if isinstance(effort, str) and effort.strip(): + return effort.strip().lower() + return _client_reasoning_effort_from_provider_aliases(payload) + + +def _client_reasoning_effort_from_provider_aliases( + payload: ResponsesRequest | ResponsesCompactRequest, +) -> str | None: + reasoning = payload.reasoning.model_dump(mode="json", exclude_none=True) if payload.reasoning is not None else None + extra = payload.model_extra + if isinstance(extra, dict): + alias_payload = dict(extra) + if reasoning is not None: + alias_payload["reasoning"] = reasoning + normalize_reasoning_aliases(alias_payload) + normalized_reasoning = alias_payload.get("reasoning") + if is_json_mapping(normalized_reasoning): + extra_effort = normalized_reasoning.get("effort") + if isinstance(extra_effort, str) and extra_effort.strip(): + return extra_effort.strip().lower() + + return None + + +def _materialize_provider_reasoning_effort( + payload: ResponsesRequest | ResponsesCompactRequest, + effort: str | None, +) -> None: + existing_effort = payload.reasoning.effort if payload.reasoning is not None else None + if effort is None or (isinstance(existing_effort, str) and existing_effort.strip()): + return + if payload.reasoning is None: + payload.reasoning = ResponsesReasoning(effort=effort) + else: + payload.reasoning.effort = effort + if isinstance(payload, ResponsesRequest): + payload._codex_lb_provider_reasoning_effort_materialized = True + + +def _client_reasoning_effort_from_model(model: str | None) -> str | None: + alias = _resolve_model_alias_parts(model) + if alias is not None: + normalized_model = model.strip().lower() if isinstance(model, str) else "" + suffix = normalized_model[len(alias[0]) + 1 :] + tokens = {token for token in suffix.split("-") if token} + if "xhigh" in tokens or "extra" in tokens: + return "xhigh" + if alias[1] is not None: + return alias[1] + return None + + +def normalize_source_reasoning_aliases(payload: dict[str, JsonValue]) -> None: + """Align effort-bearing aliases while preserving unrelated source controls.""" + provider_thinking = payload.get("thinking") + preserve_provider_thinking = False + if "thinking" in payload: + probe: dict[str, JsonValue] = {"thinking": provider_thinking} + normalize_reasoning_aliases(probe) + normalized_reasoning = probe.get("reasoning") + normalized_effort = normalized_reasoning.get("effort") if is_json_mapping(normalized_reasoning) else None + thinking_mapping = provider_thinking if is_json_mapping(provider_thinking) else None + thinking_type = thinking_mapping.get("type") if thinking_mapping is not None else None + is_inactive = thinking_mapping is not None and ( + thinking_mapping.get("enabled") is False + or (isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled") + ) + preserve_provider_thinking = ( + thinking_mapping is not None + and not is_inactive + and not (isinstance(normalized_effort, str) and bool(normalized_effort.strip())) + ) + if preserve_provider_thinking: + payload.pop("thinking", None) + normalize_reasoning_aliases(payload) + if preserve_provider_thinking and thinking_mapping is not None: + preserved_thinking = dict(thinking_mapping.items()) + preserved_effort = preserved_thinking.get("effort") + if isinstance(preserved_effort, str) and not preserved_effort.strip(): + preserved_thinking.pop("effort") + payload["thinking"] = preserved_thinking + + class ApiKeyEnforcementResult(NamedTuple): """What :func:`apply_api_key_enforcement` observed while mutating the payload. @@ -157,13 +277,18 @@ def apply_api_key_enforcement( equal the enforced value (including after ``fast`` canonicalizes to ``priority``). """ + client_reasoning_effort = payload._codex_lb_client_reasoning_effort or _client_reasoning_effort(payload) + payload._codex_lb_client_reasoning_effort = client_reasoning_effort + provider_reasoning_effort = _client_reasoning_effort_from_provider_aliases(payload) normalize_upstream_model_alias(payload, prohibit_fast_mode=prohibit_fast_mode) if api_key is None: + _materialize_provider_reasoning_effort(payload, provider_reasoning_effort) pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) return ApiKeyEnforcementResult(False, pre_normalization_effort) if api_key.enforced_model: + enforced_model_reasoning_effort = _client_reasoning_effort_from_model(api_key.enforced_model) requested_model = payload.model if requested_model != api_key.enforced_model: logger.info( @@ -174,6 +299,9 @@ def apply_api_key_enforcement( api_key.enforced_model, ) payload.model = api_key.enforced_model + if enforced_model_reasoning_effort is not None: + client_reasoning_effort = enforced_model_reasoning_effort + payload._codex_lb_client_reasoning_effort = client_reasoning_effort normalize_upstream_model_alias(payload, prohibit_fast_mode=prohibit_fast_mode) if ( responses_input_uses_lite_tools(payload.input) @@ -203,6 +331,15 @@ def apply_api_key_enforcement( api_key.enforced_reasoning_effort, ) + _materialize_provider_reasoning_effort(payload, provider_reasoning_effort) + if client_reasoning_effort is not None: + validate_reasoning_effort_access(api_key, client_reasoning_effort) + if ( + payload.reasoning is not None + and isinstance(payload.reasoning.effort, str) + and payload.reasoning.effort.strip().lower() == client_reasoning_effort + ): + payload.reasoning.effort = client_reasoning_effort pre_normalization_effort = normalize_unsupported_reasoning_effort(payload, registry=registry) service_tier_was_enforced = False @@ -319,6 +456,9 @@ def sanitize_source_chat_payload( def apply_api_key_enforcement_to_chat_payload( payload: dict[str, JsonValue], api_key: ApiKeyData | None, + *, + allowed_reasoning_effort: str | None = None, + materialize_allowed_reasoning_effort: bool = False, ) -> None: """Mirror :func:`apply_api_key_enforcement` onto a chat-completions wire payload. @@ -327,6 +467,58 @@ def apply_api_key_enforcement_to_chat_payload( applied to the outbound dict as well or the upstream receives the caller's values while accounting uses the enforced ones. """ + if allowed_reasoning_effort is not None: + wire_effort = resolve_wire_reasoning_effort(allowed_reasoning_effort) + # Chat requests can express the same setting through several provider + # aliases. Once the Responses conversion has authorized one effective + # choice, make every caller-supplied alias agree without adding fields + # that the selected source may not accept. + if "reasoning_effort" in payload: + payload["reasoning_effort"] = wire_effort + if "reasoningEffort" in payload: + payload["reasoningEffort"] = wire_effort + if "thinking" in payload: + thinking = payload["thinking"] + if isinstance(thinking, dict): + thinking_effort = thinking.get("effort") + if isinstance(thinking_effort, str) and not thinking_effort.strip(): + thinking = {**thinking} + thinking.pop("effort") + payload["thinking"] = thinking + if isinstance(thinking_effort, str) and thinking_effort.strip(): + aligned_thinking = {**thinking, "effort": wire_effort} + thinking_type = aligned_thinking.get("type") + if isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled": + aligned_thinking.pop("type") + if aligned_thinking.get("enabled") is False: + aligned_thinking.pop("enabled") + payload["thinking"] = aligned_thinking + else: + thinking_type = thinking.get("type") + is_inactive = thinking.get("enabled") is False or ( + isinstance(thinking_type, str) and thinking_type.strip().lower() == "disabled" + ) + selects_implicit_medium = thinking.get("enabled") is True or ( + isinstance(thinking_type, str) and thinking_type.strip().lower() == "enabled" + ) + if is_inactive or (selects_implicit_medium and wire_effort != "medium"): + payload.pop("thinking") + else: + payload["thinking"] = wire_effort + if "enable_thinking" in payload: + if wire_effort == "medium": + payload["enable_thinking"] = True + else: + payload.pop("enable_thinking", None) + reasoning = payload.get("reasoning") + if isinstance(reasoning, dict): + payload["reasoning"] = {**reasoning, "effort": wire_effort} + if materialize_allowed_reasoning_effort and not any( + key in payload + for key in ("reasoning_effort", "reasoningEffort", "thinking", "enable_thinking", "reasoning") + ): + payload["reasoning_effort"] = wire_effort + if api_key is None: return diff --git a/docs/api-keys.md b/docs/api-keys.md index 3cd9c13c3a..08ca388a15 100644 --- a/docs/api-keys.md +++ b/docs/api-keys.md @@ -27,6 +27,19 @@ Keys can also be scoped to specific accounts, so a key draws quota only from the ![API keys with assigned accounts](screenshots/apis-assigned-accounts.jpg) +## Reasoning effort policies + +A key can either enforce one reasoning effort or allow a selected non-empty set of client-requested efforts. +Leave the allowed-efforts selection empty to keep the existing unrestricted behavior. A request that explicitly +sets an effort outside its key's allowlist receives a `403 reasoning_effort_not_allowed` response. Requests that +omit a reasoning effort continue to use the model or upstream default. + +The policy evaluates the effort selected by the client, including supported model aliases such as `-xhigh`. +Each configured effort is distinct: allowing `high` does not allow `xhigh`, and allowing `max` does not allow +`ultra`. The proxy still rewrites an allowed `ultra` request to the upstream wire value `max`. + +![API key reasoning-effort policy](screenshots/apis-reasoning-efforts.jpg) + For wiring keys into each client, see [Client Setup](client-setup.md). --- diff --git a/docs/screenshots/apis-enforced-reasoning-before.jpg b/docs/screenshots/apis-enforced-reasoning-before.jpg new file mode 100644 index 0000000000000000000000000000000000000000..2e3767fa65634f41723318fbba26483620813200 GIT binary patch literal 276748 zcmeFZ2UJttwkRA6Hn1QdNK@&ACS97S5PA_v2oQ>tfCLDV009EnK9P=e>C#C;f|LNE zs`M_sgY*v4K~P`(&N=s-H^w<{y!VfL-aqbr&SbAO_gZtcx#paE?YZ|}KZkyP0bJ2i z(^LbTH~|2hp#A_qr%w!Ns;F2!G}Kqq)B*h^qZvSj)3*Tt7o^Fshwq{;*1V|!3BQd zHh;ktf8iMSC+<|5hre(X*ieOv+fi`=hkw9r{sFgrg8D@tN~KYBarXGt)-U+gV>&nz zVnmJ4Q4e+i8ej-e2R!&Se(E_DTr&WGJG%hDsW*R5vq=B|%3c8gEF*tU<9P=FT>b+9 zDDC=t+TVHd#2RJ&XLYBk;YoXY0AMQ@0Jvxh05E(804~7(%%cYXqHi~;S?tt)xl(@) z0A~Oka1)>jKmu$5qEt)*a2p^Fkoh?ZPyw7edGgnv3a6=mnsYQXr%%&dID7WYIl2pU zbhH;}X)j*7dgbCJ`b)I5SFT;5XJBMvVxqgu%yNy9>@x<~-v? z+KY_;$MEw5fd1UsbF}wPo!|nTq(50CE04J5*)UKWQC5nHAQ>V|IJ#mue-1!UC zeAE@diBr_xpSeJLp7z{Xnv<6SCr(nwaEAWs*<0rr7{v{&nV7lm8-kywXY@!&89|KQ zUPMM^6`sE)ZDSW*)IG(*E%^X$9}}B7J;Nh=#}<`c{P8;4-J_)ME3b@-=O3GLY9=rF z_(9~lUZ{XN=G7-^o9xuy{1yJzr@yp$_8c|$K0UR^fRks=QTcJ|G_~(kiJ_-<=_=zb z!$<}uaRa0D?yofWpBHjlXY_FK@=3T|Lw!6WDODu>z~;rLsm-4w0J>AuD(Fwq1C#+I z0Nvl(_#OWnIB@a<;PhGFO9vOIs!a1sn*mduY8iMu7r&o|am8t?sk?SYo%wn0TX=^t zX_V?sRfR??to(SFRs4tArftD4(U*1Gg0mUy8O)h4=9agHpa(zT?4>s@x97eMLtDe$#(yaKoBUsJ=2Qyk?<@;ntUu^<=f5v|?qC6%mUZV>pqVaz=8r|TKYagV9(X>d zoOK1|scBWKzhtk}s^#an zYG5*rDGWExs1z*$`(zw@`>LlGD_)7igR94CTg4!_CV5j#zQ)=izu7{yZyKlXSS-B( zS}$`nLQ+UQMW;$U*tXQ+seoJPonqPP#N+gQE18~r!KT&`p;g_9#c2e0(Xy%MjsFA~4n5%IHKCetBpYYB?E7nd-=P=J00IEqA1H zp}pe*)A+1}SjMfK98K2w;8)K89CkkHY^hZpAV~A24_i-ac(WVg0@tP7+So6n6T?#I zITA&++wrOBWZgjAdt$0nEQ~kmTx{O_r$Ev~UQiq&XmaUP|LG?>u~*cE3=9eoa`fI? z;Py9v>i^jY-C#jPp|nL%WP7ap`%;JM4m}Yw3R={AVVvLV@}s{PiBl(XMmZ{qq~lUM zu&F!{P!!OZl85W6<~-wjnSsuh4JrD<;j_l~C!_@H*b2*r4f|BkVTALaWofRvy5An3 zBG;s|SxGqw-D*LjjT8qJi#7z9pZ%qB06;w7HiB5rD`-fGWw@JZ9AHEyfz{>`Nl4w% z4=wkZq*XoKH}x4JH?``q>?u3o4#t?jmOKrCP}iEe29gKmHa5-s1)F9;i3RPa-w|9W zMiz@U`I)iQg@~kGU~nKV)#4{0u!LGpcDUrskxZoFGrD&t`Zt_)&=a#xtOQeY?OU)D+=cCRzn8?W|joQX& zEWkHOvm3MUgWSi)$J~Lp=Tb=*5)iJ%H5VK{3=ml6mpUx*vtYe7xk&7E7W|iKJwfXo z6v)SG5#YBGUwqMdOY_<7>M* z<``uFx3UNU6>-2sZ#LnpWX`6tgbpeb-vR{2H{4}dy%q}cw7k-|R0rjl(5Qtdq+T{N zZ2V&Qof{SpW7ZPE0n%Vd-|oC_OW!-ov++S<#5&Ier%7FDmRC;lK_>=5Jzj8oCtEYVK*X~0Ne!$fjzy$!{-M=rm zi7&`b?`}m{=MGeuHa3U#Ri96jQ@`48k+-HF)uztmhmxbem9W@|#|%n?k93`UBbaTP z>f6FEmk!i~wGt7=8N!IXkXoIVt|+8oJ-6#iBU72MFtI^kqL~8I=@pF~xLN{AP1GB! z>kSFC6ZD~`nq$F*qVC1Z@af%lDaqIt0rGRsRmg9*WnOY z4__^j9yPvIW36Iy$|9$GBTgqR@M|@Q7l`zOaQ7G&6AdKC=OjaPC&G{JGr7>M;c_A+ zoksV7n0OqB_Ss(z>qV#FB#WTZ$TK%wdPt-9IhX}7-2ShXh7076$G)Ed1>BaM=*KQ& zrP1YxFxzST!a)5*NUAA%eH-@}4oFAU!d7<~pBVQy>59Pm-^hU4VA0iSv2x!Y)@T7P zOJ22F`dS^AvjA?f%YCVrLp5aPV_`ax5QY?^8Ude20l~!B9q_JrW|Ul}bC^0h18ARc zJp197SB4S51bA3wCE(1s*=6-GrdyLL)zj`2*}uzl+%?NYzJ=k7O;}9eTaPu5Zbdc4 z1^^{C8wFEWegfV)U6Pw5GJe{-U7uAx-FmLf$jCTDZDPw6i}7X+_}r6IG4!g)>&Y6t zC!2|n0v4v=(PUY!Lx{z^j67R!4!=f&<+5y!sz19D;KWvHK+2Sbd+KQ6x8^S>GH^LP zHxl;6o&?@nRNYFDsS}oZ`_%5h9;h+WX#`@jBOzRUqsW|dkPXst4Ug)^HP5=|93?Jk z7Q%&c`T3DJ5y>>lRb$bdj2*67VpnxuPme_8}WA4PJi(Q9VK|a3?iDu1ry#6%9fgGEqm=s zHxDYA;5t%W{I!CrnP^pbKfWzya3(mvdBhI#O1d(F)KULD(2uXMN2otG?}wnE((Pb~ z#UT*hULze|%k(v9MEX!rT-u+l&iM6WQ0bC+lPD6UPGOQnm&mmeQq7w&b^X!`*wOEm zOrwhk!T0^)GU7l4TdrB5<$MN)EHv33Lct~OaEhL2_z9p89uFn91qmRq%ugZ<0`O-L z>t+j!SOZjFzxpi26{J(4B`?$K|NcqJr?~d}?y!E4Xn@feXTsgbOgw1F(O?A#ome@T zl%f;y7^_}BQ=LEX=$o(AwX=1l^=Jh063#_oK9Yh^MNufE?4B^tygG29-Rr8&uz1t? zRbFo`)?hY271L%pBlnue^t`UueRK9lL1DYB&knjeOeIQi`vk<4D8?oxgzmSV>x_Y zWjwFWzWu9F%R$$qa}*n2$xytLSWyFO|L_XJa*4wrb)IaFCuDmMK1C6IT*r)yriIV? zHhk6$@buFv_j!Ibr{=ra;HME)#4?m`7X((-iNxTy`CzTxQ?vyau%%q}#cjN$-RcZ^ zt8PjX>KY-5h~=;G*^5D`kw4T1zUjq(*|PO79fY5BR3?cb%u3Cf?>ZN;3^Y)|gm)j_`C4JID*X9OkaEw0q_cDa*NW zZ?Vk?=pWF-`1+rKUGblQ2J5u9AunAIqwADcb$Vv6lvXDg!?21 zzW4qj0Rg3{FM4@2r6jnoLETWBA=&i9Xa!G1QirPRb^jsHHLH;Ka7fUQ{FHT00h&8* z3KO;#&ZC}L#wD31rTcyTrPr!ni}q_zi%0J=DRo{i?j*09>a7`6cVd%g5j)(;#0}B4 z#bC!{w@(=#LvN=HwAj}TB9_};NiLbS=Gm%ATlOOq_>@}l3G+em@_aioH_jyUx_=xM z&B?B;;}wB`RAV)Dfp!NbtC6g@3Cjh=%8{}t+b1K>sQ$=*51gd=!K#02Xt}mdb5?;7 zK*Qoc__SkgaysOP&}Gj$2L3wZ#aUasu)fI5#MureDZV+C&JmFe()X80DJgNcUBfIN z?!K@tK~_RFgdu^k3Aa+X;_~F7k%{2r8|=ETfH?%a1iQe*l8o*Bqd87J`qgaSVivPi zk1Ix7^7}HWh}_v`Q(W_EJnuwc-@uH%sOc zWf!`<*DUsIP1TDRV2^p{%J}!r zaSl=>>?kSSIXj+B;o{=kH6KjlnJ`#gyL$+AtuTE&Fr}2*Yq+qLxa>*F0?#8IJC1z9 z4{U%#NC>S;;BcU`Xs&0C!NaI7+AZ_4+~ci<`7v01P%w`++>e(L0zNk6^$3SHQYc1M zySj%C^SW(`sd}8B%C2wv#*>Tc0+!gc4!n5`$!!YpO3Ptm$xAy%GdDLqu>nX-C>@#J z%1NXn2tNDXP(j6mYo5UGH%wZsZ2n`Au&c*CgkE0ZXar}4hHb&)swFbGj=k;RN3!n+ zhi)4Q2z{9bcK6_$h@=Hbn-4HOO#Owc?l4^`DB~w!1es=bzp*$e6x3y$PpN3z<{(|8 z({%YXo{HqbYaQIwdTmhVAumqPWD+(%ma@Ch95)q0@+!)`Xy!r|a*CSMZe7OP>Wnq! zcsKi{=cH?zPj*}DhgO6{ zZlfFp=TMRn_S&ghm62#P<(p60$I}HBt49pOkK#5UUgUA!*)A8vvac&d6{PNq+!iyq z%O6Y}97zznLo8)8GE0t24af~i_%7<{n7EDRds+}vGFh0zTLQIM3oRO5i!e<%*(c-3 z8!ZJk?@f?YU07M@!X%dpb%Zca5d8Nt>K$QX*L5L0?TAlFTdp3druIP-dje-Oi-q+K zFxfTTnwBd`B3nXhkdy%{Sdq1v@y;;OvaFFC*N(`99UO@2(_k4G8h z_gj`>+++CzgLc8Iy;#V@%P09QHuprT?=#KgyPe5eF>7$vCHII90|>OXnlQg|>%&fe z9}HY$GzRo`Q3dO|3iM{%A>EcBddWpyMA~q&)mKYk<(P;};{sOz>20PyUW?QjfxJCpZh z{kosfnylF%>UC{cW{!GOt;q1Q?on8$^{CK$=v3A1W$jS*)N};rch-^hIaCW8fyV@k zk1?Rb@f#s-sKS`dEQLk-$r@>vvHmW-EmA}Z^xfv#&?~WlVNocZt2Akai}Jbq!>;q~ zSwc4ZSlPC3+oYttxuEeXCdeT*1?Es;P%0{2;Of36g2+kU{#u6T1m%PnOKhb z72eo=i!>Xo@YV6IpE?$df3T5oO66{c0_lVU72J)B~Y7?m$+D&*PqGBIqDhUF0$VE%(`PcaF(@1Gg za5W8u57KO9jH{ewJf2xko|sdJb67QiKp;lxm+x){@cZ|R^ra+~%V;i60>P!KGu(O` z6vP~*vT7MVd$8>Lq1hEW`7{!o|W{njXV7p9&VTSrM6Mhdb5FB(kjgrk*uw z(OF8mI$KeeQHj~C`rti!pE5ACP*++uj$hlOu;$|{#3f@p5qzc;1{C$S$a_EJKpr^c z8g9m})~;ghZZRp4r|IzOXlo*EyDcrr?JggZKhh<3$K3$Z>!uMqAZNuoKc_Ruk9TcT ztkFW6_if)UWiHnYi~4pXd467Nd)$xSfZayJ10Xblw>Ve~VUos&mTv zjsm=N8GD)|xwOH^BhrFrJ#r+L34+q;M$NaiB|LH;rY`c|bJ&38coUnWid3D7`x?0@ z6frf8ytLJr_q)+oY+;hF(KcpnzPrHv`?@+6)?xGzdM3)oJoP3?toxCv^U$XmCMM2M zS@#SR`{`K(k64S*5PrV;hnECfy}AnT4_i6Qv3VMmKpOjr0A8 z7H#mx1kdC+7Z+)E;d`*Y+Xsw7CjeelF-?XBC;?%LBnUzyVRr0rs|6yt6}#dj-mw?Q z26NFZYA3ngaEBY2C-BRYan+KG9X^0G7E5Fqow|J;m`Ml`M`FU^^CBv+tO@%fR#BBd zM2nLD*T#Ekx?S8IKHueE%9}B6b3b-lL)n5O`w{hJTm{N&0 z*HR^WJKBfCS#dhuRPVK22SZvxn$kx!-8gXBGbGrN7Z;~x!gis7HW@!)-`B;nK|eK0 zgsaImGWPUq^D0T&n%JOYVVo}JTY8&jHKR}K?krDvmzs@wWt2BIwlLT`yM}eB>QPsa zX%gG}K~IpJ)!ozVnolZ>BzVguWYr3r2W1uGWpa*>lDj3%i`#B13g2+4nW5ttKEKKP zDe-Z6V{hqs33VWlgc4IITW_lhe4VatVH&yz5`pMCe}$;fweh?MO51-PQW~f%HW+j8%prG9xMY*~FsW zTFY3pyz`?9`RI5PhKrMElbYlIvxkfs;^JHEX`aqkD@3OXVyHllI^*NI- zg{^r&!SG{QsCvo8iC^`GtlDTk4GbVLV^po?wSd{eFCi=Q9;7@l?5XE4SFV!KcPx+KjIcIEpOk z$%;t5_!IC6i0qKX45kV1v*ZY7HOY)YJhWCt)JsAoP1f9Rw=a98sdObHB9J?iaoBw> zQSbTJxc5qF7Vnj6zAN>_vFXE|jPYzridD|;&Xds|d~+6yNtGS?LBe|%^8mL`U!3Q^ zExH*RwOPY-_K*9#nc5er@Ckd;pK` zm*;&Qw-uM=R4JmB+}7Kf#_W~hs}hv=HJ}5BxIL59GLO2i$sKDR+DC#w`fH_HM&lFw zg+tBe0(&og2BAu497+4soiJIsb(k6iO{6;GFND@@~)$F)QS?W|h!l^)n1ZrljCIyaNScrON>qvlxRD+vpH z-wq!5eihQ&#lC9W&l=GaoZ!wcH#beNywzx8pRiHVgsd=tN_ZMp;EH&NPTsqGvlHuZ z%YD;Fo{e7ztuLkw<}>>&CuSvtyMTNAzx6@MOnJD8g~Ps=Eas%~)K5~LM}Nd1NnvZT z!}B3YuyG;pSOX@A|So3GZBZN$^phlU#GcQe#h=0)wk7M1qO zJO^peC#60w(28F{xOtQ5Wr6cm}xAVF3VpxAm zbRuSKb5FSX!)Ucd&LAQ>x=5aR(KpII4^P_2aj}?WHeIzzqVa2DwGRH~b*EK}U#qx< zh|84HDf_eXcAv+~&O_5*k7Yf}P{FS>g+$7Dd>==&De0O=L*+?x8^`zbUl9txCVNLt z`Q_SZ7mlm)*Q!zK1y6_&Jr`3^t?F&gBJ+iMnxJMGgv3Z=Tdb{rn^ix{IKWEo`ZCYIAWv0jD>fl-fAGBIOLae=3Q` zzKm>#&x8df#Ve+@ha`H-Q7Rt)f3S-$Z~j`M*swurh#_Z;AA2SSSK(|XCox4k1{1bo zw|z}t5yUOUOC<2IH(oCe9rB`6$Fd^5lX4Z0IyT$`S&9cnU%0+~K%zd*@ahq58suS+ zo@nzYw|JA~+O{4xZ#Hx;JZbPu7I!B4mBlnJ2M6bUPZXE73#{I-sg>v0RBK)Bf9qc% z1TSI{rRB`yY-sV|+tKMqO~%2#Y00uH2afZx)q{7!qV`qJ@)I(m>@!-&7xCes-db5P z<<00!t_XhmVNn@&>pyPdD!M!~{Sfl}{jPDc2>UE3rrStNE_x~?1%5`U)?@__&kI|O zjnS5jMTJ4JMt8UDgSNn1NSQIC;7vYGO2654WM(>ET|`;HX<}eLc!7d(Xb%hwN}W>@ z$(kZmdsx)0JPP8W+-q^1NE~S}ndsNqMr9x&RzlAfPL7MRgf9fHFFiaI@S4kc_k05! zLeYoEWA85bxZ}bsvhvCv_V-(g_bzP^2No9>`8%)SvDXOnl9N&B4ETUUoT4~y+Q}yS z7kXScNA`wT=AqUd^YKJ?Ip4<1uk-J`7`KJ-n3zRH5f=}#sR3^QDg7(MKK@O0JSwVGPoU z0zV^^!{+WagfKLjKZsOD3(sI%qiJao`r_UZu8}pFm_cEdfgXaBF@eTJPGL_KZ_^C zt%og;i`o^e0&cJFU|7m=1Gp`I&p`wt(}05lP2)W!aeE|G-{Msv!_r2$a~xt=xp`%z z4{?t`y|hM)N!B7wBh6-9aw@xZ)lF#;g4xLU#vgF~Lti@<@?CyX!=0d*Jz42EgrIVp z&{!Wz4@WH)p{!7A?;UA2?vhhtprWo$>$sGGj8p5h(?2|hQy)*C1^-}G793hmt5csl z`Q*f39$5df9{&j-WiK1$lMM6lr}Ka<%&uz_aL_x0)5l1Em&0vL$Na-$6PE*@gs(6h zcUI5_l#;K95%Ys+v%P*0Or1yeKrb5Lr48aQZm=D7mU3G2ahl$ME3P}fDgX}Vesis zfG_oc{sc&l)gLTuAgW`kUs1-S)uP-=-ZzD1$ZxJ!W!$#mI8rxPgH&)$D~yBVE-Ctz=D^YIeic_P!y& zA2*e51{HG0&0KYe;&SNf(+9vFrK=E~&9^I9I% zmFlR3trBZGf(RnJOu&WSUL(xy#$StMB=)cd85Mr!UED_arjAE{;n;u=Csz(W=|D-p zC(*3>Tp&?@Pv5Yy9sXiX>19rlif1^#lAzNlcxWUH0@2m6IEsbVq|iXqpy8tNN|A$5 zc}&lE-ZD&E_$%oZM7`L5-MM>MET*|M)-%6!bqTQqgy-U0Z%k&QGRyD|wfUKx?aS%I zVJ;EjVf*sATe;QHa#Ig!RM#|u&)=xMUdp8Fn)R9C+4n_JTJ8~{kqDEjbhEs`Fr6*e zKk$T!7P%J6RhKyGn{GlN#2{GunwPbiJ7xnhr!ICq{?$bLkcodAZ+PTaZ4`k*=@IHN zYg$HQxE%F}!-wFnp05Lh3a?$w$xe6TfawYwmC91+*@+;K&SB*b+wnT`zQQYptV*+< zUc44>2+wg;1ohmmkT&H_>szHSa zl7hHz-Irt&mLQw#$w{h98}Wj8(e84v&y>?D{T|f;?|h+tVdnO6smxZxxT*m|ZHCoHIe5p=LujTOuIl||PGOgHqLKxSp)}FN7^X=&H9!yhsHyp=Y+8hzk zp|`*<%YnC&?AJ*NRg-4zQ>8x4qnEwKt<2Buuc}w(dt6)pV8?6oCtxgP{PU8wE)$31 z!v~EGbh>Kd)Rzy4sx9>A)zt0Fq*XU?+xGsbodP=vz;q4>QR%{4rw_~y$ki{L$yA@) zv^PE3WUMwcG$5WgK$cJ|LveA~=LBa54rkgCL0#ITi4Jo-0QX?N2laN&&dz%FWC>OI zE=?LOehymR-VS2kY7Oe)DekQIO^S#XmSL8vHI#kGtim(-CI&7W@h~-Mxx~JfD{F56 z64wEzZv7XC?o(Y%Pv5M1SWTtPDlq+eGw)vkjdlttYFq7_))1SK&MwH^b>2K)YCF!> z*2dPPzGonViwpB+^Gm$*(Zr=|w8U;Ko9wP2tK|;^UfAmn*GEXelQpGM5|JG7+&nZg zFE6mY-z@2yvV+%t5x|7117}oj0_;?b)M;#`z$)ykU+oZbinAZiIp%%Iv=8>OeV$6^ zqL%wM=_XCUFT%gjHxwFoJ$8fmGOgnX?+RWe!w>?qpXZR@+bW6P6?#t^&m4^u`03k| zFc%H-eIe%GhLXC`ZK8e?9um)l|0eeU+EaBkUy?ZgI(_YbPT%35(>HdqfmSJC^7Rr$ zsUixf5sL zIoheikb5SCR_40x|Feun?&|1b1-JaA?Q8$-_|80(LGcOV3NwIo?rG2SP(LNQ_c!_b zcgH8Z!xm@r7ug=3^(G^sd9O0$nNBG?>EtJ7W|fi--__t7W_7c|Lls8 zc6$ak<^613Qr<0;${~5*51tEWEu~Yq!yGiA$jz2sYCL1wLqE2h%Q{lM9fS6HPyu9j zR0w?*q1l+4Vg6=D>jYbZH2%RD_&&wrEr z=F4wk_^l+rEyI6XbC{XJ?Bf=L4*kLvcm)P_Bm}XLOyH0em`2$e z$toMiqk&|Jwu7;w*N=z3bdT~)De0HATQj|IVz!=5<*&I;apfSc>FmH8%0%bbDGSGz zF7unh4FIXqp>p-ToCY^j|NTm#M#G}qh`A~~v*h`u-iFGL3tcsjYAZ2^@jrH{Z=h8a z-OW7PdGIvA<3$Dj4rx!T4U9C8dkohKC?*>5k*ux)ciH9}{L7}Dz|2Arb4Z_LPToQg z8nko}%Z$p{c%Y+0`{nQ_z|kCNRlMrzUbhqgvy9QR^?oqu=Ci9dc*H4!+A~G(haERL z*;>Sz=6GL7ANs%N;rwqWE&m$>x_e_vD9FAk>5*T~Yq{Zj`K7&$lQTM`_2Y{1+}ZR! zKDmY~U#f0lYILJYgHw9QqSmaUsc+xLLnO?;FRC5R|>2 z&p2TgSEV#l*XZR_@Msj4rR7jzdGDLgh&MBaeJs*E?J9+H^45L|X@+KcRMSf9heRTt zPoJUppdH1Paop;=RwsLQtCBbA>ZkVEtjNd9OiCT!dzpM9n{iw1nJ@e$j%J6GT#8!o zqz+&&UQC&BeOdZDU+&RKOhcGh9mDZ*&(rZIKLPbIlQ79)89K`93he-7D#QW=-fMM) zCu-RfQg_4fBx8TH9esxtvwM@q?@ z`8KkOCCDzWzIKq%#1z~6P)X+xv2TML?sH_6PUO2s_O6FN9@{}=lqp6U89&B#$9*66 zI;IuBs4r4)EUVO+H9c&~#+(q*r4oBr%J@6u?dpQBapf)#>PzKeFz3Bt&cV*)r))eU zP#H2*+kh=qJ5IZr!Mm)S&$+esyvaKA3Q=JRfedUk`UrHyS|gosM%6edO3XX2dUN=- zj}-s;8{5M{wJW!N0-j&r*X`L?c`a;Yz77%2O3w}njdR$RrdM2LU)@N>Z(0Sg7s3DH0x-jKt> zhn_S(cNpTO9sr43GH=E1COQebf)NlES%I)>q^<~xzdrnS>V8O>W~$AY_0^%a6|f)7CH50=J}EJz zk7WMie&$#ECYcD|qB=}n=}1i302hHTxJKYky;E#uw;f_TV>`FrIi++J^SDxT1n$?G z+K^I+Z#C1jf}P}gFg{XbD$NBXq{Nx-n>Y2DAe;09Zhx=n)k-kxHY!R^s^mZFY<^kWn z##Xt_(-brPBhrn7sH9v6w(}n_DQVXDZ(|ClcVmr5ymMlEnuGT%WC-!-q6vHs&8M?= zEHjQQHKSgo*1M4*PA}w!$ZuqWlZ?jpFot~I2;HSbF>n2JPYeQI_=sK3g%lD<%qlbb zsAv{B!Z|b&uG0-+<^@fJMDk&PVK43ONYX)Maz+CdZ{qCtbQa5-3VqB-9R@z1+dkZm zc zsWPMQeLX_;m`c=RXOJc3WeCVdMyO{gszV>2uS>;B(Z@F^oBYTiimsqIYTYvdKf>x5! zTvO$&<3-^nW3^dhEz6}M0u29+9xybL7RKIM>Xem2C>X2F>?LW=Q0Ye|TI% z5Iem%x2;jtKu(N)!j-*?dvx3-Ip+_g5kH=v%~Q~Nl=jlNVDXAi021)&?`JRnPGbM% zcY)tr`YjrNQ<>lT@Y`zqHkH4x55KRX|3#M$X9}#nxJYN!%26ZrD0q^#NCqMcqvWc%`pN5$P`A9adlVVSahgT-=CqJ2nW?p;LkqdJycehe*%`iwUt`0|2NB+ZkXr%+sTgA{+oj_4q_ z?`POa!Ow5vDdu*_tgQ5iP%Nyca^ElKmbNXDt%>E#SYYD2%sh2GW}>j#yjByu%U4q^ z4xbPfe?4r4vJZelJCv{0d%}`xedgk^qR4cGMo@q_v6e|b1L)bMZK@y7p$WagEJW`e zYxItY8R5lupz&qxtqD$#)ZldVPVV^G4*I~5YShBKILwH&x#k}^=$;--!a5bh&k*IM zMa^Qz);koNeT*Nod@r_|LmODO3Pj>@9f^|4?_-qmB_kHyL_Q%-=Mc)wxsc>+Qb>qT zS+;D!!}{_cFE2qs_ufFvNfNS#h7X$>m556gM>W08^>lMjFB6 z)4Nzve%?0G{1We1mko!>Y@p7XsOMaS+=9GF8#BkL&j-mtX`Tt|pbdDMxGRHg1)U`1 zB3lNopj>NQCo>}=G+JLP*PG6XjTh)*bawOcglW>j7-i=vgR*FGN3rfx|Icpj19+?- zNZa1d)og)6a&}CVW{dYWU{DUCeJGhM9!@;S$UvCP4llA(`|uduk#wkkCX+SD>8jDA^1Qli_=W= z!s>30%t<=oZ!89^tV6TDo*x{dYv39wGvWOe${(&dS-lRgLt`f4ph zc$90KGL|3Mr`86SRwJ>)%c@8=K3_kV%(^!d_yhxOUHLgW!^R+C%C;i{xcxn; z)tHz&xick-~f5YSv!dqeGVJEYLOj< zwEm5YE7$B+qj-reyYA!J*w3bdCQ&kKdczZg;?0#skgePpFx|YbzPfIZV186qetuq% zeNb!P=N+AGRrAAr(baG_8NSR$|NGR`$7WSTj~G7Ah*j+-()qDgZZ0WBzOn z6u=9WH~4mTGbxJn3x_YNcKz`7U5JN|z-v}Kf=3P1Egt759k8rSP50Y(5 z--D&+`xpF0qQoAE`u+rbYv0~9fIJZ({IO$pUS8z; zj|`BF$dxE*h7|B82Ip(-xtbxG6N+Q+**vUL%NUk#Dt3uicT?)==wsEq$!XdohTI^9bZ&qWh~R$L@=h- zQ)yHZR4Kf2zvZTBT$-rlK>|F@QawHd=K{uZm8-xbqee~Yr^nd|LQh|TESDhI!)V*5 z+sd-vgD;a`JzpQNff+N_NkP!9dxmZXMQ7l6Q-zqGKD}?V-UI(y{IAwanBNF~A_X?+ z)fW=D)6AU@(TLM!>ql*5Z~;9smwCW>^Bs@F!DB+;-sDfCB4{ouE(N81jgpg)!e-PMxN(#etCb_Gw<@)Jg z7$>uNKztcvOJv!kVlVa5t}`CPcr3TL=4%Q0+nd_|v-G3g^1`jsD2T%=t>PXuiBk?H z8>FC^^V}9BQ?C-I;_-3&go~fd)r0bG6DdxjSM%MLdm5!BeZ#++rR*vSdG@*VL%tds ze;#x00W0W;l^bn3M`D{p5`U12JL{WUiTEi2poAayqd1euwnX6sV8*du8+2YG+2X!^ zx#@tyN(TI-#-LN2r3mZlKE^ahVbY>LvzFoGB#_)9o2dsG$*Ln-IHPt{xrr}P-QKlqG>6&u6 z{t%Pr?}#)kxFS5Ltt%r|mmChtQ_ai|w~OOR@-!bEoml86W%DE#7ioCs zG;yG$Puwls1`|8lb_d1|D@1fLN$BcA^^Zk(jij{dm)iViZs5rlF6FH&K6PF1Du^oA zIUtP1j#0AG^z`&bC@kksKu0+xpg>Ai2WB`jDzti#5VJ(&-0{)8=D2E}6z^}bQH*+L zAHZZmhPf#Dfs}Y@T31`)y-J|}!QOjDwVAbBf>b$`19o9dPPRw}5lj#nT(-&4fJ78v zGDsjmfXKLvizI@J3<87^Spt!Rh&DM`V1fV`d0F#xhz{q2Q7x&O_HH0{Cj#eTlRn{(IQApdNi^ut z8@)8A?3lrTt6+^?=xlDw#8ZpJ>)y6)hITp*kfZziV+T$dc!x6Zs+#^mZY*)!-Pky9 zt(Yjw)>H;Qs`)n04#2}uI*EWK(gT@iJzXk7VY#-8yd-(+c2F3+drYnOVPVqq}!~6V>oSPuNnz5ZYze1}HLu39G=&+$)JUol*&hPG)EL9oo zlSit{>W46}nqNNN&6c;n!#^?U!St-TdLo}_!mv2|m>RkH`!RE=mX<$hX6=>MeYp2D zXa5NfK&T!Ny*N+qI-wOM8;y%XfKF=q@%Dw+5~aIYw|4?%WX5_*`f0Mdyod91{5LmF zNN#96PC+N6fBl+NN;SlAbTb;Cb#eIOQT@U+r2j*GI({nkq!xP4)1(evD;cHiI}X~W zJH2Cg72_;WJ3MTsqtJT}z+eh^)A+57Ub2pHo8)|c-gue z9j#aO%~#+vFU(^QW)=&5B6ZWsl|np3g^Y2n`_ix-m@HC}Z#G8~s}D$ZMX|V|5N1iy zZvFG#8p49v?2jGx+yf~{wmLU@t4o}hpv?q^WDBcrHs^*@*f@^ zJX=yfc=h4yIK5PZTM{K4ZktKSnT!{hfX-W|A^557YR$}JN9&ZPgETT_c#`DKDLz4%}NC+!!((3+7VYNlH zvF(FW4f#jD9xMV@D_-a}#Pib78S2yPFAmzYbhW`Mq_iJ3W`Wjjh>jZ+y z>lkuc@ILIs@|uZ^`$d|3@}`uq%^wMVVhLq^Tm)17ABb;98NH&tdtzp$RjiG}!OKt^ z`^AJT4VE0XGJM9y5X$^WXc_XwGLGw@b=9`zM1ryrUmRwE`qU%;G&X~{<9q(;^KDj; z02wdjb&YB+bgftbY@GowOgUZ zkn+2ZJ~e$+a6bt&JJ=?V`NjTvKDAa&>|t)pNnilctLYMNZVOM$&8)V!#OmtSBZjgL zB*y9-yQ0!kO}?I+-$z<5KOpMMOU`xm5fTgG6Z?oE7lt7NCNP|HSkh(UtO~@LBN@Z) zeQ$c(G&aB~lqZ$A%5}d_Ib@{Vh_q3rY7c=%50fN!%T=_UStGB2A#)cXheidmPmo** zHi0iTqh(vqidv}E_F9q+`rz|O5AL^&+ww=}O%Xf)*H_3d0xZc3YZnv>?*cAm zn9nV|;7wtj8|7#PdMAH-Zon!ydekugyiC5KbE{0JLiJv7ViO-(0pa!F&6ke zwQg2&J-$#-gCKH*KU5Se=nD?tyUry*ml}j%E5xjKs|>vCOSI+{lP|SiCc$?600Kcj zcqQoW$=)|_J#&N9%_$}wq>rn}>X#`8{9{B?XX$6VHTHnhnXAur@onsAkx6Qv_duxm z@+T|00M@FHr^As1(csqt%k=8|jiB|u-eziH@T?%n44JwZF+w)W3Q_nl$+=Z!)k_nA z3|CVHE5EwVC_`Q_$Q4?{V>WytFk&F4DT$Um3Po+{ zf7svvSq)lC43#CYU0K%6s*sShJ9cjKd;X}>AwoXkzPgO6-nqP#BF-U0V2PzMLL$~v z$4jZkH#WnToiPAFbZY!Dzj-LP=A2z3G(`Hn}HpooQ=(uE(n1KfYUc zy`fdylnrZHO_p5QRNnhd$h!2X{=!PFwKB_5dV6F^-)9$BWFe-V!|?W`z7wc1S=)6e znjUDf@y8{L)0lShPMC?v4Z_^(Pxff{{jZ91@G^aKS zoa!P00s+pSNCI0Iea(lK)tdzig&C7+zbou&8344W<#bAvLK?P6}<>e%uS+|$iEnPiWZhYW6pmEqWSoU#x98xP0hDx^T#b+$V z41f0D-`ZG5l@_A-I>noYja&HKIUZ`Jp9)GgiM`J3cm813vmbFUlG$kVCtb4E3#)%v zoK-xIjBP71iNU@8692=OL+?d0jT+;@DiN&-m+qntTA4V%u6eaSS>Hzs_0AJIU3yZL zVh?@i60xc&HD04K%<_FSo!!GC6Lp_!vG%NAQWpXqZwe6{YiA!*Ol~zKDJ2998>3v8 zcgwX_47W~J&3$r9J6XeYnjX`PE(_i0qcNiy1Btr>3sUbzY-O`^#znscb%h=CWAFux z$x>)7-UQl^f)8sjhzC< ze`CEM%wz*?0_2`c2KVW*&;KbtZlPLlS!uk!tx3p@&$lm#!;|p5hnrU4w>}-+X2mjm zunCw@Sa%hpX-VT@!XtX@z(m!7CUYrr&_$JgDDLET`+OjCn;79=uu z`e)@A^Cx)o{c9&HwMs}UE{+DQxB*KpljIO+Na@?}J-pO{kv6N8yeOlPF(1g^WgWbd~ z+md#&mB@%}3jM?iGE! zOy%2?W5@D$OEbB}bF%6!^71U1(nFc~yT*=-d!2Rhiu#S9 z{Kc9>C$d7-&_Jp)&DY?SQmMS-#axBqwEvD_A5QaZR>9YI8MAzdoW_f9v9=Fi5p`5hm@DzBI+-z7CdM9!@G%+piUzCgetD6O3 z=hy1q%G`8BkazABv{~XU&A!P4;3KU9xQ(}sD-beE3(XcIBL&6Sez&hKSlkf2EdxKU zWU3F+)2UHOSSJncR9TBlTWykQ&~!0yMW#ZYcnfi*<2W|)Fe*Qs zd+Iw7^N~CML`-*Ba*nh?W?+0;*x5z1_ESGSqv;)f)3km(_ml3r-oa}vy30n_+27JN zv$}8itjNoGNc^uea{UiR{7NKemfYR=72W{esH__sYX8p>@+Yn3U!v@@D_`lQ6Bt?S z74NbIO_wFi5?_uguaAtk^b!vq<+ZLE+!AiR6U4A=!cv>w{*z8&y39zvc&Tv5SI3Z6 z6ncSFRU?CxGjN42!LA=|c3j5xl0GK5zksJVUg!z%BFBx4hkJWW##9N7IM+h>_Z&sb zf}m_5_A<#U(zoC8`1aLp(`)XciA}#PY%)LAw1`iuIAw*UN?SW3)t5(?C6o4(!gH>~ zYJ6bdjg-l$zGLN^IylumiX=fw3Ax#Z(DB|%oQzq%x;%>v4l%N1rIoX;-X{k(lPJ0d zccPY$$V?;R=hk=W3*rtATycRtiWZ&Ze#{g;y2S_YeHh(w7{=fH@i6E6!;70vgSOp= ziz<2!#pS8f`iY;%_;nAvd*=a?5A0ej#s|1P2DD+S5b{g11kbII^sn@Z<|ipZoqN+y z$`-DVSr!&YuPa-~H)1Bg4#av_?ehA06E&R*EMM_Fw9RQOuehTT5-}~hvhCQvj}q68k!Ql!%asE=ew`x$5H z>Y}1Q;!v9%-VU5xI|P@m^bD?0^}Jb^x<5AX>BGj-<_s;L0&qB-vLS?tiBWVlv*HJj zCHwRVAm-lJ88uP*A3lTF>3|{&b77X&Neaa5G*pa_hgs% zBZo})MCNt5Dyx+G^zA;7jG@Ra+ET^`AC;9rsu_EFKL5H|UH0DAJ@XYugmA-bU7EQ^ zL~5CEaS@ViGMO?$C^T7z>_8i<=d8>LIw6U7Sll=~KXP1a-;9zNe^^*)nKHl$MR=s< zo;%|nPoYHqlW9PA>Um3N$Y3jmo}E z3ZP~DOvh&r|HlE#dK)RvO7TvivBfE-8Tal}-aVn5)Fz{5)L;#e7ADJUqHadL#WW*K z5D_vHk#O;z!(m(KqcbaXeRVPI!qd-}KeNqezKK7}W#HHe71|AF(-GewYYI|Fbz+mJ3&7BGh zwkUt+FPqRrg~ywvG8@jCon(u5r6n-DZ=pj6`>2kT!8l$y&sH(~#jAoYW?07TmVLxm z#+kPJyw#Eqz+kd$?s=`x1-ijnn=Ns+W}uk~V;U0zX%ynsaBMQRSiu*D+y$er2 zwdE86o+;Z+kEOvmiU@UGS4UgTTsB-M&so3uqE0k;&6b7}0h_QTbmmT}E+V;pLv5Pl z>-4pD;oI3iCJ@9FhT##)xWm9v2k!{ zcX?+m&qPI`@q|y>v-VzBo^Mr0UhhLu-%Z?jmDv^YV=v1-tL6m%*PDb&qz^IWlX`6y zRURT`3StbzvB;cvh&w8Az-)87&0_R6ocLm^r*nPvtixA=FteZ3c7YVqX*HAbTfZwS zWc2sXE$Hou`yV6S8p3dtkP@V1YQ(v24Y36Ila5(-rPS0%)}&wvDU`fjAHh9vrno>r z@bvhh32GtT*|p$7_f}V2Fzi%HMIz3mP@6(#@>cB&NV#TU>ppgu^`v*q8^THX$96>R zg|^#JgvpCZ{Gs(7J2tRod^FGEBUy{HAeoQJf z$JYj}Hw6kKBuj6^*y9}_`u`!TY>YZ`QZz3yL%4vf=7Sf z8;H0sn;xBC_C&E}>1)x$e_gEqb6UR3iS+!UAQ#<91bFh4FdKaQ`?7q*XkWeWOp+(% zo3=I0q>GO3sgTO{YoUv6s!4&hRxg6n!nw(Y?RuNc+oL82-TWAbGa(v$o=jh5$ggO= zWu)>5W2jKodp@y~>}lKR88=w~QfDh%POrZ`rqYN`D9POT>TOv4VW=An=Z>B?8*Vuo zSV7!)?X%NY#g+MaYY~oWA}c#_dOXY{85#4%YqB<4Ufq``d-Jdh`~_Cqv4|>0qX{Fm z6L$Cp#&I&5`54(!<4 z%`h7BdC$6*N{!7F#st|wM54|aSX#$dV%2u@7}U!kX&#u+71=AZIi6u3%xKv9t*|_X zXKqcRtzlyuq*q1JGhn2aa6SS?vMP&CB$`VFiU)!B8d_g{n`1lj7i1WKI8m=(PsH@l zx6TqxjSV$8Y!rFbAXXj;&{^mj#*dh+(t8p6dTlrM^*P?=l3(JzKg#V#^;>89O#R7C z&rklDzgiP)i5M*IK`K^@gI&W|_q%4928m%r*2r-dF{HiRkh75Pyp;Brz@$I(A{lL6 zDkpXG`>q{6P*7pOGq<8lx;ZyG%RLngU4m1jLE_pBd1}%mZMN?GE}e~bl94k zZoM%0f{#EzhvgEHs?lHSG?$UD>(l%{)>WVW;V*4b7XY#zLfP21+OtiL6~1q>HG5Q4 zh{VIpX^gD|M&)6WH#A9FZ~f@>Do3kW*EC(DT|XJ~wG3l-yA!Deub-0lsolPNY}yU(i`+5k(E;%BbPzg4WFVxGi7lv3=aB zxY`6}-9w5caK+|}Zxjj?e$o*q?5{hlV~B*O*oJJx7na2mwXFJ$aI90S3Tn`nOo)`m zY9gMy=w2D&D}8+dy4t}XlvD~_3b6Fx=Vm7g_S7kJbO zSMnod{j`L|%z@@PX}no;_{(`7&($eNrYlsrsMh_~vgP%`3NnsWciLYEP2#c9gH2-SQJr2g_ZtSkrB}3ctEirHVR{T+PRIlF3tQ!(Ima1(@7fHq&+g9dPPREb z4M~r2n)Gw>^~*>&8e_iwn%4qjQY%MVw%et-YOJgMm$4XXUH-tIW~k`fpg(1Ci#k)Yj(R?|QZkPLcsvpm1I!|U zc%#|B(@M9xc&9d>>pAwUNpDWrD_&*iy_)WtrxWEBtueVc=A(e5*SbePbyrpilORlq zM9T@Zwf7%;?vu$jKK9|r%FFYwwY%REG zV&IfrMl^95EwLbO6~gl4juI0HTUj zs{_$K?nAAmUVA_3W*r(^J37RFn^}xZ?o)eJKUT+iu*j5>BBs9DmLu{GzT>oj|8Xx^ zT%xYq>*~OLD=tHXsvmN~)R43jw~pP)Vc|;CcUzq?y`UwA>$BBE7kC<=_^z0NDfV1S zp}cXkrYS{ned~h3UQAp~Gh5^sM6cnOV(+GF>aolFcjnG(Ptz&XphdTeA;`)=WXN;%Zv5R{3sr--ar{{R;#P{Q;msgg{>h>lH-4|D= z@f{WJf#zF2bsUb@d7B1TRi~m8S*O;dR3UB*O;o&qhY6mh3ctHy(`4`iR_9Bj1k!R~ zy7ITJtB5;O26Q@C{R7c*Cy@blqYJR0S$(Kmtw*GVvHX138;j#Oj!!X36CkxAGbaFG zB*;eJ)A7sd+cT~$?XvMSHNS?&XoP9ZHvo+0@oRbtY}3`#JpxIoszK(65Su>4-AqHO z)1&D*zA5qF>P_sf;=MN?=DGv%b|r*_%Y#~$V6R$9I1B2e&|H>xn?|E(x7M|f6l>;2 zq1WGcGDivK)zwu(5(l)$yan=5L1(=|rAax@g&m@4oQDjVM_v_scfLx*JPDnC0+zj0 zHs%pT*A-H^tW#?@)_{-PqLcE1Aidj)qe`U7G7;6T$wPk+cb>NqpUWl#4=aI6={65W zOk7ICBF5`B0$eHhmEJgno)+s#ErUfA{vGsiB*SBeB~|zbqm6g9rISj z@qp~m?VLWCHOX(>1(#F;Nrhtuo6xMj^KT3NRj(pB3KrIS{{q!FL-bDvxWv zh}hKhubHM;ZzNJ)@QyYv?VTEtSaJ_Y>@A)=ShM)uxDzkuZcZLY`=z)^qL4sg91A$b zldooNTri{xi1V{y;?~$T`50t)sfPJ5vmIy53R+-fIpRysb!*qN-(>9 zjPKiCcZ`>sx|Gyfm(l#e9CdsQ)NHOeMVK7P$E&Fs#NGSjmy*4&IY{FhHTPFgh!@H>-Eu zkk-S`_@JXq!H?3c{7-yX)%&;^M+@At-$lF3WURLGBT}HFN)#!`{Gj72v!05b!M!?$ z_O>7yU3pl|SF$*4%4KaaKBf03opY88#?%Ckb!}5#jkN#rldkGgDzvE8{KVrgcUin& z>EAOhZs3Zs&Sf<~yEPgb&y>|BC}0cYs{{t_?2{zDxqAS47#ey( z^TZHx`z7{EcKWMXjG8I)@9ic^KrTznMyk1J#c+QA%n@Nkpx&quo*GOoNus;XrP3`E z0*N(dCcp6vV<*D5c{ey4zL4(>}L4W*{4r9O=`GGCyzBW-8T2q14)gWJ!-Mc zEp;#e_t8+(30_lCYVMZ#))8wUMZ>N3<*21-jhD zNoN~Vshmv|zu1Ici(q-Me_2$W9a*&8?APe^4YYCTb|~V0!;w`Ab~3ppk;+oD05eMi z>ykz!5kf&5T}UU&rrHbOuie+G%ie4)I(TcT(0^oBl85m6sNloGW3y>90$Pjr6>1Hr?q`)s&Ff^HVi_%9YBe))>YdQ<9>_E#4?U)A4rX-^|Jt)+^6?C~-GrC{ zigLJ4*P_$zexBmt0!u$_sT|e0^*X6a&SI6oPJJ3|s!77*ma#!?dVh$J|D^l#KYP{t z&uQM$FBxA`&&~WI$4;%1G6Nzm2S12yD6ZhI_h50tX5(4~4ibgiM)K@A3fWUjL55`TKZaTQj!*DH zEUua2;0%s_SjkM+EnthTX(iCI>3i$B}!zEg;XSQ|fI&QIVP#Du)V6=T)-@3b7h z)KGjBJY46z>xMw`X739IuN!+69#5tG9>}c<$hW)XWk;~ zld?5DHfQ8H_Otu=MIEK$n18s;_iFOHMTP?)qaSO@EC5${q<`+>R%h*gAkRLv;Vf^; zr<#Gsx5_aV14Ccf1pr=I{HF;c<0V5pHL&#;+QTWxn9b(n{T_NTP?{wujI1s?~J;u2QpV8VB*y zlO6sfqgnTNZZq~tjozSj4RQTId%wZQ2+7Cz_ZV~Dc$cL_!D3+geO@0O!{RW?Hm!PF}K|55jvC}&B zZ|cJm+gbuOToeCchp+XkBJKZnlr|BaWvOmen znIsx?2O#{*B5;fdnfZe6z7Vgun=i1 zl^{&;a&XmX5lJq?2cC)f#OYq{mk&AwL7PM;ZcGV1o{*g@ij48lfiU!~nlLwA?~hm4 z?rw?KsO-bY6EMO)xa_FMj~DA-j*fI@s9klBjvgE9Q)%~p_#aykJQtH9e;Mx7=l5d# z^SkE1m2uHB{|2`GzYoX#cl7iBs-xcS`mZ_SL!Q)aepf~?KJI(B#J8KSq-l^{a)Zu8Y=BdbR}R7&GDp!uOvsTGO0A z`_g2eJ7Y?=Y%0j*jK;JZiJZP}9+jp}@gAK^ea5{-(JoYdwrX%4jsvM9Wom5o@>DbK zEWuPx+!by4gM z5_$H<%^#0fh$*-yjFItj_hUbCn4uv1i2C1(qv!3r>c8mHKeIA0)?tzowoRx~T_Ogh zaK~W8w7*{r@w(OMlB_(#`S2?}h8C6bW+Zur!8jKF^*M*%-I8vO8gLFz)%0;BY+D-( zFBOxD_J>fLk?s?X7X{1zgU;DPACw@ZsmFkE36hDi!c8L?oWRy3KKfG@^_WJ&mxk0-8-JSh z{L~eG)^nejd|J+EVKLh%#AVYURHjB!mqPt@mCZ#bzBH z`uZMXsZMV2kno>$JF}xd4w8;o16Y8f*o!)yTIKD8)`S|}AS10*zKLX)%wgkZLIbr< zL=n*e9vPi>J(rg3tyXTF*`h5vgjwOEM=REK-jQDP?b@5+ZOIVLsQ9FJ38bYAffW*C z#~Y$EXH+Qx!bNg=FClLC=4h?9Na zvS{%N&%BtvUq&HYv1l|V-guf_3YQhn&Qb00q=z3L|^dQrGCV zyhrwvbZo!MzjC0$5v8R1M7oDdPyD%kIV0{}8O;CkoBSQ_8`&C<-hMbNk^A-TiyLoN z4SQehDQ)P}z5V;+mw+)OtI|D%q(A@nOaGIBFeViL6Fze}PbRB>qG=@f@}Dp^EC~Mc zEy3ZE?Oy+Pe(?uxId31G+vFE4zM?IqR}4MhOvSBU|7hD`ru}%OTMm=!>S84csLC!$ zC1u4lq4Ec$2eM6-<;BAdK4O1t@Doa=b|fwqy}fy@?kI68o^Djw)NBDN?xXY+rB4xR)A`WBu>O&%|gJERo)W8o;Tw);Jsq+%|nz`H#m?BLK@CKAi76s_u4r6Ame^wb2vTf&R zHFd$S25GMYYxF6etyT-l`{4nynFY|6W1g(ImhgS6Ja+15 zt9Z?fkc*x1q+>y0BR^+QUQssLDIXq?nZw$Y)EuXOz;woJ8)GjM;Mb&pWYlFS4$SOa7zyHp(0 zqF|6J7UWguE)Gi%6V4_;JwqSlY5Tk1sqQ{es2%j!jrXn*-a&f*!)W&w3*vpd%qe44 zxs4ytU=-T+gg#zHr%I1gu`xoN_Hv4KH1{NED}+kj(Mlfyyn8=L{zLt$ zIq@v@F;SvPclgU`kmV|xgDixAdz{)M&S1bFI9V;fr)4KgwQ+8^*(RGpB18(r7rd*5ofW6uNM>Nqq*=$c=BJhAN!6ig?0M_|Q2WFiC(^JpzL^rylEyCZ>5SP)f~*=8 zl7vJ8c0PR$xvBsr*rU5cS#wuQou7&TN8Y{BruqHdUE=&_`$l6pb*^gf1Mszf7emy z7^7*M##s|X^JF{BaHFwqpRL@|eiK8jkIHQb&6f~HVm_D5tIcH}F00&4&taI9fnU?B zcc?YPS(YY&O>CNYuUJ_xr}Q?~;+psd^tGGU2=m6T_Fx_UId>K=QMG5Sl4p$O$cHKeCP_ z#evnnkj9*^*(b+c| zO+w#?MxMFSJ;QtAS3Nyl#!{-rDa%T^Z}u?Ee4OWGTm}ycOnYsJZvf2^%{;Dehu-ax zp@^ogni?CpI`m0BURszT+=gamjT0DT5!RvSse3Ql$0zH=Jz7*)H^u7Lg@qBIY7h_K z?qa@^K5`eV>LGbb8JIVs`BHujHh1o>b3u#)V_9^3sCj}#C|8U(?sV9=H6}4P+#bPj zp-A1{9*-FZ#B;>LqD6jxCbD_&0?t938#-;p=!xn9>mC=Gw;>(IpC(Mz_&>UvS0@?k z_F|V7Cgao)1y<`=bAX`aqj=B%sZTOAoGNPy-*?Pmr{*@95uaFzjylC+v#EuygE8}J z`rpK~BbeeEG`BXLC+erf&CqfP2|QoLGule4Y+yE3Oxal70Mu?BOjP#Mzn}LUI|NaN zwLC3N+?K{hH5g~IEE$T+#t1D^1AMA0s?x7-=o&iVV4MoSb@Idj|gg3-; zWdD+t#jsVPV-df#4`jDE>aP&Oi2!@2cj*rM2?Ov&!JTKawpj*|5>86XlSaW@5j)iy zR?q%eUrb3pzzb`}ekKWo^dABMpprR>kc`o}2Ia5)mmoCC*(R!(za&-5qwX;;QM+NL zDb;MEQB~Do(XVM%IBZ_?rTuNL(L05Jg2oX&do!!24jjT>kGoe(`gcyu#R3i;NSK}J z_6|os%MSO7h2jjU`K~JS^+Gg@qEg#m-VDXru;OedHofX#*>`8M3o!!yM5ET!P1Amk zYYzCaFjODMynbCLRuXJnGq^MlojN~%1kJFBCpVUrrho`|en{AFCE4c5h5jTI*>3q+7Ju_yL z$57*NN@UEGuCR24!zz`4^g7td%scn{cxx1O(w<96|J%k`EjI7}-0kA7ZH&{y($1RM zSW;eU++O8DtGi-WRCg_uc3D}>B|~hvPtWT^8@(lq(*!(b?R{z;aohM#bsxL@0EZ7w z1M`!PB`8hIZKj2fliAZ3HytNKTr0L6eEa**3yqSfX6)6V3Dtm$2UgVOp#rIuXD?;r zZ%ke1o>EH@Ye1|sjYL{>uXdwpvJV;S-q5HOqIOIo&O1lZ=44f^ay#B&k~4VABWeRU z_f0#dQ>taqPPr^qWj!E0V|i9kWl6Fn=JT^iIgM@fkVkZK-SQHXmI1q~y+g3YOmS8p zrgz6)L(hP*;laX9D-LFp1wmfxGCj?#;D99@KqSQX?a6T1KDlw-(z){i1w3AyDH=Qo zUvN}}#)G|6)}qrqchkGwn?+A31M zL^2$aEJSDbC3?kTm5zk*fE_jKtcwExU&tS5bW|}3clw!I{cc|U^j2NzIgiT8Z9xM1Yuf2~D!Zn?3S(yHcKT6y)=oCF%c=bnjn5;l0_g13FsK~+4q8j;5&@WcyLha*N+}FPtuk;00Xs&N> zm&W^+sJB3>+nxhJ8>OR(yE}nolcl=D&P=C|{?Cbg*PbSI;t z=C&cSOEVM5of#kM_*4agGH=Yy)*{)8%V=CK3N}ES)GO7x1{p#~%Pmr>WT{n>WPW*7oK4ljqXHO2kr3zcAVH2#gH@mxOdS$wI6EVE(K=| z+hr8qF@Y6}j@^`mX}eS||6NNH(188d&C<*8_!s3x#TCZ;`6jvCkOz&KvPt{?l4PBiEby*c|ot` zi-=&Pds-6~5XOExE7<*hojecU)?Ow%$STbO_vp2Ard7KlqzDU@2tY5aNiOD&tS!G& zaWqk8p!zPSnHQgkI=%0_d?Zuk-(b|iZx!J$it2?#QTB53!%Gv$!Tt= zZ^3NsJdUgp!;G$CrkIklw5#FL1z_QXLJ!pw?Kufk(3(eBc=49{c3yX?WrT2fhtKg= z;_0*7f+czC@?0ETiM9PC2hxVV<1Tv3*|BaR6Tw+RbTB2vdZeZ%m@WI1+6mJL*pj*l zEZN5CW32qIe^kiCX8-6g{1L0+7F=>cL|JF-ZaIeT{-oo7SfEy8&3KZE%al*`U1JCHj}7pPt!dR}Bb3%qS#kg+}zYbU%sUXo^ReAj*R7rTk6yjnYj zKoT)Y1)F*`UcGmnJ!%z1Ho;XQ!Pc817gWrsJ!%SS*e(l~C4ADw(=P42!oAlTRWZ;n zK2oz2m&v#HE7CsG$)VBgfew{>g;eG4Sp$0h{$=S+i(dA)@mQD_Ql9Z_plm(owYJZ# z_>V`!hdjT9kc>KNIG=_UOVp(6zp46Ef04o|X1-o!r|@{#XLZ&?cRHs^_dJDVz2lE< z^$3jqgA0JhXy;Pu?ZjSmr&WM)lgKx9Zc1vv+x-ZRc1!;EEtEwz4`$Y>Q~n>jfc@%K zV7-_N8XuEs$RO2MGHCKqiBHC_1~}@@3{~TtU}G|REc`I_TH5#UgyMMX{Q0q0mNs=G z3xf<7VjoW*n_Vh0DeIA4AjJs=QkH;P&N)Zbg<&Lk#XG6BsJwmm_B2dJ9NC;NF*56h zdQUW*P4_|QQ)&2F-DU!W z>M#cP!;=hut1ua^m>Vv^Z#WY{K})H^ zy%{om@i<@X_#Dd}v4L`tZrNJ|6i<*@$~|0YM!uCxSzJ>Z$UA{i{GBotLlb`L&oZvO z=!*YX36B$CjhF^5ls1b<-_-8zCTdaHSBTz74FR3Zs?|BO7_AT!bi67UIvkL4Pm0kF z?nGY+neza-=hhkF6kw6HgX#bhdnN|myb(>FfAZs=taeOyr>7$X9G~i2ufc&`CR^pn z0Pv?10=Y|j#GiB;pKO298Q#0}@3UMto^LR48=#-FgcG}-$dn;9@?Yuzc_F&P+$)ua zgtkE8mF&y76sGpnpdesgN%WCHw4l3=bqsrV2}YTVTqJsa=m3W*S6^HWqARRE5lb2K zWtQoaRN#Zqq*Y2rNCA|MWmDMdr}J;yXoSzqubzi?SUq0~&^=YXNs#X*MACJr8DkGu z;U7q^Wt>c%m)Ijxq4k}$g(7bRO!r+C-=_#2zaA&}IG$10H>H{l-np^LB+f$uNNQ?2 zp2d!$(u44pY@8p!9gp6aHLG5`(#IA}%FSnFkM=4+p|i0wf=$jblgNVG^NtGyE({v*Xf4@L!epY+q+7{j;#B%%f~7G zml_Dtz}so|AR*3k7NnrYgVoxnlk5eprS0ieqwylr69g^`(`jK}F9^MT`n!>3c$Ee~ zY*|eou}?UejV6xk6}iQu=eklIE$&1oq=7Jko=27xr=bx=1dGWuKK*aylEHS0 zT8K;McNIT|M2Ai7fcYl}U|TZVS3a<2u+(lD{lZpB32DRueSy=a53~^B#Y=iZ0zvadN~2=V{@Ng& z5Vq>sGF#=65wZ$va|dz(xoG=;Zr0x1QA_&GRn|!(9cBwXvBHR&lbE+aVNnQB%ksX& zC!hN`JP;bMx@NWBgv@z5CE75)+kWrP;%_Lp@!NG1HkWeAgI))BT?A~+dpN_X43#p( z4;h(zJ8N=M`qet#V9t#uh$t4p?{o8n5bB%$a>h@@%f7}g#+sARAPJl$mxu7POAH2= z(e`IQ01f~UI|i>6OYesk`sM!` zq5uD8*^55+AGjOn@()mpqEVxM%lrqaHLm|JD7cjVAO9Qq{{Kq};NMZm{C7NhUCy|I{oQ=Q5CH~639pSIY4($Vho=S}_K2#Y&Tv>=W#@>|ud|LV*y-S??^ z4Ik4at)}l_?eoHh|LM)&N~T@h_348w=0wGbZdtG*NoC6|U%!Ke1C=+|Dv2_(m3a5m zubZ($9w5>+-JJ=BV)DH3VoN#B{TB~zv!P`OZaAv@or#oSsJftO{NujY)%=%|?^ zHFjY^WX9oA_R@eruI!ENj9D4OH_lbFtNF$FkoLfu5XPAmm%u`;OblfR^}T!G*X|nX zI9V)R1E#3nnUXZ3^&)B|KQU2`dpRYVk$b``s!Uav(wC~;^20tG>o&6fS6xAkn~u3M zU$R`hg~o`+m6@tafIY3|!9>GsNs@i0y>9PC;3A6EIt6J6{~mU-C|_Mu7rin_S;+%A z7#|m_0aJs47<`E4@s9QnZftK$m68PFcw)J^&v=RXV@HW1cdF)>0qo&HsbF_YP}n>(hp@U_=R5h0T6To(IvJ#+_~=T1sNmHiud)oFgPzn?sSn3K8wl zhLfwg;^R2+wZytnmDCY&vDy6CZtGNZAOwdUj>8BtKjbTO@4Yzi<%kdd3%=B(O4DQ< z&!Y?m@uU+deFCmf0WwVlmP*5A+&kf*yJ}=#8O0#zWN))WeQ4%_@BaXQi@@v<;;@LJ zVz{N86jpWupPue9ujWd#9YU#oJ5ujiN3ld-Nq^j{We}pHxXg~2FM?I1$!_2X{*U7T zgr}hAPQZ{>)E(33V#UJ}I$qJoktMuX;1KprC%Xnnv$+{;IK|epA>z;2tK<^ScM7x^ z*DeC%XsPO?0g=PvMT^JH>!slO)l%wk-$A+i#>a9zJ8&&oI&4`>~%RJee z5H@z6t8B(9ue4NZk{7ze!ylRPC9PRG6c-U!zg4r2eSNokxgaO(m(B0KmvSlHAhfAm zZ+*RLX$L#>PUi1K%&60mc;ytHeSd>+C@)YueYQ^T6*@4*h+BzWv68#4r6Ag7F(Q4f zBkniC*YzIMB=MW*RJV9g6SPo;XQB+E(At?Gw}wohNxhn^g*F--5$BDv8P#4HZHR;q zKnrhq)!&-rqff7N55vsFE%+>PPPy2)#SR%;Pe)ubTw`lP@v`Up0aoyo%J|B;78nlg ziX*&j6URA6_yMwgvXS6r&pXODw4NhOLDog)*(W`;oVcKuCt~00l<%7iepX_0N-pce z)j{`pVtqAIQqu;=#0gK6s6wC1wQWy)#rcrYX@iAWJI9gKS>*dP-6B|V^XqASGze09 z5&C$vMN_e9cgSDT?}^xwR9TWpXS~)m*dXFl;TXY;BtwmJ#$)&s`DkhVeHCvk(9Y?x zLwHhMdfYkleTlUomu3W+y3(tS(;$knu0qwjb|XrSU3U zpdX5Vz~>B0=-W!i*sGXYR`1}XbgI>88Fv}>)3M&ODr3!cU*8Dz9;dhHnGC+rGw8i6 zv?SzJZ_bnW-aWFu6QKD>WOQ85^!Hg%qVs?z?$B-w4x(ORFccfkclp=q zN|k(8-{+eeEcESB_A?>o2FKeYV|(m<0VG`g70w75IsIyBI)5)JEOPVGsq_Y9#k57T zsVOMP!XfUuRr;P%L|jFcpK`vkh*ha2YCLj{gRf}nICD&zIc^5kUsho$9BpPg(2tvI z>7R`xfMK=9c1b>DLSXNQ=Wty&eU(y`+-y&-Qf`WxU=h?USWW1cl64FEL0|YBLfp6@ zt#dB^odYYzM*0l>EC22M^w`9%i@?~y7V)uoHK&Mf_$y5uV@ER`ZX`zhRFYXT-{wt< zaottia8}R@7Ix~FZ}TOM8;^nRkz};;v}@0Q!AZfQ#sU1n>dmN#i1O@L-s_SzvjJks;bfj3W<6UPq=92C?1Afc?4%=ApQkmXF4h$0d25w7eg3UE+gPTP-d9 z`Z@;DGu-+{OFY&Rt+5W3NX_dyqxrUtN2t|IGMhS6lsx89w!5#L%u1*FxsidoBsLxa zaZ@>zb{gMqZxz_=98F55evLndcTIlAqkurtI4F`&QL*X3CUnH-Kv26VyBq${0NC>A>2oyZ`_!G5{z3l)L?dLw2Bp*YhW7vgz* zl}1pcQ|)&7*^(aktLOYw1R*gmAVFavaR%|+;UBVEHW%Z)A1*)qZ1c71&YN=cM}^UO zVz#r%D-wndJ_0~v1Vn!u;ct-09Ftt0!1uy~JY1B*HRN8w*9PLIcon}8t{BhkkeiJO z5F8uSF!*s+1`jd)g*1XCj0u$oeN%Nm&YlMN7T{~fGy6KQ9~@m{_ z0o77)%P|QW(LQdI{p&ux^>G`@50BHP(lQ$4%u#Ki$L^g` zfB_A(Pcjnbr_A7p!K`;zb?Ao=Rkk@N>EACeWR6cQ0bb~PvFfv??;0V zl{bImQUAPd9y0tYD>;71)cnbWa)s{v8wko@#I+|f8E&0mEb~o*qtlHqceQ*fsw9uH zUfyvNTX36#Tn7A>+~bzUoy}H+=B9$cpQ{s$mC>&eV<)x`gOIdKXGehvmnuJ7(~? zX={Vc36W&MflOnFa)p#$qUQA{4Ji(E5GI?|-Yt>qqXq1dxtq8<8e@q`WbmHCbmA?C zkt+3Zv!T1pRoq996lH%|-G2-=N3!+0W78-0@9RmHIGR;uAS!Ij3^#xw{vsBx8}k;U z@Uz8q!!FMeI89QOox0iDo`XlGToHG27&ajhY@Lf`#8OWLu0D|;Wd-q{&L!4vN`lRX zjLB<|z9Iol=cq*g$JAM1-qkM$_=Z+ijD0|)-=Lr~73zshZ_lv|>*5(9xIqwuV|~W5 z6RBOvRlk>PkC|(E_CMOF)w%WFkOy(UefSq3iN*HgT)v5D$&B23uh3+@S#Tfuqaj-u z^AwBk>jVfV*V1p(lctutaXM5aRH>7^&DaBQG#G0S{G>m^P|$sp$_qcbP3LRf^r`F{ zwiI*C%ltUdxL>9FM5w?AU&Pz2Sskz~UK?Mr%?JO^L#W{YMS6P3`eyRWNjv6=EPC-% zUeep<`I7c=Zp2Vkg74Goe(?K?zXEeU4DndfAgIqJdA(8})nB{Z-fh_ReLEyh(x&#s zNweW13krJ!@}JPY#8$$_%iI&E(6CkWqzgl)meRG6KEZrWCS-v`cH8yqkOEv07_xCo zB_5S)%HKQ+Xp_I_FQxj4-3;bNp(iSOqTEt)f_eLa1rb7?FkmM(DVQ4{SG<|PI?v(+ zGpgh%6tVqv6 zvsCCqeCV3H!<^^Mz&<*{O)V~Kd)q1Hs#<&Acj?ig9Yk zYB$61m~dR$90a^#e6_CiYKces*hlqGi)+^q!u-O8nTbWaSAQ~HAu?Y0#bb315VTF# zs?xhoc~jSxJ$*iADaC4&iQ=_HEgh)Ncb@n7G0~5YWkST>pFlC4{3q6I|KY6vCRI^| ztGG5sci>dDp&>WoKV0>HRl%t?(MzYY!&7ztA3$pVRc-lC|DXSF+&Y+)>oi-F$?bzH z$_C=G9lWlo(TB0h;0<=3>*II!+xExSrd&CtUN&vtocsJF#7WBOEsJhY)#c0G*-3Y& z4&POt_@CH+UrXdaS%|i?-+?Sj#OfLFc8mjMzs4h8Jo`O5ST2G$3QPEd?>a6%^mnFH zuMt35h=yT;G6istqF2jMtfv`{U+c365K7e;PfCS^~H% z03cu4=fwXD{+^fqa!LQ`1NHx`@WNjRj_+;$l@9Lg+)K9)U%z+i(ic_l$Ru)W53rix z|MUy$PX?1Lo4uCb^+%2?2EEr#~>YIp;Sx-J1jOd|kr3%brgq z24)0b7|L^~)M0CgrkQ(q)p{2lL#B(FFmF=z@f(lc&Rk-VLOCwPXvC8Bccu%xinCLn ze7t!gcs0M=c~JNl1o<@jue5?5&;Dr6HeTvoqRTQ4kGpPQ*bMTClhQc~5$sd<98HwO z7mn^Re`;?nBGk!Yf~c0bIplzx-JUr%Y33Zch09^SI{V^udi%tvPl`h$$yetjv`SJh z-Z7CA7&X`Bmn0}zSe*YMGm+@e_O4IbF(^E8Rx`^PU!CE17mLb|mPfu48x1XyY<$sX zEOB0f)k7#c$b(|cPjz6w8VIegNzCA~%pU>LR~}d8{^-@ZYl6nR8#<)E0Z|HuohX$0 z4+gNFn}-sh={K_5PQjI4VGI!`vy?$KGlwXl#IqBlR%s>CpO+5u*RA_x2J+Q?4L7$E znWr8-TT_IkW2Rn74PVGpT3!xcd#f8w>BNpJTMrA#+se^f(l;9VD>z1rfbuIUxo#$L zy3QaOfZ-d+=V81b;YWyr`N!p!(oo;L-)h_|I(b3AcaHkr0}rS{*cPeqodpnsZTh#e z=_seI^76RQ{Fs7lR0OSsT*V30FJ*WKFV?j69Ldv5Z}b1wSl;yt+2Ya)_J%xwbT?}! z%VN@+lFa5P#`x-k?g>Y$Gp(EU23a25k(le(CF?4p5}aUykS8=#a!1#fVs*`YW`n00 zVpzQ&JU^NEgV9(&x*5;MG2u)?JY#M07n4c%gki@>d1;5doM^|vlGcKPm;k@(`3BmV$*+k6 z(pE)g7bWdn2u>;sbL^;thtBW7AOn4dX34Sv&F{@rQ&Fhsmi5ynSyr?K$zA*|6ezcE zDD}j)q_!p<($P)^IHTKl-MJbm5xcp>I^}$6StJ{KvBpyWEU5hu(bULu@=@A{8C`wj z3$2SQcGutqA7<<2;x)<^05DfCZB0^xFqE=CqDbK6*`Zlk3hg$2A$*{N>J@O5jFHU?L;h4N^`;+OT zN%@)9w-3?e@x3PpPS4$b)9n;(ADm7PA7|{BJL3~`=70QlTeBi9SViBiY3`|Q>r(x+ z6`V6SI&ZkYB)ufaz%_UvoFiQ(*vvI?%jMaC5h4|R zdrf$c0_l>3cp#?=rYHGxw8SAkWVVi4&a}(12d?8d?;&8=`On%l@vN4QYpr?57l-kt z_mtG7B(MIcoCxB)(>KaMKGU2T*y`Uz01~wux>9V_swrpmqIkkNKeCzSl6T!e##O=R zj=2?UZCi|yT~ehj$0Jq(2Ub7U3V$*kh}3?OcBrstjk3kU6Tev{$k@#W+=sF_Ip^fR z6z*uprteM;`Tu0{AG$nVJ(AvN705n<4hoCYYKBcCdo?YQz4fs$sfXlX&etgXYw5ge zxaRVdW>a&l^`4Q9s%O3mOj*l!FUv-3sfF%=P!6bvNx=(TM zn67K7rh;$z1z+_Rme9F)kuh-^p81uqhyR}8>IGzC`oC-^q{x7Ux1(p+vHnV8G)4K1ooG_ODLfAkn3Adu_nFMgxo$38|-)?jAO9m#RmtG!-h}^g{Q7*0C z_vH4YYN0{mXL~Cr;`zlml!QrxN&7%0LP~dRpM;#JaMsvcU%1`MKz?mfLjc5a$ppyC z=~Um&Xl=#)jErR9x3q-+?nku{`#n4d}H!3C@_3w6V-hHwlB1~ zTRQqftYUCL9;5J}-=L*hahlR0pvj=YiA+IPml{+}Tl+?dt#PU)+Fz8^5$E5xCa`tJ zN?VTgae^fczIW7&edxJfC>?6QELLbW@2$BK)YP!xyeqx-7)E*$&!S6L#_R@P+t&-o zL6DpiAP&gU_;_L1{KPvIYm~pmt0xbqy2|;QfY;0sxaSrYrXC-VuH5@FU>mrD4-jG; zN#5TwUHT-*Q8<;pSPYPU%2CVkVcQA{Gc8dGI~QC#y+V-4sh?U^fjhW*l$NvS27QC- znJMUTXOR$rNkTuAJynl$z1*-PuQ$%mux6C9*#~s0M41sVnz`VD=84F`NI5VpO9`JI z(s8+zsVUM6w*0;3CsXgCw1z{f<>Vbn>g-`}XefV;gF)g*iUV;BPEw1D_A@o^*O4gn zymq|qV2K+`Fy@q7_%(A%3GM1C_QfW2x@CG~NW2oeKb!n`4Y_n%05&s+GW=FI63Gy; z027Nq5;9f5*S+>i4t^>SQ`Mwt;r3H9ci*-%UBCG3#DC@2--osOz520D=Waza2Ys9S zrK@)%J*Z{I#?9X@C{mPADn=1Ph zliLIt$Eg$zG*DxZ;tSaR$;9r*A!TT^c&3>h|t-K;iS~7jQvWxBBvboEAb}UH>jW%-N&MQo1h~Xy5 zX?uoT;s`k6Z}{3$H+SFEvp5uE)}aTvnw1bqY%hp*i)B{;NbPK-{C++~Y`ga)?=Anv zK*{QUJ4yB-SpI}#!dq*xj8kJRFHUO-+J@s8EcC&8pF#UY0mt<5)K$>aM=Doa_VnWK z#Jf!+23no+Rrj5ecg%%1i0`H7Uw?=;r>Im8JY2~cO1(#sHPyM8YS~$2vkBXPG29qZ z7*GgSX1j{9`$4pw1KF#qi9eZqn~Sq&Bjfs&wI3&3&lLrmIY(vqR;zvQF+{F-3AkD& zy_oOW;E|!RGvmIcLRu~6aKZ99exh19$L;~fX_;G6aHq$I^D3YIetZ8jLVqP&yY*M1 z;b$SXWwALPLaw3XbcENyTP7xE%VH1HpyH2Z5ZoSBq4VgZ?gJ(#W|*IWVSdb}P8RTzo7zUo?L0ca>+S{$Amw#0}TBxg+pLXhopmom=$W zXCq9fKDAZ+WU5US^(MHLb&u@aA81t}dv@b9-Zsm_J6A;`M4N^I5zf9gypYd@JrHn+ zg?4jq@f|@O`50DBsTP#Aw3cZ{qMUD||BiPAe`;T35k+<~eeUbqtrFCYxO`)g<#WAY&;p1JK z*EgTd*lou?|H(9pJwB7XDN1C=J#H;KxX&}Ai=o+xo70^K`WYu)g+Ua)K5m&=#Td73#ZSD#8Z_qv5^v6dma0zkvxC$4nGym@R|Gt0l*s3yHQ~!R6Z#7U4@-80@gl zICqCW5g==_->2Z&O;_Ytd01sRZ>@6mx9n($R>|8!*J2^a7`E?i9G6Ea*B$5BiNd60 zy9dkiMWAvWn-rV_V)jjME^mGAglqgmamiL{7)~5Nakv3xg~-T5rSu^nv(|_Sjotts z+z!9tK*5)S7DZvlh!`kz(MBiTgAx_D>+H^*t`M)+u_N@2GirtnQbinR$V+z!>ON92 zbW z9fH7#{A8l`Jk*Q&*fJfhYwdlPnQx03DdMnX8%YMt76uwaC5Rh`Z(cgA7RFp?Pv8wO zDFUayCaF`z_Fa4Nsy~^IWUk0QJCX4B;qZ@yF8HWZA1AFz{bkB_#q`;!o7iBI=~hT% zpIOo<vfxp zuZ?+lh|6ZxclH9hH{3Y=B}a5Eb=`-`=LB6cVMts(vWAwQaM6B^7`QwI9W}jL1d(8a zMTWg46(Lh#}gG61#YHRyY^C#2We{I6)p4cF)@QAmZ*Ah|huFj-`GygTsIA9+uFc{u)I$|Kuqp}6r$Vqe7K!(`Vv zY$>z_dJT~!m=Nv5AQK}Zp~91K`8pfV{wTr3bTxz3W?RRg^F+rikN)brwt~Vh>4j z&BFlkVFjQJ+-+P`m;WQvWs%*eLUTBAT*r@**%x~K3Ff9aEc^AYiSuCJRqKLc6ddL_ zHe6-+?$pp<>i=)~^>3lwlZu2~d33lRRiIK+`~e=67IRCHp#Q zERS_jZFvfJfZ7AqJd1T1z5<|Rpd{Am+Lp`fcdOChSV-q!AR8z2sochXyO&|F z*Ie5$QHWSI^s%XI;57SoRBPv}LKJMi9K z$;_Xz9HjWOkhc!dwLJai^t}znyh&Za;L~9B&BkjLE6kE~0Ix&wu3t^Ql=2JXx)tl!t{i7GwmH)DU4OJFe3z5>-YWWhgNho@ zw@x-yjj0$Oh8UMxQ7t>T*9;EGTShSjJE(;qoc59=Sf);)jR%jj;Yq*^2`s^33|NJo z^jw>3;!K!ynt{BljYplH&8he_-WEzQ?{_jUSWiTxp|Y35=t?Uof)6Om_g(w_{Nton zvcF~SAzh|0*i_Dn*S$MoH7UNom}?lI5}2I6g_h!NxYBT;{%SGze4fW@x>owoS<&0FcH@}ToGs;j@@LSD}t#2oQG&o)Ospj(;ipxA0nAXDf4*VeIzAI<-y9XBT zLx&upQ~oItJ{0RA4iU>O@W#GQtum)T?vdNIiVNC{Uvr|#wm9pd=H};g2mJvB6%*0G zJ5gzRJes62ffo(IFMtFbv8#0ubj6V*v`GJtb>?8#sV*`Fx zzsf&JVxRMS3`f}c5&WV6uA%aIcQfmNARo|jIX=ZiyC;a)TRS1ox^bZ?(q(ZJ5$EFk znp_=gIIm!m1)H4IA3!;mwp6ULg%=Wg;m_@x4K-$0bMu3cIkfd&UB`NUH)_qOc`(VN z#47F!j?`hqZHVtJ)`j&Jt9A%NAEvTt1y@dM!*J^tI$2&WW%ilPb};5y!Qz#ptM;0$ zB!-7&obyAg+uvo;cQ*V(sIt>KOiYpg$a9l@U40;rc6?3odp#`i)Wg3gGD4T)?p`BksmrS|)CysM%&~u!_>V-Gm2bdGyY`a)^6oz` z_UXhsa2*|cc;}G|UT+PT5x>Xrk}*{e`?H$|G%;O_KgP!$Igs2r2)Wl3K6ZAQ{jY~| ze^-P16MG;(hia?YaoWR_J+W$?mQjg8S?HjOYLbRK6ensq7bmKlxt?+Lc<}hcnCl)W zbPpQFM&-(OsR;M@*x$OEA#^b|au&1!OBh^T1S6}u*pT<9S5+KZcoZ8CE{7U4P_NiX zRqhe;R!+%w3@wcnTF2K!7UI#IDPj)eaW~53pxjpIstey-XE2dZQdCV##+zmkFfD-{ z06^?8KeF7ZiRGG?p8=Zcp7FSE>Ri!VD+f#181DUuoCE^cWnMJntI0r$&*Ib9=J$`% zdE9c^YA$DnUFn_&e|f(^v>BDPE_PDANae;G10COrTU7@X!Wk;=Pbyk+q`%#%;zXCH zplZSHo)3r7SzlH_rdY>%$qunAxoaH|0kplJvbc2wrdm|Q1MbhW@ly2sm_lRV393Dd znj|E#eAHxFyeNGp&gj01DNyT{49PS=#ZM-SfpQRSPdPU-PnAc8!jsZ3SeZ8-EI0!z z#BhQo_5n(QNKp+p>*n}-N(}#5HYixTAe`(s?B}I0#qyB8cajeB-8gOO#s6y)DQ;l8 zmG;F`6lYehrR-sxmtTyO0RU?KoTF{-*{f?vbN_P}u>JD#`5KkRsFQ<}g=M~|fTLj3 z0(hd^3jrgLvPb~*mLnB3wi=&oY|PLIn^k+|j)bKsH7V%zGD{Xd3PP50kG`X_jg+YI z#**W+fCPJ;OkRQ}DIAXz0&7@TuhdtvcuY@Luv`NAjBrq&m*gfcmzjr(%js(a7AS|h zF)-)3!MLn!2J-({uRtFJ|83@&Qe*X8+P1C4#`zR&%bh;JaCFVbRuruVW*N!4E>R!P zIBS?OWX`ItB3ydw@*{pSX&VaBIV5XMRvqjs(h8>B`0fo1O`pS$>hKmD`aH+;#2!`T z9qUG>bvj47)ss^m7dNZ2miNL)fbu1qplXwTfRt{$`nchc=Z2qdsPR5PKH-%lL&(XC zoiR>9zDnDT&k8yO_{dWq7bK7k^1+in7nL@VT@I*!H;fkEz@wPvTq-MO!Tw-7Oe=aC%{&QWpv zt&;~XdEKaZol*(r7g9VXbp6l5;5|2=j?-XifIxPBl3p2(WK45P)TKh;c?;MBNXhkm zEk(KZ!2!X{G~Db0m@2ZN{Cz(A6IUQC$*Agl#S!Xk8%GdZBx7TZ8dM=)vO~vLo{34Wd?Fx_P591mzSD42E9jn^2YtN3=HEixg>humM990%bzZ z`D+GmV@B5++GfQ zaSuM#p@KCaCb!B!!dSPYN?rUWSF85EG6)%aMIF9^k`4=5lC%-|wOR2NpL-}{x{6BC zXdj0)mYHtR;2W_NO)Z{&`E01@3e<6a(P1nKeC7R{$=oFv3QhkP=FstcBONm;MA~hGmm#B&2V7P3PE!gt=3pt zQZat%v;W z(OY$=@m}Qj`L+mHtWbsbQBLp;uC^uTK&m1skMX7%v~?USSmW`u4^j|rRWXBip;Df? z?HQL{2!HbJ0$@3DaKM@;u6V;F+iKi#*fAL(l_a2I;^;fY+!TBSq}TFy?gc&|{FPpiMIX?6z6&oi2q(D&L(R%xLb6?Cfw!NsWYAp)69eLc={`S^J zNq-RpKyvH~iJtbItQP6$&r!Xy;^J|~C+w96^Ed6OW-0+{btTi7sH4{^4|0&5sIM(j zAsS7WsuB%4qx*`=k1OcoL@)f8YPV){fwj_Zr?5>c6=8=efbO8@V- zt>VinMd{gLx0nNiRb!Qp^#bz~Nkw32`PpI9&2#Io20N&#`6>JNu+o69{R|R*$*e=I z&=hMQO5%27o$WP73I6)twK`Sqh@_1VFp#t2EL*fMM<8>BDWb|@=rb{Oeb+Uq?K3L*q@-2S2f$5m>f zu-UbJ16^KGYV5Zh6I-jy7V1R-;_+%n&5)KW#~kY5(Wk6k>6b<&jznpR6u62>n;f_h zPrhqp_O5J40%f=uby^d7H>euOttu?%R_3kAeCYa>1 z_t_UG3LJxFV{5G4Qqp*cutMX7TtWwf{Tb-!z#24U74x$9V$Z0Gc9r`8_d~bD?(&$s z23;zbz0v>z*f0b>D}j5A%GSQh5c(~^#}k(t&xWR2Nzn~By1k?>RC;K+`rh73mMCB9 zKesq-wyb3UAkiUK*>dhH(@CF3x*@;%WegtY_7d9}i|@=U!G&+i3cF(0xBSPuhW z0~zpR7XYjsGC=9XWi25DNg|q7p-+s1H@)gwQZ63RW{;l}i5s+aJABL_-D&zDT1@Wf zG0i(_RjFeAaCH5p9&_9480J7_;)NzuHYYqTIR*riq<|*k0XR{^#mOBoiQOT1?bhoXZgp1!m1vhIdL7lji{%uo z_THK%i=->~U2T!7IUodk?AciD_6+@4)aMZ&CETI~%KP~=591-i66FP}RB@oeM3&1` ze1tpeYcn}Fp}>U2h4q+``B{xmPc+9DSJap9oc(qVc3l#}M}!ohqe14RToV%CXIwqk2oU+aykwXaqZa4RdC}ob@C!mE zlFMGqF)#CzL9A185QQNmxs1FX?~~0>FoVxyBV9X7mD;9fCIY>Bb4EFv#p3k0^GbqJCbc`>m6d!?Lp2~RRV2c@q;*l^LpJYLF> z;wO+jsY~7`!ZM7!3R;iEKLttzqeff`*3ES-M;_@aSIOaZL?IC8_e2OpLrcoFJ3Oag zyb?Mk+?apiE4)Fzni>S*3S9_H5L-3VGs#{ccUB>$sj}-=O_8XdOpb#Z5QM-|l;=28 z^MT-1b63=8iCcN6*VvE11QQ2 zPVuDiUn&QW3PxQTwpk>ZJQ#FaXZu`_004I={n!+!(cM_D{E<-FCP(KiX)Wm$2Xn>4 zZ)em7E)W5NwTEhwu5XOK5=2zad{6vG738xS8?3PUIeQi1Jo5=^Oca zO#XHQDj?1VvCE}frN%N8jS?V>;Ze1wW>Y_IaHU+BDU=N>r# zP6?giLwfN%*@2iyGA_>_&v-&gud^KqHdbzBJT$jAD7ya&HE#bQH|gT|l|+jV_-DaR3V1TTD4rSup8;OFCJ$-ten5>hBPFuvmIESSi5uDhESNDUuz{6O9gKOoLX~C zqW|Im|GmvVk?A*Or)M1fx0hfdkG+`CWuJ+l zKzj$<`c}=nr%Jyv`DhEcM+s;)$NG<<^@ltC{`U(0-S`RHnKdQV3>e3S|6f8S1`fJj z?6J31hcpIS)r{Q&D%40DmhG+B@m;ZcUtbq;P_k)B{gy*bFPmtEcgQF(asKW~iTP^Y zXvNl6{d)@+qG}zF#+ec3T<{pq&sqYt8$BFC21vgy0Y?JDakaW5apq)OThc9Z%yY34 z=GKa{4ypd7c&AnfCa){B_&l>z?pI@T1}C==2r7US(W#uMX8tnw;)&uzf3Lb zQ9t9>p`PVFATKv$AE3lWfn_ax!?AD`F?l zKZ)Qm;CTd766~kC8JG2WcCdBsJ7GlJb|^6~4Y@vg#WbtH@{6nQT$_|clfZpla(egL zF>eKw-e~tNZfrSwLUEp`wWUPb>0}}%#YL6dtF<3B@Vqrvr>=iWI{EcYvst&1m4#p9`__r(NZz|)uq)}geR=-Uhm19t^J;<& z;&GxsT>a(A*`WQ8>8BN09#uBFJ0=W|xuJ8=_VC|Yd}|1KXT*K*_8mo<^W*dcb{ZhB zFyTRdqt>(N6H|gwk|V^bs?FEl9#^eyID36xi?nz@`7ma1SOAY4o9$>%VL*vBzWvUU zQnHoEca(A@Ut+Jy7m;6QUtK5Bl!UTPJoKv60vdEWD+@;}C#efVB2rBREn67qj8Q67 z?pxUjSy?&|_|KT>T@v9E7woW6s;S(6g;3bS;igx(vlC0XVyw*RO(UyTdCzw%UEiI>6q;o&OUpKH}fLOw#Yqh2_kOU$kb$pdRoe) zbO>9}`LDOmb>H=9rc#rF#NK#xYiBkqqH@KSZMYxD^D_>4ol=N4J^EX{l5L-Vwf=P2 z_3|gvi6EAb)^1Srm4%?z%6^oDe%*9HPC25tb9g;_8w@P0SG1M5xWC@7&Gg*(nJJh>$OgT<5 zFX6w`JTNJ3Gs*F&tNNB+P^GmXCT~ub?@Ak=Ud9z%bB`EXkv++ncWoJuflJnZ^=k+c z{wVvgPA)Spz)U;7f5@a(5SyJOL2BLwX)ye5jg2j_JyDgkht@H%w;zvvrmbO)T*>y` z#&6^kSaa!%9_Cr|@t^Ol(QaLPqkkArhV<8TQtx|up}&w{92kvAnsu930F*p{ED+C1 z+Oo+7926&@azAxgLAax4&1MaxPn2Ke6qY{Dcvuyj_ArW{D~C`_?x(v7=0p>hWb`1i z=E;J&0fNRow%@Wun_X%vy;whuI?+#+%n7fseUWgLaAk+gTo@LXUB;j_^SL6aleuiN z@C2|%uj<=x4_MxFvpiH~@lyXJZPUcjZ|yl!eI0tYBe#A!au$zu4=iI4fJ)Iyx_otymHg~5+7DKAW1WMw(XBS3+L4&WV~u=m-o&& zP6a1*1Y$?oJTCpUFQgFGoskV}iYdaE)W3lPn>uh5HA#UukN5M(4uw;(bc7q$5H z$|RNbLof$+#@R>RkP;G0#s@@Xnx) z=XDFM6t6R%Z=-+}W8RXQLQQd$2PEq5X0LFMt3_r%gxm)U$VO;pvxg@mJ+1fH|CzCp zNIN;}W>fF=gwYjl|G6tZU|)9pRcJX(k*bjA&Y-hnFc0U;!RE#V`2m@` z78-M1YL~!ZFvJaF+>T8#HWk*VJbDqbAaeHOjJT9Yi+L0G8lb9689cgF0-*w2AI7rt zMo;eu5OjwKx@+Qb&XK_nnY-aPnSS|q=L`RC!imiU{sC`?AIp5G^C*Ez0-H0ZZXV4n zb1Xq*`hC?woreXE@R*kS{=?bsu@lP$;_{EeGdxGtCOtM~jwL*6CtOyYjz@6UYvify+EDZ^Fj+_%HX*6 zd|sMIuOtKCP;yNEVpBU+G3muGkk`y{kTacUjyafne)u>~djcKEDdpZLyWtGbL|u-k zRQG5*<8JiQOr|9%tO zCtkkgC6|&3$rn@vE(RSsdGX5NAUPQ)IHZV~W87Y2-qlzLr|?)nAm@}|Dg2jr5QySg zc9Y4(ST>kzY)oj@X|c07{ku2vUnem9;$KPiZ>AMr zCAK_sv(SUOl!EP9h*jxIv0tUz-$YfxX7TaED80Bm&E-b(xFoKrK!dFaWNu}D8-)8Y zhQvQ;om3>-5%8I`&HoWd@jF+PGU_c@2vjT&6;E-Twrv?IDLI@i@+lZUS5Td#0GDR? zNkt{CZjB;ap^1{do%?FIIKNMC+~d3n?#4g%24PY)GG=@s(@Y4`9*wj;=Dz*)bg?;$y+r)v9M_RNg>H?INcI4nIW&q z8m0(xpW(2}F16gW;RCg5ZF)D)>u_tYZ<{ZWxaVxQ1PH-X_Z?0(wJOdxlpft1+hp%g z9{^#w;tV$&*MlWt&IIdNs1V4eE%_OW{Bkdy_G9l>*}m4N@FjaUJKjRj2A&^{vhjDk z>GY-1dx-v3lOp~oyZL=6X4FE~crCyG(6JvMr)1emwoE-578=WwE=6MvqRTs?Zo1sM zJ$}1LtvnL<{8u;j@Fg~1LYV;rC#)N7(;1ok8!S+BoGFyxlb0U{n#`4Lf5DC0jRwCs z<6T-DSTnd}nq{>tJgMdwzHx?s?w2^4L9=Q0c%5rt1=)YwcGLZ`OrSpfsee&2-{*Z? z*7vq3V@VGQJa4@Iu;>h7TT;xXAO)(uHd!YP1U{X$;+mAgZbpP)UKL|Od(HI*+9$N| zpr;s7!Nq*WUOhDXjL(ULG6HLi-;cCG`fE~9_(*`L`5|1a4W2>&pU-WXf*=kN>t6TzxXN%kj2l&fop zGRw;33WtTFw*^FlAqWk_Z`ds}v8k7l8^E5hOAiJtT@z>^oVxLY^*7t=LE+=$fM7uq z<$W9J4UiS%mRjEKfc3ance8JKSCh=f&Ti`Gn-P1N#Ub!SxqEcLG66i=(wJ#nLhaIz ztvk0hF|=>EVC$=2+M&z}x~GnK`D&ffYGWNlx^gIB{_XH+GLLNx#A|$v^J0W?Nd$9c5O1D_HryV8QoZmXsws+toYoihx!2ZDYTm!u;J%e>2!W z6!?b^hkL8Nrtb5fCHFpRdT`$k6($d+QZOxsKaSY2tEp+nhXU&p3#0Ju5qExXblZi&|xhF0$R{8 z^}eB`V`F$V@WFE)8(}vN^h&(%Xts9JGy=%k<2fGwH8h(ZKQ!$9LdE%42gq7PG$b;& zAen5Dz5`w`!x7dN@%57R7M{6;%(%$Bbg{)*`_=;?d!=Z&g)4Zz{rgdCc_$j}@q)$9k>r9%G zZRKVFIf#w^@QA^)aL5mwU6vB`TXIOm?UVeFA9?>1`yWgEqrE_J`MO})hFcA%h_3cs z%Rz0@Pp0dPnXzFM=C6bI#8;ck43Qw_wx3M#13xZ4o(OB^@%fTb;g-*GVXeeH>Drg0 zi=~!XsMVn5$3`@ke7OYOAWGw->uY|N?}(;jdHRvTe-N>-!P!Ptvi|| z1p$Fx0RfR&P>0NYzps!lQxPbhvfxU4mR(;QA%@ppYeKTS!a#gBlGEV9j(hvT2s*@6 z0SW2ANd0i1qFp{yX!K0+p~qv`@P#A&kwA`q5Hl-KuD^<)pb$ZEa?92f2-(}P-$#B( zUb}H8xZ4fHyzxSw`lYhZbokSlfp@F`E`kW;eJ6LQA}WyQ+V514FlDiDQ@qvSTTu4E z$S3sJqGhN{zs;~aPL&vtm(vBDH+A;n`9cn_RB(^sc*N^qLw0u+jHwnJG35Xt`Seys8{7(XcuUDA>{Z9xX&+a+TvC3S!bX(SI6Cdh3x5l)XftA0yC84bWtGBv5#k6K zt~T*zi6h}6X`EHC`B00781_|hw;EBnVjHWaH$P>M#9A`Ae^JyY-{~~u_H6k7V(-19 zn%esOVeYkD8+wr{peUWtu0V)X3%vvc5(0!G9RdWUO10BLDG7!uEu=t@5+H=&Rf+@* zJ@lgVj&u;zH+Qbj%$ncx%)9P0Gr#rDyVm5NlXcEMJ3HUC_dff4_owWZfDr|wxim@` zmP+wLQBkHA9^rgD1C?VmCzb@ZG|ynaj7i@zNuW6#(aYMuEQ_^?r`X1=1`(vQvU}tV z=DkH8SNFUL@IJUZqAC1(zzMfP^r;#C+~lX6@kfp-XMvvy)ukNNtVM>_2%nAbJCiddsQ2}>X$Dpjq+z1c#jinH%2PqwqXXD0y;nlOf}vQj*uFj^$VHOL+EJ` zMwtsK;vUnZ7}pNOC#lDu@4u~eDtN+pjPZ~p%p)PvvTt3TZByHpd+fX>AhsiKF#NGp z5v%3*KVwxt-n&-~NA+0tLYe(=SKrF1JdDGooM%DV*CmHy-y#|ffUX*q9FuU(izJL2 zp|KBiRxZi-YbuA*qL^^Wn%X5j$KsGd-s6VJnxM@AV84mZ>cAZ!WWf- zrE(0YnGN|Zsbwu3pC8e@BN=BgQsU39f@hTvc-Akp3^LVLtoK>~^0zK!0~dN%g!qjr zBXS!<>o%GS>YW6dn}r{d&mfHi_*YCV^W_W^bg`@66^bhv%ku2Ygs%})>ujqp@s*VU zf#$&OUPbss^ShWn%SwxV6bu8}oSU}4uWk;V%%shB#+S=+8#TphL|n`$7moA1o+#|b zw4{VBKC9)cInG^HKv}UIupJ3kFds*)VXF|+$%cCa3Hr$+-HHea1>ak9N5f?ixlasH30;hX@8FU-`OUnpVuwW6A>Y22HX$MHXbN6*vawmXQpx3zH$L_o$p&uTMaYg zOrl$uw^HPgFuBldYRH=R6x+y-$!{ z?whOb6ZZJLH&QC{mdOJSPI(j~qNt&XwI<75Tf3`iQx0GgGDCOElw~SrY6l+4mZ4)z++Ji_2p3Y$rk14pm{=_xmRqZhvi07l@flI{3 z)Vu3PyM|YGsJ-}-RA^VvgjhhpYPS7!@FN2-N#&fLYEzftXBxk_7dm zUCN?0|HgPcf+LNX4}Ts1#0BNuD~A2@*fVW@P1nHzo)30@Zj`E@9Ne2$w;(EUXee3VwBhyLvVRduU954qPL@@pdE&y#Tk}0 znlmmiAa7et;{KO*4!^75yt~AvYvV(HRa)M!YKEnq0^}0SdKg6ygp`cG8Ni(DMUiT< z)y;0`0n>YxqNHRqlGPl75=>A7>g=J^aqhK0H%E(nER7*?UG3=%O-adJlUr=5oU~UyHYTDyP1sZT8&0x`#$%LnP-CIVucNQ?F$a(H?l#-VDXhKI;q8 zb!+EjOMmvOPm5uUjDqKd>B>Y~j3%g9?Nauf#3F>j&GoK&q{b_^3z|(2hThxP9ScC~>lzqa$9q(C|B( zJYNj$WM5RiQ}FFCx$KjGGuBS|)Da28xKp(~`i1Q>y%;l#dK?aiABR}@2NnA<4#4IDcdbGTXDp7;K}=rQ!2>4FPZ1w3vU=1v-?l2 z$@B=TKqGIJa~3=>panTQy~|{OtGpOi)G~sgLf;K7Nt9+M8_C3Bh!*CvrN}L9lV|pU z7G`EAy05uEt5#|paMO#GnxvXz7aYS+f)y=_2z}a}c#8)xmF?1KKGrIR=Lv_I<<)+S zqxHVlfZ^5BPD|1=q+xt_%FDB=Z(oW$mA$B>G%&0h*T%bKNVHuyQnbS1N`f`PPM*B*zog5^r~AT&hNigoRb^?iLc!l29&0TS20+V0-ycx$vbPkbaBuqekI5V0ho4 zZqz9o9yxS>f4PGU=D`Jl9I(#3?ZD*hRjXQ4AI)|p{{eCapENo8)o`x()^wk?Jz0}T8=Su?M!=5v6(v=~<;Bsj zpeXtIU*o&#=dE8R_5pXAs^@N!7cWfO1k~6s@F(Rwx5QH7D544WU)(7c+1wVnO3Bkd%k=cTgDwAqdT{@QBmk5d0UB>txu zpP;1OyF-k{T=u%6nwpj-tc^h}-t2x0VIk5p-9I<{bM>ccsT~bpGV{I5ay_)7K|%v@ zKP|R3rquM#@+R2s())(s!K5Ml&Rcg9&c8I;P zO8IN_yQv2xDZA9lef~<&*RJg6U7vqq-1zH({C6dP&OS%Bp811-`!9(9r)eMA+5iqd zki%RSPq%1?X-YpheUmj~+mt;!J?++X^=y)>O@QB)gyi!beuqbfifd&YgqapGE{|N^ z3va&Zb&uFKo_D^O;Uuli4(l`3pS#tvS*Po=18SSi4>3)?ChL4ET!gIuij;i0xbDUx zcX)h?vW~y)hi@Q|X&dNbO3DH1eWSXJP1DFlM2$>8`dIM$f^C^*6hd-N^jgfI=KIO2P?+CE{Gu@NvC7BfR4r+< zDB|{2z7v*FNi4&ni?qVmv2HG3Ey8qAs3z zn~9V3anrAtFAb_TDguOE$~7u0JfA#Rab(*inac;rA;|*>`W|Od4GiM>$HU5;};+569r4>`Ol6clj!g+-e6Ua=V1uC7>IRql&^GbG4PDy7((r@dBdQUeX2thTT)hYeqQZ$u~X zm9hYC%rYH@3Uwae=R2gF&eMBe(P60&mt<70JlE*J>F0$3Aeluf$*mcwvLj5jHfI63 zM$0od-}+4-Y^$G*OZF0v!)j}XpVMZLoGIP@?ZFQ+Bi)V0Xy{Kv;zK1Pw$%*T(?U9T zvD!0AoGRQOIx!5$4tUKyB`_bUxcMpPq#skS+EDy=l76ni?coGUE$sp}!lpaN*~gj0 zL%t?`!A?fulk%GveRbRzW?Q@nN3e!K^D2W7kB<1}uNWOULnd(yhIxV#z{u2N^k5%K zm5sYk1LY12Co;OCR|D)tA3o2JsOi76?0&TNOXIy##)4=S(_dAhPuw$OJuV)vDYlWD z&qK|YT^s{DXsH1_7M;Dzjhh*ln8w|=yXqB-tHpmJYpJ%>=#=SMfP(oFx~7vu1J;lc zp`phw?4KVM&}gbwhdq^L?geYfs@v#ru{$Zo^itz3i_0Oi?X#?K32Jfs1F>4qtP4+%G%~7P%)U-Gt09 zsV$b&A3N2lP>`xjWZL#2*Cw$c;s#XAZaX(mLIC9hPEKEW_S~Wj6K9#MNXYKNEwCa}#1;(R1N@a;GLNuq9`{vbS}6 zx<*yo%AU>{$)e2lP<-bgkj^~tgpq&ACyBS?4adCfVkHZ8DW!D0O!xG(EhQDsZILHP z9M>viKck~_{7jV+BdR4HSXw)p#nd;ypCW0qxp6rI+zDr|1W4@^)wHQLC$YCpM=NR)pBPeB@7BT9HnbSJ>uqtr(A; z!7tnA@+uTZa9cEl2L=uD^eP92SubyqQKvriNGGPa_Lc5s;C8n2u?`n@t$>{RdzLjn ztrT3+<385L(hGr`kaYA{6^X(+3wB8Du^#N^pwRJM@9~f62P-VPOM|TS(O=!ldqq+u z618OMh$iVjR7I3>j8FuGMD@$(-39jRI`x1Q;XJx?W zo)-T;f}nKEqVsb(I(5*`igWOzhBbLC_^ky|J+b#SlD*@1VE5!19Z`$%mvL!*uc?rL zA*|ty!_;+FC!v5gH;cREMDDqB7yTpkP|F*vfARAPjq>aCgOjb7cbZtwf#sj?vhgX8 zerIExm1e0WvQ{_+6`h;TPp_|rUf(;TzC!uVmLUjZN#Q>@e6Hu3JNy2`gVC)ApZOeK zM0*?< z{27pVe+5=@yq+$e1{UJqUZr#4tA43Se~RqDlIW7TMo@0G!kv?+O*WGD71&-cAJ0V=m`8=0>@W2^%~7AmI~K@96!ik=rVl$811lw+<;h zpml5T^H#0^u|oR#)2a7<9&l8G|qli?XPSV%p~ zZ+_tu7F9a6U1Nc2!>9QmTr*Ly&|-t671pLpsIt^|HpF+fU9Bw)yMqe8UaxJQ zEa+}&)ZN29B8P%##(fdY>t^F-x`k~Lg9FOVD?7UICS=;VsrQ|tn+dpWF?0*MU?SEG zehGc+I1<_8lMbrLS-6{Xs!j)G#&ci6Z@Sq&oDA{{kiYf(gG1HIs&#o_?rHnq$gU}0 z;fe+5lUOf|K)~z}-|mo* zEHD;E>GCZmso^Kte)_w?|Cdi36PVF)6~kiT5vtIB;2WAYF$FO9olP#By3e;N_>awN#TApIw3n`YkQVjPgxyz(*0@SkeizUsP>HUoZ!G`jJC6?AP^<-0`ySSeC6E8j_aG6e-fq z03-WWBj&&^SdQ0;&w7`WCN#j1Wzw5|;05*d{K9*R>U-7hGz*PIVrL4{`1z5i62F8f zr$XAdqx4@jC@3X)h}+2!5MkWR@lf%V1C#=INMMkULWRBEDdDcRZU(*bq1I-b+4lFLe%G@3=>`x`VH)^@H{W;Z;5E3lk6w!6mDL)K2^~(N8_m5QY-gu*|_mMFI$1Ya*%2q!1*7lE3Yq zJZb4Y;C=s#m8V=Wmx;Fo@Y38>!g9ZZB!Bu#BNHa_>;jM7yAe7K!3XmA_g7 z1=PFUo0~c!Wie|kFFhz-kq$yC|9;%f=f*!XUHni#IQ=(KuV3!6-TBuc@>91ecBHNOJ~ETfvHcmk+Q6xZ6ziD$86eWln+N4q-0KE=nZ6~A7#zKOGhsdLx28U4 zkt9|oB2k;p!67mllRf48$l132HIX}rJ@xD;adkI_rIyj?-~dU1_R@8Cy+P+te>D^M#uzm-Xv+fe$obs+OgZw@aAt(} zHn5*cra?WCmP57pDV;gv4G6?Op)M5CLfBKYT2r#9-o4*ozbfC`Y`Y{4G&M|w*ygPU z)TpOwv}4FO%A=-aQP$N`l_5;*c~> zyolzoer%gZ4v@XgjFk}8fk3U2ZEyo})^02`V`c-AO@>9b&l>SX?b(w;7BS)%Qxl5x z$NTh1=$hRR!<1Eozl*8w?`b@v{(;SAVDqJCEjg;5PGK44z!Gh3OKetmu!Jf;`QcS+ z!;zZt@}hW7Q=pOb@&`=XddqB~i2;9UP!$XJGc%S_oYpVk?5Z@F{h_5+YOC>n(hiqC-ndq_ZVemO*lcx#LUxOo(@aeW13< zcQ%;9cJSuyW2Y}BrJjRN6%;(z_8BKRxNNUcU=2l|Sye>U40mos6qfEt>ip{y>&T0L zt^5Dhm;ctR%YUtr|9qO5?dmApxo+};q^FeJX`gE8EONz-Z6B>(xtS8%3KmIV_wcYi z9%4p4;zVVa#jaRep?cSW%8U5D<%0I`n5>odT+Q!n9X0|oN6z3_TKgBUlXwy^?FGeJ zHUbD}$d1d}r(Q2Vq4?W<)cuwb-6f-(-8%H_w5tZ3^I9Olt`Hedc>#|jqt~%>60O&Pen&brTc;y%{av%7i z*G;ZekNzfD!&0ca3!k230#@3wZr}D|Q{2(;T|l^N9{S0*S8Gs zvbl2q&UiH5iq zv1D#cg>x?EkWOAT5sgKmDjBEVw!BrVj1n9qWSP5DmBwi4b9F|{I>qX(3! zVb`>}(K>BRI z*{$b0TaS_R_3hx?LuuojOfNJ0om}N}uSbM}eP|h^UCR#%#jpGh_$ujvt&}jXcDd}1 zcB_@0&Isag$NLp4}o?g)M^&9hahb6PEjx ztwmE7T=64^wF zJ>;&&TUu{@lL?0pzga0$LA>wHRA@fG}O5)4or;nYtBCPRM>|o5TBy$D~@ci-Oc#U|Q(G8d{xUIgGkoKF~$l~h# zj@*+@%VO4#%eG(Ch4RheT_M~)2)CUIOJVUsZD_lk%U)&kPIUOIw^^ch?PX}Df_#Wi zE-VC#f9RYk`>oarVag#x57-(rGe!$*B_yv%oVVGDu3=9f`pop^4L#8D9#Gw%FEns2 zD%KSSXsj?e>~XQ_2P&IbgkEW-YOROzW(^<%gE2KdH7uH9ZJX%KpZ1)kE4!+#J%O2r z*J{HFpOdb%0C_S9>O#J$v9D@r6AfbyM7_1&2*T5((KzB8qJ%46{($Az>*WJ?Z`?T; z@vin1K&FswMyGKYsq6G9hix0O%Gd3lK9h*Cp!7Jx)&Ae5Jjb*TE-oFj$6tt%MehKG zbX&=aiLn@f2i<08)e?)E)19@66;Rgvbs$p~d`wqih}nD6ad-i9G&lq?m3_s3KwM^> z%+W1OS~+YnAhTDzuQb++S{{2!tY=QZ&C|NOX~2^50wuj73%LdRSUi*i7e}P9OFj3g z)m?kCfM12XHQgsngrfN_S9KYIEi^2cE0+1%33@DgEVNSwr7%ZlkFPof!G1bjenN20 zsOfDmlA-_u6p2^>m$;9Ny%TV$9liSDnjViMe}Yg5u{Z+yvud?!E8=CRTdW8N$EDae zrnguK-yK&|auF*R9NB2H0#^9MTP_A}BZeA(XZxZnkd(*WrO$H4CYy1#x~ z5$sR0oyWc00XV~+1sxAcV-E6JLh$$D#pd+$)Fs*^FZQK6W% zUsYsQ=#<|RpdH%oMj*Mcgj83dg~Dalxcc2I5i&#jg-LT-mi0~ z1Ge1xzIcV7mL42W_ce~!68j}N`X2ZVXLq{>=R}rVw~|7+wW!IwKkVhEx0)W=$*|2P z^aN2XV4hHM?{sq$E{>ux+DegdP(5`)|0;2(OSWG+-8tn{J1P_eUnu=7Q|26MM^lhz zVeoyMmHZ%P`qPuPB^gqg`TAMebHI>i=WO}3cBt+Mx`>pV|83nBITm1sR1DsCC#r93 zTx#AGgv)SMS!ne1tCAJ5QAwH#2C1U9!`T-A;git~XC9n&Mw~H%YKWRa&0wr0`GRzw zRgvzVb-<;f$#uOu)eHwag_}mF+~!E%3R#>j!fuS*5f1W-@`zupg}H&=y&75bg<^DF zW_*(CnuuM-ni)$q@Xjy~2R13+YGKaDLEyAKj;_=aH)imGBCqV4mwtq!k(&wRTjT3( zH}QfQJYvVretd?^M-H-F*Z%V^*+ zN(rBgGCp@|B2jh#&TZF^azz6yY3@&SEw0*Nh*080>lPQ$1&+p{&>dBTAu%Z6hEG*8)t$&Knr(Q zIavtPosr}#Pw;CHgnX*{Vf529b4PFRBbh4l`D)Y^qW1?crhsCKlAVc(>naNNK-noO z@(j;RtaAm_`Q$=T9Sqv5a1rw5S}T6ET*YQqJS>w6k~f=M-q^n=iw>jpSM1keDmpc~ zFVbPa?`$87AJoUDYkKxbdcShq5-=lMT3R&9%C<~gG)jH}>F<@f+eZA57!>iXLbhN^ zwM4GEbwO$|YoCyBAeNMy;9LNO1b8&?&rnpF^%*y#CvCM%;g1%V7S*mgKPq#!oz;B} z1yK>g`G8FT$$`h67;`U?dWPBEJ0OD>v(a#xoEpfek((18C)6c#aDZ@mD^ED0J^Rx& z^Ditc%@gyrExY=92&MqQbOs-0@x#_{tBUzes6a?^BUH!EIA%f0%V~02w@k5u5gb{y zz@_og8Mgeux7;@DQrdP6V3vI8OBidY6(BaHYp~i|AS25z8SZvHZFA~7OR zV-z=KEIJVgD)Bu7 z;#=w?mpV$UA}P|I3wE+jNbhY0RTxmU(r!XX#mQ`OQruKrh*F)YX})M}F01~fO3pe2 z9QbHqE%jrZ3a*^JI%h->@I1*-DVQ4_n!H4uf_4@wB3_b_AAVy-c%#fm%tTqBa&JFc zLb87+5CITgTjT(=z`=+r??;l0h?H%xYJ)29S=@UgksLxxIWLfLaYxG8?JgZrM(wgE z%ZdA2E2Xx5^-|(D6UYnYXCjPL?2@Zr)Tgo;_RxHZyAE^A|LW?YLF z@i=PA!N5U@pXkXks?7+?C}k+HPCuzS()IpxqRQ2- zZU&LtQaSHM?^ZPMn(TrsY(5Nxr8Z<#ttxyW0x$Y>#C=jyU6M>*IGT}_0mPb&#;43? zOBTh|>3iXeigB^2H>IIvi3^#w@#7FNBHUR(XT0XDN_rLy=~|ulp;yGChBB02V^~nx z85~B#gwD^J`#7Jgqpfr;`A-%nA0)f?)m3yhlsc$hVo=@g`!V6%Pz}npK|6wI3aoW- zAXk~(hRf`?~BWMO>ax?+j5*r4v;Y&}YCTG|z}VwaQS z`r5AphRJd9R^XBTG;q0%?m5^IKI=Fnst=8khgod{7za zS0{dT#TzJcN0`)JOG}xTf6pIG@jTTJ78gGrNPBv%D~bgk>%UgJw}erco+IP)ln6v9 zuc&b{2t)JX;^nvYB;>hlZPXY|B`bQRmYy+@O>vAtTmW=6SnAecj`q^lfuR}1S=;s# zJ9kSv!aDfL15m|y9In{qxwdgC&?La*+1$pez#E-6Y)u8xN=(w_v~3$QTZF%O`~vr>NMhVP z#@&!|ieFM7AAs=axi;miWy6wk>Wh97Fcf*V!LDcN-n)LpQi>c2W~RkC4K;CvN&7=h zcEVe3wlCB>$zz|S_ug-E0JEXGTK3QqS4pw(A}c(_B2%XB43bePcqJN1q6~kg<&534 z{m#~g_i#?q?#z`C@i#u$_s?Nr-LdO24H;)0z*8I%NYM-K^UJJwScJ{10E_*5c52|8O{`TlKhKB?B&bfZq$h@lY~o{Mrnr%l`|s-d8DJz<_xO{Y1Nf#9*v_`Z zrVf|q<%;XVC?@!cBYQGa)i+F>#2u;H!8X(aSqPZIYh*(`W&V@6C0Q1pz1&b9G; zz%#?Tg@eNn!cb{{`lAIu!g@}CJkD(hPXvuKa#L+xyveUOPEXEgnrG+4RbH1v~9E^GzF% z6@kPv^M>*6xlYye-?}>j6n|pJWA*xI_yKT$n{J#hdE=IfQ=sGipCwyF9Zs zewDZP#{he1a^go!{J!||#V@?#eEeg${D|1mk3YuX|5o4t{+|>B!!NFOOxCm};g#IV z(p-w4xbf#~eoSEEBfY_N;mZBLxqThk*e*;I@f0yuUhPed%6iDg_JjDva&i2V33lF` zWfvs<15+(E5C;8bT2;&ckc|uYY&fj{@~|n)VA4X&TQFE0L;jVZ z!LZaHcNZp`8S);Ub88}>!I27?kd_7Sb^ZDA*zQcI;s(0A8C%f_3vVcDP?}kn@h-)b zLiDJH3>GewgE`rs5lBWoDS!C}`zn3=E#&KkJ9hJRk6NtSLgu*Eu1OP2CrTm2$| z$_(vG>2xo64R?I?DmtbxT7zQ|=9)nuZA0w?&85!`ZBE8>JF{BtLuW1tlgH_I+IPt& zX~u^OJ+P`y!j1F91u=3%iLfFnP6_Ykm5LovT-pSKo zD!KcO+%rppTkeIR_B%D7hPeuA%q!5j>daC`pu8br4Z37L%#wbsfgfiN*a@WuU4Hi= z%HTYIzp7Mi>Ef(gX8Im|Od_@3hG27g#*GJ8X{r1^#1NPhhc=A3T_*vmbINzgo|(1X zm`K06(q2)oC~Gte4+4j6*Bv+BX(BCj#iysf+?Ic-+Z(fR8A~c~Nq&2HsvMdW{}qKa zjHQ75kvE;+mfJ1y@|RX0AZvCUCHt+s=KUd7}8SHs0>-7{CCXiKZyz*9lO@7r8mOE>c$e^?p=L<0CD)7Hsu}9+C<3ywkWh$qHye8Klx43 zT1viccipue8a>0cILyq?t{shd7SG6d{wuubRax}&B9u+pbBoLzmKHdRyU~3zj9FGn zB2_wP7?Gr+G!>S3ISr(t(6n<5zvu#xaB<*-?ZFWK_(d#@5x?%O#EXDOn95lTaqEEi zc3bcVBgy_IbMiGMLoAOa_o z(wD6wcOU2Z6*(`B=xN8jIXpRFE^i|eI7?gZ8V)i#_U5dam{>I`?X~ctWepH&kAjX9 z`>|K`o~`nF`yant*KKZABq$ekyb;j44~4QM2wP)f;%Fhba{I4tg^v%SaX!IH)ee%L zS!wqery8W3+6~F0^c?+q))xG#Q*+#!RV~w}W+d=UnE|Q#y9Oe4s1B%YU95A{ zaHG zTr&dmCM%Cyj4cP%*=lhGZU@NHE1N}zXdr}+j?Q=trdZkOxYC=bAXci_CHgztnaz%- z1$oa=g&h@I2<24fpMYC<*cR*4h*(*2bVS`xz2HqV0>`?O0C1Pt3e*6 zJ-Fq~vGFyx&=Ig#LDi?})J#^Z;BU- zCwivVu#&UIhElRtMdLhbUe&xrGy617m*twxvL%PC)h*o>w3s&C&e_g7_&eJJg;?(@ z|FKt;hKtZH-T8ua3o5!y)iuXx^H`->-Np!^@tZwsEd<6-yj^4#%WTZ8&Acox{k#HP z^>&DNd_^@MSq$-O-6KfVzHb1Ujk1D?$k%JW!R?LI4qn2>>xp7QD(h2%Bw+S zZcdFw%uG&5@O~%q>!ambbP)>J7rOW|1Fmka3Yz)jIboWB&ON7f=XKxh!OM@MbewC8 zA`#YGL04pIG$=iNWy{aq45BVR>b@eKI zjECONaw~#3+kNQMX@5y+m8!RwfAy5-2SDTVuYRX|dE@6VPjijS^LE|3dsOgoX7l8i z6Dr3*@y%PmX8rWbW$gO>hR0`bf+yD_j!LlI{IiB9FD&lYJKQ|^`~Oz(zcKjl=?AjM z6tK2A@~@uhDwZ>sK;-0k_GM9`%FhPp8hM?_9q0@Z^Vo84Q1O+kH0qH{Rp9 z_Wl$lmL~IWBUBR;em_>JAJXYOaer#5-!U*`JM-#?dwlVqu7@B0pyK=!qw60O{eOq* zUi^9HG-kpXby`e?qiMss%^~lyP?{)l_a4O zM#tEr>ZGQ0=07Zn?0a>Wemh3Y%>E>Q^P1ZYHtpM_xFL=QNqvwFp18`UGF?h$q0=n$ zJ6p1>hLO85ExtFK*zpKw$aYHb+VrR21MUgL3hI9RM}#4thkwLjpZbjV3)Yoh7ztI9%5q~O9Z5s1;k;G^YsZ3my5|HpjSh zX(SUrWq=Tfce9%En@$RPM&}M2!o`e9Fm}_JXkE|E_0y?$#?z`LD) zIoFLBE;}FuS5CZRlg@aCe8JQ#vNpuw1`AVVP&gbDd}YkxN%n`!a`dofSQ1yJUtu#3JR|)MO}j1 zxT2|ouvnHF>2%~Hy2Tjz?DR%>3cG0y$Q|HuePv3F`kXi{f-~GA%O2F?z0m0rc5o-~|CbDvg(y$dW zaBaX0sOWH$o&tMtVI?{aL2W37^3$m8kuT`dR}y>lcUoGo$Um6rOBOM1N%`V*&Jh`; z@$inE2fnX(CTc9G^5M=w<2k-hvenLuB@z|hoY~yR;!qpqHGqC1Tx8m6c*Zs>g#|jw z7vZ3|&DduO47YS%)BODV?}Pr1wL3)4ee8W^o6cwuYT>WZCx@41tdEiL#|oI zaO>D|#V~U`KIu`7EcHbE?Volwgm=GvaC$Q7Z04!vRHyJ@t*Wdm2p{mkIa}Ofwi)2I zp1^3iDijQe&Er-gCUbYNddshSAXmQVJBK294O8DZM@)#B7e<#{UAdE_5KfPoh8U6a zN=YDL-%jy4G*|}$1K^$6i{3cL%n_nc{FPuq$z2it3KfBjG6nI7%}wG%*h#*-MVsDX z>hU~~l_NjhX%E_ti zNq9QHRc_`*&SaaICaVb=KcUZG5dkvKTiQOC{6L2*^N?C-T_IOyhW_b&s zg=#mX_e;HOIKJ^WX9G*@u3x@ibzp%W;hqm<0{wE%54&R6FHEvYh6cb_1_2-O0f|u_ zGXcn+E$3t?v5Vp_CACj^_vBGg>ZAVlRByFksrIiMB;K)Dzm}rpwH&o)YBMe-27J`? zx-cl5LKyQXoo!1$E92`wa^${%llOJY+R)87Y4cD+T(5Z-1hR2dPZ+RfHU4#IPsH-z zbR@cPz0Uns%2ii{NH)I;P*`4p5W=-eh{2GiuglkZe>QA9JYOB+K60EPzn3zRBnyl4 zSVRU6fcgw6^ng##jSHS;<^TkaBAFGFxy$C8*Is={HB4LYBjNmc*2Ua?qz z@5#E}(NfY#DeTKBEIqhlR9;a-F2Ey(Y=Q~G z1Suyu-LJ=M2bFu=hviQAy7E=+WpqoI*2KY(0fARnVXj@vmGhnE!3kk**D_Z8Kb<6d zf$dtb*dbrTO9lp_aq}OUJjkx`KY$Tjpc_Ce=#x+-Eyq6FlQt>&skQ}|(%bu1W{LM8 z9L|~{jo#oGuN$PRoZ&LOVHe+>&aMmJg4|f{RhrW=34pBSPvZ=e7aw}&oZ`z^iQ93u zfQQufjV_J^5@N*($@W{$PzngRb~%so2p10Jvu?rKT-5wlZQ+GFWC_D4l_pZcdrS$=o*7pD%>HNfYwEjLv9lY}maO{nGyt(p2 zVD;5c7b@$IMH&H@&g+=o0|WN*qpVXz{oI&gFPXZtaq%>Nx>-m()toYdJP@S}ypMNj z462I}#=MLhozv(lcX28hEGGa8YCwLQ*KT+IwE2vs`~5ZZnDZfL^o3X{uac`hnrbsG z*c_PgK=n~8hg?lxTU4-GZb-vjtW4EZ{4wC-d#jOWq+%H7?G_5|tDL2pe=@Is2A(m_ z$;qLpU4`=$z|618kr}1#S?R!-T`3Pc%Fb^jZ6+eHPWNYhP~&;$%fwLs`aN`gQDr4u@#7dz5H0s#}6Qd1yE z2@pb2>Ai*;dhgN^P=DF`?7h$Vz4zR6&$;)Eao=~md;P(hW39PX7IV!t=QHPgp6`c- z$iOE5eu(AbAt7IAAu?@11>SBnMEi?@|tx%xs_y(Qmsmr?kXg-M$>& z_RHz>t~~D7u)h!ZH~x@nmVQR7GRHQ3M^8&A=*zD11~5 z#X6-<*Bh+ntNx1fblzam(4I|J zBX;93xjeW$ZQXmQqkAK#8S5r);>r>wkEork%dnPb9ze z6Ej)+FgYZ=HjrEwjb02Qw;(8Bl9Uk*7OP5R?tsQgwYT61=CdjK28OdP*a;DxPii5r z#fH6=S^?5!#X;We8YX&f8JZPjTMGw*`MCkxVmoqo;+xMfm>C#E zVSyTkisTbC%ehIJ#?sXUiv+1A>% z+|vm4VK<4qIj?gBXK*pixM&bGH8B8PU<6qMS@yW=*ane4jup5Ju3xq`fA6A09JIV- zk*zvOWikyzJOJT=f!#FGT~FGuWk|R<5Sb5h4Y2q@H^Ta&hJH17l`+YtJl&Da`09g? z!o3-SdRTEUPBadg;{0|0GD;!+avUolfYdDQw zDyM5pV=1y~7D{O&JrDNJAaiF0WIpwLi616(;7X-F%`>b7Y`9j@X7}rnrD=7&^-NF- zxEUqUp^Y44pGEL28SXeAlX^ra`A2X4-%6a;sCj8f>T#RsJNF+W?dTjVm}(A(3Jbow zW&NNFs|{xFbT*T$a5?w&nXqVYcP-1c>Q7!c&0G>8NE;5;N1Kw!zLGPkwlO^qU3-cL zpA5Q??>}b|5f4jgwcxc5er523?h6blM?wZL1gr;@TT zFOBMOOT9eKHgINSW9>@C)(ncgn zzNv5hXZlRrWRt-arw|*R76D#(H?6ZxZL#k}_{NK%L(<<43xrNQ{_a52Oj}3S5Oel{ zET2msN9<}S4mTT{+ZTM6)B~Dd`A`ZO;46Upsz*{CRWr8iNPC-7T!?(;(3-B7TtDc> z;7_N1mQma~@o;JaYk9ZTR`Pz^&80dEAt}?>y&mC!0=XJf7^M?6JzTCh1>nBumqgcW z=44jiQKT7W?_A{RL+okTR+EqHi;QPQ;cIvuM{jz&xHbX4;Q616Nj*$=b|%2fvyhJc zx4_Wi-t?y;o9n*O+L@k87w$J7p>B*&QuOy%Df24z&Jd(E*VSyU$Iu7ZD0Qf!?)$vBiDW6*9!_$)7HMKJ>KuoS{YD{ zk*e5bhj-SbnT7KyzBz{cZ+&@xbJ4#(&PWusRf{?}quKw9EgmY$?-=6c^sC{P`l|&l zta~>U^h*X7p-~cyRV*iiiH;{CH4cEYn-`Z)tsOny#I*#V#F_o-*0B&<%M7Dzf->+dVB28Q~$fA;o;475KA-GFP9=Kh>9J^ z=5fz0rvLeH_it+C-z0B)G}k^`m=a_x5C8Zi%jC`}<;@xmt$L2%XHI?n23`7;nlIv?o2g5-9wY%XqHQ z_9-v!XvpuFT2W0L<%#p3r~fRLe|vRcb|gp`RaviP#@z|$vB}N|SqvN2Rg}?Sifi|T zZ_QX(Ex~SZKKr%Ka|wK>*PW*sMGpY~v8w;gSMhI29-F7$IrAW>rts3;t+i{fZalFZ zd=_xF?Ku3`yZ=2`!O^UI9!FvAi>tD?9h?3w&+s2^a-7wu%zzkHM1CKhrp73YqvUeP)pJ>*3$BUcvr)nEjU=_Xkh@dEx)J3+Fcc*B3ci`&?5jSS>QZL1n(ruPi6@ z-IUDn^uMR+^lP21Y)p4M`q^?*sb2BqoWkML$#b-YY? zBMi6~cuK|~N{epWVm(Rh;T)VZ#yDIC4QNfde97TN$A>91Rls|1m&9d|U#4?RIP3=C zs)EDrp#Ux5$}V+#{n*0G|1uAE*l%CJrEBM=74F2W#*|OuSBo!?j0nH;Hpdmoy+x=V z3J6bwbu`d+s5#X0mH+U?{3{0@%ikSgVH-*X(#_Rio{zJ}0%<-wjJvLPSTP6JN~#yo-@W8K?YxfZ6?^PgsdatJ z{B57zq22Mk8P#=Csl9P(=bG|v%y0W7St+^G0`_;W9Lija>R>QEZ&`Q93DA=PSy0gtvGbov z1z6C&TRta-%BIuTe{D1Vp0!b5@Jk1O*7V?)jy;v;4@D6s?r@7PzVG<8+bZdO^Jnk= z|5wwe!N_;<+b{NACd`zeTE!ycr6}3XaHo`K3PL--oDw1bT8j`sF7qtUJrT zUoQB1%_{B17I$#XmyoAThgUA$rK3Bl8_A(7cdr~~Jrn(I*bZe@bigK3uKSP^pX$vQ z4u;Ixk}Q+?th}M_btpT%_TmYD)g>s55^cumf`e5*dE;Mc_z0*M9!iTC?p^{)28yr}sOrUY@uyyF5)JwPIpB=0)OJK}-M< z8XcVv4wwnUk>h-Mm{?(r*3$@GvUFJ@%?G6 z5*-0>fMXv%ZI=jF8If0*jC|%X_EC&%niddw64BgH>z`R*|5fLtc7T4DEOsRgx67-q zm{^*lj>Dj(9-D~_Lpr|v1UzCdO?_3^-?9u_0<88!E#1)kFJ+-aR<_zEieSzGL7)Dd zz)hM~LBXqzfy|MobVrt+?>=AI=b5)^_+sYaQvE_&<3w(Z>INe7ZM24Ik;cV6w{}Ue zd55`?_L`RKfcyM#i1amWf8$SOmp>I%3nN8m#lFZ*&;qSIkr};s-L&OYj>K>NZc353 zk+HLxgXu2eZWmn7n895g&9mVTso~ib{`>WwnYZA}<_dGK@`uXKz*a6PcI3XYWAq2 zPF=H|*~(EivTJNz5ErUM5j6-X+)?b0kIKr*#M$z-M>owi?ZEed-2hq6v7lj$bGVyd zfWn1Ca&Fck#f1b*^ir8QijAP)Gu2pYGI@^;*&f;t=DzHq?wTm;+{^Ce!QuwaUvlZW{V>E8t*Kzmh(s1*D7(22M zjyr524G);vRyTa=&-!){dR2O;=t$yIB;)jm9fxcC2c5J<=u!&^7i^lB2REH38V6LU z37hr{%Heq6sYiwfd7bI?sv7 zg!)PLyA*w5D%9nSi2w{V-TPS@DcL9jsfa{smzHXzEm>F|=MjXIw$8S-si+?a zUj8!II35_&u&?!qm8ww;@Pp(8fkgWOj{G6{9UAX)AaZJT)D*0MdJN2-nla}}E<4-C zroq!P6DzW*Mq`DeoX#33Upu8GDrF5bEIXsj zt{P6_XhNfcOs-v9Q3|!+k6e2&nat8B!Stb;wW8TrZOE>$)6~CbRiE#oHhRszA>}I# zkBYVf@YyqSD*7acd|8#2ZQ$f|OVmN>Mj_pd2EuRv?BW_yW55ggEz<=He@rQWz0gqK-~ur zpG%-gHaT8X%&g3Hv^@Un1q_`ntqK2^H*Rj3atqOi2D9MM)zykI>)0c7+?r}86&H@4 z!=|a@R*JPZTdSR_QdR`m9M)uO~=F)x~V=e$8duG5lT3&ZY$? zQ?Q<}=_j_R83=^Fkk#9!&_kpvrz;gMAy6BQ#(!) zy5qX`hH6(@xSA+pUebKC{~FC<8kQOT?M;=WcE)QJP*49_a}y5jzc+_NWrB@u5O?YR zF!EflO>|)ckFLrzf8!IaJz_sb{g=z_U%ue?fLHO~9qyk!u_%ela^Z7)Jv!{^-uvcX zzT=#w!VUMp*C?cmm{`KTv zQWsOaf@pegl~y?cQsYT=G|lY)&UPY{Gv)Th7+w$FFzV?g=J+o*cZ9bSD)oPl<%$0A z%U#7K=0|?eoiRR&E!4Z`AFDLJf4^oXIO6m>x)UZYhcDN-C$EL9`uScNFc(_sGrsNW zX)t0NBi%2jXlj)Vbu(72NRlqooKO7LKJPQzxBs_7*5A-cW&4tI!I@`Y(yE-U+vyL@ID6p<;L|iP< zcrpTyZgt2>)L`Fhzi^H^!n3HlWGoIxb40PsIJQT9F{_n7)xh>*D@5E6&0#y+({dyM zhMTxt>1pP*`1=02S+b4!jZw|nHMp{eVl*TsrBYj$BI$y& z1JX9-_uK>8`Qh2xSqS?zV?})iDea;jYEci$<+2${Xp{a9b?|dr*2rY$mkZVx!IWByK*vJR)3*DhZuXZOy+KF#dG+Nz~R<$-=JeQ6Bm0x>9NFTZ`UT zH)pFq*_C~ADSlvBnuv}#Wy`4#lvK^|vfbM2d~g;;Lu&7~lrv8%Mp!RMT%@mmJJs$G z;lIRmN-fCvA#~9?kQFRhEaNzUCDp*ul=Z99t@}40N2$23juh(#ZH>8n5L0WY*|wS#(;B=r{G=!TxCgp=TVDSAy-Y{J?d8u`!KELt zN*dOcO|r|op3V8Zp(UTcy*oF;!Al(jS~)wi0|+LtX4%bFrHy>|vY*;cgx;2|F9cwO zJuaTLflC;yN|_t1Nxt7xsjbz*r&2?H>3K4jw^7H z#MOxwgORh&zn`EHjvR;9uD$5A&z*9d9@`@a&OHrYw>V7N6;KZgt)_j@zqfB8bNBShSz0}Vfj^(z7AjQ#y{g}AQp`|rJ-$XSYfBMHoK|0XyOq6nh zzf`d`dn*zhx7?PWrJg8g-yfx#+XB>ZD|nvitl4Jr8GmJT7VdxXa?vLjYbr@1UK=l* znyx8hW+ZrKd2~22V<%MF(XZywKIisIDUV9U`G*BgmQ2aI#FqG|0?~3B{L`F+I{=B6mG~hZUJ@zg z_0N$$)8=YX%kG4WpLX63Z@5MysJGFWMaM<&92e{C0Dv$>W|1~b5;J%`aO!lid4Bim z@w5$kaCPp*v6Wc$wM6oZoC+5mW8iQ+{Ow!@PTp5?uL%5UdXhh&+JG|lp|ss~Kk`hk zl4|}WQUCKIy50yr`Mik`CnGMSYL;|M%RuHGaJ13&t!u1J)bN#yX{DVaE)O4vq}MbH zr`Knh!l^S)J;|HeNZ+@!r^eTPTHDdd>?@m-FIUSV61#h#>E>KUj#A%V+Nx%V_^;0& z(ug*^@R#%2dL`}e%>2|!2V6gUmjQ096y>qJ9=jd^DCPUgZ|_xMT2tS%w%inaUilW~ ziqU4DVLrd|v{P2Dab~|Dz%IWpIf;K14WALXp-nGbdcnoh|~`Ce7!sq?z&*7OwT+V1-zCr5BEMNT755DoRbfBGo^nyhW`_* z7Rx1)dYpm9Y?>^eu`sx8T4*FWv+oy9(GIK7?z&gm6!~Ns(}dP*HsbgaBVxo8gO9NF zBJ5s3OWPI7jue5w=HIjsS*7Er3LAlm>e7?NfPfJrqA_X+z7iY=14m`6_GUEW6pEhY z5H!+dWS{?Y%C+|(N|6At#f=+I3mBLa7Q5^OUewYNX?Koa(9!FnKxq;#1e2MRnGz`- zeUP3gNUq07NxDMQJ3!IOGY2-RLGm;o%2iTDeVa?$E1}qt3r#1niTdQYt+IdA(^dKgK_@y1h4d(&I z*+p^~YWvH|B0bw>(q4+t|h&&-5?1zm4i`9+K z*m~Si60hO6c-O0`E&m;Z==W>>Xd(YW$mQ>O|7ZJu98&rB=qEwrwE{R8%$cLEozE1P z3+kG64a{6ndZXkt`F_>;DveE*z7lotpl|p7Eh*Rs^Ws+_oB141p9tqND?1LY>H9-_(fpA#01pv(Ts*7E+X&6it$(J6nn0kf-L6dA5~#&f^r2UskOT zIpn8qF)wBXv-#*E9vncn!`QVi4I2IcQ#w#~ung%Hy(Jw^=syrPr=c`Mc&+$NrB&za z_q4Fgn3?h1ln#l+?k_og(t=?GV>#h(^sm!$*Z{mSH!|_Yd~w3yuz6XM-P}BSi?<@m z@>SW_iJ{PMxI0HGb{{8|zgv4;Yt;Ec7arz&_U4#WZL-{;PU#Oi4d$L8d5sgdbF>FX zA^AZSOa?qe)RM2if9%Ei&(kMI5YZ+&-gxJYM!uLArvQe_u36R;O9Y9*HxukRUUN$# z2=qPV!P6nfnhj^gK(3s)%1d+6*X=@0mHo8ITllaB{_rI`tp zeBEkN>24teXVCz=upM}Ee9?FDz?!%ZBJy`XOeyfZMI)+j6??cMG{ea;LL4h`3zaRX zCwY79@sIlhWXo@Y3%^LT=j9dzEnrtD%&Q=a64kOfWv$_5+|)$6PYf$LJf5*sC{D36 zJbEa|v^(RSn~9(Kqh0Ww$g;8AGD%(00Re-uaM0q|HnjRyCx$95KTDIlHK48{Me^QZ z?aFhaeP!>=U}!QV>okQ7C9lt3$p*L~a5crPc3%l9W#|e6+IgYVIfl?xG#w1vd^i}~ z>N#hh!hX|b`S?W3vh5hm|$HKw67bl0f z=+}04taKP>`8?*LRbYE5^YVNayISMTh7B1La0hgI19W{))`K&;|6nj}Otk0+T@e=B zoE{8~s8UP%{Jb@^w4k_B&49AvtqKSGd_jEv|ha;}) zage#1I~x7GW5(DNGoGR1R3gm58&8I*#pY^!Ea?)vVq- zM}->>@sOY>obP5kQ?gW*64hnDcfG$mbP;cEZucG+M;w0mL2(7-B;oT$}fb^kY%C{Jw5&KK|%{wRHq`95H1Nt>O*NtdX+^Mo7m@* zZylYFL?ysn)v7(Fc3CJV78%~F>>?KsYW8g?jT)tm5xU21PYat|n_pt{%C|1{>BgF1 zVWveHMAeDHoRHO(MH7Nv9AXL59ARrk>77!c{7y4S$K-EhJ0DW zrZ)ETUW4XXb9aS(Qw$is30X34ip-a6mYkC?PkWlZCP{m{wD_g9pCB9C0^C)~;UbX3 zd<8@0lg<|~Hs18AP5mScCPoFUVt%-X-BO(9)PMAZc|T&-FCTH<94HL80+3}yMbx96 z3{$~I2k*W*y?H>r?L;Y}$(QD-YMtj(F)}ezsdrsVXa>u5pS>ng{K4(4f@tIhA4JBG z>P+0iNvD?GHsi1Z@7{=rfg%H&NcL?JxWN5fr(N3YxyYc!v=+1lU-~KsZ{utlM4h$b zAz8$L+L@H*_nREo*spAF7d{qFI1^{Vp^z5?2?Sa>`X&k5CaGpGO6oGdHOSa1j^I>2 z50TG~u`?1zt)LrcVq2$tLW6V+`dm2#9nG{#RtTTC#=f+KKSKEQ76DX=J+dtkE;u_5qTYrLb^-()IJD9NO1NU(E>9w#J(+})5WYsv^y@}UbgdLW9G~B zmJ(wXkQyYmR@;f!4N}9u5A6i-(8lwUDsw7Q63P}XhwNgfhHU4Ggl<=~BzgjDLCt=@ z(3A5yu;YAPhHWyPMUTBbrKOJZh@pqXC%p*Qfx#Hw1Xbf80>Tj9E<#6O6W+dYp1CyThb!rH ztF!@Ev6D)vf}5U0VMj9~yAdoaPLOTe*DI9-)OkKtdCz({k#}fyX&tzW zel1gt#`y}5QLND@I~*=7|H(EOio}E&Pk0N@VjBa-=4ZjDOp_GK+UyV62caCIxPl7G zC|}h$ek23d%$~1GG{-ifaI}pFkMx~MeltL&|5hA(XZ1mF@x_heeHG1pAF1$hj!}*t zwoSM(zux#mCr+O9vnDUFaPzE`1w5UmfQ`W!@k`)jrxk2 z7RbhpD*3U-cScex}(pzr4ykSFmeGc?M`;sD^g$2lpd zg9SU|#$jcXN7uxzV#|Q+#d11&(JSq=aF21xei9BhOM)|R#g%RN42j&<`l;KhirF_% z6dD~PyP9WB0n(5LYO*8&7ux;AW|e=2n!9RW+iPzqup+VA0B2Aerob)x&ZNMso)L+m zlcPTT5jR7YOf|mR#3I7`DDvW%&Yc^hKrnXyBJ|z<;$L>hmS~y&a_-M8f0oPtn=1p;*Z>X4sXlgy#H2c* zS*E_zUjimjnBfvTVjxv97i&X4(4wMhPJ+rGfV^vR?&(!DEkpAVpK$E~0kqtq$QmqA zQ0cUWb~>ZxfY7v63g@>JcZVU$(ha2Whg4hoH7}L5r096r!78DYkppzF;FDUPD8u^+ z#&~n)v1o1tzhbbwV61_CdLH|>liHoSH|~#Rfnd0&C6DB9Ai74821bz8#J!O0 zl8+yZO&}R*g56aSn8H#)DbqPDNd`Bcq~s9n;ZmwUgt?wk)^kr8bixayqr9XLY z&%hw}=8J2x_@{x^3Uk-nrtRjQAWF;94XdCCY#VOY6^0$_d~2eYTv&Z6yVPC3R%((| z;P#=*;^)e0)AENcO^Xl>pdwp%E}wB`uaAihzC4+G#65cabgv<`v^{JCT)iW5^Pc0o zM08F-E`B&)C7dlrJ0%RCW&*J7L|l6)$2QJbHwWw%_QEsE0N}Ps_o9bNtzX?-r3}bh zuojOHgwwx*MK5^JXtFaiu8!$X#XRy1wO#(bXrobYX)SM;mWXNnds>R0!?_cPo}UM_ z^ztR2gyQUenINHw`*tL8DmqnyyC?%A4$Cl%MNyBQ;R(KA`2mqp4kYtKa(sYo+lxH| z;;A;*TM0pYKJ_4d@C9b1Ye{ulng&CWc7Km7G!M4!$pf%YlE8UfK>C@yEq+`A*te}7 zUp!=-6Z`@h&Ni{{-7nm5qfw(vE9d(qA3KSZEfmC-JW$$Ux?R4en`6~iNCG%&NcM#z zaJK2e+9xip;P${oHjir2iV*@Z@xa7kHKBirKZM7ciYo}D&Kk>Wl1Ol0&8iK?gdyox zZ^g6`jeBa0${Jcs*Ix^4NkpqS=_~Q6+>&mV0Oi^rLWX0m)e-x41M@#`E?4m(d-$|* z*>O^Kp(K5-+$U?q1!(mP1qHq5H|Kyc(v~ZbPWa-(Tq<()(O^i1aOzS_yYb`f znI!geBu2TgoY-5Q9$KTaPz(;mGlx%-0|=}X4`FWZRB!|nBGN{ck2^-76ZvAK1r7Bj zmFp{EO7kr8%=Kk2qTbbtgC)`x3v^^dm1C=RE<1BcoVPuIE(gK8WTBZ)2N&a46%l=;0p z?%bLlxuuo7y+_?{eQL3ug%i4t&qkpAtkGyy5x3W>yl>}lHu*;8Ehyh8Uz&!?=}Gn- zLeXTT4uJ^Q)z?zv^yCqj4tW=V8Zpgm6F%Z#tkAF^D zM7z8i8Tp_e01*B|cie#glb^r)(nI-^&MS;?)32OCNcpO#pKNo7e4*6CudR+cmY-Xs z$#$i6kcoCb9ZA)1hX*lL95d3C;bzyGUg@;?ds-g3&e6z1@aLh2 zv9P%T9$987(456adrz}#RVk{B{haA;ukP?$MjT61oaVSSuQ$)4ryWUFCdVcml09sB zpy6!)ov`(TF2qsmzlL`4!uy6qDP;iRXwXTKy>om429(zG=Aud^%?I4rWN(4lX2V<2n$IP$r;2zV@F9b4qW27@p z8)y#UVaag}@(og!&#&s2I9Z724dDf`dIO1&)Y7RQ- z>hQuElC_N&MivDkCJlDt%vn-KIV_Ltib>7>Wox5|tBH38bs2_%Qv>h{Q>?!&mm!QF znb5tI=VXel*PeBCwcCbPylGy0Xp4P?7x59|sm1b3EiTSx=E9_t3;-}|CQX`$d8#cc zBFpIs{g>=OpO<0NAWgIW8+nOwKj`Wc&ro=sdZ#__%*kAxd2}QWdf`sQ)w;T(<%8vg zrrTx6nl=OrB9sz9(Sx>O{wMRM`c>aAB+q6O9dT3j9%;m_TGxMT=*jWl za4aIfA!lhG?8lz{@#Lqqe#5Z{k5uJeOIkJf=uCGkZ+E27dxRO7!B@G<<+7>1?P;(v zyu)i)OB+2c&)>A9)u+(C!T<7S5DGwhnvSqo}%#cjviwlO}oO}i>9mxe_BM`qs7 z9;sS1bE3=*1PV;76tQRa82k_`5>)a*m=uJ-f+_w;P0=7kwYA=_6|sgdofE$9cA| zSsyp3)BuvoPGHf#XYT%SumOM0&{zzML9vsfErQUP87hjPjz#+>j7`wE ze??WVMx?#<;OHXk-eG-rOiXD-Hp$7lj6_4!&ETuRV6d^+ONOk8f#AXf)AYYYT~ErOe@2j^cFZ?MRwvpn=LcV(&Q12Wq8))lA$fE;LmVni5*wyK87nK?dd z@r1j~XjlYtz}_-XUBXV-81y$$Q|03@5j)xg{NVA9%DLA=rKM5CS+z0W3#k3YWXcisynUX;e_#xtEe8BAI>xy zEv?xH?dI+?>TD4^RBNMN`oL%iw)8N2he|-$gIt2OBm<`@*!6{V2`QbSCFRABVYCg! z5oVFRc|q)nAR^)~e^1lWj)u63Gx9n1qbLO4|V2Bs$2RMw*bz zb$DHo`USr|mj~9JZA#sN%~UH79&gAgaTd!MEhHx%OJ>-_;4JX;Zalqp5*7n^FONuk zg_?Mgo^799An;Rg!A(%{CC`o~Pl2k++=waSQe5Y*EWx5x%zIAqkqEt0VO6wGR^rk3 zHXq}nqDC=ja`;rq^61KFTEGD*I^m2&Rpz6$K{mi*=vT|@vu#eYYRnA|&8t2EaP4m~ z(8Ug~q3W|b=@U+_oZhBTGt~hij73b8uW#?p;M_`8B6PXUW}@azPwCo)5*G5eZ;!}; z$7a3ENNHAsx#;gw%!uVRXyZC;5gPCF9o~>V(H`!y`*p=DS)(Ag)&M|sNY=$wnihC< zPBJhlex*(1j3sX>y*xh4_JqBbMKp2(5O!lkSTu3LHXn-|r554MuCTRC-GZwu?*ww=#Nj{|Hp{SkzcQwr@MGd@NU=G zD_J^^_f=>Cpt} zGp`OV?Sj|~7$gCi%)t@4ce(%KVSG$=Z(q_a{y^R04+TfMqbf&U^Z;@sXf$L5*C2OoaJPd@XdJN4IFa*2-h#9V_tA&bCRrs z^9MiQHtTz#4ni=e+oRqle7Cv^Ek$Bl72>ensEgc<(nlaOex}FxvVp=Bs~+Reh3ff> zD&y$OHh1gXDZwg(%UACKwPSN~O?3e>Rp2Xeaz51BdshHZD6szd#0>T)lCnIW1ZSs8837YPS$oDQ z_5kW7V@v&^NKX1d_IXL*3vAVhJ~M+c3hcRGd_{`RKc>zoug;}Nke?+H?J%5;&j9XHA8(ZPOuTUODYP1pdoII+(MG>IoQyB8X_Jlf}{kqvq%Fp z8AAv<$?`I)pcuNT>wI{a(3DvZ!QR}n=XluBzmH7;RF&I&a{^qfB%rVT?Q5M_qo6! z=bzQN1nQJh30L|M`c#)zqYB4aLeA7QDzIt5j@-9XC?;GsY3gT_?-diYJ~jD*MR*1| zN)Lm7@1tQ}pIIkf@OP^jsCfQ!UYT*Gced!~lM!k%)tKb1>lH3M6c~vzlFn=ka*;w0 z+|xPUG;i1EMvlpmH6rtkA(FuAPeGbQip%e4METTa&vzrFQLh!K(`ty z*WEY$W`r{x5<+5^MGR!CZX|DAk-TfQm!Dy_vS%XNmQ@a2=7GsMtZpxhUx`5FvruNe-It)+HDU(<6}=_EMpdR>yv9l$g&YtoGrd5{8f zyOo3AViP!2Ae#ipq{Y!O(1hsT9DCJFY?ZR)oz5m1Jy892sk~N`kn4Jc^tMm4xNnXz zw|!|(B=#?reV+FakH|;^!4|!u#$t36EM*<+uGe`mduwyt|9q>vPm4=o6+ivhiE>ro zZMPX#Hi>_(tlc^`j*t9Ed1JPdD>;SBUS*p-Sh)!c^& zG5=Cl_rJw67>s8X#H8}slz=bhpk+Qcv`iH#rZ_|3oj;0YgWxT6A#eD`F|4O{yhaCObyNzj97R~@VLx;ORLMIevZOe_C*zcX$`l)0hebx{VlWKMX8 zxBQOmE#Y{68en&Av2ERp*lUr$)eQkP8nlf{s$tF0K@OPgaMjBCf_%)d&I?ny1(&1X z6FumFx8)7O6fm?;ySD;@f70>Q)_G?+CQs75kePF5dr^2Y`*qyIuWb3vLpZ?N==cIO zs(Lvj-?-HVah_9mxLwMGQNbx@+5IaiqX23Ob(=8T8o;f?-FlNk zZzdQsM(ElJTWGY8h^HzJi^X?V$>cV?f;fq?aF>DZxjh$JwCdD$nDsHi(fD^@^W|~=HTLln zz2kzPuE6!JI9wD~Y#E#lyvj452J15djrZB~ZEqOdVbA#_c<`Mq)dEhkdD#h$-EjW1@Mji!Oj8nj$?N8MIF zKgjzq&^0&nkZAKe7O$!BA~SL7EpNoEX8eW|P9_ssV6`JFX~bLjXjzwidYlF_n@33^ z9Jv>{*RtVAW3h~f4ZHsd%$mV8*5?GNU415|ryOY$z0|Isb#*2(DY&9!ZApsr9|EmY zBs-P43JUYH!f)L22ui>2(Ayp7<~LCOUx3HnBDl)QfB-0Rz zv_LIuEa!t^#TwL z?+A%l?;F4zcd?P?n)z^p8|*vwl2uHC{$CHa6rT*V$DC@=ERjLbJ=j@hCr3M13TXQT)i^r_`0W9K-`+SSvqo znIO;UN)CFtOu_{WEvEHQpfh@48Cj?dH{DdrI}&flzMPD` zY`i&K0n#M8Ch7b{2=CMlb)?M(o4q_cER+{h+<5Pt88jm1x#2Be(_bi?or+CnY{>pu z|H3{~tGa}K@@NIDX)+>%McB?6pO%E@LIbuAnPkae5D1osNmG(O^Ee2hM$qY&ZG9dMHYOy*`lGPQDRlCuo*#YMgzQrnDaP*o6|3B z@?$c(LvJ2pEqR3&C#NmqaN?e%nLco*Xmdvuzi?*1?xi=vNkH}Y+6M@qjH}UqzjcpW zG@Kv#(2+VA+*@XS|w_{)s$g z_zC+F0w8(#QaRjGP7^Nm(n^q=97PdSkTrbeeV&@8}%Z_H?*ZGE| z_4hDbh0k(8g{|&^gfSm%TVAojB!af!yc-#cb17pFP{+Bf&v+ZwH>l3j{2YN&I!hF{ zp>b(uT%)oLQOzYYR~=)m2K+#3*=}yZVQ{lxD8)Es%ko*CS@ z>^y{q?>th+c9@Mtx%?L$a?D=D{s#d?OnE!iY^_f`D0OCf!qsuB*wYJ#0=_&s^rfvw z=bi?Q{n_(%CNL|`Fw+nTX7Y5^eOcf5&~7?vNhrS{Gut*tb{e%4i}a($4*~O4=8d4m zU={_ol|BaNPlOsu^^3rIO^Ak3gD^5)eN`K*8x<$j$u~b1`reyl;q1-wwvZo4*mFe} z-L}GnL<}VZ=ChX??IJ3_*LC1m6OuH~-oRm}(-5kQUV6H-hE*F+o4(}B%hc|g4#E}} zQWht_pX}BC%6Mm2zXkK6L7PtU-G6G1{c}!)my-59-LvYcS62s_*5zuQ>R1)D;K9U_ zJvGJrf%sdz-~t#AJ~h!5TFtw3Vry%S*iRX`J{W9!lAY>=-H`u3?7eqbli9j9&Wvpo zP!R#8j#LRvN`h2%q*sB28c^vZ6d@F;js>JM1QQ@M2{j1`5+E3=N-qH^q4(aA4uZ~? znLX~k&vm}vclJ5^?CU$%_nW`+T<^Qyl{YKT`>ge@=eh4YuOLU;&1x2=K}=BJ%hNFU zOnVsf3T-%&^HShfrkK4;Uu*JzW$F@uy7$aPkq%DR+r+LZ>`6k@gamvjJ#ENFL*)YB zCMwBCi7j|>?w~8N&W}+V!9ZPd=>5vH^5j!T@k_|`Qe^Eyghb`|n)I7-IFS>NSuvX+ zx)y{&Byz)Tk(& z-liqTfVc{V4!b?q)jQC6D9ZecBba>yzUPganY% z2Z9Re@tep`@F`o09sA%j7R?7kW6!$DMs4cRHgTbioo_eY4Myi>c_T>%BUuPEtT1{N zW&8#Yn9POSs&;=EP~xU1w)8>l91V^Z<~ByBCXY(n_#zcwoEk^gLLuQW62znyQ0^|l zdl5fdGKVB3t8@fIIUmc~ybYML=^UcWAS$Q1e^){3^~fWJssSrAfju}Ln-0u2OfgMV zkW`=qE@gArDHHhNk#s{!f_=z!O5%IcW=K(kr6)~mq?VWP_Esz=^OTNl!}6jzSW!Wv zCq8O95)*goozOE#T{wTohTqNh%M)&IGG1Ql)4uz1jh*sCF2VOvt<^jOgjj_NCu(T4 z|FkilOPBgg7mkgJ4<3zmFashOl3GiUi!{F9s^lm-R&5hs&A#)j-+3 z(~SW!anh}O2yQcC^u)bIQ%bR^tSzA6)Cw3Z|P`14tfi%OZ8)g2HIDn7%@6gy;rh)(42&#Ahi#N}f^-8O$8=zxsR) zROR>KxxhG|C$W$)&}3F3nwRjK>|7?~yxkIqqL|gT?aXQY`iYnPA8%yJRt_|i*V0Op zG##*omT4wtS)3%Fg3vDA^x{2h%cgvrz7LfB_si<%(`DYu4Y@0}6FpB58a{%pJ-GBx zg{#md3!UKzjopbxs*-Y!MXJe|NUUsddl(_>hfhgI*b35x5WpfR0AgQe_}uTFdhG(B z4Rx;`lnGpG{jBl=0;Z`nk2TN9Fnh%1kKcT}#&fp*0v{sjyR9mip!2!~i%C1XS&nJN zPrGWm5ygQrbTpN}4*}u8r(z324s_KQ%jY-h!=&P@B>jWZ0m-C#UH?QU>%7F+HYZQx z?n?6-95sNU3N|`u%-rJkbpe{j&@y(fU=+R}yY3BH07FBr54t4|Z5Lsd((87kL)_lb zlECf*p4IH60j-+Bmsh&XB3JU#LgIBZtBqh)12-|v$38bpUQ*h_Xstqm4w?HxSNdqs z-~0h$bS+akB~4NeWI4+4OCJEh4X_le3Un&-szyDa>}dF@|#~O zuecKk89Y2Da(#0EdxI+*)QJySPWL8N@>)TR_qWc5(Kl;;`gp;Dtk`IEB@7jH^I$xt zW~?C+Ur1%Qn6=sQ35vwo`EstX4blUORDihKCr!C@iQ_QZQErgh!Qg zC~d+P00E;rscj52UtTQCsm4IUD}+C1a_Pg-b18IvNM-+|a<$ zI{CKJ{ZzOtH_>0L&I~q__7l=oXPBtzug?(0X0NpUp{6tUewu&jNpk2Td3V$h8rvl0yi=+rya-F|HLDYbT0z|Dre-aiIX zIi(+8JQhVnSHun&3-Ial7nV`k&CRI6x!}>cQCX%pzpw~5Ei(PHH0kG)o}4Z~Fr>Q*IV7{5Upu^?n>QWzjZ)(u5$&oo zWd6WVJPrl{e`TWWWNlu7s^yldJ@*g|#ZE@-nZy`J5ieDaJkIl9&^F_r^S3?Ut{%H9 z_PCki5JehLznErTMjwU2JkUGYv#*dkq6ZkSqR4CUhbBg*17HGRAqQCgP(E8nCS~x>_bSo0v7Ct|n{_{Y z$ynXmSuxyT(`Ib~YM?VS&$?LFjZ6<0(wA+&XnBNh+U9T%J*XO7zX)CHUBF_`IHwjU z0}$APP%e?X_dfkp&$$Sa^{Oc{AXMFVEX2PxZ;Lz>d&0Zt$ROd@(AxGTB-a+Cl?>l} zbpLE<{*?aROPTdjQo!!(Gh=+J>u{iuO~>d#<7mb#EZ`ULg=|Cl_jlQCOYAyea4$vf zv?sl<3JtU`3YW>!%)`{2qG45034zIe?|hz!|quV5d1Fa z#5~Mi-L<78;RjmAq{Nn0u_Y1^9PMV&cv-F^Q78#S7;VBUDu8WQGWB9QY==>uBLPDq zowceqqCDk&cP@7{FiNi0!()Uv_#5L)AbixS{4F9$`g@9nEI z^bz8H*TZsY49$qbR6$<`Ud}P%^&4}2i~U>Daj4~;m$6Z)`)Tdgg_(YrB{F>Mxr|dscX5r8NC=opImg&adI$ZiAWp?WS5{IBV_fesxKIil~ z+l0|i`R8jL#3dhx51hm3^PQv{!>=S#f!^-^FZ@v&kh|;mEHP;UY)^VFlx_UVbeYIl zo!`a(%G7o>;9O;gNre|f_&o|=_(^JSBx=JKjI1~(C0q)9=wtB{zA@@bf`yPJ$!<>0 z@VJ>iSs`$pTg7S6f5mIkIKmxb-qr>my?O4Ol2oZ#4N5ZOI$O-8CLE4Ge%%-ElDQgZ z_Kx{`dnnJlFnoRFjotp{5DQx$3bZh<0B0Li7PhmOQ$^Za`!cioNPq0^j2E?)OlTxL zZ45i#L}4cJ7QP*Tx-DP#Eb}o1d$Y?H(Xm<#sX+pAF0pWrwTzbkh^cIxViH5qjh3Y& z(Nk(p2^XiVwT)Rdj9VdG!eD~|F-HOpzpH83Linu5OdL0x_B_oa-u$@BEA7q{Pnr8I z(0NirAFaab)2=DN_>`cqE=db7SZ?xa69^QPKsoxq(ll5NZR2x>x&D;pf zfN=~A_AtNtUc0Vvq7+b0IL_iz0Ehc`76`+U%779!`?pm z-qt^+G(Gs7R1>EDQoF?+NE!rU-3QqVLUEXI;{f@t#0ivpJFE&ZrS<{vKBVGdwIVIl2xQ#8c7 zZwr(N*qlOhwVUXxyDkVW!po@q2?#$yAT7~M!N)meoG#5{Jtgts$@8xtlbmkmuud{- z3Fb$mxmQ9+gm5CR0Es2zieR~HN{n~Swn!XvvT8!_Yg@gv?d7N4m9i!vL&Pi^ zzoze%>WUL=(>CZ-<7q54=H6R^(9Ev{Rxxn{U({7|7+B;ydZ39-0m zGlz_S8aby$(dDK)x}mLL;L%VDD_rE(E$^`yflo41-mxIU%TFQ$u2o9SZ?L0oMY77) zmFJ3v=OAzsi=kyOL<5$~Slw2H-drr%zhK9%qS(kKl{voWNB)ulk!mE{-!wmm`Aqa~ z%`O5PZ2QUreQ4bMHIN}Hci7AC;g^q>hELVG@t%9@V850>HG0B#ShP#_h;8?c)x(hT z%cb-2KGG3Y`iYDCil#!)Ar5(MQMb4?)2VXOrE0N*TWe=QX3;msnab<|uDFM!inisq z*29=UfCK>3gula{^b-(T82t+s(eg$4RKGYQL68w)Kavj`>KuL~&6B`m`vywjXl7AY zmmN5#sA6v5tdBDT$AL9`z8*wJRzkSWy!B}2^B}tSvnD=rN;+$|fP<5`I=o-nM~~>9 z3l`Rv;2{QJ_cM|p+|7XXFXvA7Th#x`e_}#9Epn8Q{uWbOt9Dh@ho-SaRV}pM}wLC+{9hN`AV^4(pVA7 zuRrKmNO&94t&=o?wn^a>^ef@eJu@_c4_xJ7@z!FKNxqT%gny%FniX+I?R zu8LDqQaK5&h*cW3%Y~*?(`*lUjFz4g;Qn_k!mT9FdSe=n9Gnmtjlu+@5-ff$GyUk$ zB{{g-nTaCHn)Kjv$JtWVTQA{r&tM~=f_Efebs};QNos2l4QR!FH>-{F!6xph{nd-A1Y6W3R7U2P4;EOs{q*wq^a?r*ny zuReXi^y7cUZT|im{?PhQ;b+>9F7EtQH|fml^mk%^SsnZjhA#a5kN79-f2d~mJ5~r3 z97`7>clih+iW>|9cUn+f)-UUxT#AFsZ@AQe&yS(HFi4zPo)91EIG(WWG|d%W@saj>%-Tc-HTJ+MlQY9G3rncYwiTA####cqR4&Mv~9!x}9ZQ-b}A=q7pCN zC@h5Wre>)52c-BfYTZ*yIQ-A1--t64M8)$cYO9uGlII~lZVO%}GR`xbK?&JMw|DL= z`2V4D@$K`127kaGr?vlV#*qK`b3p!o;vvSr>6gD`=$CsB<;wuov_jpD`~3>s%TaD7 z5{wNDw7lw;5$Z&*(SzS-qwnMXpsK9?Hiv$97}TScSeBU26X4cbZ*=>gn6&=QrGJ3^ zvLF1x&c?LP#PYWW$~J!i`@OdtTf2E&2ScBDz{D!{;WRwYNN$LRes+W?dGALke1_F` zGRqE5+Al)+K$9g0B)1pL>W7WE8;;oPTcg)*e%|+Y;@t$nL)XJhc-Ih#_^2$!TUvq6 zg}B6k7jb0lZal^IuKs-~Qa+ZFe{0mSb$Orvk2ueBg*$jAQ6$)>r_+?X}eGVhYvx@`%rnf>&H|oCw|r zXyzkf-DsR@!quYtq4TQaknbfSyU#a0eYebk83)0+6I-ZEdy5BkUQ_Zf;1c2vnank$ zc-#^%7vQro`qRo0c$MD(O$3pR9~vcZ5`9lQ>j<6>DyOUvYBL5RoXKF@PcTJ zXNX?=z`5X<^pH#Gl6BzqW_Zy-z*SEx9I39rpLCUMzMSd;;XFkY{=S=C=lQdSzS>=VKY{69%KWR}4n-5EIm(i+xAlgbH0s2Fk2yF~cu@D!qkn z=n-|F|3!c)DbH9&H`s+PYx?Nbu%ToRNI5!ETsT@$T5m7c5CUFmH@)!75J)xDz2OS} z=;e~BZ*i%2NS>p;kV+Y?(n9T5Cb=uN?JsICWs~p`nat(G=)~o=nDE*tr~D|hEXDb> z%5Dx88D?7r*$!Mj?s^P`u%*6LT#aPQFC&+y3 zA!)r_!zf8d{i@?+f^#DGes_Jyj09W8!Q)o;CvE0|aYO%sqre}x%huA1c+{yj*Ei%0 z*-30PVL5GniW83sgiE9gNMkTkBOn^8t3s3&PN|00mxp})?ExWhNpMAwW`QqFg9fP@ zsE)4M8jbk~x0z#EYa01JpITyT!EtGmmM9NKy=fq1Su3gg%JC6Ogho2soM1wZh~+_& zk)NT|*uwqvdt9IFNYS-BB9d$MF1E#7_{fSFie%)RDunNKcj!iMPQvWe8;Fiu{ z7NFI{Ot5B1h^#>iDE{eM^23!?OXL7Y$mQM3pY&@LaQ47XuNRTgRms(^JW{p0CeIc< zlgyf}UV8XJ;fwOn1xDq25Ed~vV;6u{QHxO@lLQFi5;K4Ui|y4 zBdhsZ4b=?n2+5y@9uuq_3zoT8+AvRR07*&pH}oR?u%u9zoJjbZ%;52Q+$SP?ArG%1 zEMHgj)yL6FlOfM#kqiF6G9^e|fx$S<+jg8By^!KLp80Mt!TpL2np`WFYO8Arsg?;uP4%4bd!$w@O2o$r>_s19#dCP z>w&QJ5Dn5^V?No?Xx~r9aT=<(7Fm=Uh1psK8fNO=W4isf`S1T^{X_45-~Ub2W`{vo zwe~C13yH7VgRg`BUC+%cpr~*Ux=#;%8m=Z~AZ@P8Lf@2K^+90EZ80*Y=+!|=bL1RS z#c+PY7b&c*_6lOsQDlVD!q5I3^!m!U@UY#b_q!f>we~iUd|XSsj)a6zyg_oRFM~VB z^6HP*t-SWCb$>!pVo)MIaS&_NWiyH^-7EtLgfLJ37u zCwB%?*SC)jrZGSGclY}b9Qd|&`Tx?!V`B32HGHDX$I!Dodz+0RP0W*d(SBAKwZ~6i;R4O}EKUJ6K-u z6$9PpvJ3U5vz|)3_|T1buU_{ta$0o@S1HOI+m(_3Px z;ojBg^b`7O21#Smbcp<2eWQKCT*hZQo4oC3Gk^HF@wG)WqxSwmQFLmnFlW0Ln?fA4 zTLeM(UFbzWkxOwt|IQD10D1dt#0qfV4KEzZ!@Ds)+)o|OY2Pe`epRhRcpk>(yeJxA zb`nk+Bg_%05S7?ePEE^)i;f;$jG`hV#JDO{C~C41?Yd$QtCs^btEtqyk`p7t7=>gh zYtfSQinc2cvMbwNQAr?eLv}xj;mE?G1;chsNP|VOXs$z~( zmP74m25a;yJxnE6Dr#f)t+3Ko*n%hO4^MQ|bFpYfQ3Aw-2~pB;iOGnmd(nL%uB&fp zEmaIc*Z^hcz(E3w{0U2^DpPxvB`Xc8E)B!!O_BKtI#@)>amDi^e8R_%3C^xAs z{Te5@U15?+9uekDm}pft_DixUek&cZ5?Q#E#lxuh1b{W-;T(*zy2$FG9lu%o1pOCm zBkQYhDZh&7LgJ2xWuobXPT_(euzGl`0C=!9_6rYZt7mBbfU&*^m`-jQU&m%IFSM-t zG1y`Pld98V@}*v^th3bGt z%{)u#w0?Q!f_Kgw2PKLRc)1!r(x^6!fMN>c_oVi8s+-H0_@zcp4{ao`7|v4?5~$Xw z742Kxdz*SY8Ua4hk*ixqzGgjdZI(!7^4DxmuI4{qLuiG2nSEJ zC<>y{s;rprVPo4-H&|*)u5Hw-f z3LC)+)1LZMCrvgb^BvfP;BxIo&>=_r0|$q}Tc>T8J<&E2QoH3BD>Jp574Y=9wrHXj z0fnobkVD--S62XPLI9u$UF=y8;83d9HDS_Wr)pUugk?6 z+5y)K;~8bPnp`0~F0~ULikM|}^*NYDWqcTLZ5q`Qxjx6iHZ+^QSWxx@6O(GJ_KP)3 zQpPW1;yk8<10WS3e6YsIaA}j{)7O(FQ!n+7LeFU`5I-K;?W-+k?9ke%VhzEQ7nHkR z68D(xZ3^9x+`{yu_VfkpLRV%U1lkyLe&iF42j}rdN=+q`>>c8e8A<3u=Rs?Q)h?cv zElWFPC-#BEf0?s{E>{VJsSTSUx-FhGgj;*On#!l&wE^N4A%egz&kAqYaRn)M=a4La z(T3m8lfPor6AawFeC%XSFH^^iFyWq?9vHoG~mVg_v4FDEcoJ7Bf+I}si!*u=5{Wi5yXy>FCmJzuQHz$o*l zNX6QXlu9bP`N=O43y0k1b!SjcK~(~c}jEXQP>Pv-GT@SaK_UPtL*?zvH* zQ(cy1#d3S#E!IN3|Rb>SD3D>Tyz%Pp}6z;lLWF=u#JEXlSSjO&$`RQyoA5I4t3mdql5SeuM8| z^l+=FAF%0kk+%{lv5lw!v2%fKr2F0rOC7L^dr52qjRd zE6%%%kd5n+lNFyf}>nrBxaTG^(tCqI+&)CCOd zGn?IQjjsM|vNoE*JV`R;SQ@%%WUQ|8X0mFXxww81Ii-Tl%*1As;yK-D$dbj;JUEIs zjofO}nOE)h!SL;U*JLRB-EjEE!L!r1%fLH61&21d&J{3-*M^5Zf{HkxZZ&}3M-}ad zT^jITaT3A690&oMt zV9-8MPRnzUF#e(Xs^OjRbu+VG5eM?%O)o!uPR)+nIk}q-N)cprsL-?EL=whLEAnJSA3AQ3|9H` zqap)MHqNP~%DkDg2IFJx7(tae{_bK>;?Lgkl9j#SDOxu8;W^96>DkcWZXQ9cV>lPK?9E!hI3u=_n-52ss~W})M3(eDYGo;H`z0rF2&)y7hc5l0hg*%Kt(l6W@>be^W{6Q-Mf7bLHFxL7lie>c zx$p60%ct z95G;A(Gehh!eH<5@)gW}uX9-pG@8~8ScfJ$&5x$;xTz?wUPRXq_w5KBHg z=5f9lNmp1;r5twK)-YfzFr_Q$E0`67LyWU!g5!~9)qN5NdaW2TeBC8u>(1?V1yd0f z4uqBvYLV2c=v-(uaee4!1&7TIEuOdcy*SS5dt^sBVKfZ}q#ZSkJ~Ifx8G-O(Pu!GH z?3k;8QJPStG>f_Sk{xPuizJj4H?6RM&Nx1Fsx!CgVt>)Z-EhV0dGM0gn_Sr{0DzvSyLvDE&o%O~|x4Eb=s$nbi>Oyycaf)XozYvHR zkx}Z{98$yck~0nE6Yk|3m6^dEqGsEa#q3SEQ9=ISwx1eJBEN?#pl~MAgRUc9K`nh(Tcem8q zDn0qh8TlokP!I0G0RfZS(eUJ}$iGcVc6OH@xAB^innC^ z9X5ws?O{a~7%4$q($Bfl&rQvm!v@Td&QV<4xEhru#*C!6W^{2l&zu`g_9z-MZxs=2 zJ4C;~q*1oqK_6OJ_1!p#k>!+2cdCd94m%}p%*T^4 zqz%{J7tU|Hnkbko8+za2EXL8C%E9ihvr0B}eNia2ry-sAv58z#(niz_=3djGw+9K% zrWV-g1TcO0`5)LZ`THh+Wkg|q^efZpaQ%tN)>`AGYrn}wo}c{thW`@#ukw_SziCCD z*!u5)!mGA_<)%yrY2;H)Dfu8$J^)|dVTYzFW-t9L+i|OOMU8Zm;pO|E}orUg)N1>GqR*&)vreOrR znbCnNgXr`&VwX{9pi51V@olElLN?@Bpe;nW6B{BfQLcZPUH#^)9R)kL?dbIGg1V}j zM|9U3G7N*++&CeD?>H0OvQR8-|y>`>Su9 z$ooTJlVj}3czJ9QdW z_r$gQ>Xw~Br+#*ek2$h@(`v11k?LXRpMmrbxcS9i~6{n7NcdfqM`DWam zyQiF_J>A%;##$j?srKbQ#DDF5*q2cAC-?qp9D>j1b{~`=67G6*tS3CNpxahGx?gu6 zlGVre%}|G&z$t`DLT5hyc{6{G48wW;-|?CAR+MVaeIe7GNJ3SpLQ9tNo5Oqm_IViY zQ;e3#kn~eP$@cZOq>^<8%q-e0=VTA;oD}NLydvm}bUV^p(EgiB+wfMuxkVn=V^UHg zx4OS%VOjybts}jIEjpQM5XP0NEuQdUTU>)5=3BmQCJ;Vv(&q4~y3OSK7`xwlv+j`agTpjDY{GUo+*mCh&ONZ(qs&`vG%r_HfQGkZ*NJ(IDgZzZ@6$m~P(tE$G=f zbX9!4SbQj8pZV->IhYF!-DQa}f%mmMb2+-pW@eQS$IX9I{|-;jUied(MvG zuqNHu&Dg<*T+}yGbkiiQz)e`6S4mfn$VyjW&3k+2({;oA4L8QSA90+&fYVDeL`PfB zK2xzt*9RVD(b|ks7|XkrzdM_a;Aw_`P@tj^6b0^+Zw=frYW?Ed8{x z0>4b%ERdBTK7Z=_=6Xd{#TT+fO!1l3VrPpso|<``9DI%4>tF}$Q!+&xptcc~^&pb! zgz;QY7FNerx`P@YH$S%F1Vcu75cOppd3n*|SAMA>(vdazMxSE;>Q1B0lJ8tD-3uA3 z=r=WtO{ZnMuN&w;JBZkm+kX(DZQiR}1x64YHMJNSA~}R7q)R6Ej-cCp;nn&dMoMb? z-h_HGzZ+l?c=o0g4=W&6Hk8n8O(fU~`z(~&j!|)6&r>8hqHUrG(gQ_Skq&VV z5mRA=xeHGT80olHBp#3t&r+U4!-A`f26q!+9TfEiU})`>dt#3XlnCL~G*qsVvbII$ zf00@++FOmCyfkMAI$wV)@W3D0J!F%yk+kwFlXP|8Rx}Q%W;<3V+$City4DhzF*C&U zT;JY^nSvR;V;fk11^p}2Q@F$!iLAnEU(eFXkTWWgYlT1plcjUShr$|Gc_h%4lxV#X z;?`8MRJE;`72ahcQ*YhTV1M24P!vYJ_U1cJtuxOkK&N*p6YSHNbNEBjl?Y8V9*}xv zF$KT-WWzO9Q-fxa0e+1ZJOj6#3o~Xl zM!g%nW#?h@x$vG~fVyCNR_hjthkY2WH#Ih6+^ona*?=*-w+Ch1~GI_|d_Dz2W*aX;)W@!q4A5_*0qU05<8Iu$`T9l?>R=tG3ac=tr=db%%e}Nq%Q~=h zi=neTFerH|C>`(KLilxT?+)$7V|uc`sVz&^8UIs+v{L*1-~8t2d7&!4{VoTBvdk>_{# z_a&OZwno#X3`yYffCN&LMq9V;(=gi8#rUS2gtEEQ3RWFcx~|Y7*n-o(AA^W1xMXn> zwT^f{%h^@$r;RcrQ%(a0<9QoDPDKMqW~X8gN@emfBVTa-7AyllPyTjam~dz8Qa^v{z{|8EUK4X0YM!7w%b z)e>SR0O3^qR+H)Ez5l#;Yhd!IEUe)pNFwhfN0D<7r}@4?m*h@WhJ341a&&o*Vekf`!4Q__csG^^NeA!&ZaEl)uPXV zL(-Wwk!N4#KKFfn`^})He^?_T*uokL$63GLq@vociHVA?!kiwrIlpZQPIFs~ZYDUodIS`4qCO z!}K3rWJJpU?JC*&pP%;q_GN;8zjdWAE%ZpB(EQulUF)0JAL!k0(`j{c)}7H&xJiIS zVssQ8nfY{~-}F|zZnjUeM1T-^!6^L1$1|ul(%C8l=KjK|wsrnfw>l=(?gc_gQ{4 zt!zQYhPXU7rrCdRwfRpp`?m2b^Gzb2QIZAvJ!%ktdy)&?gbYB-!@mWI4C-%>veBB- ze=Z{Ym%_-O7ybNC1mXX9^lmzEH?-msTNsY>Z6ovdQ>vI`^O8DT0`&p9M9AcpR!{U5WQmlL>-82RLnl+#DcBEr`t8LYgU-Z-R8|aT7 zk2~i=E4o2g?slND2q7XblCJGs7@Z23r}t20Od6A2LDhF+HXi#&m0vQq>W;CHE4hg# zBq_jDROx|76ODGyEgU3@c=$#NCPa)k%ndZ*sB%DB(yQAv-8nFvI|Hz*vYN&UO?r5c zxwl94<`j=0&)}%beS2zx8efEeWpYPtUp@8YEQ^%BndPg#+;Foj zaBBW~nWoXR+HqMV?G1k9D|eMXyQpK!CO-!j=|h<+&t_{6OqNjS}j@t&4-=Z#GU6@_?t7l0Tm@C+kQEJ^fUw z*QXWT2d`N7QF&qo6V%#jNHZX>ZZi)Pyilf+<=!Q~++pXmS{BHY?-~DPx|%$r^}MsWUw@?G6ztIsF|@gj z!MjRiPZz`~L#TKet+}y@g}cxP)+0_b=b)i{HdFDnu*Wt!OAt1T{kP@i)cpbKXxf&h zO3MM^t=X7xIRQHCuDZBxoRAjV<)akjV_6X(a596ka6`$huBR(5u8+!}vUM z4E6eEG6zE0YJ!VML#A39`gtTg!yMW(y8 zp*!)&IiGAxoC{>WR}{`jKeMsglv(0oH$@)|G?Awv8kG|vd^pPky6QriNc8EHM5Wj z^sL?U)6IKdC16J!x%ecsI;TNAilKZPEEMNATh)y)*s@&=;<2#TuE==R)~D67Krex& zFJo+~I_qv0UfwhxUy)RC&MyT$TN8=rcC#|mWvzntb_}Rp(y3ZcKm=gtsD=ZsRJl!9 zPI3HI+>jZws5?ZmTZpYCCMj*)$EAk#PPB+4FW}9ymc6C~vOyCeS<0x>qqW}?s~0Xo zUrxASBUW6Dx|SaGo%n)`m!H-=GFJi!xavM=HZeIi0%L3KmSmQ!kMLP< zZAnX1i{|&IUn}pMy861xT>V+5m1RFzPE0r7$d|@mLk&`w-DbPzVb^V8qi?JR{4k8t zi+cte_LNrQ2qgQ%w||)mMuCs?8_Rkjh9Z)z;r8KM0J7?E>XIfjspYZaO2*20Q|qKl znyw8LHx~yXBnWRv5NTb^Yz%Pe$OLz6$$$uJM*=fEerT=RLo;zO5kwijT*x3L=Aex# zW03`JcN>(AJ1wPWe_t9nn`LL^`opZJol7 zG^8L2J~0~AV@SrVIDj0CmIRo^r`&NC#gd7B08()rhiY_0X}n@%$gQ3@N@0s;g8E~L{+ggO}_KdnZgG;CJ0C!{KhbS_cg5|)7 zNtne1i3Vb&L%y`Fx4bI}$JbVzD^HHKfX8>MhsUjEFN{ZzT~)c2R+Adwx_3wfN!|*J zYFD#G>Y%L@2-X)bXHO(O>1_Q<~9teG_|4m%xO&29lP z8Im#;k-wIc6yw|bgb%v@V!nNPrHiDT{kqhiMGA<&;wEzmdX+!gIfqk+OCc=R$bZ|* z4kbBxcF3dgW@WbuQn_HDvJ#}^W6><=g7KZJx)oc8QJGSX9#x5f->7kNEZ?clkhl&5 z@ZmL_iP8?qb;#`yJB{rdBt>NJ&u%U_WN_XvKHt^NZG1PRw9XnU3a``iqy@Lms-1Hi z3K}`t)2k%>Y{+9;G?9Q2CIykwKl^meo3XqfJfB~8uHvE%Ttt5jsF9H4ka09(J)i;( z3}bK(H76T+Ovsi!LSd%dbCJR2A>|c-0`DZj76OYJOB{7JGDEyGxH;y9x9Sx#jnCcC z(pl4Tp&)=9p|NLp{|kbzL=!_fyvuS~(aPfz*|}v2>zNm9!k9*us0ibf=-t`ASpOHA zVfuNzRMAl9x&)U}n!ZRq+caQsU5^x;pF^;G{ZxJ|n}eN!B9d>bM0fYl?(HB_+4+PL zqcC_uXR4TPO#jf0`CcM8Kq-Ymg4`@A53%S>x?HBSnk(VBzykvM7s8e)M zTPw+>$!jXrv~6V~B~q&!P(*~&FS+=Z6AkFWhhnqiQy<5?Ir!vfJS1RwdXb4V zdx8nXgey%wzB<-q>CrQl(6Klt5@i#@Q5#3N1h84+;GNZ~NV6%Muj98NpRga2>M9#v zT5`l=D3&8*ePA`{Azvo}Q*103kmT*#_$DLteRBI4rn1CS{BA=npg^l%%|-~~OXWd2 zV*-y3Q*|#hUHs3#d7qYv2RTGgei;^SFduKp@9UGg@fzGkXA)8kku>$<-*@ob?c4C`3ctZ z*DCTpV6Hx1946NS;agyamPAYkp-Q#ZVFa7=Y)t!Q&X-D|H-@J^$9SlsAyIF( zq^@ek8kAuL5l%>eQwJ_}<@>?<`z5P)HYv6>p1$Bn6(PWTaM2`EP%^vu2lo6 z!80dUgMV1v!s`(@GL@|>7(xzmI){!u)r;{#B;8F%=iAV@8*4Aj>_c-M;PKZfz5PR; z>8X82?s#O$sy(nRdK6WszWwmQSlU#_cOICCM)&xLtG_Zu_$YND@9$;KS;#2Qrp~6W z%iWWF3}Rt+_;BEK5H@tTm3YBD&QFnhe?Fn>zVCuW#eB`C_EUG?D{6z*=lG>REDGma z8v$32Mz;ILU~4wJ(#PB5!S9~8Vn6uDgmEWa(7ek%l=Ctvk9Vl8G2y+f9=;YL%dB{Y zm^CP4GIssbjt3!XNf_$B)zUJ`VDpyk1b3+|RRugAd`(fcp|F-t$n>YtJOl)M$|d^$ z0t;IFD=Y}R@!WlXWG#Q&R=atWsJCy@)l^e$N@e34#bQ-dlQJ&vJv}WO2Xcx4MGX5k zfKo^j#u$y=;F(~yz;50~&GrD03)QvUbdU)4AHMQ78lYo!k_~3z>R`f#!{G6=k!f^< z)X%5jaTN51rj&>~3oaie7vdx8TK9QL@@H9n^Y$Aw3`Myz*dNibW>Q41)v#a4oMpgv zDphaYx#Lfy-Z>pT+yFC{u8^(~M>1#{R3n3!D5`Bxpa-@FJoY+&ny}1JGS;eTNiZB; z>oN+aZgnlF-S|OPISw@yl$IqEidR>=-=tKyFYFnhSE^c_>KL?lECnkc0Z8rixqEQj z9X)&yL>S%;P!B={v3oB+8*ErRlK8OK{#@cgIZ(_YadkjwyuVlv2)WlPefjdf_0`4Y z99>+2eX}qOkHJ(Q^#%npGd=HoGqnGw<^}F-0g43gAeV&jPl&Bb&m;)WjP6BldGY>3pU6goS*iF{8c) zcK&j?EZQnN3gQ;MJC2^g8x{^kP@ie?XD1c;MT`Y6U^yCS6JwTawk}-XLzlyweBFZm zE)VMhixz#mGDOs5I*HeF>%-7hs1tyb3Wbr;;wfe80fTq{bTF0YMptWQH2%VIkV~kJ zWgqpywnl#hR?KBdon6_q6&2XT z<{1zfrx`2#FiCwU=Yro3E`~i7i>vUMEhDSuc!$S6yL*0`eOx-d&+GHmM2iBwI)ZgAfpyCD`h zM|Wx@`Ofmq@}EUFW7a2ggurlY#2Of9Wjr1P;d?0@<3Sp=85}RDMvT-}ogMXRD`rpX z`>Ai#1<1Ia026UqSitwJ5n|K_M{WBO^vegtnLK4%GGB*X&V2Lf_7gAOIzNU9mBL6` z@30wr?piOX^vK5B9Mo3M4;UKrYwP5{213>46m)WJ9QiC6&3 zex>nu#-+$fOQ+dkTAVGC-6R^!*{Z#%h7`;~7Qp)`!l$~HM z@ftJdMzXPc zcS#g0YI{+l-@E+C&gW=#xR6zm3f#tpNk~uVb&i@kcBhS9Lc81hMneE0&>B|7H6@bp zx#wKw+|lt+uP1@p;^#H&5rKTI?5QpoplxL%??|kFJ1Y_Ou~z=xM7l``1+;^IZUsH3 zKztxkL=Ym&r<^~#slnBwNd|*lpdlgU!!52Z4EsKr;{$nSUlcLLaw~hs5dhzl&Gl;= z8ay^O-~mxoS{cwh1`eH$i&s15hMu4gc~EqjI9ejr+n+=3JL?M_!V%)TB2P8-HcJ$T zX?I3|y-Ze{YActLo8E5iSVatSg1{H*P=asXq#+9c%KceM<@QS8%7Gm$@oI8ZOx|F^_5<%mWX=_#9)`H+}r?{IBhM<)wXLNU!3KLT=}R!Bt0ZO z*m7P!{G$>_E&Gi!GYd0VaxiHFqS)?!9j^lc@Nm@o&P9pNu4s2e6>YpbW}BciZsRrx zPD3od*Q1RLu87idbDx~D^SHXzi82ZNL}@ftxEx%<^6?6hQgtN1D9q8Pl}_Je*JDda z_*!LdSiFe|TbBB4hN#g-8i0RBrvb|^L)YVpdqNb{El`cQ;F1w)9v>qIpKE&c=@e~1 zYmQ(@k*u0;>D|?JO5}vAU44IMpDa+t&5xd?c)fU_KV~u|n8jC>=q1 zA4BkFNY0)~s7QFO>?*Q1d{&xWM4BUS2rs2R+xfvzP3x@UU}Ct9KEBOwDT;r6E$FWt zK>u5c{i}#G9>yL3VRf6KI`B-GHq1gB40d{5Ap&>3-A!Zj`qBMfZJ_C5i{|~xJVU*! zXk?cnL#m|3P_c_Zoe!V`k;D;u)&fk=uivA*WgDRzp)tB>ID8^+w%={0?0hF&3nnCJ z)nSnC4u)@H30rr=QEhf{K)_UMP23hf6!C`Q)!)TYJJS* z@8dhfl#-{N79=w0AtD_)VVIJTt+=}}SKhd=trhIJXd-sS|9yme@Hk`^;bBu~R1kfac^jNo7!*Wf~2H^?uCs}R2w<^oylJKJOAAq&UZ&1 z|A+0I((*2xTK|kS78?4M|L&bL&5_wLalQgtgG`mKQnss9=hZ}6v3#!Uxz5`08_b1P zrG{20@JlCDa=JV&F*aRqt-O@@Fv2zvr}9Zv*Gre;# zNQu~3F1BOw!f%PofM1)<=T){(eBV1o{v1MN4Re_bb z8)*6868_jz1`Y?tq{h@Ajrq|tIbZ%CvKfv7n2*ScYUTnx#B0wHspM(3{^oywABbpI zMu?p8q9ectlZ}+LMG~@dU*&znxi|*W2^yia(wcEdValZYY+uw;>j&x1T?+wjDgR{? zeHlGmOq}d%9wxCr#!f5rEI17x##^VqYMglZD(wVw#kaf39s+7Kf7bs;GqWL}!~14= zx~27Tehb$Y_;w*@QsAt}9`4UVDJBn{A&oz9=gwP{NF8I@WzvK;r+QCirI=xK9>eFtXsTgfQZr2m0*74`=OU=)nOC&fChWe%Rl6wNk@9MNOX(eZmvX)z zi;sELSK`LV2sg9O_5+y6-Qi*8i!d}$}GIMZ=hTv>hZ2atv|68R5)gF0225P}1FVXT)JT z13k%aV}etTP-)hO@U`;jFS9!)XpbLptZ+_eagJuR!SRXI__lIgsVR(1W(!G)!iuV+ z;|RLV5JT`HA^{26e8pl`o?SP5UfM8nEqP4(S&fWCf1op`MJOi0rC?$)1C)GLSv}f3 z0;B>>7o3Dbl_7v8%2h;M|9;HtQw+-+*2XO_n5r(3zW}CqERu%UkyYd}U?_soWI{iy~r}%W! zQb|=J7xuHRYKrrd9K($QsLweDH~k5#u%aZr{FVs(@))OTl#$`kd=!B&0Erihy+s_b z|E!5XM}T5Q)bV2}%T6q?Q?N34p@py?W7>dMiwH6!CdP+Q%gCZh>Xwm-=2yqB6!K@Z zp(EgnJ?L)AFF;gElKd~l6rLxL^UfB9!}5<^5|jsKZ&)i8of8&ArJ?*g+f_dFIl>jW zW*3#661cM*rrX3Tu;$Xl8p!jnG^akN^0z!tWq-#UuA;*v4D8*tfSfZk)Z#d~>(4_#EMb7jQ>e)J!*T`2{QNLi0`qZ3Q832zr@nobh3= z-iK1!up{$H_lm87FR%I|dQP_;Cq#6cY#V%Yeo%uDCYr&at4GIg4hovO<1 zZg|w599x?^ac(62eE*6ug2Go5y8NcFO9Lz)KY%pHa5xY^%A+{c)Z$~aF*hZ%3CU-< zJFBD(&stO~XUkN`-C)XF>23`TUZM+NDn>a$ab0t#jpWsz)C<$}OdX)uvS!g_k z`Y72vEK0qp+^X%VUEH+fA59Ep2QW3&kC3?V+Jj~5aQ=-B%QM%DZBpFXx$gCu;>1e6 z()`qO(<_H)n(Apwi6Mh9b$NN|C>2|`G?FPIZmAi}Ah~JIVn!0o`E+eiD2Ic6+swIC zA7thN^06M$GgN~xq{W{x(_uGn%xgmQai^uFWjUbzpFE<3Ro@NX?%od7$~V_53>OMV z--Gd~vSfK`P5Mf6s&D5Jx}(#5ZAmU&nB@EO#*z$Ln$qmAViPz^Cqy`UccBaENt>P` z-NI*Q)2X5h8?vNnOjpo2mA&ZS_xKNB&mSQ36U0McB_Hh2z3Yi*;Q2%beyQ1ES(c=bzK-ygA+KQgHR)BmSBVLp%IYfpYZovoF^UX~y-Q*)JuJIL>4 z;q-a{+E{2fAX^}s@{9cAW51jE1=Olzc1`j^h`;p==m1hoBsj2kehO-}-Zee9@tIIm(7D z!Cb`;e*rs2CE2;lB#w2hPA*9eY2O9?mzvr&{+RD5a7oz z{gVN3-@E)+uy~Gkd>qGPs!{&!sM>#yCI5BX%vAK9>%o<~>q_;CCuSA)A5EP7N`u_K z%0+ECV8o?ITF6tSX2wj<1LoOi)u%)Vc)JebqK>#^fHT zQ2|$J+C1#1KO7r5mLFcWZe!+?J;Mg+cLT|aKm8&1-!l1^J`DaDTv~trXTMJ%Z&iQi zf;uux!}@zEkM9%6rxoA1piaJ{VbgnlzE3mJt{_3=#P*!c(| zX>mzZRJOd@VHa7*Tf?}N8{ic>?mzEo)AxP2(fa72j$vUbVxKTnnwHjWHD?cZor|k}=6LvY z98W&PW!u6QeW^isDB#68YnJ}>jMBTg^pfH6k*(@Px*!tw?C1GVlD==jSi4^di-Qp? z&1AY{kl(lQeAxMqJ(6LlQ`z#CHR(CyEOTUHO?7EX+NVng9s1Ui=C=?&kkycXgb%())Zr{J=z)o2Q^%FM7UlJj(4Y@e8)B|tGz zS$89PjK(M61!`|w-J?u>-efv2xJ@z9^8^+Pp{T^5O#0qCF=uXzl~W*e18>jTaNXHk zVn-qd94}$odp51T0hlN%*d;6~OVcKXgXe7~UqkYx@v@sP=!B5qExwCI!D{!T7LX8M z{?GGqE*-Bs3z6#WXN<8Rr?{KIjqo(J`v-LFXZbBmk)!5qxixH2tUj_D?A%T!$3Yql z`ZAIN4Lz;taO~Nqqo{5%Z#LS3<(6PnB{m*`-_kJDIp6N=ZreM9N0a15x9wrii=H?& zK!j#aFhH;56oA%EOQ6OPaQQ&Z-(WAe2Sa49g1q|`-leh|;MH8l+AP`f+j&XFpEug$ zg-Z~9LIC)1V%ABhYfMuwP!7BzVmw&H`Nnij6j0fJ)l8oc9A`X8iV68sdeZwIuZP$b zZ@*92n?wcMgL6+^WUgZtvoY$)P0XkZvNlNL8K=>tS?;VYize%GcDIA=&nfto3dzz* zK%8g{9cUE2wW5GX>T5NW8k7eQDb$}1jmHx(^bB7z8v>!VUYx|{q9-l(*f@_qKJ;Qp@F4KA)}u0qD#@<2~-r5Zh(RPp~R0I z-3_~P&)R3_uMOR8Fv6!1u9jrBHH)o=^o)$P-Kq5wR0{=2-_QXTu=G<<>SmHq43-kg zT~nHb=Yv;Bwy`KKyhlS*+1tw;d;-UwDecXWREzjR`Cx|h7I-GOvhrLHu1wT__q}Fk z4G*n3;T0p;Lk7oY&J~`4E&Y&lO&8aexl5R_7$8hpNS>Q;x_`3Ilu$zI?q4LFE%Zh} zFL~tan56p<+rXvq2V^kABRN4=mazxAr;Dv-r$Vlc%GEM4{i*i+hXy=W4J#UFN?u{o zC+#zirO4fn1DwJjigrw_hJe7hLKa71w{Fdrcy-v|Y_dIRK1yZG!!MzNMWn^nIDX%y zWNIW4)cT;pbW!v?3hLy<)5+14LxnTZ_QZqHm@>YxxarAIlT&ANe$H3SOJ9x_$Ii4G z79_wulZtY`1Qh9gr7{&uy1ZTUDm6 zLPqgBcKPlr4hBR!f;6@WCB5LKNglb4m{B;0`F&E2PeUWt?4;3w`;TP2G#wv*)bVk`=E{$ny=b5srlUh z;|LQFe+wY0w?f(^98@X=NwA$43v*n;%%B@MvXY{JUALQfTv>a}$)vQEHW_>Sds?1@ zL8uNm;Y||20-`x9ZjIn{L7IL1M zN35rYTV3rhGi{I%(zom%hsul0BYNky1>UP5k~pRN-_~tmBho!3r-Hit7!%4iR`18$ zDU>)bU+8DVU9znhj(wABv;P(zJ#tvNMO7v5V9cXlm6Sr7W=)!>CXR^{Ox4; zMDv%BGNZ3FC9~AI?8c3wTA7fO99jj-^d4wr|0$>I(6NwfpFt~N7~hP;gr}Wo=Ij($ zpQ0NRfr-bx%(@Q$N|Po^u3C|;nsyqbJSU8OqFHWey(xC3-fLOVuFPsEj5Bq!__RZs zhpyTqCe8{(#dM(meDG90^7ix9J@1dyUxw~bgCwnzV8ua1%QJ)K@)85>#|eYUox1=C zcyK}JlA4ao6Sz@jTIxLB8Vdjb;ls1D7!2v4+Ht7G?b*boYUW)tx-0dJ40rLy!Narj z0U|YQ_R>bUM~KMbP2F`xh9t(JM0)<I2O-aVDZBD=T+AysBB=1?J*%M`CRl)F*rb%1bHJUaSX}{Ls5$EAoF~0lkK(8+^8U@Oyy0~AZEdF*FyUNY3*HPYdD;L zcCOdtIP2({M#k4IK`S;!sLB)F`l?$i>34UtjT`zWqfdSyc0kFH_oa^W{S`f7Uuk%k z6r5pC7E$Bx)q;m#^)-1nUvze^GiAr+VfcS;H-mr!G=6VR~pP)VnCIVp@0m z5sOTF=CGueh4PlypvfX;mrol>T>pj85eCoNp6fB8Dw7{;&-1%QU9HYed)O=HphjK4 z-u&h8)%Z z`w0O2r-lVzk4rdACaK7$PYSdWAKF&sZK(pxLJx`xlvM*g!JJWKZbJz++3UqRGHc}a zd0pp9(ek}1EE_d&ucq5HH8^UGuNpa@1r@h_;;y!+Jm)ICwP4UrlY|;J48t>!c(4I} zxOAIXK5Bga!)#r%f={ix>6Hv7+(5_iozg;nz`e{AM zvr`X6uAfBt#tD>KV6pbWsH9a$tNa!fKvcGwwcJ;peEUOIQY`+hj)-KGx(I5N+mVfw zra6aQ=*SlSVt4I5=i^1EkGKaXit<5~-5~WnY1y#~BHo>-7CU86I(Xy9cE*Ij z+@c{+25B{eW4$j)2qWIserOs>RuAn;oGbS!QDdkM6YlJ-BL~}pBYnO7uG;F^lE2dQ zi4QTF;9Or_tG?}t>J;L4j*RDDgb?ZVP1+M;rFKJdEp^lz9h4!{rTVQ=)M)rm+kOf? z`R3wW-1x|8>|7~Sj)*wTpg2gcFF}=ORKQ_ioWf)}HCd#1mn>@>mI?!l@_1bVi@pGp zBdfW`MWRD<;I)!=9xg9|D+S8t1uD8<{A*6d1&gwVF*vX2YCtZU_d=>^Yf_oepE?H~ zBp|(K;`vuR-f!d?vD%+}+v=53=gn_n)!b$pkB4Er923A{9m${Thp$9T7EUbonJ=b* zZ>qpOE0ZhmDr${ow-YgCydNvMoxPt6_w>Y`*E>%rwwR27^5AV-qepvp&SZj3Nz}j6jk^WxOC08bWv-!b1?#xQ0e!H^MLt>e()7nhg&(b1M!}~gfWRxZbXBSa1qH>$u zfgD~WVK+Af9?Gz2o1N(!zT{sqfWt9QzEaSd#CpJN$oW8s7Lvs8n4MA;_X_#Tk-uZ{ z%%}MMrTcZUPKxbQYAr&ri*iswpT1&*U*RkXsL#1%vBCe2-NfkwP_w*BoaZ>>iWdlDAVY9Vxq zhUM<<YWi39hMaYy_6`V;QG=3%*;Rc?+y zySK1DHA1V5lr}HtCpLw>kK0Ucduh*VMtiq+3tDi~l_VlzApBOg&6P18?h?NFm4?1- zy~A7f)e##DBR9WyOgmrRYrRM{B}PZOv-5PZR4Hs2cMd+>AF!?1lo%FyMqvd!g$jwV z=^IexB=54R@67S!_mD_n!#ElKss5wy`XB^4zZ-QdFNT+{ZE8VydNrgdP7Z9zPyn$azwLl!MYYxQ3M%JiGl%QiDzDl_*GHSoE#A zLr%zIskhS;GVtR2{#b=h(>_^4s#F2VKDh2^!+!#(vK^@8)J&9@Gph=y?R+5J_+b4} zbb+RfnB9mARKPPuJnAVO^UyQrZGRPy$}asi8MAnCz~c43vItdvofR+`6Xn6pdG#$g zYap;}blwq#TJy9q0UJfhwtN|B4HH5Kvn*tfkiM;t`GuVlm(;P=RiyWo=RcSjR5cmYl2)viIm6JXtFrt=Tds zyV9D<)IxfA{_$TiuK%MS9)d(z!jN=zLE_D)tCHky(^ZeibaZ?W#dg_-L`KY#&}C(m zyCG$d-C)e^Z}~ZDyaw(kk60IVn=WKx-7O?1pYqq$`x4m@GfoUZW#tY9r0h$dBO6^G z*$0OG`O=00M06IPG3KX|UEXA&gNkU z51^J}pDwfCQE8s6^D!!p0#{ykN5U=C;|SD+sr#shE;1iPnsv^-)jh?kWo3+=2}^od zHI?Xkmqj3&ezJ5F0^v}aW1EiGhijAF!L7obGMBP0_v<{PrPMM~4B`pX2e(7!1u0~Y zu2!}#iR=34`UrHLudD|@G#4~C9Nchp+9}-Pt=~z>A*B<8FqmD7(DnsHKHFK}hc=of z23Z0s0&HY!5}0iU)HS<0*YUbeda3%u_20btQ3aHrKf!mUcuN^ZQnE= zGm`uOK^=h&MV3A0ZTrmv=;Z1tr>Q#U$)R(`6{(kFDTbYYg>C%z7WqwOl?5SFRUPf0 z()G&?F;n#}>BG%KitKgAroXRw$KF>>;9LYUG@W9Epf=D&Yb?&Ze!(f)m+SR0j@o^t zd@F39k%6J&79#$kigF}!KIu&S%1GG*C7x;{f6)ANiNPW|R3b7>0LKgUfa>lTB#(={ zl&*at(r+IYW#(w2b-_1=JsuaE3HHVw*cLg7FkeoEaUNGtp(-cy0A+{B?iP#K@{)0aKej+rvCXfri7Ey-ygAp0^6Sgkq@l3}mR+p{lMS>N1!|(oKzTPM zry^CBprUuBRn2KOrfR9MqR$9BMnRYERL4T^vxx^Iva1~_inBNL^35gn44NOFo3(XH zbkmU^V_?Bx%f9fve}BZ;pwz@+71Y_?Tif_cGdn;9qmP{>IL32lIi@^x*G(t(cB}cN z1@L*$qMuO3{=6KcvL&dV2DEUi|L2Y6qO%+~j!mXMLuhak)JQEmO)IgoEmXv=;^D*i zjyXuZ_kH`1W`*Yb?+;1N>{rY|wdDxqI@K0Y$L`V4N$-Y=yoSaAL}Na5D&wM!)Lh}4 z-0r!5YVi9lP9G}zlcU+rc9?z&-}L+G3bscPGi2O*C!Q@+u^I{Ye79MfdKvj0NJgcR zOYF;&vbDkTm;a)A23O^;@)>HJKO{DOh>zUgqKo+<%kr-fteHu0s&<6~#ncgH#ogMW z8#uZcn*Uw@27lWpjmqtBsA0&dqY#?F8{rA`&-}*0Zm9L~p=M#e1+$KM=zUj)6zTF< zOUoxkFIyI5>C@+|TS7|%%yXLGSF>XN7+ZRCHe2hTE6wqLD1OM5ev|x=#04>zEFKcZ zBjvtbxKDpw-<53kXN$E=hfa?gS+I9ucFHu2WF8LE&3Uy+r<9L8fa{z?$(oTkW92-k z@dgS6u$#`{Jxik61&G^6!7ulZEIwnoP8A6NC&fnI>f!Vby|+%wfF))>ckE%q@U}pbpOEfls8xk%C8<8b>-*7|NrlN z9Q|z_to*l0#*tmOZ`uG+MTbu|`*oL56l(Mt^iZJ1>zf{c!gf*+^FJ5U@-KqQ{J8Bu z2Zpq1yj>v)orN1_c0`P{q1GoDY3OhL9q`J#ZuK~ZU-}8Jelp67n>R5VkyI$?TJh-M zowZzx_?^W6$k@q1FLUU4qxj`lt%)<^^)d)<*Sje8EgrzA$`KaAy$1lbN0w8J{fGoTnUbnyf7RYb@lk zR!K=2(U_n9$n}2|@Hfc-{I!9na;4v>#v3mjJ!N>PMiBZ$&Hh@*Q+$N z>e+vrQw)6@M(1cQND2J%Vf48y&4bj6(^4|27k=SzaJ@xKJAeC6r`Xs({Jm7k@CfDA zf9HYvx3>MDY(JEk_#1^O{#uLtu8#39Wncfhg0BB+%69q>qIXD1+p>4^4UwFBVjOjC zI1)uSGO7n&WY6;&J$V#zOLDC zOz=(DKbyChY$>}C#Xk%5Ywo&e`1sh*|Irgi&SA_j=8bX!Vga~zZJHNh=v9#KX(^CT z$D8Nx0Gv!*lJh@z>Q5;>WuUq8?M!=k>B_aEKe7&c<$rWOJ9kZm5};@b>y-=8b}1VEWO|S z%%=`ymFHKQGgZ_WU(K&HOOHOj&wT-&TnMS03l=IJrHH;Ab;2`aU#&fO{M z79kUpxBbCo*q=UD>?dw|u&bl-CR+FUrM3UnB4jmT%|EBk*riQ%sFDL$aw9xG`GmSf-Qv6vNJdJzD>9VI z{2FoM9nVv6RS?(vRnP0KjAM>(lU`ixQM>(u!ccTN1LxsZX)+4|4wl6P;gywJZ>^4I zkVW>$yy2mde#7B5hCugND#tANHwHKL(KzBVmBB5O@qkikwV2FR4iw)$S??VdA=5jNO?xD9SYuw*X|{SdEBrpHpq<(I(33gcY=(G;V#ScG`(=%H~s0+W@DZhc$PMd(Fl})mQ;8!|El{m9Ee%NrpUR)PRRK8Cl7dCdPb2h^Q?Jxd6d%y zkH+;i80PV3z664HoS<)* z!c2f?2Xts8=5CuBH)NUh)D-PhFDhL~6f2P!aF%JsyBx#SeE^xlSTtkM%jOFZ^}%4} zt+9HNtXw)IPJe_iT->iU5Qppm)oUjZ+L9 zu1+FohJ>rmB*eJjTgC#`Lgmwt5Ek1v4Vp5 zWexZ{R(LM!&;tH?zMua^3-s0D$lzp*ahR|($gk7%Wl!ix8QeWr_M?(`|Cr-t(mCIP zo||VyvMHi%1BwXm%~`#WJEbJ>!)W&G6hkzw+Dt>EKH()d5ePw^oek zv?Q=b|Kv53`-VzzhuINivdQ%Wrh{=6#*rZFvH>hN79b{5M*N%ud-*hlCT^~g#S}ov zF2bOw(ImXBcd*oeh3JZQCr{1(1a5mYiG+tQg3V{VH7mzVOD<*#ewzQZ<&qI*0By|{ z^$*o+#QLAAR<(%UHo(F9)L{5c^NRkISG{ZDqA|%Y<6Ee~^^1v=qeM>B0AlJVjIIU}l?s?HD<$$e7$F0#@ z9;`DP73uS{Cm(NwZKR2uYZh1E;BCb~5b=D~zI3o* zWqiKU&-?g&wp#Cz5RJlkG?p`T*(IyEvuPlP+u5LpztCMa#xHXe_T{8M!`qWJA1iWM zCQPB9T(3vgsuEYQI1^)vwk<9bo7CZvfhHHsw);p~TjW{$U-m!`88H@Gaj0 zWI7|YH0m4h$dAcL3?ob!tdLE7{Xnm(!QC}g#a4zDNHRrRBk-_^fKk32b2+sH2skA;u*E2DQ zb6|jNxO&(1;m|6Dgk%#SlfF4Ni|AroyzF-4M&)e#5OcbtnR@@sNilTeYtfqaW-BMM zDal?_Nj%3x7s3rldyr=v$IYF-uB@W0BltHDL#rZt{WRC!{Kq``-zD)ES^4vv01Ed% z+%n2fZwNRp>nNlxlx}i3JLQ;1uch2PV+ubmcUa(N+F?FF&Fybzwr_Rr{mA;C-y0kj zYJuq=ZrwXBlIeoibI*i(50}@yJN~b++3!f;&^iLQ+hq*%pX+cQ{_LsRyPoIHSP$RS z%G*z7tnPbn$M^E5(g{Nin>XIn3{9hg%Q}nnU8GH(9BUR-Ak!SqS{;(ZcX|NB`?Rsu? ze{Z#CZ43+BQiP%kZEvKeW+b;gFuqW|$2^7!Zk`Y1sxFEvY^PyP=xq`AY8F?O%QqMn z$jO`J5K>>5K!@%E(xwri@zcvOoi@(mhTD$A3mna(!3OrsBFJ`OB z{qb={OG;c=6^-!>Y-V8l^o6~C|s z)6tgdLtJv?jpED7QbyvDXe`-?1ms^HtT;9Jv#yY4V4zJI*NySJHFK;m2;I0Aq0`I# zrIkuax3RSebRJLu#o|m6l^q6YSRODRsEw&?9UljSxcCc{s2Gc(mSsYUH1d{?LyOt` zmd`Nm#K~aAS-0&;SRgK4%1M7Qw`baMyNRv-hP-t4<2;!YT+FOxjy?Or=)uvpwH`-4 zJH^USz#=8)VH(Lp$O7fsgT{uCHQ+snk=-v1d0rp9hKhw$x*GE|MPi-P2G}%AAc$r0 zn;y6%N%w~mwusTCTTt!6kUmkoW`t7DF%!q#?!t3{UDui18tR91sIhtrg*wSL+dND6 z@@saXg7@1vrY&f_VtG1OTryNi`2hP!?QZtL%vvXNJ|Df0j|ML~XER!rA91JybsE%K zxWIy8DXnt|y1BWKm?8K~}AW ztfmELZtLjyqk@e-bO0nq!`KCk=H09O9>K#>+*X2=I*<24G{z&bXRgFcx1NtMQj*Dn zxq~G@pw+b0-AmTv#KHyXDVK37oqLe0Q=YtwLc+yuDD({Ksi|@h6LLMsm8FtD=OIt* zPP2L9eD=~?dPHcL{|3gg|rL)pVi)qYBv4@FojpV1CN1szs|jX61;8JAU_8j zhaY}*Fj{NQD{fJ6A9-&t`3$5q3Dc=xG~9xk=v5lw(=rW*DMtzWy)H|}#1s7$V)eIE zP9EF{zqZ{B6Q5dSXDVO;;TE0dKT>!E7v8wX@)z&}h#jgt`t3Rcvmw|JMA`K-36S=K z8(tbAUgT<5ripSP^03g_K+3BzC6{5vj5cMo7wGEB z`0C4)QZ5WQ-V12>ti~%vj8g5bo2{7>FD1o{(Qk(Et)zaO8}1&U-R`U|c4UQcSd}%f zne|>FL^!(i7<9JrNfmmm<%By&s2JsXg-mM~KsVVEtalW2-lZi|@+oiJ0?S|kM0_PQ zOjNKIWz@%KPG{bUOJJ$05;0A{o8pHNRoRbAoAu&8aTPfFL>4ewA9z1@Hj^pnYfh)@ z$KZ3G&N0c6%*!(=%I5j%oous2uzm|3%Hv*LIb(mMZeO7vC6QWh45tJ;W!Dpd*SamQ z+GS9sp5o&!suroH;lQTTJqgrCU3PeC`uOO-X0ZE5?)n#}YQ>I1>hYT0$Nmv?&_B|s zzpd_fS=B_%Nr5AK$loB7jLs>~uI%2;$eg-&c)QzR2on!beYs#bBynktfy+~ZPo+0? zb*!Gp^!@(8D)?BHo%?JC7PO$=X!nQ*dobX0MD15TATP+eTy9(JV@AJkbV2{6*)y>Y zG4lhLMC5WMT1Ku~@Sv%pla*Ar2Dd%_IP-p-{hLs862Zj2LJR3}_$U9JkN-A93nGj3 z;H`xT^Vau~b%@1KACxF6Yloy%)AK-@na4P!=a)i!1IQcmS&D5@!L0jM|2fV0!^BNO zdVPhI$YX=7(wEGeADqx@t~%%LF=D=5{$BJ1;M?Vs-%1o6`BCwIcIRB+*gJgGC2r#e zVJZ5Pqw(~zTU;Uwsq@z(-eoT`MSrxtA+-6bT>rOihlu>cH%?4^Zs;q`X~@CLJiVI_ zUn-4#x>YqDfH?JrDm7hfMx;fI=(~}Ox`)t${rNgzh>$gcf0NV>Qr0CJuwURNTH|aX zLL%*vYc4V7Nb(1egpVZ}iwN1KPQuaXvfU4Kzi`34_zMPCdYDUx@U(pG6F6k2o(I3)bx)x^8Z(N^&mQ1@y7d2s*6ql3=+CFjYmFAHzlqL zYsVq_ilqPzgzVT*-$m%EP3-a+jdw?Ww|e>y{BETxbAB;ee16|05ojO(p8XG{{<;W( zA5;!Ms2qM!IsBk<_#0i#zlzG?&zpg_j8pE*mghj2{Ppb3xM^ot&m50orn=u1y_IA; zxpafHyuFK+pD{fWS)?)~hv^n9r=3-g$JNA7-0pqC?Vl==dUX&}EQ%vJjmYBQ_)hpa5SCp z&0ZLOJ$cr-9NRkE{7K?+%$}y@)q(Fpvkm=+M$GbhgYuQ;xzHE2{_Hou6MY{0Kyw5E zGqc}rd+R*+)4j6yTOPe-BRuFu{!~8`dDazAYZ3t}y0Q9r7P}inUFWJGXYcopYzgd2A+xzkvhoUhpL*F;`!e~!3 zv>^5htkNXuY~3y=TOD=184RD^Y#Z8NN~)-IyIy1FRWChk8XNZIR6`Fv+$WEV2|pd< z`eJz6Jl23pxsjzAH98-%uDH$p{s2}Gl3&MKTFFAewv(Y+=T*RLOV4Yh{gmkl^{uqRJOY@ zD^6Sb@n$VMtlsOu>ckHHJ>dtpk9HMYv|`&z!=X-mrI}ElR_Pg8idwPQwa1i1fCSUV7Tff&S%rY0iCRWqK+4vi6a zEWZ-XHbKovrmCf{Pp-EHnz+vWyz|Td@dVG3m4>o*?^@a74JoCrG22IM@$l{RgA~W_ zX9s3#4TWK?z^t17z$xz>a}3&X?9M?fH8vK)yJTtm>D}g&+-(hFzcfbY!B?7rsDY)c zC;=tZL;4lt8T*oHm1Y@P$yE=JX_C?c0ex{*0R!5qW2d6Wb3Nsm! zV&$>wi^fqh07V{`Z2M{LXx;_Iw=`UQ9hV02tnr=quu1PpBL->)bx4x_0=uWY47Y8Z zfI8PVpG@tCBejvb9H*${cF6z4dC$uyL_kGn;Ntpn`Gy}9_FB}|nx#aIXI!Se%ojeq z@ugWOfvaU4pZJnFWG!mnW$z}Q!qN%-icYNk_4Lm7%{uAVy#M+lZgV_r1aLFGBv_&H zSKKCtA~6P&c3&Kso3D=a-UXe>ff;>JOEKN;_$Bgm6l*6sj+O~5c$o;)#bKgY>ubjDT9G>7#JZR}dLy9D zxe=jq{;e;SUuk^96B{tdD~)8GAPMJJ4!cMhe!O}}71v$IE^R9l$%}GYoY0;!FGE$l z&HrNWJ)oM({(bRSag-t=Me0aZs`Mr>4o$k05IWK&5K5?0>{5cj01^mQL0UqG03j5S zP6$W|HGuRQdT+isGtS)k-{1ef_j~ugb?2`2JFJy_IXP#abN1PLpMCcJ6i`8cqHWZT z*zDH8JlIi|8-16$y3xcJV(J{<&pAs=`1|;0=RpyXcgVU|j78{Dl@N`gB=%K%FN-Mc z!!i|70bkjgNjWBGQEC>Ht3>|&fx73{ObEFMu6Vj+sP_R~B2F|*sJDn}Kny3Jw|c+d zTi4AL%Y!$s&k|Q=u7Je0ve9wXv?~>F9AH73iLra+0-`x%7vk?JoD*G`+)2 z2o|Z|Gukr(Dirz##L$DJib^b(vE1|fZ+N@;r;$$M!uy}h`8)TZF2QUrHusrexk=>N z;MEiCrk0qDs@k19csi;T!{JsO-Y{rf~i->a*RDZ4gh$EsjUcyBz`M zwcJr)oDkQC=@BGykSmGM4V|s8jZt$d3(^8v3QcvJo8bynOI4;&Tt;63x5C@1fk0^| zY{2})QsyLX*PTVs1uU43T@OfqO+ZO z?t#LM49!<6mcmd6j5to@_UlQxPDUMw#s)VOXlo^UoXtP<0S;0!m8evsDXXdnzW&4i7=X@NRwm zYehfHihTtj`;)k|Bgxu!7Z1Z)hA7&9K)dmm>YDM;+?q z^2}MK_CB};R`O%IZ~^WrxLll+3$!W=a@y>P3jU;JL~`{pIFc?|z<{m30!q4{YA`e_ z?I~IxxlVv7sZKpM22UW;3pIM3PA&DRPY$KflvZ6)4Z5ZBbtyq9aK45a(_4iRjh`GRDJ5G9NoUAW9gZu88rtIWhVH}v@_;`ri`vC6r>f6?~)B& zZN@tF@why?6T?kBU7k2N=y4jPo6g*$a0mrnjlT;|-K}UE+;Ds1214tLf%&&MeANW*TN0^0r7p{%VT0Jaagfadr#ByzeRy8TNGm(m@!jHhcM6kh=q^(!{%%zZ8cU7h2_k(WCU zefy2tSw2J$bMlMdtwF@f109ij*5#BxRV>q6yfxOmK?f(k^VLP1&NGu41UNk0Te#B6 zHpSHX(UY6L9AzUtC9*yJ+=fPNdG*!M`B?khn$zH3%Yrwp`?Rjg@_) z$pEZv^l<8m&L`;&F)eWi_z%p8nL;waSWk(p-wL7{6A&8xscs0hfG$V0v@IRl3J;W8z-=->KO|9;&ga&J=;5yBC3dJynOr`(jmPQ;ZTFaYdu?iyX`kh07k>HmkfFYQLH2JHC}$R z?C5R;<7x0TPXc(Lt@xEmxHEVlR^vcUC2U-jt^R2#PH2qtq2s$2l;|uy!_D#$j`ORz z7?s$j-Ar#Zqmo0t;+}uIcJ6PFG2^`5G3kzPUGj8DUrPf+&U?8L_^YBJ%L#o_PghMs8-EdB*J9%}oTyF>@v{42DWtRK*o>6)Ls# z?^RR%GTlD1Pr6DF+}WX3S!B_kbgduST?;v9GVJb%J)hN&l6X%9Cmbb@d;uy50-=ora*+Wb&_t28TGev z3V?XV5g9qM>9J^$OO!fZ?=TOKm9zq$FjZPDeTb^qAZuFlmaxRp9_(EvY1K>2=^Qw! zgNIiI8%#plJwEN1PQqNpJKt3GjJvRiacgyRyD#c2$GYR`0RJ2r>$C#xNBn`d+9BMJ-bO;IuVa}%}ZN%py6<7H%|n3o5<-#0KqU^2bf z_!LBCX(TGA^4yvoj*7`e#m>(evW^T0mD*{IU&0Vk{7b&3#={3H>tc_lf|V`iBzaU+ z0yt%J!?B-mb(vM3-qpT|O+&s~#-EWT3TzN>ZxD+@*O!hwv(~N7CY_buT8pLdq&0bx z)_H<4>FA;gQO7^qJi}OnHAXvZ>AI^a%4Uq1S=#Q^ozk?M>hUqo1cG}eNUL?vwW^O` zv1asESgnCsScAE3rhgXb`lsAk|VFx6F%KOE%RG|D`4V>@Ll0B)V-_terI+#l56Vm}GlW zHCdZnT{>v4P=|u|l||(Ql#Bp}`5jYdMGFdS&E#jX01^YPWvqB7dtnjj@PaWj`7%~{ z;(c30oUx?NLdxR!xZTl{k>)s#m6P%+M^m?Ab#px-2lI!qGVk2-lL%fTh+2z+Rl_Iu zku^u^r)}@VhNUTOL++jqmwnJg?EMA)`iTugIO_3tG0xd}c@CtNekEN7g!e_YB_l4$6{T zQb;Zue3bzgIqs_dYT~Shy0taG9P+l8_O#VeoZ#M;n|3K) zTmF&B1l7IBLSLOcn$&sUzmZS?=#0H~L#@0EfZ!>9r~WmHQCOqMt2h34zPSdd^Yp}z?ym|T=x zmy7*e!XWK0(KTMNV_wRVnidt+L;qATvQL9YnK-bh>xCd5WIHOW$51MY{s!r9K*C zci8gyk;2#1S#0y5=;EbF>oJE~kguuR+Vsm#dirb|vGlmO7%_ zb~F3##>>70gr+Q!YLiD#=Q4iJ_+#GqO4u zVB*YiOw6>`neT1M;7eQuEDDNtHQVRAzOnMH6<01}8XAeT(=X6z3qx?SK9kcq??c%m z=o*ih&myVn$qth& z!zp^PYh_FMy5|sNn+AR>Wq`|K1`1R|U<@T!4b$sPICRf#X1<8(mLh%frS#-mMKo8r ztU0bP>sn}|gL1g7ClpHYtmCcK``zUp^_LXVR+#aivd*XID7+mjqbm$~;#1wzugskv z(`hmTGzzqu_uSwVjSj5baABv@2T8w~FTO z;3RtqAbS|bjYqTco#KO|_ZC7nU#3gJI7rCusAI4D9kgptbVU zm3!G^818g7XVnAGtVm8pm(pbn{bjlA_d@Qn7qs98hBA!tbDbF(} z<%svqd*$04V%sDm{1u=YmcJ=$H!I=Zt=z|BtbmG3oKaQnC6C4$`ff>b@l$BR!t#EM zeLQ1-r-=|a6Vo`Q8{)nax>pje1(}J6CoN61B;Ky6Io1&s43V@()$kB%sHn+srf*TNqOWEEjhF~yc=RuzUIYpSb7nx(R{CxBF9*t_*1LHC>VT7YUh>6(W9svy1V>zeKF;&bUUYwD&A2X=zeJztN7#O=%1B{Wa^({BF<3aCwizxqas70reNa$3HUlT-NXQeDtL8 zzHe5C52XMA)LlLMJV8@1jz@LQNM_c{zwW8f6(?rHcb97l-N{f)D5i!C$sKsR@UK<> z?K#1Sn|qx+=C-1-G5&pj-UI3aGr?VX8z7t;oUDtx6M02GqL!`>~%yuIn%@r`09pp=@ zJ`03;!JHY9^HFhfzt%UOK3 zj&K!w&4_AUF)4e}aeI^Ol$-l{tcF zidN@GN=LMd5$ad!7+4uL>Uw>Po=KIIvFv1wMy|iKr1UmqiKbmGCXx_}jiDYZz#m*qkdW{8lW=?RBBV{52g@D)Ac1%vh2=*{E_OIMX;7#|F2F=%zp(d-_-O+ENpr1dm59n)?kyF5yo&Q(p?EQOVThU_Iy-%C0(mp~s(`iWYpyh&5AOry4UJ_+k`VC9 z49>#)(e2Xqk5ZHxv|j05ywb&4zCy<^3B#vZA}1_$z|9(*c(<6guv~(ga6wx?0mb)7 z)i`5|bb5*Py6_Qo)fCY)+Vi?eC0M4;2)^u*4bHktsV-2JTjm1PC;xe)<>8XdVZ?#% zF4qvzJ)mg(;H7ny8dY$T=95mMGsFklgps{HUKlu=Iu)nFWAdDZ+X@vvc;pItLfo{OvoNq}VH zB9O0X?XwY4>dgXP^JHEhbS?W4HeN4!(O+r3tK4-9?ZEK)j@J4c?=cwcD}Ylo;o^hE zp7)F}psJeR?XH=2v8Nm1EK~Ub7B?&I*$;t;T_YE18vV127)%z~&c8+b#5Id&xU639 zGD%wKVaZ}G^tF!4wzk5jY3)fBOPCvC2hHvnuE!_9Sh>a@K`~Cm|iH56hlg z^;N`uJhY4=(J)M|c9EAF~6Jeu{U?dwyCX=T%tladAMTfuT8;1H`tL!L^u=?5ec zCi4xw3oLU?!(Raxv}TyD!5bu175SR-vnmwfGLxXJ261&cr|dLwDXgsYnx(1m2ems4t`Ape$D%4MWc3_yx>Ran=~1{`Jc15}zQTD-R=Ies)cGMM1%; z{=&7t|LQ1MR5fYz`u4H_ZNaQgx+a%BNRkyfcpr&6wl`c9%3I+~cdfbk5tH7 zmC8mMt+LOz!H9d3M+p+D+?_OU>A+Ry=Uv%f+SmLjBQtfE&N2I4$f!O{a9}7W$J8X! zo!|9PpM?{XJ8N963bdZ)On=96rraiwy5&yr#UE;ntk>U2?e`Kqkjas;W^VpHmRWVp z^enL%{9O9<;>t6DtDj`-rfU)XE0tGmNvXlhMg~&DhJk8Eiq!%3?cz9?3J9f7>*WFr z65LC-c9nOKFsB*jW&Buim?z+z{}z%xYml9weN5<865h?h;K=0?iih&@XWVE5e0ugr z-|+7l{J-v%|9RE_yumfV5NmZS-$)Nt1yt409>Z}Lwu>+5#xIGgLy9hxbVLMb_hlqG z=zW%d*oMbPG`w&+R>3iqOg3V#_eo!*8Lhl<^j8;v!uAjT{unI{FHQ~*^H1OX5e!R2(=gtabFI5I z*LCLQE~(+kNwD$L2fp6}*Ch4Oh_|W#%7A~BfPYni|9u)2$oSuhv)m>9+A?0}=S|AabAIQ?5UFFJ z0SBjEdS-orz1gW71WY$dFZLDKtS;(0{U~xa+O}Q*dAzp520Jc9ZPgcF`ItFW)Ui}A ze?;;7FK5?p?f$lT!|%TUYA!4MzWM*Ogz_=SrLyI?T~zD^OKZAoV2^A@I)T|e9cPl9?K0D`C&}hj&*dk$|iGvm(=K~*P8XI`ssN` z>Zcl?i(A-Ogt@8}yYWssbGCKsfI!xt{?j>q4i(i3I@3TP&fB3dO{l}0-}%Kbsqb~; z7=7qLKF*nM^y0c{>rKUG1+v4)(bLJxWWVQKvX{tza1v+c2Ma;W90|KO7ubUv*I!fM zj;_+&lYOT9@ITn;zw@VmAop!m9+SmObUx|pF);r1%aE#|dy= zQiz@r3?xGJlOkQgH)d5}Y4m6LZbsU>G0XNztOWzj_`>$bJpt6AgvacsR~s1bx`4Qi zDD603)Akv4l0=(QU1luX?4^aGK+X5`f#7QMqCDNC><}2lKvMn)dvY>7-*1IN>_)yA zJ>c9RgGUw6x^A;j&U)x6q+ax;%^MxdS8f|b$eN3jodq!sfo{u-&B%G0Gp(ZNYTMnF zVzRlRqN3fT>N^-z3ru*CdEK|>oon#h&!ZbW^R#&1Oqf505`DR)L@5vm~adklkvk?UZkUMDlZqipm?Y+6DBXl)okJF6BO01+_NjQx%}3^;gLv5(!`$hF9GlMY+zDDAPkGGhVPyoT2cdgrFz3C(i%A{FY^;`cer12H z9V}QQE(I}%a7uq1I5rJ2nsgQwzEVf9yz^R^B`@PS*qJXY@+N3e6^h1pOA--|wFEO;dO$6+FPg!>N*&9x|o+PH#={bve*Z+6F%%A_7d$?tjdoI zK0L2j*QIfgI`8)5iw1V?>H+P!dphDis#lniWOvuX6OwhRY-OtGDz*Lojj9*sba!TZ zle}RW@auio^DSc72lgtFj}P|Pd{^q?GLM%l>&$@aup?&HFz+S?ij_%ahj?x1M z!ktIYQ+w&hV%iRX1^u~AYDf`9bAohYgjpd}k_`I6aPkFB7z*Le5*`MsvRyFh-wtmj za-0_qr0;bcbj(Hg$)(kcdUqZ_)@On)o4HqRMr=pt#k~;i>=JH!v}{?YRofLC!YEk= z)NW&%M`r4GK{=h^ft83EEKRlYgYwndA%iSN;hQe3(cPImHUn#+Pj;nxRPPTg<^*^B zFuCygfH^mOL)dvnO#5-EK-O0P6R}Xcu%fqDk5k@g3n@#6GFbHKfi&jvWK>?n1-myg zPE@<*{$hSQlbG;qbYTNX%j#B6@qo$*cz8DI7IEIOd4MG3>zuZ7+PwC5gDoUTG!Tg< zD9Z9>LnXUjlWnClgjaeP>p#2JSO&ktkXSD2KOjg=4lUl!o6QH$#VWmV3p_d=o=e>^ zyf!j_&{F#qFi^S8?Ax=oB`*5qjVCDNc=1hVjf|}mO&~7&lVrS!W)ZE}{cEtK_qWrT zG5*ca#qn^ssh`HgB46nj@DUzXy;)}X!2GsCysCLhe_#!RJ%Ox~y-%PDEt6*xzB#Ng z0I$xwm22)bjpPO5w3yq77?H;y!DtRhykt4BpTXb8L743YL*4UVggrYD?g2%?ezUXP z^+}tJS$3UWDQ^@i<;m##La(oYL*lGW;|p=V(wdPrmR{^dt(VS_ZK?blI$^UWv6>xA zZ9Ru6H6%wD;@0E7RIj^1>LDxp(i}D;?T}7bkGHMh?qPQ3=(z-LsQWq)Z_T+E`DH(Z zvOQmP)Td5*aCY;!0MWByLxN{s>_NM(GVL}Sy5zk^xqR?iwiwLuD0axpt-YO`MA&~j zIP{_-Z91qh@52i@GYwv7BPFDFKwNQFY`;|Daex{t(X1*kLtYX#ih|F}vlru8>}@eMjMZN_Sy;YBMU29<7qJO(xZaQk&=H6~5h*F=eyY&W_ASd^2jc z3X4*)vN!Z(iP_O@bQ{&Hu>rSN6xfxRh>!}%#@y>C%Ja|rN0ULq?Yi8FwZrq{B23GvJH93D`?Ia7iFvb@zx1WEwgO!(Wa!Y9Xz(khoRj>A<<0c~2)G@8zjH^TWO6OyA`{zos*6KQG!zfSY0sR#n?F z(J7)L7se^qNuQGGCsvOvnkqW9yxbB5UZ$(7^0=5CFffVrwluR?#blqznRnM-LBda% zG;@vLJCk*cx81V7>lo8?Fdy@M9EdxM2SP^K+NH|q<&ie6cPnP%qpP^QH0Z0nxqv4t zXlHAubZsnw1CGko+=Oc!ac{e09RN83yV1t#VnVU*_QvB;f7R zn+%c8WQ3C}m(GE++v7E5`^r_Tm8~_#H$Gj5Sd*D!=yX@7(Pm+rb9Yos@4Z7I%vhNTt{dElOy+q`fztVQ`T2Pr z`mT~E)AFdt7RO~Ju_Cvo*;<^2eYAI+(nS+=K5|?j@jWcx?_;nistnE@FuCF5$Q#`2 zSRM7=x%d8b2)7@@ke#KF+G4lX;~u{xLKsOactC9=F*m$5ebrborr3u(@`6i$s@tB0b^lPaMYkR)$`Z$ZWUx!_vM{rfALam?eS zZR3su6f(~^ZK4RK*3A3XiqwHAdiuDACc18xE9I18H&ki3V5HU(D|GC+`-7-k?72wD zV5Wf!PC<4J_hF`CWA|E>k#R#h9xYP)FbEIu znn=(n4D=p7-96B|dWv6~lD`qVG#zYlT)PTAJ@ZNWa=`Eg{h^aqK)z#|fa2UGg1)yy zA4b}*4*IzZaQ<9()sw654xZYwKc6QmU3w5yUjo>sOFTdKkGr1#?Q8!LRS>ITrqXNt zUq==634+9kdFek9FFwocBq%OOZ84IUO#kH5u>R>-4w+=@N`6=V=g}w!pnu!ppYyDk z*h&xh3fNRy=epx{Xchhy5S?&3bfhFn?v{Ho9{d6bNcrjiC%akrY>$VEP8Z(}wEE}7 zQUzH=N1FqqKF{jtSb1B$U6NzH;{cwsYkfLqr5guZZ(;ue@bhvS5?~zR2tlq_aBG;l zCh5vut}-;tn_0wV)?*k`1N;I8Ko1L226wGk`m11*bgbp&pFftM3yqBM9av04drv30 z*xz#@(NmYdS(FKDE6>+4b?Uu#qmRB?SpLRo_eMP1h0$w+R{T{6HwiDv!YLvNT01;T z%$S{+C))0-UzNW+rFjKq+Op{^DT|oRL&Bo;5JsD4M#Sz|iF^eJ^>rjJin<*qTjdRL zR&OaRSr}Tp^}4^9`pET~l#W)dNwjJ^onmx)o#F7nowCu@#F)5lp2*5uRn?kZOHO^9 z;CA{^0w@rq2;Zw@{>&I?>`%Cq=`|XQ zoEwzH9uVv0yxq4vgNC3XxC?cC*1*6jfgL@Mc>5>`_s0d1AQ3X$FZ#)fAe)_9V<>x# zZ4$O9ezQKSk1km}B%6Kr9DG)R%CHGqB24`AG#&u$Z~8e%uQ{ z5{noP!w??ersDyrTE;hcme?bDi@2wLowk9h*j{VTtlqj1E5+Eyv_0)v=6MnsUi_R} z@1xG$Fq5K8(T||)T8{E^iIDPa*6RC2C>|}5%qV<+Y%A9>hJ4jjIV@ z`HvE-RA=MU+Ctiy`S_UDR$en-+|}?ct;^Br2-wB5GP8||UQ;+M#qFaopc>QSDU{{j zg?^hF4TEx=W(uUV*3LVEr55jpiG38>7a*3fBvP{;B zUgU--;^f8eUhA_)Et)m(Ee@dz5*ka^qN=0Us`)Gh<_e}_^uxI- zr4@tX5;xm39vvy(2qz)x?;i@gIcbO<0M3nk`D-JyisLZ^a@mz)Jc!iF!lqY zxrNXjPWG>Wi_1Vr#LEgCq>^8jw8wl`0GyNe0U6$-b3Xq=cv0jElx!CR<>%*jgu!60 z56s7u)jax!6rNrxq!}a%IHbPa54p`*HIm9zUP8EoH8equ4~)60&Z$EASCus@L+d`r zUE+C8WUqLI1>KkBqZ`UvF(54BBBu}bYmRElXW=AMKlRafD#ss!ZU%p84d2z-6DzUm z9##g1ZQxt3R}Jc$+4vZh0<}O_##kY?{PoJr1IWs|+p>n9<-11>n`@O(7Y1q=-Bl}X ztL1?e6)*V9P09xho0_N6Q9_qz2Ji>YAkHGObltPd8hRlQCxgTJ*WJfYlrFlukqWXz zc)Sg4+D4rqZf5?3JP_}L`LVeI6`XHAHq#E0Zf8j1d;V(3+QlS}+J;qgpEE!0;BqN! z^qz-?WD^3@mgHp=-qkJbIIFW17W|XH%-&fO-_)0@RpxY~wpbIlU-IdVyc{C~y;Qa1 z$tu!D>Rd7U`ugCvG&1w&x&~>9A)JCUV9Mg z>VI*97$eg$)BiN9Tyc}u6vNZmtzy|cdtPF`CrQW3S^?qQMb5I zT|5wnyV_Q_4`$z6Bn>CcW?8^$I$Y+~_eQSSoG+GtIq*3Fyhy=46?nZCW>zcfS@Tqh zG2IWIQaXkbUO4i;QMZ%Rvak2N;Iv^+vE*XgT;2;;_n^$QNuwHDPc4t2NJNRtT)Rxw zG-%5awrbb%iO5;9%Qqz8Ypq^l@z`xdo8dx<5s+9QK$V7Ac^U3w(2pt$gsKLb44~3# zabCpPqgPaJo9Td_m2Sq$Rdsm^;p?^8!Yi&ZyJsMM&=bpe&}{qm24Z2gv=2{~)<2>$ z7x;0A3OOdky+OPY!kOd4H4W7qHi&1=H+D#pR~U9k3yggBgSg$+#CZ?v=25*P_yAS9 zmP`7PvMY_lF&3l`tP3J!AKBPfBVKH1ja0H_&2OFmb@7(6_aftxL+&JxaB`Oj5RT zFJAgM5K}5CP{9?+JgZWc86&?4q}EzYul-z34DVUDX^!Hw<&?(zDpEgDvv<^PvMhe+ z#P*>ir>5TQ<=`KI!l+gBl+8Rf8?_VV=EE;t@)~w}kDs6^3jYYCYB`!3yQo9OLmuDp z_y;Ml{F(rTGNSQrqKtS{uSwsyb-}&mKZ3|2^^H4QFW+}grfIX@Xmu8JUK1I3xARBP zGMKra__*QooJTh=HRW_UTw*U_`9Z*+flJS^_T4WFUNTN`6c;^5w3q{NJGlyf2Iz%T zUUm;}*%a~imoH^VJdp~shyCnTe*`1jPw8l6EdB@}0ZQSa zDgVlVf0clLRe}G5Hoyz@X8C(1ou?l9US0o9ExjOzP3gjFOsjka>_`v1-{~qG$`|gY_Rkj zRWgy|(48q_@g3a@YHOY8zjXq#-v?Q)2v}=}r{9WLe6PW)_4<=6m#T@as*0axk>XDQ z;hksye6IYhqJIRn82eDbNjBV)TJ4JqNqP2PaJpx%oM3b|>%I-Q5Wi`f{of6@{1==E zISE#onaPjdo-E<4YG0Hd=w1xJ~%@Vo!~(*^qSPpEW;?u<~juu$Ug z_JcaW`=PL4e7Y)y?ae)@;OipKZuxR@cscdl`)&1=FPDGc^0(DL@1G6*ZOgxN0$JJD zE%qyduM4T3v(;((H3P^w^|xF8q2l}R<|9A7Hy0-F2m?)FPTT1Q4F%B4PvJSWH}@AG)e6@GkN8<)2sLNeT; zZtveu*v&f(`H=K^`}y3iZcbA@TCZQ8&v43mBxi{^*5zMG|0_fOekEkGHT8S>Z!G+? zWQ$ZqHZxlMR^nCMFn^Zq6Qh3zaa;fCTU9@23~7B8LigoJ|JD0uTB7|E-YTBb&|iKU z*5?I+{T^eg&BwC0JAEUw&umI@9pxOced@88DadI;j{1z4OAvu|(oenu*aJ#W_5%Hc zwdMp>x*s0SaL_*OJt|ITXs##3JUzA}SXY;o7=6Mwu_j-KQWzJU<+IBiY~i9kCi+Qf zCT|Jn?=jMpp4nhL5yAej;c9+j%J>z~BK73_B&kdx!!?ct!a$(fxpPW8b;LOWlM?e8 z#5be4jr^2?AAsB4GoNoraQMtW+|q7QybC;Mff0-~8!B)&o8>R+4DCH)7~MTL`9mmA z?ZdnGs5WCMD&!J-qF`Vrq+?{PbcQTu2E(7N6+459;iGJIhP(p&c<#z5?|^GE<#53= z>y#9s4(IIZmJ6ZO;O>ZykCD~_>N8fy)SkL_x-fgdKUHzO{1E7j`cO@nloi?rNg4K% zN89B1bDD;*q4x)tR1vVeA}HrWvbvgK5nHU;sMhz)9?nm!ZPUld~6&5e;U=x$$l#!N&P zGAz&-n<}(4tl{}S6)geuD9!i*v0@*ISC?$>&&_>j`tA0T)8Vll>AKo9G`IH>;&-S? z)OWDe@I*JwnmQ*{LqWWzPjA%AhLW(a<`{;A?+lN#G+k#5=veXcG+t8dyv8&CszJnL8Sjncg^VE zch~H{og+>6S~bo;s}VosFo0R6r>DMUmNy;=60mc+J}<0M>&LmOynTM3Z#7q$O%?Z$ zRN1AdZF%SDVKG#%ob-|+Ojpc3Ji!<-flXfM-Z3C*QP8Fsd?yA}m@ zObE%`=u}!-Is}!f(dRXtuYiejz^{PzMKbvEaKsJn>z3W)1Fy`r<&xOPHaL7uVq$vHN` zZM|~Qy+pj_D;Xjq z9}*+pKU+^5#Zz3wM_6x+L8L|C;rSu3P^&aUL#~*T5>O3s&@dsDd63@`8yzoHG0t2S z?Rb@B7WW9Q8TeY~nx*q@&?vVYn{L6tkPnKUXorZ+1dfzh@^4#gjwpH0J4&bKtn1Y_ z+h)P5tMR56Ci{yyt2JyQ3>|vIyR%|t54=U86EX8PBGs}b0n{{bqPik~ChSDRDYDci zOP$A-P|B@)vn0b^HNZZ4SC(J+(F=L$pBndnCcW6Hq`^c+ZV2;y+DTNR&G#;^WU+LR z_=U^SAy~>ivXoOk0xDpEn1_RjgSswpw!t5!6n;6urAS5;$G4Wqgp9t-7yghx#g!ZZ z+B6v>-X9e8av>)7JU-O+ay!y(#s(cFRt+*u-Q^_;S$pNY$W9xhjE*smiT^C%XsM&B z;$^m%GqvdZ_;w`Z%A-Vg_LH{isyip+S3V5fm9<4%WQN{d1ZTn{Iq1q%#zRo6kvsmA z8dB~4RFlC)w$Ufpr^rV5NR$BYJsfK3W;C~vEtd%)Fn+Z+JkD$-4tg{DVTDIe!Eh|Z zMk?lE2)bVZ5e@KVX20!R?iqfzJ=5TR{+d7;9y(2zg8Ibb2r9@v=_p83{g%fSxX=~9 z(B4n5`F2X9NY`q8+3iwwCE*ZZvZ|dhOX>QZ{GnP!p(?u|WvX!j7e{1cdJtVH2CSdj zPZ@4ljXyxMA}V=QyM?B>pmlFsg2aS})>a=|`*y!@(iVp;+P?U~Y$!8VlfOJaUM>w*m8{^(1w1IZ`LcWH`YVr*nwX8c~?MoB347(=Ls&Gk@BxK6yuCf5|u*fTiNFAsJQVI$Qq? zYmu$~U=IP!vg?ea@QC%Dv_YWPQRJXQ*7L1Rm&1O|7&OqFnT&1m)fm_A0S2}wrN67W zzI#x1f?S!WG`*9}msv9|WPF+3TABI-Jo#2)vwB8`eeR6WJ}#p4{mpKw+HZw$-Lh2% z`-4RwT^OXqpgOiJEXqyD8bs-~%I%VfL9gm8)SU3G9IxMQ#sb zo)_4&nwFOKX;AoZ2B&OkB?g=^pQAWm^h3&(;@V#U zgUOa~U8t-Tv8P#L>7{;G!4?@=cNWra^0?}EcBMzJBW+BJ%Z(B zr>>C^61n?5`t%R_Tis?L92iXHUH3}-FH_j@yS2N%^UC2!q3x~B`?t4$8VT@2C)wE1 zLcOCc2=snit3QWQkkBL+Fe_J;#>0S4%|Y|^{*+|ul5GhRq;}OC?-Fm$ET^LC!g!mW zIOYgHN=^3uSi|B`NVr4YyTQXw-NaA>)HgZ|M8Wwqn_)5u5!zU^`cK&8q?x&*puH#a zRwAJs>&26N;m*!Ww;#FfNbK~#L5ADKJI^FJ{k$jXhqh!c85Vh^0V~7g>cPm}X?GEKkQUjX|hHuZpzBEsHql@!3hc@xaBmKJyPJ|jE-@$T7B!uFnFk88; zal>Bs6WCzbcn7mz#g~3~cr?xeT#~U-`V~NtWoT{Skavfdu%7nu$s+GKc=vVS%ga4u z1>0R2c||VCtOpgT?OAQGl;IObKDiqrV-!_=R#L+sPwQZg{rUvkIX%kGT`^ID)cmhq zy)07VjRcxHT6vWrMTwzQsWZ8*s4zEz1RCzwa3$HxxH`V%r_%eaa@-)zxg}e4U}11O zoQxOwTxH~;{=SqXPY*dqV;3YU`H5Mp@J0q&U5`pm$*{bc_f?E}oJD_(B}&o_0=>aF zcH=8Rspi_#1H96P@fO!nfzrZDOY+n9^J!c-2d7mIr7*oo*~LNEX{S-HoUGiDi=X2Q z-xpMgve91XCc*|_AI4JOJMm>ie zyw5gvYRT{xa6m{3-yh@>w@8p^^!HvLC(nZVOIY7I?IihO2^4G^hEK5lpB2;KqgTLO zQ$zFEG*D-WBr|ajY_Z3{RSLI$pG?~XqO>k~b1oR$55*Sr(Tpx5mn0-pTzd`G_OM#| z%m_>ehLe*wFoyE;;9ZudJ%bME$0#rv4&UD?`XAIKo*GriWK19#j@&hO zDz_yYD*7~@ly>eG0L}u|J>BGsznI*ft0US~sy>*t*NrLk%~HV|VpQ)k3t_*oyds4N z>Z=T5pt|x#ItRqC*Ehb%DNFd!ikYMh+XiqY?FvJnB4QbPUEiidX_(`fhEXMwj_fZm zu~v_!HF7?vVU1jkz!}KdgpJY8+I59r1@`Vq21`n8+>_MS*AZTr4(wt53J7OR$qDxq zs=Q>%@3t3^rs0;|*MfiF-Iv$aPbAAHuc{DJ*A?y&{huvB%-C($Z3Ezd_Rx>E?tyhpi&h!9ls8~ZMHOOpfUV^9iSds+` zv$xcx%`LR{{cYzvW8;&$!;26f^c^Jyu}0lJG}I!2M+KL9wl;0VBz-~&)w~q-JxJs6 z<)}8lseuwgd&lrEPV+tQbPK+L^m==?wD9{je{7tXdyb!!6@IdHSYws@J;;mh2IpsG zyeI@ab7iF~@`I##>xF~fA}<0*wdjuKcl>J&Lm}TI2DvKAoQfzu>#qRQ6DtuTv?_Xe zMweVs=)HS(KU%a|w_U0kvA^sQdSHSFw-ZGTvV% z2VfHfJqq05Whu9%7|ou&u3P=N*`U01^<+~4Vo6O*9*td99>0l1!D5m=_?oV&==$tG zOI-J;=w7d*fKT(1fv<0rcsktlEsiWHjc`CRT5W>ISW*%ju;%Mh8=$9LirF#%@_T=< z`_0&xMnUT7>(Mui4CDjy@|!%Co46JtY*FHnz}UJtm?Jdo6tngh&*LHX09UdZdL@&v zptkfQZKL8%pE@2&3ydvJSR|T^MK&+(<|LD)Xv|6+l8`-t75d{4pzGAqEw}3{X+I>V z1z5B6n9bnnpgI1eVe7V~dMWu|4V6EdRT+k1Xu;jYNAHdASLV2}BHmOCD zwm|p8&l`VnVt(#VC|frfnl=8>XA_bzdE2cpA=*c4Wt~4ijX67|tozAhueEt8zF%FO zhgOPae5{jMnv;@b{ug_19oJU2?hn)H)M*(?ks{?3EnXzJOb3EXfrQ|+I0>2{Ejl$M z5EvYS6$_9+a0%KLmq3Bw?#11uZ{|$r+%xCgd*6HRd(Qj2pZC4_YwaX^KhIiw?X{ov z$oI?Vz?q%ytB#lLmonh!7|@FO&{!uU(xdc~-uk&c#% z8jWS(b#(AxaTt^hkQ*Ev91_LGxL?a4{^Pjv>W*XU4S|`y?Nx&cs7Jv4RPX+(@wLO1 zrJew(w1Yo0O^5$>{aRqipT;uK!4jj2ltWvJOmqG*{86-J&n@z4O};cBQ^BqAxoh%s|ZEx!OKx1P_q! zC$Hl{E)Pc{VJUk96^z$BoX@HG>)Y9~O~=_t=V)h*Ew#L*b^D+P^t<9+!Se`@mOXPJj-{bOjqe0?Wv$7_*ato&NId20fE0 zP4|R+YYhSn;jgqKPRb>TUv89yAG_BXyRKt~URE!gylX)QK7#Uzojwp(E@$B~!W+lJ zh5q_B7RmD4m6%eMy?h=|H{GRxBAp8I*No(-xt^vM*C?%+-epvOr3ey` z=vb8<$BgO2(~%kKJP7FaFNAe4YNR+ck09ve^3&bfaoV8b^OV74o#z#nGWw|_bb*^H zETEeQr}XxbWT2>8g0sLMR2R#+agJQS#%k!gZrd)ru6?QL1>72)I@z9u9C1U5;hyE9 z5-~BK6K#hiI1ylMDB5?uOyCXKm<8HnKGZ0sU3w;aSwIs^ukyU5?HH>Yp@Pvuq$QQJ z$p~nY?Q$%^y21N5ZuQ@0e5tPJ73rDID-Z594%Tp11CUcr*~oA-nc%Egxygz~J`4Kb z)gKmuMEXA-zk2b<%heO4_6>HcN}s zA#h?8?%MxXaJ>J3B73~*3` zE?o_^;AH`N9H%1Kx@wHISvq#anOF#du;Dir=+k6Ro70o~Eq%Z?7x?zu?s{ep?2e^< z0H3N+GB~P5gr4tGu7j+QE-m+sy7xU|C?rDxXwx^s6g`;bO2$TgNlp32#^g!uy2!3? zbt|RVu=ouF^f*35&pV#WI19pi`XIjn73bH1RPMS8uqf+dhZdm2D``e~c+=q}s$2!A z2&?E6T3Q59Hl@I-eJid5Zc zi$}k6J+EOLdODI8$T4dsS4|th>!@cPd?@9`9t)OW!*bb&rtXWR;GKS2exrZ2(<&#B z5RYB_>5Gkv2Ol%apd5#G01w2$QjL(xeOF`G-$Xp-l5HGZb6`ou_Xl&&7imCMGWs>K zBpAfN2;;1@3ha>^p%?b=74}=T2M_UP%Y3 z$_z_M7yGd%qnv=DyGCEuDL*~d;1_;M!u4S>;yLaGaNFQDLTS6`#NZ=mU%3#01pIL4 zyNn9n6K;a>$c*VIHZ&@InbGHU#BtL5uM}k@e{|AY?PYSR;y)$&|3IL0RQRR;fLvyC zg^#k3&}fdS!We%K^fLv!qHAtAPu7RXt`>Rv3HQ1ZHa;*eQw(H&pllMh#nm<2NY8^S zpkriwG3}EgCnl`OF5zrI2o6p4ZJ#o0Vfj=aE)(hscD9s)wXghnlU##o3X(z0GD zNYm3EVRg>KD&u`k2VD9Z9>gjtULR4Gtb5i?St&E`7@h(1GfDhuv<#7H7^jNPi{cej zU22ch=nnKKt;?8mbaAT~=V~N*uCWTp=iwIo6Ro|Kv3L~rtQSoN+B0ABw;uDY&@-x_ z^3O=m36F}p{@(iOPb8ROvae6%5BrZjD2D{z89!RC1c9D^nT-&b=+En=>CR%Xp&N7? z>|5OsnZ6}g%n#D%=P~nwWRRpJ4j&;Co!jAGKtz4nupAp7lj!^HUS(7ECN@o;9_B`x z;oShJp!xy!F!JSLp|l_@5xHgDG+6DFLN1EH1|pYSaEZJQIo>L4wuH=B`nv8kPEG$t z#<}&Xb*`o7aGf7393C-&?;hUO{E}&VIVL`bV zLle2)9@q+)MmAEnJ&H^cXHz3a)M2%Ptb}QNlA#~x6WB4vu{x?QWk!z1@i3IFdD~5C zX_>~?Cq8Js;%j0bYS&$&1-Zg9h{S-=Xs;|65SYjkwVU(OTWnL77&T>IEOaB^&9zdl zq__R2MHP0r&^`I9e9_so#<;YA$Ktm$9VDdB36ooKDt2tnoD&^G^u|sW(-|HhSgN7GlGVjPnYIu~O7>NJK`0Q+>%Tp<6ytXUHl{7A@@drMXfpbIWw77VxhG>aZ>EHAp=$Ccy@bOpyUxeGuKRXEQ*;m1o*c>&6s;b zG`+;0hzL)6-R0i76K3J!8grg1{YDkdF&o*l@IX20%+!`;B-au_6r`ki$ldhpmvvT{ z$5Fs0KIyhmlBFR*XWB-Mh3#Jd4 z?c;zJHX-LN9+s^!sWz&WcJTF0@UT}lE49ki!aRg^Lz2A_Jq8YOslIKpe3;QBqhjfC zx95SJd9U)JeS>U-;`Wx}2&*z27_LIDen?ylC0W7OH5`TcJrbE$1f6k9<~EIQ7&N| z!AV!kxzyq(R~5Ft=&r2|F)Nmt-gj4Kq82zu4D!@ zv3mgrd0#11$L1;iijMw2dOcrz6@jyG`8Z^hK#)Ps_Id+&$L6>C=3E z&50&lue3W>=0c_WdG75>=bQ%8TFIwz`O<849p3gP)i5*Z`6V!Da~`VbG?qVv8|>s4 zHsXkq8j;0&w<_GL_Gq6XicUpx8piQrn#;OR2DS-h zV%&mUDvYuZJSKP}ipa}__2Mp3S*U@topq$#A8I;Mq#avdL$7lT+&hCA4oc^sHipDB zlhB(>BCEhpZ*2GOdEN?(@X+d!7?PHvy&GBR9MdUivq;oWhrFj^jIbMI!foO-+*74t zmc=Y!!B{K8Ai0>#n~iHD_Nb%#qvx#!^o)LxGXOF0xwWPJiU~EKTxx*8<}Rd@-0pgD z3*BqqLg%n#?UYyy_fDkOcrY-K4)z(rtJdrHG)npx+TEk?OxX;@81a~CX;Co_@pqe@ zmVh@id{>@KPYIMrT%HL@Gt(6V-<8YL-ggeJ$<|)-43M&7AY1Hjry$y=*_yGV)O(~a zP3G=8wreGm1TuB4((s$JiRqq`Wd$wu8XNLf8y#_y@RdR-Lt6Hsefi9Z44y3?8x(I_ zwpXSFZ7a~3GjP8H68K9+%6v&g|fI-nhpVFCMSecmg0gumJWoTr09)151p}C znTo~40>oUi0bqV2%6tII#vW}U;=G$aJ3Cg_(GCupsTt>ia^~dOSjUL{0;pAvw~Hc3 z4R~#IUKo8eSa|K)9G)Qq_S?vg4+2zAw)H*>>DFpnA6clAIh**N$-%M}xjbAmP^O!) zaM2$DVSL88rV?UZYqId#M9#{H4`*PT$)*Hlk42cCj%;WjFF0izuR%2P%`}Z4?G>s+ zU0_{>Ue6oDJQ`s_^l$;FZ!7i5yAsw0JIgkx6K1Epkt(gX zt5a1i?jf5|tM{}9YSQ?Rhj zna2jt5qC1|^>0LsIPsSo7mx~j#Yi!0Z^HYd@NpTuIm#~|ymF#$|6ZSuPIY+Sa&Z8m z>t3kNNY_BE@95(qiQWxi&&@8j$a^+F#4zn1nK?XIZn*vkmukO0em22%LFQJmvHX== z<>lx+@m?fvpA0jZ7t6tLlkuaL902*H&A~o97iCzQ$uNi;4V~PeS++?Bl~wqL@*exq_vX${^}*<0o7}_lu?b8 zj_%|Nvz<;I(b+lr?6v_V@M@hY4_7y0q30Lt@upl9H)j~pz#L{WAZT+nz2BtK@Ju~ zJUeBv!C+r^vH*aEdVOllC2Yl}iBe`0UeG#0qGsErP%FQL=LH!h_v**)LpYc+QwrVT z!qNbME_}tT=>@xuq=lUeT7$MAW`Kq(o+P&s1$22G4##GAaGAE*yr;TFHw~kcNe_=Y zcS6p6?>yM#jWccD#F-8Nb=YDYDj@HcAA2a%1`cI3K0Sx`!9yT=D=KycW_c3j^zg&5 z{7dRn*6HOV4(noj>rU0znY37B2FytzOxcXpWaynrmmX2`_K!HbsG$PA;aF_pP*Vj4 z6d{k2CR7tpsTA9r3_tbwHR?S->7bJ*$MXPLA!anX1t5bAdk{#;AV=gd^j;j}O(seM z!$0*pA`^M;U#%3>Wp~f?;9Vlt$nMHfT#I=37?tzA(Fm-h56*j4uE){TYQEi+fL+`N z@1}nE{@O3+0(!$w$zoX-{Kn) zE#QnO(Tvtxin^}?9oN~k8)VxL`n~aU^OIQ5f~JcDKT9FQzceU~BHW!-03Acze)OEu z1cW}{x=G0S+DVgnabv}|RuKbM{h-(|0xmqh4jj*db=#!iilqZiN>{19o3R4FOku5+ zQi^m->r2$bEfNW@PJ{mL#XHCU>tOz8g~T|Iny14B1N66Io#d)tr-X70fZ-L7GF&v0 z7nGD%!H+kle1yMJ9HZVg6!HXIJbbgYKsIB{Z~^0pfVLso2ao#C z8sQ8T2H*h=rr2FAP7W8Ji3tQQ7VIyoOATRPkp+aVN%jS*kIVT~TG-D;KKBEb;H85{ z_msd;0vS0x8cjgV=k3gs4E%V%Sh7NKqTi5mP=p>H1+mMGsJR< z=Ya$#S8wGyN$ zEW|pB*3*a&y+}wN8@QEpX*7S#W#kTXtG;1&qf_0dm4t$R?P$437{YLYj3ib)O7eDI zgx?MPcr|&3=aX&c$Bs_hxpnYx=DpdQcty)bgR*YSj5ry4i?go@B1}8NyLgbnwrF4L zbofLV)3|l6|9PUo0ifsZ zD1Z6j2_I?@`MP9|tr@5Te%dyl=WUZ@1&gI!NyIEJbq8hmI(({FiJz$6dO{Efyem(5 z4SkCu4^-7NPou+~u8A}DD%Ig5shLhQEvNcQYD>Ney{Cp-3kB6=M7~?`@TRF~eSL(d zMjR%C4V%GRb9G#HHI=i>2v)EMVJe`#V$%XdB2%&Hk?neWo#HJ+ynF+qTEw0dsyH4= zCo8$MStbr9*WqT*PS=TzyfRQi6knHwv>}&Qk&O-=X*SZA(|iY?jK(IkPEfbTO(d*a zyz%F+smRc9$~ZTJkcl9+qg7Gi%n2=r;|pZQd2-+xpbc+f*)v(+@qw@Z+s#f56SFhMukwXf&H}))S?8>veleOFNb6`?8%IrU+@yfyf-_; zLTPE`z>4fq*~@WGC_D&)wM|M&lV%lrwsGIEq_>P&_-zHTg!ep74aq$p2P<_h8n@5M zH7$F>KBqLBy4Fig3npXxvDxM=Y;}I6P!qgqan?`Ho4j*(@FZ%=oWh}=hM6mWoT+F( z!)P6xka^DnOSm0++r+QbHO{Mja zG6>NgejXvguCG6yOC~&Ev#nhTHhKMG<$1WhwPUd=$%n%#l869{0)g7@`nxWH4L9<= z$k69uWE$233HOm~r}LPC(s8z`W<*AAlysG?yNKFdZW?ZSrt#nnzgxsEKX92j^`JkM z49f>SlJRP>I9GjN-&4b)fxK!g9>A)I+=$lxrVszQf8jas-{@X9<0LUh0%|zsj+?HyApi=-fm-F+wMb!TKzxIEJqj_nxQTk&xlcq%zR5s&x}KExt$V7%t7B9=!8zyU_DOV)#&| zgh&q$Zj=Dl*EAZg-ky^fD5aWZ0`HTMq&%mJ3f_0`%u-|gg9MZJYWR=$3ZJZ}$!y6t zf)WfG{lf*`m0D^k_qD8pi$Lw}FeWY?pSekF*O1Qu`Fu!Qx&@P`^6@R&3L9A!28O3%CpZ8i zPfeX5S2LmZymOslflAsZw#d*a6|Q;HUY;$xv3-Rbw^K-#)iANRM>7P{*h~KM8zR_G z^FaSB*vs9;HV~@l~xx#?yCUYnM8GDYyL}eq_G34eY0UZk+$z z!L`%}iV$Jo&mW)lItYZC1IB zDHYbKh?-%bLFFto12(3qq*oFf@ zd1~d@`GLK4B^FLTbpA~7_vlMkssi3968c!DKX3fUcGv%_zx=+H7rN&X@P_MHA?s)V ze?sW$Z@TdCTd^0f!f&bZKT}Zr<1`DGe~(R+{k}n34LY?+TsNlpFIlsHqtf3T-eHEj zPD@Lon!?##S+W04V#u%m|IaMD-ejqOUV7-%A@kIZ7}v*(CENCGd}D=Iu(RlFOqJQz zLUnyM&t|C#qlSQ(i+>zuu~I@%HMz|%sN5-OO#EYM@{`w_q67WLo9^M6x#)N6)@xRT z0pubfx29&@BDm&fS0}peA};1no+Eat6P6YI#>?4Qgqi5f`n;cx;c&i*2A%uOHfbXBejhrU z-?>7~OmtNai`%5sL0+;+QEuMl?)JS6nQW%ucLkDDx5xA8SFSt9!D6T|xXfH<$2bWt z^?`GTI(lyxd9|+;ne;c{<{vTr2QdjWVFQQhBU3xK@2||3eL0ZYw;M{`b`!2+m49{} zdjIgDc&tb(zd?0qtY$@pTI!0q#dmDn1{IwPT_-BP^M-X-8P48Th^2r{c2w0FqdvJ44s;qD@doe#Zg`L1>=o=iQa6mL=AVNlA;MZEh| zjr~vVl(B2G`?%3&&3I!jN9aP0{~i}l2`k-JQkCxXod>?G!WiBtY0U>GeD5%P@ANYC z`yldC)W}1@MZT)xURwAf_=0z#b;4xy4qb)o)9PV9zTBnyDZWR21L`A;%ZPAK4Gtny z&|6Pj4!hwpMJ>AV%kUYQF<$?g5t^9r)Cp#X&$FmcV*K&gIr2%AeQF5f!Q#Q<*^ZW4 z4W|-Bs)1^%=nhdl``w0#$nwwxrUD zJdY2g&Bc5P&*2wBGiK#8j(iR^<3gU^YeMW?WfrdIUhIXLk@?EdaLwdkEDE|Scz`jY zae}=Oq#E<=6i8sDuU~LS4!6Bz+1|G})w@4uDa?Dq#)838C6SK-g%hIq zGTklVR^15$&ZunrnRKfi(n%_qCMk0}vcM-^XtsR9_Wg{B$yK@rSDxIKZbvnsN6uPu zc~a6nVmONVSaeajeS)`8T;0i+{!s35LE8Nhw+F(lFQ%X(hVyFcac34D3!s=j%w7O} zj(0`|FPFEM3s5|8Qc`F-m3rXF*b&BJVVECWA>)M&UbCdGT&*C3pe>5A|Zuq`r-(~S40gv>j9 zH8jDl&Bl5LdgYVt!3!{Sz9}JADpr>kAZVX}YMgt(m|)?^VB1{El-gfWY*())E?y~c zm7#ceE*r^XQeco?N_a735A2V^Sk!a*>_`++Hta=Blk1I$fPk$TIhn%ojSoj1on~|r zEevx^v-hj&HA0*Z-}-qJFzmrOx!p4O2WSi&;d#a41s|;_*F JLTLt4Vfm)074G( z^`E#Kx=^dPjW#FR_2n+JX5$gDHRFjGVAZ$;&$i2T^$rb*3C8J7li&w!`O(H3pt8Fp zEQb~z!2v~Vkg!LLDd*aDx(5#)l=*EA3&ho%7{TYodiCLBZ)=^jlmppcv|B_$eM*Sk z?V?~XVCj1J?IVe|lh-r>VLVI9PlO(2RDG^v$7Uq)m$xhcwU!CSxw$o!w!wM9=vare z>P+{t&#t1)phNnm*x2%zAI8x8TG}bRlY!iEavsmDJrb)l>*scJ(kb zXFx~wzO4EzUXjY!{7O%GTN?zeTl3WfE&+YKQKerglmlGDJy9~xtZ()2qBQ2yE`(;F~3M_=- zperSC1{e&MMnfnaIN*;F04F%XZS+Jv7_O#rEv}F zHm1bTHVOtRZk6VDQK{rsZLQ?*72SAQeZyzkrV68 zJ%$*oJ&i0SxG)q9M0+8utrg%q207Po>uiL(mW70~wH@HzBS<_?(V4&)OL@^3bT zG0)YdR{qe_;UXT!`Eni<+s})w20Fh(m@a^cx<0Ep+EAl}BxTkQ)k_*)Gjg9=GQw>^ zPYzoZ>F+Zxq7ype>);=hFQ9tum%9paK zFv|}}tJxLB=&!?+&=ktWewaC6T8J3vo=PPnz_d#8KlgiQIS%ytG?cHwg54`==c2*T6m z@{JP%eUbW}v1I$T@SVE^9j@DvOhC5{&TK?9ub0sFSlUbJm;Hrq@z27jHQv}W57xwt zb8ldwQ)^jXD$QRhMy=mhP<;3AVMJ6?ZP|NCE1og3eE1ua`v1`3@B5f{v#8B2>nZNN zU#L1gqsl_E3LcNuXlpaV|8c#p_`GY0>sJvl^Ux%_dbskqb0gVNA$@qW_62F<4U$9$ z!>ZFj%;8Nb(Oz8*K5NtnZ^ry0mh8{Tn+FJ8#}Zt+8X4|HuEa7R>q2z1y(-WE6*)*} z)Y&`sVUzlu?Le`SJ!C=PYg`H4B7qDH&lNAw))1N`i`N@mRUUhN>Vz%Cr3cPv0Gt@v zNc}1@W*pb^(0rqzNeJ!ea?GwvgRXeVh?$mK791R$20YKugLM%8ctSf(HSk`+ z^lg(@;_<;y&o>sYxBV(!hp*D|8uXLK9_+tw}2XsWvo0$q7GRyuSA2R|Zyj>}u%;8PnXF!P#4ayp<}J4Z|4=w7N~_ zZfaMJw)4$>OF$D1BizPZ5@ZAG|JD+) zR&&(b7Ox$rk||aye@NcgQ@`xWSS^=Je+F|B{8Mv+GpS34Kg>zVb2-bnnyXHi+B|#% zi~wmy*Ar~tWjU?XOjN(Fk3wSV<5DXM#d0ulwiRk6OP>c!C*lLxmb1lEbXm;D#MNYl z)E>q{Q(G$&i3Y;D{+9ba4#PD+%#AHFeKe@i46#;*SL2!rm3)#Dm#vH%YV*)b#D<58 z&Tnpwgac|USNPsatTI=~)K#ZDETVKfDQl1AxrIuc7Z$-shl%O26cqm)%#~}Qjt61p z7UeqjzZRm0{#uAmbs{_+(nf|I=pG0oG+x$EPd~hykzQen)dk?zm zBHm?VUu!1t`Kf--SBg>_UjVahmN&Lku02j$+rK12U42c_t2w^omR}1RiGCB@7uD~E zTz6YBJI?W=UlKGFs+8Qmcfi-7-w7IZ#|2_yX$zv{jKQ?HuH2k=Eci{g(W`uGW}8J{ zy46|&qD1^fB`bY6C*m}Seb!Z8*AE!$Gv*)`fWc*%yWF6zU=oZP(KnbQVtdW;%FDba z2JP1~h6NrN zq*>?;5~g3RT+lPhyMb}|D!R`RFQQ9~faj~p$(SPmCAVuhNiBs0|6epE0ud|6nsd$d|ANPzR ziq7JehQycXxvh@M!DOJ+2}Y5D)P+s2(1X(fmW|<+P7i(VSGsLcm5JDpDQ)-8_pOf{ z9y$+f=w&&n$*?e$+e|^Maoh93L8d}X&uXkxGnK3z^um=f^F%8mq~l7W8-l-zPNp2} z4(OMba6?o8thl4A4Klb)N;}zmKUI|k(SFV;cw{&I7%)gEtUJ&vSX1BDo8!x9h%7;349SyHdTGD-21WJ@VEWQrUs@u9cSb?pSX$;qTF zcZW>W%vrLky)lUSeB{4ym34+TJXes}MgVwnoylo{W}y7$Fv3$?sa>Sq3!7n!OkSwJ z>4i3!mU=uN${|r>x+7f@Zg1YI6ya00h)r~9_hMU2bdj*96Kff0=)rUZtnz-Pc#<_- zdCpZNbUp+*5n+0J%OD>Al|q9r;46io(3wiAKaJXmqPxxvQBm~Gm@LRS3G$X#tJ@kIsj+A0c~N#9waV$E>uaZw-b)!wJ37x)riNmh`lmx8gi6Zi4z zFAN8lUPBKeU-%2;5T-Sv0QmF<;}uTrHelS+AwLM%&Q!y>nSr+t>i-_0-~2QM8 zhh`rQjf!f)OfUJ?Pw~fBn!;et74|R#8DPt7v*Z2`E*tT3E2+&0^AoIBiB`4LD{k+f zzx$_v>QemvwqE}|$zOWrV#LkpdGJBROnb?I+xp&KF;x_K8V*^Rh4}@60<>VXU*r)e$!Xo=?^zh|Mo%J=9 z?$&1C-}~aY=ecSk@cBnnh83OIyAHe-U#dM}5!YQiPp2v2+*lABcf_LI4kt|lHl5dY z+L>|j(!18)&;ENW5v2^nTIfxy>cPQ@b*(r;fifpZME*XlG_7G?>iXEOoG;&-{G$h- zJ8h!g#fYK6@Nz76iSTlKNf5fQ3YNxyn7coq+uFLW-(PII#TK2e)P2^w}D zve@A84G%UlD~KX--Qz6%JmN5-*&)TRe{SR}Maf}+o!yYnp0@&c&(Paq2D!l}zf_>e^h>+n~K7oOD@PcK?eTO&@QrWhrMa- z=c_D#5&5l4-$ef2qjR;*Ooiyv!T;MqZ2WZ$|91#~ofZFo8A3I!vPbiYs5q3RC0>0m z+Q3}%mxyoEi~}>W=GN7!VOUa353p3pQ4oZ>Tsw&{2Gl5=E6fFOxp@G!0?rbh({mjd zbtX6c4M~98?5dNDf$3QeOpQ-TVkq_(P{zQ>ESKAgPy57BGr3Sk<;YsVZAH~SiNIga zQr`M@N;vae3iloRNolDjZ8tu&(jsbZU+;JH=tsX>`AvVjqWzCNc0H^Rm+FQC3N#1E zz$p)RHza0=rIhquspB1?;eU#|W*=oDMj3a{%a3}~hkA+$te<~iUriT{nN4PtH=Rw` z2x1;zt4*lw(xrO;ai`~;ZA{`Ynl?ZQ^Od5D>I7KfvdP7J5kf|WOOVew37Ad2${n9C zW^tEQ|MEiGAAV-t3w4~D)C8a3CF8(7R_G@u@N5aQJkL1#&=7dw`PUt*??UtOdf<-7`By-nW~MEY=ScM<87blQ$=$j}d1y!Jp~`#ppN;_Q@>dGv3x@EbUSCx|O{%znKS=yz2X)T$KQws+ zhR!OkJoBw&ouC;horjs(HB)Ec)erUb=hMViFxw-6LtB_({!X_>dSwG~xqf{)F`e)B z&!m>hcr>vNiQ1A!9+{aH@+N{Yvt zETP++eT=8|4@>)oi*_9dV7{Ys1Wts6kEB8Pq#Pr1go8}Pv&%H??DLuu>?lX$?-cjU zZ$XG`1_r{&Tr5$pL(o1i>hO@uGH~rGK6h*aBNwR>&W?#Zv)rbvT-g}Rk*%sTLO6qa zgsd6_wiu{)K>ZDRN|p>w4- zDqr{MQ+2#!IQ-&(1Yc_4AAr;b*VKFpY`#hcN`N>phz#9%*mR^l+$D{-dlVk%C1A$K zoEwZMf@#U#)kvx19g$kK@*4%iW0yt?w)F>0+}Ld*p%yrO!!U9%b}DYq%kRqVV9Hzc zPIpSFoJ6n9R5`qli`$Fd?pMQ^dNZLN+Tq8aTUILaLN{uZn}!wdj1-iT#ti72jm+3b zDBTFbOZ;ds7$N~+L@8}zcW*MiWo)$_GsM^TAe|DhfvNp38nF>LV+dcmulj(QQx9O# z+4rm?Q0BkphE|>{H!m1k@dBCfEjvFJUS-P8N-QaVkh~6tVr>&Kv&-2pxU^$s0Qkh* z+!4WjDkXDwAne}6PnJLb^Qyf)m}u$;Ko-dd5nS1!&bhv5utS#Vfc{bVxr!;%#{Ja2 z4=e3VY_>z}(RtdH*{`_TD;;tm>gMXfp!d1nc1~rXlChqArf596>K`Qj3l2!N{KB-hwA1@x8~14BQ*qJF+9Ot;+E5KyHz{2*yD(6JyNA@x*zkoI z)^MV5=6G!(@jUY)q>=i)t!2T@aK zIGwuD<-^kQ8eK1LvTQmC{%M37*W6e~-A3RkEo8VeB+gVfw(dE~WCQ#Z2-TKGr$h>Q zHbZR-luS!O`dn9vce^_e1}sNXFXc2cWXF2e0`2FXt$FvgErGn85n$vX&WnuxnVYz| zXxe445GZJfc#V0Ox~h4?TT|W}|3GuEy%(_4G?aRk#k z%?&VD)5+aRr`FY#>J=rvtW2|DF{62KK+Efc17DmcD>lib~E$<;!#m~)!cE{&8>Pt7ybtMI`OVD zUK4%_KtxhYa4mFANlsn;Y_$e3vL?8pFnLPnJM<2|&Ojpbz%p>hD`s%l)wgQ$ec(iJ z@&4>#&@2Kh(8**8$@I#QC$Gdw)uLb8z0e_>4CgZPcN28yNq6{X3%E*vp(Pp$WJ%d!c3CbVsW=$&Dkb~lvK z?zNSTEfJO^BT0zbd>UaHKs#!L_FY`2h!%fO@tXd(`o5CP-?%r(MSI`XwT{AH%9HrN zVLiWR`EcpyzdYX8G`z~@$cS}@d}F_1J%1q6blgnDXw*@BmlcPUq6fUnZ9rnvp7$5U zn9nWO?8m<<@HJb0nmMd4SU2DJBs?c|HwoeA4xaz-$i6{C1Yni`%|t3o7u4 zZx?0xc2@1xdcmDcg331)!oQtme%XTlrSACOsdA|>v^IA`_nQh(Rk) zDinX~K<5Y8|0@2!BmU>v`j1T&qsIJik3=`~pJ#cG)ZlLqMcah+Z_|A*S$XEqlhHr@ z?@bGlzdVHhgByLafPmhl`vH3aN5fQ|qWhwOg(sp`LKozWXr@cYwgF@i(TsqjwSdjE zbJqsi?S`{3h4qNqpD&+W-kn*0?n@IaT&}-dbRIS1lM<3N@`M~d4Cz~cMBp}#5uhU! zdO`U-MA%5+?v59co#f=b$h@>zo#GctPTHk!uH0cA2_#x0JBpr`;1>|ql)k%d3kDeI zJUQq|+ihCfZXBE&61ly%E+N4!Dx}<7)WYB81p!b~kBLGQB!eT8Mk1qw9xxjbq77Y# z>>Rt!(s!JBJ}@MGFNbQ9^k1?bT!7KlN2wa`>s&Dn4&QljV_&=V$VesRkxPPx6XZ(W zj0uW8iN!RgdNEM5m%5>@5S@JqncCLwzXbU6)CS!DT(ULOD!Fu&VR!0grZmt6 zZt_Pw?Pj4;xYwoR>F+i+F;;IOc=?&GqbS{=lbg6(n+YAe+8}*}ivM0qrbj+@tu8Qp zRkjq=$w7omh#oP|NQd@s61y+TA++K%MHnFYJ3RtgoRjRkOdvfl-AUujMd8Ehs!IquWG7j1G2hrf2AO!!sw%wCdO|z^x0h{+YxC; zJUy}7E*CPYUny)A&TUqE6?R0vQcSRVtvsXnN&!&FCYS9qO-;Ag>nuNf@w=nq^3MMk zsk><5nnQ4ktM>y&iLaa4XB>N~v?n?`Kq>(Gf)0Z3NJ3MRzopD!4O8-DJlMKH{GEX6 z*pDQ$hOZRA_!BM|*2$H&Z}xk(npMZ%{{!FV+ySt>%JG<7DZ`2Fp@CmJ`Wjw7ld$EcvI9bDFOdtyW~zZ1RkKcT|WX572Wm?t|Yvd8f#l<~^*u zGcFZMM!zgK@3=Y48(XjhvPOBT?Xs$F9e)DrS*pyw~-fbFTTW?|k2x{I##Ove#aF?X}l>_Ve7&eY?^8 z@ECzv^lY}$<5k{hPKnlP=P*ULD$CD^=p1}Uth>DOEjcI>J->$n30dDx+XCRFv@Vrf zS4`|$JlpWk>eu%w#5Qk-MVIcN>U$EyzlRqm z<4JnF$v27Ui_cRHAj!}Y@ z%f=k0Ze8hmfdB|PzPV6S6TZNX&{cR2t)+XLc|TNm)>-sSj|8Cz7X7yB-xbjT(A0OY9abqaZDQsuMz-TD}riO}CS@G|aud564>Hc;yZkE8-9pQ1E zC1UpT(?D+~9!NKX_Y2EFS3d1UYN=SEv=jFO^IGtZbEIF8U#Wk z#a(_J*sg^kFzAyDB2NIk^*Dc_@9>Lw`!)Nvtdk|&hI3qhYPR=Cig20S<;u!VLrKJ} zFw*r6eXwdM&yWW7#Zx$thSUifYj?yqaeHH1cKceXahydO5mr+L0hNmfEnW?@&!gu7 z?le}X_t@x*)lNlNs`KL)nqk3P&Y9CJo*#VFc^gNVbW7Oitms*3Hde^xL>sZ?6rE!R za^5KMaJ`-KOzJbWBFxECVyec&V6*&@)7HssW2qRd02p@4x-K>mw@-fr}skr5It(tf{;B% z$R+6I!0>u|^pObr8%!o*SlAlTvZjOHITa8LoTq@MH5X_eE_N9*vFaXE`_8DRikxPE zm?kcjsoZd{?36DosqT-SUVP8{Q_{jRjdc^1QcKPRGPK&@3B9vSh_t#zIIb^_;-Gr&|mi)OA3U;f&>yo(OJp_bd%tlh*YI6%>Nb~UJ(otoJPJ(yBY7!CAAkhWntA7)a zu|R@~z|#+^JgtwSjO{C{@2seh-8m)S`B$fOFRopu`>5+K3fK!XiIC_;AMh~*Da3(v zu>R=1wn|#wr`XV0oQRV|ap+xqO0J^=dR9|g08sGL8f2{Sw&SCw&jWNx^YR;$CF2cQ zr3?i~Kg4WKMr>z3C7N=R1nGgQfQD2$CLR_RzBF3)_T=XHgLic(?zmW82n~BYX)H*Ej<6{#^qcaW4 zZnok{TL6Lj4tz|DW^GVT{bRh3^fxOi4h;;;+md@8x=gCLp-C`R`*!#R89O#pbxyQqRyF6vD`e{g~(p~(3D0%fr{vSJ)5G` zR(JX5^!JK9?wA2>pK`O&k;L$ZhD=@)DNiko79Bzxrr=U$C$kkcOcg!=u@^>n`CIps zJ-*fTEgK(;UBHnl7TK{H21gkgwVRr-R7q;B>}jt;@1x-sC*XN2_!>0i6@#O4^qRhM z)q60Fjoo)|3zjMbi_+;i^Yh9U!i*0yNFNoOE+X5wjpFqY-BJLA+KvW|3HI|}p94%8 zHrQo|#d=mpG$`*5j!q>SU>+CiJyYW*CJJf8k!2yCV2!cby^E-_Glb92zJaO4c|r!7 zF*PG=^b+dBR~*|5RA+Y{5>q29wT3F!M+q(m+HijoH@VR7BWW1UDM%{Sqa5hg-a&31zv$1{ri;ff?L{^~@ajUs?L{SjKLQ zq(5xH%7efsq*qjkic9KPJ3@X2r58sAjRd`4S?1@W43t8IV!2$w>q>#Z2>%DOOg1Jw zGcmdzVrcg>%kV2k#a%Dw6#dv?$*PO!nEZrwLVLj#Yf(nFP(c6h}s z9LXbt@!AsGzq$rk^$Ua7y|y@txfN!ei!pg&9=LFK%O#h76)WMMbiK7j<4l%~Gp_@P zbyO28mL4W((7eX@XU$z1KhxBxk^hCx(q|4m$n$B&9|<<^p*Bb zbFTZs;#wM$33IGe{o1E~wDyPAzR?B92(O8?;`O%TpFCk5cVAwD-D!b6G;5ulv?7Vw z2ViwFB_FCH@mpzlkE{c#VDhpqhepCchmvT&Tn$RLQ@yBlfJh(c(lut4lmN$D-W^-` zl)#NH4}ceDZaC~+`*(npR`Y-L#Qr#nFiMD$e4_9%DlM2qFtT-ElQ{|&f_ zzjjpKdhk2DF}2DfV+FxPF|oXY$0(tSV_)~FX&aAJ$>DXmFuugknj?!soFbrphL99$ z-RTzF(TF1fnFNp<9sqkA8_kcD`%dI;quZx zG(QN4(h0}HBr$}(K$!@cqYSR0Neg=>VZn{} zcL`%`g1_?PcXtnKhmzOei_|8^AVHZk9|sz=nCN%QKFT%J&r718lGe?3oLY9)+z%^EHU zig?pMp|d|b>}T2dIlRk$;$xQ<^I~d}oy}^_-cv+0V&NE9RIQpj`UKEtwt$k76HNb5 zV}a&XJxj9TGmv>k+47@_y7$SFC|%n*+n+52@*fZ2$^WZl%`K0^pLh#Y*@%7}3~5 zExkaO4upRVaL?Bn=e5anEu(NvgF#j}?$g#~fYctbZ0g8VP0Yn2DcTIHE8d?F0`SMU z+NK0&@B=Czd%flPoFCg{rKtu^Ou<&>&@wkgbP89Jm0n6Tx@{+Z_iiBL(xZZMi^U{# z^i2j2o`f8$Tk|beE9$dt(Di% zpy_)H43RE2?@H!Rb&}^UK{zBwB#|aoa8s=^zYwWiB-VA2EAkhIQI%#Bk9*e;Dy)#1 z5uR@*L#ni9sr)S()R;beVpJn9cvMAdSn>)@pg2>Uc~gYAKE35kXnEQ&e$8M|&zZ@6 zjwjSZ1I#CegVf%U3F92yD*Z*}EymP>Uc=Un6>T_NI$*2}5@@XKcj@u8hrm1QXO+TX z%|=kk!#zI3&IIrp!-O=|;IA1HS!oycFGpsFuV&I1UIlvCuZ8OC*BBB=U`h&M1Vp={ z(J!B_&R?-v#(OIO3*QfnEOJ^P#?nL*6{VF+Ioi|wJZWrfawhhYv5&fpAB{MR*G~LE z{?KpjFfD?qWC%T{_AV6G1otEJtWWm%xh`17#u~;sxGi#+k^m!BO$FJOAT8q748l6q zBjuSh3fi5V%oWOLSrqo~CJcA2Fakx~^W~V?2@uExp;lVld6XmF**x(R$g4S7ZU`_g zlo^$Bu5Xwx#dweqf>D&VPmghU_c2CW`wH~{%nD~Pw?MKfk^+l;PKPXc>Ze+6XPKQz zRI$Bu6QS*);dGAvEzrQ{=v=bt*MQ@Yto6KUdf~^tpr*m}D5G<&$xJ!{o1GM^Kv7)> zheq}g$4uUG4XT_R>#LQ-7tY|F{)S3-+UC@$Y~{RZnmS~H!jukRs(N+f@wJ2L;&Obs zmfrFhj3-|+Bf&vMVEF}y;O1JAki9tj6wP*vUGsWCH#Z4EzNRo>$=F$a;VWKEhB1^G zrP8gGy)MD06W!9>FD{L*-}T}Ee~QcXG&MCSiV=wpLZK*yMG%9G*geP97{hNP^s|Iqp<+HVb@v%k2*C*++F^5N}alC^tOFbNl|pLZH@>d!wuEja)Ntq zxmvNMvP~71n(;LDMENk`xyGQ&A;S;5Q0yS2;eh~?SyX7eR(9+%p#0m_i(gpCUBWTa z*Ex9&HP0m^+GtuF&3q6q3vu(s*NRXU()?09u6r}Q+nUStd2J3u^@U>~av#>DGARW% zsYCoUNLF!a#+I51;961-1AP3-^shbOPgc$BuN(9Gff%GRlRUVqPrf~dGkcBo(L zPIUq6)KZtYFDa&!u_3=GCq(5*!@xalTIyB$)%UA;UB7#HUs94)Ue1Utr_F1AFqS5f zsG$LMnxF-~Ondl`MgE^2U68I|qT)X4*K)@)wakxdcy=ja?O`L~?ofR`e<6Awwh-%( z8{&t->hESt+r#deL`wrX4^v1!wbdcm#Tyg8VUa}>Y_sXk&#pl~5tD~p-W0@QbVZXB zJxaxU*AadfzYKn3FJcjm;6habl7#+sM+-eTu1qd}k{WN0S1oKv*Lyl*weRj{Bq z_emD_j4oU6k43Oea3|1)tibQzmp0pH%NGbE9cT{)p5+E^XZx@cQU zaYiho0-yMPlF!yk)EU6b?uJbiA%{jmCWsn~^PqPP(?@H9d3u&hv*GHqejX0yi8gg8 zxg(Z#pRW&U-(~OgMWB;+djzu+=G-&z@zL^dz?O)`2_7qM@?3n0NC#h|!}D?>eryYH zR|sq{!lL-qa&Y~?)Wvx-n8+-Llm@)}18U`s!x&fcpCSd63mr`L>1@}S)ua-E34sgfx zS}j{&m$KGpM-Nu1ztYs~Y=77=xyGGH1b70aaB1v%;wUzr%x_{S=PwtyRRoHz@;fFZ z)dXAf^ttu`4aHKjV@$#*nY+U{nK0#4Mc=zy;6?g$$5Xf0Nm=9VX}*!y6Fv-z^AyAV zRV0q<=AQ*Om`yHOnAqOCorcOY3`nUG!3K-W7$sl77qas2B1~HUbh`dGE98*cMXT)K z%SOEmoUG3~N<9e@G8h%S8Jh^-0RXZuPe)QZLlIMfpDycqCJpSlTvtp^QgSdpZ_Rr% zv9myE|3%&XU3I<d^2v75|K0&|`{Av?6Wa}=W7RDFg5J=A^sVC4#LT?Vy{O21`WKejwi7-ToxOGD zqNtDi!qS7oUOT*<<;GIs2qS=pWrx61u?ye zx3>LO1t;soPo$WV(ozwzXHJXv3nKWnK@0W$w1dGfEZRnWYDe!I`<5a%r+?R{p57nt zF_<|2^``cvaITu*M&&Oo3yUM#jX}^uYvWT`9?g#T3WA=;{=WTy~ zLoa^Ur+b>r7&7!2)Ob}8w{5dHcf9<&(en9Sr{)*l+*)D=-U2_(_^Zc_t4?N?o^&VV zN+jq?S6dhLgj8blA?#CA?5pZo9~tjFM@#MkmY2j$w_@BKbn*#0+uG-X=SrL78iiry z5bHybucOzGVJC{GnEmMNWThL`(`KRe0>p*({eTy_(}VAre*QZw5pLT*vwZb;Zj`#e zCvBfO@a!`~$tPoZ1Kp@iMnC=$vpec^40~W}=TZ2D#S;|9)UszbTI{o%k!4K2^mktT zuf16w{oA$9^o_6HJ#K`jT-^N|FFx9~vuJ60x#=0J-R2G7CoMiT z?eV!eg*eFGT?S}={gzpELX-JFgD2hj5A^xJ?O*;FuAOL7S=N)x9S%^N;aRm;zyH_84(%2A|`LfXxq`{FD%dd-gda*bBK>$_IFS}g*a0Me3eK>ufSsk=HfVfKKeFX z0Bz2PHPhNg=Gr_HKuWjT-4)Tc)9?JugvDrsZb?+ucTQMuYHmV-Mn&|O7kc=lgj%=s zW*kizGHbqjC~iDn>_$8AgkPei>QK}gp%8nHjHlQ|r?LM4KG(ec``AI z0Mj1ZY6>zwD^0XmXU!TM-a8660L6++4)Ta=Vv>|Q$yU~arY_rNti_YePLMy%YE-qto5NDdYVl? z%9{b{<{lRR7nluc{Ls!SUXvc9wTFm=kZats<=HFZ&6OE5PtYKQ3C$M`_$WY3_}%nMf+ zD3R?CyB_}no?1S!5w5dX4E3Ibsb= z+SkEDST`&e8|_Tj74pAs4Am5HrGiAgGz}+}eudh{&GzkCc{aWBGnI`pN#^ih_mztW z_dGDE-7?BP0ie>(-e@wf(f=h-N2+4AY77C-z$>VLrFdt)go+4 ztSgKj#tsm#pcKxj1O}S~hW+UHeKVny28JFbke)O_kl|OkTB+BI;3Ac9!*1~MLEKOv z_k#!mE+-`kJL+2iSJ%_4#HjqVrG$@r`Ld`&?Q^g>^gHd?BKazv96vTXyhD)$PgAn` z!t#SpGuvqKK%7+$a;AD?L`ZY2Te5JrXS(Yt<2)}TT(dIDB|j(WlT>BZ^MnzoFf19l zG%wApN$L_`!;d7gv;5z4MYLTizdF3w)QtT*nw7qVG>9td4;f z0U?D*^MZu3a(SvBMb5b5+n@hS%m0;|?l%o9(P#sZ)w9nH9K;s#?lyc-J^hbxV@zGu0VrS(g~NrlzlpGuPtNlY@{;*asKZ_m zBxi4h27O_%%iiE|dRro60JTGJMdL(sthwOloFKeMha)lR5uze14T9Wnf%hWhhemYJ zCCZ;(%Gd5E`sp!@1=4jc@eE#Cy*3zK(#$zpW~k|3)z+*YWAEFbX-G43Op}T{9IxA} zba2Cy?VKeY^^Hv+^2z#VDbw0gGqxhOO34HslD4=BETaQge?>F?7C%^l7F$AG%DFij zF1{pCn8t%g`NV6Hp;C@rTiE7`pqFmdcDs8-O(7YE;;UQ;_ufY62*W1qrBUQ6?++$7 zV`HBe)-4oVc5r|L6ST-mC|uI_74DaI5HUlTsnlcwlg1e!#C4Iml@a13yJ@^hTOzZX z3q3xz>sd#fU*QOPHP8Agr+(E&`Y9Py1h1z`P>zOYClxACvM_RKF%@a3o2Bif0S1F< zTy^ToxN$tOC;^ZJSHVS?w%_;kyOy-mj4%1wjVR2s< zI1)vVGn5At833)>fkATpX@rAfqR3C6=W#>?E({rQ2XDiAn?OusC*g zQlfIrGl(gqO`<54>AbJ1w(+ycrn_*)v&nNk_pFL*-oq~iP)fw7Jxp3~vJfrBfMCy_ z-5dHt!@T)4ZPA~w_N{E?Ay(j*u;o>Gpf^ZouDPQFm$e3tId;yi!iX9(CymHZByF|x zq%d--gmy+c9G|=vHK+m%=e?N_uBrNKZfYF~j*UPX4M*%qR)Vqh#Mamyy$LBxp)5z_ zSo+JK1ZI=gmNvZsIZ}Z2=Ys|Ex9JK{1u#kw z5O95lGW&%^j*c54HdLas#M!NpwRE*2C-&EK;);8EJDn7!BIOsB_HH8_JI_vtOyLew zv2%wro}H5}#ofkPrwL)7^sv-60qe6v#(r9&CxQ(Rq=JK0<18z;j9-*SJyDUZ!6yxi zWcg-hH@b~uhAT2wQAwx@p z&*2!Z!K12bft!IX$4oN?rG*`L$m*Ogc3U`J+@XE@orkU!#t5W<-IgNLE{~l7uzd4}Q1Ab? zuyRWcN^SHFEg}*_N2-*gTLhXJ$UaWIvZjXNa;?y2mUS_Ql~n~AI@;%olk&sqiaQmA z8R|OF9BdZZ)#nw~z1-p z`;~6SmJuL4LW9TfG( z$qH8nZfVV9P%4bl5k$B+VCxx>9(BG~(%XeDAJGQ-*(|96lAMzjuBFVT~dGI&-&Vd*cfgWtuMH+0Dy?43o<%+ z`S^*q35`Osx9{w+$1WAaa@Ez7UyW(KaLDhdA-fFHP<_MEp}e-?xLH*f#~EGujbI-O zIU|et#)3KtJHOq1snN^!402!5by>Pr%~pS0y@Uh(Pn!W{?i?S7($YUXxN8u0^3+XanrrXZ8-LM6kaW5fV! zZobhXvdWR0elfjdT#8;Hfe>J|hxCu*|G?7m^M7HpmpL{YorPe*@BJ0pJr@Cj+1~Ei z{LBmpYu>H7Ep)nL;F1TsRfTOsUdm;N*6x@W@b-3+80G^h!A^@T3m2>cg47q86ceH~ z9??6VEl`!U%PlXO5^L@y5OcB;$Pe|U!xs|4eax_0cF5Lj_4b_$?T_9k*$~zhzn%(Z zEE`J`2vJ0bkr+x0|F+hSzksJk`@oWR&tRR)v(i?cmi@|;kX)FymA0@jnpO-deAFXk z&XPEz1 z^S=EoJfA!&3E}WAG9bo20fqq@TQv;D=O6Spy*8}YztA^~@G8I-7UX$|nT8cY$jqIV9cJ)$6vi9m{GjYy@w=hZE#TC!OF|iaD#9Q>I;O9k9*gRg1EnI9 zO+cn(4mPBw;XNl4@5EsmLeCT{UDjsRoc~jwu6NW+Avi^lk?5<$t~NZ|Q+-?)LA@Fh2bd@!n3bXks;p1%?PpEdI{6*sy+lHqYs@S9G%2 zbRs9kIwjMH*6lk-$Tt1HEE}I$e}^Y>AL4Ra`gHK_>hjO0985-}BDWLdmJ|I%8&YuJ z>7JfS$Vcg@fXHQ@NLawmTic5(%ym=#lyX3z);tqg;sk9Isz_h$Lz!(9?6i5o&VMHL zoH!USXQS)V3w=Fvm@U-)qlHhSfp3{2zi|>Yqht!`aDJ{zWJ!EXtSs*(b@xqKHdd{^ zgzJ5$;!g5Cl1k_!>8KR1Wk4kC_`}TjgO~o7oDOYAN6&;wlt)^C4yUUszuLEOJ}oQQ z>7iXQWQvPFEIqUtul>k{MYN>2i0Z*`C&;Xc>Pw%jvkzOd2k*%twuk$)nUavD)S4)= zaUba)b$s>ST<>e|?IheIMZh;bbC`d4bpl`xNW(kG|%!wzPr!JjtL zA8IF>^!&wMZ~V=0hC2VvUO(6P52IgG?8Dkf1k+^K#aHe<7PGTZKUg(&nsD+nvP~mF zwr(jfQy4oqD$7y=TlFe1%`3eu?Sw zcN_nPM4MB(Yvl0jf8kT}-<|S@`C(Y>xZB|JL#DIsH#81MqRD)s>ls#7S=$DiutCY= z@om*B6O-+cR;M4s!Y1Pul(^@mgw^coyN7lpRW8ru;%M1h2jgsl**{>D|uG zso3x#O-Du(2W8A!ouE-ytyb|u}BRaZQ)!hQYKuw*q?9? z5`Ol+`r|j>$g`e1yzuk)AE4E8$BD+GBT*;kL*wqX&uoSlo(s9~B>e1;b{8JM$^P-f zd8^QS?SCOp`rzZ~a&YrEOgPV3*T0_+cDUCGP&Xb5vAh`Xa^Pf4Jz|;u@q_Uj*RNR5 z0sZeh1T}=Daeg$a1R}21pW|sKL%k<@i<&RZMrulUm|OUQ7fFoi~lA#JpV&9 z35~%731o?hK9y1sQRC>HOT*^P8}z5khFpmJwf_qXqiRKFnH4G8GK9~p@c!u#Qov9} zP`BU-&{~^QSo?(sZ(Q&2ddfHN72QegP;siS=#^PdPHI0vx^HC!Xbe>amnce_c5OR= zfK1i3@owWcEv3ZG^TW$)wA|>LEa&^;ZcFL7*cV7&8hb<9@j~zs)PF$1f+8xc=T!aU z^D?HNI9<_AlPcTWKY6`q{>KjvRq>j>@2gEi6Gkpt5YXF>J|^{JR>>XD#~4&Jk`&2i z$~z%ZX=Wlh!O&LYFu<{AdO#a=ID(VgDRVKC;OnyEe|d;cSbzU4<=Gc_Sf2iE^#82- zpEDXJ&m8Tfp1yMBW6Mi$>yTf0ex@++?wpFing1-YM&{fHp@8itLjE4*o?7*=%mIt7If~S_(Nh!77nm~%9)0L} z?eb%_^YgY&f;lned4zPPN-VPR2D^|=U6;rn*}yowwaP&8=M_3|(~=Em>s_Na_nyim z+>55eG59s`es7>B%}E^b6!wv|5IUaUI9dwHmr!))4H`++nhjS)xiN@|O9sgx;Ji_V zf#o3CI@cjIZ17k-O7cY$F|;o;_1m#zh#?#gC>LhtF>%P+oUYIxj(_amGprXHWzW7j zypU^IE$pO92g30vy)s}i9Pdb=v@^vO0b$zNF_rHUKPTf~a=^UDORM!TI$Mul*1Kb3 zmdEP9urzCe!DDGN_~-Qk`I0sGWwF)mJ}|?W52)&f+hndPJedFjiJB@Zsl( zq(0iiSEwEiEB-bETsCK-CYX_gneC~g-xOb3=P);-u?mU_HSHvqC}`KO>pCX;6L<>K ztI15c_j%fP_N|?d?*!ayD_J*!-Bmw*X-FId}P?jTP6QnO~?0CZHS-|(F5dC_T)d# z{vTIUboHVK<}wh8KrW9%<#=#BK9+10tX=~ZIi8lbDc!ck`D3`+Elq0g9lzkaaJ4;x zFSG25=KiUDyJWpQvAywDmlD_XLEQHidSh`L$4#h+d#}sRzxtn1;azd>#hm0BC$ZlK zif@XU*GDjOpBS?McTMY*Wy)u`Evd}p3JcV@E%AgT9$fx;v7qp`BH6RpEDX7&amLqk z;pia+WjU2S^Baffncu2~+8w#I|8{VWbM6ZZs;#zQ)wDYc!|NM~YGVdt%6~=l8$ax) z2evzYZK$THpm(Yz4rO2(;UZp2pNhRZSF$P-qhvV9jyg*M&RDz*SBYl`5jg@jkL?p3%CDz@bin`hDZ0;_HX_y_-$bR9FPAa(}DT6&+Co~-?+`^tEA$l z)cO5g4W)nQpZQk>tl?E*;gIso45;;OS^ZyLI05Cx(sKE~uuMBmQVVB|DjpcFG`jNM zey7t}-?0t}ZXti$3b7U!6G5$_Ks7pBo-;-s2ZIHY)ScTvb_#z0Ut_O*x-j3!xIpIv zvu;^Y3Z7whtY6Q{XWt@kb!8+`DUM@<&RM%^3Ul?;Jv$q-grD^be-D^LZ+6z_hk)+A zNi5T#zAi!~gmPt)40?615h2ffGCL7wT`v0CUU~-7K#2;4(O1M!AzbLR8D&^{!vq3V zeVNIgdA(V|wvxcv^_i8K>?c7WPl;>ue(f-jqOF`;i$oWhk*yk4%OM&)k+O5)7YA~A z65xWCF-$5do#)=a=h#Nvx;<9X(s{)j+$=ZlmaPd5sF9tfW~C{MG?=65;B`Q*Tqlt= z`TeaEen8{WLgSQ{=OYPh>?gKUcuC~R@&l#@=Ik1K2Sd}f*4LM8p8BlvI??w}t(Yv>dYyc8BYVmzpETAMDO+o^YcEnhH-B{OBPLURl{j$X$6ZO@@wSPJ+p5y={ny+O zdl!ALN?S49OdiP`wm(tiU_u?k6szaWt^4PUI;8&6%nKSxrdDi$G$E|=rLqC8A+1FL z&j5jBMg6j6#gz}*59uDIWm=fI=eroJi?4;@{r^RncVLiLis$VpsP)?ynaHZqHL|`f zf?}%waL7@c(jCebGw&;oQpB#Lrpxy%JuO0&nTgvB)DyY~MO>wWK8z|fDO|Qiw35y3 zJAHDn#cxsu&ri)-h!2qYTMS0p<_ZV!iZwli{A-ES%0ZX6K^IalPKqtN`9O$$AlbeS zT|P;DGQ~U|46g#xW9mU^&~(9tR|YSI#-3?E;LZ0)Odtpb0+(XYVR{m7S>GlkrvphI zD^qj&^*^b_97Ell-L}&6zA_m3qm=9Z)hqd*um7jXNB=bG=wkftEYp>A$w@2Jw_xVv zXM_Lz#s059}_^HRptG!hf_I=F0f#>2EIMj6U!o)g`f=))16`Mfgi`^0=P>v35{C0V2iR~?gdLx4}g->xzMbWML~$w&dcF8+Pw zcjG2ZVt*}?b%*G~oOI~KZAr^ugg=nK(V;`cU2ebJvASO;JiMrGELX*c<%k8tL}&Vh@mbM-#fy3;K|W3GU?>oyq|TMybqtXsb)*Gx>f8QXo~W zG}q*eKwgSmVCi^ImL}rY$aI4CZxYFAz{$^sUMI}$K~s0jjPIA?pdPk9E0VEOOv=bJ z!K{His;r!uQD+FY2<1D8N(I{@Dw1^#)moEyqUNG?M+S>gP`T^xAW2r0+Qv}MgGV`N zvoOQ}ZkMCP>};8}x0A%qwht6C*wWSmm4zq6R+nE=`X!FW&ueME@L|>Be9s6}*3uOE zT;w{?6-%@>c897abESnCd*VqKxy|5AUvf;{rrx$Yr~{jgC!!1N_P!}dcDB@eg>@uN zkQ`i`vt?&z)IwBVnSWv7=`UNLx)FUeJBY79a{BJF5Y6OqB`36E4sxys$TYK#led!a#v`jYf+JC)gs zgT*dVk-7E;Y0acUd-usyWMjnkI@j24Y3!a751lE(Rf}SEE5FR^TI+F$wj8;Q#}Yge z{K{o+&Izc|G~lG&*_6=Qqjm&X@g}rXSq~m;!Z6abSI}P6<)$XHp=*b8P0q8%Xogpg z<@9`?z#geG+yQb;+?9$jDoqNM=3K)ZfM2~?+@S* zZ5VZf_ocW;NdU!ZmtT^LZIcLTPwZURUc*y43`PwG2<~>4H2}9f_dX@II3l~Sbw^f@(QM@Y4^aN#?-aV)`}D%sMn-Ksb5jGyr(A?B|usH-(|a9dr1jNxuvfBC*N*Gd*Z z_E#e^sSRwL7S(F-)wk^|@Xm=@B3o4PUZODoUHOHDqdV39+Pk2H4YRkUtvCS;#qwp9 z5j>Vjpraav{try=%azN|=W|2OeB4jP}%EGo5guU`Z=+IF_Yv2Q2 z?|gh3EPok4JT}t4o>Ke zqhwy8gPgC@2BAEls5cN3NQoPg`%X%zX?w@FN)v>P#@@SNw&ojlbX@S_7bZZRQ}GK6 z{Ke_DgbcC+p~a_sWVOzqXvr4?H!jhkv0V6<40(U-p8SWkb_Pot6SxicA@P*h?Poq* zT7F_WK(){pa&2`<+JNK7#At>29ugO^(-Y%s)y@pazC8X+Vd1lnJ@ZOEFepwdHG*bDioAR)};q z`z{CQ<4=@NjrFmC6F+!d?}sSBV`Gv{fC4~veXA_1J$yZ@hM?V)b^g?fX5yN;KFf#E z0V^vW@}7QfW=rdy$Q+q>a+w;KTjfbZcx<`dP1s*5KIfnZxb0rtj`nG4@vza~@?hxT>x8G>E3m9$91qQ1e_<2F#0KoQeR_EGo zmp(MUG@P0$0kZh|I~1tuwwM5o>wz-V*K7Y^3u3X!9er-ZW?(nQA$+RW`#F}0tkf=lhgnh6;M^6af8s_DfNI~8gv zU>b*E0tCdtAco9lLGxsfMJKW~VcHcx{DZe7#zB2b3%rQk$x4vQnU#W1w zWg7flm$;`P;Qr+#aoCIlq$3_q=}i4eRC$L-EAIO0iyL`+p^}(hdvc|=Le^x6V*)Oj z2&>Io2OMV~kk(Sn?z7J(uO=!0Ve|FAS{Nk0IQa5a*8rGF0-~bqQJt-^OM#hA5<$>| zSO@`Y%>tZ@Iz>egi%Z|dLu!Lb#Y5rC+@_AwE4HS9Uk(nUrNK;(1 zIh<{fZ&h)6dbKUgKkm}l?M$N)k^pK;)+xb?5Ku6iWxaX|I*VS^Si4+3)y6?gNoR&E zUJZUTo!lo>zWKVhZz%vyAwU#Kt7@mb8lq5l~k=9 zwxH4~6OO3F=!q?6OY`7l9hv0Xx&*Td8^rKzS9Fh@8vYN=F#0d9Ay=>7u&)XG!6~WI zaatJCd)bDUp(xri3HfS#{kzeURmEFAgv7M$7HuidKvMxP+aG=&Q`qo~)=Hrh`zm{Yjfoc_5vzU( z48`s!afJjM|9D{38Cuk|_sz1av$=~)xKc*uHh4W}JDRoSq5YAN|AzuD0jU^F&ybo@ z|I~UxtF!&HYL>q!WFn-||th=1+A*5N=Ku4@colSzOjc$s?9OdAJ!t4`*U;}u(2Gs7?m{CL0{CHQWzuz%F$rb1}%{>S2R4q3s7dgkT#nVJ<+V@QP za9~ROwY8M8M7zBWf!NFg7Z+vP1HuEyOl^XSDK}TdHSy#U^-Q6p>yU|od-7nzQ{(vV z#~$`oeA#{UNB&JUdIkAjxnU2Ek5IoimjmI3%jzOX{gf; zL|V+!mMbvk2I5J%$v292pA}C%{-&^t4-;I0WnkO_gB$F-iD1gePyuY;Jw=*b_W3QK z3};I5hJ5mp2sB3OVFG`p97hCh6HZQ@mAWbN#NmbWt_F^&!2&&{t}10`b4NNq@|Axe zlGIvnEhP)&q99_J2nhi3faw#FkB4Cu)$Ewi?Bpeow$!jTPcI33PRflxa9eAHvcd8A z)tOH>v`slMk*1B#$*=KuY)PJKLb&B_Ala|&4Y&}lDGGBTAr11S+ z|F~`OoJf9dT^^qSZ*f%sEm#W00O<)NqEmX zNA)$Mw{Fd?Ht-51;yy0k&YB3{)9_e8#HwlV2Ik7C0h4I(jP zFDxtaYrAD4*dbAWQ9j((Mwy|!x=d}%2C}w^Up-;wb5-|>jNZ5n4}a_aKEz=NM7c|x-&X7ND}(g+ zV9SP4*NdExSP+=OOtCm_ephQp!lXmIOM1<@mGnXcZB(=+(dSZ-+=(Z6YZbyqH{E?N z`_R*!_UoL3`Aynh%9Jw#9eSp>g7qeJE=ZkZ z*qYcax1m+>j)h0dVEM+CQcepA#l#y*9Z*j{B6S$dZ_rYs36$5KPY{q4ki4#P;oabe z_x?_1OE<{jxq4CXZlz(4Ukl9b;0TPjhqc`oqwYb(W7e9prpYcPEAoiI^Q{X$ZvNhQLZekvN8o{*aJew2N{1!^I$C^;1K}C}5v;rK_8fb~d z8RyK=5gTmaHjLul`MW7gi)f{)ix*{y{Xr|5F^V3{?Iyv%tL&$ab6!h~qmQ%lS9~Ya zk@)0|NJUx9=i*MZdN#z6%)Ln(vQAi4-ix{L`WHdE|1ultiOuhH^lGVwT)$?h&*nrC zW{XQ#*7NSBqOpiv6?C(jS%PxJP)jE&?3PiI+S5VAy4Y3(!b#+*q#YZ zp&cbP(@W%Hq$WqIuX4uguPIDkM6Nt2ARAwHBsu@OmlTy5AQr-}Rap!h0AvS@fn5h* z#lK`$sJ5z05~+-^02|*oI8cHdttRC5%Q2D1-4Jbt4QRlcK1}<=!%Iem6h_mzzIzz+TD&Ho>NrY>0zS6v{sDX5Wt<^IH=n!V?N;jEE&K# z-wDM~(OqOUePH!^IJ*h~12iB&(5xFBEY*hm_6hEr9)J}7wx`K7o?6u+0vlba`Tb;+ za1Jr6f`V|=2kJqg_)-3u826={Lp!4NBYE4LM6Y1y@`<^=29RfevB^+M0(VU$d~uE2 z8LB;G*QvoCd2)&<(=8$~x{SG#j_EXz;}{i3&4&|O1bo5L``uvaTbkqN9HkJAKm(jS zecgVavjGJ{#p@fTk=@q%kR*T@JOaDYo?Si2WY#&(MdgO(I0Xui@|fPo-J^xBw!2T3 zj_1f^o2i9by+4mcUX0zqd6i=N4BE?eb*&4uVseRnBk|(R<}r{vW^tC)S{5IySVr+GNdeW2?Y^&PYI?_`ENl#6BwePc8zib^TBCuF{48Bga5NmeuK!15R(i|D0@?reAi8-P$~Zkc|{@F>gJ z(dZ1!a)_5?Y|PGwnAhW2V4KTe;B_mP0(osgwMxv7r_NSC&HPZYm!87zxi|sJid3`_ z<(6mjbl0PDY`gJic3jny8c*4lp1p)mF+ayTDR?`>u47``j;D{L-+ZeY{TrdBL#t47?uGv;F8mxYTW{K^~Zvk9MR z=K(E&Xbe_pM{2uJp_e}G{Pxnt5N>inH5L?{@Sh(XAVqly=FY+0y`9O5hm?z&R{qKF z3cex}pAgBg2mluGV0`!Cf;AeSPD zxpe~b_(dtgsIYf>Gm3)qy)155wHocfYRkS-k1jzY?doE(5Nrbt8ViQpz)r1FkWSD> zDiFFNR6_Z;os8R)G`BJdnv{k2PNsBcYaDpV5cjyw$k{+VY zbj?(pu{^HKRmqQtlCp}jv!*Qsp7=Wh!II8*oN11jv*vTOtOGPu#_&)pO+7u;62Y8I zs76(u{T{UTpeI#kW~MbDI+3Mug!k4A@(}8f{hccQ$aCeSPcx0(JO*GrrJ8>n0|Q7LUK25f9hqE!DrFElje-YMhN>HLQGFz zf1g}wy>c59Cpt&cCVoXq@%7^o)v$_}uL@aqsQDf&8s6$)UFSSind>pF)fl;9QjZv- zBf<8j->wb=ZBxHwsOnU1zJ3Z4Wzc1e$W;H$*)*Qdkl$edt2qXhN)j$hmj=rW?fZ}-K6+qj!upmVH)D%mDF0>Zpk2hZ)KvtiXfGQ$))oWS_YT+X?GM{BoRzt}cF~jzF5c_Dlprte3)>{c%c?{2u7zdv? z>KnAg4nv`C=-G~XlJOHO^pC;pqLAzEcSF|dHVC4cYMUQA)aq^`VWk<12h)x(Q=Zi z06Uz(-A^WAzwXd;4fk}PR%}OBCsP=U2gF^oObEF|^rn0E>H#aN`Mqn6W#sSDao{eV z#Z2X%CbIv4U!kOcNuo}Q5nke$siZ`O3@jPj%XSaoixjE1P2bl-nK^u5;m~_pINk9g z1>jBSkS^?I^dIo@^%VD+J$f!?a9$9>aR6TwGD|CG)L(x^8PlROFWQIEw+4Oz6{Wi|WIaN=52t}0k30072z@z%8ay>RQsYBTxT6aEw zVRlh8nqM?_jBC(T6jYBcuw9Ib@^2La&k2!fP-b3Esxs)-}7%V(%Jl#|7FXLAJq$4~&`? zWwd=$G{QcVX~)Z|QMu%cUBpDX#|fziEt@Ux>L#< zwB?|k#;JRkxnC{3&nur?2W6Sn&+l1c&n*hMQ{lT(u}3uKm85h=8-nmae%QW?PSDPS zmhjP73prl5ItT9DnCc-_-UC~2eSTxR83iZjQEfzH+~=O2CbgyvRA&@xcLY(^mJl&M zqFF{&h~0^nRB1E2QxM@GZHLQ_BkMNi*2CH=hH0S6Y=40&e+BLrSKRRS$I;lMWBvR`tqRkdt6lfK}Lf64sk{6p9a|FfUpo_zD;6!&Ll^t8u+x?#|* z(p{jVlQ;b%Bl>&uKe1m=KBhbI$9+8wIkw9g@-4OKCkEbS`s-GJ3s>z;4z}H6I=dMq z)&io)cVcJf6nhPvcIgK!96gxgB-HiGB=5d@!B0%ecN6dkPuh3b$3V8{6MUrHzkd0I zi*W5|iac|!DXe_5;rCKKa?T|}Wi8PZ{cXf}U{^C(R;ljzdS*(C$_xUR@8%4Fv_1NA zR0y39;BsHwt1<9q8LDaUsN8g48lQ+B75z4HuuIEwc9Kx0bXjg&CT~@|)H`UwWp-|$ z;p2^J(|ywR+Sd;5vrzjV#hr{tENoYgdd54t3WG!M{#gr7`4Zvt-krHVYH3w{$LL{J zLt?7=ce)FIRU;kF2~GRz+l4Q;a}7+a`Ll_ve^ujfMV|QZ%VsJV-Z^%EFSb`JFa zCh>PQJvTiK!rsp*WmA%ycAVG#Y_iqgJ-xS>fB9u+pf*tJWl;+mL7UO9v4WCX$Pl z@@9Gb?#4=NrtM_QRVkM=lJ4Ywi_kXJ~fv+asfiGkCAwwLO@n^TBE6#JFSe zB)8P2&DYOC*)w+|=QwiK1Ez24{Zrt72QCeBMQeRt2I)``P(v^4Y!f=V2Y+RWfCN_x z)f*0)(vA#&W{3-UQP)EK5t((eik89R@4tmVGsKwAK5@`x0vSk;PEWsgrbLQZQ{%i4yAC*W)D)53{| zcA8d-4L_N@V#nWscpsyslrxJcsC64ut zY{%2=ic8qkDWNFU_wAb@>!$xjotl0_H7hjf9izYNR_DyaG4`fcTKak=%6KcKlV* zkrQs=@{NavToG)M=$t@~$T0{61X@s*{^+AHx#$lrI+E#G;{FIoGSL?mbK@26>@e9c z>5$ZlCDytWiG|iyFIx_awJzRbBBVKapo>v6F40kbBwoiT|5%wtLt#I#$gZ>2D`fZY zHB8ThZgmMy`6`xI97CICh=N;)-jvm((v5^)nSr5PgHED3*1$LCE9MUt{V@L6lH2;U z%l-Ks0uZ)pqrA4J$Uz^PGq`%T&}QkHc0i@+l*c-mPbMTgC+hBFl&Tyo90jGt?xk`l zqGF;DxF|#{A3WPWS0sQ~>L_bV75KG!L_2Hy+91>gNeyEjU6m!y2Cfn5bRdB z98KsfR&Hf|g+iesKJ{m-g*}n7;-a28mrmusR#-&CM8GfN#>Xvp^hgD&z_!!}>CS7} zcF#*WlPsRH&md=B!Cm=CLc_Y+>Uvr+nZJ&dK)fpLNRl#i%+hg5H(YMJ?V5p~7w4-m zR}83?ke#R7q+LnAl<)#&a)y_EcmyuNt7AvOap;Jt7lp^|d(;5(n8QLu) zU(cV+Z=Jvcme$rL4_4;**^MhYx|bNc){##cbod$Be)JT@`zLGX!$%^o6|VQHAZ|?g zi{&3)p^tg)#AB0Ow%WYkbk&N)Ek{)7>PWI+pkL@#~p4yw1<>p)LMja2y)6H#@P6wp4RlP8iiqQEm^FEq|~>rtqgB5O9`*r0kN~psiddy z6&zaFlCnOa)4Kb!3&)efw_&|}UHW1HGjwMQt(z)gipHYar%lt)Hk~wcdEp)v9|PQE zA1gkuOr;LU-%g5-x=oEKFVL}R%T4Uf{QB%V)V$A?&xrw_StV>TJMTV)%J<7TR7?2Q zBs*2JTnYD2T%u6=*7Ni}6-!CO=WsSTd~;??wO{Y0#Z@{CVz65oP8|zu{Ke0 zL2B*}ky)sE0-uE)Haf6t4@Y|oPJ6+IGGf5>4RSNj6qmRZ>YwztQrYHIIid`!WhDxo zv*sWm?Xd?ann&l;M$drkDgDKS80mD<)vNf=;fb#|-(6Q`*J_O0Syd*-?qxcMl8h++ zVEoXJcbPVARDY+_Z%6L$D}O?9Bg)E2Kq;D;-2<;&=TbCM(8D7*4+lU4JAsh>!*_bJ zwvAu&u#4x`m_!WJ4Z>SZrrUhV#tJqHEM$%69_L5a@kNzqhmT+XtFp%a7c=XBFo4W=F!WG6G>;++C^zRnGnZAn z3Wdes`w6*W>5mW1l%2WW#vJz}XHL#_CGd3Uxi5A+DyGW!LafNicg0ayqCXdIva%1j z8kDl3*!HR6V8s8;L&j`@N^RbyR06tSG_D*Zl)Y>HqGA}K`~ z*0v;!&;KeyfnEhaNW`kAyn_2e-np)xFCBUsuw6mDolmLQa{LT=hqG4YiA^3^?aEt; zbi#NK-J*={^pDw3gjnZR@N{z!c5*yW(6%9uppr+O4$fuJzO~%!yqw(U5s6ty15*Cl z8Uz;2a@mR)cwA7OTzdaP+_p|OirI_M8S!a0Cy<3G_3^{432LpT(IT5X5oopGQyr%S!g_dYwQS{y*Ah!<+ zZDXiM5MFtDgq=5cm8NHmiy}B0?D0pF)MDnF{AOY?Fj9d=O^KA_y#@BiB$2xx`J@S5 zJ7r?#?xyDH)~2U2Cy!o$PL9-_FDWJC-Ue6vM)$S zrScBNf2l5@*$qj%J%V-nVlF-jNh|CBf@ow(VCSqSEJ$4g!77Qd8tJ?7(!9dGc?B(A zNFlMBaFH8M6fbCNc^Z=ZtgTl z$8n&cGb~ZDkY&P$wtEeL94 z1%k!Pq|mP0-j=|43^Mj`>U`G7q*$6-DWF#O)vIQ>0MM&53uZ6! z5VEDyt^riAxQuG_^EX7_W;gY3@*_;d;}5b`n?h%F4vz|?pIf+P!gar0%UjWr3Nq_F zIy^WBaA9Qh6v@fW&wi1v`-Bl)Ug+J3B;oVEs$4CH$vj%b;cn*;QqGN(w)VLXWp$}6 zK*vVP(pT4Jy(=aVP9Lqo1{D1ky{G66`d5NsBJk&0TRb-`I-tL11t-$8&jU+ep~ZsRBs7ntZVcNmQjZWc0FRca`M&P z`)u#Z49`H6j8jvb)TUH{CZy?z-W2~=K@*QFpiXes>`HxYj*7|-9^U5#s%_0P`Fddg z>iy2au?|g-7FNj@_OWPTcxglpE+4^`I}%$*Vi0@&ND5A0cY45rJ3l9{d$}sYV;NNQ6r)!?k1{dQP@ot#<42ewfpgcz3~&l(dcKxn7wvP=6@Xkj8-=SV ze`7W|Qo8e;h2*tC!O1Es$3RU5&yVu@%{yD=Pu#?*XIwe*a58P|)V;Cr**e;ZN?bBM zSqT26AcQ{~Cm$X!tuHfu$Rw9g^Qpl+nsDKF56~Y$baHK^^sb*|;fq;y1Lk+t)zR|O@Y`xU88%J-=&eO>@eoPWAGOCDsFVX% zgxNPckdf%M4u({3|9T@fjX;7ETNi3kynN6vxf>KVm7mhhWLnmuRw}+WWSfh2PhTDB zn;mgR>w+N|* zvGby;RGaGh&Pg@+YhgU)n3&w5G#wG|##J#~Y{PteP=~2envtl9K(Aeh?X^y=1LCE) zC}g*EMakK5k~TEsGm)y&e=9$Q!y>SRYWp$DpiQwl0Y)YG!0@pcsPAr$u8DxZpK-y~ zz#)jK9jPOOE@9SXI53$8=}Nxo9{}JJN&*Z;3J`At32AK0yMSt-ip@G98^rm%Nm_yP zLw+(7R~at$?rGSJOM0&Qa&gq;N|t5Ku|nPl_Pd#Bp3rhsVDp?Sbkfq0f0kPyIQYR@ zhvW#sY$%%|hS?=IGdFv9?9{5l!{Vwrw=(Ne@i*1*>y-IcE zXH17M7`>4twt6%Ab?bT$iMoOo(T~ouxARHW*&-}PLwogsnj8mueuq`Z9-;wZg`l$h z>=iEg;n{QLRQ%VOJ5f;y@gjQ%qr!DyKW@8j52Q zVmEt+ZAD=$rNW~DH1EH*%Y517A(0aAi;yKP$~bRb$tT`7>~{SMlI7QdiV6k+v9x3} ztBbno?Yih}s?6=|q0UQH7K`bg4~1~yt*j)GX!n?W=fIW4ghF2_Y3V}<6`0E2XMrf% zR!EP$++wPj4ybg9im|mAFX_?M-k!hY^URp<+hgOi)t1lrFrb~n46--=3dhNx!4hIfT3cSumJ zj@U_(jl0>&73i-6b|h#wK}sA2mAu*JIN*mkN=MFs1lZM$=>;N=MBTV^611U_6&U2& z-fHQuj>qT_ia|ecmp!@3Fd_n$_$EC0=!m{Sn(T&;al9myHn_^XGURf#($RPCt1FBgj+XChlTFOhc31-4;@vlT*%`dF@L{Tmad6cal zO{QNQY8|1tk)_RpzSH^hJ!DO%8uWTab2@FFqOL&p+)`9kwaD0jKxfPtzV52j{76Ix zj=-fdV1=EiWKS{}83i-b=Q)0}P|Au56s!vz36^K)dHMTS-QoaXqp(Mb9P5_OW{=@w z7rCL`yklx$NTvG=)r--ogSNppzEF^zm+3ZJ^l7A2H$qPjJVmaxwT<~QIi1_EB11?G ztVet!{!%$u=&=o}jdFQ2+H6;T@5Cwbof+A78jywRphwY`g# z%<37^_N(lB)UOf?)}dGp)@^;KL+@@1Q2K$M&qKgpYE_*=1nniPrYh1M>~3fb@ki93 zS5W*;cf-HdBHB)-;)%m@l(Dt-nsFAvzZBCmmwG)_XL`d|EAhu9F=d5R{j%g0bzY&L z7dr)y=y2wl<;7vSKTa;wSP1&8an61a4$i__6@X_&kV1s|%>C4DE1VOVWG7Mur{}Dqeh@ z7o_c*v_T9D)FWreYDtiNe%GX=wwaZoQSoSNG`zem5h_UwxDI==EcczRSZvRHLKiTc zj`@^>0rLi2HgjqlV}L^COCE~-;K1w0-Ct_Kx@y>Y3EM1!!v3s( z-n*TOC*pM!ROj37ET~}(CB<*^Yw7smW%oZiulY0x;3RH!d?@v{w`o06jgqQjY(rEE zXh5ad;Vqso%)MLv2q~?`UJ@;?u#+{(*7+lJs34GI7W89`nQ!&I_lfe+Pa$6KlTXwM zu>mqZZP0z7$OJv-vgiN$iF|;ZZFbCSN-Nc_wa?I1k1)p@WnR{yBzxWY$0!?3;!@=0 z?eP=N;pvtVD4k6`No9%Fm0qfAhw|5tJoC<53oVMM4(=4`HYUHT9k6=opL{Yt8xdAi z`1tAXUP{oG`6#u$V{sk7k(n+XL%yQUw(=}^;(){{t#_{#?ZqF@^Clv;TXve1Hm&(gtkV8VMxCX)vrKN%fT)y1 zkE}L!OPvczXxN;a8?&wcIo7RJ6J>>sUZw=yCcg4g;7zs7fo2Kzu9W4Hd2nI@=W+^n ziJ7k|%V6i--Tdq8Nmf^D_gBdx`|(o!KgY~!@8x;ONTKuq!n3=-=J#9G$>v2ew)H^R zh5;cbxL(&dG_#<=r~ut>$5)Il@0e@|H7sT$c8zS>VK;|<)PQY?_54^D+i0L!C^&g| zz2$g2LrLnDrEIF=kWH%O?0wCUxwNM3N}NBkB&9rWMq2Z{SfvEApK$k1ugE5uXY}XV z$v1WkV{(#Y58?1scL`xvwCteIKy6=zJNg2$x3w)KX*s#BM3KY>(Hh7Fq7ZYru@-tZ zf{ojDwp_}rwV-z(vkgdNsAV^5QcHklPwz1oHRP=r`7Od>q~rm*M2+I!rf2B2ku38| zq0{k;U2rfud*&CStvj9?Q_uIEZlXN5t|d5VdpzL{oF+D!i0`vM^mbb=4mYwe*#O;* zFw#09ET(Uo5YQVb4qYk6tXq2zW~@GRevB(G8R!|%iCYJNGkiHNxhDfitwf?URD0hA zeD_DkB00d|7FBJgJEGl;S}GFwF*v{4@1&o?sr4`6^o!P?%Hf|cehTY<`0>YH;eYe# zu)glaR5=x(U}{ZsKktT?YHk(iz|&_3fd1{saQr$~_7ACiImN8#mGzq*4Z(GcyQ6cd zOvr$2uS!WChb2w%+7a(`vl!G3$BXT#44Yw%%u3_l3!m(7f9)!WTpPdY37?W84J8Smza1rQ9DYuSIdwiWS+M9SOI? zu-`4v^0cb`YzZJbA%|4rBE!;DH+)OfuNJ^kP$SVU3Wj^q|OBHmNn?`s$C4!%z}p@7AyYBo)SqS}a<WUiqn`dh~qIyf)cfPKb(+XA#B5@FUA#e zqiwd0SB2!}C+ntIyJGRBH&`_SRiR+)*5uM;q$QiUF4U?{mA1 zCB}OBcO?9=a}dQXWA9uxcCUds$69iL+?Ac6%~|0igQ<-OcUU4x;b?kRxwXk=kaMF+ z&s6heah(C@$L$TdO;eJ*X5-@AWHL_JPu7|wAfK4}V>2VBa<_E(tkY0EYfi~9uv(B} zjdn1U?VyFz*N7cY>trfFeWw#%7Ml%AnG42(caP!+VgGU> z_)m7ZIY0MooZCX38QB?euy+$Ct1k(~qo>}Lg-hRkzZoTZbm|K-3>i!N7odB#Q;V(; z#25pXD1c|g-sx2;2^5GSkVhEI z)JRCqA$cXmE2X!J{#sV^)_QvF+U+lT&It-{rvyB%ak3;;7h$J-O(vkCoV@6d8V7AA z$)B#u<+~3|xupwq8tY%j5&ForE|*yArpcFrZZ*x%^ghR!jzMc4L~dGp9&R+>I1xPd z^*BRR9>3gA{yMcCGbc#X^&aGG|=|9B9k^ld%(oy%_R-Q4z0VyBs7;ibEV zs~SJaOxLkQ-VNKnewzg| z(Q?P4DlY&$GvBOqB=P540uN|v;@S9`Pq1DrVEQRBfmT(NPlH=93*)|@L}7!L#aH&D zwabe)f~rV&|9tza*DPOaZy#Ay9V-cKY*_^d4c2^-Jsi8yr*u^8{YT^5=)Q^nXt~MH zCb_H)vjqIr@(Qs(n{@L=$VAKHQIpkVs6$)qlf=S{Oa}P4?p-cS-h?r zpTT~((=`6G%V~cTtEiow@Nqq8a5dV?A$OyW4lUPycIMIFv>Ci7|7h$VJwN`tiGhF7 z>Y-2UnR5gcju`x>-4VX=gDVXUj%I&x<*%+=KMP_ByFuJ#ibe+1v5t@b)yvC&(@NgE zOaF6}T7HcAKP;h5s()DW=UVt@mi$k(aH@XFl7q>a?{t5y^EVBDh`7Jjxxo4#B58}Z zsryqVEkqvw6ksYEkumLJ*0qI~-yn;M5lAk*PO$>=@9!h_rw7p&y{}<_Q@o!2XD(G8r*gK1#y5dqJ?R7MHEBsSzTQrC~`O5M`Y>Paa9MKhh z$PRrmofBfnhZc^z2JmIkrCO_ZdMkg5jTEyw9-&FTV%8sBme<%;HWwE;=%h~X-c?&- zb5MRrnxusJyB6rWZ-tU*8EC>5IH_*hMxtE!z^)#)EE*Li=LzxjTZ%a8Kvj;sYK~?& zDV^&cKT^kg{pV9!Ynp{zAi3v$QvJ-rZ;#K@;p?x}*xYApnRj@pce2j^Z-BVRr|I1{$T@C+b-ojIFjVbzof?8+^r?Oy4Y z8uDD06zJsFmTu;wHhL9NAVV?J(j}2F_Y<@d7cb8XoVa2ieD5>O|Fu(xE`)XUWeITf z)N|iwbm!@Q5q>SHhyMM_%q2;1-0Ue5-RU35Nss<(Z_X!tqjXr_+{`)4!?$q$%a=L6 z+q4%vqwA5!w*=}yFDWcS85h0e@lQ??Nu4d{A)es0k;Pp2S?FyAPN<~dgiek-A*}+& z30Y>6x)SwlPW^r6)7+5@3;eer92^1U|`BaH#c#AQsUc{#n#_#`oLT~PULnCXQ1TD9HpN%ORe_Jr)M zldtLM-U{BMGclz*W$5%!FKhkMfO83K#;JI{S6^it!DOW>uMC=lZO$!9jiy=&OfFd$ zB%`7Vl3RzarZlzUNp0OJnl-sf{HS#&c8#v2SER%Z+^UV*N)vs>Zxu#Xh*jWcy^K1f z))vUSF==U9ymBxtw~$N&vErUFi?qK1bxIK&2I=t%i-;&I{{Y45s_X4BiVVa%37gHn z*!`vBzVStCb&?r<_mKF5nT{E;H)oiNhHRV6l0UM@`^=kNW`f?=8SOON7c>D*efN99QF@I&9q zV0wB^d^9cMV75?oNhg;4{`S;WE7o~}u|qa;JpU5`Z_Ff{Uij6iOR+7-Z+CN^QRdvo zq#ppKv3gQa^YzFluZ+PSjo!n+0Ezoxml$R-ap@^;b2}I4I5ss?^+^+k6}cOZA3miR zSt)*{&3h566|ZlXV$83(`8LZMJDkNgi2>LXtOz+!h&s1dHX%o8qdWg;^X-gc$M@o5 zl^@j_G3za=nmFK^A%KRNOtR~2xR$)=nj%r$)8t+~=)gQUC;KW2t17b26VKbh1ZDEh z@tKggr`NBXkbUxnYcqGVL^q_QSDUH9Gy$q#0&oFY zd>KyhkJZ^;oBJ&2c;)eSRr{t_qi4k{AV^*Evo!|l+3BV-P`G}Ty+N@(1vD=@aR&h$ zGtD~P6B%Q+QZxI4$uNV8$H^sccV*CCc%h7KZmoe0I!q|o>IcZ1ddMfmQ40Lw=FOW0 zrZIBy$x0$!<_-2+aXyzgGzMWioqcKx+b+gTQ0ck8+?sgqSbE>tw^{ZdZx-b%{bD(A#~92k zU-9HX&@zTEgu>+305lp*n89jxl585X=sgCJ6^<5A`#@YefYnm{y1N++78z@(;Vox3 z3h?LK0caT3J~TAeb@|suv;AzemLqV$c*O0X;w8a|Xli6RC=^*CoUAAldP1%T69{hWSnYy9t3kN%R9;9r; z8LjM2;f8NHZ{Eywy!RMtRLc0ID!s~LrDCg$wBkqx31dyHX~^6)ik)NlPyV;lkG#?e z>1idV`V|F4FiH6Uox+?G*r{`{8!MsFb|T`*H%%|i!@M^yu-NkhAIcdwyXDA?`JlG8 z*vj;rfD+8vtW|%l)hAHe|L-So8rlp35dNrF%M+vz$%LUn3f{6h{`QS|q zC$FfV@L=R8LefHl9Pw*M`vUY;v13=74RDf8FtIIZ#kE=`eW4A^*F$reY!}=}-A*g|j=C{X7D1aK%PvkXLN) zeS8{pE4SLf4KOMe(*ccA$<`nwQfMY?Nj9zoOpcAmkYZ>|##DFlmTxkl7ls!iq-Em> ztOUC*WXJ}$k``$zy#O(-K!Lu0sI$LFLvcUD?}9bVFzH)@j)a~>-R%ser(xHQKUrSU z{Wd<N$1Tj@0xGt4a9e%@; zMg%PlL^UbJ#rde30nw%L{b+*@qxgUP16FhHDqXSe|fg2nUBEsnj2 zNjM2R+MUv<@q~`ynp8EN!@y`|s>nMpRwqm6vw=&bJFrcL4wY#wMgap@cVnc@8CV>kxbpzVl zJ7~Ysr`K;G_F2aE!>lisTxRgR8m*BEkAqFE97@J=)L=96Kv)U`r}#lC(=?QQGkpkX?WMsN;PkY~4o7 zXmKcPJ`JgiWr0B3zfr6#Y4--=@-x`uxI6gu{DVs7F%g->K#o#Xx zF9s(OxVb&I&R_p}Yvi{mfV10J?wZHv&xTY($kKU2^DNCnORB?vX#dfM{d$FHqelsB z8LJ^{64R-MCVa$%)Xr4jx5cAdUJxWCn^)W`@_C2x%sZMa=m) zkaKyL%gGtoWaNyd&)?`)#>SKI!a}^>H*}(4F=i<n@#A{9+&^PpMi9@12^ za$xk9Su>70psv4Dd3%r~x2(!d!uR7-Xu(1fQIY}lNTOT?CdF3Pz(>4N5583b z05iC`c*%mmjYb}um)9C0@d)!SyX?^H^VBNAk+wyE-lzUX%SAUE+!9%&)I7+ozEhx1 z>cS)PQz!QPm2|7OCN?p>HE#aA5b~XBg-HzVwSghn6hEqGsK+F7kqjwSWbNy4)1r90 z<6j(YP01tAIe#4zd|c5S6(hKqvO^)Ex`6YbHWs&>PMKKG(ESp>$PlFfZB2)1xVXHX zu&lU=`+4?Ltet?yZ7lVG9h#1oAFi{MTXF2_<843t8sd%O5liA3);|Nk$ zzp(wve*tc9K7$0oZD$DzegOXUhqvfn6jY?Vkg3m32YAU*^?K^qkagD!1719(iL0M~ zzLksYAs<4xD`1BV;C#+@IR)EQ@7YnY(e8%amZ0t8;E&O}T6gm@_$n;~#d<04)%Dx5 zGh-}Mq&qm-V{3C8Tx2HqbMlylS3WwR4VrucA&^UI2x5Ok*-C<*f39Bs4g$NP+0ixT z?Ekx#{@KXgrLa6Wn5~LRcYx6)t!InkKi-ye{t99&J>DHrPf1w-E7^3_3)&c`8BBR(mG#JH zSES3EcoBHAFjSL-0`=PdVcWdwd|cn2=}CKwJ35AoZt5)n1>X7I__!@sRQm#`^e{x{ zW~C{?*19{xA;W!?0apO$F4%%?Lib;nNzJDWSs?~ymb+Ll zUtZNXtyXG4kYgMDH8MplfnZBanU)`357`-ww9l&wMIxmOtde#09?S{5D|hix)@{b- z9_TsFvntxs?vac6JaaB~>uU@Lse_@gIUeBOyQhtv4?fg6-=<1>(dHL`>w*j2YbtUu z!nfg1r0IN!mqSgxi3*%pwV1T%^XB1|{z;ANa|NY+#@10#R8b$iV*xFe*^ximAm+Tj zLXw2FpS>h^8}{xCodfNDhBIC&m-DRLkWO|epK+Onk?R;@-m6xuDSAFChd#O-FFVtmN9o3(S;8Ufn&Z#`hfivpAQPw-Ms_VyWl2uN!$K4 zTi;dloxmAM1olSc6{CdvUI@Dwqfxg;oJVWK@=$lf9TX1*@@dhMnSR>|xxjU!q0Sm} znCmR(4HVU#tVvs#+sC+br)SwTqsN;z2)PbbE%tVt+*06b1|`o$5!U9OJk7ulOvTL! zefN^GIY`^GL96B3>NCtEPdKVh$L}+74wJ62Xsj+(o&e_Xa88b^NrJJiZ_dM{%WSWGYbuT-wKfxzNVH8p z;l2w=f7Ah;9+fWth6#F2Eg8=QPW44m*DtPDIyf1UVcLf87I=v1ee;}H-*TugDL(bs z4ihGk%ol6B;BE3N-n6+crUz_`^53=&Pate=7B*%U7u|nGnc{$2X4-VF*=A~*5PkZ4 zTWdYEp>0g=u=wP=Pv$KHT*2_F!7r8iEl+K_qdn{ixdpIT70plKgy{~$T2rC;lNaAM zRaRdBtJ*c_bvI-3+~1}|bQ;V?q-YW+$?aj~y1{vgOb}|aLyG+Ebk&MUTCkg#wwch# zOsWUI>l>6=@tOA>@)03@U8U8bKx&w4AOmaSp{pw{LrEAmddMB0PLu8Q)#Jtjh55M@O&68q1J9tuf$t|1}))koVJj(Qw&3f zbl0(E3)7DP8hJnbRQ7Wc98KG#(gY^1VATzLjoLHwtU&lPElp%uz8SJuu{y$q!Fb`? z=RVcyWIi{FB=DFPj>D1|WHwecT`uMB=CuW@jJ$piIMb>1AW<9Ppg&TwaaKe$;lH(a z-BC@Y+dhtUq>P9t)k4)!lul?m(nFgNLQOzTKuD+w0RhD_GxQ({5F`|(ngj?^0)(m} zNN*B)8KgHErHP=vgZIw7ckdr>t+(!dYrXaEJ!_rwCEwZSoVE7;oqfKu&;EY2?NU)oNdE9>&j@O%=EF${fzS8t7^ zbKL3G$%=0qosictg#}Otr7}t%c)?i1wxg{(UEg3pou%=Z*T6tuEHmrA?-5emk(A@c z#kDXY)|3Q=01g^Zz2SV0mN8-`Wd5-=auSUR3tq%aN)~l0Sn&Vmq`klL!_ z>?KgdW23I|2nP{bF$sVrV0uI1Aaw8`lBRf}A}tclbg|if1!W zX5z?5A~J3$b3h%^aTRh{gx;R`Kd-;~$0+3=X$7wA+Y6)$TmgX&UGuA%$47i$AKbc+ z=S^IK$$PQdkiN_)e|7%jZ2h`ABa9_V5iO{xH#W=Gw!Zo}RA_7q%*fJcLBS%GHxwzF z)6KUk(bytFlNlwy&J7#PXNb%adpBBq5Cz}nxlw*4X0#&4>`bx~M`fFQFviE>0fiCe zGRX1+Z+XA%)tu=ow5SV!2U&)D6G?eaBX?vl&y30CW>0WQ0)n+PB@=}P>abm(Hst}8 z;+gF*rtp(GlaYK^+M>;W6wVTRm8ez1mAG>t`XjOmIUs9=u3o_hCDwI~yS>>p@wLiT zb7{gc6@)*Oyp~_Xv`AauPYMjggkyMq3cyVgr{k|1rB`jY)}>q&Q-9F1?H>(7-5Mz?IA`&7oSVgP%Dr z|70czWOhr5Nqs9_d;8ajA*{JBb>!Wf74Ao?^d6Xj;LbVN3G)+59jByQV47<|Ri|oG zj9DW$8*cUQKFY*h@Wi=0F`<=b!=&)ni~Jv~5>#H0eHv}6h+zk*Ufxqe!> zYi99YP#PjbaxJUk_!-MRX;kaUPgU0~gR>z?!w7U&TQWJKzBFI3K#TtnG^Uk_CfY96 zEY9VHe(F(s8sC1UwzY@4BZFGHVH;y~yh~4pz@*6LVb`%7ApxD8^bC|S%C#IU$`qD- z)*$n8TZh))t$!--S64R%-f*l|^Wvkyr-o6H8zIW{4U429Uz@RP#ENWR{M#6b4=cQ# z?A*(;1$V?UV)j?(nz)aBA_S$r)xJLejJ zSK7QK9GaR&X?@rLNsK}ObF*M#`QqTU9RVN;v|hi_`-PG+FIkq3s_F`VU-()1YQIfy ze>>YY$xq*|=`)l$TXbr|Vz;d~{Kp5`nF01YO%i3?;*R@0a0It=tI(OOYG|wz=9mrF z-PsP9DamMR<^oJB*sUa0$gE1N>7gWwW$D0_Pi>FNWJx#>frB7tEUJfm5aaZaN98?=fo%mFeUUX)0|)iIl(4>* z3O+>|Kmum?YcDT!+AO<}z~Ksp?B0ABwJanYPEPzCh4uENEfF;wYlT!=2q%^GsGN_K zh1-n@O~r+nC~K<$Lb)tZJ#I_ij*$pQgTV!^hzJp(tM+ii1brjiEw+&{Yo<$gr))JK zx!@$!3*wPi-jhwqNUw5YA-O@b%vvmy8FGNlYk+f|{jm0>;qqIxw&(YwUzFBpB7CCG zxaVN&G-fwVa+^7Xth{yjA)-v6)R2rko>rH|bzxC7IJa6v#72QM(|R>rP1DqO0C~Fp z2OmH98|UjBmwc>Czf3LI9hX`kQg1UI`S{aVO|$=4kN4mMt~aX+F?4A+u@k|_LEs*P zl}sDMp!5_>9sGK>=zu(#9_KJP{yxc;U| zL~ss0r`G;A`lHR>aopSds1*zHGaXHc8}PLwL!)nT0P7`1HrbGI|Z7DfBO`J!uH!O;D))TQ@B!F#2jLka3>D~ z>R#6c0(qG&%W5PkAdZ*QzJeRe`w{L{cfH+2NP!*Dzu+=Vy*g%vlSn2ecg^ay-%M7)w(o3t>2~!4UT|5zHtc71*x|WkqylBOUU`pM z_)5hNro`EEVdw|<3)6W03}d#S0uZq_x4-LRR+$}MgeblNp1;)CS%1t)DwAU5G)?G9 z$hJ3dnlVHzJq^06KG&w|2cd7~BO_;vw81{`aOGmQXn(d-T5}YINAxsh3HvJWl;ZGQ zVC@aZo@F-%i(Fu)Y712Tj_k8?Lw(k%ew~Re?TX%1H#71vqRim(10Fub%OhMKhAd1_ zKdVS?b2>T)(P-`pzy`C%5e&%)lWJJ6%<9ssDs?gd%MnGVXPHVK^xP6!+nFia(&o{E zYzsZVCt*}Y4A*}??u1DN0da@I%#u?f#NPW-|~;nGFSC3w&9vt%&B*?lI%E$ z>JJ^`Zi^~6mszYy@qlD6n5#d#iOw*~70Fk^5K56>(E8D`27*#(7`a%W5KEAhQ{iE~ zU|w2xaN3CR5JP7dRmp(gzbm3`|L(3(56Nke10KO3k8CRHr!?ex8mFrGH#2s5z{c=_m8=IZBUi*2EN@XcI~=PYV$ zz3oZ6vfjbrwUQlTZ((%XMzs-MDJ?zKKFo6*1%beG`H~uyF;i-qE{@o?yPd(p882Xw zo+8!<`}6u!SQT)$)E>A(vdZv(=W=8LcdcF7_Bryh5pU}YOkM*u8s{wvxv(&j%be-3 z55wm7Gnr4Hn2XkpU0Gu34)X~jNkmJjb0!yqQ70X@n=b|Scvos3eeF?Gj+JL0s@w*@ zoN)}zwOg>YsF*FDL%Ki2dENm39<&N3s9CKb_z>I1bZ6{R$%SJ*P(l-xsselu0Cdzq)AOtZ|;H`DqCal`LlChJ!!ZR)(l% zraEm`5o2qMeTQ7@sO1_Z|tB;lr z+`vg~hdS!Pv+c@L*55gWU%q$)m~}l@t%f8~jatr5ef%P;uag?vORK{VHvGO)H#3}B zMl2u?oa(lCg-_5?wz?-jdJl-HE(O>M(bzGsN9&8_eEwmbDvW)y+t%-KfTaj|&^-GGOHdWl>IX zM+yrMCQ8nF&o+piE&?h)6zPyP8KdPS80q8?p>C*H?H!%bkO0@FL77J67#{K>edbL< zNtfCOfLKG_JX}e*{Nc02jS3~`0CeZ)MK_0Q$P(i%mlpfX%hMdMwS3{hVG<5jX!SwX zR3T7z}k zWV;T{wF@MeR|T*0(pl!ya(}`LDL2w?7NU2S<-w6JK`l(EJUev3xnx<{zaWE|HPmDa zm_=&akilr>FsI{MG*c9X#5F!?UGN-gd~v`#TEjg*x6p#Didy36Og{(4sUY5)9u@hy zKl9jfhdnt)=0T9%U2_%WbXz@jWh|IA4HtNxoM!Z6#91d^@KPF)-*LL&&RZGBoS+i? zY$~Uut(N+EJJk}d))JHn?LlWgd!eB5$-TExftAvJgXNU00MwLhPfhUm#p{ZNNj3_p zcN-yofk>#1lPK9G{+DBX?+n9KkyX6UzB`Zget2C+ed$PDR^5=L{pAbf8ccC^Iz`f_ zedKI83E{^6fJBilvBjuIk4B#eIF_H}ELx=o)DTDw-EAhm7%?dSZC6o(wmvk^>eHT@ z&gV($-GVZ5VJZ_D&I9ec-g!q1*HJ!%DIhHwu$S7grJr4HanTN13mcomRXM z7HLuAwzC@7WJtKk`mK?h6U%0w_!OoduaO_z9eKR$`OJ};2tmf69c(&WOKY6NXgDM9 z)`5W3#Mdx`Gs3eF1j30iK0af)O(2r5K|fXJaecYOMBgV=Cs{DuYI!&|48f55zAdoegL ze9EI&B-WHmdn%3PsNS5m_%wAm^=K$}BkB(NlC8mmSeFR*TRtO>d6PvXH;en3Jr3!IMFpS4Pf4GDS$r_OywRWX6?E-kY|_Y5 zVRJ2TXMF6K4+-9FTN!#UQ?-55JY9DMYKtw1kz8;{609;=2cK@uj2tV!=~<1EE+8y= z4rymIJw0c7_@G&{(VJh2DsC0eyeZf*skO0FjUBR5 zmU{Xt*|6!gpeP9^oFWB-kdLkAluOx3<~h2D#};HU^QyVmv}m($5Xcmr(~leP!)>0D z1@hmsEmz{3%hi2)yb{;7XdT4-2gKsnho#f|n6m{(Xk|r&dZnd~_X3008#H6#S4R{j zh%FbAe%A3x2@sr+*fa4Eo^vSbEe;H-54mH(ZF9A_w=FdvLX&lTz<(*QvN%;_UN8 zRzR;E7H_zlER9PN%nx!}n@O4O2B;2q#B!aqsKE<333#Krs(m)18K?`g>A20;KRDb3 zeDB1;Qv_5D%1w>1qmB<(D)V{UaAhUfd^T1^EyolYM%U$VCKag^^LX*=SrY}@ z4Wddt)V<1y!*W%gnL^&RcJ?VX00^U4aSMfOc97^2BSCu^L*rQvGb?Y(WBO<5lW^2z ztmAzJWfLP@zOlZ&6k^F5TxX&*-8bWFP1b&BbDL=GUfbX8>Oh?*!}^lI0}6yv#ho5a zut>OR05t;~E-ynWQ4Stn?>Ly)zgb-?;jUciTKH4*h>T+GE-RBPFQzlhd^d9n@O$;# z_cvu0=ZCP%TD=c4LShIa((20Dmv4JPnUTw-t4d+$8|50s^nSKz-zYGa=N~#-LK_fz zJ8a?UWq$HBVQFzGsgPpaG>s9Mjb`DQvd9TvSOU`TVBGr_YE=~Wj5W_SG}XRBRx?Ef zN7h-yD8FgsXDMlv+RRqL6$WxvDLfZg@4tvYX&BM0vT?rfF*1iLm#@wl;$UvvOicW=JpVB<91N^#YbUTG`T=1D@)Ggp!`{UOA{EAnxqN7csD=L-T7Arm~m;Q zTyz6bfZ_Gxu&^=7Dn-!Ksh-y{tfXS)lIryNzP>sey&Sg_KTH zeA?&z7tez3&43;}CJ5CXWr`YL2SchPa$t5n^Ik~|vEOd9jM;7^0kzrc($R@@zVWZ1 zam>K3iH}RJ?y_aLhrb(RUA^7jIG;gL#^3f?h2-~QWphva92-8!uRvZW#bM2J6cO|s zs`}>@7~W+`f%8RXQL<^rr2Mx$1$a)vK%Kq{IItKdp2C#OFKH1}TXM-RKlW^5xkB_+ z6>^dTlNUHgmNbUWbfqtnI_#B_;4Q6i8Kf`U}E;PLy;g3j-~18^P=o*P^^^UR>~C(wz#w}OA}qTRB5 zVmTmT5Xf6S0ovCd_TszDS79&s+uB8l`_ILD>^}>dgsaATCL`nc=MY!ZS^Ppzz2ZPi z)GNz=g^Jlf$Hw7dWD`*cEUmL3 zdga^y9MAoErYZ(?zE@%O8ih(VMOb;XPkDUFnZ%zitonL_T0=I`W zM1=R={zT7C$K$V$yh$qoMek30aQ1wgYNBT|ke_@H6umFLd2a;)a{n)zeQm*=DiA9_ zNb{s-dYpg?sO}p6J8h;^f{aB^=qp|4$xVq7U7@+Fgm~s|CS#f0@v(c7%V+xz*7-5M zBp~^t`jH5O+08p`w@=L3*cxIbWv9P=d((2c7Iy`hSOD-1$9z=NxpxehXYfy7_BS2#Mz}mwW&TH6`D2xt=?I<`IacR^711r}X>ob5Tt1>JRNg zalh9*-#1`hbMeyN-+i`w+dUbO$lg}mBPe1PbSUCh7@EWnNb&%O)->!%A7ralzuY@h z&2ynYJ3)W;fqGYYfb+au1MUCIxp~?_{}%fP6|&EAPx}Ud_76Pk2J}nF1Wfv!lc>bx zc3i-o%>Z3r{$|HLUGILg(>^+$g#QW>`-?D$;t3d36g9bhPwtzU_vD@*w-a9o`gN~6 z_?{#U*oFgmxdy`QU1ALQ^&M>NR@il5z4l#-PQw~+I&kCL;nF@a2n45 literal 0 HcmV?d00001 diff --git a/docs/screenshots/apis-reasoning-efforts.jpg b/docs/screenshots/apis-reasoning-efforts.jpg new file mode 100644 index 0000000000000000000000000000000000000000..413053e0be08eea3a1232cd4e46c631f043a5502 GIT binary patch literal 266960 zcmeFZ2Ut_xvM?M(P(e_dbYG=PXi`EI8-(6N2p#DXdT7$@w9o_$B{T_Al2D`@sv^A# zB=jmxdQqBU8Md+xdSf4=`e??7f&R+%+x_N-a6_g+~?BS$lU^O~v}ssIWK z0Dyx02RQmbF`}WYY^`sor>db1{#DQpAj8S40Dzmj55iFG_9YWjvrDIDeziC*v$6B` zJf{Bu$b9#Pj=2K>gQEYS&i^UJ`jx(NoVNXyzIL4U z_wn>0*U>*tBa94{$!U9X`ij$ErEUKzZRd$Nt{+XVbIr}w_qeZPIF|UVgF6&Lrl-jt zE`T?{5TFLQeLQ~hcQSZj006mN0D$t@U-E2I0f6dI0Dx`mFL`_~0DyB(0f4H$zvTTz zC!RJ4o1g4XlJN;gM*v_a9{`{;1pt`d0{}GUKg-DIAINrzT*O6|%Y*#q1aJj704@PE z0PX-gfEYO?3AhT70LUDT1C#-jCr%vyli?)!pXxLf)yb1oG^b8cpFT@-_AD(8EiD~A z<9RxI26|fB^B2xDFfp^Ru$(=|%65U7jggs!`Ird>CArN>sxwqnXPD_|>6rg%IC=wM zIDP6g?M+Gw9>56(3Q7iwqh`QGvbxElQ5>7%??8F-1T_WKsncg@$mL$=0Th&E>8a10 zJ9UbRmgWR4fZ_ys4Acyar!F%wOBfj0oMvIYWtfnS8Q_tW20@@+j~>V8yeuu7J#)d> z);_VHSL(LIbJPbuSw#6;Hg-8>Zy(3704er|l>aO_#r}fK2KEzkIX;n=2Y8{+zMrRPz3YtzkhxKuZT-pr zCz+F$r?B{&N>a+tesSP(43|Id{cq|!0{*)7AK%E=>HijmP?3!Vz!j`^86ZjZ5OA9Q zpxytCKHw(ciYkCo6F?KoH4k^hMWnCXu6)mP1d#v4lM!O6oR5Os&?j}Af4UIeAktmy zu5>DV9uli~E_@M7!Z;!#u1~Ov^~E+d4t_H7dL4X&Cj-uv4Z7@`JCbXp_5B7<0wyO> z+F~$+9?t`JHWtJvh{pZ>9r&;tT`qx^gXx*@>O_`_i zs!efUBpdsGB=Z3F=r^Ywr2y1+R{_Vw>C`3Rn2C4(677`#d(!q-cK@dyf6`O*2ni`@ z$LZI;)@Ch6s}`|*yWaM7F}lh;?>&3>Z^->bC;lo;aqFGp#JAaFG(&aj?56jp@MH9l znMMhZAJzX8nwgWf_1}*<)3YG(_NOv`qUg_pvd{18dR!T~I-9X^-}4xG zWji5<$&dCKz?F90Wf)UKpH)$V>)$z&0 zI%4zATlth6YVMs6snHUM+-RAoTcHkayu2YzACCa08q~YmhamBw>#nH4*f4p&#QVA& ze3;VaO>?dyu%G4Kf(7sa;()%*F@;~$2OCIu@Z1Uy3G*r2G>ISI(XH$Y{ANwRmx*OC z@YeNFeEmtXB7oTSjxA5NdwrRn?Ol}AeyPs=XP?x1D$-~?cdC$wOd&pAAcj$29$qMH zrf){(EO&+3Sr{+&H_r8a_3t5E-*DKj+@M2m`zWFtf#c%f)!P=!#`|u)=v|d`Df=iy zXUufMk1_|seeuUO-JJu15UH~^sgn}rg-qcWtv7AbzVkQO+ef&J&@$zTzLT!(?EuQQ zC_>`-bgi=x-yEAGe59`SS_!?43RYxulu*%Dyh<_XdJ=?sW{wra;ZV;YH(L@k>@Co( z7)KnwJK~N5W04E-P>b_NV+Eo!+K9g+Dhc=Ja zi5F(HS1mVe^h>Y)(tH2#qs@y;XSSXp!Ucb0FIlH#EvLBi74<{Y$yhQnCh*PhN7&-L zhnQA!v9RbX>XM43lV++IIcfEh?umLP5}tr|NArFJDat$NRIqTPpN)+SSIwR}^n zXrg++K4(On^<MzP*S$ck+RQwy79-W*8Fve+VsoMI0?PC zl)h&_{lIlTY}g4We|#KkmI6MhY0 zdASrUYAk>+m6T5?PP)*&Pr$y#!^FUS*S;J2Kiv3zoYEX_rQ>*BQ|Gl|!UwRCDNn8o z8kt_(KjZOc`D&{~)rXG)At&R8X_k}N&2!hR_MErHyZ!I{Y<#6FC_8MEu4wo4*kO*T zDZN%2I(3n-kQ{Kc?sw(@iXuC%5VyBNnyBuA)Lo7W$&4ZdMplI6jc}i$?mUgf@#6E5 z_tN8##=Il*o0z=2FJJuLlm~}JBGK^3theu&E~wQ%z!0@b_wVpinY(qOHZ-PkJjf{_Gqu9!j%lVal)M_r447N%=EyX~f9szYvR66k)qkcK6%%#X|LdMsrZ zI(LnodstZ3Ex!zkly4@5ZJfK3y%zt@Z~}X`IB9%?(=7R=mrg+Tj2%o|5T~Xn9_lan zfY+l%@pknr*R61oT~DJfNJz#O)Ld>6)p39Oh0ZzAI>j_;LE(#;EA^0;hc z3ZT10ldP2sN}5)3v&Yg&ixT%s0&a}e&uQws8UQh3%tgmI?aDd{#2-2Qn>#zfw&5>M zaA>}E!P;W;|HdaFMk8H6EUbnc7#j$&U^E@-#nkhfl{l9wsXmATYmT;9LKG zWbV-}>5T4ZuDTr*Y^i6ZGLfN4HRF3P;md@qb5i~1L1-nCID;Ke8wPbDxSuj^ZK(l4 zW^r&uEdoS(^U)>1pW!%0!z}?p1*y0a9*TBQFn@)Xl@uPApY5iCxJeA$T2ozh4{)47 z`6Cbuonh|*+)A6wh9mA5Y7`y4t-nf8nebx(yZ+eUVTW zE1!du&!Bm^u41W?6pXB`el+o`7F+(O{%?+|Lf<|=U_UuS3H)>0|AxeQC~V;U$2ri| z!BtCwoxBgUa&-7)yxL0P3~TfvaPXZ^!>_q(P%-o3WM6r=nlaYf5a)|V+C{1xt>LVw5`wsm`XD}3AsU-e!M+{)cNo!#f>(%C30E>UU|O?Oyn^`&^yVuK^c zE|eM=B3a5SSlK-+jIxTGoKf%#%i_NxQ`tY>;R2;EnVY0N@gU zhJ4R{B6RUk`OdKchYf=HBAG1ap*>xh8U%&s4ZmU$16`(P;+J%9%FFp*?r3_<_K*$$ z;LW`@=qjpN8_WD6F$4sLzl^Ahs*jo8meKtqozB|2);m0ZD@4sJ+MPI&djJd&$!BQ(Hk3@EwXDwN53 z1Q4BVlM3wDAC!?MzI<(4b9x^#XpB*v+VKdw-9pa}bw7sbon+fAx$!fl&U#`+OCug3--MfNu# zOX5HS$XKH;F%_d876v+xM{BV9SQC%5n=e|$Lo7j4aZijs41h{I3!xoWND>qZ9;h8e z@LUAD)_#a>PY=1D`=S|o&xK};`)zgQrm=>YJ3_*upa54qu;gJ8&%h*hD8SHsFW+6! z64R*fH_-BUy~Z1^eK2Z}4Q(Ihg=tMJ0eR!4RN(f6{Vx{hA_hxy?y{jtk#|>W^}K`Z zUl!%ZnOnqolaT4Di^KB;Y5n@Os$EYctoTQ*Mqa!z3EP&rTrrUnhbh8AeY%OU?+G~E ze6dSP0j_>Qo4}f12K!Ll9)ht>NMwT9)Pe5ak4W=~??x;x@dd?0WEGXuafF5TFq6Q% zd=JHtYFm5=9ax5=w6!%?N7-l9SU8=iyGM>(8T>&~R(5%BOd7xUq=B1U@4n|D9+hS2dZunM%<|pOgr26$F7$oI4N$o>v1dwfjig zj!LxIFEZ8TvVd6{vI{?OIn-E-oK6rtlN`YE@mo!qT-&|-qvcA~7MpAhZ&s(6>hKG( zj^$=xUJ@6+GZI^@CSbbhQa%N>sRvZ!AJ&P8qwNni^bx|m&{PKDT@m%ge&Xc7Wfj9Re#k!$vO`gMt8 zvkCov)SX1SJ2cRD#M3Cy{#a|xg#q3WLxL<0h~cr1s)yAl?UY}Q87xa!jVKk_>2R$u zXkO9c^U#hT;8{f1)bTF`@;JTB>JBWi+YM*(L%M1pi-&1Czt(KM?>8bU^+Rssu`ImM z0g_+9d?bOu<4{jOSjPLU>%)UXwj;pMtCgVN_HqoO)3Z33tq+oE?jO z`8>Oy6PMEy@Nljgv-H&@F!%F4y^*W*ed`x!x}cqyyqGfHg-5*a*A3;Dlib1c2#((O zexr9=ISrcF$~b^yfwV^PeDB+%O)cMcHHke}b7)^5#d*jwRC3-Qky*8l@^phnj$EI$ z$t&^ZO`7$OS&!vY%c+7r$_$cRK3Q_ZNzLEyTjsb^k@uqpKH-B$cqk(zxQ0SdX+FS(#=q(oba zKVx~-uWK3WzSW@vH89{eglMK-7f|F=#rY=_my&~wcXSuRarTl1@qMjcMXx(lH4zBL zH{>Pk_@%{vFHoE~0w@VK+uJtwX>7AI1E@Ie4?pNxn4XFH?n1x!j!E#H@yB^P`xs(; z4tl*|HE4ES7 zY2MIzovwQSg)HFnZTgGNa`5Y2>B%Z}Ah7d_`oH{sb2 z@tVUT8y5RGSp*^*BjCn8^lXE}y+eqDbX3ih*^ui`QecKCI#Gko*}mjQZ~^mHDvz)FXXA!`Oy)7wbV*XGqd%_t4)lV|i-WH$0ud3f*)b!mwW zbp*x<3E#d8MzeUGs$4gO1@_(~RNFZ%g-A*>@2QX7@lCbJY4zPZWz#VDE-lARvAE8; zhWp|?w7(0TFW&aL+~>_OE#K?F1zT5NR8opu#q*t7HTv%4Ybbx7#U;e2U603StbIrK z1k2uvPT{s)YP4E1$4JD3DO?0H_P}!93KOiyVqQ`yY2{_^Dw9_!SENLzcS`DTVr>vK zKd*fRh!b*dwA0qQTUz;!@N;BEmSpT{w>Yq%?OgWw8FyC7a~{KE|# zF;Hd2h|beUAw|m0PeU_J9V4f93r^XUi|85n=Qi|fSgods?mS2ws%QC)&t1*APwDVW zCi!_a#PPhdkZEl`(hYXeref^B6uc1n`G+<#eZvdV^E{LV49X-KdqR6-S7I5y4@+Mxa|uL&ld_XfB>pWa;Hh<1lB zE zlUK$WHHWK%Dk^bk@4lv~0^bX05$MyRki777hLZ#Q-p(6bifofW92SeONAzXLBZY_N zqC%(bK3SS7E*BeUw)3H>25<#?o7w~^u=WoYf!dL6Q2~zom%lCGwW9TNAu63N(iAGk7Z3kVBigCE{Fn?!q49Tc9l@Oz&UgROOr`R#1CA?94 zy?M`szm{Yr>aTIHeB7)ii<4x)U@%Qv7e&)|VNL`I1hMBpkUx-jB)M!z;q|PRCx>Czm(0zk0AyBpxB^ocMK1lkKRBWDB z+nF~RUI_Xu_m$Bi{|T7I&Q_g&h%}sNz={xa-p)2>HTAeUr)YJ#;(eRTIl#X0%QvHv2`McVA|yffdwn*f%hismJjsRG*^XO zOiEbO$9ikF0u86ouRBv&>s`r-Ue^WO1>@aNpM4CV5Knc~ki0d=;)3?DAkL%fT7#y$ z1#$Ii73*t_n1nB1MdaWy&sP}-T7)gi3Eoo$lO0prkmu*r0Oya5_ncsoi=$ZE{{DoE z&sv#A(YTsKV7WNo<3C@szni^WZ3?uhf?h{nyjWT)Zxbi2qt`{ugBPUdp`KKk zyXU{UHQwb4l{IA<&|T)n8bD!uJ*;Bd4cEFT>Cm=5Q4~Y`$_b{0k-~hcwMyEy!f6Sg z?2(Z_BDKz$N>GE!We#Qd3#0FeM*`Bb)#4t#0KCQfi)n6dQJQ{x09au5YX7169f^nun@%*BY z#%GBSa4KNSj;;eCr8bI*O*x9B&Q&DIitRJjb%y}O@;D`WFLd!6_00(n4cVowi^x=s zr8%TpOkMmqmPe4(g}OijoqEupD6U_vn{;Q3?FlOY8k0mVgdtFa-*7=3sb7*xIxLR> zr%ZKWlYSwQYp=BYadl=J3ENIpmA=K-4NGA}x_(kYKZ9{vZE>du(1pkPw^`WvH z&N&)qcy2;wYH6W_d}mRdOmTqVy&`M3#@TdV&$@9@w>kq0J3ITxav9myMU3EsnZEdd zpyt=I=d3<%)APNy6iQ{BVCT#+nRiLc)JlWgzn?FyJuI@*O&@h?+~f7TQ+`ds8lHJ* zF@}ou$J+6oNPK!!=la*OO`gjm2E)d7O7U7yi1{@Z{dbZbA~Z3k6`+pXNzGJ8{~*~{ z!y{KASA=I~fc*owwxZ$%XjdRG+nRFL>`rGOqYjyv zD4GF_QO`o$tWaijb^8%u-fUef>G`JECJ^0{uymcLS`Pw=fxD%IoA*~hG@rd0<|$6X z!w^fgiHWH?Z6^$E6&;c*pjAWfB~SCY%F3;<^9Ykd^mc-Vzi1MQHz6H$AKfo>>v&cY z$2L-o6E-~&^3ipkrBaBc#6t#ey{N>*%s9=`BE*<;kbS9Wpai4SZ9-{RZ`L`Xd&^rf zm>_zN>hTo^jivlZwtdWE>dt(IMtc^;1Uq1QiG%V%y7S2wl$Qds?2WHlTrTTEH+^TR z7AgAtb;n}9hpXOH;v(VGe4x$zxZ3D?-`%hJ?<6clJH%+Jl=;ER)54OO{yZ3|+ls~- ziLwhx`xj*6uBEyL3AB8bB$&AqS@f$?Mf~kwV(mT+ue?^{g(aF#tsMcNba7?Zzv_i~ z^>&L@YA0z1YL2DayPhTPs>}p1HRStOh_wZa`n>gr2T}j_GN)byxT0aZD>lnfFz*g0 z2ozRqPS)l4H?cJ`4zD5SEaAq+rny1IUvRe2$Oj4T^?kc$6~bli0Zw(K)@}T{nzPP$ zEc>^?rWD6*Q&Xq`Qxf_1iYF|UV{KQs+=5VUk;rfTaHOhSeT6RD)e5yE(ic{yK1qJ@ ztzmEpY#qlt5hpR0L|134^ul#8ZnD=W^*hZ`)HF&+`b;3EJwOH zx!fqSzoBCBsfT{>PSEw*J3%2|f^tP?wNcXZ4_A1>8$OS^+#u&Fr84+ z2i!T$*{*wY>h_y%VLheMa1f=Sk!iPY@^dWQfA#s zD)qull^__Cdz24nn0BKpH2FDliaRM^$n)A(V5vUuzpt?#=% zmqK3@!@Fy*Dl6JI=f4}uw|}eS>l5iTS8pPRE{a+&8!c7`%%6Jk!X4CjSq!GIwFC&S z&)9YLqGl6Epv(P8;PO(epTtUx+A>(1kobIM8(MOHPDDz3f6T7h@(MmxQOMG0l+n1d zt|k%T;m=bNRQO7=b!*s$E@QZmHE;#}GBwuCXy5^m2&-f%z)me35}&YH6i+rZN&9_$ z`rN%ZW*s%U7?ozef~rH37naohu9ULa@6lK39Hf)%F@y$VUS~sW0vs-n#~a^`oSr0V z1ink=s55v9{E$DacXK#bYqwJF+E_#y%G*kJ&z{@GaN&u>dQzwdOSM)=PZ_9xSfi-h z8-il3Zxk2mQQWxq^@7!m+f?zOOQlB3C^zlr>l#u~ib)je~cm4+7&Ob>ay@N90{? zv+z~Y1*L|uRWZhiaW$3L&=QOI)fE;Twgb=K0c&s>(n>!-PAci>g0NM#+#ag^IzVMg z3uKL1ktO+VKv&m6@$4BYU#siQ0}2b=mUadGaV2?h1sTnCd0y{{f{C)}nbZl<6!feM zwsP`n5&zW8yf^#3SzE`2KEWNH#Vl4qL8Xf*JF|s~fxQk}v(L7Cy`kQA$r;x#8g38H za;ofcfjp{-or!&Vj1rsR9;t;6g~a&ll&>X^;}q)bO7oSGR}PNh&IS0+B+>8tftw)i zd6`dYEh%Q@EOVU7K&-*R{g}7}nEZOD25WqccVFG)p?3sF`Ox@dkCfZRVKqtj2E>Mk z`Itacwf8GK0zmRzJNhHrExk*gEq>_|u0gjaF41oboUy+(yC>3p00ydd1Z6*oBK2!X892CqhN|%_ zPp2g(f6?fPSA#^XC3DhWZ8~o>=~QdbP|3}-Qr*Q_DP<^JOMZs-d~~a&x9rKq=|W$U zBvJLVW+#7YCqWfokIf2@QaJ?r%5u63wcJZQXR%Urw%Xu5i8m=L8no%+4fdX>EuU9W zQRzcu#16b7P1O`n`@T|1uy|qrd{f8Gw$KBf7eFwO1(vkVNWQ7o3gabwP^>Y!u6dE& zx!_U|iDy@6wQSRA#XnyGO&ZgT@};wc&T(JQ{4(ht8~ZqT7M@=OFJ)3X6a)%bIv(1+ zV>I0o7J0An1}e&%Qil05YDMZ9&6i4Uzk3k)E<$t&R0a;k3IV6$kXp^<5)ds^h_O0y z!6Mo3$Bz4n@2eMvhu7SNyS*FG^&eYn?!s=bX!yuO#I$#;(~FSH9wXlL?;_13T=+nJe98fGjA6QI&FYuPqqG>N$16kP9e6G1tx)MdREVOK= zx8sMAH(=Oe4>VZ)Ps)mz%g9 z1g378V|iajZo&wKx-jt&s21lb<+6QKRvJidoR7x0FEZq3zc z8CCJUvjvt3gi!|`rvyU1dToRBl7dA5Ou76kT~pq$=2%efm`Y?FjfW31YGr2y44p{F zr7xwJZ~T^8c|karKXXY-l+#!-8g*WnUD2X;$0}qcGD|8T%J_QJNgomeX|Uy*xW;8v z!JblGES&seJnSY}E2mPgvu*DUx?Cx?8a8E%Vxb*ATxJzF#!l<)IWHMbiQ`{eZMRUf z=DzN7%JBsfyFfRv2|6ngLnQ%9dO;;A_0g{~e3H$A)$Vw+2j)43D+??dwat`R&(=$V zLD$rfucib-uH8$>TnH&q^-V1Pv|_R)O(cJI4`#Pq&6K=? zT|>2cCavU#Onf7$Bh^ff_l=@<4yNj>Wsq&*a}kRJcBUJ~F`ELLX)sA{E6x@ZkFyP! zYqENX=4y2Au3>#t(V;iNK+>aSE6wq26SRL4F2VhMEn(NFhb}a;43$aa`$KUUq%V7u z;u(4D6iW||0CnU~@{hr~=7XhA@cQTVxr7OjYJz9QGZ$M&T&hzR`EG%mV`CiYJ9Y#Z z3%qO}peodABv(V?r}JZOz{7$ORmr60jZbL85r90a{key&_|a{aTqOEj+d^LtcU@J? zd`x0$K;u^4Xza(5VKZP*Nc{Gon&GYk)I1|GJ8l=!nE6BJ2yor`0j&+3iyY$6oecoV zb_mx87jF`}jsVe7W#OLAtPsLbSx%oaRWK7j7-ljP-q>q36&#n^|3S&`IW+ zvJNZW+mO=7@;g*5#>FrkI$`k%@CE2Z-=pY2a!{a zb80Niz1RcH9*uE}3yaxT$luOyyrEyV0$<2_L)88ug>RNN>APS}o+CvN%&+z*g&Qk+D*&c7Kmihk-_VkC<@@@Jjo-b<{b|1kSwn4PW}l>|*z9 zO+oT8P!i2Ye;yB8wwCZ}6@EIDhcDg2=rI3GMIdBx|VTu6PJp5CNCYy4sc; z#}xQfG%D#Hx1jjWXjgLc0;C?s&s+lM0pTXdvHY|6tit9H5KFn+3p6w?MsQ{UHx1=ShKHJm5x`)sr40Z2 zF+Yv7SFi~JA)7`vu=;JD9pN%bSLSsD&;c#xyb`fG2$_MC^S;tkIe@OY#`LNT6r^Gn zp~cmM6V8|<9GW9stzPbK|7h_1r+$fe`SvA8V&DCpyWFip=;9v<_5_M9mUsxJv|wax zvd4X1W$`i+t&rl4meGv0G*v3VE(mWGn;2$&3H4^T zu0*p`9tNTzWC)6f^TZlgO6f8b5D<*!$#Pjg5>n&~6YAF!MJkr+5sMdj-tj~hgWXN@ z1vJ4B&fgTxc{vWIQiUy2G2{*8UVNT!+A!s!mYYU)R>)^LgD50Y1gvPz6dw;_Yadh+ z)sB}=nM zZ=Wp}eL<}*dHYeX6Z@r()5ZQV-<>%0op00o@1}1`41_D3r@z(ul;b?_Gp}swr<6bO zm3gkax3U5MGCnPTisSs}>Xt!h<1Waa(Dpsb7*#YA>V`wLw zaLxmsc|!j4a!yVrj#h!BN(S(6Rhj(EtX18MPz>LXz`mjWs~5TCop4ea zxwx02?jPQ?ACFG@?_2*Gse;x9ZJDj9xoGDki%LN%Un<@*S;ox>s#y5U{-HtnNYNgR zySV#HWCFhhiPv3oVF|hbJ4BmRvTIl%)*2Gs%`~T8NDY~8?i7Hdg$ir8m;OY z4v`gP@^MS&amm@)u^=}~h|6y$2<*HD1`MgVO4(2Mx##NT-aolb%k?DYwDhIl@iSuo zTazA^^J!zCj?D^bny3?mF{YOk)imVD!M&f5m zLUb&tf<?z_FY*-o3)W5e`q(uk$+!zq4Tb;(Gz8$o3*E(V40iIPu3gNTZV|)s5x0Z;$x-82 zo|lm@kotH6A%GMBz0Y{q`gD%Clk4Fr?k(z@wg91< z01OS%oy8Y~@E?PjnKFHm|K_mJLoa36Dijl`+n~vUk_^7v$j;@L*GhO9P-3>|_t2$t zAdB_ElLh57R~TJUYeZ~ET?1{QA@1f6524QYVBYuvSxCI(f?>Oq6iI1iPLfgv}1W zvrgOkpUYyHB6Q|i2+RLZ4DJ8BhEvc`$7Fr5!XhXsLI6kvyF^rIeQJ+DKC=yQW&2N0L(8iNpZVv)jcr}7Qf7r6#sHA6#hV~UC#K^Q}36~ za{4^uWc7Vr9t(TdHOTiG*r;4&wjTi+C&J*A~EBa zP1#oJW4;^#QcrO{dAD;#)hsY9Gw0&5MH#ftY<)vg|B>-;sSijJQDwK|V(RV&&C*ZT zlS}<4^ZBzg`rSQoDPRP|G(Y#@1KR$up+9EKUmV6CHuQ%J{a5FMK1v@)UY;@lUB_NX zJyl)`+#~@GAYReq+HpVK)STzN`(VmHGXCB4 zk;c`*BtziJP*b>_^Oxw_pf^$JigyA8Vtx4+YKol%Z&*2b=OC9O$IiC3fj%-zNKSLKF_ z`1Xj5T{&djQ={M?*vUozXS3L*q0QaW>Kidy4kn&MIS@V-l@;Y@r(gV6g2sIeTDLO(cCx3Vo&4i5ah^mR4+gUd78>{x%P3AUD)@Hlj8H|4 z;3fW2KOHdCB3|AoOfJK8B^@^s)H4qY76e%p*+sqc)7tXyiXcN;{E2rLRN4mxK%85b zpZ!bfq-1UGU=pn4-pYP<;(5G690>${zdu%2bZ{muF`k80+6pXa1O|hrkdWM6EiE9$=Id_XHYzF?Vv$|9K}nrxZxwZXGge%CN(r z~0Kv z8ek*lz#F5_r-j!QR z=%asqEcI<8h+u2@1)BZeY&y0>3aIY@-uzi$O4{GAKg;^mySTZpMz1)@PUXGb_z_^S zm+AE|Xw1f+u>e%bJQU;andGom5GEaGC}|y!?j{vudZznyXrBYEcL9p3RNL`MrD%OEC^#f zEZOe}5Miif&p|JOMS1<8o;pYX7r@#wt>3#Z9MBzHIVY6GF*tkHBL?W=5L{&EoTjH$ z5;Rr115%uXyRq?}n=lm_?po_jST{+{Rgk}@VDh!wB|vJ)x&^8}{^-LH2PfUsdL;M# zhU>_C8{U#vroCpAREu|jJ5H&~Y{vagkGmi`E=A_3vwa8+L5|sdya7v1yuW*wzex(` z>1(*r=-gF0>Oh>@*k|SqEZ~--D04Njz;;lrLMW_ids@y$GcR%a`H~rTqn}fgr-JV}bLK zTjId<`gsfS-O9>j`0D{D?L^^w{0PSg;{0UR+n(3ww5hYg$iL^bkTth&mgCUTG=fArJd>N^*Y45L~Hed@Xz_}Lr81Rq&R@7clfcBn7@5g}|UtAkzur;|lMI6G_ ze zOOKm7G@ULwgU%*s&V|XngUjj{sEa%RNIVC;3p70?tIccwZVNKOTQQNe%)IXEozFe{ zIZwAk3cM@7*eI)?UHPQ5DCO-}>DXQSy3Q}*29u#=Wkn zsxGNjv4jrMp$|W#6qI0x*Y+zTGS=b)0k5vAPgDXj~lq65HcNtyuM z1-4Pv&n(DV!ndetC8iT&#YtuyXtR|?i@W2%BD&k{CGj>^DuD}I4)p=B`e!LGU0Aul<(?H{GSx8*DanQ}81uT{Q> za{vc3d*{Gy0&#?bmH`|ta;5RH2k08ep^4@7{b>Ww<|sO!XO)Aht7Q^qyE@CcmbEkd z=WMA%D<)+oQN3d{;e`fmx+D3nHR+41wmd7k7K4q`yorvzxfR=Z{truP-s%Y^yHMe| zpt3$KbX`GRR@*cE&*xo6C>a|s%xVfp7pW4JSQ2sqK$QrnUbx5)Nmnmk%-ia=($2Oj zq=Om=VPr`o{MK^hTEa9E??n~X5n2eP9TzGXSy|cj3pTr!X#C^ru=IhH?w7@kq5k!r zcZxuKE+)xw>l!0pZ@jS@uycZB*^7z)cxh+MLjE!TW0w1ZRxCZ*%io&(w0ozT1k zaG?F3ss-JyKa#lo$t}{u~ ze7>C6CTz`RSw4(yCt#fW9td96g9lr=!a{}ccxz>kR5)V*=ZzB)5iwn|P6-~&aaz=o zVXh{IN&9-BvTv27LSo?~F=v?)!-9q!rLQQXmA(8Y%c8ISwuAF=#SVT|?GY%k_sC_A zsKkIWC`(1)JrI}BeUcv825h0eQrOx{(qZ2+X)4Y+91YwvO zea{hB0?blzpnzn*&7qYZ_ZhQDhc>p!xcE(b-}Ag2OTZle8}W)w@%2ws>WnleOGa!o zF2*Z_K~4I*a$%%RBLxd57AimDiuW>rQ z^O2QzD7P}6gS-{C{LPgyM*x@i{R%$mv(9(NoM4BlPZf&fW8;d^u?YfdV+YofEtF%} zl?;61wVlnKrEZ~%4Rpe%XK{GM#4@|LRu~&L4k+st8eHm_D$mfK>QIlEp3*HhYBAx7 z?MHgnQz==#7206z?C2-CLyh3FBFXignDAGWeWju3VAk?gysIRs21Y0ejzbNow>m~F zz<2WI52~^K`m>5GAt#|GvghSf9@W^}<>4(25pWImt|Zg_+NSW56AJDu&X~E=%~`iS zBRO!@#^yVH^{JPc|IafRND2ZTdwfh0tPD#Ixfu-~QeF|RN0K+!uZuCS{+^&dj=!ZX zL=8mJ^jakp)M{j8(zIAQ3#Ro`=QBRUeB>6856XF)+y?3#=4b9iMkVAIK& z6BI+W6f;T;SNfh<9wIqMu0@@-DHQj7H?)FY{Z=}pMQJT{VL98NGi^|0pOc=*kyM{V zEJFrJ-|qBN1FGW|;7Sou_(rt<*AMXxlwHoAi>H%LLk}|U!9Y?T?&Jei=erCDwp
  • sP;D z3Ez=fQ2dYukqW+xzV=#k`vTXyorP-8UXngOU2Ax8k&w?QTqtym4#+#?r;2{)3sCW` zgA4g35-!{FO-0k`Qqk>n6uVyq$Z(xd^gEdn72K-2dKE_FZsC&?4iTQ*BsPHBW5#A( zYO?iS+|($(ThzGNK*gtqcqh*#iC9Fc_dV8VW4~~_g`6-@19fZggQM?3;2Z=L@VZsT zdjLVuID|0|+wHAQcM4`v?Vl^Hu-nb5v;$%yq&|TOB{VgkK1KPyS6y?nqhAHWd!z$z zye(xEu1lV4*iigoK80E!p--O1k8kPJDkWXl9(ytes^-hec^kT&vcDhF=YeQ3C7&Ve z;DSc==w0JL(F_*`5vZo#&I@FMYNWRb{<_!5ow|7#g$ z4S7{Oy<68O!L_3HA#U>E-r0929{=qtr?^+`TGoyTX5z!OFRr$1^l|p+d?9)-P6H-_07phYTgJ9lvtB!I2ta5Tu8jKo0Y@8bC|G!}+=~3@??^OY_(qY*o1U zTMPtqdTM7h;2pOkhVMvPoCh%xur6IaQqHLCeaJ-Gd3M^;$GLayx4=uLjxm=kd9!9R zt$6`d|OZ->)4HF%N1{QJGBfr z>4sH?U;1^Ou*NA#b&=H%UGgU^3!l!TlD(Jo!()`-SDNpZ`2Uu z2y&rsYHgtwlBT$`--TeU;yueI(i@;(BC9bK*PG*skUsSDgpAWt$UGA!Z0Gw6`WGeS zlkmb7)~-|9E_ty$!~~jtnLZZ!(E9k%%S{u--X_8B)ySA&w2}oTUy^t1M2+pAP@ znG4*wbbpH*{4{BI@JyiEqouAof2I`7QJChxhRTrxZI{jT8F}2nu4v!+LP<+txylcd z`q#a49DMUpw_IgsKJ(Se?xH)U6mCicInz+HEw8hzAh0IQd&Uq5Hwltp`6kBb6{2xk zYPm`qP-cD=7O`MD3h>Lvr-;kfo30!$b!q;vJ)~OIi=lW{=xStTKUHo@z_Sf&%o4zG zcB+(RMS2!XLQwd%;(L?e(w^RtYr2>Ki%OfHJE-li&?1kNP)uaO$37Qk3mdf5hE0gV zc?8I+OP#Py=s=+DzV?f(Ck{;{gsPnTP{0iGIY(U=Sy`*SFW%2h zt)i%c&U%Zp$D|29vF}PU0*Zufc>T~lXK9%IG@ElGAyx!zZJX&Y5>%lerAJTx@|HE| zxu#@!8^RjdQZ8Q%ZxXMpA?nU)3El4Y60(Hr;M zT+-8M(OhU6!_(+Z&(~S{Qh>X-j*fRM5|wuLTwd(+PxZecN~+KX46ly5>gu_?OUmh@ z+Of(C;=S1RzOK0&y$ZIjl@>QuD$P=jtl>gr35=uFYtmN!^vo(?rlV#6!DAQI#2;Td z`Sgs+^qE=xmY!NJG`!7hJBwuQG`@bN*iMLDciMN#VdM>FHjBV^#WFnE{7S3LMVR-z zZVBCoyKjFJ*S+xPtc*!LXAe$;+8u-Xp#%+)ta!B_~4OKGd4^k>Y^IR7S7jW>P%lWnyB6A zJ>hZXjvj{Mz<+Zw6ZhGokMrL*%!WUw=)-sr%?7hQ!3*)ha(x-l{2`HQW)X-8bJE=i@Cu~lhhzQ^C) z<@((r?b5DmRF`7Jl=zQ>*9q-(JfzSVtJDJ1d>XNa?;} z2Hq}w20$$V$77ndi0OwFo}HWb^#(I%cH_6FddI@oBkrz}*-OjfLfN@gHH+Zaw-Df0Z5&1<_-gg-i246w;ak2&iB>;hQ%t7O zOFrj2JTZIeH_y+;9a4RUf)j*_^e4HgpG;;N+X*HHmIF{^G|Jlal znB={!ph>jp^F<`^Cud697>7&ydZ=Hvkg~Fj>S5gX#nFb-RJx7XYY!^`d7d`bn(i4aA!=r@}x%RlcS%fju^fQ`hQ z$k+eX43vpdMMQ z7XfhW6Ci44J4QMhgbJ39WL#c+>A)<7kJcnBvEyT}w~}2euY6;9x=8A|^TKWNDY+X+ zb;vWLn_-eyVRNm-(5CqKOHZo?Ugo>?^c|*o;r(S>TPpUoXU5~=0-0cu?}D$t#L7C) zhwyH1D9PP5+YopoEM8JOq<3!Wbm14Oi60JgKUn-c~?oUf)6I*bIY9d%>O9FbJ3u6Z{N*!cMFE7_)sCH9Dd z(k{;@_Kk)blS{d{P~RNJN|&x<{dp9|39Ws!hcX)IlDMy<%X0QDX}toEtDb{gyf{!0ZfctR^?QxS`!6c?1xd z-#Rs6){g@Kv>P=P-%13rN&y)!y+4q`GoGWMph>BjjtOwEVonKpd29dx4E^bve)LcF zSNC4<98TJn+uh#4N8kRG_l?Cg{E+q4#Tcg2dfq`f#l5F$I5e^9CsCkMRLUOpFo++f zCnEB`vWwg3Vlz2X03;`Hbo%wIOHub(nL$^xsZSRmdPwdMCg1kuX8eW2&=+4Y@x!IQ3 z3;p4VHr!b^;)Jo7=+9jE^&q9B{IM(?=Irh*#+A@2i^$~H<*r-@F2z-YbCymilo@e& zv2p27i{@O4fR&D-v7{=k$R$Bj<`cFP~-zp>=b9KgmO zpZQMw@sHcTU_SlhAN{Y+_^W#S7(bRf5aEx#lB{_%oHzYXlM~L zqWNcl@!YRWCW!x5`>TG>|JFrV;Dl}X_i!u3qR8p{0|@1^oVtey;rhnHGh`(;@aMt* zhNt<{f-Xi>9-KFdTK<(uulMhFe#!Dr3%ba2oBz?(q1?xVEG*~CIP5J0ekq}ku8_~= zX68Cmg#xkcDhtF|Kp@>Hw#3N!ym-xkGQX0#fmj~i;POC=sL7U!5R&D7vhCz6%p;luC z?luR$qp_>{y-Dc8H*!CF8M0jdUkd#H{ht5I5l(%{H1v_1_aD8hjv>q0tUqIR5Q{&9 zc;8oKmc0Xy(Ri7OkT*;~x*tJ1uD(tNR*VAJ>S_sZI<;+lJXUIXF-16{Wb?EduVEW4 z|JP_zEICSZtF3CeJ4f%U5^vOu+LJ*qM}sSUV-(^`_F)+|UXr^$x&SVzLF>2@(hW_p zAXHamkq!$!zt#)6#?t=yuiXECa?W3u{?HA|!}i39j?Qff89f6Z5^jr&i+qL@F1j<= zPllS{!hG9!0`%&|*&V*z4^>!clO)1QQ1*qsVjQ54Dvqd&qWii{wqjpW#g?iQr+o9s zQ@pO%-B+}ys!Ygc{nS9wGo1tIqRrGqtI^G>(!~A8EKa}vC-?hT{`FUlF=kM`d&=F( z^W80H%8p+g1SBV?i6dbbsRdrjgq8L6>+AAIuLnPGXn(;MvTKsyDDJcs)4Y&4aLDn+ zM*ZyNd$2OPT2(IuDq2xQ8R9hRpwa3<&xbp$5lGszECXua=57X*j2&xZRdZ9zV1=6=4SB{mDOb63?lm9G2QZ z5O{ybihh=XwJ{~K`|7#AG17*FUw3e#O!c^YRC*#vWq~T)NvOse#jYMDnys>rjCK5Y zbI2~>3x0mSC(nvXXQI0up3_)h=MY@}HPrW6h_nwRabeKMKmuU_)@JzK{T&ARh)*=I zePS#(r@k1%y8n#R94|#_Po#f>%l(FNMl0POWJZ6P*-DVR`uc1We?5NqZ?Ec~Sd&mH z>O)*O=LeoB+L)G*0W9W*Lz`m?WZJmhU7M8FpVu6 z=2zxsJ3!;O9Sa{^b&PItq6`-F49%}7I41M}K^^?d2&pYC=luE~aLwUc&avaZ&$8a% z8tN_Q2O9;I!MrzPXX&Uo(dNw=hu)g$<1@)wSt+K&0n1(t0sfm>Y@Udrcsdt%?Tu!% zcP!8))}{3gu|7^13YsV~`KgghAa3|xBS)Js#ccjHoo4^Gf?`TmDhmdO;=G<6+>|j$ znH=;13GoDj&2>u}e}~CUbcm3w+Ee#F^%vyUwfCVlxzM_+$?RHY@i^h~K^xJg*6&F3>v*+l+%sjb(1eOkWjRK3N8s z1BLbtYKEV&;Fe14?=P{HNPeWlmu>)=wHlb12ANfhQZk&FAn>y>t~SdCF{NXJgVVDR zBnZ7rL$-NUo|xskIGlEBAm1g@t!FeO2dNwkwn&);PRGiG%%?M%JK#*-G(>Nw=RdW! z7x`@SXYdR?fBI)S&WYDd%I&|k|12GgS93tY;Y$C_J_R=tdiZ)wF8R!Ewic1jJFNkO z8;lWeN1l&z2suG7wZ5u_JZVo1%{xa<+h2>D_N?O+) z?;ptJ_}mdEH$!R+4#o$(HxfF%atf$4dM`gJ@i`OBb4J|2_sT>q&g9-&r@28WA?vxX zIblJ-q;;R}uf+EiznT^~*!7cMX2rI z;?wLQO?u`s@xhF+;8h=jdmUZ`h6V_h&Cphm1Zf~3<6gkMI!$-=U%pJkaaz){J`vwo zPByFj+?|p30E>`fzmcu6dy9#Zx1`9IXJTtkAD%~jam>QwFu_Wf-`D+oDaIi+(E?tR z?0+@mK|jN|B(20$-5tAD)oU??(E_ec(Ak%eMUVQZyfp`1pFVs}?@%aF=&&g40OjHk zp~|r9Dk>+wM1OpyO7#iOWG^s?PI& zZVdnZ>@$Bv%h{a%P*$G^-(-<}{%5p2|KF5Vrs~1B99CGE;dJt{@HzSf#kt&%TFg|d6|21qneXkLPqC}>-=y4_a`yq{_m|{H^cgxD zI&6%4arVDfsEx=hQIOBl8A`ckbT9Uc*O_PyCW#Pov@3Z&1ol4Fu5#)uMEIwDU@1Aqb1-M^21@8U21NPju8FyFK{_HWR)=c zCO2(O*@?y>}=I#5eiVY_gHiYDv=uq85z~xk4c9V9q zV52+@(=$i5n-ykJS1c=P^O<>8e7$A`V#4sel!rfHE>V$r@Sakqfz~m{@oJFjqU!@$ zkMzlfYF2$(dS2`;ZbVuCVyFbR_01Q|mWIu-d4}P(_!4v@c>K!E+B*p!Nw?Q8IU0Pu zJH55oIfoOKd(>l9qg5JmxMy#BY#CdXvH;Axk>*(kG!HBnZH6*jA zPcv`Jimr6w`$HV-(#HXl z-({o%%`7dU%484$43c32=Jyi=eP?SoW^CGkjYY%Y8OHa`g!u;7K`1l|D0sA{P1UZ-r~=A{S)CEEV56l z2a9&=dOjQe(NBC&_}x=1Lz3`bX7&kI(^Ho)@4vx2RSs4=#3(86}V0 z<~j%am{pjEZqqSqUii)=m{lQ9BgQ%>;{aX~2HI={$xe=SQvM;7hEB=Vzi$!X96 zrS!Ve<6csJdPQ-9pb1EysM|kIJ%}A_;_LF2keb(~Kyx0Np}4u4 z?TbH`6}|j6xs~J`8iT}{o@r<)OAKMfkJZ@OgW~Y##!FQ$c!;b;7WUky zN-b9?MCmVA3`m?c2URmMcW0~6%{)QWTA{=gl8eUvZ!|d@c4L3h zclDPGWG3|%)f!WR^A*7Q3%{YYgdJC(@JAhvxu#Z;1-qOJqKi^yJL)^9ur{4B%~nJH4Of!$@|gDWZc1 z=u!($INfjO%$F$yoCr62HMU)$2Nfy$sg_3#KBww{RJqp3$67s1lHY-(r)w2)QF{Ol z+uEs{O!B~ljz_?(ywLWLFv*W)rA8%Vb*w1s$QwDD)WSBrMECTx6_vx13!2PN9J^D! zCvLDL_nvY{SGi(K8F+UakNK zyI^tHXl71f<;WYlw#kFo8x||;RNpS4S&i|JZ*rPAWYyiPTpKLY_vS`8VM-m5!8*yp!xKEDewU z@*49mcmOxRl^&R|SC-5EJ`5EvF889yE{k+l02&|;?N?FlMJN`oCNF8Z_Z{3X8-Phl z1Pnjt9}>!P9|7bbBMKh|iOg8*w6*s-45^)XJ7jaQpu~#H!I7#aHk4d-8QNc~b{=B5 z&!lkOU05!b%yVDZE@RB2fLzHr2tJ_>yyaJr@$w=>zmCI){pPDmX^v057ARqA758u| zTYn}}c4AjG$+Td{DbZzatVIZ$>eR48S=21-k!O+}!rzOR20Ripb?&fyyxsju5c70K z?@=KR0muowOveH?NEHCjIq1mtPlIF2sUsJ^v4k<{9d_OmuRT56uk46+oaw8hwnrt& zI@eHVlbkOp){e0F#J>w z;i1C9n;T3EZLYSpA0&rRLTecMZp^+UV~zR3Dp5}TX3fbyU*_EgUp^HPaz_TH^z}q; z00iE%=LB-SznyJbzw}G+$x71CN4(^3Ecc&%QuO}&pPa&uAbvcKw)n3%`!OtB$rCoMh_zr?@~8D!@gV&k>(S-y8CwO7QxOPnoJ;(qec zXUZq?yL@_1oV!|?cp3pXb{{G;giQY8xp6+^f1}I_a1E5$ItDHb^jmHFJof~7X=uUV zgLq+VYhW}RsKEo|1l%~ zXG;9fl=v5>#Q(7iE`Q*~_oruUX{n}60QtM;j_qn#|~pIW^+_b*Hh6ge$&SfJ9h!yH~FAqYi?_v?tJyw}{jGBbmw z+S3pPTfd+BJ(ctQjbssCGT>3?waxQS=Lhkne4~Ek3V~ER<^4AFXCSZuhSomBvdj5q zF^iVj)qYf`KjxLUWPkUyxpnFT_vO2LLda2WE*eAoO%_{>e6C2*vTIw8kpmr;}YPmOn7ICNUPu zhu0lR*$J|->3IwzCcE~=7(LoJFiRjTspN&zEo9~vcXayaDA=mYsXf<%^53a68jmi~ z0Rp43qZh!x{?B&(KibWIP!mr*M-+E*AlS04R6OjJWNGm}G7kgDJnf+~6`n03_fy2Y z63=;7v@GCqLJ~Uoe7Bb1WBI9N5E?W|m8#aR9BS(kzWRbF2E6nt+*HB~xid7~gWmu? zf9Ue;H)!0s?@T?=EDc>mGAW0vG)Zd5qt4fv#Y%77{2 z5?}8i3v>DhI`jvM&I|y-ayE`;15;{|@<;hg&2+Mr!Xmr_CMz={46<@dorp_Q>JH*k zneU_m&Bn`v%2V~C+-hGA?9T%QV^_~nzYa8xq|EB3lHCk7&>}e<0b7RaDa{?l^k=^W zNLgh(#69UYk#y{awMM*p@+Fzv83>dL@?2f_>Rf;0R;(XbX12nSW0+s=?deYMEyqJ>;*aKG{r8UNm_ecmqVXY!$D?x%u}Bp8#Mi27yh zLVyR%s%&*VmDHfXV^pdVx&?xkR()eJ>vP;_#$X#eHzpYATlzbMA}OZhTKRmdVM~xg z9IRt2T>EXEtbB~Ch_wU(8BC{-1&b^nB*+>tE>5~2YbgW1xwV#f;ujYu0|9-HM6uV@ z>>+##@Cq9v4P0{b9N}_5s7f+eIk~!0!}*0!R2MG9cd@tgK?0H0BZ*Zo-^0%=X8ep2 z+&&Y#cs^`v>rCNpM}J*QTAl~@TJtKlOS=fBBfbjaO^@u1rrFf$#EM#+j)<(j@?_BC z0h1I~#^2y$DKDD_oO2ktyKX~*DToOb&@Swu7;fv>l z9nrPC$pDf8+^Qn)xU9p+voWVM;P!~OGdIq^eM&Af;bRDck07j*?K?)8(Zu1Mr@OHE zsRKzmOUS`yS;p}v7wY)Z;oZo-i+5w!@*Qal+dvWh;37dp?U+#C@)cu-jD1*?dj;9o z7gP4EAQ~>)Hvp2`z_zVjMSrf|;ubCbWbM(TU;(VOlrTiD18{&1AQ~8$z`1c$y`+1N z#Fz+EUCOF67OX5^7tO~KJNVYe6Wvmz4wJx4cZK}$b{NGv&A=Gqnp10{F&9i4&}+E6 z^)*hH5E_y+A-#5ox_Fdw34=L%*|mf?bPx?eAycmcOBH065z-3CO^vxNs}{>atgX>+ z6yM0zo!}T@*Qo-p3xWt;af=8dRcN6D(ajXle@?#uKiQ~Oz zF5G4v2~%I>)BMQH)js=Lmsi6-*=^K7$jcTEP~x$CQ+JU-m`yv@uegW=ym(_e7yx0M z2+h}Zn=GtV=hczev=UY=aw(+wA_dn+lBWr&R5lpj4-87jR1$kXw9f1c-v{pq(<#ZMk53 zOx*|EAMBgCNyP`Y4>IY&lXMf@^^d$vm+IulYMn?I#g^JJVR(=Am=XsxCF{#U;aw*x zmI6sI4yN+0W$v_1AFvKOst8fK;O6rFub-CWGkGi3&WoVGx_1TVxuuYF(sinR7Ek#Y zw6Xj%*(U#j%jiEiKmN%-|9aBcEx63Ik_`IEewgES>D7dt_!{HV5c3P|2kkZr@BYcF z{vUO^aqWS*V%tHh&5g06g4F4w>6065@ok*%366G@UzvCMSGSS3EaxjO+U@JZ?MdYb z@=A6=q+NnwK$18jNqCjhTj3OWX{BKpoEBReS*i@v$?0D)hkNefX~(uqt1*FPr)f?D2p1cXhp4oVfn< zrvZEZaIn2)?x$~Ue)V4YRR;Rg<)Z6+?~d;Lo0!~SfD2yCgw06v-T&8a;Q#M${gk|X z9OWu^Khbn)Y@*$At16fLDmzxBR(ujEl@CS$NUpU@>mBl)#NzZ;6%oeQS_rh9QPkgP zI=T#!`n0c=H>W~(thwS^JrE}hX}gDr*t#FurbBsST~S-qJrv+%JT&sti^a>rbL={2 zz~p@d=qXrdkx9kjRO?Gm0f;;@h-8*_{Sdu6)K+ISc6Uv4XW4OFboJWRq-D9g3KvDM zhcj^3Zh2jAWuEj`16%*SA^jIJLRKV8Mt#o4%f~0NQii!cS}?8!Q!?k1@F*S$tXvoP z8_R9L0-e+I-5}|Mo?Er4sj%PSIfE*~L%tj9;{Z`{^h{U%Mry4mATztR#{zie%L4Jz zoH||>wLYW2w;z4+z^=l*f5xTY!@`0^N>%H;<6Co6jdVq6LK{CJA733T9oi?@OOwq; zaX)ZQeVSLj5FSB-7^|(l0j(E*OPo z{Af&rM0k{M4}OM)qK&gFCCd6^(^MXHU9@bpcOW&zVoSk+8TXNj;2Riqm z=ctI<>Lp=fV_$s50!mq^3}L{RXWG~Yi;O%HC52#ve(6p%K$g2dxv_wZ-7-+FQwBZ9 zwCD+@6PinSKs14EF$ZVwHe^xtCmXVqh_AVPT>BA9p4XT!Gbn|*mF-EgEe#!|XC=q= z^jIw>XIl{s^okzq4-7C#R3diMq|a#X??fcY!-Sr9zS+a!9vgqHP(R7t0C|u%zELxTDce{is_>e}xoG zs2=X9g$AlInYVMJANCaMGHbXaEvol=gj!#?w@J_D$_-A(a3dP#$7UB$<|(V_z+Tg| z#!U1xqynM6{ee?%ln89QD_02D8&B}_2fLwZZF?5h-ZS9OZv)Ai0oE(XUKD1 zBf%X&7vA3~Bg~=>Ci!cjWeg*v>9u_#7PVZumXvG21r=Z$p}XMwM%K~JQC+RUI&U3< z@M-1xJiOwjC8eEnfZ%C+vTWBBq+8`_ zPNm&dIIz~B2N0AN1Seb)xz)7SJM!acmanvQGs+Cboa}2|AGD5h8xf05aYL;S z5o0E6nr+LDx5Zb*9`t@6n-lpCX(jg!UK2TZ98}I<4jC@Qj*{;6g6BW>*D~SLe*e1K z&cT2n23tz5fb2rdUrKKjR?!7hVsyQr1dC{K>JU>klybu_F{9?{Q-)2C+OnYx(Px06 z@BtbB?G$~@nSvZ#Ha4a(KN8`u(W^hOe{5QzyPqbWei-NT!il-Yv;DY?VD*rUtv)@u z$bCcGA||PP=MplJiS&lnoatRl)!A)hxD5J49GoES`nnz5Y#K0kUoP-aa!)`RHYYuT`zaZu~~JN9?E*^_Y&l~_yj{ET8%MV zC`*Z;dPy-nI|8Kc*BBOmAk)AnRJIJklgVkaT*^X-sov z#_D&PZteZ6*X%Odi;B-`g@q4G&MrH-7kwI9Oe1&VXiUa#%nZ_>%Fa!ns%F(T&4 zga|JO-Rw8+(`4A?A#j&VwupvZzbiGn7QYYbz=xKSB+bmYN$4RCqf)r$b?Khj0*yY1d1G@|re9Py!A~sm|Qp#`RD^Ezdbxs9P%J4IR-C_hwKU_S zXVXiclCWaaUw7Sif~Je|Oaa-sxtKsGTq{W>8xC7YAb6eUi_%GXbXUvZL33r`dAdU4 z;zNoh(5ex8y`UgQln95jGn87X&QbMv!;j$lfR1`H-SZ}YYSDFcgOYg3oCq$CGEEz|w#MW}!Xxh_>8QC4(VSF!=y zdf2k>l_kN_hWe64?y;O|vr(`$<%USw`UfqjVM}XEFR`LjiMK`MC6>n$Ap{%uw|8*N ztv8Vx96vuO#eUCfb-UW%FFz(GkyLssHyB6&WLr(cAuyQH~HH}A`<)u3UpjQ_-ILvEkaN+2sejCK_lCMTtGAzJV1PCt?2!sdo`1=i}drs*t z*_ZkFbenwN%#1`tUNC%FSCrzpCu>} zn2Fu9m=_rLt};HS>ia`sEi85_boc=Ka8D+ky>wj8j+vz)tM!%8hpaFmv-^(YWoLvh z{CBLN@0-N`{&wQGjTVEmaR!3+7zN_EAf3GsDS~9apooV-%W)Uz0s7187e^^74hdqC zBh1e$0-2dbc*l6RWB7F)g$hwJ1=gO2o0h!NQ&Cz7eN04 z!4X0~BjuKeEVSA89K)nW6MvYn6rp{s<-Yo1Q~xc+L{U%Bb4Mv;6}?Vvucjm7`wTp; zm=DI4O9)^TPzDYk0%kvVL?_g+HaCfGeClYO=R$;Ze}ru5)q^`?^Rz0{WnxtL#a8*u zN<1#GS&#p=#tKsBo?UoA)`~oh2=XzH&4a7-X^_|ok3}97 zB4KD+r!bc~hI!MP4)suNSOzct;vr%YCB&H^QKIqtqeL*xUJoFZt6#n6xf^xTp3&5v zhw}F))dc~ixCyvk9SIUbP17ay3ZaIYwa3P@^vJWss@nFQsTHyNjo^IYSBWBrpwU`c zkR@!O;CA_Iq?fk|RQBLvZxKhxo2Ep02$B?bSgRW<>Qc5R_=;=lhA2!gj<{| z@LL+lxwiKFzCe+~4N115q2)B9(90Yu?G9gg)f5+?Dot@Fy(a`?a>s5h6|sastN&HSN~$?tS%IJ4kf@j!oB4yXV7sZw57uY znS<-y2`y-#b0eDvSH(2Tx=q@Q2s|3I`*}=O%`qEDm*gJ$%#*UuogQ~0k+>WTkO2H* z@hVK%PnGAjy1AgTdL^VymE;ogsX(u3`1OWI za~%ASav{ffLxonV`@;PA*1c~mh41o7ikK`ntZy`P$yFKJa#%mi8yR+?c5@tWCcAND%>x1D%O-hym2*U z=7NGJ#0Wlab-=)%63DBiMXkBZ^B^t5jINkp<$@@l-f6}kreb}Kph?61H@vtH-8`!X z2`lA9i4qg?qAIU!GNzAaQD}1vIS5;bopBDI=S(*4xN!DWz_Mfqs4G?K8_PT7Yu>@) zNmVV?9So`v*1_rT1ASLQQ6zJ5fEO@Smvrl(l=7{(!Q!J;*1I#hEd!L8NUledqSo5A zuF(T2J7ss&15(`*U%U4oykIpuRwL;m%|6pr^XYAH-l(>ZT$gzH52P86L_E>_-}nfAP;%Pdcl9qYf0yI|Z`zlYI_R$bq4)taVm z1?T*`fLwcWuH#~hBcYk2SI>_`%yU%6Z+c^)RC_eJQ-4P>z|$tXh%vLSY$>r5Qs}hd zFXxmVlBI$nG3}y@X~*j#R#eL*~zbs*0&^qH$L6qz6Mkh{JepTOCWqg`N5{FD3G+uxYZSn`(g6C8OV0NeN`-02^$v6+G$IN!@wu>c z7`#nhMvk65XgA^li3Bd-1;YuLs$}*alm4dnt%*tTz143fY_y1~NyNo>SGCcx=a{He7oDFfRr5t=_Y1q2Jf+@8sr!X=Nu6n=f$mi zLTF$^TQBpm2|d(&i4*{zxMAQ+Ot5!?`F)b`h+c*;p>*?tn6u&7`0h51R5o&RvX@?M zZA}T1+_>FfHXvl1d~YaP+IdL_C}UvBvv+xA!`p4lS_wc^Dd`D4{UWnnP{CYZCP?P!*Y(sJ zEJ%9w%@6RcQn;RnMl1rAP}06F3+Ym&=jTV&=MViV;P~2^A!%Kg+ZX*=a@n+p%%9&H zCs858V;B^;zIr_^>}mLvt&XicFVT&C!TgSwHyY|yw zFrDE`gWHD1hTAz2rAo#Exlq#?b?<@?+*ONL-Da{~P-vvj1G-s6Gz=X~hsgc5^HuFa!FNP(BGGPf72iqGmBs8yvAhdzZVLd%Pr7u}RIu{^U?<_{JEuxK z8R`6F(&&hD?}4SHpum#t35??bWv%7z%1l~Co+GwNErwO3j8h^ zjb@Z}ux34Q!DZUWoqZ$c5?bWD<%q9^2j=NkC;5~-{`K#@sK57ru}=qF8CRJ#=YRAU zp-i|y{2zeh-KKw*fqh2{RJVk%HYZvj^UcAkg+05|`a{!;>d+|1*Y>>Oy5-UdSW?~{ z`PEPZn>@lEj~x1zH#@Vwkhf0*DWV-Yru;>s+z)YPf6%TQZdEXK5s6;>l&r`!YGb8C z1WSGR9v|KP&t3H=_H_18yIpgh?PTDZp`iCfFL-EI8?+~XHFp@`BxEW2UcZ7|@YW+X zQf#Qp0{iuBPb582>tlUKuLXD1N1_R3P=!(iAG7ng({}_xdH9Y62CC(Y{n=`Qw@r&G#y=$Zs&nmZSy~ zGYCFi=q{sEWt+cqMYO4`XyR6t5*Uj!3|p~gqzf?G-ao3`K!%==VJnYyn^^B%Ol`yN zEi=fBs@midyQ8?JxC>+AH;{`{=}zfCdKfhLLA0|RJrHiu1ZF-@N3AhWUp5@E@W!q< zcL{y?+N|QoJ{|rY3)I;yMX{{00lFSQO#`w@re?E;d}%Z$!kva^GuomDL&{cKC0HX$ zOh4rVWjMJzN{7El8WI!FJV<-5B@|20oFyg<4JHF;;uN!XYH+2Kd(Rd__=D{al#|a9Zl^ zf+hDQM)`6~#o3m~V&Me}?>v6tx{lj4wj{@2zN-(@7lIWK^cS%q1_UD{8;>_XsQnyr zUE#x^{>z(1$%gQT@}9MD5g&@Vu`jj=_Gu>0bk3zc0>642JB3u# zw=O<3`VBP~rhc=w>5Zli0>R3vZZ#dr@!lQV&V&Oql3L65a9Mx6EpI^y0@-v0*xJ=O zTpsMpds}U1b{&_;tWPl?iAce6fV-)_R&nwM^P4;N=f-*LmmgcpI62e2`)M+)lS)G5 z;iZ7hY~B3fi(fhP>0>GBb8cFXw1lU9l8?EfS6>S+`78x2p@tzl+{?qL?b^&S!me)e z&7=0gd0{90BCUd6b41AnOTm*x^) zx`y_AoTPSCEh7t1p{C7HfxcfJPhi5D2B(F)ce$3Ep+`NN%<@QQh-p3PX$ATC0Z5T( zEr%jN%2xI&B7k#SZK?m2U5IFN@qHBMEvma885QiraS`wAcOxe5w+^sg%4{t%G?P&q zabugY2Ubvjc{3X+zRa!MZXy0`^yKUERVQ)$7zuKW(eHC6K@f_7rQT> zrN4Ji1d0%+W71x_C!B7^ZBAotMy>KwLOB&^+JmP74(-x#&&ng5!Hvnxjm+M2L))jwQuT2NmA|jp z43%2jnct|tzdUvnwXMF#h@g192yj6JC0{kTbH8E@gwI7mpQW2hZT68K#@l|Gu89G! zZ#~HV&>wtI*bvb3>E3%`xCWCGrMa1yKC_?!fktsNkXPd$B@83QV<=ru6$~KWxZJ$* zk_1psU+3Uk3<2NTFF`_2_r(aIwxqNpW${(V_jtoz+8cgY0^NVFqWQies*c4%)6%tY zXF-R6bnA}BB}Y3r!qEhAVXKOwA8T-rw-uIjrQ2l~OwrbfT3DfsD9PM~i}TtWA|geaO4ZP$gpR%{0qFt>p#+c)fq*0sq$?^VNHicIUFjqQ z2oNxYfYKoX(mP1+y@Tk9-@J3and{7)GryTR*O}jS&Hg7l*{ke4d#$zCde;5i_uMRL zS^eSQ)M#YIwbuvx!Gap~#W0eFDrm7V{TEX)Z;2^f^h?UB#*YN5{YrU1A#5I>GhVP~ zwo=q17~kU@!Rj&+F9|!J;U2hc_a^KYlbCSN4*%CmK@^r>R+v3x%%~qle`#U@rtqZx zgcYk4HYZ^Shx~_9(q#>vC+L?g`3&9Feezrhxx)2|b1WGFpN9=Rg+li>{8^Ksk! zi7EZX)VFW~WT(oL0-g3Av}{S)+6fFN#dD@->c{ydhGtM(NM#~o<-52hz5TmX_MH@r ziohazpKPIL8Sz6H;w`Sg^^^ueI6SawRJ#sK;nlT&1MZHXndQiVz(@B_^Ov=Rn+M$w z`iBZv|EiFsuVjO5BaK07MRf|^g5>(-@fnii>cM5t$Lmz!_$3D0#ROK zTgbD-;;3@*3whp&)Y*@C$Aw*0NnvBWOYK}gw6a2qG*+#06KLkS5m{>1-PLhvKYH92 zeabjqOt*-g_!*u^7;ftMndLJdk(U~p?HrD$_OuZ8 zCHm*dd~vpXvHcaAnSU{{fE(*4h+RBu0C{iuzNH1$N z=I)*dP&v*qZW!m0%6S*lO+&8ccYs8VTmj2rB_$5A{b?7+mtQRDoSXEIavSDdMAzgy zU6{4U)f@1z7DDke_yXM+p=M4X_=u%sX@eJ96uyQQEizxaKpn8u^eQp4b-i`#nyPE9 z=4Fvpqgkpb!-X2Sq1(eVCg)aC+;ja$qR}$9dFhYuwY_HI`hHXebd74>eDaDny26+9 z2&j6v0bB}1D4yqBmZ@N$gjQsdA2n@l!>q1B#Bjc8VmLzIAN$kcD|=&3904{AzSe6( z?)9EpKso1%Ou}>F91W7nGR~5%V(;e0UB9wFidzCBmaI=d#&|jX)F=m?SalMLJ(rl{ z7n~)Qy82GnntJVP6!8a>r`*3bVf=sl`xTF%UcHT=4NlBoZbPOYG#9~#gAcoyOI(_3 z_5WmIVu#aWf3wtgm=FfQ46&KBxBmR&9s6i~a6LoK!}}jTk26?hU-3Y&p-r(c?TaVj z+1~0cg{asKl~@Q2U(I>r=^q-)b|0&RWlX-*db%VTp34~Sv5%TUib1Y?4F0_1rt_!; zv!Dmjb&qSmf{mI>9v~aveZBf8UtshXkJ_@W^zttG#1wXjc@G2%*udjgA&W*jYN^;S zh|rJ_{$tU+Tbg;6ad1X|g7`voo?XNoJzx41_c4}z%WuIn(dg-vmj^uof=|e)u`KSF z*h9a%x!j$fmD3J%MOo^!F|zq}FEMc^Q^jg!x~m+@?fa@NKG?UrX|DvHp#;FfTmow3 zlQp$Y0=;>Juq-nWOum=(903W|Yo^8^`94cv6TiGzRdIr&PbLlQegAx9uRd&{a+>KL;TbKH2Y%WOpN%Iz2+*RFx==oTCR z5QGtueA1EjXs9lq6V0f@xVK}i!%#r@?na?&0)?$lAiZEpA2Nm;4(VTrKX2aqGrkHC zfzp^8AKVpjiAz}qHTy`K;W-$%Kh=BgaUoUw{BaO?m72c0F96^HVl1&FL(V~yBsi{@ zH-nx$2e_o>;_D%La}1^%#_{8{neS(`&X2crvxkMbuOfH=JhDR~?^eZdD4)iJBBq;W z;v7Ge)ZfJonizeK%rhTm7`zASS7@GC&(HbElJu1;WD%sL@$N|l7taNP;4}oHjw)v$ z)B)Qw*-Pi`^6C^#`Y?~NVzB^&F9o@ySQA=g_f+$vnt#oH*r`iks>SunAwVwN0)H-i zgr_UdSiFmPG4{Po>Z%>NbAl_Qknqu6FZC9G>pay60Fc~LZ&i|&e{|YnzubMiAbLG` zTJp73sMVV8W z1cU@J9*TMOYA{_(I3Oy@7v&1WM^vt??D&aopPyTtAAsooh_wq=W{}2sfEMq6Kc1n> z=BRU_%>5<-ft%Z{na_mv$yvh+dRijJq%jX0u5KNLw=<(HY z4&P3LfDm8gAo`2!?EaU(fND%%nYjL* z+~zN`vk>yJ>GKI=cCqJg6#f0`zwhiX#xXcF(^H8)s@t+f>@eV#lDGLC){pr|HWeF7 z_F{owOnSL_>wn|54!fwN_=HO_stzmoBrKc^3xbLrEZ3$58&<7ZW!n z<41`OdFffMj{X7cI21Z%v|P0!s^Rc)nQfAqW%jmvCUp?sPXa&c`#kVHA+n{9)I>yj z12~`Yow_iv!ah)6;Tm+g)+9jco*Y57w9NcW?nE?59ZqYFB7)F*a7XHf`i#`r+BSiG z5P*OZ5W=GaMk8z;(9Z)OV=LS2aE8X?4~m$4?{F^Z_xGNsY5Ei|(Jwq2p77IRH%5>s zYZN6uZVKo9?Q+3hrsJ*XSN%Q3G{UgSWaN{RA^S+My0^C;n_?|aiM2CbFqSO=ygDy$ z5aN7W`|;fEsAyHu9PI%W>&qP!+4cbK5dbCI7w<@@emD4;RA@7jGVUh$Dp=m_J+jB8 zHqol5w4abHe+L~e1CaG`$54|^B`gSJ@QOvD<|tx_ z`-@NR%8q4%4TdM#KuZ9RC{wt@DJqx&6_~RsoAUe2aZ-nt{^F83K_Xro;2&F7ENa}{ z(W^dSld2%1Z`VHsMnTvaKn7uu#4}(rTge5Ul85+-8ZPa|S9S!Hr4e1Yj9V_jA~n`e zy>|iJifoSo%0Fi6%IR-t$CEI&14WYD@$5paxjKu>S5^wNxWl7@FXs2m zSKLrclm6Rko2=gh6oP5S(zq|k3UikQxUDO1ft`h5e*pt31!3hx;tKm90kx3M>XT_0ZAIJ{c*6S2e=4+Rp1qiZV@r~$W3+hJn5u-(k8hc7{>N_ce~#Ve zG*m?XDy`|+L#7LVFY-E$&p_%zT`PgItnZHOMwU4WzR#U9KE`QOZbZR)>02E_-$@Wd za!wnL=5Ahc&Yizwdy-rP?l9~OsN)7G8@zw^-{1agS^^)QfS-D$$Id7E^}(wwGtPZ8 zO92$}x;XBK_gpW7M!2W7Z7u$N@;Ff=oxuQ|GNp8e>6k#OupjX2h#;qY;o(`A1q)?| zMXO|@75IcvS1|+x8=RU#`P_tj06)xepUQ<*3}WQKUTM}6~7xp*hL3Tt<6gu zMH37ao>)fB_eEtgGWWyJ2j!Uk1sT8MmfHJZqiQ2Qs`&4%@c(~j|MJ2~pO*i-hg>HA z{_hW%%{TS65r4DDEa-1`DLa?=+f2`GQ-8hl!HT z=UlW&g&Rf{ax>EpQhq=4Kg<5A?8QH;`oF(%{8zNgi`><2-|+Kh(Iz$Zs2=Wc$Q$s+ zTDvYzL7FhCkBaLoH_4%NQdCx~?mv$C|D=}iZ|_-8CS$_>coCkgJt6wX0q%bd4f12rZO z+wC{Z%q--Jv6tp5yN@zAT|vg7WecyjRNJ>>IrGZ!d~7%Y7_(uAqX3DRmJjwI#<^XzCY ztvbc$n0q|MD7s7Ig-W;9bd>9Vrk(21C^TAKP_ZcMcH8*!dQQBjHNSei<(4Qu+s&Xw zbPc{EKrz;yjTc~ScKf{hM!i&2U%~v^V%3!mu+ldG02Htnr{B>Ut5r*2ETRlqE0Y(d zS96c-{dPF((XU+{LRRfQ-9wcdq+-JP5*?Fs^L{I7z+#y>HV33jEST}B;}FbsQvmOd zdeg_)4K>qcNy{@M55pOKHl4Afc`I`XjpgMWkSZO%wA3lQ@`4P+NlcTgz-|jBonzd|v31m!m#j@1Ab1`84 z825Cv(S*|a)36Qw%Cb#gQ*{ zGCVTcUEf~1pH__3Z(=5sy5B0{x-Ck9b|8&lVJQKJPgH@1?J$#A^iZUn9e^POZP>N; z-c~*4S$TGxgP^SC)#_NJ?vc^VgV!oTJE-R#Z6%iL;)XCgbV7Q_Ri}s0goOr%vR#*I zS5;W5VIO$#P;jT+Og)isYi(Nfn}CRK(;!;RC*Wons{hrirRJhT@8(+{^9yNr(p_V9 zTeYsKbUKym^q?T>8rjpckkvd9ReQfsu0boVhIl~YdA^y{S%|?2h)h|Uf}kBv){g8D zJrg;~5W~cw&P?h^s*o$DZ3-P4B1-O@On8kl+e0s97|F187kQ>~Bl%9i2?S9H9i;TZa^+hbS4a506` zq!enO;L;3@W%pn3W3e;gsoQhnoFo_47~Ajyy;7AI&IiOa3SN)*)9!VK;W)c@aje|k zt7kD1tF!MXR+C~ZRNeP(e}1bcDMmAu$k?f6L@JWvAkH>EKPja zN`%xh4`~`X8@KC7jgf@4C!l?z@;S}}9S9DGvelFDtFy79G^o7d;{rd5H#`@oW-cm< zSEnac6$|uiy$b!sgs7LtxMro7@kCW*RZQgN_$j?H2utCV12a~sAURdUz_k)u>gdQn zx6&`BY!ca!&`mvf16XpU!HPzi2Y>JtAn+CvZ3Wq>oz2mX4E=SA2?s?%iD)9=K3qxL zL@Yh1>FqC*@}S+}T9hoG2LRlGCBSP!tK*rD`p8wVx`)7S4}Hw2N3icnm3Yadg`9#b zd&Bvx3b&4!E~;Fs5`*hAZ;TOK)=4V24{F=-_xDcU4$lakmmKSB2(3yNVBL1)uIEU$a|~Iu|2^pHCo~%B zAL&_nSvlw*NN%mB21gv;L_58d>M~nVOqB2Ya~q59>S~o0K)Qob@9T9td;kLYoPh2_ zo}xC<+V>^=o=dmxT&(pm%P9Ep!E3#x#TUXZq~Q#UxocF(kV1v~L|Hp}?G`jvDp`k; z^uI2yv!7t>!AR`XpYU#m^28(Q+y|NJ>ObTCX>GX@Tdz*tCXL)~u?1KVg?fRnx^})4 z=NxJUAfCZdxtiyoyj$BuX~m(^m@?c0M-H9KV|-h?e}{Pg^)k?xI8_fB+> zZ(SR`yW~;Np%)9w3Y6$Ad@X;j8IpHmIa;AA#n6yLt~oqGN?8&fnYi5-VEH)Asp$Ji zj$6(8%jMf&_+(eWUa#}gbCCAQf}(HK&f&wUIB43Pqwwe3#{d9<^qI)9KfSq>v11-J zXt*oHm_YP2tZUJZr`O~cWw1S9@P6sYPghr=XV@ndHQ6ljhc`(h(f(1oWf&a@%(!X-?OEqi)Rhv@`ur$cx5Z?pLe>W`m2i+HH_Y~n z>HJYf@V9Pj^o@|<@!(RA1MbL`Is}i}Vlm1w?mR*QVIum^8(N8<$;(DfDAyG?A( z51^Z6l4XHZdWiYp?-yjdkn^lKqBDZu^@wi;Mv+BszD0|FUj&tNBP=@})ux=fksA8O znDuGpncaf$&Fx$&!?~*M)D-f;AA5T66#pP7R`QekJbP!DdBbc2rcb&lVIBnwXQ(_! zMyDgwRIGPqL8N7@!60wIl1KQ0*yw?FWSuOn%~p*VcNbjwT6bexi%wxKwB5R7)7ErE zGFx7ki5srGpLc6gPu%TIq(v+nPS%XC>8V%ux=K;bCGENX;&N6c?rk_+eSETAn3dIT zG5=cGYfdbpPkZRj^=y1LS$)8D24&55Yj6sgkzA;*OM6gdXI8dj>Q=D$6j8$pGb6u*hJ1u|6p*z%XqJtcn0{-mK5{v!EQ%%sHy$u~%i0rqGay^CP|1EBHb z7*)2VV6WoK#de!7RQ-WhlV=BQLX@3`bLX!bgY>7G6*Ua_{n#O_J_3bz0{2-j-}wLn zse@r1{&N>IP8$v_@9DSd!SM`~H$X8#%V_^8B<_l>GrD@-akzZ9ZMdbBn1M(;?e?*B zKB7?io8BYLoJ#`FbnN`Nvn@lQ764};N$4^we+&k34hB}4f0ce3GJi!_LVX@%cn*ov@4LIC#I-Iv=}zrL|8& zWg@_cOqnL=3}T4w67z{7E@`)>pZZ+1C=5*aG>ET?1C)O=cX7yZYt@ujV#g_%Gu+i$ z3~a+L`4)=joAH2(WSI-}0fRx-(pl^~(Mv*OZP{K5*|_!0 zM6=7};)dKvDiEwG8i-WvpDvE)+R;GRU9y~Tuxx3CoUC{*qNk&L4ou_Y*8$q)V1G?MOI&s5^R_WLycmWfNUc ztI9#O2h??Fo-z{MK+r`NzV91LLie8Dy`o-Yd#@mKEK=w{t4hIYnnAf{vA>X?2+>~91ChQ(V z8}QP^*SlEBLj{^S=?H=k#aG#9W@g&AVm4+dL@y%&mu7Avc-ti&Z)`c3#VmVxOI{tD z6VWWLAdF)VujsjsHrq^C-Z+z2Mk1XVkHTpTechLiQdb}0^gVA#;>ShEw}z~}vh0o9 z48Ju8kZ_CG_SEiMuzi?7Vr8u&LKLQk`;aTcHYG&z;}JqtwyLb3Lm)`jo36i?c7^Ef z3-8=XC!pwljuDAYExuNKwQ-AMj_&gK!YmANizCs@Wk}Yr6$`T0>kY5&RPPR_Eo;LX zwyx~x(}{_YZ?$~ledg!(#4OJnR;rktA0F{V;w3HmzUbQv|GCwgj(Br=39rb>X2ozR zD?72Pn9e);p)$BI6mPskFSs_z@3Sd+CwJKtn_A$S*`Aa2To6~RtdY(@971yQVX>S& z4r%_ft?~+pta?5i)r@9k>j0VPGCs+EOg~}W&s#HHjMWz?$U{@=tAQOX7uZ+`zbQ_# z-{tXEm5Ud9*F|)_4p%Uj&im3dWiT&Uo*j5}>nlKkCHu|$!y7x(Iv2k^`FZX`L#K`M zDZ0$q3it`dDZ4p*E(6q+h96BRzY9y)07!L(JzLfYP1J7*#0kz%AazbQ+kTo<_MJA@)+_Ndyn`GIwG@2J?t5 z^`4H@ak;nTrrv7tjaEb1kukl-5JbVv_31^@VfiI_(-lU{m8D8dcQx#Z(h_(L3Iypp zfbT?;2N1oA96X4V5>@Q5US(4YHb~H^^vkv%O#pD4^z69wbd!4qY#ds`u}uul{sclS zl%ZhvqaE;JDSyDBWhd?v$(}jaOi`XLce(L+N~(1mC7hvmYN*_{=y5)j)1fcUj^`DG z%0kQ64(qt9+bLOU(lA0#q%=ju*QPGDvo)F)=UiC|TDL25GoRE7y;0V(N02(te9VYM zB1EG#D1_u;Tyyly62&J@pwk+Xo`GZMWNd3a$gk^99C+ad@kwg6a7Mh##U_4SM&ZH> zG_q&X3jyg!pVqsinR&yf{n!h5Jg&2?Y;F=YOd*sy)yW7AFM5yNr2?->#RP>FR3m&of{ zu|K%==Y+KbtvB!^SA)v@{90GpCWlTjXO{O^xf>c94Fj&~=*cF|%4k_(i0;|x>5k!q z#Hpk==ZJ-M>`%|Zu8%OZLLIzix@1mJI`OjIQ*ZPJ7C~c#c(wf*NRs1T+;%_0IE*B5 zq5D0Cej)ZsK5lNGwMflAiq#ySu;$y4Vs`HN)*bQsve(Quc{FR+1otvyf{}RyD}=iM z38doKp+K-+lb%jEg8V5+Y>M5p*(EMP?5asZ1_!%)Y4DdE-%ZDFo`Ukbp=GkKqM>S^ z(oKflE6+_4F{!}up)`MmweZNcI%O`;X*BQB!#XY#sMtDR#jGTYo6BM*YRa-Hd6+G= z`xn!n@VJALbGCGS#iA~&d~X+Px4fP&U?A4^u1;)0GcRK&k6lTnF@BlKW|^O)KfuRc zHEFD=zx4+#)ba=oj*j%VSt+z;T^OJ8cN;c&@-e2@X8B(^OI6pkIp*GjuyBal6 z*zoj?Ly$=Nm-l1#j76IO$gH+3m|<2QRGwjA5h|gx^ampLbzHM}7yGZ`qc@SYQ8L%%Fy7bZ1f|pmqkboK&vF{1tV^Zw%?<$v7@c*bZ_EO&=Anr`k?hknlpw2vc@ zPc%aj6@Ml)>&YOc0D$zBR#syiXY8fTshp*^n3S^6?(cnM;`h#jndTZiQn07092zXi zA~<)+g5Q%wi!ydAXMsH?5WRPHpC>5 zA8NF%CRf;tdCe`|W^$^ajlDRAW|Leh;k zDfWYyIBv#k!sKR$EKTlDQ=zfFVPvLZdzt)Lhlr@aIIT{&sWS$anj2^XE5FBt*``p!q}A3 zz)Y*;dmXriMjbh(Q|m$abY`H?6E(WnWf8T4aaM+Y7$HVHHb}7Z8EMAU$1z?r&IWNxUh_v54gZQtQfgVVPXEVZWggduCEw}s-;QbH=SH^XqZT+E&lkhC7MvbTXq)^Vs zClY9Gd`-D|Utb-4!l3pMZQoTRGl(Li&JZt|N>19`J@?Ni!Ofw;m2-^U)$~0|#*MYX1@Wn4%*cB3s zmky}R!)x!hS#mSOF0K6lUd(F!3Svxx+N-NoOPblCch&oUwz=ehY&ouKNF}*n?ABBc?kg-g${&Ky>x)ih z4Jo=fDDM>NvKELfKLZ)avlP7L-V>Dq5Qu7rI?mzb2sqeM3{YRU$KWFqX@bhuUm!4G zB3kd+Hu7!MSBz$DPMp>~Tp*;HO<@RW9gocnr25+$7oG?M52hm|S;t5wg0UhMJ<CwCoYS+4z;}xVi zfVy3Ge*V~K7lOckzB<<8TSni3de)kVtLJ!*3R@%^G#RsC}U&R7yq4_F2qoojqhL%{lWmN2dF#AJYfOw!kS<+l~*-$kjp=^>_Vh zq|Nae=5Ru$bjw0Qx@Al5$j#v%E~t?7!mG35Nd_WsfeqTsAOe$ zaFuYkstFRIu7PSomq;sSgL@s}(?~JH+|(>&bBmv;#2dM#pn;Sw2QChOiBUgoOwO<< zL9;`x%`psbpFknPc0GpWAC%~1GgVI>EWX@|0fkFN59e9v?kataYME9K%FfmwYf8v5 z`#qk)t}zl`8?sVuU}x*RGY}MtRd!DIk8>0^1Y&!C4`+tRX8JleFoTpmD{8mA>9lK+ z!mDs@a9`{LA+t|h*NXLxt%|WZbqg=4MO7O}yZ4;YqYC^b22XXC`S{c-t}NST<-zX> zXU~R3d*O;nmU&$nTiIKk39^z#UKbAMSZScvW{xlmdBE*mggmiLEd+T;cPJ`Xw zxx-WRrl%jwt%>Knf)PzKjv+K1xS%Yfxc6FRyGGsLlvo`SN;~jX(1WO^igdj;1MrG& zdFq0pH}?7Nr`3yDb_JeY6|u2mi%D$BSQJ|xgb3NYRH=mFqd!<(Oz zlLeu_$ZBSX<@WfcS4)8@1^(?Hp1W)NX7dfCbas@#Hj8ph#q|=%RjJWuOn1eX1DxxS z6!zS110#z4G*_)-yzafS%G0lJ&1O0j*D~9COKa~6wVTw_;>uKr1q;~k0QS6SH5t&1!AV0D*czPcy>-YsC?Yb5Wo`Ua zw&@t?UHtK>g#*n2p{Aw`^vX0(#LrUNkW`}_0eiyefe@#;Hj7*>Rkp%hemzn;zpGQZ z2VLIPdG9>`bV^|q7Zxf&=8jl^iYMn}y%4OEZfKT{jdb3Ay6EV`gHQ|X6nTIqTMn*B z7!lPqnizZne9@u`fv9b5%oiTv2Ix^n$7D+OFoMk9o14a--R0!^i7Z2r@xm007SP96 zC4;gq?W|ecnpFJ7z|S zmvQ?;x;(}Hd6idJfI9V7Nz5W*=8Zf8iu`I&OuaU2Ar&z8+vmAToO6%YIHU2z50 zRGo<%Lx%}#NQh&Pxjx9<_O~;}ePw zKG9(_#K)de0uU$YT?TeKH0KQy;#}igsWOYg@`A0NSjN8yLztfD>@s*rM}z6SYdWFqNKXnUzj+%@v}&20PacfI_ap~Ox~2D@ddRw6Pz+k)Zg*R6^? zS)6KDN{btoM#~M*dIyBrfv2t(1cKCM8NR#>uq}|3VV;XmWWZ}hZ_Izb+J3(8w9UxEU7r!Tz)GV~;8x`y{%&6K zL(Cxl6IsQybqTOZuOVd_(m@o^aAd$*oqE>!U0xnuWB2EMJdiD#n-h3n9sxn`DlK6^ zxd9{^W`+!v&}(IIaLhMRQaew+Nu8)s35sC}f@=lTJQ<4S7y0y@m}|ZrCZUW|^EjJ> z_7yogDB0F5d7LbG@q7)(;&}jGQo8fOm#S>?qEHNerT`U-nub^ohbK`ktv*X2ZT_rP z;dMW}J;>Lv?)9mo&-|84_Q#^Gn%xTRr$V`21R`&p2GwsvjD{-T;2JsPDJbV};#zl= zyrTPMwJ>DW5CN?YHE*&-nNGP!DY;r|S5#+G%WrI5;_iIp8`@h2u`CsLDWT`Hd^EqH z_9W2K9*9{s0(#lMR)?qK#bihGM%7d=%d4bIHW2mC;JdGzB#r zo^smpBq&?;I5Wzl+awwKm=L<`W;gA@<=-)gbX7y9Z)`Xuv8grXRBF$ zfZcWfG~$xjTMLEg4+z-;A+72VeMXRycW%UxomPKYySwSN$6jcR(&aaodmqtF4kCUr zeKeDWv_e;`P4Nu{)7!}y1kbI~i9?Z#A@RJHc^=Rmx?mRm_WZj`|9_nE`DnRXw?X8SuDAbL_a_}K^0nTzgZGtb z&o`aVG;g2}iN3>mu4n$8o*$Cb^ilXalg!`i`j>RT|9nE0O zvA^Q8CZhh4IkxO?g&&g)Nc#LAspenF%WCMW@a$i?_MiCkKa$W({UcQTkA1&R4O}TM zo5!&$CN%j!?H0s_1H(k?cRoJ;i^Thx<3A1XxLxjl@$Q>tqc2t?ONu}5l6C%I1m|5- z*s<@?)6;wL^lD}#1Uy&4OKkTFXzV;me8zGLVRF6<{m>fFQNBA-0oOE8F}z_eY->b< zaHUq8yru+ar9E92wRKtyK74=rvZi1M8?Y;yvCOQNpvV0vz7HmJeAyTs-1)_12_JOG4($LAj)7bW z-`yW&x~UU1nuho8f-7+5SiJ*Zy~(SHYv<Y@cGLQR&s+`dt=mLedo{Fv?`4)@8CO z7L^p|?(SPCppskq#_e?cc%@UAymwI&;%Bs2<{mow)DztVQU%d`-%BJD#>feu=->`kYm+y!rHVVezk~0z-<%0Kv^5} z#TvaUD5ae}^O>5yZ09#nC5npF+%9#W|GCZJ$uQROD|Owm>sd)JyrRK+3yMM>1)Av_ z0EZv4!vKvMmP_%)i>Hh8e(+Uot5Kq#xKm_eE6+*Y*-f8b%MR2m>R)}w71yDIDgzf6 zhh!yABm##SOfI$z^3F&agVPoKxzeUqvnJDTs0QGap|&4z8iUqDC@;Sx?j);(*?TEa zhk7F9bl%|_6M?LRQB?90&VN8ESXe(XvS_9QnDmk|wt}2c;$dtGt5xzi{9*zb8^_!^ zps@Qz!VI?*Rdqwt|ZU*dNirN8DV8CD_eU}PqOcu#{%*@P#2k%<)B=jm$3L<1+v4#vDSpCXk ztBu?|{%_0*XP2E=?A$~xgG==!^wchR1cKzFWE+TZsx}!a-ou9e5uf&Mf0boNzuS1( zas5ew#j*)!oT#uw)?1RptI-T$6S%r=vCmd~-}UeW&EC(8YugLFTDIjQE)$7dVvijj z2Fu?I)FW};%* zeFT%cn_2OM%6{>3jnSno1&(__S-^Mu<uxq@+()_3X|Javf*N2?%K(lw22lVB)~XbOaTHoZc1)7ml# zk|y2_s?S*b=G$klUKnPln6ggmpxt;Q3Pgf`Lc*e!GDw4O_}z9o!X4`R@PLJ~rfIj! z=_JemZyjB(9hW!8UfkGoeZTMV)51%pJ6Yj7{(jWf!0eAvtJ(OJmR+*xOtu*wpDt!i zV}}aE1=tTSW(ANHS1pEr;6KofX5;o>v1C4LNvDA)08d^`d@9wj^(j4^QQ`^M5fG zOt8blg$VK8h8k6QO|&WGZXa#YxH|drj^*PW$lVv^qdDa=`pkVIM&-{zv!h$Esw?fe z&=dYoS&q`7f_7McSFUZnhZHefo4ZyLgtID=@~9umX%Tb!C+Z1d zut0c-*;gchBPt+aV|sM{ zzKI~gU=AJBiwx)JBMAVlsR0Zz(&5#ISdQjdEEp39M3okO zB9BgnHbQ+1^4v^XO!;MxF{a*mG+MKV zjT9uffg5;H;vJ3tpdopi7ig3R*;e-?B!F78Z(`n`zeQ&D6YDWg9Nj`rVIpWMjRZsz zTfcOet;FaugA1x?&?!~U;mSKwmr?;j$9&pE1K-_hthzahbhLTR3OoolxvgJzGa~SK z$a?)7XY?5lQ$Jd~m|FpgN7i)4FS|e+)5j218dtw%qn0zC?CDE0&ySZ>v387qUJ&*y z?#{`BWJ{~49o0DWcOyCvJ@hIO` zD(UWz_2smm(Z86sHxuha1lctfQc2Qm*Y0ZAU0HC|Kg7mNY`?Y~5G{FqHXN|z^HsCX z{T;qL@uHxyNb(@o0Nm6=pcvz{)->(q+jrgsIV4>iJG4UvxW^WG0>VpNeWAvaa9@M? z^=!FXMYiHq&S6hW4D12ITUG!e1ong|37LKbqa)bakOHgiYO`fa#;!w(jqmGwe@HyX zKRIfB*PevcbI#7zJ3|Dye;O@lMd=Orynb9ud~s1y8sVs4a4Y=i?)ccqp_sgdO+~MT zD@pSY3r*SYsgiv?Dh~$nSlv#B2n$2PaEig|;<~5eLYIBgdXPFVF3}@&6N@z|Pchwc zWq<<(rUJeXdWC;u&v(As;zk5nISH)<16{zacU^#0y3WkClD>{ z+QAB8FLZ_CjA>)43MI#u@7yZ<;xAMFu#%z3ZPg4ntR%iFXlC~-fvWss3iTek_&kYY zDXN~T4DvGZFV!n711?hK-1B3m2wjZpPLjY0kL^P3>is#pqrt+?BeZ6F&(_5|gC;x! zuL{m%MLz8mV|}Ajm7bmz?%oE0bu(P!ahySBJVG+X=^Q%T3wDE8_;j3eWmqd;+O?UA z#$QajvPa;sj^1FnbM@5K-A6diP|qOpUIffS+6F5foM^#x=|4VkzHs9w;=)%u-S^?A z{`{*nU}R9&?Vc{a;U^9f^jCFpdzaI*BcH`8D)w+u7ER`?XU!K_2Z_@XX zkH46{9&jbBnFR-^?Gl$GnB%h$rkTecll^Sf(ltra}DoqI>ouPJuSDb-tQagJCgpzw7AHU zM!hOYR#9r$gK{*BBuNa;-r*-$sXKbs%n;&}!1;_Q9%eoGs!rf9rZ?!wFtnY6!w?%R z$*sJ~v8xP%(PHle05S~o8CEZ_(qBiNx22oN)t1H&0inEP2)R8p<3)fUj+Y4OoYEB8 zvI5`HT1^E|c_0_ZNkLv+R~7<{~zp?R9ni|GhXPXr#dx`7vgKLv_(qJ!NA$o%c{5hX)$EO-RBk@ELL3s_VFca(V? z1sRm(Luh3k*l^5i36GH2L9ip#8nEXov?lYiQljE+r6ZN1rH$oK5CsNpjBgCLNbTBD z&IPv`x<}ENeI43SdNs2U*9t-LZlhC%!+RjQ6k~B}!;Z~Aw$A@}lmA`9zoY_16msFw z;VKbjyRJ}6Ip(+p(UTNN!iTQtt8-ZHGR_IGxjqFL~i5n5P7kk zlY=J0gN|40pcq{R*H4f1F3;XG{!;aHNC1$YkvxrF)>@zPlkAc|_0=Zgdp!rw@SZg* ziM5cB1E&b9>*AoI0vV+%?P_!3n3r4+NZZ|}6-ifYhjpNlg;x?J0AJfK-h?P|O6a8O zB<2V>Ql2U zTFdaPa%#aT*Ji$GyckOV@VEYoU}2hVxqmj@L-GD*6rd2ud3G{3Uc15p_0f)xr_{v^ zD%S=a%Tx1d`_wNd4*JD(EcfnA>DYD|V_wkpaI|(mR~?!q2QegEz7 ziVFAS;T6aBL(ebsn446)`jU3_)|fji*1`H#)arzRM@#gGrHKA(W#*gZ%?}#)F3^xl zz$bpxferXxvt8Q`pHoH>DTMguITqod=06KEu*xlj1GdwF)iV52t_6fh(w${qDWBsa z!fwU_{%OZ~iuZsS287!pUm0uzWw!4hl)+Fp@dA^{r%(S_Or^|qS0?W7PLC?99xqZs zUm2i{owjtP`aeMC1*%62@g6IY*XV8rs`#78AteK7%k`i5y0NyXXQ0r0J3>T}tTS zQV6{T2mwM-Itc*`y{f2`0AT?JBs6J(BqSIhfq-D4gApkK5}I`BiqZsJZ}z!+?{mgI zXYco%^S?Rj{5}D7^@^C4CvfRB> zqrYu=fZ?+iu<8u(HaDWdEGtIZk{T#7_1$?!FG(M=yzm_JxUalkuP6O?=7Ktli)d*# zEuB|Z0bKU!y@(F^=uXmS3ya6UezBzaUS?42Q6EU9fWffv;_O zJgy#Q9fGaJ?;dmvpwYCZ3Od}gf&Leo1+fJT2167*LFH@k)*0E}&O4LO9mWt~>lsm# zPntm37LrqqjhrkpY0lQ|3)WHWlE3x1W}}_Q>BijWmM>Wiw9vr3Epd#?A*Lx8DYj}+ zYeDCN_!fU|>2&|~=tn zrv#ik>unD=AHTzZGPJo+h2y7Vuer(0|7+MDO|8%z{?>DxET zUAd>HGu?!BtI-b8+aCTAUD>sL`LP*miIA%S6wSd2?+rQW_+kQ;dEn*wQeP496q!hoW} zU=vYI6WN1tWT#lJ@IQ||XlP1$I45uMzFQ=$PUrmDEKpNk?zwEiRp*6iRe_HI zfrgw+@_P%n&|jEDNU^KRJ6G>W;0Ud=4YyQfB;KRg-*dcnyNGtvj42PF{JP@nPY62`Ca)7qs`9i*eKT$$pE$^&AO@pqTA3QRL}NBwit$K7eKxg1BfXN> z1DV;*X&k1XYFMYHPqE+(yaJ79Un5t>${uME`Rs=2ysadc+*yV8iyG`HiAR^i0TY|{ z`x4;cUo9(cmJ%hR^S>zif4SmW8+)88{DVQ_8T-o}E!DXQOnm(XdSh2) zJ&6ZC-WjQQ&J8c}=P@XDVU^=Jax3h(?byiMk1PZ@ZiKunxpCw+$?IUkgaHWaR)!4D z#RB4UA-R~TM~&+>2ye7YW5e+`{-e#K&AQv$ zNS@uaX7QWv$k+comYHz`znJ@Cr$<>q{auxME`sbNyZ-eH3Z{;uqKh<|HE^;0#hX&6P24~so zEkqkiu)kC#{YQg__rLraUK?H2fM!sJzaIM6NfPmOE9{fZhCRdeGo9Oo*QG9E%>YHk#n^; zz7}OCySrhEe$&g!PKvK=7jMQu%;xhOwh94Cp6Zef<4cr5gj8!WmkEJciqIx}hPC30 z--}7beChmiT;W7{I{C^J)oUa=GVe8nBNP)h}m> ze>c_1y78#s$E5mf&ihk8lz#)yMCo60+*;JfwK;lpP-0JjP6&H@*G`_NQdOFneX`W1 zqU}EU$?PazMMHzdcNY@_#_iV9!!|G4F-;Rc=F>2?_8leon`EO(+wLm5;$i0Gyu4*b zWw`%ir-#9!S)Id%Sy_d6m4-}jjv4b|AVm1aDWL~{&+7jD!hdQa?bL%E%Yd?9-2)>- zuG|^3CGyO$K!?_^Ks-9OVj%>@n7S#nOT&EX=IeY9&r3n+#2ICsiY+O1cKSHNffYP? zp>yS9XV@QKoaAck6&#>*F57yNG2A_z;*w}2n4#+0SGz_$IUR76{0!CR@(a$W@6B;W9XS^hYvI-T9!fPNBu;K2 zLeN|OOBuDrbv5!tpZwN>27~0Td}XXzaB~ZH<|QkL$G(GCesHKq(wxomboFC!A#qNB z9y5MfIdG@(3w+`o!s04j`NsZ_ZA#uw>e!pawQSqITPcW~Zs}6fF^c&Kb8%}M4z`ZI z$?@R9Dn$-ApS|&3uFm}nGhViKQUL>A{iOiVi#E}WSi!iFAOP+NGCWsn+YeBqJw^$M z-TCysX5;Rgwx#~Xy})hbb?2e)l~;a=A@8?9zxroo1PL z>)o{|R*OZ96Px5F@pg-b#t==jauGe1fb6*xxn@w9CxLs--Azg5H)=tLuS+5TWanfyt?r1diHE13=P^tNhOinP8QaP%yL zcLX%yObs0b$W|CheRC)Wm6n0I87^P9yW^gn6tQ~*fwX-+`d6q^+iHOtOY)Z%f6{BB$InN8%xK<)3y3P0IvmmLkqf?!Q_F6{PvFCqnXsW-bCEAJC_=XGkzuwl;iWx1c{7_ z$p+fHE>G+IekNb$YyaH~0xxIuaweCRa!{)PyG<3dZpBOe4>vsC{4AbXV+ghwu+y(t z6gwPgx!(+JD6Fbkmxe{^v+ntRLBw8gjDq5@4PyMBZW(OR1yV|~&?HeH!AR3At zN_N7*{I6v{asm@u6f)C=_H5kR#kCgR$f_7cZzUyDdD~OjTMuD)l zVL%^0`gam*a~_E-hCbvq+j8q*6i%so*UFv)Z)?h>)%DqbN0@9VX4v=S;nub+Mimd9 zZa+Vx^ZP&l)Bmfje}e+P{9A?1sfATjA)`T~XNY_&DHRw5@ah~E=Wo;bC__qPIozgx$S{|DRre`eKxq~7|!5h4Hg{FeWQ zYml02sXDIi*BN-IhR}E@AnQHL-9Wp2+t5Qc$a<$Tc9r6qHTR%;(Cw2lUd7!cAP*pJad%PI3Ii0&kOn9tnCq{XHgX{dQR=QI0?<~OgA*JT0JCPs=l zkO?DL3EE-qWDnn5x0V_CQ0roPuQP(}?ZmGgr4YWW8XxUQSP0S&vhT5Zs^gqq?KFc8 zJd2%kGChaM8XdAFE|1~+^6L(M9rq*NX&f{FTgv%V=2h*OFem2SfVyqRho+_9TnYhs za4N`Yn&7JG9G30Fqc%1p>U1njFOW(;I#@m>!)2LZ zJ4_F6#Nk25?;G-wi>Td@IQlS^#{TAemL2n0gty%H@#Jdp53Q%DgL*jxdWANeKwEGd zDVtB2{<-(l`1|$cTrJ~Er5gU4*Lpav@FRq;mA~Yp4)n6FQyf$hyp1Lb-D>4rS93&K zZdYmtZB=THX}xJIRF=|e)yEQ|{Dduk?)rr=Z&KVMf)yAc`@ft}5agT6m?SqN2a*q?b(kuRCbwz+(lZ`|U)2 zHL%M)nWY6WFZhecxtTThKl6+%q)B#!AIe#u$B;Ao3lBwZZ~{bi{uV$mT%PA4QvcRNkH!r@4$=Pt5A{G6XLfwxi3}j ztOnd%cq4NvE=Ae<)!CsoN%*{zba=A=j1S#ALe;rg;QbTbn;D4RgGW_7LVaOx1m8+Z zzCD35*5HIuU+d^t`vX^ssbf|5)msvw zdb&yobV$4PXye0VX~xB2$-Y#3dr-q;PXdP|U@@Rksb5q4hYd?1-;(Acw~Uu6YaG97 zzkX>8G9({>iRm8od5N1KqyxeBhL$UYsD+V*D9W(2NBh%}=E){l_YA6(U(P3hSd>c? z#ma7dABt)Q1A(yWsv=G#zZ;un%rU4*2iy0(!t#LeJ=8u`ck#Ijy1k9j(Z$9#Gje6% zAtubX$emkpSh4m^9hvsC(g;h_=5qqd699PL*A-@$#bWu3-^9$-qTDFBBX}BOYYs&{wt7sAN8kff@7Ju)? zdA=}eIIj?!R!<_dIp8ODKTyH0I$N+kd&~Vo6owFLtyV zW@TB|^SNAGTheCuvs(F3z#wg@!w;{#+U*;b<>o3)1r_)V@Hn(Xsg_9Lx2fjbm$(92 z^mdnnpyC(`9sNBN^Len%q&hauVSbIt7ZJGmRnaH{wf90wud0!;qOc1SXtml3{;Z+4 z&!THFO%3i8i?Z>qQBLJqnh4p>VGGjs*(D@^9t7?WcS1fT{gX3=DvC`|cm^2#-tfUX z5^vTc`>X?5>o1B&;Tc&{QS$}OAJ!Bb>u-Xo)OsVYQ@i z$VnXXN_f{B_owDRc#o3qA53k2U$3SJsF`ot;oJ6FJ5r(ctD-BDQ3&}0uG`})bA{Kk zU&lXjRa|jh3WychmvI?!+F^72=<#WiV6aOjuUT?>w)p1nu|VVf+h1ryfun@<^fjR` z^)8}*gfRKs1-hh|eG6T-98_(!0EB0%LcHb~q<-5PLo3HxQ2WoacI;Q2UoS)waaN!YBemsO2K$BO3>}3o$XsEA( z4R)?ttL>A75gP15+U`tZmDv14I9B|?4aqj|7-O?ALOFGGSmN`9m?0QqAPb-q#9ioO zva$e{7MPGC8-@&Y*j(XV^xjIAeg2;5{1SXg)rFz45(Jz&4$q|}Er zGsMRr!6s#ng}s~#As=#2Pd>EhwwPC3)5R-m9&7n+El)N1aYx=|#JbV(l|qBiIu~x9 zXqf@kL3;EeL2=2e0EJ4fsw$F+T>JA_dt|@BG%4oSvB^<5Km7cvwR^HyjhQy2yP*1^ zgGO-SVXp2dvJA|}2Nqv@9ogl1)%(hoHEf%PHZhLN=SsVS8Cp7~Emy6VFs%;D(?R+$ z9a^}NKn~Z*(B57b_GSO~dnZ*pH4gOK)pbJIjGBv#*DgU62n~(txA+<$OKE=GFYQGy zB|H6P7tXcNwb!xRV_#XBUTbM@HOVpRR~y-s*2`$a#*YWtnHYsU?(dGH3oRl$a;grc z)Shf?e%)TK=VW%X7j4IF9rneT(!3+tJThd+uI3)!4)V9#moFyW4dE!tx;wvbbqbot zKlgBbJ`*4qTyNuX-rSD@ag&mE8d z?!t#=L02^G&YD58Qbac^Aaj@XX)=jWtiWGM6c5B=;FDMJ@ur74P^!$DX!hh&M^8$` zse#Fw&}~QzbueDWj{+Uw$F(9@Wl`8kO9o5P8JmnE8#+OArn)bpSo^o``SdF}KoB)| zchjHUg)(m24b5H{lMJxmv}f>yOCTIJ7?5Zk39yl(Sv163$;Yyi61`fNI1!Kes4C0a@*22FUxNxS_`~5ZPHXoIENkUfaE6)f?DnZtZDV%%vj7=cC#YSz6&Q z?{E%2K(3DV1A^mo{R5GlNm()&f-`Am3e`fVe)>YYy0+Io1`)kmphqL6b98v7DKOit z@o49r5j%OZy-}^pvq|BJVK4)%%5Y9b;de})J^$ZPF7yvW@c&}N*}ocLo$L6kD&s-C zTYY9ZLc%|hRX;yd`P=_u|9{{7Z)F&cf34qtF8JX1-y4seSNLlg2IQ~x`>z9z{bWY# z1KuN-R97WE#+QZ^S{+$go}KTB5SwqNf8{88munpMKM}=d@IHZg;+)cB{Y&qv}rOszXF<_Uxb;U2N~eYSUy4}M6WO7O!k!;FI%(S z_R)|sKBW3!fD#OjfkNk2I*JU>9RR@?x*>+YPXcHzF0EmjfecQ7^F}zFLMm|+3e3sOc zcUn#tA@>2mt}6s$OU%NDTSSMoHZI6W4_JZ9W9mwF=#m@j5zzhWP+F?^qs*tW5|wp& zCQ-fPJi@3}UCI(BEhZ-TK)b-;2_h1AdX{cmekin({%HX|)?#IWIlbnnJ(SoOF}@tB zwP;a}HEuF)#=9(l`>Hk`T#0zHV~X$*d7ij!SV&3F@LK(46K0Ysj^Sp50fF`!iB6w9 zsXNsAOv#hzi>aS1g`#Q|f1XVDn5@&)f{{>!qKN`|MJA7_+{CXf z54cDrxq9>H{g!Q)ZB*Ygo{4lBD!kNmS_aIjS|!^9zP6FcN$vvw(mJ@_O;_|QJbX_( zdMN)a2TuhLM-#A1uAs-VJ z0-o+|*k`bU zi_WT(=urue&O~l5hCioi>)uRQzwF_UdtTvSZ-#Z(t<^Rrg!n9g4Ogr@jwG?@#5q^@ zU$kY3;8c)QrDDsO?7skGrPYBSZjHJJuQ&4yR4DXfFgB$thT(oBw@~{YQ2fuNM<<+K zyc@KiPI9bM87b`-&yC+YQ0*^(QjHCE?Hmvs)+UN&iwOO zDH?334x!Xi-o=2{kQ~;d52~OSc^aUtmvfgOM6(ucaV=Td0ZP=wT1IFqL&R$<--N@U zqQD|;{t8-zTzMsg4gw?R!7f9YS_0uv^e6s#__pTR4oF~02{y?Uz>67PZZjot#VU`_~)vZC&tZBxL|a>v;R zF;{9;q`P;61Kq%IOi0jcQ~~bwv1R#xc0ns@w`qv@f-DhnqH+q#{UZI2IR$GHo5xq4?DAGpJ}5p0?HK~ z6DO~}nC;SUC5`8uy@wJE0dGd5q9F75-GNF(ke8-ipyQQ>GoCtjiX+Yy#`3TQ&)LjY zn#ok2+?9z@Nvkj*gAWuah_8i;zC8-0I>{;lFvH(=u+b47YsDyl=He;6Xx%4N_61r9lR*8C$k9r^*z- zF*a;2AFlN7>b5MGXMgwu8JAnjtf1MM$;}P2GTR%+Sgk^=@G}iFBFcUV%%mzY$2%qI ze2hT?$C0`jfTVLK<$DR7mun{9>AkCWg+wt{~^RTv6zy#HFU(H$V@LEkGup zOG~hY&2gHYERz3bB@QlHDJj z-+$gb>vW)8U9XwaShNh4ZL^F?P=6yDXVFsU0ai|Tl{Xcw_>|1akiENOm^bGcZ^9!+ zp`U*Y&@vNWAl~{~Iz1*uY@?=Cw75k&0@Ij;_pxUyC&!KZdG zSLmd7DF{QHhEH|TMGC6c4i?kvonAcA9}L>o%Mwk^g}w7)cbOYUvKV@1(7-QVdHU{K zIn)iXrP|{G<*|R%{Ms8?lKmM8;Lx1oGqbaxc!XpGBS#wiLb#(-I$K z!a?-|K|S2PyWUtSxNj$DzVY})gzfVFE%iT-1-{;!?N3m4$(H|Kle5TdB|LHKG8)V5 zCM2hYo~0vkMV|NDg}J+!EtrFAO$g(_M)iivEW#X~#=?_`#!gVvw@dN?Mq;qxv`k*e zYDAqrYAtO|@^wq*GZZUj6|T5vcX@m)(;)fA`fa=6CbfR4o9YA|`N=GtmBm)SLYMHF zwQH}TT%Ek066Dr77lZ_y_ud6L(p=aLWoQMI-x8*wzA;=y?g0b52#-h7SFJ6Q-1a>ae%m7n7y)W~x|r1~YgbQJN8}4U8m2k#P7g zw2yHW&Sj(qSC1EO+flXDyZH&28ko;gaPqevmj%wiuB&nX{p``x>;1fIKapAS%;3(; zb#@agf?$7AtqK7r^kk5*e1!ydwfulDj*-~B`W$*}E#ggKdZfZ0{3NKKPi=?<273p* zRIqd*BxC2}O;B`>RVHWfGgH;&5Gwk^^4*D)@F$@iV{kr?qx_M{qS^`n7lj@5AFQ@r zCgy=2#1M{=wAeBuLQJTUtxLQcbETc1t1A5VbaVC0yNC?M$Uquu3D)*>BLN0Sn`D~o zaMGT#XFD6y_`Z?AS6Lk3)J=~N8N|jDMf*`gJH4~Ridzy%bzG`c+-p<=x}PFOe2B|9 zN_P~dRGuGZefDm3oF~P2*U(4J+Z-S!dGpI&J*s^gUL&OdfEI`w9P*{GG~P1pcq)ZC z$9MN72(=U96ctj8ta!W4NWhkCxJ;~{W!f{q9F)UQey+rA7ppg)ppsoNf*m^XY{`aS z*#3g`Iv&A}=vDK;k9*!D!wyX)yfOk%WS{O=Px{l(ft^8i`xY_c*yaM-_b@{@u=N(= zhy8v(E=IIj@WoBf3*6X_M}A98h?u2&NKK|>H91ptaM>+6B^w0hCKXl zCt4V7vee!an~J;sRU0nE3oA3wC=9o=jUWU71S{NSw-pRTE=&At~=AsSAyNWaHqlr zI_r7tZE1!@STbf=)Ka7>FF5n2WDk-vY6*l@%3e=NE-g~s2f^3s;}RV^<%lJ70Kg?> zsN=R_48fKl9gVI}uO%!TUAp%*aGFGY_u_|jukg2^xFF$%4lv(spY_!MApGklhjp>J z7_;;1;j^)yhfvkKY2Z{!;(-6!hVg*&J%u-Qpw4-Fdq(mw80J+}sRmnU;nTRnq!o3h zDL1&d)>Q-J$<3gVkI-TSf->%tl4-7V<~-KNxN|!sHfYA$$Ii zcVFvpCJ7w*IX3@t!wb#+$tHplj|Xen&H12Nq8+w@XHsh(%`>rt#W<`J>R2pe z>>~Jj)!keN)DbRTI61~e zJY$f&$Uc8Au!!t@9XlT9emToAGgVANj>oh@lLNtrX~dh*OHuh%R}GK1)#&BtbH7+) zv(PJ)qf?6u?@fOz%ejopimLDsYgKz#?F3>BPns;^3K9JZrbX+NBl<_zmYIg>;45Od zcK*Hck2pu{-onFuU!2KQdW?Xg#enLjkF3$o#gyi&>T{3OrN@|Q6z`>=Tm*6a+ADM8 zE8;C|vO-o4Y52?MP=67kJZfU#Dqh70sU)`OuqJ+3vUeZ;EP5y@JOkzL$xa_xmQ;|e zMGx6@nhTXhDmiqINPxFvYpg8y1vo<^?H8|&M|hKabwpQ|{U}ok2}~p*<4NN5d7j~6 zdLBW1L=(FJ2ZKN&YndEa(?ocTz*;7&!Ap`4Z1q!zeZxYzp)=OYbEfK{K&JbpbochrCJZC}D>dW$-|GGJ){{6KlY&TWvY`EhKF1QSAogbEVD;#3irJ{dQi#zi#l0dp% zgcn`bE5?DR{b^&@Id;%f^FoEr9PO$hU1rrz>3oZSk@?7z8t(Dq@8Qj2v>-$~`o-`p z`!X4k{V+zrxwC62i#k{qDptDTGTAt~qfAr0S=Wc?7j$!^j>=8!5KO0kR<1RDOQ#qg z=7Xx}(y|LhE~Vu$H?-57Qmglbx&X5WE`4!L0)U#rPfK1L$Z{1YHh{tQ^9Il^;gHwO z8YEN(_pEG8m{*!9PyVAUClIll>?yPm80Yu6q~#HzEs`*hx)!>dst~Io?6nRt1|6@1 zE%c^N1Fe9-slu&RSOfG zK*V1ex&7fZY{?83CE@zrPB-})p~i8+3#fAUW6YDXXDl80JfqQr;1W?VgsIy=BT`3_0R<}ST z^b@RKhZA?2YNmc1b31W`89xd>nEpR&{?F0)|HGMZ3M$5chqx`f>gnW~Q@<~6FNT}Y zpa1QOyLy6S-+XvS@#aBM@rtMZH~rtnD?Q>T$%Q$5;~p$UnKwIvK2#tgqah=2s|=LP zQcagB_c5!=Q;a%1753=HMIVOL4T^P2cy?XWgce`yMfwnENKYyUBLREYJ({Aylmm!G zmrvnav@GAbqX6#RyFczw`hV;*?=$@^ph$kR%C&6S0gXnRIz9SSDXJ|LNOX}~SG~IiHNA96ZFtCRrEy~F6aYg`znpTI-vXcPP4JD}KiKkgzOEFPJPt{i z16sQuzX#|vL=mLAhmkfz^J@jDyh<(F4^s;T@9H)8gjJB=^P*+0?{$wzIeF;9Jc%f; zZA;S}cy}c(oNGtXL$Px`bM~Eh8XTb^v#HUW`pA`Y%*0J3DiakH|K*r<{`Hs*#0cr| z?d*=dN%p_dks&-IOzYv)% z4KS#)<3-deJbi_5ynPUaE|QJD6hO^sP}jUn=myt>mwyzHgoF8Nl?brKMAldrYKMPcf=w3edv zRUU-ST}N6c#Us2D5j-U~<4~9t2#z%ycI};qAgLVq%(e+#`tA#K2cqJBlX_euqL|we zPO%SX(#F>#J+y=^kMlKw`$N-Vgff?-@!eN?(99DZ7i*6FbIHcvuC0H4KgDA*_H2m* z*8uRjEGe3#yswmorc{Gw({w`W7L&Nl6i*QcrFUv1DJ7yhR8TrL`$5%w1f2QQ^cXnhB5Ezrm8&gol z!PK}g!Q2{z4Q<-m251*56gylDX=zEfVW;9_bgOjl$_^7XdBkJZ~<*`6vC~x0(b4M zN^Q~b$MPFkq}PhF@hhO?^4# zi=sp=dp=+|97;u+n+uPx@-d8X^+8brNU8N@j@He3TzoV4O7|hJ0la~|X@#q`gcz8| zVg`nq!eX+Fah5JqX>hPc;KF1k^1K`x9zOHvp5$WE*}E&)aQinleTX-Gpx}q^rkI}3 zP)Fyov-590*w(8EDC(h}jlRIruT7W<#Q2mnL8U*+;$7%I)$!&zQBrnsQlsXH|&J$s2N<{fJtO~Sv+FS0K`3_bohVw&)jr5W)9%gy>It0RMzU5Ep{Cesl5`YHJ8#1h{aQpWH zY?=yF3o-f@znkxIrR)DB{?h9ApZ-Oz^Z#a` z-W6%qall}uV8CMEtvLGF$0p|REs$g_{zFU8VU8J{V`T}?iQ2*stV?rw#HGy4cZ3(j z#Wbs+Bl7{RyEiYR!|B~j!~X2teBC;G4sS>N(*wV;{D5coPwu2PC>CYYzV+juFfyQy zX&!nYRuChEqrz}uB9)wX^1T{Avc4F;ucESlg9K-}Tyrc+X7=gnaEy;Fnpd$r80w=Q z(owdiQEhFG*2iY{r$S00FmPxve0}Ub>csSVZnKo9*XOfj!D#RDbQI7X2f4C!fmL{_U1PktdZhdU`QSP&@;wl3vBBOJ2e*CMPG;38kXp+~Z-X{IgzdhgPfJ&8>hOS!j#c zP?Jxo4`Fh3igO1J1TJ3E`{mg0r+_Hs!savTg>styzEwz?5X^78N6LjR-T6dE@q@w` zI+}1f4|3}sN-}zIq~ncgj!RWu%{4#7Z(on{X049or&Fa z^D+o*8y%xal4bW~R%AZr#`@c~!waU*aPvhdMOxUbJsk0H0h&Jv`07Hm1y-gwqfM}= z^_ArOI;-#4cL$y?ZfT3>IoG3(MB?gKUikjViK=N&NZjIz=AdpAF2i8_4!z&Jy@Il2 z$^vozFtOc+O@<1m45P=7f@!L`Xda#EUHUx570(2HLVD4CbKPSKY&L%?vt37ZfI#OY zQBCL~UwSgIZ|?%YbQnz03OL`kCy&Z%Gh(=VLL~= zLL6!Ij?9?SwbAjT7d87=Js$oGQ0*V}{NG#uH^CU4XR%#o>Zm$@{%BsI?U4P`v;SUy z|8%RfVYD54+_$Ox0XxiDlJj@3L6)$bUG`&34*Ny)g+uA?gJs{W1=Y}1w3_>IHQNNX zEMEv1R`d4NYqpw3-pRzTzfp+cMe*UWtA`hC`?YK2;=|O_xfX9#Xgxt^p*z!&+Q5;Q ztU#TrIg{}$LGQoGFaM^6bNgFtKfbkHo-Xb$ZlTVol;+aH~#c|T?}U$)Ce2P zbKi{;Um)dHkJ(4>0^AA#!X zoYvgb3U4l;0o5dpS@90JyX9v6Xx6Ik>|B(B=ELLSr*IR}r?X8E^J)Hu zTN`~@{A`Xv;Rtz20kj$tm@#u;=S+b|=?;L3=O&mXEzmhkaP7%pA^_G{FS%Z(oLmUVQ^3K!E5 zC-+}8&OUZ1n=|%)8wi}v@yn;%sggS!?>-Z1Q($LvPvnIqNdsUoDy@QrLpUfY#_XOo zu6p!)^Kf5uJB=V*7zQjwnL9J9gRSaU_P52f2pizR{H?w2>b=%hc>$n^twQHlCy;_< zt7$|v3C`s44WW`Om@)g(nkAneRCqP$55X%dvIsDiXArTfDw@8fgMk3t;te;YW|uD= zJN92E*q%xVHyM{za$NNuR-5lV1(?|y44LSJyYpVQ5=aW?951WU`vD!+kUPI`eoOjfrORnj5Lgj zZHW#LjZ@ZOYxC9}6St3?`(5zES zKz9I*Y{(@>M6$l*9DlL?Gn;+xs7dFEjwlI>abDEPR;LqC#XPCWS2!lw&7t}JEU<21 z%BbUp;JbADkul7dPsU-gEB1T)edGFgRCcYMzgnHi*oy3Ld~axz}>=2>xl%$U?-K&s0#vE{5yt`GD06AA47N`xGcS`zv!O!5pnCq!7=*94QRg zut(=pO0n!Bjvlub$@(GQ+`8a&#{+Egg9n`8FBX9A2Y}%&DX(5-NqKw>8*!chgpR3S zD?SZfXkd9gC!3UDP)ta0pfSfueyK%eVF4&TCh~>xB30m&xO-`qgO1r!R@pRG$?4Ou z13@mVRxV6I{)P?k>d(DC$~@|I^7RXx$G@6Hb-hQQy{B-?-nF&UP|!L2d~Apr$#J5J zFbtzR=d%lv3lf-L&1Z6~+GEs`M_bXtOJ^L@Hrw8syGjDQh47Z^;QunQNIOZ z8l1HWhUMw??hrL^KIGM#oVSamd=$BvbffLX`U#r4)0GTkQ16mOm-wiL*eVzd?C1DE z_;Q#5+o;cdg;Js^vSrbxSI?eVkRqI*P{h8^`tb} znz{!ND-uhFvn|ifc^jxQc`=0K`5wDy-(p!L-;It^T!%Mja9~b>Q+j}GA_r{W)R4tR z5y21yU1?ZH=vv~k2|K5gPGJ$iu zK1*ht*M^LLvX!X_OzPdGZ`yC-0>M4{$03v}qTUTtgd470gRsce8|!4^lem-cf@M5- z)85L3uCZG;nG8ky66IcWG90`3GPOG|qQ3B`_#$+TzrDd` zNJ7n&xG$j95A4s?Gb7Mb`9&LMuo|}Gcbuam^t?Xq8S8OFauv+zv8Sa1{B(c~F6;RU zO~A0Gov9Mv6h{7o0B0yErBY%Q=<@x2*RP{&cgp^x@Qkyxjk+e5X^~|qQ$i0ZtDPJU zKQMg(o{hc6DNclo+R2r1#EKOfdVCRwr@=Uul@P)nQ0H70W-IQpvl?--rZ4(azjgLC zPxAz&jz9^hlQe`T^nw~E zs^BX!duC0d=GIaTQfd2~+U1vp-)tAa@_^w%yJ5qg=b41jE-{}`v0coGbZO2VX=&{} z>C;~EFYnT2LJ3|A!>GxdJbwv-@I3A5jNo(RsS>=um&tMzPuvAyzy1%{@E(|}_mQ`w z^SB$kTft)H*}BE3_|!;^HUMn1aqpxc%>!jLOLguocT)SlNO5Cm9Y)vg^;-jC0^hxutQhXc(aO9l^Q6&W%*uQ;QkEREwdgu& z^ug5T*zf;18Tq#@n)iCz9=vG}l(;AS;OKU3jMK&5gI>4mEWPTag}4PJ;X8l%u$w_m z!@*tmE&UFi6Z6^!_i2289ur@)(K&YP_?%TQ`hA+gi^&?(q#GjwrR|#cFN{}hjJ37~ z@;)g^D8>Il&0I!GwWug_cqKK!J_)qpT~kU;=4&t!5&0rD~k0$ z*n97=rnhbH7ni+&3lUJND>UhXRB0A^6_U^rK2 zKtc=%B=lY-G^GkyZgj2X*?XV;?sxBf&Ux;==kcFB1Lm(~X3Y7UbIdtDUmMOT+{nW> zo)h$kBzD}U511M-Uf*+Dbfu5K3fBhF!Qt^dl)jTW#;R7r9Kt0LHz~hHK;gx6QZk9` zWRyfj$9?l1nJ+98!+u{_#xjd`+u&G%%z@k0nN7}?9TF8_#<-4G3H-g4kG{L&J6Jn3 zu0On5Vve*r77?~4I8*n9<&HNKV;b(0yT}t2B+(!g-WE4MQ)^C1mDsKnl};^hwyimb zR4G8I%pm-o1U7WS*8w49GxF%&8~k^dvulv{WMiPsmFi1af;PB(O0uf2`VBy9N3<7a z*;4w>wl+nTy*Mso57#TF)E#j=;?Qb2-3b%v5Q>}`cI9f(-!>3YJk7}BY2S7$ecv_+ z?H6^=sNhh5@nmTp^{utQa2DG}^Jwz(q8(`{{5&NcaBVHN>F5?u&kY+)X`K^Ew9m09 z$XI1QX(Y8``+$!JpxX2K6D!I76saM{&;4 z;-*bF>+fm(7|*2Dt(XjoUbFhmw=tBa?v(JZWO8G1f*@K#TAOW$FW6JSL>y6<^uG7% zPFCLJ1BS;r*ekB1hn_HEgRBYt0*GDI1o_r*m#6sFq7{d zH>j8{XyYy^O&rcI@&eI!Ra|cY8@XXUn9M#!5IKyKEtora2t0qtM3e4COyCkVT*%;; zPGP*|LdZ|#VDMx{I^B&?Hxd_#*txLBcfmN5eQ>3Z3pKej(?4P^y8P&)!yu=^7naws zi7zaf&<0i|U;qz|nw#b3mJHu_#;Xnv~7xX@YQjC`JXfFypFtTyM?oEE%)qT$&RS1yku1+leC z;Gyi7DQW4dHSRa3NnbMHyVa63F?_QlFyRY5VCXG!YcuG{Qg$oP$)*cOwnowy~2jul-NLKujr8N(A+Vu9=VdtcJ?e( zB6a)9oux+uNPNz8+|B%0wR>97QCn@$aFA>DdUd3Svo-FxW2?BE@z63OOo!UJ71e7mU@etvk*c>J zM?;GnX6NESZjZ}ULhWMGvRGH2sp*L4>iAhkz`}T~vtR0`ONwqAk%0*G0 z(jXU+i{sYlq6Z-T1%de28Mx))YK|#&PPbOmB%^yWn7Fj_3_v)ZNga8%8YJ$`^GRW; z!8^voBP;ca7x5j3KAn2XO2f)Da>s35yzs`-VW7i3NSrhwwO|XCr`KK4`Prn@GaIAc z1X1joq~vPZl}}QF&*n`bhb%2#<`n48VP@x<2_{WPrBM`}`)?t7S5EFT?pJN6ng(w| z`@=D15(%fvlf>?D+2ohTV^6eW`8HNI-?ZnDuwPh`u1T4@mNs60d62aaqFs`KzvP~Y zwx{XnxJmdxd*`IgZ|+6a&fDB7ESi8}x|)9@X{{Rs4%yNu+HajIIf^7uo9%f zH>dqdf5y=j6UU@^cEUD^#;R3|VE#_mOL?LdloM1$cddFq-8GKn3eWzy!&*i23^tF& zVxDEt%87(>d}Ed+eKeabI9AF3P3?)ig~BDXP9E0spq@d zI-zbKq^%lVA#k(&-5Yt`juaS(xx9yRjYFf{#-P2$9g80M4c(flwTEbzXLT&=6v)67 zb6~hcz*DI}71PEE%Ipf&&bs>a(+LMoV7{k7X$qzmnn=EAqTcY4`fPEDc#KbL|J2hv zF$VlNJyOSZ`$~vv2d0T9`&R_Zfq#R!ysvLuPNb66qClsomghXeAJ)FSV()AU{*dTa z=W~2}kx{vOSo6e#k@osCqD~v}ep9bs#Co1?QU2Ba)9@{aCI@it8~#5HRW;F+*{KEb zZTwC?HuO;PX!XIp%}tw6`@S243dyqE2AW$A3PHXs-Fw7|rszhj))Tew#*pLTakKL7gNSVHNgCp^oIkVRdp z{(FzepQgX|J}_<7WnItm9joAn+=NgZwNkwASE975u1z|Lg_(k=|to8`^0N0Ev}x z6M3=Dl0W=$WtQoWE5`zr;gZ=>K?(>tT5o#>UcVn`H7(`$_@gW!+a^99?GF_`5b3IQ zygmQHtI4p_@Q3X0zm_Fk6udWc{jZ9S?h`zl+`M*vTJD3r&QF*1+MM3i!&P2KbEjPX z2Pd)d-TZc-Bjf93e@lxS;RiA=Z6OBwdb^Xfh`+C#_7}58VEKQ{Cij=K>|4L@wrna_ zvJ_03gdV+F>)>}t8WYP@u}Jd}$rksRb(o#au%_!zzv-8QE~ThkBQ`hOf&Bb(z_A-v zJc#3|rCslSX(AG3M^vU-%*pZKH99h|S;ZJk!zOU-im%Jbq%9MC-F>IqZfwF;lTZsUen6*J0qS_G!mz!hSaK zeVw6jbbjn8%dE(DU=n=^Y>Sl!TkD{im;&L5!sWxOntP zW0e-d=+v1+8}^Pk%#4=yZ$UL8nrH9e_fCN)#;Lo}ZC*ahr3JE_ffhI0huyq=Y(v3l zE79xo3}*7OOP!^C@dj)4)+MJz($?8Rwnm!mxuHO>SO%TY@@T`rftS8;Jfa=mYE76& z7SKt`g1~)|obUJVsRt)3v3?v$u2^?B8KOLyvW>7ploG-N-;HRI<{4|23zXpc2J?-= zSMLf{^9H$Q)hN^V;rOcdg-c&VZ$&kX*v53m(Rxdk( zEcp}sod_7ybbOj?DxA_aA717_Xb|YZ<#PNPVBn|uYu#< zQ`=70^qbq>3P6RiNK?D9;Gz{0G>e&7tSsmJ1$YNfWQ- zIMvXCys4HQkBaGzl~UbqoIdh<5=V`S@kwviK4PG4o{R*)&F9`4tmPc0TDq#=h zN5OUCp<5==$U4PMeHt`{GbHy3ufWUddLeuyuTu+RF|b8xQG$H#}@>`}G1QRAz-a zBM+bz)YVOH9|rhqkeD*@VZbm)y%aC3+xFthd7o~-`c%E}lqB!kKDs47mK#uLn8oOq zMcW(Gt(0X}U72#W-=!}`w5U%lwwIm9#a4gu5uAH-;`zVx9si|szE8UT3iD9lO8eTg zA_!l*=?g=_Sr3VY%G9graS`xE_#F_uh8covbJ zaPdLpMkMAWrY`X-Bk`Y%nt^B08S7rN(HeP)< zqTYV2snsJ5RI0q{((8D#i>*N(-%eiy?mB%p8aJY$c0waRD&|MUe081wipN|;dI1ks zw9nx4FuvnJ`J&fl{D0}jJXTH};DXe*c-WMYS}5O+;!N$@`bk^SCzaQS&_V_I1xctJ z1#*Qp$~N~Dmr7OiNrV}qr*c}qvo(|5a?>9_FMHso`N0)w!8^0&t@-aFF+rtd!*dPa z%x{P5Jjy#hVyktyBR@fJVh3dNUnmrnQDgghBOo{M+oen!T1?fHzuED$?hl$^F5spB zdD&S@a#cJu-+2o!h}eEq5BmD%6k5^imP_iy}meNwoSelm@eS2Wy zPI7NxTzw#4bmlXi=_ed9Y+5EgKTiKqb^j%|xe~F3_f6Y=SbcKcJ*fH^(UKs3BjhOB z+y>uz!byb5>Fr?P)Fs_DH!RI`HAxQ;srT3`m%~a6 zA70JbB^vDIWaD3Hp%KPe6@RK_X}3F$wxc^gs1=d!5?Xg``BVnZL?LL)W`wrc8OfHK zp?Ls-0~WTbsP3sOxewAA=Mj?S*)FUrf!do!&E%{l`@l}`PbR9r@FL=E64WFrOvbE~ z<0g+warJ3X(#|S$;7Nn)`9XxM!Fu*9Gy->tYNb)@-r?u-MLsrQKC_GmU_FD=Y$g`U z{#MW=zE>uUc;#)YLLy~h#EKZ8& z4Dp>&r|?efLfjma&gQaGBcfts-m}iG(VN>W^$hU_=(jBki$n|;>9m4Vqsrc zFja#KL;~lk$NG&u){5=m;X%x5kXL(0_+i=71od=iLVBq&etu-B{*^B=V%fegSPS+t zM09v%4LB;E;aBWabhWi85#Jt{Ri?j*y$ul}GUb8VnbdC9P|Pmfa0Q?EX-MKUvv7 zC*xi)!js9rdDgZJE+mjIGZs=hY32DmSV?dMgYKC&rM~B?%wwoY=C3dZfv+8Oft1P? z6{Tu>r72b=SxNFR*Dv82v+ZG~zoeEOzsTjKbkE^L_3$Sx7_{XxY#4->ONzD7+)$hOv`&c)JjS`H=ji6v>FtAg0D73CE@)MX#JNnQ`KY z>&uHg5*Lb-Y=FG$cZo`>UAALt!DJq#9th%0C7vGzfEbqfS;ZN%O@OP!(WlFaj}IL` zc*+{HdIrChVPSnfr{4h0lOT%L@U$ISvtn+uu8+e!y5Ks@SPAXSJ?gk5ViQxg`LH(p z%sko!#OV!l-NIV}M4xGcDKgq0F4DK|M?QIxovI?sWnYwZLh%;f$g_yl5Kt8R)Y*m| z`dK$0HKXmnGS)By*568tmyWnL97|dH03#l6h{_{+W=C#V2v_v<=CgAOY|X`$fwU=~ z_;!GbLd3yrr~+@;Fv7(6by3kh?^6a+INgdwLp=-$90CCNzFv|ZP2pnH3&r^*>J(y< z4-d~wxXZ|ftW{evx<5XUh$Bf>12C%$@?e^iuMg4V5#C6ukav$I7LtYF z5E0u^_hkAj9o@BxL+1Ifyz&Ue_ndytPyamqr))-`)Cf&%9mikp?O$bHX3XZF!qY*OI8F6iURHkzI`ysmnUSCpni z5unNm@Ky$=;?}LAybXbRx*L7?wg=q;2;&Be1`Z1hQ$ozjUqC$ByGD2Q&H{P`;@T35 z84pLEIal?@$ONbXq%T|MX+O{tHD#v7OCjWE$tQJ&4oU!nuP$9ZtlIPLg55CA7}0^& zr%}UOFR$K;9h6Z%5x~aY`dByr@EB}i9DF&cGzrgBc(FfEzeQUw&$st=LQ|TSF?vg( z>8ARn{D5_DSeI>km$*c)Bs9EZP*tCb)K(8fl3!`(XW6TIb$SIE*2 zm$Pk zV`Yrw>=mIl^vP1PwI3>W`z81c@YEY1u{xe&(eZe`D4^=ygOs?qVt8>v9m0QBN4viQ z$kTurI6DNia`z0G-5rN#fJN;dgN(+Zr#8S}Smq7NgH`0`JzeQaHN%RrKI1(i4fNF7 zO4^7(Rac-iOpVr>>N>K+t=|8zXN$9{{&rMrC_Z)io}{EZ2Rrq|=Ebpd{x-vD0l%HW zX2o>|`#`sOMLV=64->m~jI=(1=N-zSS3Ikx?a;QKKA=Y-h7TY>DFt9>pNXV>-vU$H zRv9ouayDOizW><26gBxz2cPzh?lW17UgL)Ufz4$3i1Cm#k}gbsv4)oP_&VG*vITmi z_*H!!eacUL8-rTJ$@;l}6qHE%HYRpCAG6KHJiPcv;s44?Xpo7NSuw6oXFJnYArEu* z*I_wu{l8D6=0ADlKm5o3`+Qj{C-`)y^w?`o(j@=>^2txCPrR5Q%#3qC#SGm9fA-Fi zV541I@p^x~k>%-7Tom=k-#_Z(|0RDYJIMRXW^w$xUv7MN^Gl0cyj6~<(|HZHZ*d#$ z8!6(83xn$6Y3kVp?P!#1-7a}W2_xCi2s^f`BI)kx6Yqfmc%^96cf#`WiY$=Ah!j!Z z2z5yeyar(7A6WMDXlY^KXKXmfyyT76N|H>GSz(2 zt~e5aF&%VTPE!Xo1NN_01g*NYd;Kxr|2-JMf_T&R0VyVji-#0gqe;LtDN>>jwUY0b z`#w0*(7T45Q9meibbxkuO49C17ZVCF-rMod{=)Km4B_{g;$;Anv9SRoi)y)9N ztx}Yg{e8d`f3ClJz= zw8>7T+f@HqUTH7&tJ0iio&?37!mOO)S+GuUn9O?Bzy2&Pct}y*l|W<+(kP3)_PnRV zaZC4sRcK$6R5y2HTwSLn*(YeS+T}?B1bbtO3@xv|uTOwYY|dg$BR1xd5#-Rn3WRZr zq>f&>g)3vSj|*`)OKCgMbpoE|Zk*Aq%AxFGLe2>j`^B{E;N2?e)gQ66|KG+Df0JB5gn)FWpQ=HVnA!*+``te}?*E~Em)~Ct z-;IAgw~pnXOlAjiO^kTsBcx45%cmo9&c%$h<`nivg|H1eGL_*!t{bN*j;79Jrq>fw zRwmwf1`ozATg#B+up$``>UH}q%&+lK%q8g@N{Z-?Vr*)eXjs5gQPyFR%M5viGJa zW^4SuU+HR9ck8VxJ9>g}pfb$oCj;9o%qSy)#OdFqo)Y%RqJc%gH%@6U_QLG$qI0oJ za|B`kbTZtc_f{%Mb-6G86kwIC3ij6??F_a)?)l-i`-3uX5)6%X=Kz}=3{*e;zUWf5 zqR!k%UERmx^FGhlWVg*GRlUteJc@mTs=~qaQ@z(0y?J1kMJJYRl+`P>sa7I7t4N=Q zRBOYbKAIkD-?W!*8-DbBF=C@!vYWFsf({gOMrWFHnp??0-++=qhk>_Ri3#>+gMUGn z!6oG{59KIfGcA}U)?Nph){E^YOBp=|=GPkI`G$`o(OGPz_o6b~t0ZQ0WmEAueqSJW$$y2YN z5R6T|`=x39{T)nQ!VJ=&YhAzaS9vvy<4=0tDwVloOACr}P?O$<1-dU3y_FC1!Rs^B z6nENhHFRL23Z{lUFomheL(LUecQ|bXyp1>Va0c(WQitDMQadE~w1rM6hYTyc>#4iO z*Z^Fe4^SN3EsNKrb=8B(7wy`0_T|@kDY2p{DqB75a+2&I=apyDVzcJpJz<{X~{Z=5C#OSqV@(Ab^P8z4Wp_THde+Z zF*>WxOCO6OG$ry1TQ;`NLL?<&zu4sY6I7J*+)#jd3d1V+>it`6IScUOQsp{tj3q_H zCA==i*+fS-&mUX8V(O4#I`QlGuH+vYx2DtX@4FK>qCcg)Z};AxOH1Z|<~S)m|A%M) zHx%BQ?SU7Ga@T=7#;M#>)=Ox)dg}n+)y(|XsdH1OM+Icrk3RqT_w(<> z0RuzpwZ&L|oc^Qg{tdUeTy?r5!;k4f5^~wNGG}wlI}QLw&Z-X+z@O4TqEY^2=eFa% z0hX*^zqW_}Df!q#cdFnSB8`(+M?0AJX}WBVAqoylI%wM_#~ndprZh`=^S)DfAnr`m zoa?|w%hlRvAA&JFe7tiMshk=>b5-S)#rDD~I(=Ykeb_>}KNxK*@!5 zd8PSOKtPfBIQwNk(PvIo0LeAG^RwiOi|U^>a=H|y(og8cVVy`4q!mI- ztmp62$CpJ%WBP{DT{8&Q^`zAiae+M6pH_j=i>HQ)eS+v*aB*o*%ZeA5da?7gxH&Az z%FvfG4XB*j{ZMXq+E2flm+GQjh1?Mt!xPFt2HBl`gp1(SnXIY%c|RdiNA^r`IY>h^ z)+MnKm-m!0-@XEd(!+K*>N69bU1wqW*MnsO;??53Ty!PvZPJ;RnobOp5tsWqdu%i~ z&FN)?TBF6K#+pf%kPPmt9C!Y^!HcAXz@)yP6MCg1y4eP22oe4tOshdx+cSBFk zm98Ik@=hI|!4=`@#Lf>B#Wh!6wz2|qCC6G}7}1hvPQd6{O7S?Z#&Q-&C^;(%c*R9p zt+KdeYs;hNl#g;xj~JTuOcy5er`Cnw7BwQcCa?l(#NftHKJaP7q`p2fN;PSQieoa- zH}js-ZX_m8@@Io6z?)U_NTxzD|MeX~DOBWXE6hx0TAI`vm-^yayBh;5qgtvGw^Q+s zlF(8#Xe7(=5X>ymyaX12rFA(Ez-BT}wrp1uC~+0Wis}3&U6w@^3#Y`!%r?4!ZFO4Z z-fZKnY`;_&G^5Ujx)0$S6+PELO!3i#c5Iz`qf=d8e$1%eK@kNHRt0wA`Fvj?TN;^N z^T3se!_>Xg`7H=u!9vtv!cs6rSUQ6Mq6fF6+_W?8OC{N7%q8S(%z!R^ymq7TWkbA` zA?$ds>;_w@^FvF${gbYU;zZ8ke&3Oq;LnuU2cZ z;5|Dk)f;bgsntjr`w<0NxZR3kByyD6It*t{VTT$!(C5Lp2EBmEoRr*r6&Hx@2DQgPW*d4|6xQ~=L05cY4|mcWq8tUPtb>J!G#h&c`DxgjjT)+O(I z=na@wbgCXj6fvtEK=(Jh1Av}XcupX8*mppq_#u-?jj&LN{sNfYYDa}YAZ_>4R+yp; z8SNjib>b_jwsf4gl0aaH*ei0g8;P%eM+(wpRlGD>4)-#vdf@HX$~6=|EZv)MDV~b3 zu+(^;u&&Zk${@CAm{+)Evox^ptmVKA@>RtN^bj35X&lT4bdvo zlg{l6_N_kEc$I#8qXuq;E|IbOHwnSQ5n+SIrP^LeQ4mgY2fCoH9=o3NH24KEVwS0+ zTADw(Vp$O0-PPPsf2x6sT z56)y2=h0_BFi z3fIm3V~O^scsznU3QbJyq^XoTF@TTuW^vFkD7GJX*EMF@diPkEi<8au_H5A%RiJ|D z*6iE|lloFWV$bN?f@h__AryKF&62dK++SGurd~aEz=f2k8o-PVIP*^p7Gyt2(jP)A zm86U1h0{lK0g)aVG<$EW;VrE;rA4U+zg52Cp&8D5fML2Ba@?B;fBQ1oJX@kotQWn4 z-aA~GBL+)!L97XJt8W7^Pe;2acaI$>cy}C8uFB; z9bdG+gzvm@Ej2Zr72;k7R(2P!k=RA%yQY-+3-yLb2K45vKr44TUn3P*AN51dR`P`C zf3Pt-?qb*w=A3{A2SuJozn2hjLDZ!tvM3!rSm&5Cr2#b1h4N?6I%Nbkj z;w`LAXlroIGRIr%u?_*dx24@_l{A}gqs9wjH+i$Zf>4U=YE@_4Y=fpt5PTuNR zac12hdJM8P;M7x|bH&1ga+WH&};&=L4(6H5g~bbPaS34(+5V! zww#!`)?Ol?c8tETj41Qk^}%<1ipq$6z>)J3XHu_$jU-**mSsYyIZR%Wt)Tc9mWKI( z>|K7}FD$z`O`mDuKJ@pk3FYe*vBWPdvA5QJLT?}K2I^dKb9u3(*SwQ|uQ;tiBv?Nc zMm6ss1mG~tQNTNv1l`PAbe(EwLN}ar>sTw*d>9b~z>raEzihIOSAJpndD_b@y3$L{sb8aHSc>Bo z_wkL8L)|@6@j2f2z)5`xsWX4-ka|rHUq|jq2?!1}4c@$8C+ck#hb~1tcHkOto$AlJ z7`UawyAGmq3)4o1=H}DDoESK5G`DL&cI}CsQaUV2iobmEvt-LM4#`6S=aDvl?glCw z%7?jm%P&}A+T&W?K04sW?$Qy}W9N2q@rNr-rPIA75?ocd4?>lmR@FXMd5wewE*uE- zlME6nMA$A%ly2JkZ(`t>QIoFHfg(4(`J}%5LCc zb*sbk?TY*fV4QaFGi@8J+AI3yom}5fDDg`EXFXn&eK!(fX~Z;zqhJlxsQeYq6NYic z3$s1Un}C9CTv7>O2%0om?nC8XKJL-$)#U#8)Fi$kzu*$x!gi5yD%6J^Ac1{~;2EK_ z^{^(Ug6cpHbyJYn;N4gJG*@ zB1@~68~c{i@MRUj{c+_w!)}i$C#Q(P9j^jGh+;CM*z)Yn4E9sjqc03mOyDr5D#Gh# zhEa4;_CeP-<)wA(FYhznOXnaz$rCDu=E~v`W>(=eFT-UcrWVs!$o#a{EO+b|7QI1o zWXod@aJ8|tJ8^tyYaXNUo8zW?Jf#5IFAD@mygcD)B1P45Ws50tx3*0_Rmnzkgs7j1 z(ebqLNNp6Qt3opfr$mAasLF-QZ-X*5hq5yTmyoZjj<~rfBBxrBY>oq5AGGwQfTDJU{nbAJr$MS3C}>iTy-Dz*kw_NYL@Sg zBKn7;qxm||3z0?oeh>#pyIe;5V>UOy&A>myWByPsZ!CBjTH~M~f=~-2vu(l@(v%DiH z>AmMmsB>PzF4+;Lq}YQ8=j300D9y%KjB<*=)6WOF{>sSF~2)yr0IBKF7=Uir38|oZV>b0Xw<*fvdquDtFXW(dnD0q{)$aLFL zm)g|T@M>EIPX3+3bxXS5amF($I!}~jZof1_Ba_N7rh#fsOn6Agg_-EFy;FR29oV!m z2(-!+%Yby%A84bn{96o*zX*0U`_9o`fBiUy{NruW%KOKzgb5abgM%)?Esx44)a=hR zf~`HvlnWt!c_=r%+#_*)jNS%jRMv>yUt>eSuiX1tR~7W!p@6u7F#PQVoy+IVm^L-c|Fp&v0guEz zp{15cNjRi~OO0o7(X@Hl>y zYgRG31D?GBMI5zIyMFTQ()E^pFRMx()G)QJaNcsb6l9LQv z90TRS6dz^ONCX(Zv_ZhvMvT6wl*|JMKwda6o02ib@yJz>ZEdj0QYSUe2~VFwg)U~z zI3WOtZrxdmaAe%5^z1llWjsd*E%4-|QkzCfrrL9>nM(T#VbJ#Jo-+7(mkfX&vl}NR zf_&VI>-yCZ+z;yrlG(&nyis;!DA8Eivp7aPBd)YFVDvbbu7pMcPaZ}sAVgN^9_u?n z6^A{sr2$F7i7b`Z21a%_lTzM~%P16y-8B1`?35^ zK3uigP}XZabNL%2>*Y>uyG~e;54Pms(U&UOiPfsmsujn&!?HO+eX2}kG@i`t{$feI5 zq?db|_?wu1OADaJCL9&os-2YbHj9i}8Ljiv6nbG933@wIW5Fnnmru$Q!wSO!V5Zv^ z$nLUUQ%c`}C^CaA%ti6tr;rHk773sS&UYbi%cES`o(Cy@0N2C}+ekQgV1%Q1`BrC! z`bb0hl+8-$XFk&U<}e#y^-YwFxOahzlqv({1H;(|fyc1#QU^<_tfTK$1^)c*G&LaW z0d3wFObx+c)`}~Pj!cEhxu7l$pvqsvE4oZq?M}omqMAO;$hUIAhG$P)xIXGqWk}8x zDvYm1ENg=-{R?NQnvShpg4M&gp!8#~=tG!F8GdR+RXe&JGqN$?kBsQgS~^mUVL59Ub5wV%9xiX0J%S=;(`N*6T2H?q8{(8`kH=ijW>z;sgM~`rm#}FEeh%D#Ls4W9)giYo~1q6t#Zp*;)PmO}J zkDKJvB>dL`=9@I3+o(^PIZK5At}6P(DuPMqkf$H z7pJ`?VFiY9DO5+SIoO;#P1C17(R|@nj9S!aq6?1r9?aR8h&?3%S>WWG(XCE{RLwQ9 zLGT(}3oiK$Ks__aN!I2K^NPa#khr1^sJwIUvaREB)h>-k&R^e?x)P(z-3F zgUV!oRX%J2Q>6uc$o6p6RWm{)PoS34>q!!;T9=bgFJ&(oH`^ilG7mO!j>;lymI(4- zpn+3yEH`$H24mO&()^t>TvAFd0-fvcd|F6iG7uxm#>ZAa9}XBCzw7;B|62IelAOEO z1A>jWWu3Q>OLXaVX`x0{$(@Fdra1=yh+SU)jHUtu{cZMBb>Eg#?xnh&bC8)T)%f z8P1p>>etpb%K*v(-{;6yX^{!PR=BFyIo2z}{amSwWo&`zJYLfFL*e1~F>|;#rC{c` z0t}PMqWf!)=M*}63wV}i+^CDTWF=Tm7I-e*8h{+^unp?u1`D~PHlWp>lQyDJ9|y&3 zBGxpn@CI`vi|Q(hM9ZO_a^f3oA60e4VPB)N*U=mOPDM_lN0x3j0|zfeNv$C9+NjAu zrVK8&G=<%qU#Od1-nd_sz|NrvtptOoE9#h{zQHn+$8mhkgl*4ztR3?reIBwLI#P*< z_pf*eSU$fz<@1>&b1Ox=k3Fu(A_)W83S3F?aP2;l+FfmMyrLonwX(@( zzxHSkuy?dv*4gzJq9cq!X*}JgG$(w0>V%rbqfME_i{xZ5EFVuDl$jR|b3;y*Ij!Dk z+&{o3hix+-kqzzQ0m(Ue?{R>Y@rAGeLQ z!1=D+hn>nP=q?PC+aZ__x-KE){oRmZJ}LudH`#s~@VStARwQVfXWB!^Iy27C&X*Vc z8+=q^0Pkr|stzvD31%(=ihyLQe8tyS9qqhXi`E`)>ArDzmc(abI~!CX!O1K9s9wm~ z@}kD#M^tZ}@aKe!6yedsZrOcZ=7W@hGT)+9F$c77;94I61K)bS%ZT2`{~DEXsi-Yt z)j9b_(!QmXR^*b6on2^i4KcnnxmqPUxK--4Z;2kmGX+dqVdmhO3}OI)K~{}s#V!9j^AwZt5^n2Sv9PXc_9l%CSaNNCx211FB=(Q?a~gRsLyG&{ zYv4Ksyr_$J`mv&gpTdI9#1}++m@iLlRz$aWqqkJ(M#43_cR19J4NbVZ?-XNZ!aGWQ zqp&YJGxBCXg9SCeuz=cz#R~^+yXf64JY1+N>#}Igk z68B)E{8DoDGNxhMbqcP!z^Hj}@#-TEv;F23MnYU7;UtBM5Dg9`nN%h=!h;Xnv8pM#Nm zcC{f@BSz&gUXKe>;Sv-yeQat-l&Co}xifsaQm(N=5RKEZoF|dfQj}ebhDvc^WLv+@XSvB1q4i>Hk7a1&wV{3w_oleyfyS0G~Fl;*^`_bOd!+ zb>#*m_$0dY-YHv(kO%Mq0GqGdP|Bor;$~Lea;BFOo3_*@8p&QKmShvpJLe!$g*OO~ z%JdoTvlMAR4RdpD5$3K zJN=ff6b(ZhV#X44q?L0&SxsI#V4&jujnP`nB>zxh(zc$NbhFe3s38@?vjEBYm{p83 z-d@$xNd_z>ErBtcBB(jhBK;g4c42u57en|Vbl+22>vD#9EnmALIT zAd~B64R{qRE{R&HQ(zfF~SxBo?i*zNU7Cx^XpgU=QB8wA{l(iwJ2_adKjA{X9HCY&6J({OtC-*;1ld&w5 zgyxpewf%$g1T+Q~%O9BI_d3O}ts!dOv+0H~nEt5oj=;R$acGMH`w}diD&A3_kb7x( zAb~&5+Hv$yoAfpz9ueZXgy6I~HfSy=ygHflI%{^UIBv`|$bXx*7_6qpc6=#)$mevG zocNM_Pp{pnB#rLg&bdkp#rnjqyLRL;-J-A~-n>#nov)nBs=QXF@2`X!dgN+|NblX7 zEJjdJ6`Um3jBUD=dL^0DN@LM6QwaC`WW#uc2sORlhtuoC1yk=E7g%xhETALSB6P?CpCgpJ6*21 zq$P+3v8F}TXh7qdx@Xzd^mGEA7Df-L37XU1tBO>DO5#d%`p|lHngI(n=in&eq%(u3 zy}zbD$G`t&`pl}63Tk>}Z97&5IiwD}i$-Rb*2t|sE7sE<7=xFZ=;ss+`iv#5N(kiN zX;NuY_RzYtXEVHN)uv|F?B`%#t8gm^+vmX~(TDj6KU}$d=o?()A2+jqT>59Y?_UZ0 z*?a?7iFkjVVi+rB7YR#$oqUL={~%ZWFF|;zI1O!3u^G&`?!SSUjie7ALNw*N6VAT3 zm+F+H7VSC+l-gf=om{zE<4BKx;$>xcMAG{8hE(aMahKFa9pWNcmAl^aJMKxdOlP;z z)YO!(KGb#EwPQQXrdCuvbe`+%*xzSMErmSgnGCz(a>Ngt@q`)L>={)*7aWl?u@6ltN940_e)f=rlx`k+AilV9WW!acySmi7h z(}gj69*o_pE&HKF|Gzhx7lu*S+2`ykC z2@smn2~7ip4qM$)f`EVt5SoOZK#)$Tn_dG_Lhn_&pn!no#og}N_w4)bDfj%|yZ5^O zNao6#bFGyz$C`7jF~|5mgY>RvbdvPVse1N)L9lkX`YQ$u>)Orw=;{4?uMNuf&*+A( z(wR|?{+nvD8&?HgYTnMipXa>I6>w=d{fD0;F8{5me6q>y zS2J?*8$aw-?jQc~>;4BVe@88j6mz_rxIWxktvh~~j-LGL;xE_YQUU_+%^v7$lrCwm z3gt|H+(ri9$Xyax9rfc$ZTO_M7Lqrod`6`6{a9`NYUY*t40CFQ4D4T=T@lbQAx{+I;=d))>0)HPGp_=6@+R7%ot^#fgr6qmQullLtZlTlpE`g?QseU;t zL)*ZV1g_Ug@`&v`-rXaNFL>^)wfb_E$}dhkbrJ)UbWS=HE1jz5VZ?HM! zit^F0Y@B}Q+rj5E#ayVe?gW7A$^2!8K5<1@{kG%5*yiUq%bR8`N)T#D*J361q;!}h z6xIqsL0iG7)B?qmdM5T$NvDb~EQOREchQ3g+gZ^8RN`1Y2rlCxxRu=_>RXq+1bf9j zI2a38+5&OaL_8*UOG6)gy3^6ss(&swi=rsDZL|7g$zz8m1L3(N%*ZP_h0qN|8PYHe zrRc922bibzP{i-06g)-H*@g!~9>KP2?+%9ma@v|`9XTq`xYpEC3#kUK6%*ZhjM+0n zlP1iMD2=)>B~6v>+eNP*wsT>ch0Q*!OR&lk*X}aD<}rA^N(L95;ywaw!px0WrMtZ! zXjQG%(~90~jR3b%0M07w;@9(5Uexs!14FBX2Jt)M4`O8hGbr73jRE z`qpg+?1*yx_NR-x1EOufd=U*{WGrUYjugc!B$jSEgj#4r!oyph)9GdJL11sjbL<~_ zOGXiw@4l{8$PRk76 zukY9T+}~6*pt;;wwb4`rWl4}OBn{?u$`IC7RK`hBg7%&0eWhW)XnUx8Ap%FZn3Xn3 z%oeto%y`_Mgoww#7Z0ectP31IE9PNk!-`mnU0ht->K`_`00^ZD97-cRYi%d0Ohlm; z=!z#ZdL$y(2UMu#R`jr4i(7dSYl8&BsO>;)tSfl25*qe`VSx_msnF4y$rv<4$ahHCK8$Gz(1DoEzS`zlxPtuI`Q$)2~5rn(P5!*!v^uMz>K2AcB{ z78q{lNRu(MOac0>66eMhx$cCC#@dX54NUo;&&WT#E`d*vgfXGiKG`9--vS*Ki+td` zW!qKHiLdhD%T7veH#o-=7ONoOH2Ns>pKX7$nYUe6GwX*zpK5X6{>tDe`~@`l3eCW9 zl+CTQNN!HF#&$ZZLuCyC%~CWy{*%Sd)1>!JZbC^%!Cg|=g#ZBEd5Wg)M} zZ)W}GgBg^HAjTIM8QYufvUGRS+*};PRNH09_Q?`)o0qISELY>L6-CM?-YJ3I1LZ!l zJ4=h^M5!aE-ExqCR2qyc0uBI((b>;|o-8++w?r z5oxUwqlq+%lA#qk6Tft2;vfwZy;vxY6!(N5uCsBlyR&=@HID3 zg;gh%Dcx=0u#-ucfr1>&I0fRT_5n<7mtZ=y^ph9sh`rB^6~~m~DULSc@KrswKK-VA zFm&}Sc8QqRYA}EtC3=j=j3hA)Q{An3xN?$+-s(L@dPpN)6(m_Bio&ckn{9v}DQPcu zbiqhC37)CMk%c)M9n8n$!b+|l)ZD6({k`3k2r7@J=YcG3psedM36gQwb*lnMT zOze?g@A6iZ71M~Es^A@x8A^FoUC+J7nAqJvCRC=>qo-bOzJ{B{h`X^_`&Zl6T*Zagj|DW*s8gq>+jO z3oRMr^(G>tk`ah@{ESr~;YWU#r5=eMB&xgh-=Ir2Sel&Ev9sc?4?Zm(`3o9vg&WUSB*G=i4CT26YYFSvQ}sK z0xTTXO(aX%HYTWBIY?f`K-#B;J>N?KbrXF~Zl=K(IK{Zai6baV&&*qksrcZT+-Q3C zP!|wM##&UM2pPz6SeF;FG`*DE{5fM_)y4Ln1t(S%$PL9x-Oblq>W=1)5+}k;zDRY; zd46>MMRFX`aFg9j7w?Ov4y<&&CTk@Yn2Ea+DPUEwLgst%C|&4RhG!fR(B+nNnAPBE zb_)%AiB;Q$dS0W{245gTXDO!;M6~Y-CNFgfYz?i8k$WbbpU%e%2+S1l<*zeZAUGDf zSC6*rw1P~U?Wb&??FyTZLnCu7$Uc#6i7VaSBxjilG{Vd=*6A7`N6x>aA2wYC5^=_5 zbLBb-v(kHv#6`~&DV&81xU4f`Cg#|Vw!dLU`NNF! z8`zg<{ik78@m~;gFHC)LV(9oowSUP@>aU3Tm5{1^BiYvdmSc0v8XRgpYOzl7IyxMv zfAFG*?hLDHM@>AW<%<=|V309g0$^1j=wPlTduY{Y9BLCM6ZVlqjitc&lB+eWhJ8$2 z=PA%#x>Tl%V;+-~os|&3<36j8K!y>feJYs^V2~kEJprjJsnJi}dIP{FYgV)Elgm`~ z57+GmbqJ+rriVMkjf_P7#Mw04I50uT$crSc*tzxD%H(4Wi8ieCppu7-B3qKWZWS;n zl~Gi2JP+rw3{|0eEP{vfyl!4m?mh5kN*Gu-q`|*~+;wn<=}ZZ#M#9;>;y7JfndEz4 zt_Q!8i`J7EW@6nHdx@Hj+e!-H>9WP+v;z4WVELn1lN~S!o`2_{ZN*)Hpn6CrwjvOe9%CI%Xysp0v3^Xnl}vlL>zHvQ|FO#_W{jTXTOXAT`d79t)OYbfIb4c z&U(<9hATgXjDs6{t}-zE>NvYw`|CRzUG>=it?M8B>NtB+_3Jy@q5BMHkFG=%E9^(e z7`s?Ifhk+8#S-bJoFBj!e9sasE86?rI4y>_;m7Kn8}%$gON6yl-n5mBs!G!C@G7Vk z_RlA=nXPpBSKn{H$zNf#=D~4O!q_gd7otVS3t%401hx``*AWvYp)H0Do~W_V94tJv zSeCOC#w%mzPQgdAa>Wm4Wmhprf)V;Zp<^KC?jz_H3e5Lji;8DPZ?u%`d?echH@bms=L(}?d|MV!^7QT61;0B4^8C zzVr1wf4y4?XCe{MZYq3{>s-qMGK6DHwCB?n?_9-=_h!4whaCfsuPkLC;u%w37NdZM zOF=EWR=tyiDg*w}ggDz5`3u{hG`t%XZ#O|bc}Id?Pme5}Pnz?{duFb$_rX9g%>kvV z0!8oGb|rm^h7O&^(w0lv@0WwT@_fx(`+HR)3%vRjUiLSYqIY;vs5P_Q<=FW9v3zXi z8I>lvC-)XNr^XmnB4tb}1&dokTjJg)QX_2GjTz1Rw~&Q*TqvTFdLpmosV;}{DCg%N zzB1gY*zEGSTE-z{+1OvtQ+qX8q^P^Q7o_N{0m#5r7-L5TeOIA$UTTgA%Q@Umu#Bh( zyUmv}i*)g5jV3r23wT>5lX)z?J<0`64j}b=TiaTWGvw85Pn#&6q~# zMn%3jqg{3&E)qAmNGl{MDR)y%X!-)$HO5)FgHxzN(Cc>BD1_&$bjQVqQ>;vU{-we= z`YSyQ;<1B0Yj$I4YjCIV(;ZL0&cd@Q#V94K=qGsfIKCKfVm?`AZoHy&=e+I1Y ztWM>m@R9wK8JsyKP0Jj>FwLtbZw>n_2*pf>S=mtp)HN}_+7e%*AajfI?hTTDckq%w z>k5@UD|ZVFMbD|MT+%|r_-{y~DPQ))^z!ZI%-RXe-8%ZHo8DkMsY7(N~xw>}ZY!85e& z5xgC{$0ZMTDY+BGi@WG!FTI8cMyY~QFTQi0Xixg$rV^SMR_5>H-_^fk7DrF7(vHy# zBDz4m_aZ{{X}-m>;M8K%!j5~vam3-JNTyq}CLS-{9*(@7OwSDC&3q!$$jRB#Uo5Gz z`yxtxNZL7EEp_k%oI*!QgwSoMm=fAdh3FKS`}y^Hr+^cP5=zJ-QQtuSz>g>kxI-UmGFn2TtXLN+dgzjo9txrrmPI=TgibAjW1LT=5&-A zP?=MFBdNT_hQx5FnlLuID!V0dhFFDGa-?VKn_Y6vsqYqrMi}u=+C&Me)1eySz?Otc zdJnhTZEipC6?I&3awjf_CMty!CrKdBjFcEHei~l(sR_+EJC6#3$ZisF8-6=1lc8uH zXSt`0GdU6vqtn-^r;)>H-h@SLuK7J;SXY0>r(OLUUGAhhG;&3uc&M$`DRJ!(_TPZ0pgM=J$Nhu)|kO?E457eAH4fM&_kM4?XKH$=$ zLn|LKi+*XTlg_2m7O=jDc^)U9YsW9aVA)ZwyzG$sByhoJkE%DlqN3AFO$EeUepl?C zz92aUP%Jst64Eb{8R4jC|52RL(E^WIc$8L3qeB?F_c0S?JgUs*py~rYG(FWjE}Wq3}tI_;Sq!{b_rn)8gPS={=w*0MT7= zg<^p96Sm~Mk!<71aSCj$1z4Vt#sc*klb%H`S8?KbXM=<3JR@QoHbEZ7K@P}#C&qC_ zstOBq;~~n}nDjKx&zxuu4M`^IzRk0N)D6=6o(Kqk5 zKvlJ4yx9%Hho&nGA6kP|F>{VJ4}wR8aY@xeAYLh8KW%oRo#N#>I}xdMZd8Z@{sHiM zl?4i91}W(GDvC16)6t10RjFRG?>)l4_(eTHV4C|B`h)Y(nQEs= z^c)yid%DNmyZ6^#zj@$mtZS+(s&;)wdL@z^#dC$zQ;jg6OJzgbS4iEbOp%?j6T!2b7tg`=p|N4k0hDJR6U%^O6tVK_fB7s z4ojm%&pc;7BAH5N%$eePzC{hyhYf96h3~%n@YKA%>}caN8@K-9qxFi>nr#d6YwdBY zZ4_U%p9F6I=Na=;%|d4m9b+nsY;Zobb@H#+%A9_n+bsvY&QwpVJ_~5sXIc3-=Ca!> z3=CeSRx(sTNp8V17B{}#>>jp4A{datZuw+8n%Q<*-{^i@R@DrZFj741t2A6zax2(0 zT#x67;j-5G$Hf<|bFI7Fy!OZ{+NHKJ#G6q^LU?oo8q0dENBD~jTA%f@O=q^XEIVz> znKen6MW86TVG$`7g7 z_mT1t7?s*OzSWM;m>FY~vfmdfa6$}df!$|VieKuz`ydOr=n24U$a-f@jL&L!dpN7k zsX$dC5)=g_j`ltZZy`rbbBSs^&XJb9!@p5I-9}kEPj`y&**uTTRYg^~&b^5fzqWQN zBL56`qaoj^-urtaVHwpECu!p|jtQ*HMMPoqXjaPJ%Q=+ytis7NjYRK}F-Ej_Q&}jd zUblL5ty4J^(EBi?WkD9$H93|XAWc2{NTkF0_U`byeM7wbh~U)*Ma3rr&n+to*e0%* zn^-;}<7^0Tu~O!G;Zy@}0J+q87P`m3`8j3f{8*R-=Oa<_d$&6FcEwlGZRFJU7BN4GX&;KIfHa<;FzzDv@5d!DQo!YgA^D0i~$kLI9^)Z(i)ULV3`@ zShhtPE^*IOP(~WbTZ;A~zns%IIKB~I?0-)2?4FY|$QxI)?* zp{-nTHi7eF&3lIxV)>8tQ3;}vaAcZ*^bsXPML}je> z_jj&(h9WCm!Rcc;tg#lgjwQKA2)OA!Yw%r)X+ngpu)a_Y$p-~%h3=i^$Y1aTU9u1{ ze*VbsN$Dwr2W^~|iL8ukHcCcDO9_3x-ZN_CS&*}V;QRK6YZLk5Z9TqP&4>nIK z>_c{m8t1@fd_pc|vF0_fo#B;ZE^px0NW5sQ2;jnR=iUO7pKe!RVqu8S>^^Qx3 zZuD!j`t?ClJ|Oc%AEe8^0lIU7bnOZ`U+~;m72AvT>A!8zI@ptR zDU4j9T)5Ix8z@>r%W5`Q-}ELWlEXS)6&JUEN+bI~5HtrEM`#i#xfI^Z0CC3nS> zRBuRPxCb9kNvALWC?nIudZ?(7PAU@7K!vN(d9qM~CSf4k4=)6AdDF6;tVMvQ9iLm} zIEFdqm&+0Ww$b=mRpfgD0pQVk`!aou{=2r{+!Wu4{h+^kDvX16i|9*;ziys^?WvzS zR|1EfFG9O@J*;o?rMU=tSE;OhoJ9#Wcm=j@0A2TmNRLc5ZEc!>F~UH;0s15yDWcPr z%O0lt5*amNeyHA2v%_czhH>7cRDzBh-7m>sGl^}@fMGkk5;=2;%moTN==7|r2UNV7 z_~6ZBN{T2tmXU9V1yMo$6$%+y5hD^)RU;rETGhg#^k2xM$I3TcJEKTP^7;|i6fnDf zYYQH)B-2l*TD|Fb=|RcOwGkhn=~`~wMcvzTTluz8%aN$9Y@Eq{VpQOlvqx)pj8=^A zulzvK>sFY9fLO^XFCG1Zv~yE^is&RyfVn8=IVGu+tAK%!ZEKCNVG`IO>M^aCFERUp z8W0XA?$C$>LJfjPhZ)Un40}uo_l_EzEK+BAgg z=xji?>8_w=1r_zwWv9Y=o(c{_dp)e(9%@a)3Q3(xeK0=3i=ROhm36DGI(`};3L zSOwOMW};M4#U`ES6Dj3N>Twwk$q-bs^NyCk@N45oHxHLR2sQnZa!s@pfGNu`&|Eic zO|@}gPORXE!6y1B04?v2H=4QnjV_M_c4RL*>Cwz=t-Ff1RFw zHzDI$$aY-ODHkgrC-jPtF^Ul04yd(rnm$e-S?ET=z0-SWU&3R&^MYX!gf^@J<|M}+ zfCImT^d6>p>-B0$8T0Qk>wf?m&^a!aab7~#{_nDioH3Lkm^hWvnav^-9+V-HgatBK z&F@9{z%_&1vV)y28gC44A4>AXSMMa1CItx z$ZlxliDfeD!+YJg$54HJEs@QfUZV>DULi#C3FDlPK6awDz_ReK45r2r&YmguE`9E- zd4b5Q@8g(qLZuCd@s_QvEhLA{hpU%DDSxI&l+?Tvl{BH z>OYt=tIpl1%dRDUis94$%sg2n{%VUYe0ST|+ z+3{zOGiHb!r(`bI^&(ONqc#L;Bj08qC9EF1f|gX~Tm-ik^g0<5aijX7xy%$kqMr{b z)iOrF7bWPGdDFP?qt3$FmA%$LlBtn~_D(;*eufTg4|N-7qx_RTuoESvpIBu894C_62#q3qA(M?byJZVxy(FMnNwLaJQizQF$}sby zoyCac^74AEvOBI@SkN&XFSuk-%mguNi;t1s4P`O(pKzx1n_SEO$U*EV^8l{(@@Ddq zTziwvjMmLuU-J@3*-qRdw6h@jNQoYK_BbLz&>_kdU*!~i#z}z&a*Yem3dup$OIf=) zM*z2s2x3Nr%z&H9PC$;amk)?b*@EuOQyQj3!&h+|&AeO1Y~t}(4rSM|j|=PY4?^3}U4;$XKExV6j?I$SoYg3d~U9!M*i>!-Z zJ5n~@&C`2?HMv-u@Jw{rO;jf0UUqvc-IclPI_~gcU!Y!kOC%<^+D47ie7Lu2L&6(R zs+lxtw<~DP9T;U(c5i^Xb%hn>60rbY5D-Iu6<0CS-Ywq*g3{}bukN6Mxbl%kD11*Q zM?Kpm4eA=nF%EX_#eb4_ZuwxGjtk!cb1OOCkUkw*kcPl=kdgCo1OPpIGDdnyvV7QX z-Ay-*{J8iW{bh1CuA!{Pd@MM<$QVE^^{LmQvB$N*w4Hso`u&R;2R|vCpm7RN7>c~ zXtp1Y{Z=@m)0*bxS+41~Qf;?1ejs(nsn-9qA3Wf;$8QBRn6JIDN4F}5%5D6UwEt?E z_(UGD>+PA}3qPMgW2ay@mGoy=LI-{YM%ktVqqs11@BfBHr1*P5Fmi5sQE0qW4}!mA zyj02xH_E(w^uvlJ?S@nq|o)T;z=Uo zG-g4I8!VvhF1;z}7zOO3eQf_3d?yobVSzM>Iq{b;J-lg$eQj=?YDsep8wpH z#j@0IRXR$?Hq1WV5=?IIC!%bQ*&RP$9jAX&f-U@49<+lGe`SvQmGtdI?Xf@b zRt>i$e>Fc3yw9}yyk?4>UJ)+Rw4q3M4-DD5>JP}jQ_!^fcpgsCiD#KySeeOBwKd2h zK*@FPFJ&p+%rx;+g0A^qz4E17(tA3e16SEIo?Xp@mkU>MDDi80@1b>h(z=A~QFL2? z{H4GrjEGrx!9S@EKRJ9co`66G$MfsDNa3@xvimY}uT73z%!ga7hRiH~V1$oJRH^om z6HO^phwhEL<7ui$VnCXm;?<}(V^}>Xd=Qvmj(Sg!IJMnYl=AtGd)>dvLetgn%mEI`f z?!^j`0@x_{@8Rsu;XdsWV&Tg17F!OZ!-LC9w#W9bj7Ze6qnDNw^G*$|4zPB8BH{w& zX0>-wC|T*QnwYEo*Je2V3MY`;h57q%PKwNgM^w_Hc6*qzyTS$08%{Vn;gvTi*LZJ; zyQH2~<_^Z$7*GKW!)frB8#xpE9LO)^UA@(7dg9`&sg)gB2+HWS)BA)sfdqmIJlNDFnjB=N{ zkRH~muc?7c)5hwH>@V4|2Ur`F=VsN1Z8-IhJ<2KVX$LuGltM(VG4ss9e8v|5lwo=( zY%~q9)1ax}b{{?*eyO3jZ`&ug zZgaVcAKSyPO-zrJvI(LSKMh|ImQ3bcTDV`j;aQe05 z-EP*-EszcAb0Ky@p*@pEfdlo|s4v0sm>Y`LzC!;}{0j6_zsq ztdAkpFQrMyA;s!{XOml3S_YELD_7%?j1`_^5!-g{|{Yb#++%QsiJJo{{6u4O3z%*(*CQ?dotDl{!c>`K+1$maICl zj8&XI)*-il@jkfl-hG7xRb~6a>0=iBDGgA4rPnZhmHxH?84WhAk2ktQNjD-roS`jS zI*su!d-Y}%O}TpryFWS^SZ5TN0L%)63BB>D4s=3>e(!{sX#drrD-_GmKTdI{nu&TS zrW0erAv76+kr9}O*i=kRVCjU<&%3C$XEP?v1tMZK3z{vWL=>X*Zzd%n6ws;m?sUp! z3F!>`B>R4&SRyEXTec3nI(&vNjP;s}A)x>P`pTeY$OjF}(|y+Rl_7RpDy#ORS?J*@ z2Uy_fm^6A1t>27yLBDJ@sX|hSTf{_49Gyh!AK!lS%>OZf5dJf^jE1p3-6+;)%(QPk zg|vKEiLq|__Idb+h@Xw>pUvUNm-%MC0bc%Wh`&<*9|FMgKmI?RVc)KjLe77S5B>Yy zBA0;ZJLxV9!=h@w2BA`z_x4|6nn?yP-t(>Oudx?f*m1JhS%nm zHHhS1Mbn+s&FMv+HJ-+}rkmb3|FjbKFLf3yI%X~@T>?B^B2OENkDKS}E{6YJ*zewb ztA9!WFkgi z5>AbNgLz2(xw?_&<*LTicfLfqqMt!v{)vMYGp~OJk&yTmPiA>kWv1}wvU(!;($hJJ z^)1NCzh`o4y!~w&4L9TjxZJ&cgte%Slzg;{^^79qETypV?#F{c$SXTCo>|}bzsuwQ zi9Q^+8#P|KlTxy zJ5V@+DJZsIn2&G&!FBitQsnQ4{*og4yEMK_BR)-a=GDxHF+Lq{1czRiPGnxzPDT1;30=zoo^Av$*_{6L(-2 z+5^WKf17{x;Ju*9T_~<7qnCHa{@a{y{qxNK9sP+eivK-A@HY8qIvVF9#w8JEVb#|% zi(xpP@;9Q4^qqPsX6qLY6$$+57j^rmKMw_QCCHuTQkw=bm7VPq{`ZjB8zukr75pl_V+Gj-8VvSLpIjTweYT@H zxERlyv86^DjU&2qckh1QHFA3CQL-XA_9#rCls5e=Gds*I2c+`lL>^c@E1N>khyYyR zoe%K~8(X)zwWn5RKJl{VbN!>@F}pn6lJ~xowNl-aln{O(w4lT=9l0q_MiK$;DVxdr z<9;C%Dg9p=zA|Jle`Qz&WvrTW8CINCWOs1+XnR&>A?RHqt2Fci+JvXUR~#>g2vO`Y zunG~(p>dI7%&fEC@${SxTEFztfE_v+EHQq5ko7AA(9?u z@rjwJ*{oE!mJdsXTHQknK|}t=3wG;y0Y}@7$xH(JiIZuFV!)P_qtI;fY-eHlviIw8 zQVBBzgCyp;GgFHZ@r5zLRqUh~h8zI4rFshXFmexZ0+uO`$Rg(UjqZ(M&~E;ot4JlRlQ@IPwtf*ua2W!iu-HSu z7fX{0!l4cxMBuQ8FbpvZLaM0Z3oHGgXXEG#nL~tliD9r{$LkBn%bc|ArEX*OhjM%Q zneH`YVZxJ4#d?S|5IwU&7zQhkPvShf((V7IBvGb)JLIt#r|__XelARaT%95mICY+v zSbhty1MxhG`0N4hVkDD5*eFYO zHj`|=ejL2%;>*${FOmf#>3)I`%CCI)x#Io{d}$m*J&!dC!Ka%)!hyFNsdA{p8=c z%>88uRW!xHzV^}`0@YKB4KG)9?C_SygdwmXu{3ZSCbYwAXYT$mTC8s5B&lA;SKLU3 zt6EZUbVHjpfy>QEr%}@blqvg=Vx5#Wh6jjQXT@b*vU)I<^1#vDKwrzEp~PnlCgrYs zq`VpsL{A2a$2S9trN%0oTIuZ9AdHg3Xy`Y zr|c@D8$PC5Rbk3-$)~r)239ul?}P5YqFBzG_Zxr@(eN2Qa2A+`yLdK8&_vKAByw5l zamUDSwK;@&-f0nuI$rwO5591vfjnak$Td z$Ldx-5q$3E)$7gdyIA6`kcOakixs}h@tpUT(8SbC_45K-*P`f)l+JUecJW;fO;xe% z&I5y~=B3)#)XK@x8NA+qy&D zej_$~sZUQh+MXMj3K5O!qhc;84yj|OL z**056Zh-PDwgoEax%9(1Oe0$^_WHXb$+-_)O_MZQn)WuA+Kjh+SsZ~hSuO7Jf|8K_ z;WmujJU(Q-f60TAMGiBKvdM)(yZC3tFaA@@W8dbA{~L?%vBzj5(|iFDxibTS;#fGx zGIbHRm6Z|Ky->#sEH7GzszUWAbdRhger4#Sh6Swq3GFqpo!K-_sx67W+5J%2$F4E6 zp+CgbsZy`PH8nc|RUu|;oh;)WUFk3)(J)4_dDSn9K%6oL@XaQE zk{XImsGu=%wsw-u_7a_&?WSt|qV~acxIh7=Km@{p0BJ5)@EYw$s+C4k-i1;WTl&^7 zXI;9RBsG{4LLSVs0Rl_o!@!-nrHHI3WH#%%8qhe})ohce!}HdIpj`s#D?g#gTd0tUzdgWI~DOM4YoavPn?@dSEx9skm4fyFPHyv=$HK6~BQqu0@oJVMY!5hgT z?H7fo74q4hX!oZSjq+-e+VcAG#i)6t#SKWlB~J@AaZ-5GD9h~$fw zA&oW>Wu>8}%SoD{EvCcR&Jm9xfzIk1#)6#1eRo(o8sPNY*=M1AtUQfj5EwV5A||pa zeef&8k+oU~}yk$bIgmg62)hFEfc2t4h8kLa1RER|>SkS8W2&Mf#OGD>ZC5AKC$hO~;} z)2&o3#ELbQhTbQGYY#)a`5!j5q`L62_o?WHoRC(&egOOJartMqGu2b8&yKb4pIiC8 zbf#9GLFe|*C;juaYWG#^Z{|KW=bPcjjvT$S z>u;^TbAW#50R7GZ`ke#xzZVCn8_6MeeYDBhn4H| z`zem$GsTHgf%ewQN4{u-gC<2REgJgqr+URo5Ff>$;kdUu`GSFJEsd_IaXQXurfQEGfk`e*RwMG-k;P%#M!v9 z!5ky9G99D?uI4&GOtKPk)z-)3jyI}ab6ng1F>!Dw$T?qErQ>{u>|xA~Fg)?4QwCUl zZTqDK-Tp3swtT7`+yc6pN7*{-tGA{Wbp@yTRYi;WrwEm%V%g&LtShPq+>@d0yH~dZ zGK{TnzLEGWcn;@(anMGXB$t6G7M3*J$?)4!^)U14N|A`5^9wgtMbJ1oo+M?SnasAI zO1FZ06i2JH%v-hO`i6N6Cv4j+$a9hV=1iC4EG!QD1-}644(1ogs5CXv>Pni4KuNGp zMp&o$GZ@TYYQDP3MSoY@hYw$oP_pN)Vg}RQbov6_?J6wscybkO|HH$1rs8&pQc0t8 zW{JKQe7@Ya@w8JIamxXLnN!O^28Sejon;&`zc15SGE$`Z+!f;1$PQwG)wKujoswtN3sv6>6gLco#aWw@>+8^O{)AG0PA)n!#`FbBLH@)r|0`T>O*~Gb$u_^n)q~Kq#A@ zQlPVG9er=>oLJjg-a&YWgjuUlg%MymXX~Sy#yp9DyU;c~i(PYGWb#vci4+%fNDkl| zxFh1b`(H^jIeE(R-jK}MoS)*Q>OZ2Z4Fz*cd-%_{YA-0r%1e#YpsMWwe&;qvWG&a| zfG*;rXcqye{DHP0`Hn0)v87dKN`Z^VF;XPk-i^#Rmkl>eJn|F3kJh|=>e;lMc&^n! z+jaU!*>`%Nd;~`qRd`04uHH(i3&;a|7E(KrU>;wToamRcx?NZ!oU6q4t|c{udUP zJHiwEl`IaK{M62a50x7Owt7C+W46*Po-}{1xZy3IMcxG;XLzpi$`c$ z@%yR%f)VGttp0!2D|)KG&jaPnFAZ|5y=+lQsW(f6HY73~i?Q|)VK-huoYaj?9rr_p zzq*AEhW$#Fw4t`MYBeFWe)$q^RluHE199J`H##ukhkx=({g0YIqlxzAT8NP6Eo@&f zALKcoTBW=2Y5>w*`Qc>jx#g_4Cdva>;Lf>rp8>VI|idy&F@QlfU)%p+8$^{-!nu zk1IvqYTZB4a(P>Y?yFrn`0!7Z>D;}2?wU)(I(L*;(4PYA<^H6~KkyTKxnK?Uv>cr* z=r+?Y^j{YD4x1Py)Q3vNzC6Ho!S?!e$GiZ8YGYi-_e%c?5u67GkE_W4+H-$`>|-L4zkk|Pf4QhmQ_XHo7tIqjgwXS3_9Ju`#_5bnW-xTxqCxaNrTDOGAN5#Wtd2Zbx zB~?bn3@g+^dRyXLy8ti@<7DBKfgr}YriKssrcKV`a5p}bUo?!s(6f#y!gM44V#=UX z*Qaian%&T^41->Yeq0|pPdF~y!&?%*T(KpjR@QR3B!z8A0IetJd?)^nJJQvMdN%yu z2wz|8e&@^`h(BTP;&3loOVE2}au|b0nUpR}$i^t;ASrEONEaBa_1uQjtNvQo5wn)1 zMQg_ow9`4bYlf8K6*n*xx&+mW&T$+e(;lg^ZhK8Ud)S+r7C3a892VUXnmJxY)3LF z7)hci->Z4E1TiwYDME5*Wybv<_TD?Lsch>T$FYteib#>7SDMnLgdzgYNRe)UPynWUi`(G(R=TE=Y8JaeV)1RbKmDP`RA;2 za`w*7I%}`J&)#c&zk=>8dyt=w({xXiQNxF+h2BLYQr=LrvDpMv*@0+f@Xn3D(9ryA zM#BGm_xL}R{GFL+0-DExU5(AM+*ww};SYIH{L%>44n{t!vZb!`n0zE_0Hjr~2o2O6;e7_2IJ8sd(;=)`~2-}cVvbP%!sAQzEYE_pmR zlgE6*;#Xk_rdZ{RI{nn}S3%BNXE_z}>=Dz!FGBP$W&SMutHQ1v&rR#vJs`0o3srC4 zIYA9Ayb*(CO-6f!MWjrRS(Dj(0BE}2*Cdfg`a$+|K)M!h*W;Dgm;=Br5UQGzC3UEonH>m_7lFYO#r`g zOH2Ly#nOH_i@mw0XeoQ@T3g!m(Nk@jlUiin(^~zum57*#RFf)X-WYM>S;S7QFC1BG zI3e`NR9pi)A`hw3m1P)&_(4$WA%Y;q*N*0`0Pl?TjPx=Z1&c->2`0;0%p^OL7Pb!i zJ`^=S7_t0XzSqqmbD#zH2|l$0%1O=+R&pR z=dRpUy9shnkjK|b&mX16c6z#>p#Ku{#f|3ZNMKc?&64(Fq^6M$}S?&_ZRI9O}3Ayg@~OOw?~(! z+)Ry0&&$h0gDYe~hIf0Va|6XKJilv=e^j6v+(N|1y&+oeI)1PBV^#c;3Nc^64!r(h zz(RLmv_8n!b9w9bIjV7?GeRaccLi8uRF?HEN`b)puDLc>*0S8B;g{Zic`7GPU&N5t zj={08lWf{6ZUS?Cfv03fB{}eu*~MefYb7{@>{0Y4+c+ zvOfN{@N`d6)Z{^*RE%V!(=hZ&KXnxNV?JMrc7?tYq|^O;;_0`l&oKW+v@1ydjUZi$ zh6b&*Sd|vL&8+^_R!{Vri*7WV#4p(10fCwFW(9}#WRkj9CcJYho&&FQ4tnSAX)M^N zG5LDi#sqwic=Kqw`3>4QZh+}=(%_kljKg!dn{nD>c)y4p^*Pqo`$8_gLZVx}IJtJO zGi93L^a>~^{#ct#)y{AnfEw()3qANz>fs3{-6hW)tSQ8CetKdKNi6cO!o{L}FK}^Y zS}e@P)P?i; z_jN0*le!txLt-yYR+TrZH^Ce~Q6r5_!eeBCveGvdo03D)*@Lb$>l@Ye4bSts133y} z_!)0DZBig9@AD^3qGaB%$Wh`xeG*8WkH{_S?$PB!%@DToYrSof>rZ5nC}2;`gXVEo z4O`>XWntG$v=jeH1`wmJz>3*;^dbEI(6fxi@ymoE4WE+1LF@w12`(ehMo?_$f{Mg} zv`EWixSjoyJEYr$?pi*aRRBoO66v2@+}ArOb?XsCWNB}Dg=+)JZ~pdzs4X!*=g6@z zw*X}us9cvlFye9-n-acYQJoh2dhoWog%~e<$7E5}P(%5LyJ>jk+J0_By|`sik5Ga( zxQ+i>`9;yja>w$%TV{ck7A{hi_V5wym{I97L?U%3%=^wvxNv>L-55-lVyUXdCn-)=5m674~{(__nz6Uys$NAReIh{@Gu1{e%2@$nWeUxj= zIj((8TZd0$wM^|i`F8U$2hcLJ>K?&b;3@)R2(X5Om~dMH{6*v2FF}q?ea4TSBt}tJ zztfksjAM@lL8m)X^k(@Nw}PcH3fztKTVkrx*Rphu^wzr=krI#q*%apaL|~mOzC5$SZlLm~RqT;v5vEz97lw>uSmP>^ixrCAVEouJ z*`68hFQ6YCv8 z#;z2({vU>K9{W5B-di|sxy)35vIkP+d2p|Z)&&E?E8C>QWtQ5{+RUXbyA>S-DQK3Y zdsfR``PhDNe~q^ID0L7P-W{FJg~(CXI!1P+s>Nii?|JiI%eW}sIFxvM5RieDI<9=) z{{(hO7t=|yeXf$@D;OKc#MnWoFKhHx@9_E{S}w4k2;oXZ7p5C0=VwRMA*nm>Oi-mG z&=u*YkC&2W;1{g&TT03EMg6RY0_NY&7Scev$K@8%XjR5|fgm%RE8 zLA+;16FcTS{mt>C^%kpKQ-L`hz=7)2AZCUXBgDq!mo=UOUPi9NmFyOk}a?#HO zA!qh8c&X|$1pY$BJ#kZc+sjR}eLYzc!A1vQ&3R48PaGgwS3#222xCf8+crxfmM z13R3ng=+0(xaxO4+^kNOC^HcQWV{E-CAwE-MQ4X|Lo@rN=Dl2$@Z#F;)e0WN@!zA+ zGg!6^Z{Jje58*bPW)7-RtMw zNZa@aZl~L;?nJ&>BMgAFlH;^Mj&Q7Pj4Y#p+CZoyi8H8jvNov-J7g+fmml3<5|ZnO z83vIA>{2Ma7ayLD-o9t$m~y(~?Bu>_|7vs94v5FIB5X9h7Z5h`_Epb-V428aKU|uU z$eC|nS#sKnowt$pV@C*oagVt8l&WumQyW}V2wtBeU8q(ssWqP%VOVnVy_f=Tf&;sH zo#ozIY_T01hWiG5pv)-;S(t)P!ult;o5p11Y)+5Xd-+H(_B`Frc5H}hp>Hwr+By}f z1<;~Gvhd%ug7nR?g z2$NZPw0jCV&`X`7S+UNyxFM~TY#C+yd`-TA2F8gWAD)o@HedR; z6@Dksqt=1@!SI9r3ndQt#@}{1|A6LyokkY_Mi+Wdg67O$t7SF)Z5mnh8(rvABF%SY z(%AVR9ip+Ip1(-$^wG=LXE0AHhKMfiBWO2Xea3wh;_i`db%U$L+R9G(lQTtpw+j-( zIeDYYAZFG4nFg5r!J2U~RXV($b?rcNU|+@gS?)}f>xPT=24kY$uBUzVCD4cdAxuHu z%UD36YT32KX=Gcr(!_UySuJlW)N7=TI)s*UaooUOS)AQs0>I_iqw4~MRIb6{p&7lw zvt6C6rP8AXxEqnTskLVlS*;N8>fQ}RbZUl5oe?*DX{*qm0iq{!{Q zx>37o3a6y2U5Z+E+ia$PTUE*K(A%NnrdE;rCpWVfZwG+1>L<#N%C71q*>H08~7ZnWRpEHo2N&B)+2w}ILo_xC;% z^3|qmEtR3~+N%xQH4O|rnL1c>YPQ@?Fs_Z76gaypd!gdwN4a=@_5J+_gT7%iF5d6t zBS6jBY3@|THjsMDny%*kvlth zevRo0z{|8s3Kc!^$-oXkvYS~}ru5WyMvluzwLBmRf{GyW`D0MKm!-aSOnJp)Yx)|Y zPqqZn1D%F=FC%7QaaVkSvJ%K#N3_)y!cd1dv}t~*GFwGfJgXa}wfyWLWdnM1U*&Kx zlPzpOB?HRERw&j`;iOB+ir9R^9BwqIqdn~5J{}uqcyp&jGxj4(sV+(QRBbx4h9E7_ z(R_b?JWW6vppyH(9eE5Nuu4{a{EA`F6qB9G+?%6u-<-&?{lMlShFJd0}B;;bZrN z96@i_BO)&iJ70Y6tq;ynv+@U4nM9@)j;Hz8IK5-I_lryUu1p6zS7%7|9Qp?$NtD2+ z#)9D5u+KD+OG0sl=1sxP+s~-$D0}lloQ~vWqMNPlJiox|(KGlk@+wCmj8j~oAi4{c zI9+voQ*!?bGhb+rzs90N!#3ST&uU9Qfe1NDm58j4=vDYGoNW?BsNUne!&06#JyKOib!=7G6(}BMc2r@Ux3Z&VH)w zYd(y5@dOP@uUM2ly>XHNv2F0yTV2V9GkD0UFDrrjgN&EK33uTEZP&shT* z+e}rXoO5uM8W1`8=D5|KVZmVbkU|t|3fH}&R~n$U0TAAOoxjl8k(nfxy>be?42tuV z6UuiwwDP$s>86lg=*||C!6nbYF`zU$X*;g>sXr8Ojo=99AdKYIL^B=Q|x4JrQSo&ljjUCgU ztvn!0^1sJ*bx10!+Sf{qRF{nSM*+ppaC2K*TT%D4Vs5;~Oq!ZBm3nu0ofsBQcv)?^ zpw-*2-*HM*Rf@?gwEjMvg|7gT)GMo3TQjtMR2IpnNIGzM(uo~?6kVQb(q8_a$L-#z zF#t7NCgF!Oo!1L`fT{`YLnWq1xrsAp8ns}MV-TC?t?^6wtkUNRChS3tuPtuN8{~j* zdMeWI8#A!nFD;x_?vkdYQth8{hmoqvYDOBRU{t3aQAs@SHgk;3#oeuBW$36eQy|A z)IMBATd?W)u+3J<{4#G}L>>xN6IKH=KVEueANN*~$@$FLiDV{M&zKP2cqgS&N9}?n z-o7??mojL&Csz2#_~57FRyn2^3|7q7WV4IzZA;65BO44tg(k*dGS)dg%q-pb(AqI2 z7Gh+VNPTxT==E{to}RP@6WWSgT$GjdN5=L=9Pr8h=L&LB!@`tGu^lE-|1A@kEu;-2 zRb=84`qS0}1dy~BoZ&zHLG@wR%XgQB|-x%Ak61?u3O^Q0`ty1qWSroVVtD(E}q&wCI~St zjWxcSI?yj>r=c;(-8R#~AJFm|fAr^7v+AG{CW>;Ux3VW?&aQvRAPTtQs;KdDur6w7 z)nBOSE-o(AfVnC6FEMg|o_*7VLEg5Rbo&9F&&cR?(fH8YN;~b?j`#wslD5)4VWvX4 zIqFJDcI@}oYTO2en-3)@r=`^7?>znxQzp$d8YO5+4T-z5@Z{MT{Re!9*x6^S0Lzar2susfLR|zGIVm6cZ zsge+ts<>emJ<2mEUfii`C4Fadj~%w!-6=F4>-zn$Q?+UX{jlBjycaA|F)>FE<|zen zq^|Q~Sjb>omZL|8U16?kz|rkR|KTzP@;3hqO!J)*(8qBO0-ZgN=lAnBt%5TJ&Bw#W zZoyVxZ)m?y#V4)UN_Dw0^VAQc2NC(s`;MA!raoF1VXqsxWIb^m`LZBc=*Wv=d#v37 zNvi~`94@+@@0o)l2O;`748|DmPubjVi#BtC&Ub^f!4)^A>ftMg@2%60Hc+2wP<<(? z)99w+N8Qa@sE_A!gx(A$Yq%A~!N75y5h01oO1p^cw`2D=&tw#yS1If7;t+O+qm4OW z)B%aQ1En71;3c#zW?C(`ik@{MyWvLNE(Fd@0d*Y8`v&V$5u-Y6bNVp!w3crfJ3f}d zP%jzfJsQn?P=1vxHGFW%O>4P`VWQP&CbiYj&6^egknC+rzAPK8i_DL)3ENFZcK`tz zL7^`;USTe}lqFqTyf92DUlC<$N)pMhBwuArsguoyu6cBM=XmNq7>@xS##cDTH?5f7 zBT<>JZzbtjRZLZZ;2?(5^?r;!baq??f6}ZIQ>o2KsaFp}2~1?E6l|!TmmfC4%CaLi z_yRX36!DV=E1j@vFO_kJEDvJLf~f^N=R#jhB2P+})jF48mb>CEe`f#%8+E@psO^($ zjtz;;w>)q2G{qr*ZR&mOd6InjDIgb5@pWlj*6>c;d`el35mx}rAg22>u#>!qs z9wq&t2Vr^Pz1!Sg{A7QrCuJ+nrjM(~L=!qj+aIIddWRHJ-Y_UFmb?-*josolUNC{@ zP(WZrSxp7O=p+_$5a}-^^$Oz`394dezFKl0np3IV;D$8c2I5Q(M{`BjbJw{y@ z4tqsJy%(MI_GCBs)e_uYJ05V-@1H{dTuL0B|1iUUS&i|vT*Y(y~ZCOJf9l<}0yM3my z+3rWpw0Pek87o^yn7n;grknGi3C0a;OWMHWD6v2gQ#&f~O*li_A)EntxtmRR&SwWZ1u=Ssm1cB`wU#-L z56*Ac?CtTavSQo0e_0UGjE6zWpvrJG1q4!cQPS)zN0Qu1_syzJy#OVeXYJ~ze4H?5 zt{@f&B_t-X+ZsG0#?XcuoVP~}psIQqs^bhPoT1zFi`Cb>Y?hMjc1$#qR=b&>^9y5| z!87UvXlrUNC8-t~B}bIQE+kOSG_&78#LQGmjfY-!f6g}~ds8j$%{&Tgi^zf2d;bh6 z&Ka6dr7x)Mdf~KXRG1n)8fKG-)I&Zz94vF{6Y1z~cT&X)mJQFTV&q?&IxKFMpI6USna5{1KC;g7^P7c4D-+to z*@5B%6p@dbtSk6&k`6ZS31{DQlISq6J;cSxt16qopx4u&e4VZf9Fct2Re5X$$R&Uh zQSmew&zz-F7&|YI6A4(y5INj1%2o1>`}>nfejZ02Xa|=V6BD^YU?xB!x?I_?BCXxI zEP(E9D&^&(=PoqDzjH7&Qvijuejd7aMji;vo8!EULW#@>*w)nV{djQc$(eS42DCJ- zoC|btqWe6_5rHi-$pSDH#+x%Ro;FA;+<>B6AyMih{$U;v(>8^4a%qWLnSk z4(8lwGV+TfI2apYmAOaRZSi@?d3V3V*7xiu8xJm#FD$TWkaQx8m{W@I-tjEW0qFRU zJ3d&K6w&_lh@pb;^m!PB8|q@(f+o3=ZU^iTjZ1_qu(0G2p^A~fA4-U1@oggXZzO!3{r)}YgG!&uTjwNgZMH9@ z0DsKqpQjc$3FGgC(oCtlU-*CI$nlgKa{*+jam$vCzQ^I9ZTSZutwRB_QPCWyTp;cY zIAo{#N9hy$FaI}-?*WRW_a!5Gq^qP0n3M$NkRlMfYXy#%)%8X+y`>#~aLUS4-gG!;h-^LA(m*50lCRe2ep2 zBuRXtq?Q}ys(`V=9cF=;#L)tH)n8s>r1V`vWk;@M;YSjeM?7nv`fy)e9FBF6`@S_v zy2*g-`4Kw>UeqmJ)^Nh8(McXM^7R$CC&LV-Z;C8$Pi=Bsu*sbnd|#ee+Wj-F)`Bf- zwawd1PGf!-#zCVsdZsLo)d8i$I3QG6sSE0*UWF*q*B^Wz z6?=N|4Pc2G3S2NGNEwU>akw?7D{I#Hqy$jKIvq?aKux>8S&?#4`7nzt&|02s@lve2 z{YmZcp0Qb|h>X;dhGm6@?(Qc_xGt|j6Tu*Zr*FC%Yan*EVB1hJwEtTcTw&Z(@LDkLz5r{fSqzHdwyulDJ z_#(JBKZ&KjUf3iFZGs+&sLy>0kBsQ2D6|B~mI~*Q#Ev>1@1Dn(I7x>gDHX;}9`TwX8J z3T$rBQ2FF=Io%{G0Mj(!2xpN8)_YMDwwrc?5|}Nk%c3iEd%<(O8EHRG*KkA`$cfY> z=h}QP;r}4Wa?yM?y!t%OP_&4De)2~7CSz`(@XqdmyLD1_iL z4cd<_-b&JF@_d`gGF5ZZid36gr^Oo7tmSOOB7-EfQ>H{AT$C7Pk4CeeWu#&+-8ySS&xNz>D679XGWX8;!XPTd2niuD3-$4X2OQV&fsi7uDE_Bz z@`@C#79%WJ)+>@$=clQhklp>Ox*=-1ca)6CcPf6Y9;hD3n4LF#d!Tf|DlM&Yx%t%r z?SY6`2;s@`sj4LwdztMkrZ7OrO%1~zKuKT}8X~ZNq#lt6(~r}WG+4>yWL8+*T1+I> z-NHF-@06eu7Uz*J!m$hu#ri`8$7rn!RvGq5>EZeDj|a=<3%lC-H)W{(qM0oJvrKB_ zdd`wGv%Y0VDXU4z_xH850^&mqKJ*BPk43WIH26g2OAbaV_YBA->ZmOqk@#+q4Xwrm z8+{DmF49XL)X|mN7YH0zI{k?Rv*A+iIL5lX@NtE(Q?=!#d!ZOdr?=$CyH|-^loC2) z>_Jy3eWS+dl}qHYVKs>HEczar9$ijjKi2GeHRT-VG<#JiqJZF|J`_-U zr>f)SLY8}W#rv3yc~w_PwUqQt8_Gn9zGe@WhNr75kLE9b%y{*$sO8^zcQ0jlP1!0V z(5wL^pl-Kp%^`2;QVuONT9qJnc@mJy5dLAb}bHi%kTB~#CaX`+i>TR9h6fn%mn%BK{Q{R3(+I7qw9c_V?NQ@ztAkn-1}et+P70@8J4rnk-8fV0}Axq zKkmmrRx;DpRstWi6u3=xHb!2VfKegMgtf1Rg@2@>SGniCd;b)84Go4r6pM1HPOYCqbTBe28!TOVWFD>(7JGCI1A#v<9%>Y(v!Df-3NuPL9CkfHZtSI zF&|-%L(`tpcZ5;fv3Ro>%wI4tlb=5$#9^ zxMW;Lo~|C5yJhKa3efL-7VU=uaLjV?`Px_e)@c+UWC3Wk7e-$S0HG7in6|?`tO09h zPdgsb8j(P6F68E8IeTx#u^7}h6v4j;fJ-6DeXS#;Wl0#%Y7H}leYGOSpK@2#GL6oB z9_^_Utu~hob592u6h4lumQ-HZU~f!%aiz&Jsj^-vStiq|vGA<7)=!QC#c}3bHm`Y^ z06*#}0HCW8l>S;qA?%jg`0P#W1mvJAqH4WF@1ZHbyr;5h@_>I0vkd`64NV5ob$fx8 zI=knJ%uhXPmg6scN^ke=F@6(HvRq}Yq^C1$x3O9V3LJ-hWdVth(s8`8XW^^SN-4_y zayQHCg{6aKz-JonMZIG4`$ZK8mKN&@%_o1|7W-D;A0T|&JBRXih3VBE7)#?1+Dt?D zN>ASY18V<|tN5KX+j!V78H-uyUk*9Kybd8FQop1uCjOdbDeln5`m6M7nx*E?f1ZBH zKK&oB4RKqi%OYxA_CuA^&1>aK_%7w$d&x}v$;F+;vhI)^C(r@yBO02ou?j}NvTI1hJq$kneA1;JsXpfMclWE~qf_5Kf?UM5np==+- zqQ$HljqI`uBpBqZ##TyTsoO6q>i4_|W-WQw&+ZBBSKg(fUoP7$ND_$459*HzM8&e8 zQ%Nk-&B2?Rxk=Zw4a~jAJHt9BCpkE4jEW(aM`H>?vyo3zA8cW@=nWREW`=BP@@iUW zu61N>&g0S^6NT#|M9)+Rz98$*EK~O{c0`xS8*EanO$4IK3TMmMcDWaz3qO|xj!2|p zPVOCFx$p4asP4ld!0;u8OTELU@5u*H10qG|rUH2MsI4Zyr@99;7;=+MI zdj42QjtyC{Mc;$PNyE$sHDO!X+c{YM!o940$36k6+)>|ANmz~kix*SV8y16-mmm8e z?h+k(ASRA9ab$f(qRfVya#NE^;eOR81kv+J({gRCdHcwnp^qX1g*^`BPy=m|xM9V1 z23Z+1-`Rv<5ipGTd8=K6&+(EA?gMCS7lpfp{L3qg^As(u!7$vq0F)~iI82Z`WoT|+ zs}bQC6Yan$fxhru1&{R3XZVEUvhUj1Y4^X=!t5W|e_7mj8dFYTIjd$opA#s~R{%Gw z1$mdb&${}%^}fIQ7P_YCsNGgs04(3_>>OtcmEv!d$@Kb6(?4q%toJgjsZ!(sOZU@I9a6?K3h4W=UPPhYQjjA!q0jzdTMgiq8E^m{Hs8IS+WoU*fs~JVD zonF$TgUME{WVsAAZ0zR3G@5OVo)0lWpa0>GdsYb(3n%ZrwcNy8#`uBJbIrn$ILYf@zD z);V$DFB6pXnI>Pu5!9_x#zaNpP+Cdd5+5lRHaJ@f`~cF@d5|pse)Y#$j2WM^7bH9Z zIk2_&iu-3MnIB_Ym9h5l&EPK2WGclv4N>8*$CpvK!{w{JbIIAC!N5W==Uugqj?!Gp z&q_^7DkX|~?a3t06hTo5MRD=fmbUQ3YILpAMIt{su}m4@M0*)iIRqA6+ZUIbOWz zjY7Atx#oo@oGH-hvUcqC(d@;EqCt4+{kr>l!@^@R2QZPiP4U&u*tY0sErC(BY(_Yc zC61o1hE3(J58syE?M_X?a+=lwxmnidNl%pK*k_vFnvV-hr+Xd~#UnzH74$=DS~|kk zhLZ3@Zujx==^n`y6e4~|u^X^*461j_RehOnSz+a$u|bRO>5?Q=KdI zyMD}_iyoDsHajLJ^H~|`&OIS{iH5>Lj@;1cS_&9@C$L<9u-EXR@#M=lm0sse_RrkI zwF=nQ&ev{@C$y#o1LmD8W5#u`8f9@?cU<&_${$aBSkZ5KuruEw`l*m&zGl}Lf%YC@ z0fwycs`m>EJSlJAg7pcx>;U>As)9r`_g>Gp7}(J*8gDR)vF6&kg&lpGTJoSI8@WA- zVgLZ@kJ+=m=n~HKkv};bi6=3%V|e1?2=L4;SznXRp@DT@y*T$f0jeR3yVBP!hw;vx z5xpK(LfiW}ndFlRJW$tFP?trxf0MWAHs|maeWtSA=m0tQ=cht)c%H3`I}0u(W$OuT zn429YRj_FT3Q7$|GeR2L6MCDPV0?M-o`98=YToG{uW2gCm?iAGpfs~UXJR;Vkfbwc!$;=K2_V;~JpbBl_|$rmipMAqsz z>h&4y|A3Ld=Q#H>waDq|I|sq#RIJZ`nBty5I{R~obe8KyVce~w9fS=0s*|`9rS1F^?5HE& z+aXNsW7gnvWZoJp_U!L~{lFi$F?|bhupo7clIGhYvxPB*AX0+$XBu1BwT{&$j{|e8 z)4HiLHa_~rS!Ju|*mIc?%osEh2^EZ2bx`&zyB0Ppr@qb}krilx>4;XWns{qJY?jk& z(<~?%&dVF^t)&U)+JC7k3v$<|jJeD{^SbVJ-`s2Ar!~%cRufCi)Zul{nAV`oY|w<6 zge>nX02r~DoM=}~L`BA=FR@lI{P5H5?D-?z<`~3~GQa~3nMSouxf2*aLK5E&7qOsH0`aS{`X zsl{NBNy!vEgwpKR-3v_-0}+N_N2 zj6cKpdV}4x*QRDHkJ({vJIm3VE^0{5iLwz+&!(LYsY}t#kRo&@)}RWZj$@~Dik6m_ zXji0|&jRG3Wyvim!?tNhy(!)M z8~&~XKD2DxPpllJZBoXlu5DP*jJw#G6QsErUzK}DO8EYu(K`Oq7M#gbI+hoV7~oOP z-+V?*Gvgqr0`w&H!MzU~ey5wHaeFB&W`tte>)AgZr8%ETU~;-T=pgv6SQTor10*zz zF)Tl~0a=L|4RPi87Us;Rz5(qi+OfuBid>o5YJ5rjrTngP%60lWHz8?yW*6k6Q+V7h zpPy+Mm34r~`B8IrxN;s-%plqIraG4v70F%WSV3DRBh;2_M!A8qPksfDB-<{)MnuW6 z)-=u?yt8swW8K3!I`tsTE3<^%MHJQQqelyR4+?QD>vo?Fth0eW5pnP3s9{75X!c6u zNH$>u8Wyp+RQDiv2?N^Cj^qUt?)eqQp}}#+*L3uC!cpq^*X?yc7sN4q2pOuYvXXB@ zBqT~ubnmZg{Ih?w`YaU&RTn0#d9vUks6$1U34m~ z7F8o=^HQI~E3VEV3pS(T)>N<1!MZvI1uJF}9Y#~Ly+3fmsxP!0aKCHy;b#0O+8vmT zS4b8X206uy%prNKV~qHq7zVbH;V1(f6i zgb~+kmS)$mF>LH?V~EI`w$h?2j0MBsoP66kCUMXehNWOg5 zy<|RH1}ZZ%)=xl0gUvohmMLGk?j_k}Y?&-Zwrg6otsB2@?Aq!{7$}PauLiGTpHC#3~t}1Vq%gxmNnq1fXS9U?0O?7$oTUa zci&d_{w6ENE2cC&3;l;=#r)d7a~6xk!Bj|x_E+1s)TD(!W(fveA!b@irUox?$1rJ^ zP8g*vO6qOaG9K1h>|5_EILaMrT6Et#G5mMfnt!V-nE0 zLg_gejI7e_?Chp#*zWzcmM_vDFz=l`2*6Ag+{bDvE0<_2++LP3GUcUm%ue|O%9fxl z)9^xwBSaQ~-4d8KU)!*|e)vu%@&fTw@wOCEs^8Gz(vNR8bcYwa5YgClWG>j|A%@nwn<|t zwNkA}YdSYxM4=aPjqg&i_|)n47&T6AP`4p9`Oy&N(A6H67ZIKt+&3BtI(X2u-KiYD z{p{rukt=@II3<+6S)E8l){ebfN$!tVwHw!q5%eg0oH(62Ln~oTNo}d&490kwd1;Ti z7|6v{^?K)HtH@9Cvtk7N25cC!BI%-)oWiJBIeqytDEoFf&fCt%t)y2ZF~3UUvcUmh z%v69}2+)sF)iH_6h4W0k;<~*<`i;+5V(K%op zo#uR(xOHw|9Q9 zQCb=!6_cTY6?TdSXX^i@Ehm zVLPoHhb;L)8u7KlAZ3Uch~XM)oH738D4gA&WVT?-e!8ieD-SbniL=1G?Cb21;LUW| zLlU^RB=UQiZ~~~p#}Q#ea-w~=xtVy+wj>#oMS8dImlYU03~P6|RGSlzMtij$U0=|YD{oFcLt!LZe~!&sqpLo24V#kvFKTmlJW7tywl}?mBe{KCk)?( zPHDaott32{R?c%4@O!)2ik497An>VIr6ibQI%1xq77ek zN89$HG$_iKd2_Ic^d%VkX?0oPx`3=%HsfH9@6M~zXTlb#WX9nO2gO6@@mQ@4CsS7I za3O+wd29IXm(LPdMf77H%&`?hai}4#T0LzSYRmMsoY1Jhd%bl$%mtahk0d2wpk zjXOKLWU}4QpxHvz)gMfl($O~Nh}O}~ix?F|xZ)LFoKD2-?P0V^rc&P@vh%fxb>rP~ z#BwnD`d*<7leFF(*_FC@!EBeZE=d8|g=%R5Za5P%H~2$sgrR(qwUpJOL)p39#&|Nt z`g-VXXMJGSc7#esj=L=DQ2uz5yW@}~wckMS^1I(BU8l$HMEWyV3vYKtIBpxlFk@F* zVdVqlWf?wV0!!1p54aU;)Z5iRpE^B;T%f)Nv3Rgj6PFfM77?w)8Rr!9a(D@6dN~^@ zf!>sCW(KKLiHL{}Z%<91-zoGhlh@M&7}(T%JS%v!=W0b(X+3Zxb4W+Jbmo!@@y?|# zYMhZyi`ms9`7H8xX&FFPFkD7{t*%I%=7u+NgTv4``rv#uzuSyt#RToxhi&W9(D%mr zmCzK3z)~5etZk7PR)gAOAvmaVlVeM)GLEKCqZQ@Fs4EN(^6-@S>T|<%ree)9*7nSD zFfD+n3wp)Y8oUed*B?;nQah#H&L;+5OfCQ^n=2jB(!@&rGc+*t30EftI6Ol{m<1EQ zZ(3lD@@rZWfn;PNrp-N|TpKRqm8%_g1EMLfV0jI@eTa-LoFLQf&a8TXHkR=v& zJdZb$b%u;~207lcgbLUoyV217xalsCJ9K&b5fxSP>kj#s`393{(4f|O&^q(vu`^BQ z-;Z4XinHHIMW|LYQR*}J15*)BiT*n0uvI+}ry>!&q;8J>Cy@0vE)+V&YdQ)U%2`iE z&$7!%w{lIB5Xngg?McU+BZ=44m%^r0a6q9?)7tx>`!Rd7guiTU2BRF<6wY--yp2{o zs`5pMnA-_E1mursu=#gv<;jbE6(r{0dHJtApfvUcjs!-EMjdYc{!Mbf-qYQj&=TaQ z09xjS$y4?RuD$&#ByK(A+{uDY&MC}VbxbMBUl-W9+>*L}1i7f`{Z-h{qd(}AeX^P6 zj#2_VlQ>*EV5tYwAs2^mBd{6i_GY zHlxAvH%AtY1MFurDdr!I`o}ELKg>mc71BT_#{E=9uEj2U3Drq#^sX1V2|28Y4!+xL z{VMdJp03n#CtOt9kz=;Fy7y_%urgk9Ct=~CpY>P4o^a254Bio%b&&F>V*XUjpBD4q z)Rht&%YXAWiCe~vp?~im0-M${g7?`M9`4(T5OXAYhF#J3=h8rxu*Sc05?klLa(`uh z&0zA;{`6&7sR^}k-EOF#4vg?R?+F$+UB0ZKq#t6VWJqa4cAtfuyIhy3ry#}>{o+-> zM_7%=cTaw3ovTtZzeBI6)yK8=;rsVzujkA$_eM=-oIj! z^mlds4*Ib0uzDX~Oz9tpK71p%gr5rDWGbLYdNVG#DEyR$=AQ@9{<~)X=T9bEZ_=MU z%TNh#ayj`s9>kg2HX52}Dk_nHfZYSVxPAlBn$h+5+e?Q^ALiR^?-@HjRhDf&aDUlp?V(r3b$9!SmxaC3IR-&HtDG6G zng!JGW1&C4|5HDI+Q*;$UN;F#`;PfMh(YVI`?Zog8FAez)jy5R*tg*l zeAQ`ae(|zjJng^e^M7j>qWuf7^nYm=f`$u;iinYiR;zpC2HwoQt?|p=x;Qq7Gi5h~ zGsw-XPS#>$CDY3L>I3v$-Tm8lxK*9z{aE^PSH;gzbE99#`Cyh(bbj2|KR#LRDNj46 zw|n(O$+L>cYoviq(QKUwIeWbib#aQ%UbD-v|9R`58u)kDz>|evt@(^S{TGYP@a)b_ z4-^RgP8$5H)$_TrhjgDA6D0`#Lxk5;sxLVaIm9DKNTAT&~k9 zG~ayQSVeR7@gMAPP7Hp$?l7;QGxh`R7T}82ukw#O;zR7IOlEy^m0meGD)`=lmrJo?tS z6ECZYZim5;`19Mpv5H)HN^@4jX`3$i`f#p?M|sbq=Cvafbn^JBoj3ZQS@Zt(6Tb>f zF5c><@>k+KW2MfqCXKCNWQX!-@2xv74^up%qkB~bTiaswSD;&&K<@xZP6Z#qO=1)Q zW@mGkI=8-F_GAC}4;!CpJcV>t(g+_nRlf>9=)61ePhO|z@HzXnxHf0mZ#Dj)6X?eN z>eO;(I5o1rFW4}$R~YE*ccjH7wEP^CvKhkzIb3SHa@6>h4C~g{JkP85sQ>Sj-a6Cj zXPOehV^v~y_}=;5U{c(dz(rjjA-Uoc@lR=JJl^)R8Et#Mbn!{J44x<lM> zZJcxjP^b)UJpf3bil?HaptJaxY&TdfSWZ^>fJO&hTnsJ~&j=ZJDaf@Nu%PH0thS>f z(0z-^D8m28-kV1?m8E;bv7D+D%0fn&ESU|H2mt|cCTuIk(GcYCe3Z+GAIe&2n2|G_$(wa-2$ z`|R`Vy`Sgz{C>l%tHN+-D%Dj+ptKv*#GBp7 z4PUKos@>N+FCS}JNtzBdsBVn%ars(94{P^=KTl(FQ;yGcWk(i7O8G1l$T}^ z91(%cGwd#st7t%-{J@zi+&v5;a5AW0F+}3>n!FGjDVpNQ90ikO;2*f1GM$+*jb~In z8JC>Tz;&QRx|BTREZMtiB(!8BzIo zMdSN-d$5UZy=UBNR#vo|{HFQGtf(S(3&3?f1J1lmPAjf&zTd9?p1HnE5i#yBcidDo zX3fuz3X6;U(w^KS4U4GN6e%e$7y(<@GVO_pzs8E4Lh)XRq7amNo6>glCUJfZrv+b!lZRD2GUH(p7_YZOmRIzNo*>6b zx~k1WX7Ql3rIn*jg1AM!Qcq0Q&Xq3k=R*MMc@5N7qA>?`#T|pmz>r;D>oGV4cR+ZM zB+|5|Z;hleexzo-TSOYn0Juh)f=x`>9=Z|+9crhV>mQ)>>sAM)Ihfd_w7Gm^Ae_E< z<1^B8Y?o)UMNWXRSpr=r6>&S%x# z|6$+4k`q??!E&!%w*A@&ExO?41HM=SxBk3`F97mq)9~N-_iwB37MpRh#>Q*eGi5*i zW4AuSiJ3=1Jr3|x?qDk?A4%VuVNi^tUywv$RU76SviA2@k33HCHAPM}!e~&a^VX4H zwI+2E6`$hAdLI^Yv{z)wu<;p3=5L_hagLS#*BNe?ziGI&L>-b>uLzuqsHzuQ8-i%s z^A;oWFqW#2ZMiaCaDDXAyLr_eD7lj1q(B-TiyK=8F*KTg#w8%Mxa4)n_39ZWxM=w z*GS2n9VNdYWr@PwzQNp3%VcFcRYK-MG&+@L7Jx&_IoKozBhFHJp_jp6s_5s-)QDSC zq_A2?KlHl##DlTOWY+z;GCC-_YD6K$kPaQHbIDPK`S;_N!{+58UtQHoLR?w#3E4e& z_jvwTCh%e=AD)66f{W&UHI^uKAUcvWR3U?sKI6`#?9L#kH4Ub~c%WFGZL+mw4H}k| z)`V?hJ3FoPB;{~gyE$UzCISkTsc3s+d)cI4)JN@O0t1dlp&?L5;z4eNU~>e20it2w z$5#HK%}D~2amLcLniN%(&!AytA|#5^r6JUy-cX;Mylb6RbfGMww&0|^;uZ{d-q&!U zD3zIcQK)t`(renx%rUDu`_y@Zqy1|lMr6j?!0tWvy;Q|un;p7=s@=MnGv*W`q*<&fq&^crxvsU&XXkm;Aj= z9emDQWW16v5lim*)mlT!F-zK%A*&G^jn|{vUd7Fz@v|oSSBqMOc7{xJ9h@f@@dCTR z*Oxb_{QR~VmeyBCqbS3ixy{R9sGYFLRjnD zSFKoPWe;<83(BlhG30VAWipYhy)fY5jqQwKZsw4Z8Epnu2g4FLVn$=`mtE{aZ@T1g zV5vlVrXR`~{}eyyhNaw?_qsvQiqsS$cA{wvjy_^!l``# zHEl{6{v=IRFx8S@LpkQO=OcrZ#k-xIYvH9c7TjX0eosEU*An`upByDpDwmtj?ziMo zoh&#~s9PIT$(T6=3^e+JJpB?~-zhwgnyRz7pJk;~xsv&j)pWh^c~rwy;u5KJ)YrUT z)b36q{5C?kwk`qh8+5mVL;V8eU>3&<3riykw}mGuR?Luz?>bPvWB#;Wh2W*DXl&GI z1aovja*P(~dOkZVWeiixeqPhd(Ph>fZqw zBW2!`ZdbyEoG#JR@6~oht!8mh)>#U>Y=`o3;$&G(|D7CTipT5A_9c3BbM|)kmwJCa zmu%}?O*j3MMV703Z6Pt+KK|)lX1+F(*plIjFkm-XB0x`%HPG^%{g~40=o&}h-L+19 zWTVh0`yX+xGNdwKy_C7f!d8;7`wq4TCI_4^P1Caf@U{dQgxH#+8ka6ble#C*#LL8l z=nX*(cS^MBjS}0&`DLMTnr=dG`>sEjwl&``G0kA|e55V5J3}m-SBX>Pb@LrC;lGCTyrNz`30#;%PAfIq##ByL*Ei6C&bG_4tbG9DGy)aaZt zsQCTL;HvW-H*4w2db&fYA$F;>Bo&sdZzfED3Z@1*zi*zbHWwn$4B%BwG~(uby~DV$ z0*@w@EJnFnUly1C^uC{h-Fqi(!AKHH&9P-TP$728{{PK%obZY>PQJPLS8K}lFAt-y zul5`QQg<$16%ZQ(^qQUb8;cW~t$Z?TVl(*uSh&MEh_#1bWA9nRZVv;{I1s30xKosk zleSr*C14XC4KyX|qqWDq6&H1e{H zhOybFYmPNz?``L_yQ*EebdSdhtRe_dTxI4X%{!R(bOQ8>J-;9_!qPD*od^0D8rih= z^(b-N`_f!tG96xckL6eRu=XSE386SpSHxO|P-)i9(V$3(w@CtX-(wj0lB{k9)LXWX zU0k|YWH=kIUTWqB%T49sl+&)B-9H07oO#Db4gtr+D&pQ)$5nr}_Bxf9LUCHPWr|#< zZq5&H@|S(PflTWDP?` z5vWL%mQlWTj{VV*(j{{aRq7Sh%i&`<1iaMn9Jo|4e#O@9iZNCu&KUI4=EfIBu6m8y za)?&6bJ(Nn2j6ZT0{XBm2m2Sc&$yj&laUx(Z+~A|#_y!a$SZsYTzNR~A+#upWud6I zIA0^%s7^7KF?YF@t+_!J%m|UUxU+uw`Io$uF{v!=v+qt?_n)#M75X#g_tUZI^v$a> zFww*>(|8v3CB1WIK`hQCnwLacGUXD7`r%(3*Dn3AyG%x9xYuR5_}oIUwf_du|-m# zpj$|cSWMxSP`_K_{%iR@mLr8Xx#HNL^&7LUuvR|$L$ufXo9?^pOHt1Ie-h*xb@2Az z&VbZ(ftdD^zlexdM%4uSM%7R?BJ-VH_15myznrK6F)30MUZ+76JZtf-mhX4>eaEu4 ziu~ZsD*^XrKsc@m&m!YYDRjIo_-MFI;krRl2uQ3aVb`Olta@$#cEf!*1vVBdS3Bz9 zv-U~+1E_OO%$KNc?-nJgXJ}awq&f8>OcVAjci^u1>#9!y#6y7h1Qb(2qCO&sWyZ6N z%=n0k4pI*ZMWgySo&@?EGEm1hJ?O0L9S3JY^p1Tz@DKnj<(={Um{N*=t@IK)tYGwt zHVVG_-azf~b`=u#7y<=T?dWrQ)-P9+32$c12d6qkZ~BzP$1g>$(Hm#3a_K9Ys*I@W zx@6&dAH~{lBQLCIuGyslNPN`XYIu z;ho>a@pcj8)rVuAS4lPww`*_1a-T8efODyJywawYm#-bKbjz6oqsNz#VL*?kO@uF~ zh!bi|nZD4lar?*k1`DrGZN3w3nK?NkNs4?GxDtX&g`T^Q{FdKkvf4tK&Bv+o#?;`v zhT<{WuS~hjWelZ3`1IWrd|~! ztqiEOhP^a;-N;I)G{09U43tMcOMEp!1S6-=_M%C^I{x*3Bp9ODAz3|ca0qDc^}oJf z>9hZ9!R-RI$f-c8QZKSTG!)VF;Q0Rulg=$}% zw?q!yj0k z6Okg+Te)L$cImGUK`$enifA|}l0yRPr|n%BL#lCmk_J?G zvWqo*hN0%#sJT?vCN;eYoIlp}G?Imutf4alkA1QE&d=~8%qIhehA;Q3^>^oalNTrw z^U6#a52z+Qaq*=_L6S&q+Dn} zc=+XFxMHkC>DbOxAZX#Nfu)ISi`DB6yvnO3NdrP9oWQi?Xsu6>BQ&WZ`U}L$VyE}q zoLu(!_h2!QfCN!qz-x$XbFL`JGsQmw8Fk9SEq+ODQZXcWgxn%e&E zoyU}8om29f46Ekx`M_88DEO#r4lF03@Hr}b1NJbia=1KOA_Q-gdGOPlsVKg2GiDKrUTo6V?VU~8HX!m}InH326 zb{-d-@hKQ9vLp9u)YNl7s!Gicq0GAE)DT_3a@Fa8CnVWhFq|IX6gx{?-{I-`4vbaI=krRQQ z-Aj}Swj`7%b#sJ9fXXo<*~{Tow?E0AV-=GpE0(%ErWI;GTvOtSm_$Rs&M#}!qvanc ze3BUM^-GJ_I;D>^ivan+< zX0OjpJ&!PJCZ1OIXj-EaiP?2K@t5<4DaJa%tz?O+FS!S zD0`DEI@)9f9mxqB;^&X$(?JTawsEa-SMTgG#N2-^wd_j{4gzTg(dgh_Oc;NqgZS$m ztzI+e@k}jvl5YI~Gc&`nt|O-qiHq|K zZCw;K4F!i^D(u$dwM@22@{f5a4m``1NWtbxB$WS}wA_eq)#>5e%a7%`tHALu_U4M; zoyC{Xk_TZFs{pvt+{O{M_N_X)BZdFcyQQZKC5mFl_4?tSFNO8c&`&&$N$a5tu)MxHaTmUSps+g$k~v@T!fkcXO%i69F$5B( zgW)m{!FphLka9S|=+5&O)g2!9xCOc~X{o;=g67U@7;9MJqjPHpFI{aTSbuqA$#Jff z?RQn>UE^z2f5zkGf$ueHFBjU?A>2(#mkyf8<$76tUwskMb&_?* z)pPy0bK+o$<@D+`#i2$L-|U?}As-TXMWwB&mQ5|0@OGx>91A(w&NP3JV@0gu5V{ zoVTh+CF2jQ{c9cN)B|oB+-~UJg-m{kK^CQ2UKHM0G9|K|P3f8$5h*_&d$DR4cf8-b z*6p;~2cml4SVNO(Qb`JY`3<&dl_BNwcx7|oJZ`+>_|`(y+oeonTWf3G*1BHRtk~6z zsKo-U9B>hzk#{w#()X9N%Wu&JhIojL&tU2`)zavcbVYw8(KOv^A?=j-FA3NCr??IQ znduuCTi=$?A2=N>d7h?o{D~IY-8|ov8HY5CM}8pZnKxCQ z*}+L2?+U6D3`(t%d$IrF4^Xe=AJH+%-`?xDWo&tl@6FTpOA5h+mlGh{Td;N6tof^; z*k8Z>fBIO=|H4jw5hs22$D-{7i-ef6^3T^JzLopxDva*y83UC6(>M6vH1J1MR}iJq zf{@|d8}OyxN8T{U_@7bp4SK4346-)D=ZhvUqzfm+-{wRLuznWDTD%*+UwTx8^Ex^3vDBN@?05V7=3XBUEZx5(HnFt0%w1IdSw?Uj4^*?Jdldk1lk&?z zG3vzWbBJ^`>mQH)+I8RSyH;=imqdEz7jn&?)3JZzyf^FUs%jGCcR(oT$Nw%K{L|~- z1InB`!~$)1&rw>|l(Zq6E`7@Isl1*%L$=u=K=?u>l~`fg z<(|3I{cJ7&>Px)t^bLpTL%?J`@Ba99Leqsm4kQ-=r~irbl8=0UZ&7lQ_t7`g*LIcK zmkt4ogB9Qdr}lCV z0S^n-6fIUDnE0ktr_GwL#hZ3vuZ}AIN2mX5je>tBBoiz=-Pq zUjieJM?LJ!95GOI3Ncj+PL;vPmZwUh69@24)b~$e>okoXp%p*+ligFAsza70^6uzy zU?F=RTgWcQyg%a0ce{B4j$2;5(_q!_pr25uHZ;I6`UdEU+9{l$9C_9=5>Y>0o@?)m zFKrmsgo@}$^=E=0ljX^9tp7J}|NKfW!m4amwzaW}86WF~ggAs!!=~e~6muJF9aPnH zBrq6o5jwR-s3i?>PT^CWJqgDeVnq?`)W9L^Xp9OFH*384-F&yQ8e2koP|A4Yi z2`TD?a>|-@qn3kLwJS%GQS`j>T!hsS@+7a#6fCYG1=b&9c1x% zu^MzR=e)-}%{Y+-3`PaqV#yJ@+aauhq@0WTW86UFRo$?Tte(>`ojItS`#%q-jod}8 z1ui>%E{hRdP_k35zw(}MN503p7s3q2#g}IASElK}p!~S*^5=d(E?(lFFZ(GnB&(`cY4+ia0I`(6hlXrbXJQ^|Sugcj*4&8LUjWVD|-^ttD=8lfN zO2?DepibPu1L#T7HMAX-y=_d>>kW-?54~b5aBH@aFEcIF=xQJVF6j!q>mL*pV75OL z>zcd@_|x0_?G64{%9TL2P_FRoOWS0nvAFLXy?q5%BqdEA&XT0kKiigicS<&yvlliB zH`s%|<9(kxsu{zsqgGI@?}Tf^V6vcd_QItRI36Ut>{SAJ+q2f6Bjq|Lf@FU6{Pq{c z@E{rJ^e6s`JV%yqacO}3)U#zus$fB-YEx{NZDdUTZDtQA8JaSC&f7--Z?_FTI-g=) zJe1?rU7lC~e15(r^mq2zjz7q{-u?E&{$!^vF^KZb0&%tV*DI|`g?GSVt%E!1ukTC) zxHwVpqs5(1hr4PjuoSXiHdfm7a zX?*9QA7V4k?JFjOPklecLtCJb<54ZFkB35_{#ovGHu%h?w$d~7LVt%dZk7d5>gML($|(dwql)c&9Vu=aYF2t8J6HDWwuYRMMVy5RD# zG6V_@Ib%z`I#`@I81?KW9+?5P%`-VH5U_<6*w2@-)i4}@5@`EWz+*MOaV5VDWxphYpwokv-P8)}u;^{8&TO)!YFH~!f1TUHXTqj6O1O2s zJ$zuB8`U$OZC+=svH||h6^3?$_TbXCWW$+HKc=FCJQLXcAc?{+z1=rC)Fc07`5)-( z3}?7Q9qZ@sJU(f-ZTuMQ++IFRnJ^PB%E!HDw_GsX5!mQ^c?Yc?BZ0f~9J}_AjCVW) zP)mA)l$YlR(LIQ}GmR{gUoP*I_eNbN@TyKB!@yoQvtFAnl|Y{#bpE)Z`E8&%U)5OY zo6TgEigxJk9xNxH&Y(G7ri)rwST|`}>NqqDTxdC^PyMB3uz56Y{$tDXudCX8Eq!## zL7?RU!94fO4TR;^zS*B_xi zlCr!L!lsmmp=)sId8;o7wFJY~;O>f0WYVe2cyRi0foy4t3SKE;z%=rv~O0u4sV{gX%K?SkoQ^k zONo4aHKI_-@axxuD%jfOL&OwaMQ2x-b!qBo%`FDiTfZcxbxA(=w3N#tNSFgJ$hgMm z#9jL%)Pxw^{ zZts+k!buVm>_$t6IrBZBRFuqwQ^Aq9!jAIMWtAlzHuXlPy(Sl>g6ot<=Ui1Rv6Bod znQe2hv+2WA6hS6PU@-JdcsuA)bFf2ue_1Jjpp!z0jp%y5ovh>AoAOs zref#wNj~f0XW3{(dBA96qBV)fgM3heN+3&yxi@`;_Wy8yT-^=rKh}tv;oj>~Oy2M> zheZ@Yg4&doEE0QHa>)_=v10R!5rS@cTl>*OXI5ZkW#jmZN;?h}LK1%s?Pd#IDT=NG zpKscRY`Et)dWLc2qyGDsSi$jI@~w(5dO%Bo+IW#KQ>ne90e&?@r1h0T-2=x(tK%7txMkHrV~!X4 z>;A7!s~@wNf3iPTskWD;8V&7Oh?a z8;%qrHA1sL>-!n;5nGxgd}Y3oCF`oUoSiC+c2$am(b%Q~%*+uZ0~dP-E*KD}c6SR7 zWxP1X?Bn{sS zf;uf@Ut&v4cKGUMy{sGOOzXZcqrV0k@5qOE6jVayZ)l-8>)QGBL%_nNxJsXgdje*) zk5wznzy1IrlglL7*%$BRsF?c&hCTm51x@OTT+-XbCh=YKgI?aBey% z1?2LWne9`OyrkGc@UWLMIz30Dj3wpB*w6LS+{`?Sit|a{FNUNMsUs9Ce4SozGo>Es zej>}ubVpY9>$&S=AN*Ta^3T=A_08-jSm=5c)x*a9?;Z2w6(STXL4$l8KY#~8&d|9Z ziFYWWV!jpiua-kiGD|nBqQbl~4d)!mw&_1=Upn%S%)loH#(2Flxg6@uimYZm6+=t~ zLHYSCg#m<`5yr|%4uc8CSzp^yh6$?s1s0ri-pPZv@mNF=#)mciYJDWS(RqE)USa?bvvOHi@#!IDSwDBYU8(k^x%KmN2|ru-}&N(B3UOYI$Uq(nep|oG|M2Z zC>Tjz$Pi_?mzR?znaQ3N84gue5?G1zT02*ELWgYa-+zC0qTnr0(*oI zFI)r4f8j+wD=){1sM0ZF;-wUsuMfhr+)v&a=-roI9~CK8CsHi~YYkC7*ozI{!7qUO zeddE?0upUw9iC<5vox$0%Ac|>YP|g%H0ajh@?=B&b%F{P|0f3mzQYDjf{D(>mz_25 zfXui}F)>#Ha3mw#!pq{z521uZz)iD* zHuqH<4(a`gu0z1C*5Dz)>H*>J1II+q{SxXoj;rb8c~o7AKLijSIVglKM|l8g($ryv zOYLLh7%M(-Xl5xQEnq_g-QZKeOH5{%)R_-Vb*(*{wNbA*2{yRfeuiY$EX-_pm*Hkm z7p5cfs*sIYar}6YmV0te`IUgydWXZ#MdNV4v7~Q%je>|Q3^R4m;4-AW@99U=A<>jx zDaSfV+l=yCT``ZfeVFUKZQ)gCXP-$w+S$18KLVYZ7!W~)Ik*^pdS_rwBiSIF%n=_T zh*X{+YRHu(6=kF6a|0USpJgg@Z<;x!Xe{Q z$^y#3#%ui9kkt$deWthGwaXo>u1+8J}M$jaZ@w7vk(v+<1Zu@z1uvY_;Zg@lMKAr@8~mFG{b$;r%tG zj(RvHZ-<03l8%2N4OcmbS#`@$E5ySrd7Rm1ttt@MNT)gr?GvRC`eF=o-%^a@~{74y=rh=CcEF>A~g}jVamArUyC?KGhJ~#Af9W)LNMDztQ#eyU$h0MWjkA zQXW%p?U>+Ye1gbs5Pe*#j+IxY!MJ{)&7vLP1m8bm>Hg}PzvS-xbn3d@gHMl`ktylB zzurq3y-+^cSNZd)pZ}`vf5+i}Tax0D$EidcdFRn4Tn|IW+Z_-jq$R|XcMtZn7a>*Jcd2AA9G z`|byX^~qmKcfki$djXrte_ZjX)!k~| z%KD4?Q|ArNmwwE8j}E+3)4bx-{a?O+u!d<|yh6yZ zrDLF2;k3OtY-t%+h|-@=z~nlzA)FMKcTzQOzS;>`mWfXbb3H`SwYz#~)IfEpyd1Tr z7=*XS2CUrBe*VAS*Q(CBlCc~Hagbl~zofDvQc|2vtP_7$358=lvrA_e0f2u!kcXw; zzEKAiT=)-Qfcx!Imj>VU zn>7fYc@vu(@9HN1A&Disoa5MmIFKCgOf6=Mf4y2Jte^g?Twt+}<~NW2-ou&x%UABc z5XsS-Prrh0eYBkMpEjg_vQntnpPJ9_IK_4E^AWCJa5=Mo`8Vn_j@b_Vx7q$*t?A$R z@4T8*+3UNZ2JfJqm;jh`QHi0pYdbRB{FIVeZ;Qg9=_>|z)=gU%r-x_z`OT!|*_fPo zDMa=+K=}+PZ@QYrQ+kDfL(B#dDc%+MytE3Pxn~Q?xJA-~Z{FR|3sN$U40(5iIo}pW ziNR0k`j$7`>Nje$wkAq%``VPkY2aQESVQgu!kHe5Mz4Kv*~Igau76**VI1lx8w7=d zu;tjk3?7dEjT*IIWA$|;z(~D)p-VkdZWRBWy#~bqk<~O_FpUc^Yf>x4kOmiw2Nq! z*LAYd+oNUHpx$nJw3cObS>My2%Dg^VpO&sVSF*=Qk`pj2A$3^VOo>_czHocf$ivUi zo56dPksgjNnt-eC`lPs9zDrmtj{ElIQN_I`7;PczBu>e!hdj3uHa7+OVpb0y0f)$Fn=T`@Bt^+*;{*(p7Ab*+lI_@8_gx8CfvHH=3kQ@azVC$~BND(>8qMo1=p_>u=+R5x)mkMK0aaX>|kYaP~T zqiMptabP5`-8a`xeE;^*!;J+NSz3 z4yE#Pg+_c4j_m4C<5p(kYSo?c3$)*MC8$nhIf%2I31h#^?F-GmsniGGxfzZ}aUh#w zNeK+vjgCmYmwdhk%2w>vW-iL2T%?T>Q_NGG>3OE@?#N9=#U%W5 zRMP#d1zDl!#Oipl8V4^7sX`e=Y&0MWy+iU0*sF1Xxc+dEaN;zS4Kf+kA-R2RTAjuB zEWs=9YGGvm>(unc z6`w$8<*@bT;T|I~MUy(R-D3#69z;uQvQD*4zo6T)YMKWOML4NiS`wX>#S>nZqed^T z86<_usN49Cl51-2XDe-gyCeJ2sU*8>Aj(K3;1AX)pw`P~BLQ&KC0aKh>L8lh=PlhM0zJmJQfKlc7!A$iy`DkXnoR{1-t_H%z! zU>Yk2eqen8&j9W2EB#(U+X%dVEvhBqrL@Tp9=}&G@R4ik8Vq|c_yFU|MfttL?@u7H znW1nq>-P$OJfrj3e?Q{?)xCHnu;BM0aV6w0qw$HkWT0xrp9gGeBlR!iZ2PA_T>F3F zkahoUkpTXWb(CGfuT_t}X8$IH%%KpX{E0&7q!C)BB*n|lW+5?9df>E9^7ttlbd*Ms z(Qn_vVzCk+RVY+X4+_y`A9SKgeh4DPuV(EOlkxVTYu_3u=eM)m6RHS4fPw?KI8!`J ziP}@?q&V`rw8UOCQ(|^_+T-VMpI=4R@SbJe4Rbvht_szD2*z&cdS$|4Od|o!JCPpg zIqWG^cnbT~#EvwxvuERJfY{{ndU)>{NSoDi?qi%Z7<}B_NJlL>Xd_I1NHh2OD8AQZ z!lL%(R{fmtgx{`v)af$PZ6*_SH6+RZE==*V<#n*0Gv>U)W9J<+xazg79#ohMCyXXO zrBZ*V`FMv=<7dmzM)TeW8Ced`_Si_bDhyn6BWMfsn1e)5pv??{E2QD;vf!=H$)w8YR;YOu1{HmGyfl+^Pu)i4TuEQh_1MK- zWl6?MgiW;7bkxxSIrX9k)nWynJMx_hu-@f=*Atf;#^nOd z4l=3Mk-=*(j4KVt!v*f9VVihOEGxE+@etD#t??AE@7qZo=;Bs8YcVUvG*4CVEi*G6 zRX9&#?kCxv&bs9U6j@>+4Or@pDlCpFzZTo3Q=omUQ{={@6R^-kUF+z4_tN{&tM5k! zf%qrMT95QoHK9Rz)NxHH565JG!|A}E|6w>R2V?a=&N~ATYLwAgLq4S=V-Oh{Wv|=; zM+Nn|n(#>B1&vX%b!7C38{a31yk&C5#N!mQlz3|Cas@pCsf+I?H;4iZoH|Lrm0*% z18y(61y`mH)w%o6H(9D${9^j*H?#Dw@%8-H(m!ABzDs{}~_z~rq}uyLE*7mhJ9srVVx?qpY7O>(38Hpz7p9P zrkG+;qaG^6xcTt>jan5;(pP_*j%76xgjMiuQhMD2&dMu4YEYHAPs*w}Prc*z`qOMa z?t7Q%kg~Rn@>r7{7!W$Kg!a%d;>u>aDbm+z|?>E)beIc=gP>x|8J_&>kk7U0hrNOUy(Q8Y|N^ z9>jf<0)Ep0rY5!EB3u#C_}!Z%x?U&iUJs?Em!n#XwaR@oXB8ytTFfE;z$GY_=f(Bp znomUFyfNP9>L9vAdKUuG|A=~kF)uICZy{E2-^P>4>wkNhZkRWJ`vFBvQtqyQi;7H6T%Wdq1{pkwoj9LXzVHE_%GFnLxB5Dc=fl@n(tV)kAzG~LlD z_l{+VkID;e+g5#>Em6p^^bZoTG8!*ywsKZixS9K>l<77rirge$6-)p8fjzCz)>?B| z_Z{cc#xf6v&7}zq zD16DvOue1IB>Ao^;J7W<0Bv4n-E=}E^)h?|!Zur|m|2O=%sozw_W})%SaqD9?0~|_ z6?%>}8~&wq+Vnm`XS(}?)Kn5fprMjd$dxp>y>)smQEPu5U_oQaecij>=3Yg9v{cb*E@b= zdurnlK)cR?NK?e74*}(|Gc#1H&O8Br~VfLmXY2dgs^aR)E<7QT+NJ(W1;1p~%8 z&7S5o>wy((oG42bvG*@80UtaGFygt^*2!i21af9N_dl`8{wWFVf3pqzhrgb`b>aU| zG|HLNjr5sFvpQFf81INyhR<8W2AYWk;_W-O9(X#_bNe3du42eQaBIbLzm94knf9@# zL*Fy8Ma^_XN4rMN0smuLLzJIgzOR9(m9>$@Q3)fTPwtG@oiDSM9BQWgKWU4-O{yC< zR4^?o!x}nCPW4^kQhk1Q@i(Wp)gr5oNV(cmX;B>6q+D(b=b4zwH++P->p3Q!sJ&Jm=?^nFE{X zF`gf~iKz;xG|bJx&q0HL^SS9wIl^Ce=t$<=Foh51n)oC$x`?E1muf>^jV<#kkdFug zvzmCru$MZte=2L9u0HW&X=$3*PoF>0MUAkAlPpmV)y8sSPMMawD>4vru)Fm&bR+a~ zb3xH;SG*)pmal!KV{m=;F|W#_kBNDd9aH#JKP4*ke&QzUfMJAMQ&an>aWVW`c)5Gf zriCNXU>;WhM20b9!F`%x@n%KufZl7gYBLj2^AL#l`dQzxq~+Z?3g!p;`&i%$CVhl^ zd~>-MHiomex3zplMwhuE&6e3_pMl$+33(q#tt|^~-q*J-#o=_*RQhHU5*wNh(w64E z$1btmI#QIaFW)trNlM}5kqISY^0EYpaXHaPYDF!QU7zEUdyTm39|VVlm}gi=0@pc- zrew7Z=e;Tw_)PLfeTB7)duuSCbtOd#SJ}`+<#ld(qv!u+7Z@$~a=b!^)8zbU$^k5( zxisA#LA>J_lgKNm7$PPXyp=`AJ%c2+>mHTm-x=FV@&6WQM@dJTV-M?byaLJZn}C-u#BlUj49)K zn$BfTzy zvNzodGm3K0yaLs^5t!TcFk&fl%Y%pZr zsfUtr=F%AcRQrx%$!u@k{Ann^B2y7*=>WIVCk96!-~P zWiy4posO*6pQDO*=ur~~u*nZwPD3(YMTrYDAL0buw0A`6M@F?udm7=`$xi8OFSp(> zBr|%*%X9F^uIR|WtAB(KS{3-mqFinJ^yY%5s{(104NwakEIRD?2dwUFLe}L?CE6 zVGxp-gGZ#TPx?_n%bf!{Rg%Z9-~{{-@H9uRh!pBdCeaKh>=CvMl1?}NEgcD2v1xBPwJ@#F z?k_SbJ+Us_8eX_^8z29+{?xZc>86)JOScw^poI2>HSzrv)dk)!bGMVMO7m zL12x|1T#6Nxj9e7QpsPxP@e|vA17ln=cWr}KET>Orb2eyTz#Zmbv9w=q~Mqpr`IF( zYM-95KlKq5Njc_yqL+4m>e3tyGbfiRBxGv!+(p8ZE4d#|;!^X#3q_FB*L zyg!&3y68;k(P$n_BWl#uSOAr4=)j$uO&AP=<%ZY$TfPKOFKB!p^s{a3N_tn}5a0T( zeM^Xf^CM+1a3%nmZN8x%+A7Eb^7oq&n)(UViG;U%dMBdi7*n;<=dF%eh%;T&nViZ7;0vY2`WX&>xpc zOh0V$EdC{>tgD%L>lx-mkh!Tm7y=vC;+9ADJWLWGcSjhDmr>V>DkwH-Agy!n7RE4v z3;wAeByNvikA=w#w^Ra_oGly>!EhT`vC-*I0kCo)3Ib5z>RP0jY>^#JdS#k~(bNkF z)r*6}ea<(Qcv7fWSNtp$1}sbQJ&rw|MOug3EEN>Kc_dt7$4e8TXDV*)^2IigRj11j z{N=8e-h!HtJ3)-P0`8mKOkYX=DW#T%CEw5E5#N> zc;h3z>=xYzH0&jMQ;k;V%yZ6+5Ct5{DPPO8JZ zNu@fRq{9UTybvAj&%@+g@`k2`r9qGnC{mG%J_1*eW8G=I73(+Cabaeh5=qivFz-Qp z9hZ{MWNo{C@Ms~k&nums6=^-tbgVu;> z8-;j!Iu|(iwa)3E^=M{fvBOQ)Qdd-xW)AbaQz#H~nS49b%d_}G#esFewYJRMF`(qN zSj*W}DG_+~r|eTJj6Xy{;&W0|4Z`A7qbX#e>~X~(LAKlopw=I4iB@|8N$*`4dUpf&EDU2zkgz3$?EKB zOHH(whr?a^or^YPZMnsIwFSC|f&H0oxZ47LC$G7{;M5g;V0dR{-cycvG=GVSKXcm! zyfTBtZT%xzFJI30y!x1YdjYdWE?3Rhr>qB&!#JB+F|J5V&m#X@*;npglLED+l{;}mUS9&&;%EPvtkQ1+R-Mdh{&XiSU{Lx4c`AFm}N zO97=vsv@>hE-Q2A9G3|9Re;5fL(GLswuo_C$t2ZlfHL0pOn;vqE-od|*zGdxat*9P zLph;7prp8Fl6=ijh8*Zv@9svVD>+DG2yE{!dD~&sE8KNC{E-)=S|&TV$ksjKMWQ)2 zEBgIN(%_(z^*99_d<)<&QhVafhU0{95!D`F?3r({;EDC-(#uAotlB-D+?`vhTtVQYE@`zEas3bhxgHKy`G^oQP$dfl>!PX+U4(f=8 zKo{#G-t<6_Zq;o44;Gl6Gg(7fq#?>|X|8dryebEVAgza_enY11T~oP~QP^QvB|!|i zjj!5TmkmMbyG(pI@94v;#w}oV&nq%~h;0Z52~^Ud=>nQ<25#~0aTO5dhE^tM*zD72 zscKi^p3zv25o{k4HWQDT$t0Arl}FYbO)B5X>$le>hpW}HXJ5g!?`y%J7Umb6J#$CqI-g<+ScP1-{^NHjc&TvMeV6s2)CJdkibrFqaLz z!P$z6#B*v|^;oxXi^9q9CvZ`~flhR0g@<)ddQgqGYDDnZP447AmE<_Th$>CAIHM6h zbucXXYa?Wow)?1~)t4)jYZVz?5xGMHoZtb2qxDh^aFejNd!*?W{hY7@E)R=8=W;$RGCJ4Ffqzt{MdoWUcp#J$!^t$lA^)|BM=SBM|HCh@AwRW&b@s6#V24{lxJ$;i(NvV~* zobeR{`rLN)dEj88h5EkLyl>Uf=eUK7>D6>6bGm~d|BzTlskq5eljB#h_blRM7D`tfsAmGg}XeAQqz4rY3ltfK?>7nPj3`n-xWmlNk~6druh zwd3HTsy4I)UWXrCF5lZXOZpf=Y8a$o>mMj6i5Xy43(tAPIzzcrVxdz8^O0e&x6_Z4}jM5wq?cTs?cwb|>WX6IxI= z-sDrR`khH$+xF7_AzVaz%*RLJ4o6FzzudRKL!Xp4^eA}$^6PO`?g~{CJGfZSxCmac zCjfhm2>r~$PEMZxiW%%4)TDCjiBojhP0d1Fz0T1&TrN62tfDaZkxU+{$0fC%0UfB1 z3p;mgj7@|{+pO+=T&Bx=RG)y(tq!t~$sPJZBT(ls)YHMBHjaR=yv9?o`6;{>Y+)YJ*@Va}$^_{> zC-ptQ!p(0QK+b~Vr2&Q@Q#^07Nt(@$I#)MaEM@^II#W!C727sl*BSHSxcpyLAUkFM zJL(JC8+s1z1lZAGft1cef3wq z)3L9&2L}hVc^F)}vWuknKGsYIYCbq=D#;tIFGpn92 z?fhdf5r5@;{*GP8V}-KS&I;*G>vL6|AN-@Z9_>q;T)(5n71G%}xxa9la?hOG<8Z|# z)0S^N@w4vqlOVe9J=BzYLXf7=CS2piVT$>f9@;lUf;uWL$~Io8%81~Ue>qW>@c2eriJ^J1sMe;blAosws|f!^t8K92dv>xeN7~Y zst24}osl8iTN*A-^YS0N)TmH@y4>8*G_c+tmXE{Kt-{WRQ#u?;UfJoDUxkgV^Ej^G>on8_nC!bO8dxpVtD+h3rx`)AmbvS#Ixi-aGNw zhr`?lmR%<*Jx16G;lNFY7d;l7!*Pnq4~y*Zb|V3*$eG#p6{pgSknyWz#?HAnlO4ny z65(^sC5{4oAzU)FyVGO5xvOC@<-AAD@|g>e9mJ?gi?D0i^`1+dK_cjs{*E^5Jh)qi zrP8Cm*9+_ikvcwQC(xAo>zY%|{O*)BPYr84zS8R6vOWx3X=(9VW~hwv<4E5p=ByYO zF%M}Kwu$qIq2{lH-u%iT>DIg66n;?_GTYZbaZQb2*GXK71PLlLQvEVrtc~W}2q-Tu zW%4oulGc@*G2->`b3d4xR|A6E2h(ky$J0sP6oWyyj|pW> zB$6GfaGf$;or4N88r;!(-!@TXJanw}8IPtGB~SQS($A?q2Z?;Lsb-LXHUA2dG4*CNB0iW z>~A)0)m&=c%b*5OrhG!u-U9 zeKYTEl|cj}fM+@TcofZfwOMZm#|OcBtEQts?v(S^%hyMMYz_h&FBJm0Xl>8t?M)Kp z<*tIueGTAY^%ydp_>hxbewJ_KvJI`F+uS8BCEi-0+cmQpQD>&_bl*)y3z#>f!BA zz~F1nHD`yjfreVA=WFhW0g9)lh6YW>*40&A*9BUS;SV-&t^GQTpNG$87EZnO$66C2 zj2u2&j9N^0+ecvohXy=;F}5S7^TP{}R%U~aR$YU*(BOkw93#4nho8`9iu^e#o`TKC z2FnN~reYcd1RVL85~ZB?;^Zn}n1m(Yx| zo}P;(cmmQ-AzPCi+8Zu8sig$1#ko~Zcb&kew9fl{dxxSmy6Ahd8R@#7Nd6x4=+elF zY?7+BW5Jw+wx4RytdIfv;D$&5-V? z7B%|=K&j-*TdAS!W89vV#+Rkl67?p0+2%Bk1X#RcG?ws#aL-$oV&7anl)_qb&xokd z*ltn7yas8@fA>ozZ49h*7t&|29dRz01aYT>Zu+vrp$B-T6S@Lh6Z$*~k|-pIbw`%zp{>-f9Ldj}+wP5$`g*-es+@@5W7OSA#c11$-%ustbgU{6nkrk)no+IQ zNmI5-A4t3^Em?D?I>i&0kXBu2Fka2vsch(Kzk0XZC5}{u9&oRLUY{eOcTlwt0*5@jqWMO?AE8v`Vt>3lA&5ch5I?{BG)X$Bp1qB0+`Vf&4k$sqW02fRPuM zM&2jYuX$k6<7Buo@0rQPI17j@gU_D=UK2))oj2&owujM}MYup=Xt2nzp=gIFRvioi zuFcR{M*|gjCaIo=$(7-D@UI!vDskn0J}aa8sW6PSXu+-ZE64KjvKAW(VbtLVctyWr z^$Ij$M+M@Uq&ASY>gfBf;VTAhx~8Y$f~B2;u(0?wl2>{~nQ&`tR))aF84AD~ELX~|28xh8Kw2$V=Wzi2c6srm7Q5gckiz^|e zG8pd+z->DnAmx8}P{il7xFI*)>p|-`TjbNJ)zV351(KaLq+IL3V2Er@d*QNu!V)6* zOeA(dUl=h-722V>vwbA-opY`Y@=jUrbn%bbnD$HTY%8PWL(4S3NT*ra4eGQ&aX}zy zkc`E6Fv;)utP7CX;EY?M2ONRH#?h-Lh{yPA&jQ|dGvmdRu2$p~%JKGDqDv6OlZerQq`L9qPIh_~e zxhB4=9-vM6Q3-ylX<<-UWaD$vPX+NJ&O0` zIcwy8=f1qHIMT=GNvcF|@R91%Aba67sn^~qTozYAN-2D4N?}>_0`E|f=W_8f0fwz^ zgOjd9^t>AMKaM2#Wth7#%#MG=<`RNEV!mPb*xiFB!^<~JEFI%s?$Lj( zh^;ZkB)L=ur||{93K(Mxj2XKW($v@B+#&TKv}_;Xkz!sZ9ogBRh8;9oZy}<%`0cbEz62Aqj*(y zQ0eTod*k;wKap`Uc1Br<&Ya?XX#g{B+Vu&{1|-3~8B%;VFU|8}(&?+P_q-*;B=DOVaBo(2eobT5Lrw6# z!fM?!za-^bTh#emb#zYcoJsmFY2rbJ-Ncy}5^XVkqVlu%5&H5`=iK!v=RXE|>`AbD zrHy#@eJy`_Po{yPoGa#12N|^Yunx1kEPrvyE99GrLL|5Ya+7SiQH%huCV!)4$AWKQ zBIoH*trhsh`HM@*FIhYkwn;q+A#S>&WiYrH#E2m8kwACP2v=T95bS%BWUaRgB^!z1_*F$!?zg7xXka4`oTU47~mG}*Y}okQzD`GJNn<=&$Dd|rU1 zHAq{$lT!C7Ctq_DNp_@%Dsmrp=|`XBjeFW_ zk_LEb`(V$s>p7+r!D^$pxp9HHwnC*IKD~o|&7xFq_CobTFIwXMAo_KJ$K)7lwonAE zt8B;HS8{a-XQ~r4OK@LIjS2<6W6N2>KigC+V>&nk*`i^&ShI7vNxV8}XdQBGM#l`i zq_dViv6E`*{W{3_qXz(htKD$BGdAsSY3Q0+ohedi1&6U!VpcnN_J?*+*N@biLK-ej zm0EelEqTlol-p7l1J7QbiSNkx=5UcXRJ$)#_CCwB5*~h~F~y#|WDfCk6?!@}KozNA zJZ&DdvL?^D;w?ZW;Pa!zKZGLMelg}B*X$nrb?=vmeT7*|4|>Jz^H_;P`DfwHRg&I~ zYHZp?PcC(?H6NqpR3yAtexvT5>=ox%HV@EkG0R)lGC(o#%R6K5ysx(=(8;E7Q8#uP zTirGotnrimQ;rM&_G$t%q6npym3H`>8s_R5#T6IhG1o%PZu9GNfA8~rWJpTh)FdgD`<_liuRi32 z_t03H0O2+Gv>1gQwLVLpld_oN0NU$)gE`ZzYTq&)&Ir_ij__{Ku1*hT0`djfLZ9w? zg2p4Gf`vW$Qi?eD@7SoWgfcQx-7aBf25dSHZw)w6+d<383J?n9T3He%IQugw{4K5fZ$vHpyx-=JprceB<<+OjKv` zj;m_t=QDRXI8N_gx!17~S!^RS9vLguvh&O3pN@Yi`rCF^Zn7!)!)L3Pkjn6Lw~|*Z zUwQxOz_I?0n7s%G-c@)m*Go&1CJ$55V4T&{%gANVm^<1Pd;|~XXf4M9Avq@8ZJ-l+ zs0hIkI+lfTuTf^L<-EPRKne+_R>kZFd=QfIt~x(c;re>ro84I+?1ys8m(w{O6}{dI z{s=l}!K`W%m}WZ#)BEzRi`ectC>_a?o-UEGjifVuc=&Xv4EsIAc{^C<7dA1Qf|eau zdw^@_=vxgD_?08PaJurWWoH7yJH=+qi=RM3s;lOILOrZG@7HbWU9*7_GLm+A7vNrM zCwcI3A|1*WqoLghxxRZmVX+kIa#JMj+4#w1`@+s@s{3$9N_6?z9a{ZoP-}?UXgk{T z-R3)rTXuPG)UA-!*Qc~S8!g`6ZGFE`WEU4wv+zR6J`bAI3)JaBvcwyA>^^f|v;>d$ zv(B2p0@x|-3*7l$AlWnvi1l*i>dex=#|8bx`v1}2Q|S{b+F_N>#GKue&Ta489IiAD zz3tUw7E5H*Z|MU(!IQ+Ux=`e}(<_bdADuQ1E3F;B62Ar^NjpIxXYAUj-2NA}=244_ zW;!Ox+G>8@9wTkAkSOkZi6}z)us78T+*qba1AN1HOZb(;kW0BV?OfNn_bTsYg^gWl zH*<5bxoIy8?LAWj_AQlt&(8O;{ggUjQ`=I>$d52T!RX|_vH*$u@|C#E+Y7xlGr?>w z)y)>!-gZdSQFrP76=yaZegrCGjUx3A`sZ}K_+GzKX;0yx>%h=AYU7s7VXU?P-6rl! zt& zc<;NmkJmPY=`UU<&pJ)tISx7h6)Wn`t?T=^ue
    ^yKafTsf!VoB;|}apT)xkWS**xt zO8#M{>1KW72zysu+Xynk34ZA>Bl*d@9RJcH$MG}4FF&w#U1WJz{ydTk{`P5}{B3aM z5ldF{Wt~sV7^<+;X11Vx?v}|WbXe{nXY0nf)v3WK`(HVZO??o~06EuWTygc9CkPFC zzvx&;1=*({dlaFS$la&&8tnB0$4Jn@y^T*!k%tFPE_>J9*H^ch!9hp)pQiRs1^X`E zBK^woQ}DM(wV#Ui+U~zt>Lo-yx$$E13MTen4*t3DpAY`I?CZn#`^Eo3J6HbM%>U8G zUu=`KbQmFUhvyn+nQK-`Jn;N5(?hd%#VN2@YL_x@Fr)n zxo2Axs8eliF;xHJVbphDstIxy2_KqY{ZnD%?Sn^O3VW+$f2o&rRP^jiy>k+uj(&K_ z^kbhzlE$Z8i{=KWcD0P9{gX}qZ0>KcUG|O2ZOtN0wvl7cVcFInLEK$rn>6Li?gBD< z&-dKLY|E+$I^om*pw?sZ6T3eU{Z~4QtoiSs$F$atq*zfe7vJ>n{>s6<*)S65?s96q zDf!Ym2jZ!)nQTk?6V8}bzR%kcDm7a55*T)_D{O?B3;qI5FH}gtOF~2w(>yoa?R^@E8$<3#n66x;^tFVL?JOn(p zZe7TOB@BY03@(R^W_ZdDhi1_?%n!@=N9(351rc9;OdzdWkzm7FV^@?b`sih4m!EmC z-X>tmyMn0Tao?@y)m=`}F3oSD3k6M^mVP_qmOAf3>QNDruRp6y8l7zDSLke0q$tDdh z3MWFd29Sh^yW!Vw22=)lc_*J@u1}<1{7`jE&EW@=G{T5mu>sB|ix{2u#NTekh|X59 zrq87`Acj7`tCtn`o%XkHo~f_`X=^CB7OsUaVW>WUy2X6)II}i+mM4Adtp`Kn zc9X{SQCIR|yq}2)1@v~HvJ!GiUD%o^;Q7EfD|?qkG;(A0gqrek?)WT&c*cCvDne#4 zYKXFRh8z)>`|U+7gd47GSzk-ZKN2o)yVh=5*L?*TWQ@2`b9&|0-SRum(jgj4Ms6y^nUhep2z1 zW9RpaH*?=k8SDQhg+b||3v469O3GP1NQe3Cw)~Z;D`4+Aciv&6v*`C?qoyXik{)dh zpKOCS;p!mLg^jw5mE*4b(wBqj4vVE~v^+Hi}m8kz# zP#H(>TmF^f{12hSV+Qli-V1=)t)OS)YePC6e5Fc;bt@q5DL>Kx=Q|A`XGHpAIAmNN zO@B$z{gopT4zvIVv(p!AD&6YSVtu!4a2@(Vl?IfzjPl3vt8gl~;j1<6rV<-m;=M78 zHU;N;frrD91Y;Cu$9EBBT3{$Sc~F$>St?t@gybkaRqZ}TJ^i_&@^cWNqA+H~#rKTm z{CH^*TJ8q5Y{C#taI3`+#S`szh!x(xzF!m_?-!Qs1(t?^DDO4U1Uw0>YYi)oqhoQ# zQxlk+R$(>Ez5ozVx(5__dr z%6l$66!LH!Nk~?>)Xr9DG1+|D_}(P%EejI#A|)3t zKm{8V+6oL)!HVGV87qgew429Eo5$|9g%~jm6Ub^lBjXYl;-RlPCaHZo+uPTowz$B& zs*<4q>^1ed-Tx3pnYZBZl zx0P^!w-O{*IW4%rq5eAjWt4X&!ZCMi?=CBx-R4i=&ouTyEqoo8i8jgf1ZNDgFmv<^ z)KRV@_{*q68SNuHYqg`n>$>^Trsd&q+`8bUfuJm>m}@~$U@W4{p$s?yoyQi3(fD0_ zLff}Z_kDt;9@ZIDPxuTe(0E(b?9VxCND$OZc~ObUcZ|i}M}$&^^Ms0L>j?NR3~Xla zBeMs)9~+btvvG*)wTLRj7DjR+cm?2@B@Xapmr%%l%gj1Rv%Gsipqo$??Ya71{o8@8hy0VGgdktrMOur6Dw|+J6=0@iF z#7w9zhgP~z{wS+Z5jHmaR-x{pon6SnvRwRIqaM4NA-3vU4;u|e(LD5vu7OQFRcj=2 z->{FQ?3MTDM&UlFTg=YO7OI>xN?Ng=!_gW9AN$O^YxXK(S|Gd}_J&5YVMU z9aM*}cF3^38k{A~tXF0tw8-_c>OYve+dA<^dKFng`yc_b;2xyuWx_54RO?W^a*KOf z77^GKa*5$Xu>TQA-Ib|L_CA{uHO+i7No0H31021F8k{rsF_lqo0JnyuXNLvl1zwGT zTE9)s@vj_Pr5ZaS=$6KJMqv8%H{pCgvyd_=s<`EhROL5&$@fN&h9v3@<4Dibe@-{v zxndgFr+4UgLk=*Lme`eUm<#R9{!I1$&&Y8p;(A{8JmMSkBU$rP#yvcV^Agn8A#`h_ zZIXT&NTqC_0Ax$%*>3w|Ek;$C%X{ZJ`rq5Lv=H(DYPFG@ovoo*AwH;5hARPOJzJfk zN<5LUGdc1IOfQoUQb{}6FmtBii@rF-xYiAE;SS9Gbzr@GYfWQ%PcB?H)^u7f=&FtTo8r-i`Kb*5R4kp^GBeQT*Rau-+Rtf=;Uu!H0UV#n3H zrlhgeaa=1c-`=ro($zK=y!Q~36W2cN0+OVXO#g(2ibk^^qZ2kJyKn7XM@SvTW*o*Q zEPKJ~o@NL?y$i3ApOlun6uNAs*k?Zx?I~$zBW%`^BoUsKwMDh4&In^+Ug#s1IL1{9 z%eoyNuYf?Hai6X8g%9o8)L&A&(v7ygpv5n~l74S<>)4LXyC-6>ru3yZoesX`Z(sZ3 zZ2Rl_Wi9l+=;ny`AmMKG@2kd>gGO;@Tdc>=KwwNgDID`eq!pS1r_yu{d7kycReugJ zF$Lt-0399lU+v-;Haq!{k=MFUY)^I&%Xp+EdR0p*WK+nQLrH=XaP>`9Bj4@Oxz3%U z(V(uNDcGo_K=3DYu&cO+X0Lo*<&YUdD0Un34F2(oO2hi%9vX!p5 zq?vWEa?62R>DKS<;}juQHy7)PsQiovh@cshkJgfDY$dRGtX&4{>!wTDz>!d7rQOQb zmQ~QG+*U=3u7Kzd-0rC2p;CaP0m1>3&NPm47P)0-XXBZ+rOd42@54}v6J<9|4+!oF zD(6wOrjc@sa?~%AE2x&v_UB_0)J^lVt1)`@(lC9Bs#rFve@n*^jFdl=W=!WiL)5_S zPqwLX3y$Ac&Jg%yXB4g#mN%duG-oDoBag2y$v?Zi#4C3Vv!Qc=QZSNFDOXukgc(_fU0vv&C>!*V#goZ7Rxo zro_cn$BTb^oW<$@U-(h$e3|?%BraTF$KSz0ric)j%RePEPWG>md<+J&pqQ*TvE`*l z-;Qbj%7J;!9**Ni!uf-O6NA`JS+@%<+bZx*qV|-zu2!0>_03ddnd^HlzQ=>5y+*F- zx!@k-9CWx+H6qcOpLojFy`YdjaW_d#qBK1747glj32!&7t_lnlsM(ZC*aMDBQ6gka zNAQL*nK0~%tMz3sf|!i}QI?DwT*$5^iqgSMxWIa#XW4cp7wvfs?f zd*x+-UAU*c>|Su~Q1j1%?|646p6d{ovFFT5-!&M%c9J44EkKaBUFcCS@2U+ECYJf~ zzTM$8F3Xg^6xeb;#-^WCZrofR7*613tlJ$rvzFAuD$~V1p7)O$Cae6)G1BzdPaJ`7 zBsC>fE5<~wGr*J#mHfV74fB&N9pRn$Ly2DZgMtz6vqL*-(6SR?PFcPsQ(YMZ)f-ik z7TDcBbx~cFXsxQn4AuJaT1YD8LEFRSkZZ6Z6F+3lJq?!NbYc028YO{>orLyj6rtQ; z4rnjPV2Ac+<}3A7gF&ytua+bRk7ie>W%;Hc68TL*=B6<(Cz1dmHltHY0t*ERId6tm z&ph@5@54&1bitNw549dq!q1Lvq{}Qr${p~d5CpMmzPKnw307i&XV}cFQQ7Hq^$A6h z`+C<|l(CWNE(R|rP`LN}#8n(^@CR<2PweYRR{+V$b&SW?D4W4lCknjf51(wfHLX^F zt@k|Fb?pYBwji-Fagq(z=T5%O405(*`wK8DNGtubKpkSqIau>b>|*iN)j+WFM@9Q* z$*`*_?MXrKRxSSy)iAW0%LTp{RKM?Oz*-{^NarUS_t$|9L)T zm>5vv3;o^H>EEc15CyX-{{vT>IZi6-bUZIfL51_Bkxlwcei8x}v%S!9Ls(c`JX%!f z#=nDJelc5r5Q?%G5hX{X$rQRKeME%k7~jVR6MQ0yXV!{(rq%%1tr0J0biXr3RyV#( z_knEn9WiwB6TOjpKZ6w^;vU3?(wjc8u7TG+dfYzP?wtKH{U zLp3Lf-M><7{fD)#Kl!fzzO}A@cXFp3;_kDEmqr&6sx=5X=BVWax5ZH7sB?;X8d+CY zIh%XrL%DGFWM)WHh)6HIx{Oz>Bq8UL%}0|A?Ho*{GJ82+t}XxJe}V1#zvE}Yw)zIK z5~j;(25hoEdyAVO>~P=6Zd67ZhT`wubmRovXO*wb6o*U0e-iPeqHqrPdnUVL?$^}| zreCd<$vFcDTV7?)@#;na?(J+o*%mk~@YG zqa;!;LW-+F82s*ur>-vNBEYJosH9S+N+9lIN3i^sqr>MqI6KzAIIK$Z_*aN)Ug0`r z^&1|vd#V;R_ajcrjh-P5kLTcB z!l5L8|23Q^J$xSD?mhG0fsA{udN&R+5|kBN#b`6>ozKru)xdM9mNt8+a@N}CrP%BH z6cgZ4M*0OF4W96}qDSpDe*xA7|Hi8u!HZO%7ySba&$%FMUNi9zFua76rC&DX*Zv5@ z1L%{p)i!Vb%MrDpf8pc-^hs){wp8*|Is1<0W25!DtTX~E%|%rx6Bb5^hgT=jqLB{^ z9<dKmm(8v*fa(R=Ni8B&j5xnjLjkj6(+x`Yi1=A zggJlOJm=QW5#O{Z(BTr=6>mcYKNhVE;~q7p z&Y8b+I&RN*G4BtBUk$Z17vEV?7U6kuBkCB_LsR-@z|o0yV3{9)N5k|~wA7myum0r& zIlZC(Z1^|Z|I-xz8}Ev*?7SkZN{1%Qf;BRpR(wlRnx>ws{6Dx3zVDH6t6Aggmgngoe?XQntx9tzGzeGy%gYaPI*V&4DM~YDjxKKX-UB>N z*CmA0#%_}fYnmXf;&qk-C5omZ<~s4VLZ}Wc5M=lo$#k4Kl2qLA?l1be7UMR zmfOYT&VjjKC9p&LxaZi}3xLfA@p0mYKi+t`_>SE+${K+#@>zlKL zI$fXF8)p<4AGt@%KC<&AA%r>-RngTuyrLXbFd2_ViXy+SxZBP7M3#@nm@GS}N{Hs4 zC2_Bmy-Hxs1mWnAU=^M95lxr7L7cyguYXOsmu>NbTO|qBV*tOJCDZr0Jg882QB^ov zY@POY!;Qc0)xw5%8Rb1F*eLfhHD#8~!lza%+HwauY|s8^0(^Pa;SuRgGoyl}HY!GNF2gdlD z;u?q5L9_X8yMlo+31Lg99(x_s&k3uX$xbYxWx8OYQ05Uu(2=)k{hkTXyL`a))!PBe zNkWt{JBxejG}&t-a~S8&2;Hn?dQYz>b>&B5#+g@&(2`VXs5#b9>E7 zQY5VPM(ytqicy_bJ+|p)CS$#WzPy8X#ytqq0Dd&6R}h<$cqmC%wAz2RQLPpCK}f=6 z$(oBPG}L`_j^0y9$V>A4PG2653R`1=9hkx7h?-{3lU{|~HJANbl4R`VRAqo((qyhS znF2%i4+{B(gUbwmYl5!>GnArId|#fi7IH2jH|w(fn6}ng52g(s{jKoD)=d4zd|o@c zPY-8d4M!es)b+zR{S%T##0pYzNzGteV~03iOoVK`XgZlMTfRn^CFTZ*j?}9osx`uC zr8dk~naG94g@uxqcR9vX5HWWt%zphJ+in<2W?DGOCbH9t(C&2ZcRDKCiob9?{D*}r z9b>2cxE7cblfSko_y}qP``f?*&iAj#_a~x1uE|1zr0f7|p`!1OLz%Ft00DfX&jj6ex`-_l*MY$unT};Vcn_jGk~w{kAx?KxB#Ju7rpYK@ zUAd?$7SM6#fUmN)UUy9z>liNevZa0l@`G=sqr!1E%;kSYINBt!<+2isG-e_n^2?|Z!x|$S#^6!gBtJ`H_ zYr+6H+iOvN`4bw_*GctPi()3isw_O6du@#kl45e8gF&%c(0II(j+M#oT%E4L&Uc|A zV`V@Q@#WCk$&zhjps>{i!?PQ+jIJkoi9-pA7(j3u4`!-@Ae_0ZeR7gKNZqcW+_7lf z?JFzw7n--QV3X2UC(^@r38(dm$4ki#7WT`$e3_EEl}#p#l|~cuW%+?_FSr=XyTW}d zVrQEoEBmc2ElhSP$|M)N-3150j2^us-2>k>X(<7ya$*E8j9 z#faND!*59ed%@#`?09sRWP1?RyaHfQj2;asEGf*7|Hy!PW(oOgi0iU19euWDpWDyN zHCg}EGiRTUv+B_X3Jl2cz!|zkgVK0AXL>+Uk}F*ee)M;8dA2GkFylgW zpt)}X*A^ZTfWJG&X}bu3;n*r{X(4XO1Bxgm1W7`$%!8@v!HmojXC{&Bog-iT~6 zP%4{IOYl{Lc&eCqJ)Uj8ZO>OkxTZ!#r-SdamwkLE8Dk;a!+Qz^X@5$;8(@>)rl@*!kB&-V6CDts*ECt+NJY~&} z!%u+oOgvMwG-gQff_bM$jjd-8TAkI&kBTyY9YTpj>s-_Q2=$AFj;}b~Jp}d2mfv({ z$gqsDsMS6m1y!qZjSOS^BZigE8vYkH*&yqzspN`riY$RS^lZerKTy0GPe3`C7dnc_5qKgV8!}M;p$u#ydX`s!5qH(G7I|S zCM=@Z60rAUmxOmmqY)9yU#m*+F@ID~2#$?L)UPhqnsT024*?i}29Ihg2?tDLE*FE^ zNut7uMx)M!3C*>&QZY~_cWsXD1%#kt{=)_#`|k)3Z#);Fz!Tiqxy|`(R%{LZ&Wu2n z*11Z-(RVcri=lfO_6yFL8!~vT!Ayq`KG!3=K>;o##*_{BHP+$T1pwHJ!Io{%`g|iQ z*a9Lk!!n)Nl1Nr$N5?ht30d4Po$h&<3kW9kD3|u}gbxRW23`x9J^WeT`t(&Vp-=Ed zX@@*2kLCIk#-y1JOAZwvSYR(ztM!s*ml!Dl?O_NDZ40;|163tH>%EaBnf{{fL+@)7 zZEK|1g~f=C;RR~hea6})<(Ix-LCM-)3GMfGl3iH^i1TOT6;-t+i)2%SZDTG`Umb8I z!te$0jDQXSh0WH4y$qmk2UzI`7Xt-9%E;e@G`zjg4S6;J`kSeQiC3!r7}h5He>V32y)?GwCE<0BuG_!e1^?e`bP4~IrCmLJ)FjK5=$CWkmDXrLR)RRn zGcMQY1Ty6G4aG8$z$ZFA5OKY2PKR_yW~Au(tdgl`PG!@FiK*uX5Td9wa@J3p!)N1!^aPqQ-QX_CL{~~0ky6JW^8b|#e?bh64rGqR$_*cy0 zF)TEaJZ<$F@mN^+oa#&HOPCD}vt}rNIX#hPT3rzLR89pIAzwI-8fXB{0KBI>Lwvoe zj54_2%VZ#yfo9=%N!}NA*qMp!BuaxJ0#=<@NvBKM(OJg*Hv|6{UjLi&^oyA<7@KI7 zqU{KrC(+Kc!-0B>$Ws5ne&MmZ|3hmOq*JYTabNdR_60w7!6c_ zDt%_*`TEr5L#M0%tnmM)m7Lr12b2{>f8zKDQq&_euA5qQb8>2SnQCU}!u&U|RjGDzRzgjg<9jo=nBb7Jn!qb2Gv{Zw2J)Fd*5~A2dL%D=smr zt!g;axaN!B?ipBO^FK}=z!xv(e*d!D^h8_Pr}}USng$qL4=A<<1gp0%#HWR*jQ zL6ERgRY|?rkqvOgnFk!h(X!S8OzFjf04k7r&U-CIQCA;^Z_jky%N;E{e6}l90bS{- z4tgWd*#@pWc=7PvN4Tn)T~6*S?r`c@amUE#(58&)#cWO#{5q9Zt($Hie915$)zw*| zi3-R)1ux~b9XKX}v)%3)|H+q4K+p0K(V(_UNusX9lx z=Mfq+6PX0bzQ%?IuA8YAH>og(m(a0^>m+wew;A0otx=o@+Y1r@JcY4LLnp*i%C9u} zdR;bqCLi>CKwO`=Fl4R;MT|U5^j@onRQKL*vS6kMlVsp_M_i)4gZUKawJzC?9mLD9 z$Sjw{nD2{fdmYoBLG29S&TsS$$SyBr3I^%K>=tMR&W;}JyZxg{at>`<{(9bW6h_lj zOK8}T%%zyOGu|%|3$(^emR>XG()fhjvfdL_^bw!&^E;ovGe9XK>6 zNxDhpz%_Qsd)Oc(OrUDZU=W(7F+bzDiNyl0-T3|k=W=k%;7-xKPl&=H-59W%a)BLm z#sHFrp>DcjV3|^o5ML3mM+EnbGBKg%IrXilP`l3HTd!@U85TEzz3;(Vb0?V#F3W+H z{j`%3nj^vXANn)ZoxY+kHk_tIKFCf4DIhXnf&8Z#cWBjyHmw_#ZB1HGw2Hn~M^>Or zTJ1u5c%`IXi$|ja;T6wwchq{VBXcZD;N2$USr~AkH~6OQfwNm18(Bf0hz@4QG3a#{ z$9HG@upvtD+}H3%ix1>B4b%8}3{jS+C*Glb)KaoqHNs2}|3+j4Ay)(gK0i&H|FjsE z7M@N~vy=jY=XVG|=&G*H+M0&YnV6!H)l;ReOBYQeD(yc6Y;;MeMB|E*{vY-I@39yYDA|gmuoYXPv#*UT5#MzDq(YSXM~SKNxxndTQo+^&DnEU%xvq zf7mw(UlQoPGd_DC^9}`^lf!1ZKQ*kQw@FB9?4KO5S><<6jm$1 z!~h@7OM_BNL0PqZUI^Z&C;{P#dGi2mN&-&IT2;MT-(&Xxv1;O_p(rKllV5ezGIR=> zy>SX!{BFq7iuLY}^o^}Rv?f_F?d4(sRexrvQZ#7r{wF?NscPuX<=rMSF?HReOoLJZ zaN`V)cLoCJjIDG0#sxtuXFc=u@`9BaiWx;L=h3Vz;e0QsA{9b)^4->ry?aL zzVRN6zGm=2y=Ks%tw!mU6p>_fTCoOw3{m5hzCvu)&^!frs!4I5X z1rxL}+vScgCytn}E=)SdZZjp!Y;-VAPYr1g%(RPSJegpzx>({z z^!A%PUWYDouQy?j--@rLFG`}5G%`o>%-e(O*rWV>89`GR3}9T`Q0dcy?^(HMhrWM;1e8 z9BSGN#~%TXVbn%}v$85u3tM4B&~kdcDI=laCMZ=JoQrkij`2q?+X9tM2Vb8#l*%>- zza+YvAI&bP)?$NHHA_tNLoNiN*pO5@RZj1TQ{% z?;i9P%k&|vdPDPZ_lV^{ju)%FVW7SzIT4kghiK15D2T9ksshKcaMl=_l=@E&Wkx|i@V!&-pkGLq>Z)t;=ylJbW7dgJ zQ(=chpMoy;Ytx&e`t3Z`g(|PtMH=#|dh@o;4#cyh2Wu@BbW&6{CK(;g;nkC|qt;8k zd?jiSVPQPss37b%24#XZAwD>&+yJWsI0=Y;HJFnnsWh+i#m-`N&{?@=-#h z3>sDAR;g0Gt#;+xB_3fl6mvWh;z# zKhUnhr5}dh73jra(34t8NFy@~LI+=q@}uUmUe^!9ed*bj*yw(GQ-z{lVR^kNy)CoS zs4

    bQ>qGKwa;yNFqcmub7>_HnTW|3T7wQ$+!4<*v-+ZMI!Xw0$`KB z=$9I)Yim58z)B~-pxU^Eno!E0t1ue zHfewNDd-$crq5?}pWJ#{aj5K_NRmbm*hkvwEol+W}$@%Ce({p%zAXkpxh`ospGA<*nFN%deD>iTrdV|TP5NR6- zUUd@1k|v>ArF-gy+BB3>yp(6{q~J`44kd0Yy*r_Xh067!DnHnp8>MUvKWE8QZ#v&WS{m8f z@=Ucl$JWHYOg(;wUQMbChf_y5ipMR9Wa@u#rdb;1K1=0f>P!)C}LZ=J~$m*#fZ9P%Xb? z0%s)|2inGtE&85hFhh8d>2F>Hg*WaBQTEJ^jG$4w5%>5Xd)9~(_=C;sw)=2T(%Ts<8E|{-o;R(h+sa9T%TFsuz}~U{*L&6c?9IMfm$<_xGaL zgDiEG21K2TTS=|4>f_CBG&H=N+H2dTn(nDvQXMgHW+tn2cbR7X&NH0ONJjXX5% zMbc;y&Z5AkqNG24f6nwp_@@i*hV-MjylqYo08bm?GUZlb(8ZtUYqK_SxE$HdPYo}U zU&c4j5Y(%u6=O31#3P}DPLXEX``^U7b1+PZ7y3YMH`g-e?tmzj za&<7SD9!?N2Z1NN3(@Pj6%%Q>#>&d-DdZB|8u)LCDE}ACfJo*kws9bqsOC{jzs$}*>;L~15V{}vVL7OqSmUC2#ZWR3qc&`T){w!nF0)Z>wgm=-ueOm!xp-8< zm{o#uQ>v)On+F-Yz>$GjD8m%EBb0(4hZ%uDq*b2hiaW#)hwz&gqKY1CbQ`}*stFF4 z8KNH>SvZFrZAw)rjSURMWn^2F2jNHtZTs?rsBM@pPf%WlreK>~{N(5JRi;pQWT@3N zPW3htZa0+N-p9|QXvs+A+Kej_(2jYym^5#!twD0lr-_#kZxMtsSZ;~(pzrB5b-rjb zeNW-zc78JTWxBF`DSe$1#f|NOV+}Xc1P_k->}w&ttY~DurzU&0^?R?350p#0tfgQX z)3uCLX_^;xd?GZ}S#e~y3sDgg=k=!TQ0bOIecL{`uI|1Z*OQ{jA6SM6rWJ*AO(xcw z_`D4>Ot=ZsJb`IS(lfn(8*W@iHuLHlRM$qMnJ-yS?6O2qTew^>d}<3e<>97N)s}i< znV)WPcor^MsGr&0j?>tgHifvb*5^b$Ji5h&Ahxs4Yq1{;ZfAqR&_U;W_`R=2sWphd zGpN#}ZGZj2*egUsZ(Vm?gS3`s-17muJlUhc%a4d3sWT|q8l1?*?4|;SFdHXr&hq3J zGjYS&n(}!h^teR+j7haxn2pLD7yw`#iaXhW)0yb5la*03zw`t+AAIS;zEOI1%BQ{&49U}jed_%z zAcoVJ6MRj%jjVhyUoQfoBHO?6P0`XQk$S(a|E@G;X1EgT%zm<=-mdBBPypWFU!F!| zklSjk0%@=~2`|c)bIsy6z?nwi!RlGOnx5f1E%(Bo*d}N#HRDb2Tg%!TDto=|Xwg{~ zjIHF=p}e9_d+85%&7E(5i0jo^NHJPt-Jx4k&DbzUv9dafZa;KoQyv&5rz(A-b3hH% ziCmOc$Ooe}Ewx7DZ6JD4-WIWCWy71*X~o9DNL!|?cVvZq(|6Y@wz`-E0sd_QHEQ^PF8L4JX>w5XP>=!#3KUfzGWy}$7MO9}YZ*ev+k@Pn| zLA+Qrl@q)n?XuxtinT{hFJcceH(JDZbVEfA8ZgK0k@0mN4co=kca&!^9Ls4kkLYzO zQ$iyEd~k*q(R~3r(kDYZ7jJxCa=Bde4C3BprFcZ6*_;d*%HM{=J+kGApvh2+Yxd-ORj0 zWdj?hYt|m-+lgEDmET7)1g0$eGWw;U!E7tyWy}7*V`W0zUdO%`dPAK0Z84; zMJkDlEokEe1ujEO=&WRy$ln{41qJ%y(k~V{U91sZHj@4;7Qq=M_&ll@(gR|cb%YDE zPkF3Ak}|XGOr=%^8`|DkAfqPQyHZx%+J!>il}4|ev_RSpi0RBQt+o{Rill*p5qL$G zSjge>rs1>}StZs>odDm99lf($UtBPB-nPH;W?KH>wUt1yHUkRUwkcPMTWP7nk)p4C z*H6^m{}}Y(%1>}t{i-Lesr!ac0Y{!Q29rxd;#}A(=RHDmr}irGXYYZY{AGzb<6G9} zaO7hZd3k5`X~X()h8FSjza;)K;*U8Q4L>LPIUd8=*^>^>ryd{00;nEVjkjd!hi^-t z=S5@YOtoeL^fk{yb*4s;*mji{knOlAO^80f`486COkPU*N{Qi?sfd9%`r#{N?a);K z;x)2JtN+tR1*b%9=urw;G}z9m#aFigUsBXwz(uYQ&tJb*Dg>T~D(=6z$KP$|u;XT% zTIW>NQ+7%FUPWc)m1hM)czm|j#Itl8BEN*_{8c(P}0mhf)Ud{->b_Tg}l zLl$cE#a)I(MHWLcVg-4fUYTvQW(K&qR1Yvs$eLbFP%u_*T8^|?nDs6;vayEcYxE5& zAM9C^EoqfFV?Ic2i)Xb1p3FT~dZ@DEyRp3IYw!=o4<9V`_Eomti@#m#a%A`DeA##E zDf;MK^)}J%$JuUjZ}S3o3gD?}bWE0!ZZXdFhu`yi^`A9KFC8ONk@rVi#=c26k=sjs zq1m$*Cf8GeoWikUufg>Sn&j2Xi$8$=N$i$i2gv`__tBY}k-olJp~fC8FTj8Q*BvFJ z1u?EBGMHaW%@2(uskL=J_l4qo84|P7{U{@z_hEdMayV0E&hZ5T=^w&lxxsX@UFE%<3Ck6#}BglMA zQh|WySAXBgPu0@=->!AY=begKM_86q!VQ@~9NVzGO1NitM*L&}an86>u6E?4ALpA) z8;{gfcZ>M*GoIEJ!cjml?KBnn1C6*3V~5Cn_19M^`(~QE@_ExVxu_M3@12=0yAL`e zv%73E%Dj5V^_Ng5S6iIJ3%?qf{2Lg`{P{*Eze=Bfni_9lCOLcYT1i+f?CkdCrezZe zwpA9_B$J(dTEyeJKI^m-l@75vL&fB{d)rZ|*OFcxl1j zYBz_uo{D!5)MF4Vmipn2j5e1sn{@F_c}S#c-q96yW!VPCK;Je;H!||Lz(8lJhaV&3 z(#EWJu2(%=2ohTe(VBkAavCT-M27OlgRHgfe#7<)h3)S)>O=2&}b z(Ft!xWPA@{!$D$i1coCfM0E?r5+_{qBJt~X7Y2i? zBRNSTBQKaH#-D>oYplSIIfm-#zKPB=ujC!%sur{I7{W2 z*DE6;fqt(-TOc5sE;NQ)#aCDl)xJS)7quXB+e>R)R$r-!(WX+qtx^E~ktH77N; zeP@%9|B4*?#4Bc^B5T={g~f8w(uPkJuljTkRVz1P5+YL+rDoZoz_~S&UK-{^95Nva zNqC0}DcUZOS&FoQzMOH=M?}e4ljG=-&CNL18!|=GT3aS%owr&Rq9jz%eYXdl*@}jf zjP*kaV=c!KH26A0LrVak|E0S^Bzm#H6JW_3lVR{Qt^-uZ=+a>#O|fXPg9K?5m*qP)~ZeZ%uuafxsWdc_aGS_3Y) zi@CIs=F7BwvD}mw%vc9=B$jJd_Nua4ciwOx__RJhyT#n``)9xSx#Fc(WOiHsMeE`R z^x>%0N9szdlw4>y4}~HDGAkXxV}PAipG_JpyIeKS`^sm()4exOa_}F|9Q7SDfIwdlDt-hF_&u-xxvM{hxFlprbjLlbN&kJiYEX4h7QpXsFhi^w9=ywdebeU zi1a{z()&cftWm5PBJ2Y*sgKGxD}r`}nu;&2m-cf7+=wL4!#h+W^biL<5F<83+gO+@ z-$?or77i24^^)b4twBnt)}U{)?Iz9=E!qU4DkAcYYrJRHE%qFlubRbKS43!%Z5r3V zSn8Y1>tW;PsJW?oY4Qq8yq4m13(;0jp%qWSQWs8ap1#;52e6u(3d>%x-S)U@|IX!E zb^h|p&#lhWa$Wd$m`@D-KHzHW4$V5uw$kK5yKmHM5}^i(3Q~!hzy+pkqip(Zl%M^a z+g_>>y~`rip4}i<)S0;(x3Nmnt`aHFMWawLRb3h7u(W5kcI2-^NuPOk3B+kiIbzc4q&&tYv z7=;KINhQ087{8C;=8-;pp6fr9{b&z#&Mh))5_iDsWJ zkWaJZkVGYD`QfLZ_isS0pdU?FeyZXrY=_#1b|B?t#n1aUXP^9(@P~iT)PyKgUm&4( zp!XlTxIh*$(CDo>qP8yci*h{prG-mrQ*2sf*Gq93WxWp*J*Uy?e8M`FNEd%^Srt$4qKT(FklY-|Nh+u{lvA~Bq_msG8p@jcSH3;k z!OShKQ*_c#b96C#EcO2SRvMu_|+&cxO zN@N0B`NfVh8+tbwbzP)x!9b4bc^fi?o{bD1nKFj`&}0vBdjur}|cv_zty}av6n7?iwA1Or9 zGkZ*77BJGdR(MP$w|q+n7IZC${8Dr*FW3QG>dC}Jb4<(2wLH<(h6+Ff=!^%#u!go5 znzpmoG&_qbY@AV-P2)3`4q%jmQG8vk*gRhS#TFp8wEM1+IvsXQxlu!Rkp_BR45=2M zck5GTuniPt(v6yjN=dhXTg25R2L#N>+3QQpJOwO^b=GYCK4dfpTJs6h@@hJwW_QEX z8T9WWH&`<`Wr7Oy3rBXeU#~mGcR#XAPp#eZl*!u;tLO+uF~A%!26*XaO~8^A4x62` z6c)hq9nC&q)9;`P41AsL{7r`}`RIhL zLLO6izgn}tUO6)w+LH57e%;Jv_l<5|GclUb*^)n`!p|A9GIJ1=8Own1WD`~-Qq@sSQ**v z8Izxa<~gd!`nmMO#3FC-z3m#_vD7Ax%PA4ozJ|4=qF#m6837@cXsLiuqvloYk(F(1 zJ3w?0@Voz$~WPrr0Sk<$3&cFodV(P{~_7 zaA_e{o(%{=1;KlmS*0S3V{m*X{{8`x+DzIeuiuY&aIZ5ef(x{AE6J+`N`bZ5kiBOr zB;TeNybXg+8$k7Xvq*~Uy0%&c?dSF#4pCu#Y}Diaam7A|d9s(Pl(O|GH56NCj7~VL z2d@d-CL6!Lt4n|D>NIU7y((o$l5~^ELeU<=XTUQ}pM#Oh1S%Cp!|@@MOo$-LoF&(l zLzqqQU~eBK-Fa+G^r%pmTu_cfBU9Qk=%uMrDMAZzDLu>J7WJ%kTSnF@Y^udU*M=rc zU>K`w$->32d%gm{eMJqcks({h*P)TYSu8AT7PC2}G;99gGhYSmZDLPu^{CeuILeId zM}`MsL)r&2%4XC$1x4J-cl#@W<7)2~y$AV&xA6l*PA|AzTHY0M3tv944jT7er^8*# zvJxV7#G3b1n4@;&CQEf!YV5(N(dVnK22Dzj`?H|E+PZ9vX<7*jH>$EwGM=;vqm(FcGj&2kg(Nta!u2fBF|~%d0FG154YL5K=7D|B~<` z`rfjW1a7}sK01^8OVS_x|Bq~J zUnEidj207?5UZ>4wYp2g9yx>avZS>d9E+Q%4-{lLo3yO!d1L=Ia3ihNCH6YWx1hO0 zjnVVsPX~(jST<>9&%JtiUs#+_{7EI+Iji!rXM@|P@5kPfgko*i)Mscgb}tjSN9^Uk zR`iue{~N{;k7-_dW#LK*!?)MJzwtgr#wDn;Sb;i8mE`^(oom0AU%%aExW{YEb)s@o zc4ptqJAKEr>yd6B;mxFP#%>ZzGDPJ(_A#zyogS*vJh_vz1UO^N?}wd&4ik39JvDny zL9IZvc$K3@ki>n@yt$1or2V2*GY#(F3mHhFCnV)WeAQe{n`vc zV^~X5n2!i5;nvi#f%W`=^ah|i*_636@4uMQ%YEL*{aB}dyLH^iS zJ^)|x&-we`ZF=(OJmY^~Ev)*>d@hSn)1qkYHon15g&f&JJ7m|v9f}yij$lX z@19?&&#o(jeH_E&q-Q;9XUqzwi_D_$q`o$bYPimMgn+j}UFMaV!d9MdZjCNe;Q5Zw zjcpB-1(kHH!*4>6La4MsZRam9-L)017F7IGa?&YKNvhoLf)Ia`y^O0@s$Ncs`{$2s)eEd@G!H>W9 z*}tj!vCze@MgQH|kif z0CZGgNiR1b*G*ht*;J!^|DUS$Zxs6Ft>ylFN^qYSbm7-*H+6r0YdJrk5&%E*5Vv=!YwQg1qhHCQTTaJ6002qAPo8=N2k8 znj+GSoNkKLOOR7rUi=1M9q#R@lhjduFo&w*jSZTT5V<|nydjCSWjS~-+Q;8g@ubRpLJVix0=LdHj(5j*R~DaFDFEJkBYY>(zoC))n7Y!}6Zb}9@k#OKSqUL>EY|bBO0|uUHTMfUG z9c@6WVDaO;1B2nRF^W{%x=3?oVLkLQZ~U)qCm@RsZ06~|#1eLZeX#v6cb#eE-sOx$ zqd|Vl=D7T{;n@V#kZyV!nZQ0&w(0PEiiDC!*%Pj=7J*%aM5@TIq_+joT3xAc(pFVj z(KpBJfT^lFz^T8G<}&~z0Y+&ZbFC~tuX@BjIsyTh{TC9s50w7GYl%mPzt;LU8VKz= zS?Rm7mEU$etui0-E17@-Y6tpY{BO_!rSd0aP^r2|Q^>RwPJLyw-J^7jtE)_6H&`Oo z+f=>V2hp+NT|EkLgCHRb(+?NnF0l8?ZE$sd{X>WBt#Y-79QJ}1NI<-HT03w$?F4H8 z5ay$PsH#c_zU9)UE}GQpNbn!*bpS`nq7w2mvzCrgA@=GkmB{1RmhLS@N*VU1fI1IW zQ6}RP+KN-qN><+~=p=kCYImFV6!grewAlC*bnvL!5Qr>&+`T3#*JZzRq?er9q()t%$ZpZ!#ssKmM@-NGj&tyj?T z&L}u|LX_D$d{XlUsajpsr2{u7wriT~1hQ-J4o=A^tXWS{YD3(Rq0(WMIo5ldU^7~? zUj>sG*3CNw9W#z|tu9EM%>8vI(#HJ4k|&n;a0a1Eekw6ycMeBS7KTHCFI=u~eS82e z?aCfvoBTX$cb^&;I$VBS037B_CkB-}TYq0Z_2?_btxv z-AT5jsiI&xyc03gA?FPQ+Wj4)I{&IUl~2U+nTrkIqdWy25zJ54 zUv*1>!{dlzRPgL+^JE$$xg30K%(LJO#j@7yfR;^gbEV|WapjI&0hOB4lbUQF>x2a> zHn_%G)@q)A*;e)v0p1b8j`MyNLLEc|lgbaj7bZkRY4(GL(v;A*f7)j<=wG%C0y;}d zaR>Dw^zJpGWUDSGNiqvEf*)&jb>ZZFvMu6s2N#}y|Cx_;juQ0hfx%eE#*hBD2bXur zeZrd7z0%LQUwG+t_WsigFWx8I|9oE8L!xo=%jcqnU@`Aeh(zeJTK)3@; zcVoi*$1gvUjt9z2J_x*LLM{X9$MU8?`HlES=Se)a{}jY?J%{twk)!`~^@hN!#qPz; z77Wjbh?)2bhx|Z0>1NgQS0N6zqMUBbSCTje3f9~=R>@zd%u!zD`?%CFTxWG?UhVB_ zPr5G7o8;TUIEI}q(>&1E-fgZ3C2FX^lzV%umzS71h>JyCX9-&UZ|6DUWP$B7^f6ZcyZp{sda~35#Nmq-A!L1M&wqq zwFXezi&6XUh&f7iFt!P-wuzs0`(%3x+TpLQ%6)K>mDb!VkY8X%^&~q9;*^^7U43D; zU6#wfj8%GhzmQ*&#X;V#f`@4;WlM@F2Nj>d5AoVMjaXKfSWKk$#w{H?jg~FW$bS3e zlG>pHFNS3VdL0W{KhA!71v`=UjsA!HRF;Cl4^T`#LwYZOHVbzLdpYY3?h7FlO2}<) zEPv=BJnq)t@o+OOg`f)|-9@NdbW+e~ZHfq9*eo-h`V|h@!YchWdwXT6yA zRJIC@@9U^8m1qieK|7Ql=gX~VW^kj??!D|MJW^NP%M5LuFZDCFPs_rLdznhJoohq( z$eMjK6i^<%BCIUtfvFc$Z6Ya>2BsDuLuqY2yDfo5xDd>WD^C(t>8M8GjnA23_IzRB z4yobzf8IF=GlFTr*;o$cqLj`#Z~%rKU2Pb>8v8a)s}N&ehMPoCJS2?d#BX0 z30$B~LEExFSc-c*N;v4NkVx8DU7GTlcQZVy8fJ%!v6;BaUYXQRF4il15@n}J?@9sA zU9$)nZPL`Rs+4ToivQ5JFnS8Q7jRO)oKsisADFf<77}#2-U-yokNXpKN6n3%Y2zw6sd`q z!;jLXT}a1Fvh*EBZ+lP609y&6r1=)3F1d>{K4d=REri3U%{uX3`w)sx>awt+otmP` zL|OZG>knfDimGtV%udm?7!&s>6Lp`XYL)$1L+enkE~m`}UM4yLv8rV2$5W zHN_ayTo)o!cgLd3_@?zj3Z57p#fusY~iqMZ98# z`rVKR&g|)}b$;~cl$%F+14WHond@Eoi_Zb^U}Lm|_ISu4D6EP#h3 z0_V{=2r`ORA6Zu{0lSo5T=}i5D1w5)h}C2$ic@*VoS*#-x5&bXhGvgro@w2ao>*>2 zRf&osAk1{Fp7e71pbBsGo^asq%RKOe8?&iWUS1}9aGQ*XTDCjj&fq4}>{krSAL1f$ zw7AP^9XqP`?sKWM%4-Jo?%Y_uwWUo<)y^v4WaCL2uC?e;XT*L!zi^{*uFd4PU5_U$ z>@n6X;n|j|q2sK#v#fG5jLeX1`Gx^P-?f^`uyaeK9d#E3EeIinaM5&}=G0OE@`qfV z2pyBxZBmZPRLXW1M!r0Pm5e<}utzd!~g{Y7vR+9lrW}BHhNy_~$TbT_< zrn5a7shz{A7ie>+QZ>lQPb~&j@q9TSkZv+IotYZkiCa=6Wb1VbBqn)-4d6QjZsbWXnsD`?ZdPRPE>w%8Nl+ zQB+l$5xSYhS88GhE?-8?3iLAaafYdnH{HmA%7bsL`Xy3);uR1U(#nE+-c2j-k)(NL zep5O8HL0YpuC^k=QQ?ixw~x22pMVDvq?De+1{WUc{QK!Q&V&F;Rp~O5}Ux{ zyjXYqWwi!Ydq+_`<&UfPcNUc2ZV`hXytiQf%y;Ie#B*Ph{>9u#A`1JVltlIYlVApG zWz|%<=Wsla;1rM4*|-1Iqm;j!|6g{v0|Jqdk=(x-%X%xr%psYGFV=E_w@F6f0;RiO ze3Gc&{ikW^z@0`D;6`9CV3{4(R2H-Ra7NlX@F%tPtjgnVy!_k7d!auQmpIx0NBB?| ziGh?jF)4n)8<#$G@(@z+*48inLt1*rowqkF7g(f5H%fy?uJ$?Kx-p@$Q5qVtb1u>y zXfjdJ@4>sDiTko&+Gc1pIp^*kY;LK=TzdOk-1v&nz=?2J+O4q?eM%GSV_0&Bm=HXT z1#7}{wvOWI&5Y8%7HErQ@2zEy;5#~23K!Hf>H{JXa4|>42#)=j+w^@Y`IGs$dt!;O zZdJDEsW{{DVRSw{pprA?Fj1Gko{>?XKa=5)%XV|jfWvh|pQsm` zIF%6ZA-cNc)xx{*mA<&{hH8c&O6+aargI8DBfczxp>c=Z@8OhdobxBH-7D~?{SMnk zWA*dawiuZ@B)wH)vM0ZLVXAzjTdG_@m)CoVA z%G`0wp#bfHh6=D!>^`CneZ1rdklX@WLf$>`M$Uph-)W9}WD2m^EUIXdyBmhHngF<_ zi7SvMWui>gSJXxc7jx3)`U8$Oz1_O7?c~x4tAZajx3cpWv{L%AM5L$+@_K^NnCn!3KB#z zyN7UL8=|J1U0;2tX%tkr%l2+aOs^Rg7fwe<2lybw`ujUIT!7te2yu`N>ix`Tqr({# z6&3=suazn*Ck_BSAGwu?X9L&UIdb+%NM8}qA$HkK@a_H$m*6)3;wLFNUe%#zI;9;P zdm!oY=x&p3y={|BrTx|6u$k{m93932L-))j<_E+hLRZXchc#NYlFSaG1N`GQD+;@9 zdG<-bFlNs-RF!CKgTUP$(fqLuWtr(CHznUPXx<>Ly1wbB!kt6>GMoc~;6#LBaC(gd z9+e=j;H%(ocpVyogc6iG9j(>~(l$u{=-^In(rerYk6@oX?!#^+mp>lju&vNt@gL#7 zGW(1Z3P3^J_Q#AKUfGFDc`ROcKI*&i8G~hfjw70k)$m3x*e3}skl{IcTh8LX3`{Vv zs6ad}$!wx7l>8PhIwl}`@R};OdcL)3ot~Y|-YB)VGk;)Y_#kmS;bEPv7&RM)(LhtE zpDE?Y4ZoDZA13Ks&8J~qPaQI@{y=%|?qUpNFOJ+Y0;&_Rkl$2>fa?X{Bwd-n#f(&p zCM`VqbkpiyTb*Lhopa;x(uNcGBb+<~u(#AEsy5*Rd3W3j2crD2;SJjChf$Ud` zzFp+56g*n8CF5W$q7uvS4IFS6)YMQY&vwjLdbGER&O9Sjk_t9cq*dwM*!13^7J-?2 z;uTk;gVK?c-!vM$3hwXk*8CK$xr2Eu-_BBmTL6fJToVTNJ@dV~0 zQHf^S4j;%VNFw2++;AoGXHhN7+xi<}{U48PLoUCraap768Vg73?UhCVo8I&mf%bnn+%s(*G{IPc+-`{M@Ig^#}`{%O?T)|bv+ z`MFK9<&l~%T1PGw;H>SCjcN*W)ALW)W5cW4XM{*=;Do5-#==|4_9#PLHovn6-(H>h zUP0@;_l2b4qgrZW*|L5A%m3q-S5!oHg-u(~>CaOkK^dto%uE&uJmigIzITiO`0nc$ z;wk#&W$EW~m({=;HW7V}o|x-Y>R~uS$QOl<9J?YNtY@ zqoa{kS0?t4ZttEQy01Ppg`OC9_ zk-!jOg-{3>B1oKU`QQLU2}{6M!smEy2{}|1E%CpzpG!+LZ+c2E_s!sjO-rTfI-Z+| zUeU*6&@E9P)3rKzQ(d@250$pc14LP&Pa*(vLt2Ps?dxRLIASXf7 z9x+pJs=QiXRC@-@P?&-sD;3h=fpKBW$X9W*>-u@lTtB z|MH6dW%jDSee%D3-0L&hex2205OcsRkz;DT)-URgIZcsKH@>3 z_ka2mo-r7&MCy*h6Peq!Sa3&H&mbpO;OSX=d1VmMbI`rw!&_9NRvBDCBKAnb33_5B zJw2l~W(5L0{?ohiZ&P88{itHrDo&MjmK}a$=madx>I0_S*eqd416{cPm#3f>O5efg zArp7s`DUULo+--~eDd)aQ?oy(XXYK6(DI<(d~Q_khxZ(ble}AY$-i8Z2fgX&tPHz& zDu~rw8@>vIe{Wfc|4G2RznfL{OM0jGaa?D`n9LX#@LYW7);Ga54;oLVCI>I{u-1n>^ABpq z&9>m}%EaqA>w22>MQuZ};~M*~#bX=MxTd!FM`h`{^_?eKNtW81VNtw30olaqiz|IE z2cHJ#l=ME33E3`Z>m(cf1`dSbk@nX0im{|%UDCdPfU#NUE`(n7HTCuPM<%8o@6V3R z&a&HjaH(QPEV%61@nmm%q=LdrDU$LHOk3WTSO{v=n8w`L*(4f5PkwkNN(MbI;v(cS zg)>8mv^0$vI5B0_YsM7OXB9rpV3m)2#el|SJBIUD_$2xd@2^1iqN0h%DC2L`J5dC{ zA`gSF&xS5FVq1={y!jseK8&36D&IJrfvg>LG&134NL>&s zH`(en^`c82nlOVi;j07&q#MNLh{Kk%aNnYNx z_T97GCLMx~P6maLkw*j~BV;sTUD;mX$y44QK@)d74J_nyuChR_V6`x7q>g3c`<@vH z5OFG|Q**y)Hh!wR!ef8)n!$78+r_sGV0|G)oqp+PCLN;i)NEY4+`#rpAic{hPJP0_ zXoj>tLO-SSvEJNX+{j?1#Z#!eSOzi)IjXl`+8vudcxKixuCG0N80wsw*V#4r0Y8KZ=RO!db>eTd>pH$Lf`d zsjlfx{LV}r1WcmPK)m8T_*%&{dax0nj2w)@ELmM`%Z zCANbMUl(E#7Pi>fL#WTFR4RbL6YYl<)+#7kL;ZS7E3pOkKyIBRrK^@nU|RIG@kq2UdC2)>~@ z(}8Z@TGPW#Glp@meF%tQ4zk=UqrmGd3dY@zZ0t#_lx4CCEbF``P81S?Or5q6GV~C3 z77-Q?MJ7)kx_Jyf=R2YgTHr*)Di=ITc7$rvt208C76KD4n2qs?@vEej5sQU;shojp=qc|NRSdBbfpMHM1cfCRU}ja>7eMGocDX@Tj$UBy?@T1 zv(|b3-2a|+ul+oG?`vQ8y|2C3+W41Lx6VY!WQ3=Mksa&^!CE7LY+E4GP*C7NCdlNR z(!l7x+52I%m-m(wneG@|)ZvrVB-hQ*P^=$rlTY0K=2u=^KEu=DJ`DhLvX zos_HWGg;|{MuF`4f+nf4yZ_$Lns9G}=YP>VNBNnM{eS)J?LYnOqRG4HrVq*=j+px0 z5sHIrKOM#R5BtZ~DIg5r%Oc}U#A3L)Zl@)yB)-EAM^CNXjksH7qGaZ&7Unj?iHZaH zWX1k6ez$P6B4uhk@r+(wT55)}E{f%dxIC+O6_4jaN7p4W7>AInp&inWov{^=DranZ zuV@~iUHg#ZMXGir)r{z`p+?HxV9j=H>k$EZjUP$MjPzIsNdMjc%PyA42;dwWhAMM{_<7Lz#;9$0-5n6tt`}Ojjts}H|)(t z5I%Tr;`AGa!%RM#1r)xSx#~J=V!}m{#j%bw4kjTXH>O>6&iLccDGNAP@8Fg2)oaL+ zO!(qqX6KAQi(%ADnt-2YT7)ItQqapZ*Gfwf7824lnH1L$ijo$*8!DInmsESf+q=ar zA>q&7`L`SP^W_A$H47tW^?2eE3!eGzdQ{a|1h;NL;d=cQP-X+<1S)T)u&jtwM-0bL zlRs{sJxl5`d1RwII2LlcC>}^I{WT*hI5Dj&B|LQoa~8PbV(r$kGesXlhSrEelyB9e zVQ77d$W|DTS2Vr2QUajGYVal&3?3XSYJPLc5vq>A(#`PZklk1`a$?>F9KQDC)Qh7J zvX2XUlbGzIRecKb2Aat{4(BMOn^k1#;_sDrJ*&k$~_ zS8mAT;lO!YlfV^ke{YD-Z$;ItJ9h^5kA<+=;ncHOLP}D4UOQfwmW#<+URhK5pwl`` zfAhusdLb9VQZ%uK?XxyM%g56>9-L@AaCY+a#uLo#2Wd-K(Zx{Tf-8kjrMY z;bbotrz)@LfQ&n9&98MDtBzdb-wab_FBSMbK?P6-8lpBRFuk>zmAm!Fg>D0*jBzLu zs-P#shgVsi9PE|I-ACaW^l#xGfaF8CPomRLYHd@~$q36OCP6h1`zk37B)*yx23V9? z36L8am~9=bIi!-*lhxD97v}s%lmeH*5*L=Z8*VIT&4^k|J7GkWUwp3MhVED+FRF6K zp(Ys`RoE`S6$-JpKe8E0n#Tlti^&j7;0)882-IGaY22s693h?@`J6RX;+I>buKTu> z)l1uMef&4FRFC-l;Qh^~SJ<^ygM(Pe`<;fsP4jhJCT?bO5ZglAY1xo8x)!$WT@-~4 z^{7^F-t320hA=eUOHz#N47w-aKDFt$Zm&u2PyKRp1Wm2*R);BkpQbXhNe?g&g!>>4 zgPU6-mmItK6S&xBkME23Hrhnu(b2%T^{&69636m%D?i8l3{oOaDh^U}uFrq-&`{OR zNe3bNK+x|+A!8 zeigAAvH*g92=z>0`13@1f8w|rITRBn#J<>;9CsOKxY+`JX6cJX3QBna0Su__Q{f(e7x1u*+fqDOE-p3!wrpN98<72H*xCpr?nk! zwI_2D6)^%k4bP{tgl;*olYrFaccasEG*+KpIi&(T-Oo#nu!KnGuzTe!GLg?SNz{F{ zPl5%Hdp0B6xivYa5PPFwLkAjj)=Zfl_vByT^h4jF52V~h#WB5Fm4l`J1$R$v_)&Yx zdl@`JH>&jGKvaxIm zc+En8_pQ5%>4c8Vkn`Tvj&Kg!r<>H0jQa4~RV!y(!%QvAk7_(KoQ$&V19Xn#+iBrF ziHoA%77McZZz1la<4AW<#r0R0H#35kpYQ?}Ftq%)QAPv&b>MQL0t*L^oRCftTylKT zU=w4U(2JXK8X|5gI>SP?6xvT3)}MRpM$bL>3Yb%$)e?m&F5Ky;Qt2>xElLQ06AFru zT{&yDgREFnZ66f{RR#T?)cTOAVrD!Y6&0LCxsn==o8)yR@uJS2HSk~EJ9E42MTTyG zmbwP+Wcu+dp9P_ueqlwlnpgtA`#N{FSdj*eo)ubJnJ1wAZCY4F2&NP*PF!|zW17dt zb(pn|-+hAc%igpFHo7mFFp0bnMX-Nble_Rg69RFzi2%2;;kUfaO+lwL46&7u}k+Gwm0es0d zNutI}2WUo}TrMLOKoA%n-dXu90KiHIdw;TBkWrjWzEQL12?=O7hC_=8Y70cNByrQU zqe12OK-)ldvB}TuY$XIrJtz-&+Sgv0;DuI8N=G^nbE-BS@_2VJ6t7PQkO^!BQ>OMM zM?xvrvQk5c33qmJq8hJAdVQz%eFdWxQbB3TG z-uc@aPUD&!Tdoz@c>Gc4+_NS}b&Vndb{5mM+4~v}pJ>8F?&Ke=l^+Yc%F0y_D76nQB$i=7d=FmS4wtXHNLnirAgwW`i(73z03AGVC; zY7qeq?_3bZ7D4L8PIkZ2pFQ8_^PcTDv7imm`=zAZUZH8(nLp8&AKP)e^7RRDiAQ)l#sqTtxJ-3!-gB-qdKvM6dLjP}dh-(3I{*%`P5mLx(+{57ds{)5-}r z>}wI`+qQX_g}XB81v>$<2j#xPYzF3CGr_v(R!s)Rn0%pHqS;Qua}(r`ytNr~*yRyh z)q*E&<&B;pPoMBS6{?9f2 zp2%i_y>D^q`pU*^j+yaC_*@cc6{>XNje%6|X;wd?*NT<9n`&N~tv|f1NcxQj*a1TGl z5z&8DyRhh?=45vLkVX2RGLQdHEpE$|10&Fs;{v^t86PvZJeOYV+dp;i>ivVp90})qlv*HO5`0u>x5r{y*`t$64rpq7(LdIorX3)TG8e zT<>bqlf{cp&z~|c=vf`NRd^6zaOB#x0rtFX#U=Thg;+U(ENWR+qywO@G0S5%Sa7)k z)ZBYUoHR^L&%eCS(#4hz+}r&VC3XDo%>F-o{l{_Mxe(*HOASM8hm)Cd*WT;G78j@8 zYy|y610%xt1QwR_@yvFG*Cd{qCtKsNkbLP8Bb3sjZT-AW6>;b4A#DfFu0-uc5g@)c z87WQxF66g+DpdZ%$8|zRGu8_;&7ZMR)aUsgb|}^&`SMy>3!~2qZ}1tYteClyTIUC0 z5h=x}(K}lH&kQs|(=QGI0XigObaFvk#)yAugw}q2%>mnrdiN(-1Pt`;N>;{)-`ZrR z9Mn88sm<|_7?2kTo33LpzQr@|@wu^PIREWlZHa51QUz@x=?C+&ik>}@p}pL2Qp4|$ zz`a(Q_?V#jL7cC7vjC#maV5PG|C{o~cl-ZgCi#TleK`5hKgxYM{rsp}ggoBTOXX(5 zkbzPoPuwlYYW}!=pyl9*ILvzFbI(JKBKIK^uAGo-6|vh}dLX6Xj*jhH!t4c*_Fq!6 zG2i#$>K0fBr+RwOHZRY2xDKMuC%hjNu5(5BT_heiL6l9dFbvey^z-*!<95@|T>sdZ z@m8zmOZ8$ztXxTW__g+L7ohTv@}OQN&2FgaS`;s{Da)2K9@O?&xc4|0dnN=Mh@v^v z{Rp&Oa$wGIQa=Yxdxu-*o%V}2_IC;mb+N55wAo3{>+Bi_Va&{^M*5G=i#GL(i%Ctd z2f|n-CGNr>kcijY`VM%Ga}+?^zGbyX*e+2D*_ZhZ<@DKja|N<8NU3{FSabhVlQHn~l4F|o z(mXHbA^l{blfA-RXMw;F8PF7K;Kw(iV6NPPr0zyO2txJeC%IXr^g-@VFrcE^*qrg{ zor9nD+){7HeBQU?_V~H*{PvN$utfps3nCbXKaJ5 z71O{*eXRVm4kT5La9-*34E&W?PE-HU4~6xp3WY>|1kbDdT$(>g8H+8%o^TSVBnatX zISMk)YI)W7uvs}tq42>2(_W8Yc6inErxAdH31(K*SEq!A`aWE!LB+yW=OA&7Cu;;V z9A2}Cu4Z}bJXZ*hY^TvHgNNto@6d@Zg9NmgKs8(m;30`mM|?+ZxT9yQ-TLXPi~jPP zfAAPBgBA6S+20&aB@Yk3H0#BpX7Nnz%^jBlxnYdKevO~v=b>1qAx}nS?;In;F-s3a z(kHB{_wSNtQ~XffC-QPR;!r1>zS+$Ss&LVK1y#Wf#B5`e!`UT?+A(C|Ck#2oi=P+$ zxq5|=S!Uu8Y8G5v>#C8K_rpAfM?g%{n43%{WB5ete<8x0>kg_^t6H>0xXn?vFmAk6 zF5+WgCP3UHp(35t2dTqJ{)6lu%;zL8e7!B0|Kdv5#55ct?p0a&`QuOStGt2z=yxS0 zwbuAX;b2pl!#@~VYU5u7yPPYMzVDRL-h1lfisNxyPZlmB()0WI92V>fbI1ygrjYlj zP{Zz6qc@ZkEVXOU?ktdg1FGpnt@rM1YJBbVrpAKEw@gb47NW9IaCYho9)y^4gEPc33fN4skX;gacf*cqE%G-%Em@{V) zJ3wvA>RR&7!W02U_%#xjC6}!gcqRYP#~+%oFDzVlYBBs2$ZK{*{ps8OSWjs+9uq8U zMiQ+{tMA>aIX%(|R_UZ`pmGv&Ixi|qNqz6qv2&R|@A>ld25%XoN#TirOgu1(XVrG= zfTPo!wd#-MRk=LT+l^!*C}@m7G4ScZ@lvi(KLD4}#D=GuG=L%A2=8s35uB9FG3%Z; z_~zFxAFlKp%zNY$j7WVonfskWz;NYvot)R(TRYphq!Nazw^?+t(F5Ch?!{x5QPq&iT{qu1YjJ0!P_chG1B4SG>%>eD;T| zoq9Fw=OKF_=YRIV@H z|9+zatHpX`a|>YEKw~XZ5wd1-e?F8O@hxGzP-;j&ncpDf4(n29)X1}zY|pA#t+rNhGsLav#t9A&?7I@?1ySD|i(|+Fs!{(~^%`jTbHJ;x zoxJ$PXwm)pS+|#lUvY2INhLagf=OCbri%-F*1F$KTQ3JtQNIfgXqi2qbM<;@yL@uI zbP0 z54~;HTvSg{!b?EK{bN3G2n6Ev*2%rIa}cYNg28LRG!@UE4-J4Il9wgR6T`X=O#_&}9a|W@h2NO40%?HyR)FLw<~8_TvfB zk1t2MnO-W~>b?^EpusM25M2wpvw?o(0GRsY=AP;IKgx8!2yyl~cenKV;vGKEF?dl~ zgLCXjR-)?qP?P#o)=&{D>-ew9SFb|jKRm6qZ}Se;|Hh=O^3+5sV-zLhPnFiBeJ>LS zhD2wD#&>$=3C&Z7KH#;=(FwxdOv%+}NsJ^dHfZi7RrZWNXdPF#V)z%?8U|`0O>_gX zIpo|cyu7-9I+-6z;$+&J5;$|FGbxle89JO1QOambhn=T6Fd>vAIm$w*cp$y9&v>VK zU#Ch@o9PixyAcQCXr=ouPcUh^awlj=ugdx1wbnOiL zZD_+0&&8jZB{eq*&ou&4_IwfjGn`zZ%7^wM9VP8ofek}=IVq`qQc^NO!I{}b52fLPk&wC$ z$0)PCMrlLzG`}TdX*`-U4RdX+O}4)D@H*;<-eOL~QIARYFYOLhmJ>WaO)}>cwB@-y zL!=dlq^(>|p?6zfk9Q9~GwLW`Ss!FT=3&rPaF=ZW&bhPv*E#l1lHB0XB?ULxsqht| zX8P7^oljva4G!AaOatz37cKJrc3gXjFRqqyYG8=IPX8N+^nz#o6SAu;f0aJv5y)Ld zdnSM4s%9KMG#7E{Fb`3|dYr(_TDuWPIdT*CW~uJN5xbzq8lo=yXx$F;g*d#KfiPxj zd#>||yT=gc+`zBxbfM@RGNVb_uk-zjTJ6xlWziY`uz-N8ryTrmFl{kGq<0v`sq-&| zV~$nD&^LNjaPG7Er#Y&WpKRbxW2^>JrHy1{@Q9O#oCZ?YUC63s(gPk>%r$ z=|QdB1_qxSC$#rMO9c=gpu$#C;?i(=pc1gbf6+pZc6yBY(g3v-ig2RJH8Q%0zbaY` z$^IoZThrI_CVknsD)qu^t3!WDxx`hJJh|)lDkP{}B-C$l2GeWA*m~zqztX3VekZ6g znQZvV4SeyYi%X^6As=M;2>UaqGG*fg-=@PM#FC`CM>9nlG65j3z>a9q2G^9gT$3bl zPUAe4)wwc~W~Yr#Dngh_4vumfe*ffj(i>A>BIvutLx|f1gXPRfbVB09)%-3`n_+8Z zs2@ssVa_-2^n2=cJRDrgus0)DezXK?xXyk}_4gL zrj+hi?LPU-pAY?h+|F4VkeEW&mVg)oLt8o?@N?F8Fyrp1r|+oM5#d9`ru_CHlMK#G z&jZT~1uhH*Y{~|+rw^NGO?)c$bf|>A@y*s6sXQd?P`DtMJ4l`>J4mZ8XG|n6H0FPI z>ffeC{|*UC=w{oN7-~Z*o?Jh>&Pa}Xb8q?Sslk#OllR!vT`X0|?bEG3hPTSv6M@Zl zO*6|i$B1XzXpb=5u2jq#y;!Vhk$nIyj@8Rc?T##Ox;|J)DoLw_rRhxbG|~SnV zBM5pafYtcXl@d(%m@VW)ApbGg>z#rG&~@SnRs7Pet$)R*oP@+QW$9DMq^*>&jyg9_ z=ciSI@kI({)%bqb+>om-7g+3B$;YR8EiG41HRW7uim;+7y^ts_)tJz~J}fCv8PcVc zrg|f6CQ1M=^Zg#JKZ=hV$p08pG2jOTmU$r_ddkmxUOnNotH6FIP*7>ev{>H^nV2RQDQm_{bKCfOQNV-LdPjuJsOKy@98(FA zMmo?n2xtKm7WL&S?1!>wp+e`m1IHlhGQ|~TF&tZ6VICq437Q@h2VH2-wZmGy`L)sq z%iiA7s?=tlPc1@;QzL;lCK7l+)w{pHv4g5qx{Gv`sO1=7OwXm68kRsk5L`DU?PO@! z;fN!;!40+ifSMRqlLH8qlQ_As=rrBz)sK2*nHsjSJ~-;${Sz}qd9T110k;dtk0$s$ z-uOG{XH#SiZOrr^8FlRRq^qIQo? z_LZxNhG0%uFhmvIaFvvLb`PeWA>PwieBh7Rl3J5^CdRHw@jxxnEPSP($;?-3jXa6T zJThhw_G8GIMI0Kzp>7`-?0MG8^H16I<{`6J`2(pEI5&tzrC_2chL`*$N}Uj6!O9ul``xLib@CZ$@EQw(87ckK5y?Lc=SABnq%p6b(}FsuW$womQP`I`(6HOWX0JfI=l*lAlEQ#^a~ zC0|Qxe2b`zT?hjol7WM84(wiN&#TN>6vDZQZA+I3dFcRt?y8tQSsr}O`!A`7z!2ZK z(1iux@SCoNxv#03@NQEe4vwzf#Jw83nF|g$rObWq&E!%BDk>My0woZ{`*X?9!!;R- zPV=5e!l+~Awsf;tLsu!O-~NX)bANAtAMnmAx9t(;!-O!;V@T_2GH}FFCXK5RbBx+0 zEUTbv=L|)Xve%z3^?Co`9QP(tyZ|e29}aw|h|0G{cZ6f=vow@+I+xdJjWcu%lEi+Y zdhuyTniZv@*FO9{lPjOr+0xAfIWAQ4Gv9gBEpAkkaEE+5&f8m(KD-B%_$kPC)9Ank zJ&KVFcfst1qUzU-5AKXpG($P)UGLH{0e%*7o=p!G7O4B^4``vot8d!O zIEW#HbwS?r^jb=Wu@~8sdG@xDvuPMOtg_v-i|^JBNsDn1TM7$EvQ68-pRHxy*Q=eu zq&|qZlr<&gEiF^Vx6Mck-a%{q*l+iu&$svXW~WQt>IpcyBcbfR``bSMyZ-lDVE@>| zgS?#hQmLQbL)EfC8{VrY3_gACgZJ%^zV`S#rIQ$< { expect(await screen.findByRole("button", { name: /Created from APIs page/i })).toBeInTheDocument(); }); + it("creates a key with a selectable reasoning-effort policy", async () => { + const user = userEvent.setup(); + let requestBody: unknown; + server.use( + http.post("/api/api-keys/", async ({ request }) => { + requestBody = await request.json(); + return HttpResponse.json(createApiKeyCreateResponse({ name: "Selectable effort key" })); + }), + ); + renderWithProviders(); + + await user.click(await screen.findByRole("button", { name: "Create API Key" })); + const createDialog = await screen.findByRole("dialog", { name: "Create API key" }); + await user.type(within(createDialog).getByLabelText("Name"), "Selectable effort key"); + await user.click(within(createDialog).getByRole("button", { name: "Allowed efforts: All efforts" })); + await user.click(screen.getByRole("menuitemcheckbox", { name: /^Low$/ })); + await user.keyboard("{Escape}"); + + expect(within(createDialog).getByLabelText("Enforced Effort")).toBeDisabled(); + + await user.click(within(createDialog).getByRole("button", { name: "Create" })); + await waitFor(() => { + expect(requestBody).toMatchObject({ + allowedReasoningEfforts: ["low"], + enforcedReasoningEffort: null, + }); + }); + }); + it("edits, toggles, regenerates, and deletes the selected key", async () => { const user = userEvent.setup(); renderWithProviders(); diff --git a/frontend/src/components/copy-button.test.tsx b/frontend/src/components/copy-button.test.tsx index 6332d26a19..9043615e9b 100644 --- a/frontend/src/components/copy-button.test.tsx +++ b/frontend/src/components/copy-button.test.tsx @@ -53,6 +53,59 @@ describe("CopyButton", () => { expect(screen.getByRole("button", { name: "Copy" })).toBeInTheDocument(); }); + it("clears the feedback timer when unmounted", async () => { + const writeText = vi.fn().mockResolvedValue(undefined); + Object.defineProperty(window, "isSecureContext", { + configurable: true, + value: true, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); + + const { unmount } = render(); + await act(async () => { + fireEvent.click(screen.getByRole("button", { name: "Copy" })); + await Promise.resolve(); + }); + + expect(vi.getTimerCount()).toBe(1); + unmount(); + expect(vi.getTimerCount()).toBe(0); + }); + + it("ignores clipboard completion after unmount", async () => { + let resolveWrite!: () => void; + const writeText = vi.fn( + () => + new Promise((resolve) => { + resolveWrite = resolve; + }), + ); + Object.defineProperty(window, "isSecureContext", { + configurable: true, + value: true, + }); + Object.defineProperty(navigator, "clipboard", { + configurable: true, + value: { writeText }, + }); + + const { unmount } = render(); + fireEvent.click(screen.getByRole("button", { name: "Copy" })); + expect(writeText).toHaveBeenCalledWith("secret-value"); + + unmount(); + await act(async () => { + resolveWrite(); + await Promise.resolve(); + }); + + expect(toastSuccess).not.toHaveBeenCalled(); + expect(vi.getTimerCount()).toBe(0); + }); + it("shows error toast when clipboard write fails", async () => { const writeText = vi.fn().mockRejectedValue(new Error("clipboard blocked")); Object.defineProperty(window, "isSecureContext", { diff --git a/frontend/src/components/copy-button.tsx b/frontend/src/components/copy-button.tsx index 3d0469406a..3b458da589 100644 --- a/frontend/src/components/copy-button.tsx +++ b/frontend/src/components/copy-button.tsx @@ -1,5 +1,5 @@ import { Check, Copy } from "lucide-react"; -import { useState, type MouseEvent } from "react"; +import { useEffect, useRef, useState, type MouseEvent } from "react"; import { useTranslation } from "react-i18next"; import { toast } from "sonner"; @@ -15,8 +15,20 @@ export type CopyButtonProps = { export function CopyButton({ value, label, iconOnly = false }: CopyButtonProps) { const { t } = useTranslation(); const [copied, setCopied] = useState(false); + const mountedRef = useRef(false); + const resetTimerRef = useRef | null>(null); const labelText = label ?? t("components.copyButton.copy"); + useEffect(() => { + mountedRef.current = true; + return () => { + mountedRef.current = false; + if (resetTimerRef.current !== null) { + clearTimeout(resetTimerRef.current); + } + }; + }, []); + const handleCopy = async (event: MouseEvent) => { const trigger = event.currentTarget; const dialogContainer = trigger.closest("[role='dialog']"); @@ -25,16 +37,27 @@ export function CopyButton({ value, label, iconOnly = false }: CopyButtonProps) const copiedToClipboard = await copyToClipboard(value, { container: dialogContainer instanceof HTMLElement ? dialogContainer : undefined, }); + if (!mountedRef.current) { + return; + } if (copiedToClipboard) { setCopied(true); toast.success(t("components.copyButton.toasts.copied")); - setTimeout(() => setCopied(false), 1200); + if (resetTimerRef.current !== null) { + clearTimeout(resetTimerRef.current); + } + resetTimerRef.current = setTimeout(() => { + resetTimerRef.current = null; + setCopied(false); + }, 1200); return; } toast.error(t("components.copyButton.toasts.failed")); } catch { - toast.error(t("components.copyButton.toasts.failed")); + if (mountedRef.current) { + toast.error(t("components.copyButton.toasts.failed")); + } } }; const copiedLabel = t("components.copyButton.copied"); diff --git a/frontend/src/features/api-keys/components/api-key-create-dialog.test.tsx b/frontend/src/features/api-keys/components/api-key-create-dialog.test.tsx index 7e8535892c..af78496569 100644 --- a/frontend/src/features/api-keys/components/api-key-create-dialog.test.tsx +++ b/frontend/src/features/api-keys/components/api-key-create-dialog.test.tsx @@ -11,6 +11,14 @@ import { renderWithProviders } from "@/test/utils"; import { ApiKeyCreateDialog } from "./api-key-create-dialog"; describe("ApiKeyCreateDialog", () => { + it("labels the reasoning effort trigger with its field and state", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("button", { name: "Allowed efforts: All efforts" })).toBeInTheDocument(); + }); + it("shows the codex /model checkbox unchecked by default", () => { renderWithProviders( 0 + ? { allowedReasoningEfforts: draft.selectedReasoningEfforts } + : {}), enforcedServiceTier: draft.enforcedServiceTier === "none" ? null : draft.enforcedServiceTier as ServiceTierType, trafficClass: draft.trafficClass, transportPolicyOverride: draft.transportPolicyOverride, @@ -209,7 +215,11 @@ function ApiKeyCreateForm({ busy, onClose, onSubmit }: ApiKeyCreateFormProps) {

    - 0} + onValueChange={(enforcedReasoningEffort) => updateDraft({ enforcedReasoningEffort, selectedReasoningEfforts: [] })} + > @@ -226,6 +236,18 @@ function ApiKeyCreateForm({ busy, onClose, onSubmit }: ApiKeyCreateFormProps) {
    +
    +

    {t("apiKeys.form.allowedReasoningEfforts")}

    + updateDraft({ + selectedReasoningEfforts, + enforcedReasoningEffort: selectedReasoningEfforts.length > 0 ? "none" : draft.enforcedReasoningEffort, + })} + /> +
    +
    updateDraft({ enforcedReasoningEffort })}> +
    +
    +
    {t("apiKeys.form.allowedReasoningEfforts")}
    + updateDraft({ + selectedReasoningEfforts, + enforcedReasoningEffort: selectedReasoningEfforts.length > 0 ? "none" : draft.enforcedReasoningEffort, + })} + /> +
    +
    {t("apiKeys.form.enforcedServiceTier")}
    •  + +
    Kevin Lin
    Kevin Lin
    💻 ⚠️    		 Borealin
    Borealin
    💻 ⚠️    		 BrenticusMaximus
    BrenticusMaximus
    💻 ⚠️    		Sakthimaran
    Sakthimaran
    💻 ⚠️    		Evan
    Evan
    💻
    diff --git a/app/modules/api_keys/schemas.py b/app/modules/api_keys/schemas.py index 6947c3dfc0..10aa0dfe10 100644 --- a/app/modules/api_keys/schemas.py +++ b/app/modules/api_keys/schemas.py @@ -33,7 +33,7 @@ class ApiKeyCreateRequest(DashboardModel): default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) allowed_reasoning_efforts: list[str] | None = None - enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") + enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|(ultra)?fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None usage_sections: str | None = None @@ -53,7 +53,7 @@ class ApiKeyUpdateRequest(DashboardModel): default=None, pattern=r"(?i)^(none|minimal|low|medium|high|xhigh|max|ultra)$" ) allowed_reasoning_efforts: list[str] | None = None - enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|fast)$") + enforced_service_tier: str | None = Field(default=None, pattern=r"(?i)^(auto|default|priority|flex|(ultra)?fast)$") traffic_class: str | None = Field(default=None, pattern=r"(?i)^(foreground|opportunistic)$") transport_policy_override: str | None = None usage_sections: str | None = None diff --git a/app/modules/api_keys/service.py b/app/modules/api_keys/service.py index b6e3ca8f1a..a68bce8db0 100644 --- a/app/modules/api_keys/service.py +++ b/app/modules/api_keys/service.py @@ -1461,7 +1461,7 @@ def _normalize_model_slug(value: str | None) -> str | None: _REASONING_EFFORT_ORDER = ("minimal", "low", "medium", "high", "xhigh", "max", "ultra") _SUPPORTED_REASONING_EFFORTS = frozenset({"none", *_REASONING_EFFORT_ORDER}) _SUPPORTED_SELECTABLE_REASONING_EFFORTS = frozenset(_REASONING_EFFORT_ORDER) -_SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default", "priority", "flex"}) +_SUPPORTED_SERVICE_TIERS = frozenset({"auto", "default", "priority", "flex", "ultrafast"}) def _normalize_expires_at(value: datetime | None) -> datetime | None: diff --git a/docs/screenshots/api-key-ultrafast-after.jpg b/docs/screenshots/api-key-ultrafast-after.jpg new file mode 100644 index 0000000000000000000000000000000000000000..e9c89139dc53e02471e8ad1d1de1cfae63dbf78d GIT binary patch literal 83038 zcmeFZ30RZavM3r`aX{2YCS}l;NeP021O&th!i*S52txuQbASM0lF`-{lpvu41SAY< z0s;PzAVYuzffi*JB#;Cd6(KSSD1s=8+_-z6v+sLnpY!egy?ej+&bzyk{HwC6R;^m~ z*IN14P}zLB`5N@IwYil!Xv-E5Xp86t+I+L+rIo3v$7Kh5b1PfupM(yO2)3OBfr7CS zI0uW1zc{9g(MJ_0 zs$8V3v`x8I*1Hbwwe&z=r9T6HK;<@|-$H5&;MYy*JKj!y8;aC4B+$$9Kga59G z=S*-=Zje6R>dQMyy~Q3Qp996%PJi$B^g`YwVH3J7$16$IM)``_}e=7KXt@Lg+T#KON%et1;%<9gvkiuwPn6 z`iGD$TSaBIiR}^-+a`iOAc;B)9LEI-;{fS~-I& za{=by2oEP__Ur0FJgqVV7aH?9>)a>TdU1|nS5l`OhNB&mx zA?j9(NVvSH)}OZit!9x5qOaSwi-~Ajk|M>n{%9msnYlg5HuY#$}hZBkD7 zbGsY;y)uZBYz#AATrbnv93*M^?@j+FkZpH7u@u8uM&VBHq4uefdH22}H+_50slRgl zl$M`p*Th=*>419B(&GdE3Fo$ zyuU()-V1Y`vR4Syk!EHEYv%bPh5Dd`&3guP&TGQl+fC%eiIp;v@`W;870;o4E<<~B z+)!&OlHN|N?ypQI0CJs5k~m(PoctuBP5|jdu_P>Cho7u81WMO5o61>kt0%6FmJ}P+ z%*{L9fA#LD14_%tM)TNwX#fMF%PdFma=n9b?}v2|j^d3gZD*-uW>NMl7l__F=eJ4o z3d4;;XiJ{_ysCuju^H{lV_-1quBB(%LO80vC4zVwc->P{T@`$SU5>SqtWuGgX~=)e zk{5mlQop|i3jzd*<=hf|;tJ0ED|BW57n+sv0^H3%;6rU`wv(wu%xgj%Umo8iG1XUC zNp*_tF`jY}C~Vyid@zlT42H7P%`*H8#C58awIM8ojEvN6DOl}Ch6*>YFa>=W+u&eC zU@$D^3}GE_0UyOd2tqM+0t$!R?laP$)@dEAM~)OkB`=jk)NYT~#v^qRb&cNL57JNo zpIF{SIV!>2`|t=oNM@P))M|ZraogUE(?hb{!*yuN+&87#Zscj(_gmk_)uyA4hq}FG zmb`?9mRJLRfvgcr5<+r>6xJw*o3IO8t)(@N^X6{_qqVRpIssmkAWg7+UE+xMV1#~Y z?&y7+?S!%bj2^Rba3rpDC81VbJ=3W2>Tt(~&)X&V?QjfYxHN?wnp(e)JX89%-sytP zrJkaX9WSZAFi5_zMI9L}qraPy1Mg3NKi7XEDrQjk?TGm%s8_(}3wkS*hI-G2Hw{D$ z*5xQf8+#uwa_xfFeuQt_YmM*-bwDY1tzXt+sUeN907`GWhs63 z@)O*~lcgcYc|G<1jR`84fGY)<36g^2<&G+z_Z@)TmD|ETea&t|tu0qmJwool=(;Ng z;}fUut*K8(qI)vT-0_~0FiAvb+8;5qg-kp~&TMF?w90(QrBO}`=OY*WJXV#Y2UF{n z#|&Fzj@mn_KZZyGhD?m6>*xI3*iiGi3Mc}1r!)|}MC6SWKS+R|C<-PkS+3@#HAJ3k zG$^^P@arVKlvc~{wsYvo|Da7D$wHkF<1<)5QFXC^QxcQaU)lP;}GFYrCD;YB^!MuHM2~;LgT)9EMbW!R3na?k*ppfvQ zAc;Ze$q1$`#gk#nj{YpU?>vL!=%^ET5V7x^O;Dxbi!wrteu}0HI~*q2%SU&mjJF{X zHotJFj7v_V8V5rnDb9j|ejQa;($)EhMA<54^cpwp)z;oWK)Y%iAZX!$KIv{$Yo3*o zYIzz13)2OJW^YRG^KM1RpFrC{p>eMDlLhI;fo}Gy_#Vmv6{Sp>*0{U)eoGnKb&dhq zwFl*!Mq&UDXGJTPIWSYPTgQvS*y3PI(IL974;dXT6|z6ipP=|dLczh$HGFJU9lcBSiQ3t zBXJI}-ib&nxzVZM1s^uDSdny1ZG|Z^zdVZ2@Gym`(1TgTCt-y*9<{8q542Y~j|?8!+PJn-uu%vZyMRs z=W6?Ltdk-04L&vL;?n_*MQp}uRD2!S+lns#3Eu9u-{g^SxS=4s@gDje)FN~=rq!v! zFb~%8DUhisn4{3HQKFk3oV0qcw5suQ&e1=jc}3^J0{Up0NJ087vo|F+=hTNS?1~xO zT*mC4S?8+o7_f3+qyvPA@J3>kRa}8at(Wl0`p(Yl;a3K%<>w!BTd{S4$gaW0((2=O z$Xb005NOzI&XQ`*9ME!wQ4MUE1dsiVO9>K4Bk_*KkL}VQtE^*$X|bA5JXDF4&xo>D zwzqQ-%V9-&j#V1R;8o5T^jCoQ?FU}=P&Co45UzeY)LF>R_D9Py{s*AmLqiJIKrY>WU} za~olxS=b&dpUTLxo4be7a8oC7whznhz6;N~>;%`fl=Y<*g}{dvQJUmj^yjfxe;kTd zGoHMufu@(KM~1E-#@aRVOICH1gj!sUtF#PS71?^GjJ`Tvt!vv+);EqX1?W#(e2eRi zCZqt3xk;aK8*kTDAeV=;ox+)Pqyj=L;FHMwd?X%sM zi@6A|fxW&P?`%)(X}^cSe9UDjru%XxdawZ44}UanZmVdjqC4+bUddO3MC{(y#Y+SL zZQ;ZdDhkM52g5Q+B=13VM*qZf?oD92);9zC}3U24k=L{5*Sr5CN4bbOij z#L*6+sqWPe2`74m=clH1-9>%=x#{MljZ#bH6=q7!N~)Nqd?z)Z@FiSzFjV}Iycq1di2e&qLqMu1oUfni;vT0;RB9;w5#)dN(lyYYy8qR z_0T)b2ECZHCj7YPBXkDG$p+l6z*0_wDX5TifKAW?HmW3Ef&-;gJ4;p@#l{5CqOR~t z)0WRdkeEi-LG4BSk}WOTn_HNQtsYoTkEV0%dNpSE%bGC|P|G?E7`#>%#N~shIoxHN3@H*iC90S7j0ytWuoWiblLVxpjMAVy-8Ybc;i6L= z0%*v~pAZKbWUZUnW;-)w2KBiMpYq40Z!8}t=jDL&GGi&xJqLNCYRP-#xJcfGUQZ|> zNJYCazM3tSfD8*mw4WRuYQDzSV-`_OJB4LEBm|U&sOb< zNUHq;<9`5Kzbs&3Sg23Rv`9#%TVr|3?`L?29^%RpAhzWdQTs8W-@dmRYRZPRP(~}FC5gGS`cu@eGE>8{5IodLw`=sNC z%WkQ*0KkTF;(1)yB?9^g>y_)^t7FsC9d`zieT|aK%IcEt?O24pxF>wl zboTo;ZQ3+Ax9<5owO;|9Qjw5Bh-BbAS*)eNf_*O(T<<~GZAkC0yj2t;kcK#VBf|2V zaD1Cqd~dAR&G7m z5^@iJyZ62L8@%k0hCq)G7lbH!Hy51hY+!uF#0t?Te`?#y&@*o4YQT8u$t^ULr* z6Nv|{y}46kf=!U;U(+kPzsVW-Wk$Pg-2OB9wMzmuTWB{H@=5W}Wa}}Vzn1=gB1C1a z@)b;i_}5ym=3UGAKL0iIkp$>PZEDBH>`Q%a%;833&>s?C{U4`~7yK41NBlbQNMd_z z$hg8^a*`a1@8nV8o1mV5#FmZ_wP5JBPA$FVfv%LaYlV!cvo*^9w`MGNeJP5_FzL4x zcIOMbfX@gU+y5Oc|3}Q1mHFNFzC+?MuRm{nrFZp$)J`&EbxPv=Kj?v~onfUJulDRN z+5}a#suTzL^Dbo)B@qY%V4^hdb97S6BrLMb6}7Gt~Xh9c6k2Jx{u`n27_v8vBQ z783XC4Dm(l^&@?I3%}SU_APNm?YPh-6Jp2~*a}9<0E|P-NaH=TD5Md9f2YipJiT;W z=m@+?A90lJ~Gw-GynK2X1xnP^tjlJNgQV( zOg2HJJ-=`@g??&8oSyQV<>ARNnGR9*=zL!tyZDHP4Q&@6KSS*qy*HAR0`ag zf{x!QYrcMK{i{U?!5-;VEc8+;`c^l|z>g3Q?_9n_v@>-q zk~0h@9@ifN7bhIYzUd&%l|$F_eucdL@8zEVn6^Vsh_hobH7z53{T88D0f3)o8-Rg| z7vm?_+$*{^L71p55>mz$Na1ys@s7y3Zxx?PuKfJu6lXON`PK1J^uKuiWuJff#y`+6 zBzTeb^*jrk%W1@F!vljDiXj2rQCjwK?N`CC|5#7`3)i6=*J>K9wDfms45uLN_ULJ& zu8&!dr{=s3Bo2L#IP-1TF~FpkU~yKyIIWlV7E{``3F=xhxZTs9-n#=`B-!s?Amj3*afD|0!udP-Y}NSxM$!NQ&(wG6uwRO=lvR3ERGFCFgt z9Y0RQKZ~kTW_6U-BYKfBkLm)7ZKK7?m=OcbUXOxm`c-vx*u4f2*Qq7Xj)uy+o}lsT z$y!zn*P7=yW&0d?Y7n$3xhDp-u!sxco7uf=!`;>@T7kondcoW7CY&Ma+BuoH6WNYM zJlqE})I|INwb7dXT3~LjL*^=`WjK=X&>?{;b$tPnxfg-{c9sf5hL))XehaXt4}F*!u4(mC;!FjN|yFw zq-vOB!N06m|Vvkw(LT8=TScVB-<&9q`VjC{!hBt3_U ztQYe1@z2^0RJ2vGPUcnrruEcHwUkoR?-(5LrcRk-V1)KtkQwfp`tdMJn_9YdKCf86 z%P3ag9%`W&x**W!4IqN=eVRO$4n_MmH{Vr{UK|a2*D3SVygWO(C94xxZ?$bzg4i*w z;v@rw_sWF6QwDqLWQ_wQCnEag=br@=XqNmtt3)^H<0*=(Z>BL6pTtO&TA@Q%tiRi- zTqQjwSqr6HxybLL&^AG%A*Fk1$wL9noI+=&k23s^;q&#jdDKmBQ6L&z6#W}7s%DC42mo1 zN9Z(LVK3!o6r1;&Ut(!FQv-FBDasAFqtY1c#QbF=|94JCob?^b7t>=mPAg?;96}Y!!LKFkB%; zBimPp3t8~GnsnC-w&2uE;hA+&mQdd5b}{8eKAN5d&RJC)|4`lxyMu>&|0*jmT6^J@ z(T(h}ZAiCwI#p>S2US0p@Ar(n2B;J^)Q2HPBQJ5|f{`cbmJbQi)}+#shh-6^8+Wag zl}*AM!;@No=)Orx1N;fj`g36i1~%L=Lk)CIefpY$nS9%}tD^ray>{nt^2L{`!_Qv+ zLeOt}=NX!Y@syE^4R!y1eyFeC6}#m6tk%wE59G{%&81FM<@0hr-@Fl5_MYRu&2H3I zkcS37s`4MV=U%Y4tZFI@$bYGt$UT5hA760K@N3R$izd71Rn4q-fE^xIc5X$z9Fey5 z4z`F_<(Zi;EeQ?Qy0l570v5^2*!7e7qgwrzXDKS^xgN4Q^HlSJ$;W$^hu#IV+3zCH z5IJdT-g`^08w9Pd1FboD1B8rd3NUz33*UoHZfZPAA!xuJF-)|4Tx-Py7<%O?g>_GF-3ox{*Cy6uA}~b-__ncDspp5bR>dFu@&shj;0hV67-4*RmQ|MGA_vYNKSX z*r>h$tN+5$99O&S5++9>wyEsCRTk2awz1rx^CQ+{h7ELbdk2G)7h#Za0!O{6MWPPm85h!*C5m zNBoUumcLNOr>xS(MiJt`?Y`V81kly_0`?3erGbb~Yq!#E;LAPsE~y;OU}&Y+oegjc zc5P)^<&$4H>!)xhO7W$IinP@w#SQJ8F@KR+UnoS`4ngMa74Db4(Il~*mOg{)^&>dp z&)=0(ON0>Vg$JLAiw~8}&i#_MUI~XNXojYi-B&q`VS2xo< zZWessotfu0hUu7<#padu6OpjyWnw8WWZzH9ekld(u<(qr=LMHje`+Iw-;LBb9o??^ zp`2mAJ#<^(i*A|mLi^`Mx<1oF{F43Aw+^E1)BkM{!Ck(N1NyFiu#=iyl$NybnLkn- zUSP7D6ejq1z;AwOPvvcqmLLF!TBbo`6PaqnmFoqLG@3ac4yi z9Xy?Lf1@xzx8Glk6Z{LuUu|weX1x=0^#z<)nr)?A>gkXOvxD7Tj2bvSpV@+w%oK&UAO;DDC zSAgFBDnperf;ys|fmBhPG7mAQ$*^bZ5?9+BX7*{2Vna4Tte;k?WY1{V2I$SEnXv;k zFG`19F;l(=W+qV(HJmCz@Y(vCHg3n;?^(+U#YTIx=tZ#x(cBdP+h{a)OpP{T(9#n{ ztBRkoO3z?G@}Nq^5`AC4t^_cU3}J1jAWc3W8B8xrzkhW6F)KhR`00wXP<^0?4CxXI z3>SI%y{lbuGb)QdPt-1@fsM>98F7*1sT5|QEB!qTzAl&HW}fZ;YWAoXS4qD=!LXmR z9w%o|QB*rjshfeSohsuW+~3})9#vI&=Pe6hwrEd2@{_>?hg zEO0gvQFF0Ixtb-IO6IgqWu+$iCrwu5YSVu3p>S$^{;Y5AWxn90-qnxs)S_38u;99G znBtkU3itTtA1W*4!&;gfxVU7~kFcYn#OpSzn zd0oG+OHH&u2?m3)aL2iQ+>(_8oJ8W|tXcJYPm4Y{`xTXvZ*~S|rv}aQy(4-{BaSEX z$pzA8U64&sQ~iwgwfh6$+M6HN?PrB{6&5t^twMG~oL)r7SVSx^@d5~xClplK$-a#^d?t7l zNXtCk_Vd2m=b+8prm^*lqcz+*hVCY4t80W)r`D(1*R^%XHwW_b?1yU2jl*#Z!(H_) z>K|!Bsg?7UT-=3Np=QQ+gx2FSe_^se<*|hlGtdVuZ#flsxRTV1grJ)B-vK_7VXL&8 z%YoigZY!A^UlSp;We10zO6%^(i^1M_4XM|(C@!S)DMI?oxvYKtqT!P* z-@n)Pi^^21;koJn)jpfJdcl~m9wE>_8d#-=R1V@Q76?U?A9X1ggz1e9^_VT2pxbt* zDqk8me*a~$0H3kiYj|IOXkm2|Lbu@gP6l1k<*yDlXU7SrxrF%hSrhT3} zgH@I4TN7xfsbWrD9b+Nqr(f3{X!};k-UQWjZ-VX)OKyTbYn5z*oY(VypRZ7lI&`5K zD&M1DW}N^g<&c&3XII@1zF>vcGM@vIg>1|5m-1tqAn~9@{@|6wt0&auuaYnID4?a9 z)QKi3D#1!Uc?E3xlg_=&Cb_3K+m^$MnazCXOz4tLTyfd}+|A*Yfnl;S6NpDpoupZh?$zF)519%%pad-A{kM*Rp5`^RB#p_W|T#W{KF zulY5D@*P7e&Rr4lm#J%&s!>(i2pKt?_NiA}?`?lOLB(Bm6%@NO3+3Cn@TcitLd?2D z8&^=qK=8RIH4!X1bA*6*bxcuH*YSLM6!)_QL=tsh40-TH%=xJs+{@0vBtG(JIH#l3 ze>NjBt}KrgZN+F~Hl2%Wu|!MUj7;K@UXZ~YLm*(kWu$5x6(LA-BehLM$;gENYIr$9 zHbFMHZMUPa%=vH_)7tXh-4_>560RRwW=kD*$+&HCOqQ#b5Q%V;1i!KtI1K1R(N5GV zc3jxAy+H~c5n(yv2QkNdGm!d?St-&)DjHZpTWHIK0t`6(YiC`7crXR{G>BNxC_RF= zqy?UvK8_!QPd21-9^n4?FAb^gp6JY6Sx)phl6m6Xab#@FIo;p?P}-ifoK&osj}lB$ z$k0eGXVoka&_ZN5nsgLQYs%=}*IhJxzr!k4c*3S8-Fh~naK6|eFFV7bQ?X_Z-*Br6 z@y91)R|8!HjtvZkU}a7=3~aA$wpx-yAIaEoi(-~Hb)Pdz=#9;3D*ota3?(65JPOu_ zdhPtTG6vj{5z|?CYa{gW0fkSQ4%{VL@)C|*M%32HrjC}TrVYKZy`O`1zywEi2R(ie zV3z&WySy-&G>A_kWSeU{e;%nwW;b)no;LoL42l#aI!CNoghbo)l-Ska2ctQbvw`z7 z%<88mZleG*q9Ty{0wDrig^cFI{K|moqk1xMBTX9l0yP3*O*etirXhMbFh!Z|+O7dLcaPMKZ$P!5V*5~kZ1C%tdJq(}U4c4*YXe4cfYCHUq= z1_y3}I>wu;KA+R+0BwI|RnxLTV?A;T*t0m9NWZz=q1B|9X`QQ*l*>zF#sX9Y(o;vA z5A=m?$QoUX_9C=#l)fN)i>Squf#9sf_M5=^gS>EbF995ihT8(_cIUTNU2-+NV0S#- z`g*&OCC5fl!?yb=lfT=Es4P?&rmwD^C4_yHEzv*>TXgN&o;mxK9%JfEOXFv)3F+CK1q>5l&#af@!xa_hd5^p8g*{(TVHdF7vwu>(2Qt=2|Ko(S{7ARR5F zRFh>Zy-J2BWekOa}G9fjz#=HnEYa1 z{EkDQxfMSfjdFv){Ey+}89%!sd!m$oHDu=V`1g#pj|KDXO`eu=ta!<^Q9_yV6e_6)FGs=Pxw)*yPkSR1f-T z#Ew=~wPdQ+!mJfuknwHap!sLW7L~rOWCUpxz(p}MORr;c2Vb%+G6sEUHKRdM43&pe z;xe~trV|m6FNESC+>TZ!fQBG+hm+b?w_fWx%@@?+k0=|)q)JVQklx3v}n_!vgTxf1w(iRU~l^KlR2B9=%%Jd-tqeO;G*K{ zc8R@%FZC@Nhp-fgWY1GrSvk-!AYQUBUMMdtlJuA?y-7@oEv1!}4(W83&d7hNJeZ>` z>0se!`1;J^bi>-0XJ6Y-@CzDIdEgy0OeHem7AIQ5MG;bzd_VPA1=C_2 z>ad%|wfbVP1wPQ!xz!BE^ib_ruF|JT;Uh?)_gc#Evt5yhH~n+5bQ79Q=ByVbV|nJ# zTwTkvK-awSN-D8DFp_)co4T$kx4YxSi6fB8RhF zX#ptHzbX~aXN(={GpryPSs}vdb&jJW*oVn4XY%w)XEK7R<5S@<1%=7cJ8Wc_neFe? zzA^D?*M`a{4w|V?{+hL--#+5yeZrYy$^Y1uq5Q>HSx1Lt5m9;5#6v4*cOxQv@sp<3 zFuAhNPD{y5w^r*h?A^#ISO&yZh*36_FKx9SJL$088iwSrIdJ`&MZsZAn5oC56(22A zq9j`0{F5he_o}9x7Tha(i(n?_$fwj+T5>I=cEZgjf>BtQF<~}y1z)&xYN(dtEJQ~u zGpptd#P5uQ8`6SvG6Lbl5EF#kVx4@w5J;xQDO;Rz8q1)&&Qi==6bHe9fyyjNdOjfB zBX(3XXZw>f9Lx1Ye>+kuQg09(Z5Ti!b-c7(#l_nP0{n^fIpV1dT#Fiwk0m@uHbgD6x)_&Li#g|3F_WE%M=2eP`3T}?=R$h+uYJLS-v(rpgI$hHI zvh+c%nrha-z!SyT%A!h0ZWI%_bGj0Jx1#0Eju78I1#Ct;70%LSNXx8sH=d~)xFBz( za3r~4p=DCrOyG`ZN+q5Y#ZkEG?LNQA^=DbbJzkgegQthDrg`kXZ)2<^Z{*hilYF=F z*!r#+^fOwkv(WvJlQx_bamqn-;Juu}C}$?`90FG!c2o@O7|+53JWH-y^D$B6$Y=_^ zE`XmUfxf)H7rADTMazWrYKY>YgYku$&_T~x5Z1onIvPDk{#36k;@KVmC39mq;(C&^sD%Kt-B2~pk{3{ zlE@hC&aSETvo*`G8DWMI8Jq$RW*HlP%Gw({ErG7ED{bJa4SSw{{h|5b_Y!7+g$hr0 zY%jp!UWhHqzUdl~VN+$XAJMAG(kDK>m7!;$8OF$`R%gAAS^fUe0zHbn41+=;O46Y( zRAF-qZ!H?TRfq^&eK-u+OwT{MmDYgkaf>QxY9CdA=;aKm4j!EFo>%>-nxbrzW6i-6 zM6-s80<~^r)Yj?c*Xa5LMR(=ClFnX>(b-5ce!=h32G;b z5p*Rzh&=Q%389_Bw9D*|D9}f1jp;4TW52BBxvd>1t<9s?i9y*Ijfx(>NX>vN#A1Qj;mw zi3SyWhPooi(9UE+pLV9pF-@jJz}d8ZhMjk2QGEYH=ePP%=6A?FPINvDMK4c7_l(AT z)FTv=wIDYR%ALw)k$Ph(3y|UVN=doX{HKGB`I9mI<7^N1_yXyD(^wF#e)NG4q;1K5 zcRx@iil#-(RXC|5(vX_+{3yVCSI*u`TG22)go$CEzlHvh7DtmwIEa)j}Y z&S2FJAf4N~Yf4k-d+DR8%B({vTcxxx%9I9C=Q%$c@mnVP0M$lV^$Ho0us9fyyf|Cp z{H2=vJZULOKFz{sKZ^SCE)8-$CR~6+Q+OudA81FGy~`@H>H5MV#NgNX z!9&xo>gB~LDN^@DLw|0#suy`Fso_{zL+pt-|8ctrET<}?W3D?Z)+M4g+I>eQQBB(V zvlHc*BF`+!$pnrx*g0q@h1`+v96>0ta0|EgHCw5NS($3{Wg?wdF+iPofi^y@E56;v zrfZ`&dqB*{jgiiA?Bk4xE%Y@PLTwpN$Nf3$dG~}@4V{_hOv&?UG!Dd@o2QkeL8~bI z5c=@ugls(feODAF#hS7ZILq zw4p#~`}|IT?@r2w+{8CA4bdt`BGTiiPDOWGp)QBW!<-HRN))eigE2L4t`%#Cj$Ukp z+CgNRMj%hREz?{MRnlX3m5X-xZP!S!OTFD48zt7-PU|BDXO6=@7s7!(WBSby=A}o<+o^FaSUh;TV@M`sG4N)gUwCwi8DJmws=LE3aa7 z3g(VIEF^>P>f8-~OY(ji_SO8VC`=I0tn-Oq z^%@-IrAx053q;81G(U*e6*1tD5mpHRb z?d}}>Z#FUfe|1*d@~1g>{oGHzmrm>wRBXEh-WF>6((!XfRqUlgDSUmyKyD)#Z>qlVo{&~d^g z@uXk3ki~d;Ip}o5F)q}Ot-;bwmf(nd5$-DBzukUx~#8E|tvJx3e?BDiv59)qg zZjCO<{HP)E&%6CUUgGa{keyd@AC-JioiE}1^?siJi<+`#qCZM0NIp1Q8$9U68hU!V zr9`m(=iL)YzEM&;QyNi^B|(p*_7`p&ZLs(@GZiwHr0{3c9++{NyZhmY&cbA9iL1*V4p7F^|NM|#RYlXEH#&VJt>478U=3`r z>m#68*5@yq3-_1Jav1-m1YcRn`e1K$Hu7k)xu$Ab*i$VWEv`FhY0Gf)`5LFs98H-2 zZ7Z#;HOvb6Y3OUc%$we+Pqzie9cQe%8P(>iwMqu4Ke}$ zvd*UFJ!jXOa6V9pX&E(i+zUm(TeO9cs%0UTQaKzXNKgQZYe$y!>az8#-p%FAn=GCShrBh}t*cM>kFRDp&f;2lFrAJG(3?4q=aRHaS*j+yQX0=zDxE~rv4b<1-bywc@ z2~Bx|+rDNSv%A)xAC-T$ki)DR2(B$|;lc?a3nXyvw!1mhq;SP52#nJ(ocbixTWC+D zU;51%Gv{9uZByb6uc_AmdgeW&+11Hh*rj@`9vWLc5wB`$`M7~+Lm`KF7WNO7tt11k zWEV7Spl(4usH)P)M4#<)Xp!(~y2(?xgQRt6;3Wp*L0MgiW|zIb0nS*h*y?m_NZ#19 z(pFeL!Dz+p@GH*0IkP|;yKWtupk8M*=tr1VT6|M!G=fl}5bSlbT7v$3cIuy z+BK-(e(yuGk1+aoVdip+s-TV@*oRl^ytI@~z42a(S_lqmg4$~9NF{aWbj=s6S%PUr zr3t`-&!Rof12t{BJEMT8?+wMuz_<%Kfaq}6QYPbE5o@>|;aN%mUqr#xK}-iTJVzn(j9j8Az90QFwHkN*Va`jn0)BL1T<<{@a%G*Dt`6tU9eghI z6$$-)E71ge$f3r)`r`3elUS~JcWU_};EJg~VJrqG+r=9;?C?8a*6V2e&G<3z?>PJ4 z$=chH#rT=pH#e`S=kP)^usq8^LpB2t{k<*<|1FmZrL1_BM5eOqSgv_Vp0lPrLq|&QT zHkD}yhGdn}uOHsww!YRg_zlXtP=x&2s+O1cBBFjobC^#TShA@Iy8Mcii>d|qgv8O9 z^F+IjVCLCNHQU$RT%1RkzmLz(02z@`?!Dp&^*P6091$PVe`KrgwEY1T2emDYo8Ubxl4>fIE{$7VCN9(tAxnkTl)ZUQE zM$B6$2U>n_w;sQ}$En1AekPw6U;J(^V%`1i^f!2WwZe7$JiIpk+o3{8<#!H;?@29)xGv# zJZfgloG^9%fK8FCPi7vo;qbvS5eE=pbWCzgK) zyi07Kw^i)W1~zQ!2NmQ)?zVoL{Y-xSgL8XZ$)xD$gVA^OzYWKJF`kqXY`O&|1K0^L zKQ!6>HoOsSe7dBCijXR54}*_>{0$Us+xXO@JL z+kr*9>2pP+T8F>Cl6y004O)tL1$$RvgK0pJEbC<1C)W(aN}JF4v+XNlKU_4=9~{#{ z=7Lw5BM0atZm}1VT+uQ*zD%Jx&LPgsY?;{uymh@e63HZ+l_cjbw@c%we#ox2MpSEy zaE5PATio-$>_tG4qXn)c(k#PDFY8@6`&n!zZPm=mQM11#Y>6h_Cx&7oV}Je8;=co1 ztiG#nf-GM>d8X^V36i>A6!QAb0fk-X!nxJU4?W}WvuB3BTz<57()3LcrJ5QDo+w_d zA>44&bO&Z&z-Zkb_@z8zG!T+LuK~5YMtM*-FnnMtLqq5j=kngcEK$dxk8k$btx58k z#B1@txAChmwyQ7h)v;lVJ-03VohGBm2d90EtUH!FJmdSB)qDjm!JvWg%~Kb+qz%L!dw!<(0Tcay6kY1ATN#VI@+i{i;jtAxFof7@4Xj zMtc>NTWM6C$FtEgOS$cLRX~UNTuQe)SyH#W38IiqY5Mj1r>nTeW@Ima>VrqK1su{R zj(4_2ohui;RkQ$XzC>!!7m@*{px5-{t-~ej;Z(u;f_yDplj%=7= zY{au?43r>vH0M>;0U^{Z2!AIRb^C4K+0PpgaEYJVu)S3g*(45AXZgnNAlrTa*f#0+ zv_W^{@GH+U9L{PXx|AzrDut0G+rtmh+kZ#JsMcKeUsX$Z2wow3Gy9FTWV8gi&Xp~^ zn`<^I_dd@`N}>oBbAS<)FEG1&cJGc^`(G+yLx#ut?!RYI=$2*sS0mHyu5f81Ur7#c z`D_KH7VCl!{S2^0H3A2b2TD1g4GVVp=9V0NTu^aDe?)3Uy!d~x_ugSmWox`JGuRbp z5D{rQ(n$cN1O&teA#@026H0&)q$M;%iB#K&)M&tfgrWurkdPoHK!U)CN|lm8K)MK_ z7e$I-`!eUuoI2-u?(==$z0Y&+J^Qb{*Iv7=wf5TYe&65w`)xwG8)K6<(MLMFD6?); zV^g=QvPA^ylHD+C;$(crsk)sz7&T?x5s{H8_V)<&RX@@*06^= z+J^TG+hkV^$9l{7ZKEBlmrScPUr|Y&K@yM4>K!7!Ob?$|U26u3NZOdI_<{VVd+T1s zkTDJdhKYTS+MYu^qPVkpXLVAJG`;LoYtK1@3!6+mRgtM0Xqa21W^o~z^XZ|Q1QmoX zeuIaS`tOoW9rAOM>LkiL`-4~HUDNBPoh0`8|(gW4Y4t@-TyI*Q`C zee7iCt-taw{K@d-jgiHmdSK;lyPtUDqmAcJVo)u_aJ}%~4 zfP2#e3SP6wK#wh6uoauhTKmDYMH=YBS?7p~2npA6(4gdNnm4I?RVjh1cQxC2m~{0h z3SY~avBLKE)`EJ3)5Ah|^MM0envN2azIQafw0(geJn-F^D{Z+d3g>^BDdCrad(l$; zhJhLD3?EHd2|m7%21#n5@GY&4P^*YnHB=pEMJ%g5VPG;U=V#%*Lk3sTi~&aoX&}lX zLo@MH((3}}0Y>xW^p#TX;W9U=f*2MN(`^QaYnhu}z9pkyu7Y(?0p3>@6lW)wbFDYxw8$&EGAJWXwk3X9tu94%$^w zM=}$JHJWaYtUnikm-Il%EbJoXGg%%8p?a{L0|T;-6_U~{sDcJS_ne1ZQqNWkU@ zwazJ@r5@NaTDztIdWbM#PjZ}}dNeln}YL4HmF)T;dA1;bKqy6qdLXNmwaN;@;=M0?rj5eTZRo(8^$rUl690DjSqsh|SI;y;rc1 zBS|Ijl3$>S!wr)^3$^%#e|>(H%pT72fMLlbgCunEgQ*yMP57%&q%@J5P_iyNt};@+ zdbd3{Rc-UdO+#1er+g!IYKwJ{H{edK$+7Kx>(g(g-&Xp0()mgXZh!78?yzRpZ4wFycmnc?K zRQy^9GTK}lnQB{(`*|7i{1rMjxlSbE?*|;5M4f3adR_PA_4WVDD_)Xvx_lFQxil=; zMm_41!?tw32tT~oZ=`PIE+K*T#>A5x#f$wJ{oAk)&v}gX51Zl*oz65bQF2{R%lKm1 zd29yV+l6<8YWB71R!DoJsF7wmndzdp3_e{W@@!m0}%;L@0FFQxkjphkWDANp9`XBk3cN;hbD) z{`b6~l7`vkMDqZW#c`*;@gpaOiSxWKMwg(!>*abT8)9qeMRSYC-X z=ZH@9(8whh9_cX9e(7=$fBbfS`lqF79y~N@94^jQIc|uazG;#a2$(pUuA%cE5-0Lj z^KM+`|6sCIRc$0qN~|2tc)cw(+|Rx2^bkSnJ?EL&J81OEY66JZQWG$!i;arRE2=Io)C>(EM=SsHEAvHLRh2dXyah{Hmj~hL)v{^&@K7y;nn#PWMxq)Y%!1BezHadt3 zxZQpWbZJ8jXal`GXMC3Can_ygl;O6F7#pC7iWfiTBckMEYmvMj?qCoEQk6i1TZ^dO zO_RAYfL3z7oAYefT{jRNkT@O4On3DkErDDPG3t4-BAl1_i`yIvv7S_~Q#mJc$R1i~ zna@-UCo^1cP2@&Ce_ZqOF#Sumc!vU(IX^-cJ*K%4j zRlsgY`^q*$!qR2?>Rk8|v6!D84Nyx;6J7{+lvM58)V(|4KJ}QCHfU)zRNaSY+R%EO ztYi0989mXC&!X0Sl>6Mu-oIEO(zyfa7u>j?K@V~pSa%Hbs6nPRn?*;Iep#n z=y0bIY9~p6Y{~~12t@Zz)XBS?_WSE07E>Kg8!U%;yK~5ECU?JN3>{q(V$SJ-4_A(# z)H9Prbbvv}cN``d2@DvT_^~;0aq*BTlde;>ZF*~k--HHJA0u2uuv~Y`z%C|g>0SfX zRe@_jmMSJ?MVZcNZL|u=D>FfcN1J?=ho7dc61@4r>9m6xPM{OT)gvX&V0#I<0^1fF z#drGn4&aQ6TYIBQb)5OunT9!Fb9p+QeSKpa$_K96&9zDl#XAg9frWPuN=@ETmxXT#3H>ek=R~^NxSHMe zRRMX3K-l8@kKcr@aT-@kF2!7HoE_gnh(v1rkXXqzeCK~4Yxj%6IoXdo`>uz}Wcj7I zgcn65%X~1%+s6VF2uTT3UM^)=rRe!ogl+EmnZaOcMv_ajgVnZKPDr__50&q&5s1dTEGrU12Sh)GSHDZzZmL_*p%WZGaFZTAXd_v#- zfvY)*+EZ3!4LF1#?trCb*54nieYb2^Woo^n*N~nBD=jhCkd&i{zo*JSpnaeL+8=Io z3y&`yQ9m+;OPAJ7^WS+>RJME5))sp z#*QXO6jWNZaY^jqef$LKl;Xw=Ooq7Gm=_9^@*v^EFzL zMbaQmW<*mQFfizh5L1~lF|qD!XV@5L8VOansym9D;#_sJyj-s--bbx_OmHnt$hmUA zzugzhyi#{P^3SF_fuP9!amnPZ-uIB^=Ilw$YX)&jwZB#TuDs;qtH!RzM20TpoMh(K zzh&qzukeOFk%+0$Vi?&&fVimMQhn*DNUxFYoC# zw||hAR6_k&u=Oq}DH$)|MZIUFw-%h*A1KT}wB^XdkKOl|c3-)<2}z?7&Vae-Ytv`1 z86CIX3UBmZ%B>DB%CB+}ZJn`qopjWNLfbJQDqj>gaVl$k%(&~%ALjQ`0iWJqT_2KK z*Ag4Kdhf0r5c%bI-$#S($eJJ=?XO1MGTWhFZX>Tz{iy<_<@~zJ%SoBi*>Foz= zg5RQoV2&8j7478jQg-<9vK0qdb24oZ8Z%o8E8dI6lE~wU#-MMp<#Nsf3B~^FUXR z(qVcN9XkFw%R!Y-ih0JJKLqt?Aa#7xk3pxZv|*(~{;^zXHqXtkOoWa%07d!8*~@$E zctXpW>9psYgc3wc+kKB7)V-4y)u!WMrNQKB)Hu%a#^O2f;249hFyg@y4AHHWM14s? zG)owaI7m-mZ?ttf#$%ZUF1umimGJ619yCMp#0{rymV9mx)Ag9!?I`jkmiy`Uyy5nf z;3zT5Mw}fZEYl#`d&bi364xNv%O4#PSoQb2pgoAi+UVlE3-!pX&#X_I=B-49k3O&+rm4u#)HgwwT85 z4Rv|&lNWDTa#*fKj+Rx-W*|Hn?sN=lic@zlRgw14G_ap4mCf8(R}WK#+7Qkvu`__n zNo|8*(uR_thZ!`um?k5pJ{uU6u6;={*-R&bv1JUi|5)5lDXF+%ml768 zcyAOZ^qQstE)8aL-(A)z$8IBITzoGFqQ34wXjf4}C-n@WA3f zxE1g%jl-m2bQgK!LhJg%B9p4L#DR7Ti-b3%iV3NLuparzfNULs5A$_jKXHMjcW(>A zw>9+(^SD44H*m1@p%zrQtMf|2sybYQvE5E-Ew+x$LPYLphdt58&5+QH*zu zIr92M@w5ibWmg}iqWyj#?MvNH9qA%*pJ_Z%QTPvf|9QR=rgB6inMgR#ttx)%Q+Qr( z+df?)J&i*4w(d+#dMcHxtdH~Blg-nlbLq#drRoea4CXPp+OG#o7F6An20bGL21l~& z1g3`kO$h3iac6Mtnpp{2ePDl*)ABN*tgwuV`$Fh^GdpEsRUSRNG+Wy=i6xDKmcbg! z@y~10#`jAfLi8=i8mCWK(mo~7p4Cq^9?rMy{;ai#VjM)Go;oWscgcz z$A<-1{r_T+ZPQT+>bV2wL%=nxlQkn zBmgg6ZOwrKkqjLQv9F^dRBXb)1Xg4%E_T$!xDCuQ;Y!Cp(6&%wnU7M;?&2TUTIZdH z(BGssQPJ-ZK)1MXbX#8Xh$o-XfeF)6bQ9r}$R_T_t) z#Og~er0!ELHPh`dyptaC60?D?r0lq32sDn>CnyLAKw75*NPw0GZm% z+`ppS&kBS;H;aK}=Y(^a1T&sPQ{W%=XVji)Ostc7q+*Z#Bbzet7w{v*kgkD+m(Rxme zFzfFt`yrmS=Gtg`<>gSP6arFpteKa9C7sxY&K*pC?Y|^`IN}pu79{6@y9f!LF!`vG zl$2$%;u{sxb?}^RvsMoeW1TF3Ug&tJLOh;-! zty>$!g-!P&blbC2mgD!V=eN!bXiRI*JHH>Z4!_OjEbm0F&?(E*>nc<@a(DEf(%T?v5*aunG3=M zd`wb5BP2Nmqw(dl*iV@6sxqO=_qumz6aZKVgfd-@#;_3&-*TIy#RW8q86%Tc;< z2(^>xn?};-Ku3gj4aIj`D$0n9GZ_UIS+X)Jtr0nznT7=qf{@Y`b(s-w4P3mv!@U)X zr#mdtF!`bq(b{>S?3yc|?(dQK6Hb9$T`k{)QrC`er$FM-7AzIY5G-ZFg^u->$=fL$ zw5NXL>%Qy(D$)nwGO6sxElXVZDIYdou{(AqvmHvNhg1>!cG&e&( zf`U|5fsO%$s6A+QS3%rq;-Dq;J{-njbO9}ezjj)h(9j%@H)`OvH1 zSDjqQJE>+alPH&;vu#B^3G>?wsPTH^o0JoO3o$&mw%g_iqYV~X9H7&2VErKB_%0t) zY7KmD0=mr+#b!Yd9ys<*$iJsUyv}+U2K7se&o^C(sMeaF8~Zw|Ksp0WnF>8CBM@3_yKVsR9DAY zW}{vLXk9`nO+ihdEzv6VBD?PRO~pKNfDbM84$J!Xg1yU`SNbh$@+3zEOin|uF+)$3 zX!)kgZd-`fV>=Fhu$d2BctjGGg=FCI^o>CU5?jK&is*{8f2A+pKDu4JIBEDUDb)lW z@|SV)+gSPYWp74shDz+2Ygv0P|AAk)jt~-RI5)l){F&YXp}60JQ0z#GV^p-m9+Vq=Zy84 zQ*9RP4HIu!vkQN0$#`)kxu1nzTRRQ_sm{i!~W zhiA@nZw4yXTMr{?ZE;YgjN0!m4|4TCrGFZ zF#d9H_WU$VzkO(AZd9w&%~$syKJ6yzq)pLON}E6)z=*(P52qI}nzbH4M|KrU8{|r}>;pEc7_hU58%GD^bJ9-kqQS9A^;0O{ zr`|rPh!(>~B|+`5X9`bfG6UuFFZx5nNdh`0jb;5`r9%E`3`wdsXiI!3`Yi_cdywug z`-Qz6-e+CQy(WhGBS^PP`_bd4J=!cHMD?6ujgq9;(WEw}dznuqv6Vhni;`P5(SNsV z0k(2wxPTD|dqzu@stAxgOCz){&YufS$91+CO?r;16$TL5b8s*QUiIM|oIpE)fw+`n zPK^d<-;@(Lhlw_J-|_lJ=~Yi0tYs>#oica@gAn>4e<>6Ih6bS~*tL*4jvj#_WqDv_ z$XI$k+hvGSknjLuDt8H27JD%4yVu|t+K>7hNdlPzT=?hSv_rRxz?tf5spfU zk>2z_3F!;v4A1$Eo=GY68k$q~)I>3`bt(xo{8=&ke`=b4-1r~y!R(q&V~@ixSlWiT z#H*{}Ln97gs#CCJnuLb6+iczD`-$4}%&Lq9EF+gz=G3;DGJeM_d1A%e+U!q3kwUV& zuKVr$M3g;QGux12SJ=K4f~+PHeej2)nD}TmR-Yau)uOR(BNX~(Mc;VJ(2}KAV?W?8 zZp4s6XGLhzcdo|Qw=aTL0?%;uOMY>iPk5-`JVXaFhYMw^$E_8;V=uodDx%zGcW1y@ zQq2&iVC}gPaPw0)OxW#4J|oaU%kyIp{neAY6=Uy0 z-{8sbozj|{x#jayo>cu-2EdD$CNq|I%Lz$$9_mRR%jph{Z@I|ix(4#}(A5Q{KJ6L1 z)Y8qlh%Oo&xtHj^s+un(GM(WEUAaK1hQ>U0k<^Hok7mj{I-bK=8j8Ma;{eT-fW-dW zHrpri|FPTuZ2kKH-0=NV9H?wl3Bj(tl3QM>FKsJF1vTh&L)+p`a}if4+%)e%QtlLSVThB-F0pzJ^MK zp+C@mS{HTHaSMy>qJ>a_yj>0! zH0?%zG121I(Z*<(6d$R&ba?HX_(?V-E?ln)J}T8{kUgYwl;2y&gr+S7k#Goug)^Z@ z$8Lo*1VZ83>g!GYc7aX2Hkz~f{uhYu@XOe&4`;{S94>B=!w#`YzNQtXE5Ed$sou14 z=m=%upHKTEr5F4Ks@_0y<+od4LWdO(YtRVJR@0v)OtYSg8@4}qcJq$8Mt{r2dBeg$ z@Z(&xrxWN{r#Wz{LT|qS43Sux{VvcjA35U?4VC54K9tmxZ&1s)V0u5Ag$W8LV*A04 zg&(?RO54Y8F{t%wxXNFfhj}kEO;On%h~Z?Jfr@gah(U(UzPnv;hrPGEua0b*mUU4! zl72s@Fx(-I)eln$mylu&%Dc2wn~zL|x=`s*sM~B3wK#XNgg~;snpCo`#vafrTo>iB zefsK;%?;8^Kmzp;f&Xz?wY1i4y8E9WS_(M^o?^BH#D-ha%Udh8QhW6;VR)w)roMXw zFI}_Q9n0SveL2-5*LvyjK|k4(f9=`)zn;iW|4~@|Umq7g%iHQ3U+I@6nfHsny?c zJ6y@%!!Lf~x)I>g2!9auh(gBWOJR2TLwHZ{6Ff3_qL^Vjj}ekJt|H+l&7Sm|Z2NZD zb{51cSU-nX$WJ1IDtsG~vTXJgTyErYK7u%g7 zkU&(qoSRmjPOU)e6LhUS^)JDg4kChSP$ubtfKS(O<|m<%p($7J8N$4#@5Vmu8S#UcD{y(>*Gr`wO*{rpO&ZXr zeS)?ZnZqS;U7>Fd-X2Iox{llp-AH%k5?Y)2P8${j0*$Y1VyQ`YG?;az!E%kARpSG+ z9KfSd^BD0m%r$;{iiyoOMh=1tnRvlJFKaSnZmGa&np28ZYVsp4=EsYpTXS5c#YL~` z=|_AMdXR{WDIyK4YfEcXKDw!ZgotXr1W+qmoj(dw?^-^(Ifrz7T|iK*HFoT;7ki_r z0EJH8o}-*5Mi3kf{BqiIW=lMC7tvg(4W6)MDktw{b4C!Dh|;@>+dBWxllU%+-(j|s zUe@m)H0GWYL;a<~Yop=ILzK0M^{vc?_@``rk9~Z+W7##?cZ#=uIGZ^et-EnB_GHgS z;G*!UQy=Ey?o5S8y35>gh{=U=}AYmOD*sr_s-Gif%YmLT(HBUnS z+H#VA27vnE(Yf;p{lVPs3-T6m5e%2y>5+`%mg?k76R zrg1p(XUdZv-ZN_%OY6Ymo3e${qI46p5Qa%`q)hEusXw4byRR^gc{wb)z1}^#LD%AI z+XOxp1O)YSgJUYM;kjvSC;t+D8NRvy)$44rV>#$>csEM~Ge1slp^~lab@uT{auQSe6_x1_rPTKNv(T4Qy&3*=!>ubfr7VuJ4)`hqs*fEUjK8;M zD<nGX?;*noU;!3D(j_)h*2WAgr{ATpSH9kT+?f= zX-iu0qHp;JN{E|4$y9{R-inBo+M>U;@hlJ*5C3{M=D{knO{BU3oYdzBOj;~Z-$WTH zC0P>1jmpK_#x0yiH4Hlt6dOC$Apf8FhxSn|G1p8VRPC+sD>S}TWZ^#h!7P&Yh#=;M zIYM(&fVz2iaE1v*Gk5>E43Z@wj@HWp&}6>YN}S?zrj}NNYyeo zG15_gF0sqi0dU+a3K_Ly%t#~J>!h}a2^2x@JZZEQkck*QUN;u0oSD9%#<;`cI6J#n zjaOx@HXi!00eFXt4$V*T(4gZv$$o+cra=K}Hb>W{DfT%_Le|aQ8=&P(US5n7ioP=! ze8G+}gGTM8J>s6%$;U)nfw5)`)K$or9PlQ%f z>7Ey+EZVtvw^_mBX6W7+Zjm62*R#hJ?QZrbo)S?(IO^N7{ z*OSq*DHHOZW#R&^_Lv(hy~$5W0&_J#Gduccrpw#XfHcU^d-f4^1C1)zBTOvAWa5{u zC&!F~IY~5PT68JaWXKQWKP!VrVgzYlcr;l~1Aa)^x3T_Ik~>=>I)h!Efgg0nx|n;% zoBPdqM{7%wjpFu!+nXuJhPbEbvw1#YorfL zsiwBSB74InPB(#y93pyEil?D9W1guJHVfd~CneVJ<67*2rq4U)CPpN#9`mpz>-nSA zMZ1k1j79n#Vdf4Fw(_2t&aI?Q1e!+Bdbm?OW=}!V(s$hbarlT zlcZGk3hQF*o*8E?7f<5b@N3*@{Nln`FG+baQ8i6j3xZWkEN%IG7w`byN}m)$n8L>h+Vi1_Awco!gWoP z%jkB0d^K;}d1$$2lpVeMuLjA#e^&p`te}X0P2u6>aYbGAQxHDaVk`3`mK8m?a^(77 z`idy7?eZe5n98uq7rj!~^a#3R5jrKgfb)Cd9@OnxcmC$y*U6oO|K0wR_PCH&6LBN( zPhmSx7^TEu>+S$qaU0c4ro=KJ_`MRzz57i_diLD<(H)Nt-Z2gP{+kfxM{5XWjCtzC zTB7@$^CA8DEK`X4RDV>_at-*yciqBXXwCw0`GYJaDam=+Hmdo4>xhxKHa+KEvjnKN zSwg@O*%LjM0JT&fO3pTRoT<+9v6lHwC7|Dtg-2kO>Tu*qeH)cpkstM>^vHUF8pV0i zfbwZ4`Jqnvsmt&eBa<4@JgL!z)P<)*+G)AqST1>VBvR7ZdOyeqZ!Yf%70VQfCZZBj zw|KZKsGnpW7$(Ey4QwOWqf1Zw_Zf6}A3vK#1T>omOlx_L`fk)I)Mo`ZkA0NMlyJ3F zs!JfknefTTk*Y5~*vz-?X#928Pf;gTK4;p^?g&p=PJK(I?8~&TFki&jni~%BG3(0C zNRr$#@n$ZRCOR;$TiY2cFB26xN|#<1XOHp7>HR*GDoYC01!1Fl_OxzTwmu#j|4 zDRB zF~|BRu+wDnCG2jUgPH5W1&tG3%bOsZ=>S%9uN&!;+N6JFR|DnRH9=abZnX_8?B+}W zi4R!K?+*=RO6b1gdyUOb-d=?`8eNVSFvCEOmEhC7&Ky2%7lvu;C+NYVYpYudYrJa) zhO|Ara{_fE2l~2K<|k+5d49lr^24QBH)$3Bx70jnbhz|kQk|ZB7_*d4fKQ9xcvedN@CLj@vq$&h7YUy6cgk+HT7$TNi2Ey7%__ zck1qS^yU#)h;d^1L@B(-L2=qsoO^nw(x;2j*L`*!Ti`|(jQiP!{I~3XbsK0s698mq z6$P*VTjlV%Gu+@RA6b!Go^`!# zTleFI248;@df#yCn-Dzy$=}znTNrtruj=YPa*H##09tY!s*7!Rd6BUYoKaGtW%?p3 z*(W(ssyrk@6z=ZzmR(&K#MK3kM%jHcLqUUjNvJ z1`kW`qUHk*`=SmK>|Nd2mwYqk6sh|HrK)-wDAjuUJ*{mgp8H&mYUj1f>C^iisgrR( zE z&DKHAE5i&cZ>6ML@hvQ(y^5F2t*t(Cz{PcjGuYp8>;a!qxIJjIOO|sb9;291Ast z=-AY-`|#uLCzMRTnJ_U+!^&?$7|Y$*Z$i&Mk6BZOvqDiA^y0?3&4>Gq7BAoX0^&qv z*H4;{7DP)jT=XHVYCJ}(L;NBq0FKA%Irx_L@v!>HXlz0PHBU`wWB8$JL{hfgh$@(l zVBRg)>fe@dnTic!I}{xB83>D3Gyz~3Z&5e|8=r0-*3gEX+K?5!IU_&nmjp`CIGDPf zxfjb%6SsErVP+!(c&!N!(oYXqMf&}x5ij&NsD}2@pl?FT+$iU3dw%Wvt-3I$BvK}W z8QdK|JK!)}`eaO%A9mEGzCcyCP0W>C{2~j$t=z^1kM8_T7^4;zCJjg-Tem?jCF|p| zhL;3dRA!H1MzeRLv#3+(uuM?G=%DbO-OvkdV+0f(A_0o_>d)9okhx1pAq?xhA`w{d zl|Jib9+bE8CKEd~&Bn8w_h*pf4KQIJ`dKB?o-Va=0)HzqEGHXkVuGS-y8U8;+O2Sq zveL#I8!yOhE4qO>f~Jzt<^-1)5543cYz9Y2ra}Rj#?<)peHS^Yd!NoFQxEaX1_y1a zX&q|spGgDl#mx(?(>2cR$?-Gb+f6D`r^42rX89X=H&j#@T9t^)3w)qv8!(M3^DKDz zZ}<0Q3E}4cbzkqQe*xsM4)C-~Wez>Q-&uiaSMHR?l9rxAH@fKR1E)60kkA(gQL@=v znx*-C$MlO}n(EtQ^3K8!9lL~ z8O1Snu*2r|+p)1Y^HggA0bdudiMrf`QNg}hm}h~whNyd+rX$-DH z&r-k7WW#;*&?f_4q2*6&%vrj%vt4+O`*cQD9}q6q+uRj;WRxboj&xCoVx5igm0cgp zI%sw*>+#cj(1Of5YZ2?1cLf=<4B{)wEeu#LBLXHZ9~5Qd*e19E{23dT#wW~ z1uoL1OoZLy9{{lmb*3^XG1%#_20(Ne_X`=mTmH5+aK5s zq2m8o{U3K6xc_M;)v$dE`ah83FiQ$Z$ho_Ut{tO=eiLFm@9Ju5X`d$gEs;pQ#v!*e zw!NdCAE?s#fX3*f7QwG+^~k2(Pq3jQ3jqd_Nax^iuHRZz14D#q%=sw>YlMXB3AAM# zrh)cgh$H}bCa%ci@ zzQ=w*Xqh&*7RPC&W9&O@LrQO%71hAP0=h6`C5LEAP@4_So{xr=FKuqwhFyaNZjRXn zBa;K6?>9PbA2{_=F(3t5ZN@g+qySfa&)}4!l~Y~U`qwd4DD@D>Z$huZJP-JRaxFwO z@e~6Z1!BJT_+{i|m(Nq|t!s@Hi|yNA>VhhvMxFs)LA>{rqje-wtFy!#Q-tcdqT#JPQgB z=jqBB6({ieO!E!f#>U5$$MvnaBC7fE@gG8Bz3VFrR*hw@yr`(V$?C$uQHwzx6}!#b zC6vR?2gyZ48hXHn?yvZHwCvJ^J>?4*ippnpHVnu4|gpVfVFp2F6E^LujqfYVnr|f z@+LlQq5!#{&KL}&77y7CyKHcqQZV*9(aeX>L;(QD$3BXtBsx(e&(&aa1Cq8oWzz5s zi3HR%CMWR_8|wQE>YgrMzr2N4KT+i`{=qyqu-LmRDsT@izIt{j@TQ{;i4{i?y$VtS zM3KjK7`JMm$o?X4M*(B!kYNV^W_!5P$qPz&;_IC$ADY3(&49<-=oND!&tVyLtaX|3 zJZ>8l-h*LI#6gC+(B_cr-u)kL%*EWX8wxVeD+(D0%LZ4`)xlKI(NSMWq5+7keESOZ zp)<#EHa+a_ed$iwRnS!(94s)1%Cx&+|AgCNL+wlbB+z(3VlYN?^?>Hs8H>>Zz5Uh# zntDrkMFC5bz=IX@h9Hc7w6}xmZL8&H)sOhv=7FiziV)5#%Yc(blvi+;tJr`>YCGL6 zB38m%>}kU8$BwJ9WSseYiMq4|3RB8;w7+2Pv>kr|*w1_@EmJ!XKp;;#i^Ji#96gZK zxnPdrs4NMu==v^YHJy!QI*K*jW4q>M$ZF%5O zHIXn8rEY=|7tJ0?>NDx85hFIZgLL9QxP{WN29ye1w_K5kFNiuhJ+|Y=fBY;6)(=1Z zT<$hE^|{^n?=mLWwe5j6s4n-6?>+&ah04@B)nC4=T%LJ{7-Jc|{~ZB5{p7zzf5k%o zuWN}_^}rZnmrn&G#q(0BAXQS}S1wKYOMkAro&;Af1l^qtJWxfnC5<~KMnuDFOA0(+ z5z4bS>wE#b&W~2;E8~f4F%?6Fr41EV1zsT1{LuVfLjo*>aIZXD=_0*&ws|VGd}0Y` zA`27^Ue(<>;I?-@ugzgX_cWoE_s$!Y-GWs;vm9JmB>0?hSBGsC^Yr@8Mqyrp7u=*L ztLzYGAN!`6i*~@}A&(JZ446Uy0`Vrbt&HpLs6J%r`?$v1SWVNVWNGD5#?hgHHV2V2 zr}_f7-IVI(XB(rK@}8Faeg3;BGrr`KXIM%$>u?@UZ?Jb<_K&N{eyO=%FB6Y}w)+KfM$}HyrVl5U=!C0f zI4j-_)M7C(q&ab!;uZ{9`TL#YHj5Cpq^%jD-*HT?sPLu)V1Kd2LCu9dzLBASTnC+s zz$o}$5Eqs0v3mdIj|=?mapt^xF~CXn!$V$XLv6E%Ry&^FvlWvkHr5QWbtjf#UN`jt z?Up^>wB8O&9nWYe7~|c1ck`HP;cdc#k;a14U99{lm6yDy7I42iXU>P{s+0EofZ9yRfCCCW zlcaZ%mt^J>?U8xTLp>VMq^2TJ@2#ZVr|cs~#(iyLaKB&TP1G*@9OLe@!fo})+CCna zZ5XSlMnvqJB8?jv4oaN^#Oq8AjuOPRo1o1=t;<}Qif;5PwyOv13{gCM?UYM4{ESFE znm40*hG^>rz)!t)KDLh3s045eUfp#B^O?}tT%-3RNw$XBi*qYCig%6w@koE`C8_Iy zpZ>3|xY}0AS-c!3evq(;u6I_5$}$vo)YT6qvwj zXG$cIejJq zxiOHzLoD%5D}A_24sOCh$w`MaC>z5}mMPIL#3PUnElN{pxpGKu{EBV7fJIG7FTR|u z?$D_+?>-$s&5|E_J=S7IO5l%;UiJ`(t3@x2T(pE>C}|2sp#^wdi)ki2>LibccsJu# z+w#lkIWxZ_WPK7*G7b+Ttz`a0_ftnraUo9bY+2=DsfWS7!vAj14@)QR|9; zT*fQwze1n9Q;<;Cc;%5l!x7zKQJHI!0^>_f*Ox&NZ?gp0NUbaz)pTtgG+!ZXB~Z7* zW?C~9#&pa4e4s`jZTOV-i@nV~_s6vtM2ybdjj-i-UX=*l1EkHLDZ;XsW}DAT-J`D>Gw^Ei4n;?KGQ|V_suMf{5YrM z!bdu&soIn}6_^zu^-J!Y++jt+ zLE{AxL!)VkLWK2AzoJx6FTlOza6l+h!zwuS)Y50<6-E&oPXAg_LDpmM%?0Pb@>}<0 z0WMU5La^7bw7aDTH2p@K(Iq9r-zyO0QR`tT7*W}GNH-)h4i`K=rb5!kI?j-cBm$nu z18|uNIWic|YNvu8vuo$XD;9)Hyj_Yfg`NO1Kvy~8lLNuO!sNq&<9UWe&Bnwd>hhGL zl1DLDY-UU5m7=O)WR}wxgiU6uK-hWm`cLNh$fNW5MPYjx8?>P*Vp1pKEqjY+g74J(DK7DUmOE zBEhIWQ>CRKaNc{;Lo@n;yvt$QigK+jA7XFGw4HhZZFYM=1jMmClpGm%?;!T(phGBO zu_#rhG^lc9hM9`?l^)@nSkET}sOtEb_bUlV+S=z6{l}%Vh@RyM{FToizs7^bV`ast zj)gSF2BqElmanM>j*my6`GyNQS4DWH8FscI-WRnV(=Cdu=z}!|Wr?M@-E!XIbx-wI zzUzVyXH=wB_|(JwhX&+DSp)r1ddDLb*(XDzx)LojFY>*ztAc>mEJw0$!IX_k7Gfo< z3TJlhIfI^YM4__2)XdRTjwyZ&-%h~=LEo*cXIof_feu=ocz=L;F$#3817T&jy`a2~l0XG8GwyrxE)w#2E=dE- zk615t6llUhm4(Yr4B|lZ5donDjQd*RCsi?L)z|A$a+7bUtl`QrPth>^5;XKO#wP7- zZ2q{y>iJfnwYCYQ3CKLfF3K^^bN+QAE&Q|SXOG{m>mDpP_fuT*?8|+qvdo>){w0on z6EK?7OkI&YsehsryMlNVI@5FfkqJg`&i4_a@SGu1J=8u|+yO(9!s8v{zDhs+DRNvT z=8iv3F=O6kNv??b+}c5-8S<#XZh+->+ij_c1cRJPu7``woE`?K6`#WLtB2=~0yrpyC`BXvP;83EP*uu|ukh0Q9`9dkUHlhakx;M~~aZ|u;f)(xYq z;2nU;ug7=XTEV-W=zU#N0z1`rC3RY?JL!tq8ENXIpEOrMgY$~##OHoF zf9}j0^5V&;fJ|aV4;}5i)ayno2TPV0Cc2~v@&)>vD{*=t)<{y&me!Q3<<`&3>{&WH zuiDTA-t7i)YbGVB9NnDPL~SP?6|f?hG`&ZhyHLd^VQjvlevVIz*D#iB<5~D2`g(r7 z&q(wvatw8rA}vQ%qAOP1W=SJQV`jDwz4qk>*+hm{_xc%b!AKZ@hBWoc^E*;0adf2q zG-`lCR_eu4!cWyz>`p10oh(|Ql#s8u_)*Ku&)1ksn@5uK@}v-0Cbz@BE#oLD>46)L zqBZhap)1pZb%I;Jmy;ZH&wYtw=dx_C78V>HZdh1sf~0T=x=?~AkUE<-Kd+&ukG(T+ zCTp~+Cy>E2Lb6t2p}N%-xi!X_c-UwoO|wf@BjO59-f^| z)_T`kd+m3%xBOnWz`A^9L-H7|;1q$RZb%-8*H&QmWA>9>kl(-=F7!WLdQwN@4jmL; zU^S+So95q*U97v2(W{niBpIqsh<%JChor^!MbtIF=ZUYWQQ|0WBy_og^0Z8i^%ZCyBeb3A8Fm<4AZdBijVJ?(@#pVxWxBt>OK%yk>S#f&Vnve zi)mLqdBVYEA{8w1 zo7b-K;rTrMZf)ZvkKB4aW|D!1Q}|p$TYih93ohNP6QRqHi6&h$H*DF1e{zm~RUyK0 z?Z*7+33fQrI#9vwDmEma%${6Ct6{Qwe^?c&bHx-pGIn{T&r8UNd#(-Yl9mxm>`w0V zG!0g?((@;v*O?tSSW4oeEn|H~t#Yp{3E!~$7JhgX^ zpgP~5>XGFIgXPRfq-fP6a=-7w@7@2VAs%IlJhJ%_bY6T*c31Ol(0@ww{b#{_PVAe* zBA5{HObdpjON^_U?OOz|YoitO9Q>2pZ6)vD4Xo|DEZ#$8@#qV;;8@$L_SDS85+ra6 z@80iv%sSigL-AZg*5S?aksRHqIgXNtz&v#A`0lMx_gi)YAFj`D`Zt{+Q~L3XLlyJ8 zpIh|+($B{=XBTRR4Q!&J-Oh;XE?>$Q(q6?5RN+2?iX6iTn`#TAwO;-EZdceuBf6d8 z*IkCzSbihjf5tLbhj*qBTq+BcUoUT&=RD8pwe|c{>)U|VzZ)%?=a8sk;I~i4{=T{AIMRRXbMc|>6?OytXBR#-UM0_YThYt+Q{(L)0C`yb0dQ7V z^KYae)V@vWO0g0QxylbHKP+WaZGNQoR{ws5mPVGrM7>wZlGZ!R@`>FUsALQz6DjA# z%WG};H1|7(N&COSK<YDBmZ zVFc{odK7r>JOep&wYeIqnt)SOhvS5tHv@lHDrLtsoYB&r5)io7)+v!*W;PlB=!t8t z7*1VvwG3xOC5Ia>?0V`JF^`l(G?XF?pMqy=s@*eCOzd5+=&HI)4#I#w{-1q*j2vBh zfK2gvlibAC&Yr+hTIH5(rLV_qV{=!%7fKSC#A|!M&X(HR#iHQj3v=%-eg)p1Pam6m2ZB(zHUuEtI1{+ zt&o6xo9hGFgDS;7=I4cc&}U-D79*hm9c z&DfaxGyQ!1caVjo+FLDQ2F4YP!vGV;UtchIGVfs-+cewh^`-a&u=5R0?E|iv88`-Y zZRGXTvNG;kbn;?3QX;+mJwvX7a4p6Kwti1JMh{5{>zTGoo96W}i+H9>4-rTSg~5nT z@X3BTJ~UW~Yx#;-tltD;J!7y*-Hr62%f%*1eJ}2RfB1cW{5EC3+5XZq7m`8)*~SxP ze`yNGLPH=J3SdfXi(C1G1M+>Kn*W`{K27je2|T^Bx40F`M{;$*Gli#0WfwMYKVy^m zw#XdoAu@UNMF30B=Q;AfXL)i*KOzO<&|a<2{sL%T$kYQ}TozRJ)&;hR4+egPP}z^5 zbE5xjDg)AY;r^0gk)^8K-n0%~Rq1h-$%f(* zu;BXvfs*`*zEgQh9U7=a)>;8UxXf$>cXqjAR+U49tk_m#uxAB^T(6T~u*LDr{xCFG zO)Pd7f-qkMgDMQ5(h?U$tDk8-YO*(}A^+_2Mtz1a!a$+5RVBjwNSo-1pbK*wdIRD4 zcA|p~eFI9Os@J-Vj1Y6wQc<6yssWjpLKC%Y79(gSFY!V>Pu-7+g*sDo#Ay-HL40tr zgd=)5G=`oY-zlJN@oYB4WXu?{0*Awq3OF}d-f;l_YeQQm#B|8HNK!@%P>HK)a&#Dn zO3WfquhT!ne?3vd7&qb`8t3t-=h=)}ItStj7aUG|K)ie$Y3|_=5ueavK|Kz3N~ll` zwxK4N=GI#@Nj)MAggBp%f=T;V>0NNkADEntjGoIj?8$T~Y%5&NX_y?biUy+HqU?*+4?EfM4{5B+W9&<-_b2FCuJV$` zS^A+ox+B_R687ZL59$Blv>VW$GxT#lq4Qd1ulemr`x zc6#^iKT);Bpx-45eaX-IWSIC4zW8$v1)go$)?3qOy4cdeWB53GZm9=o0Ez>1{n+6m z-mlm+@4spU7X@~Ud#T(duGBB!=5OtT&4dl@zWCQNX)pIutp%jG<2fI#)Y6e)@76=# z;EJp*)-(V|b0B07m^`HaXCCQKJoQeN?_r==?Iiw#?_KUUy37Q1ffs#u>p!S+C+Wn(&TIM|3_M8wFsPMr4J>cz z(&l^OsdmKDaF!DU+{Afm#es9p9dKRmhv3Yo<3STI1~CuoJfkqi&MB03u#cHEO>!1x z!Z;2XyuU^a_TET#Gx(u|#-!|q%{L*77iuXumB(r|Y}Ci->A@I1o}3QGKpvyLXb3oy z@-^OjNgZXo%a$CH4KC&RCo*U!i>hnyW%;#pc6pQHuIB@T|}4W0naKm0au z61o5jT`DkP$<$UVcF6Y>K*>mFi|A|0YsJ3qa(+9GR0x5c_>`3rK3P_E!6jeDhw!tt zhrEQ96T}c9LCj@eeWMZoR>3XoE`{|}k)h!p!(HyK#Q;7M#9|E$$y9A1WnE`H-%eX{ zO*WdClJ@8tFX)^cB$*3~yu~Hr6U_OpswL~oaj!X0nM__{@YV<>Cd0S4cl!1}|NY&A zYTchGclYFSMp300xoAu4s-zSHIlQaCBy3vZ&R>8@ldK4I>G~98e)6hheS%d3ue)j0 z^nYTV-Pl9_0c*2$ntoI|bpkz3K^x#nPHq?AZ4E9%RbKbkKZKlD&F_#ap5iz$;8kXD zR+lFbf9s<1)PI%8QJH@pCX~omD;ru(Hc!Vo^z4Q6{1(jX^?TAfpbo5n|7W@dd4DDP08ud1dWs(lb)#|dmYsRz z=jA0CVS=3qn+Cj1FGshSd5L}bmq>@*MJ?aifTf=9F!5R%-76g~fjQBvOEUO{Nc7*E z;;zjipBr1(+3iM)LqvHmsCErky}i}yQ=EA(9Fc7zwo>Q?hvz6n2Fwap^u-h++{43j zMjdOfaEK9N-x4Wwb>Zc}-YEpr@f;7}5W9C^Yb3JaQa{egaSn6M@D9_hvpj?@ZRrg; zk8%5A6(#e{h~xGGRA&~m#J*EkvOLP@hy9FVtwTccZQa^a_z+tp_b8-L6s9oMv9(-B z&c4(RniB*aLN*8-rndgli_S~#H3d`qPWnJb(ei2-V(uop5j_lG6dTjl( z-p$l+ue!&?(zWnHokt)2HY$BHm&h)p{`4TLx_P0mtwFyaP46yQU9EfOI=WQo7O`_f zPcx-IU%PIF&wAWG%5;xe!ztd-vPh2th_#8063eyR*@D2A%@m54@cLYTte(OXK<6Cc zloJsIR7_h;{^1E=>c{g`F|pPyB`88Gkm1CWMn?D8fT-vjr|yQ@qX;I_T|GHZIRn@` zosJT0qI;b!{4cIcMK*~M+br8FH`N+vKRHb{nmN2;ZTCI>#M z2Ar6mFGcoFnS778);CnU?75aTDl|9dq&>{laOFd2-@BQ!=uC%9y%|-CMS1#6_fZ8T+CHou{#n}h|$T>E3Sfm)|xSvGW zavxPX(MtueeK>>cYfnrFt(I?MP(Z+&Xhx)kk%-X;3v0JGj(h~!?T;A{zm*&rW%ps} z0#|6&wTQ@Ru=n*hRw5$2?vzQuw$>*Whro~~{Fz;s|JX|UPfUD&ljUDukAK@WSqIBE z-fw|=iFXozmNfW^E&0#mq}l9jn(YZ8B>8A zGvYd;1DU#2;`_YNpXtQ<3BW(j#2uV#DaIld+g4 zVLw2`_;B2}C(pN4gAglJv*Cz$iUG@AdcRb0kYdSA|>sk9MEWCb`t7|8Y8Ntax3kw56|OWOQp zgW+v<4uPe~$D&ryWL1fj>4P{cM@lRIN%~964@ljsQ}4WRX3vq&H|JjsFEOqB^a@n< z<>pHVNrBG%{%H@@YKyTHB@5 zSY#v?79l@c6pq~hfbJqMWXUlMg^bhRZ5x9+{_ZvY%Rl_yd3x3^YK!c=Ls@{bmKZ4$(y?ckx{_N3eoJold z?VLsylkfrH8>=)-cZN1EudF}mn#D&DOFbbM4mI9KYtJ{E6h3|lh~nf~!mOeUL^w>E zo;tlc*$3r+muq3_t=px?7sEVm96uNqHcTWr4+o~jHAptWSQ-HAEt5Kwm9Cm0jd1xY zRVsmi6o*z$k6_YQcGZ3NTp}XGuyF3l$_WQpMA!qbB)yHtdq{a7p0KWbFkC&%G_Iv| zV63aI&)PucrRSV*=wx`P>VyP+Mw=5IY=))=SG|5~3l5~}2#>Bl3460$5f}#AxVzGz zCzyk1yL39Ley&%+wKX=5x2sW8D#134*_+VeE%2H7>^ zYVn=&7X}OqCY_qwqpl050%i~-QUaxh-;HUC9fE))cHv1C7~ZGLt$U`pb>*(KnHoKO zo#$O%;4eJUrbSkQBASOpVWONf*L|%VW3XuIyRt|98Fq6BA0XV)5t;EuovFNnb?dvh z5$9rk2N!LW!>pK70t|J$MTOS@aBuw zG{O07diAGkk$Sw&T{tLgGM##9u5T&YaEMvQy(U$gdsQg$uCgP_a)ls`{wMXY`P5*|k@8CJ@`B}lV9NUXGA)QLT%Rste zd0R)EYUk>-t>lC!YNUjVj#L`N+*SeKA<-zqfYh~BQSLV7hfVUVq<1RFI2_L zA5U|{K@$qej_*2hWlu~5A-(AY?`g4xn1<%;)|}d_*mU`V1MMvc{?o?O=IjMB=tAS@ zW-}_%r9TGt{94k~{OPC3ZHPd54cnN)gCkC!xgmh%bs%mOJ6U@67Q!IZBM4{45Ax`rcpO|~z*fT?+_~=`S8A!J7@p4r zW6rn6)~zbARpQ5w#c29cl0vf=4Np6hQG_S>z+i`>dBXu9Cinf@pq^Y!XNgufnNHqc z9$yOabm~?UKTdd;fwkO!O*N@V&oQmffXcpwKoeqd4zZQv0Xi^b~OV+TuBH9qP_6ETc z|2EMc5~VFBVPKk8QRSWoiEv+8lfTY?)58h7DB>{~@Bn9|0#1ut@na{PTS~m45*0=v zB26=UvB*Xp+qTdx z#(gFw{ zxM3UOI=sCaMSIU%q90r(Mq~zC+I8O4@gtPl%_@BT zck-TjUq^u&n)e$7+@+ATdSlRnhVD63i*Yp|(9`vKSpC+abN~2(841lfBT{YVjVr>B z>Rwr6NC6KerRpE@wwtgKx}2oT zrw$LGDS1f$u39b@B`GFhrmVY{5Y29H(=m9tx<_5sQjf3S-ZL#KIWBZ=9q|#wBhd@J zfJb<{vczeClM`{-^l&c|_k@VN#&ph5u*Ot!Ot7lAI}q^-lm1Y99oh;z-HtQNlr}+d z?MvJu7_l>2Pme?OL*&sX(vE73<4z6nv3MTMXE^~NExOPQU6)VnnZ7@+a&EI=(BM{e zG?jQC*jB2%;H)9IG?#?+5112)Rj~kUwXJ<74oA#h`W#v3^J`%+`VcIh8oX3znBr2m z^eKwc>D%_V|M4693#tG5Q34R{U13G^e-`aMKr}cwo7ev}t^nkDGwuer=n`z7P7+M= z|Mru_K@fNPQ^g3ol#Z~2(k)u_h6?qcJw{f&z0n-(Nb=xoV{qBL$)kLOBQ;}QAjJ&s zIJYexdPJ|i^RvlmXo`fnQ@VFSzdh%#FCYvvye+(qq+bb~wb`3cRfp*Gh<(aW{_`CF|a^z@K{SKmRM6#^_6>$Wv&AVLOkQYrfdC=+U`&Owjubm-#?fb zIB)sa_WL^<0@P;#AD;f76Yzaoa;r>sj;oiC9&u`okyg;6?)eB}7^E-tJ?%De5)7RT z>9-Rh$+XHxo?WU=y{k51Or|z;yu8hGX|h6G{;*?(kzE1=q2O-9#SpAI*S*D?Uv4Lm z5HV%RPdnB&QwI3KrJy0j^JvEi__Z{p{8#E@|PwMjI#h{liLJU=ilnM zn|}EL4(1zY;F!*z8GP2T(jD$+3-Y~DuW~9ra_5+vVWkOkcW zoX&3TqLKCQuM$G0(*b8qweg6;xwsPzE<6Z zf=|4&U^T@lj>(@==gn{94`_G3Mpa$k*%a>E*(ihczaBZyNk9J!bJjo3`n@polK+_G zODb6+0`OyGeb|YTtWB`4Ve&Aqoi_OCFDNTS-V^660-d-0`GC7E=RlY3+a`aa;=i!x z;|F;YA7BvpC|w=?4kd60c#HE+w^z^9RGOo`MUK)lNrX>-J`-Ay2U|DZe<0#s*FptB zECy~#ard-4+0Nm+Phj88C=CH8#nY)7sV0H8bzvgMA#O}P>enzRg}&&=TiotBD=BV0 z(wG6byN4+0?hFEM^6X-@9v4DdO3FgLF&7Pfg}q8{vv?FlqBUMrEakp)p^p1WznBU1 zJU6xziXh~w@9T-suiXu&&V{_{l~;!_Q8b=Bv}J6oq{k&vhBk!lEqfkd+jvX)-7cfEW{}tK8pQls^=0FdE zd3$TgTl=#7?;@{w>$dn1Y8Q6jE|HkIDTLzmFvues-yfWv@J4MtPZIe?^|-ceN?~GhaTECQ z8x`QQTh5$~lTYv6mqgj2szEi-6L4$l!GrsDne9;37(bX&P?`_sDP`C_TbsF<@Yt26 z+a6=X?30@?6}-0lHsu?Y6pwoH!V^Lj&?d?^DhjIWy>89f=~TPpcd0v6Y{4S!o(cop zD%XzzV2)p@*lbvnXV;1q{T5F#=5iNSdP>wH6tt|PVe`#iM3sJq-PCVXuY9?~%Kd_m z0Bxe~P%Tk|O-B??jFJ+POn^T9N@b&%rTKcn{rcAMMV4<=);m4^y!uZHHa~4r#l<~t z#w;=RD{*Hf1!=r(w--sdTim2)PKGt1P*y0?NuLwNbB2>71+RMt9)XmcZkR-A-Vh&G zRuU-hG?Oj_puW8eil1^%bk^#&yu8&yG1#P1qOo+Cs$l!1 z`&IBX>=>C?Mr)-h$7;^4x)ahc>8?q73m-0+C=oj>6 zyMmHeM5NqJ`dre|QbXm$LQL%@Qe#s#BiN!&JFAt!dnwMdgzVUgaO{umB^JNF=1I<5 z^*5JhRd<4$k%htLdR$IN_2uojLtd_{%qOFjGK@(1vLl$AuR$AO+R_gSZnfz)-YU|k z_2qSFIK8v8GMhdCZJs-pyG-$7PFU>CG?90*nKyh;pLptK0iqd+0+a7oT3y8P4`Pxd+R7n(4KhpQfc9Dz=>tzjFyt_Uh1)7(m%84xm=Usfb0& zT=nfgl>F*$1hAG;5Esm}0#k5TB@sqsgCFQR^NXig?@}K&=624%h%OyTO!eFa)OMpv zNq3elsF`ir>oU-xtnpI+PbZfEy|KIRJ0`+=tgS}OCz*j5qxRp#P`5QeD^3T-QdOOA zMrjj^&52Q5oT;Xb8FYGNAG5P~NYkAha{qYx1NLZ)pzNLTpXole{GCgandMvfZmlN1 zyA#Q7lUwBxs{;>U7YP2bY8CJt&4)^85BMYUqFY2Wv%nDDd7Qo0tI!mj=hSHIs@WbA zz7oD%U|ZRA;|c)Q-qZ&vz0%wP4j@OU?rGXpRAPr6tVQJvicAh@yOib_pW^rF%T(a= zYg$mVHRhn;&~k7lAm%lKW@i8V%R~YK9W_2AWt!^PPvvd!zSq2 z+edbbpDA1iKXD$q=Eis{G<|h$jq&;Mo!bLDX{EbtcW!sm7@sqV9e;m?s1@kp4dH(5 z#&3v#0)J3dz1?eA`{JyxN3QnUJMH+_zrQPbx(IN#r+30@sm~1k{1n4m@w2w+1NXl0 zJFuu;UH1QfIr&Hw&>OccVa0gJ0$+$>_fcXx>QY7kqG7DE%z&fx2k2jUAG^O<(*ILw zEkE~Y(oix>Ul>o@%7y~I7%Bh+;WS7&QUq9C>f(?c0(^X^mHuj%s%{Mo=5n$(mi_E zxL8Z0TBr$q@UBtcJPSPlW=W8!q5HQ_dLPpALKBX&aXcizJz`g)jFlNmYM+1WBCpc; zNkw)P-V9Hx7kLw>KASy|r21YQW7P5dZHd=mtpFBfcl#Gyf`RtfAyv`Bx4on|-Nt<0 z-kD49jYVvTVc@r5B~m~;iW8?JM{zKtK~*m*1?jE-+*Wct)bH0$y3rJh(DuWy#}g@y zu~cOL)RjR}nPTaGdY$IBy}*HvKRA^(&Ox`mvUf& z$he?+Ay>O<4~T5ojX5w?C=LP>Rm*gjfp)OJ02YtF^1LVjpT2_Iq8qDVVgYu&lM!WYjFXAKUi*TCVz{B#Zz&hnyNB`?L-_alK{$x7#XFq`5n6m8^ zEK zY3u*Z@ST2Ou=t)S>EEktcb_|8ibxWH)hp$~s7rDpE*_rFsGJ&N4a1Fs*p0pwY7NB0 z3vDYB|1;*?%0_V8{H56lTAM`=fB1%gbGv`#v^$nN-y@HnuA60iW&n}a)hjkmt6`SY zDYo4Z!~{L znDiw^8M&DX;D(weLZ6nX28$Uc@T-dy@|iNf!^K6H+7xIX8!d8qj`du&`j zxV`g&ZmmeWa1S5BCqFJN4L1oM-@vNN?WuVoHs_F*W9}eIvu8)>>ulW67d94Qn#JVz zW?dzl#XzfKNkP>t7ax5d3o*I_-UmC3A8%#&jCo6(RVgRkm;9zM61uYOE8B}&^RN1_ z;gVOoz&W`G|N7Pw*lP6F$g__i<+wSC&02ZTWC71ahc8j(x}!=9aZ|_dUGr^OR!vmk zz009FnwBCK=^P<-ovIiY!+{FZdFkOlI|lzjzVGew)(o1A>1T;^Qbv`aPAcgf_vQ`|X%QryM4 z-YIMI2GYNni?(kGrLQvdFn|F4Z~A;M$i)wr|5V|U2Db4~Qd+lzAj@^ZWwx6?1S#HdE_M)7ud2B7xrFbH zXw*lL*E+}hlZOC_@W8g(_78g-ei(Ixut?T~@oRKCIwLBnNDM`x{=w$s$39{gJNc3V zg|A|*9NsunuBC??j%;b}~^ZXOBo^DQ@IIj3@ysPLQz_9AHs zaCVKH91Q`DF*J!6Y;7`TXeK!8yxW~!I%1`pMa{6aU7woveu$w_%544+O~>|LAw~A2 zL!NPXw{SX|d|WX9#AMegF8)vg5HO{wk$QIG;T}Hs7nTUKoQ^tx#RE#t3K?lM#d%yx z`yhs7Yc}xHI-la2?O-zP?GL?}O>*}Ae94U&^lZO6rW zQT}#VUMQXoL2-kh>6%{X2EOvnD=5$m=3AVA!(!8K>zdbqD zPt+%S_A3y!`wQi|jR`J$ zN2v`@I-~Xn$e;rY+^TOHI-zVs6Zb{vtJ@}ZLZ?uS6&PpBIQdPbCBTtxN|$1lf5!2s zkqW}N!y&?817cA2vPV6o$rx;8V$789c(pTAaZ^a%Xkry>j;O&+k*T|6Td{Hn*s+pTeAc<%3;&X_wJ_#2oTUEM7&iXJ zSCq1|9GQ&SvYOI~L+8O{`@}8phlN2WuZoO$(NSipW?Rg8T~iUoCf)}+Ue8B11irTC z;iFv%JuA>6tMUI8=#RJeLa;)CG_<6HC$?ZM}(c%vL zvO#TbXhCm2bBQt1{KH#MTkrYF?DYGK9PC@Wlk<(!(GKE@gE`(BWaB;Q<~gCV(-g8m zGZ!zU$7qBZL&QWwSO+ zhMQcc8xet6=ZOcw>Bcar<5Y9?NHW64NxqQ4JN>S!5=_Fo?A8sP6b$a8Q@a`zK1^wH zDY|K_PtvRr?yl4fY}!JvT*HS^!eg?7-dhZb0Dt3Li`2VpuB+PMoY*exh# zc!c5@#QIz6Xo~eZFkW9BQ1E+ZYptK@q z)<9m8Fu0{blqEm1o|sck8FfyM^*`8^g@_ZHJ43M{t`(xkpbdgdvbHyzO2SQiY_Krr zyM1sj)lL^e1e;lDv$>UFZ+U_VHH684F(V)x`uwO^e@JKV980bn)zbAT(a9Vkl{Q!x zTTN?_wC-P%HI|G9aL3Kqx4^CPzN`dqO)vFHN`kR$&D49_f7a3Yc9 z?Nb~{nYc@<=rNt?1t_$(tMla1xja~A%oJZlwpx=H`TFRoXuavQ{k0ZhEyL5A^M}3l zRPLn<6`8&|1wbh7o7O-j{}<+9nulC-Q+7F&u8_d1F_BNg>hR)ut(k7! z@TqaWa(;~Do9p}HmFTusYF}ah9<9UO|4|MOYQN{Plg?Q=1Q3+?K9^1!*pZWelc66z zC2RII{r{&0K6FdMMPzF;#%F70#Aa2J5fb4Iu5WIi|5F>e-(~vlS^m)PGW_xFzC|$q zyYb$Dj0Exh+XDVtj?0ho;hb|;mE!w=bM2?kPN;bx7QJ4L^|r@{b2U0pM=W5xoNx>Z z#R#5>UikGnGekK|eyaD&^#Vwzb3v$>M0S1y1x*JWQGpE_s`LWLlCfRx()e&UZ>Wbs=_>fTN5~JC55_*xOmLrL|9m}p zTM+Fl=+Tk)$ZKqwx8H-liF@;uAuh$s_{ppTb~N>2YnaFE;sulXw>%r zpc%XJ?G_ZiYbHK^@;>tLUfj4-YmGf^&?jzkdiZ9Cki-K%RCHQhJPk&)+9F5&kfEyD zjNw*0LLlD|&o%-7Y!hI27xSn$w)XrrL`JCYTQSpEt4kIyp{iXk7bZ!goUF=hsmDuv z!*muRHxYUY1^Qi0hov{_=C1dys8Mp9J;&a}MFqO<(Kuq!s@1@{q%$K*YpZfDwHQnB zFbsHS?8%fNWgEz6dEcwmt)3RkqX#>Wat^uJDEr<=4NS5vD1c%+wEB|Y@v+pGvE#yw z5eiM_TZ@W1T)4zuQCCxG%(^|^gOd0TCNgv`r`V{9A5#o5k{b{}<--lYq@au#Gp??X zr5kup1ySE(~6?M3$l4Z1=fUUdSFZrgE_-E%D2SiKsBh{02kw$gZA+L~EWWnwRhjpA) zrD?rV__YGdrv*Jix%HQeIyy3P5X#VufaO`UP`*MLosoesuF(Pp(O$VP58L}MEcpth zL@hpk^(|1#o{N7_%^&{!ud5gsM{0VN9H(alJ+2QwtA{1J9w!mZGolg0aj!0as_r|! zTrDPTx@wbRkzlR92j*HZ0}dMpVOpkx$ykHnDg}9Bkz31G1<$h`mUD}K=2nOO>^UCci z*XZXYGUg;*oNo+G%T!Hx@!0beslW?(6pA)AwyN4dHMx2JT9v^qBl8a>yDVID`p~1E z6xaAI8;RN4_f>MNsUK2ae7BWd{du?{`!o%(AVxbq&j`vqW+YqVNfIO7oVlZ@^k_!< z1t)ZUibWgytyd0wYqR$uDRqNGn3wcWVRoEDFIAm)0s71G*)GrQFdw&Oz%AwLBfa?6 z1zaCiE_TQZHu9m2EpxG)6$f7=5}sKN4u~s!ry9>S?IxC(XYvK^cZ4EN zVi7DJiKWaK=aFi!lXpjxyJldvkn~8krd&+~+umTcYygsW+yTKPNqbZFM(XNI->XN1 z)+VMTN59*iCa&Jlk<%d^x;^nVl2BSR#3C@$(1I}>I(+pb=%yZw5!CO4n+|M2VoJB~YE^6TbN-jAT(FI;!E zb`g%XwFh3#SmWPYbg;dWzxG?8G5=?;jcswlM5|zp_DlIp&BH+%-7eFk8+WA3<&qzo z+N?<#le*Cq$)IZ1w_^WMe4NemZ*?GxM~|;uDlqCb za@iW}eL&LiF#QOEm|&5%Y{QSDn-A0D)2DR0wLNOI$;laoZ7Jmk%1_Yb%0Gg7%1Heh zK^Xuv0C>Vr*Y-asV0JxgS2{(tw|4Z9mFp9EgFP8|7H-P9y{x-0#1 z;198DR~oXDb0Apv;zwzZvJk@$dH$|~-hsTDcQ>{2GBb>C(0u0Iz-l!jB+JF*M!3Y2 z-B*-Rwr=A{O=r3XJs%Q2v+sq(B>*&g)FjG*FMrG^o23P+YUoW5^JDd%i2A#C;^JKT z)H2I#4oyBW(Z#Q6z;D>5x*jPj@_LO5Kqsukld=$9<~@kYqvDDH5cl>E_{<{6oYh=9 zrWSTgH`eF_n}8d8kVshNhph?7-?jU7=M%o03Bw zu5^J2nm=5p57`F~avWKnD(~Pa-T$-o-!<{-{{PEz@f{u-+tAY%A7K;!%0iE1V+gN1 zp7(I@y`Crop#KCt4A_P%4wQ}TG5W)W^bZpL6YJ}+s_|%}M~AVzGS>%2#$iLv(Z+mc zV|k0iAGAMLwu6lHJMj7^dXo=7f|6&SY#rjE_uhQ|H_Pet`&-lL6t4UZC|<6sOXf86L@*;Rf%&D z;>>_mgP>2(dXsPYkbx$eDi1aZI2n2ZuS(?{!um)O}IkpM_H{*R#CoEO+CG1qA7Q= zZFAs|Idr|)xDi-z#la+Qv$uu;EK)r`#dU$0mZ!8@bT!Sp13#k?s)18Q3NrZsHbWu0 zV@KoQ5I6LF^XSwlbvc6P{ag#@Gtgs?lX0<;r!44Za_rssg@vp`TQ;F>=&U>o+Y)7A z5^rQN>2MGp;i13-CR<;1HAKAE9)d!LJIeRpOq=QnmhoP@@gPoZhRhqu9@PXhnl?5? z@U|;UyZ1mHo`@tHdJ{r81G^SmemsD9%G~5hB$almWow(Q5>S@?R`z6EK{3!aLZqM` z^JoY*R2HBA3S-R)sMq(xB~SEmIy$=MpXtg!^C)reFVe<=f`UgW^ls*(MIcAW^Uqwg zyLM53sdn+A@^9b}zbVe*eV+ZsYX7D?*q8#z{60T!O=IKDG6EI-PEWhkL{+o>nn_@=YAVP}7GSzls*2#_LQ>%5 zHWdrxf^?yeoYK3ErHUIVB|Y^EiJpu(F3twP{`~u?`(bl6kc2GNts9o5?E|x8Yqc{C zTW=#5Y=z2iyk?UvJkjM@#=J7_x2)}z8#cs~C9)Q4S{9$UK5^MIi+UdM*;M*_EIgdXA&< zU~AU9W0@|ZCaf-a0y%M={k&0kIjba_1qH=Lnqpz68D14%Ao2NODwMlon@oJ@b8E(9 z=G_{pmd<6{&HbDOp-WHk8+tmk-ne;ZkDS3 z2=WYe*_i#XXO+D65p+kgX{q{1YVh*OL*GCo$IbQsoI&HE@SX`KGzrpM|G?s4X^rJ> z-WG0CN^E`G;==Hy&rGp1Y5fi{k_>0@qA-`IFpf57L>uY?Xh2WAcj@FBobW7ss8Dw4 zlMmFMeAg;tsVDHuyT^>ssnDM0m9965cxrG;n^#l&$jba_kR{-6!FqE`OvM#ygw6tT zjCTzu;#>Ev2b&HnAUPpnax2H6HY9iR_yu!k;c3xN1m>(a2gG( zngkrgiU$_pcDp~+MogA)%|{(6vfiP3w#)My9r7F1r(eI(0X{EZsh;ifGj5wKnV53j z4785+f&vaKMZVQZjLY1G&pjooj1?HHqjiIF6J|08SB~En2OLYjG}C{F3TW?dbbt%X z7b?b2zwXd|a!2{c5BEJu?aqMzcYW_#dos!L5mdTiA$Laa=+_`Fbw;%;BO>GD2o;Yf1xfp90D(h?1Kl$N8RPH|U{Is8~bx7Oqp6rm*vwLYJe5G!UDs^Q6iX%L$nvT5w0o!GHw`^Mm6Okl4MZJ(-VQM!YrNzFfEjsT{!vtdA zDFu7uHS;P@>uRcafdhl6Ufk`6(M2W>TL|4UdHSY9Uul8;n`r{QgNe)AV{>XKPf=_o zN#4Mjm}Bge$3nLCf7x(ki)O0F(F2cY6iQUpG)l-ctZ9-gvcO&QhwW?41h@5O{+O84 zq)^~3Q>TgSIUK$%B+pNGAa~KBaIETHIJ=<)XvKDQtV=M!02%x)=aH&YQ)ed>d3= z9q4G#sJAwY-`Xe&q>^0wb3TF=@9Ey!3Irf!rUv~GPe}2(RF1mscoK=%i-8J&Du13@ z)IUuvL%Ed2BfZr5?_pE@h@hRk{ij%_e4lEfVhJ1)mNGo_@~a$^FCiluI$ZLHa?Hk< zMa#Ap_27Hn$D^c%D^jriiVCfa=l(=}-;E;wC+PlCmU%hJae~%5Ut)T%+%=7pB-K6x zJ$7wo+F1p}b?|kA13w;Y!<2YH-E~7BcurAu+RPk|-1rE>;tXz1_pKgZ4SxT6T(&v0 z;);v?@bmtbH)WdZ_QTi8<8LZWY|F^OmdP%!O?y>FQQ<@K1zQiwQt$10QtchKbot+y z#Z>Q>=A6nntNPoV0kMnK&ItxiEL;j#jp3~F6pTxaPtVYhV)7oN@xCb1TdLu!H7C=2 z=U13MBCo=C9LRr3NxS@~`tSE!;wk}|8!t^rSkIg*$h$5Qw?7pi15sg%TJL@b1epH1 zRsI(C*V-Id_E37#vb;?u9NlD-g1~YJCibHK zwDdBh+#+XL^EKfDn|uMi%kd0nU`Ey1>hOk~dcqO^%$+QD^YdQfrg}Abw3q z>EsG8?(K5}f)Cr(Z)T-@1Qi;+V7wYC*ddWN?$C>a`gG&_dDq^j3lh|vj-$Ew0<$Be zI^p&EYc36TR$1v3zHNCJ>~li;|7-8Nqngaty}>$)ID?2N0i8=z0-}^qMV&FB2!V-7 zh(Lm)^aKI~k{~FGBMdEZKmq9*T1X&B2_!^{pddj|O6Wxdq!$&Gs`pjro=;}YS?jF3 z?)}!c)_3>^c(Zx--oO2nm%X3;dp_R_i_Zcd0g8n)&B=_I0#nB0CR@YfBiLd3+Og{> zk1U)s_ZBlNGZFlXZg@A31Q9;2J;Vp7V*>QARJ-!Krr+Nk$zZZ`T|{?g5-%{LHdQp4 zDF;|5Y=w3V-S)WFvI>X8YmB_OCS=GKbYL{WVcQ;q#pIiVam}Syz}a-17Av0CVw4nz z$8Cp@iA|KogIesA;Tui{&Bm_}=w@^5dxTkZg)B4}Q6a^?c?ps)^d1h&8RSXMxYB!X zr=ii6wr4EN%rvvq_df}I+Aj2|BRve}V(+@N7?YK`B0rmRtJu9SP=$pV=iFurSt4}b z-NCr}fOhl*9OnW-9i#7cfgV4-NWdKFDrD2vFrZIjs~`!}__9<(g2*YI61LiU^Y6&%={%SD(r9c)>q*qXnLD>{lw*g{h#DRZnC1r}6Q zeK`~5?6T4)HYSy-rLukCWckQ}#u7Fue$Sbz@n3*N)*{^aXpQJbi=F3H;t{JHfY-axkyU%GMpqt%0N!N&zYH|Pp)e(dC>_>uWW@DMt!-3gax6fE)#kE_N z7U@Pf-S>?4B*JW;cJk3Pfo_4Cz5-AbqqQL;Gg20TupXCtjcnA#kJWU7IV=|+W+a>D zj1`BJt62$*vf>@bj`Bsff{}L0-ak1tYkd;a*|Dfbu7$bhYN*-Pqo2+ErgE0o)7hrp zd4T3y!Ae9mCPexuT4egx-_I-ldBQo?;(ZZ++B5_73PNHrS>ZF|$uElF$Cy|5zR-(} zJ>V9iHkwO^SSVj|0_t7)@%0(bl0W?EOn~brnEVU7o}*UcglF_POuE0fK}fM-?#wC265D=AF_Xy%8j;`T@t8#PJYb>O0u-ivgNZz&%)s_Y&P?J_3GyNTk)|`8y zaUH#Bzp5Y02KdJxeYU|wgJj4)6N+bnz3zN>I`L}*`N`YQszks1#|<~5%y-9teCw~R zPlxSD`SJX&<99P%bH{dt z+}pP3^#{8dDGqW~3bX2}}hJ8B`Evy(h4dfz~ z-jHpZIhUq_cFOd0pc44BW^4oROc5a#wX zlBIM;WW4z+ni~`IO5b`!WSWgJk*7X9`TYdl*jdLc@9B>uKCXz^-<^PoETaxbhcLFS zmojaSW-xLtDSIV8ofwSQ>Z(nkh9Zt9f0#dB6rZSB_@dG3spvU@l8jOJ;nif%u7Jg} zx10BxDCa+hhd#Ft4Ph0V$Su}@6^YH%;b3X%9`z%NSM@rR>TcN@+13;ba6tO-h?NsX zwkoyS<0rwrtV|AF<*aq`q}?WcPiCsdT=U6G5oR_`i8@1JCX|d;{pJx$b}6Uq`|mI7 zuK|BiUXabWFE|}{-|P7R(j26qJ(e*abJBmHs>MR1v~KBcE>slMzemwaq{Znp)Dcwo zy?I%AuVph8vPJ9X^vCn%wd%&A>}EQ<~&USr)mv!$7A z@%jS5iG)gX0bJuQX~QocWTc+EoUp>$a@^|c2lTJ4erSmIjvG&Scv}lIX{P$8e>)na zz_US-=t17cBk4M zXQnDF&&~Z@^V$i$jOWZWz8v@~N09`B)@J^+E9rS)hJC~!T8~?*=htxk@ z2XgHh&j8j*QaW9%bMvKMd3J98iCwUW!z6aXy(iEbj`W1$50k&zd-TikZ`po)Cwmk% z2R^d@3u2#9`|{(@U;{%ha1wR8qvmi1OS*jnb6%NctO z!Q!XXwdxO@)T!E*C;ouktFD-N9@D0sKI9`3P51y5KBRLKAg5H(59c!d8g*O~vFomU z1W?iR_+1#zNrA>!gU6j(|2Q9yHbG^e>04w`F zk?!lvuE}s+L=k1}5Jq-$vuA!5OFiTFT2zcOuQ#8N>Q}A$9?u%HlV=J(lIL^NCazdU zYwKKk&7a!C9@H%7WMKPO%UBt=R~SoF{B@A>L-sKl%r+QXgA+3i_nhvVo*Swrz7HMn3iA(;M{mQbD2b3M=VjXh#rRJHL zE<;A~oC#30y6o#uV&3BUVnmF2YTrUO9#tq;4}I)q8I#0f&nJZ^qM8jDn$!0qr+fhi z9l6c$;8J&@GajszdXs>>7Q&2}g8aH;q0^Blbe-mw9*yI7)RB8{Z4XklH-6^-qHU@z zFK+MCksHw4l2$-PP~KoD+VKd!T6&~>s)#aO$jzoF1F+j_mhJwS^L;(AMc6I z^>ad@(Gyw*EVo+vfiqTOUG}pLMDxY_b`-KyyZ-m0fxcN>WBsX5Vx*7LT1PF-|1d$@ zv0D@Db+Kc1a=HBenZ|X=UpJxaxLj^4S{oA4YK$A_biiG(9DP;k*Q*V!j=3!5RM(84 zSMfw&=Ix7!srul8M;Ep$^uY}_c~Z@?x<##!gJVy}o<7DZ#dLSrOhz&-qs#}lqKdUO z)L92~khs931jlx_+^*-(JgPyNtv0Lv(kBfkFVZ-KjNDv1nen;w{nj70%t;Xnj&bgX zZ7=?w5Z7v%l9w!UxU*?|>9=k)+oMI=M*{QA8ej?4D_im|1? zUm9hasqe2>Y&X1A>sxVP2ul?q>BGN>^%t9WE5M=CIq`D!lcG@BR zJ;LNw@IHTg7lMi>Gq+?QY0m6T{bl6JT3w2G8LQ$EdW%nGyqOlq>|)y#eu$NnhbG$h zaeP=0e`x>LOzMSEsTaed+6y8~j^{aFw zw%?+~ly9rw%%E{7f;0|E6R0nvPuchUY8b|(!JN5_cR+&k*sUaji}ZJq;pM`I)GM!3 z#Cb`p^6y+;m@jo4hJHBdZ08us7gk{^ZoY~nOtAPRXsh;)-Y3P)zU($a@w+woYXFBa zG3lp>jN2Dm>F%T6HdGi>1l&8SUs!IOD2eb;JC~-MOV?A6*L68psQ)3I6_Fo0DA;yK zyf|#WYqD2|5d-b^pg4TTJE08Ty^|NqLO*c(s+*O)+TR9bwPyu z23W+z{ob&i8Dc04^WNuLmvwRv`qFS+EA4+*Lrwh3c!X{^%~rK&i2X!=UgKE$mAC%f z70cH7xmR1SZxs7~`M3Yp)x6K)7!9kNrhd5ZaRtu{bdk2>4NiR$oBkIkqZtgbyNSYj zKfe|jpI}1O(vG@0M+G)O*Bej&4Nva@uuii|BLc{-!u!D2cDcN}=o-vomD%}lUNQg0 z4*$16Jr;sEd`q((?mQ8o|3>30mf>RrRjB~YcVCWqzP;F}|6K7yoRGQgxR`8at0E(# zFEZjLb{?o6!-1qmj~$qczkaOOA`VA&LGI+gLgE)pvjycR^yiO?;kbbrw{iV|a>n&q z+yNR$f5!*TfCTRASc36^W}QLx(%azjSz)$}-(%poLo`l7*hf@zU&F$V8iPIE0;439z~kG6efWC}0djvbL0n9v@l`VL{jT(l#S;XQ0}*Lk zDFqY${jI}Q5x;I)FG+-TQjjFgfW19cXe6!+EzJHh%eMrAmqLEDn|f?Ah5>=}hs{Bs zZvsHUER+)`dQ|?}+~`REx0LXo$rZn1&}){TigRu* zkl!a;&FUUTq3{_o2+YIJ=ZCVdPf4viPanAT_?n%u*n`_Lme_XBNG9{HnW7uD`Qo!+ zrIm&*okQUtizwp9?2lhQ0778!4+7mXhfV58lW)Qi$zpMDj+rNaH;XBscLmM;uY(0_MWMEU;)QrGw_ zp#BB~jkZ#c12%1y{-TV6qke&B%U2&%Ov`+$7Gm9HS*y0Q@ZN;1MIuOVw2JHc(KsPv zadvH+)0;yb){pM4nJwOauv1-*Mw*dhMcsriUq}z3G?Xv(>g;bWRk+0t*`S_#O(FK8 z9*7KPXY$*0qgnN0B5u`|IK&UUU_1BN^;`4s?2Z2@N?w|Eywxd~Kaz)sG?w65no|0s z!NCBV=Vu%FbLlY7{a3saqum^K*NdvT`$YtddpjBn_7+~5;&HwmMgndZU9}*ICeKsMbvX(zExBT@`(Fd_AaYQ)FERR$$6v3`qxL#$IB+ zwa~2;cxPSr-8%Av>8n@?1kIL9p~28K*=sq?g2)Zt!?QObSd`2oIwpt&Z~pnhY-=j# z;4(;2H?1^*#DoysZE~F6vpSvp-<>93BERifT!|nWp3Hny&m z1uK2qY_n@0S1)fMxl=+gtH$g{f0iBXEnHjMxnXpihH_!DtJ8sw+QLPjBQa%?xD2Fv z#c@GpMeRmu0|_$@|7S()lWo1>Q=$DIzAo*bE^Ti+k;Q4@!|J}QeXr^i_#exi{VmFy zO#2`dniXfEW_IO+WOCpIF7cl%2HwlALZ2f`?L#8USpw5h+5MkadV}JNZ-5H10iAC< z-L?U$Z_xSb<-b}<^#-UUO5q)-zc-I9G#-9NE)h2o+P^pp(p#v)_xI@^vz+W-ivHHg zOD2=rdx`_Sv6^$d>XAj>%1^pJCzexKAs+$pCrl8IKowyqI?ON@`>tl>f#|-M$*01cG z24=u@mC{KuzSg&4qI>j)s!E7}m!o#_G`wo7+<_$kjEC*>t?i3->)T1=pMl*BC|{Pq zuY~9q?fty7dZWkrYzDR$r44J#)*mSmL93>7!~DdD-#xHD#{>)<6DHB=|FgZ?IPhkk4oj#7A&jqH3}a2;k1}zmKqhXJE{L z;sDQp1R|wtA73x8FF8S?YIcJ(z)@bq_pecpnN9UCZKd&mHxVAL;WIiLnr(nef(&9s zl@QUsv}K$05bR&P>qt*AiHHTPp6A)yvd*Ty>8m(YKz+HvPwdkh0$xdmsH(EpL_8qi zJ^e?(Q`_K$lQv-W4Llp9ZNO>+JR6$HG~KexuYG&B12wd`?=P=#fB98ROhIMr_G(4s za|I#GpjD<$AU9p9G2dYofR-lZ%EB}jKCmWUrF$sWZwBrK8~}mF0>O$}91>V@2uR_X z?F)gi-_Nx*;hA?={5^hpA#$&EEr&}gHo$iMkSS7VgI#LVM96SZ-j;#cGN39oUcKJ$ z=+;_{#^T-wFss)P=POYWkgkGD#XDL%p#7i3K1lTZ=wG_NgB3fca?~_C6XKZ}oL4R| zGqyt_de{OZX|zcakS9N+6y9CwCG(njjBRQPRNyQH;O{t zyCve?T|DzKstx0#P&2T%^@m+UL0y}U8yPEv?VK_a84ddKo^jQ8YKM*|MX;jc;K{}H zuhmq4w@|rirMq}%N*a%|w4O+3alg0ncesgaem{yvXy-oM?GG|o2)^uo=$G!?4%5Qc ze9iJ1vK$ZNwBO3<2FJH*zg~>HWouHFYF>^PmLQ=AkGXbf;$X)s>suk1m#ksXnwXzh z0s@~F8v|B_()Ll?GGqeno1hrtylPS4)9Yf!-;02Y32RCp6}w^2-UoTbm(NhnH@EGt z)2gET4jd5#i&9Sr8IV+^3&NhG7(YkzPHWBykjcSP7=;smAmx2wN!zo?Cs1tBVH1O# zU0iL;QumtvnZ2!T{L3Sv*q?sq@6mN~GGixDsNRb{$51bR^ZR(+BYn8u}3P?$8}agJylXRamTp0IY2BGAk#?G_Sbb5JBs}Li80m zi_6zLHwTm-vX9f9pHb_(t=!19|}BRAgnmjfhq>1}DNBA7=P# z=D#aEfHqw*`+1^+tBi{}--#EfM~})Nw4&!u>xvByE?g{4O%rAxW+0xNTlCMzkpqX{ z2i>+a&@>p1C@4hyp^xBD_9hw&MNKkj4UW?VrHh14_xCzR&!KfAHQ*s1 zp0yZ0`KD{87tj|nXbYjbd}s&phP3<;z{!0BYhY6u6YZ~6MqOr&V~H*?+f;# zuwe0DwT+Tot;SQOd)2AxGVHwc7mPL`4WnTLwFc+os)3D;9jLuN9VRnLj zlD>Q!LBC*E$UEymfq zF#J>kOH&aCM?HxoMHksfuJny$`}JKG4&L@h@b0R)cH{{T5$f-GNhuypVF&JizIR>p zYP+hW$T33HTCbZ6IkE7jO$F%|I++)f484^hFf`MYk(J+@a4?i#Du_eFu0l@SDf~&@ zi6qGGwx?TvkOv>`eCNG6X**K$#`B}tCVC*=+`(8axj6@oj(KLLgg_*}OW)*FHKf$L zAXhn5h;OpA)O}ldYkn;_^4dFq#ssq-C^ncqA;U*q;L>z(`28rk}T9q|_N0TZtyVqMzj0*7)Hoha+QMca)e zXB@-P=Pay5EJl!PV*8Z3Ypr4o4+EJ+L9NxQ0>LpcjcMDP4I3d#e z6%Pd@AAB&+^X<(nE8TxMD=90fIg1%nWSFV}A;&O*;+@e%?auC_O_uo(CDuMpSK^f- z6Sn;LEz?tIX1nbi1ql|!oE|&<&o=u0$Dd0i?x~O=(eS>c`J+Z&5-q%i3=743X=X5( zgo4H#eh85dpGw;iSL1SejI20Rf?Xn0gH;M5&WH2OAOVI32g8!w?ng7uLR<9sxhBA`_i%Hi0W;il zrVp|q106$ehr^1-YQhCEl4wR{QkY#VCxTn@&a{wU-c#VZfbvra6_K2(1#!sA)8a7p zzG-A5{%-in0{O+nYA)-On3JRxD#El!xm(1>OEK5|DI!okICtKc)&=RLsgVchtkf3E zReboEg(}m(8LvfmmBCf`<)_TrYzojleT)omtIzDP*hiD;_)uzrR<3EU$S26L1r)i* z)<6Py(1@CUWGt(97g?`8sa%dokombOOo_xJ6@&W1d9aRp$zIwIo0+W&M(O59oWr1O z&tmm`sH=qj?PiaoNU9i#3S2btkz9XPQLjo-<1Phbg0m|nG8r0_!mbUX8X}F?RIiJ7 zXr?N0w3?R)Zghh~(h3O22{)${X8Ckb5>sTQNc8?b{qog%>)e7L%6CqA7FXv-AR&a3 z9#f!>05%^x%Bj6wDQH1fJQXuG_wP8Y8Dsmj`>_mgHd+Sr zdU^T9whxvC2uJn%U0@@F3^d!b6w-koihpp8~ZV6%Cg9f zIlf1)aOH)rhV3&UV5M9FzKD(?4_tVH<6`CMl(TcsJN9V;wYOztD)pjV z?Kb5KDoG&SCvQ;CPx$x5TEJ|~MRq!@LXESxhN_G;51mo~2%#q(vQh2S01iqoGn;MK zf8sg$XFe`ZWTtf&U{|xiRPyWApR{>2;$4HlKjyLT$EUWxHA6d2xM2ATdACrtXqo(% z1Mk;T?ro-Ct~!S+jPkqReXbsEvwGDljkTsSo?VKh1_gf-yW7!=?>=KE-`ZA>-s5s; zSBiXOWYnVOdgB0AZNK=7;mTas7^Iy7^ zP2AC&+)YqWIdiG2Y4crSCPCRVX(DgI)ITkJO_4JjNj#GvM!wS%xFq*pnfyoL>R~~W zZ63J5Nh&JNbWH{d??Zy_j83|d(N@JK=nG1sSu}}HV#f`zSXK6StM@qZJ7Bhx+303r zj+;H@=}6!h9hZ`LD2Ug%tMFuH?whl$BCR$<`$b1XtGWLGyp~?M2{oq#WxW!4guTZ+ z6=Vw|Y2b68K(K?k)G>M4+Q`x|IC@*NI$QztHi{eaPQ8+oLbu^{Uf0_PD({;0%JU>B zCClfTW^ge%pv07+0D{{J$e5DYXlY@y)k>3N-fv{m`i$>?u5kZ&iw|$;&V`-xhSl4x zl5p1TXH9zt;xjYJsof3-X1)ZY0(xSt&!=NaFZmHZD7NoK{(FSd^MNQss$jKjW z%I&u0ASny=*KS>TU9E7GlK8G0vGyD(Q)uf%NWcD=Sg`w}PCBo@+rOrxv-ZM;;xXOg z=NiZPvFcr&d>l(;9aYAfqi}QvPppZjG0rQ8r52tZ&vZho1UBTCP%we2M#c~B4pujq z1r(Ik6}EPw+DzGbd0`kWnJWgc_NiPn>$uA!fu5PDdea{H`=0 z*|=ZgH0_PDa)Pjl{dwVhqC%(@_4fWpdveZp6rJ4;KU?I7=i^gurA`)f+G=u2$+zXw zCBT9vvCXXfwwt*z%`VEU(X9AnFT^jgRx@N+V_G!anW(RrO?`9hw)LyCIigyxZdAMI z4BB?4(m}DXr|S}aK?+wv;u!w8Z9@0P4v$1Q5?cJz^YruYOTAzuLuOK9-K>1@@Y#o_ zHChO7xUc3J+c zawl_YBrM{EEmo`9#g3dfA2SQ=Z{tUwMpa;{!@g3@|Bt_2{H;Kvt2!+CRzD`Us84e0RPn|_!m%yob;S{2}EZiyphQcw{G$J(BK(04$~ z?ub{KXA3X~Y<3UtPM**RL}V6qB8%N%n?aJsXss~)u-W%$wC})~l+Gx6li$mZ{-u4D a=s?Aba_ukMUv9nkx1;)h{o$!k-Tw)x{9^Y2 literal 0 HcmV?d00001 diff --git a/docs/screenshots/api-key-ultrafast-before.jpg b/docs/screenshots/api-key-ultrafast-before.jpg new file mode 100644 index 0000000000000000000000000000000000000000..54ecb306f2aa4ce27e38029af353c75593ab2f85 GIT binary patch literal 82778 zcmeFZ2Ut_vwkRCMj<`2M1Zi$VQG%e9fK&?!Jzz){p(L=SlTZTzrRd&naSNCrB`8%* zs0j&D0t5tXv>+%Yp`%C#0Y!>p;botF&Mp7D_kQPm|9k)U-}lZ;)>>oDG3K0OjyYx< zbFR5Hu=NJ?qm6~N1!&tg5NMm|2ikhOZNS>x+~b0ygN3yn;!i{yNaS{$0)c{pBS?;x z=YMi`as6rcn?FPRfxGAx9{R`gKXIaX*M)z?4g&RR{wFm5mG@q6pKvb`!F$oALJ}n| zVpduNOZ)u^AO8dP{1Z0$1CELajS$gX_yZ{X8>4pqh9P zNOtI7{Z#IQKtEgufvR8rtKYxMB=jQb;$NcMA-Zq(^#y@eia;O<7Z6D56$rG??Jqdd z?LXn|ClN|P#8-&u=LZS|`G9@`S%ZQ>ULY+Iqysty(gx{o4TH=@fW!rqW{i4 zJ9qBbv2)+<-MjYe-?x9i_&#xQi310JlsF)HKwSLCLqAGNNrS=Q{XfXa9+H+lC=HhW zBa&@mqA)vl?%la_ue5}?g!Dh`wjP5d_lRBJd0%WB1hidpo0#Obt>+-6Kh$aGHj(>V zZb3V?Z5P|QYxkbL`$Tx$kDzU0Vj$7`{yjTH-aB`KMCctmC3hW^I;y=}8VpJ8K8HeM zQqp>K^k9w`J;TD&ACy<@kx}+0{YoD{bl!_0t6QmJ4hNp?Rdu@d4$yR-^4!pKaAj{pWM~anx2Mf(|j}AW%u`|#LRk*sBV;98Zw`3Tw4|c z=`J2!@rd4_{Co7}n1ja(9rZ+#l05g{Nm4Ju&yONvq-5XzV)+l~$Nx;at&x$RF^-kc zljzU#W61WsXL5dg-H#nw^^HQIUZF?kJpu1ET5*|IUQ)c(K;lh`gWME$5;A+@-S4(+ zPE5+p3Juq(EszF{N%5GRG8cy)eee*kxke#jHO&v6R(Ns3p1?HaFEccqzKOpvP@l7B z8`TMlHoolbyCZI&%I6!+%Mp=wEEbro<|2=Kq<;&*JS~gYPQE|QDp9QwB9X?23?D?d zte5?Y3N37@YZp6|Xe6F)V3T-9`Jk*<46a}6RIax@v{pCZ-MehtC8B&PJn&|7wAPFv z+k$sBi@pU42NU%c&N%t^W?O9)FT44ob?%4KWiF6P_g$4__9h%yzznbCPTD zC5o0#LdKdx@nkrlLnUT0sfZ7BrrfhxMd|M0j9Vibi(U2RQN49;Vn$ktSRJ6I!Mo!= z1;-Sml=A7v_HzzAILZnHuXI1N-W;0Uqo@*Ap-DdzT#pcJ7zbr=LNnj&ez6gxRO|ev zrW5b*Fmc?ThUw}jR;NIj$~Dl5XsxNISrZFE$e4zb{yjr=uA>J%ds^WGPMCYa->Ta= z^Fb(Go>`=XqGL~A06wOydIL54CD0D3M)qvxrfha}j+Gpk?qhLLQlxR2&?B&JRA*@E zwT#?s>zR~tix+gFVaIhl&Jr|G2BB3_aAaJN@18k>>&On0-vSxCVX! z4Zm|bfEA!#oNH>A^}f%dc$!ztCcY*4%sr83HUI!0@}QK}Xo(Ka_;u|pY zzu5P^YAnwrSd7Y{XANUDVYd3=&+E#9QK3yX!KaiSkn6gwLrdfMb=2y<`MM}HLTV6F z^Fweq*{wke>mp;eV0LiFT2Nl$*f6Cl(MBk2R{2V;~B8 z?6<>s;Ka3gD%a30G~cH|nBe6aW%WFT03@*m8?Job>v2Tu^&gBuAhn@)f~Y%oLr&Vm zhk{fhQY-YiJUE|4lS-st)scMV-$2#`gBJ-C)JQWLJIEweLqDmS5}KHlQ>a`2d{^NV zky+KJ0t*YwcVZ0xuAI{8zArJcxMbyl^eRE4q0Umr|A?Z=QjG=Q|FBbAMy?~4SgsNc zg#kvRLGx5hd!~j;MpiYwAB6~s9`Wi=>8(VyrKmM3O>=`}5>5}c^bEGsjj?Sh>8Ak)KJ}@Rin9s>wd&OrHI`;8kp@OkV0UvT~=h306OwE2% z1?s+mxjA{8l9MjAM1c>y-$M>kle~ZOz#hdluqXQl9mtTENqzu0!f^&&9LQdDeoelYZWAl^_b*f9^;hf^ZTHC)H0rc~uZ60jIL7g_b*8qIW$LpP%$n zYWIq9odUY#W-JnZ*E0F(@+ad@8q)~2jcq}l#?!v@bMvSdlvImb&mK;VyfYs48Vwus zrq}jVXDQkY9U2-M%DZZxRqB$*xx(Zo2{9Na!njeb*h}%dtyb&h#hmavdlJUcRk7)1 zH5i}(4~4D!&t=Q;N9p{2ZG|k}%g?|ST$)yvO{);EL@PfL!R=e=em644By$CKCvpcL zE9`jRj?}x&Z>Bnrl!oJ*!7}^ee=kIjjH=GR`y7mEOLKy~E1_1a#9Zv)Mj30yoT?v4 zq>k5qOk}%<>oZc`_2UfL0)*WM`1?4}&uJsZvyMl#972~LSp5R9u{^mxtRZ$!LbH9w z6=W4P+t!Vt>((RRQ|>*zf`JR?^NA2<8>bI$!3Cm+AC|O+rKNtS{DwSESarplQwKhHf7-EPuH4aRM7HndoBH}8Ic_*idxg%`3WTaG)=)a z=-$$Lg3vyMMpmCCFXf_YQHlXeFleb2#kugq5nc4v?9Z}xeXpm#pY_u})Uh*Y%uuC> zlC5SaTVbmU39fig(4a{IVNWD3_s({#XYdoraXW1f`SMo5%T_4#zHAFCPB8d16ta1C z2C6=lU!1!4?rb52-&)=tPbiW`>*@3BC^z`I%fqbfLsFxxOZ2$I=8aYT!~TV})zlKT z6pik#41y3*^*dmok=Gg|m&8s-&fmqUyQ$IYb`2d8zlBM^;Dph!I^ z;y;a!KURuTHJ-Spj%SsqkwaI3(N^`GqL12&Q<~lCo`tD}6-eSX4c%+}j=jhWd;vYRBCwk0vAf|)*7lJ_V@|A5ITN$=;kWpMH` z=hI`X2X%jnhMdm?ym}A#etTzkQlj-PK=_!+RY>-&8}AHeLj4kt{WdEmnylz7_?4Cm zRACX~JNSK9m`tsy@h8?sT70aBx`&&xa(=wiW!OkU=`BPg(=V}J7XI13E^j%2CvaQf z2@4P=m96YhCqGwGyFkt)F+)xvEQ<8Tv891SL$^36Asp^3k5i>z4S_m z5iU3%^)V$&X$G7Z+FEVEydyp@T+CZ=!A2X)!r%Yed{~(1{UFG*@_X0ZA(boWQ3!p za$kDeT{CSP7<>aCZ+hBvT-L^V3#8`o;EXiX(c!aB+}=<3e|-F(4+#DZMuL6qX-!F; zK(Ft;sP%OrC{s2Y!35Lx9LUM`l`KXGz2Hz+ z6#r7;j=-+GJlBBSyD+G7Y%&EHDs$p?)HCaQ)pG~+{OH4_t6v$T^Gj*riP7-*6b%67 zX!JYTRT#1b%II70N`!cckG~-n7gh^jT3_h2ej^$Vf1=6k+azUqF0LdYOMtV!EHSW`E8QmQj}Q1|cRaLS4;f%_^|M$k_N@sslS|^xf|wMx z9S^(b_b9&}d9WIFWL-F2z4wE>|G|=%5vg9sze359W(G&LJ?n+)7pjVeh}{5oZu|U@ zs5MXP`J>PKA%diPypDr-T8NI4+WWfGdZ$(%FsjC`Znri?n%__1hS^YJQWm5}`Tz_( zk~xLmJFeO*G;;B>6rR(%UHuwr)TnChCRNRLV&y%@bQM+*FTqlDc4z(8rv;oWETP@T zIzEXZ*QNOZ%Gu)h1}8lB%$`IZVfiW3ujQg^QV?0V`my`sUkzQ*Zu0P)d0Vly>A*QzA{d*{CIpnbkRQ>(0l*%Lb864-cP9%GGVJ z=YIR$$S*C5_+?wVEU(ujyWV3U{C^pN?RrmOr0dckkx1}kIZ9av%S^E3V4ja@l^oI;rj3 z+{iAF-!NkG%IcE5|D)tF5n8wR>fIpv%9-dWuuTRen1bE{W&b@Q>$G-5ozt;JnpCaO zz1)wn@W};xa{hguSNNavBT`L!tfo40rufWH0$aQP@1;_pBGQy&ukIzve*Uv4DqPkR zt|Of`NxOdf`*aO-(7U$JI^;%t&(%TCl%8rm!ZDY&n@nc5Xc&b2YO;Mithla^jlg8P z2-t)sKoXFYlsf&k7XG@~>aoZOjBV&K#BFHgJOZ%LKP<}n5puTS)fd_=kxsL7-a=u5 zE7iUNa#DIU9sY(W!i#BY;#PI5377MWG8?C@a@eG}eK&sE`LWLy&@ITa?9}MjilvGM zm{Y)szwj@8Bc%o)`5C!qfq->F~ziTaEIxJEX*VTqd$TA#V!}hNkzZXUaGL%pcDE&Cbol3Ek+0(AX|J=CO|J#H-)U zpC&Vg`MF1w6_RHg$}9^jJx+qV$iP#B2I%2~JV|~(H`&fmr=<}4eKBeqXDZloCA%#n zG2kiFHoS%xv>`(e!Es=@Tc8&dk1bI895W)yrs~cr>6O2pHP@a)1!7CA9d8@^Wul{5 z+3EbkxB9=429f-RJJwZLTVXS!PY?mE;_lx9RU)dojQAT&jTC@t=QUC$b*^BOl#$%M zaei?fuj~VRn^(|Ta&6c>L0A5ZN5rj2ayY9sa#gIT5$#cByh?6*MT&Rp3MQw)s^3NaMzLJWxE+)Nx59_+UafczM+gWdHv2B!E_#hx);W&_LGeT5$JWWICj^i$x0p`=$ zl%cw1v!TWcdUc38N+Un20C2cEWp1Ts^Vy}{8`x-w34-lwb%W&~vsevxD~m50@_WRR#99M|H;op4;)uRr8z>3wj- zLGNFCzzs!cWGb3VqErb`x}jtg9UAg_!mP1-cp=chs69lXXkaz@oT7Yp75O8=)GYe?l4yCTHFT!I}L7*QgAP~SmAc20?T z*N!kFFNivJP}Apz;QS`D6|EC}<(5xisJXFFFU8M_CfU*8*7{j%>P?X@G9Osw#JlL? z2JXB1R#@F#KjF&%1`z0sG#4OhX6gK9=W$;D)7he8A|t|WCjs8_{Uu)MmYv$9B@8Ti zIu!Xhc-ZGbTp(Hd6)d3Ofsf-eP5Z7rIF11C$6O#x$?Qp@0r2*$ddtOX9tYN8H^g@? zDVa&6rJX_RECJXh5*I9^04DY36!qA~>ofJ8Oeyi|>!hdcvZa90;| zrJ6;~^WbAc+6LcyT~18_ub<7rz)?n7^Gi`C+1A}|@lJ(OvT%5?_H{VC<<#ydyjM!y zNmYhB%_)P%z1h-dQ5k-{qX6NlRXuHp{{7p~;X}(3)y5E^JJ;Wf{_BdXqf9#&iJyx{ z+Slx7E}hnQXL;2dO%C?w@s?70SSZqrpWE>g%4hsQ%SQ=(#9GI}u5aW-{imI6cSUx& z>|ciG_AyH9jHbD=Nk}LI{bUQ|pFZYM`mouTx?n8txJ1pTLL<+JD^*!#W9eFC6bfZR zY>iUViJqJPx$p*d0b$qdjeM%c0{Bx7cVcK!pV}pwu=+lmD%pm_Zj$;P$bC6P@mcW z6`HhDmQ4`b#Ay<1F=Jc$jb879OT2j(jv2}OK~^fFaM04hqJ6e_f~r{8GL%JNS1OJ8 zNY1iK`#*(U@A@`O$<2h+`U_HD4LD|MVfX%&_aN0!VUqB)n3?&Mqj()*r>2uFz)US= znd2`}%KO^o%9ScKJmXW8^B&JH(a@p){9_WcQPH~VKA@1TWo^n z{ev>Hzm!Z}8{6E4AQcs)!ZKf4Gg9!)*l0tTiR{F)u$RJSB=&L;OvR)JoN^TD$D&0K z2mz1p*|!9)Ze&(Qo;6gCT9gSrTp520Ne>m7v*F>5Kzm6nHPzCz;|`BZyK=#J=Z$`e zEl{T8@Ud$IxleC{?7Ls~!_xpK48HTFbc=~wJ0`>Vq^{21$?%Khq4Jl=SO_--_Q_#8 zbmDO9n{Xi7h0IAP%_uQ2trYH%xgASn+;vF26zvo*J#6x}YxTv6%<@C6&6NpqJjK}U9;prqbhx!i|VD+7>(KTMVDgok* zh!f7Vn>>qm0$6BLldOgkZDb;k4rCb`K82}Jot0yW%)22dS2hweau@j+uUc~NZR5}B zn`xj45)mNX1LyR$EeGQ0tiT>97p_~T9b(s55%p(&gM zmGI$oN_{uJ>Du!5>BPh}w-9g5!r}<^{lk$2y3dhYerGJ+h3br~QX7};+n_$4k6TMA ziN{^!Q60@FxrFBu1)Nq}KtB=oK_j)X#H;w(ozhXJ!I5J?><*^5W?c;0m2V*gc&T+x zALFT)-g5-b1I(N21aqfgsSPz`9m z+XQ1)1MJ#|E8#KDGv`~)^diYAWf&nr3rm8AT`*W@E6_{1`sx8*H<+%5gu^CW+v@PF zep7I{0ZAKGyP$@RrhnR>zHaOY6qK*ebZP{#oVo3~X)X)9x{V&eBmt?`^sz>n*=h{|6X9!S5S|+?cnd^g{pM#|WYWb?4?|#53ye#;50yCW5E8ifVs=;P z6pl!sFSq`(w}TNG9pn9DpS3+dq1cg?Q&%vINqNyvbTlE%zbAaU#6ccN?nTwTj%2Y4 zXNb1aAyimQmHDt_NM}FU$Pqek@ZAw6 zM;U($xBH;mWTQoL1AEOm3OiuekbNX!yl<0rYA@qLYl@J#Rjhwkz9P6gg7pf+=1zO?hWZFUL zOPamN?X~^~hws;SAi;djj^fmzR%RcH8~4=y?lXnS1!WR5f#Pj|EU)iz)I@ZEd6>_5iW-$lLm&b=ENjg1a3@^yS>#uJyz(mty1-?n-T;z4h&K^p3n zO&O#Io&GJ&-hOVr=e2EEYWx~kMlJ_hclsw!#vQEqosc*6cL4tDyq8=~wEX3%nJtiC zky=F7w%}|lt*pJ#wTLSfu6HX1FsN#(74y&`dP6(2I1}!!^&Unx@8CszJDA%(oit0$ zoIOB8@Tu1Vkgmh*e&e_u>1%skXz;=~BUVl_i96eeoG-<34g8w5qJPMb#886e&P3d| zhTelWm(Wv92{$-L_7Pz^t9bsH`dc&3_mt@x``y9S1kJ4&fm z^Ii$Gumzy4)lf3KyCgn1dhvBb*wUsL-^V?B3$%E)!GHQLUdZYGxvb8>LQ{8MKO>8G z_=eHxehF(_f-<+Er%62L*UwD_CiepDe=C5_DN#mO6h$q|8t%Rv=nhD8y0m@w}()xJ@t>1yc~`t6XRb=902g~TIMW4On3 z8%TMZdp336q!u~R6qQw+8tOVp@O~Iuw84%UC*H2fu+SG9<&|BX_e5HL&+vz{_*UEO{5oB}+$(c)|Gc7mZi{pK11gBlb?=fqJT&bL zj5?^tiv4rjv(B^~nH|)+_Jn_HRg3PO`m8-oJtLJqSK3ADpw=j~bMB^~>WaU<$L*dv zD6&b4J}@wVs;}*y?cLmuKVvm!6Om?jvFB2$ZVv`|&YRnTs3y->82Mq04xJ%%+A|?O zqDuCWW0aJ4Y?aTx|L98E=4CXod#>u@F;4JUr^ei5oT*c1f-%sy97|wni8Qt&oF1W%rj$NzC7JRyKOs#i4D1|yQQXg5tIhF8ejtBqt zjR);l=11gp6m3?@?&hTc)YZP|L3$9@$_wf(vfxbWtxb>7*lOghGg2el@->gQ1*+=U0(AxL*#dn%)Vl>LjYGDP`_qhn_~|vB;+u^r zFr(RYbRI~sb#bV^pU=r)kO6bjNPbA5gs2xI>qik59-h&bNt4h<9`mJe5?)JLJ>H84srp&^((7v3fIL#mAV zPq`P#s^Qa#94a9To~mfAG%Fx8oTkEfitz~Cn*$ido%aygVb}UT6P$LB5K~jtly&>; zl<&1Z=D=HfydQM)cUD<@-~p%SHZ?um^s3GjEKvN8<3185mc465QNicw4cElO`Lxb5 z9|%Ilzn{<`gX)OKL0JIUY$a&se6~Ncfh&lyA?)!?RnL0)xdT!IoL+jn)TZ$8zbxJV zxVL`g?{VA})!y70Y^0u##mkprEb8b27ggPngg2yewEBwH#63{N9ggE_z11q`j`dO< z`Z6y-^+ZaC!zEU&tt##ZFD@fo6|0fOj$9u{i#MDi5?Av+kb4fF6x0yrJKj}(EE<_9 ziJAA%53h0o$h6O|**TNKUyMd!Q&IMpt@0TKJ)PQ30WeLaP5|Fa zJeBs(kyC1g{ejgT{9WnRA=NhRvjO-^#m;lyo>R$`x=+OH?|{A1euYzFoXVAFMRmq8 zb?!k%LHSPg4Lz2&9v%pC-r@PNDImNc#+@3?WHMQ_OC|XIQ%MenhWD8dy778F67qB= z;4;THFf3uD0Fq$7&`?seQ#VhvKWbMb4oWWz3Wxgo5PgRA3o4*iseVSYOGbmQ^9Ahm0=Xpdvsl^H6VY)ePR4JB z# zOHxm}w!D^4#zy@z@mPtq3IBc@>(C=(!^XEps3Z^PCtO0JzupN8Mbx|6eNPKZcY#@p}J-Z?RW1_jq?0Jd-{n zs*Gok{_+pRq3B`nq6L#S5Kq!@tjju`sd9>&q~!1cI3GM&6ZypTJNfL*3Rv4(crEWy z_=MrlyRpaub~P5y3|Qd605B$DKLA)|W3JT8u!WKDVCck82C~eEE}KLaxVb~6#ubdO zdI^WqiocfYMpsKnsLb&g<_Ql>*cg6a5(Cg!2QELBRDZ_5Fui03g;3@p+YAkp-rn(K~`m$Va_LGO-o_@a@%P?|5UaZH>Waspo8wEjPZ)PJh%Pd^0;kP7)a@iEO}*rI<0JL7Ct6A?pl-RuvVe**MJ6x3yjPsi2^8Xsjm3XtDVtsY+uvEZHQUUGcPVq`6oYJ})l z7ldtX3c_Y%RVQOd3sbqJA{X>k36Zg%+0rw*qnjl`{=OU+7gOvKMjf`c^`puvmOTti zj;R=h6O^-WFwmkEfkLpo(q}%b!wS;WirT4sa1Dh-qbj`8P_JN%p1N8%NItbI{E7B& z54-c6idoQ*$^ZpwQQT9yTz-7UT`kPbPXncVbKRlRG}&j}^NE?3F!V1(-nq$^QMz88 zgQltv;DEp_u3o6~<@l3*pC-O1l%-|UWxo`g`l z836w?%d@`Qb(j?AUHLdY+}L+eQB1i@pg9bl2IE8yE2VKg#-U7D6+M}6ehwh*SE{i(2d$x+wckZ0s02&V|yY`K( zQ4}a%Zk$_|FzVv z@EmSd>a2tvRt7jubY+niW41u**Y0F!BsP{6Xda4LMPrSoq+zSe7LI*}lHl(A5MR0C z7DzcsU||rd7@R;l;{V9ipGeRM23OVC49?AIN!#DpQ}Tw2%qFN(fqoKnL5mSuOnJ+y z6%Jlj<02Vo0noS*#-h3w6Z|Sv!kdEzpSGR9Ic1fs*6#ITv0zl6awka(6TU z`nMBgQa>6=DhkiY3cyiQT!n`0aG*WclH2+6w}jy2l`=O*ALdqKcR+FNtHsMSp|f`o zQoq*)PqE}CKjrbW79(}*BY%{5j>~fCPpd8S_%c>~`M3w5+A=0YA}n zXO0_8>|?UXjI&h-77rz13J~A-#X&f6g~tW&d8?J>-HgH*uB6h-;pfHA;T~k+Naob| zl;@N+)*Pl;H)_L&4)JGyrlhhY*R)#|C<|6hHk<;kUrA)q@NHT3k-6Am^$60Q2ZiA$ zu!prU{B<6%VtNsB}zx4XqwmBn-2XZD`p!ZW|sg6kej)XX`L3sJowbZ;r z4a_oukff$~9eAo`BGEtIqFb9ic63)Q+F!NE zTBTkkOcm!?j}A~cZLJI^)yNI#YkU^&&o=h~wKWf|Hd6S?V z>;VGw6FL0)n~45k7gliJ#=z#TPNeEuJFMlBi&48tJH#q=s)s{oh^7p+el8@z=r5yl z4yvvu{BP=2{%b&&ZDH~VgP!WTVezmJwkEzCU=uXJ949pZP;3Yt+mW4-;P1n*;Mp*^ zMvc{rUf1NJ0Gx7yi*iq$>}x$)q<_5R4(yNLjG)lcrnHH z3C}C8M@E*vgCn|Np=im(1*g+#lCB+w1 z^9~mF+qH)=h0G4;R^kgG0}JQ}0ncDGvLwcE4DQ)IJyMlew*?Bxn&=iLu+%+0;nOM< zr$Su2>~#HTFdrH`mh@)GAPq0iDb24is+dyBz-#876dPzuO;nB5KuI4UK#Vn%m>SvF zi8%?Z-?~m1S)v(iUV$zo8h1SZ!5uBIaErGW&4Z`R%nJ&puFTj5T+7uIwvMXL z;)AZO&w9Ls6gsgE>EYv$FG%GIrC4R`p8FQPc)1Gy&Wxw&z1WwRYA-wq&GdFsk`=TyY4wVYBqDsCcE~+J zcwiMcDYxq^8@gGJa8G(^GBOD7ZK zoltuBHO=l57-iqp(e|y#M$2h)IPi|jLyU{hBeM)^h--BzS*y_54Z?8drN|Yp!Jpyk zE3&_ib>^=fQrGIrzqYG-!dg>0g{M%W5P|{v7W~wicST#fNWp<$i*fJX56unRnxiev z%}u#|TuENH;M|F>%*8tg5E~(U4S<=&eqVTYzKW4DBu)mm5Mu>7i<&98Id*S{bfrat z<824&B&8WcSf(;i+X0X=wR+7g*jdNiVp6nVU?*1Gy`1U?%#lVe)R6U{OPiUd4}6$6 zNWMPJR$%q}f4}?6V=*HMaS(NO)1O`bcJ*b>6%_9%m_3CKBXIC{ua$dAO!R*!Ic zszcfr9$H&`xi;eASn3|S!89)BbirUbGhRw_e4ZF{vOdj~>Zqr*Y<(g=BP0&Mi0sCsl<_joFnlOkBR*8dF-qZWqoUe~_02y`_CCe46I{ zEbNPgr+#4+Q=`Twc1cip$5<^{Fm&8!x*H?x#nxRCbkZu$rm9@R;JkELwP8d6tle}! zN<#oRjzpVVzKepo4&|4*8or@DYzCN=*cTR70>l+b+z3nNE&U>B6hqRmuVLZqsMB2Q z-pWA^EXb)eOgpPWCvxI9{DJNP0RQ3iNAX0*f~N(?GXVNf{m+$9jrGfTd>}+E?AG{! zj~17aKEGS)y>xj=p0JYi@-fl72UZ|tCTeVZ6elnlqzPztcGS=#kTdhLtYV(}2Q~j6BH!^htJV3#|6&IJYr#bJ4*vUsX}kAzZscNKw(n@9F+NB4 z7Budsi=akXC%=oQZe|NQp8P1jO0BjV+o>pI@wKVB6Yybs&IcNyu_()Z?8$XMnUXx`Q1*k?2LJ(@TObZY0nn3KFa zUOofA_Bb}}C@boZUYbLC*(Yv{0+@OE*bAL5E@*jjuELKkb4qLnahm+~&%go9te&_X zQ%~GRJ>*Apv-7~vr=t9OMrqHK!KoIt;~6}(+0Lh*K#LytT{U9y;NmLt{0x2!ZA?Dm z{iw2K{_VC&c%Iyi=K~*?(oiJcSSB1zE__6B4U|LqRkbqe@Bsy^#^yk`f}u2wxxpiexE<1i5H% z5g=FAdt9BewdBa55&8OyGG+I9=21Sv;IQZ!Nvi}%LpL}3(7eyUvx-{y8=hE9p-KF( z9--d;hyxz$5>P$tIPTB1G+5`;?5Jh952 zu5%HQy}?Zdreia*W&^yDXY0q?pB^b&HSJfRUcVh?Exv2kxh6y)iFix`jQVuJ7UZPkrk^B@J3$O z4H;EqNMKvy2$aDX@&XJvy5q-OuUZqiROT~V%b=p*q+x%r`x%|eo-s^DpVJGLQ8P!7 zf}7DYP`jk{^>=W7-BC;5$t{ri$%nSa%w>~}934xTS8$8B7gru-G z0^w=@f;^{wm*c^XCzc3IYdxzCMreWnfp6pCE@9Tre2T0P2uSx3miW+_IztKNMO3>A zvCBi%csl0D4YdlU#bhw2tRwA4&v(ZNqFrF*EHlD}sj(o73DSfqYb^?7_`N@^3j4!kK##$C)3pRISI-vRLD*W+}zf_S(s}c_Pe!rNw-h# za-1gF_^?S3)5`fK^;bZVInCsRGLx>`z4<+6>RSj{Zz(7~y{fNf)3zC=N5)%SEQ&u$ zr8U^gvF)3B1%}0+Z3p}2RvYB+eLyR{2>Q0bb{zb;6#Ypc*aDqY`1(ua=9zZ2gfTs} z)UUUCpiI|xXtkhv2H`lY;k==uSrT8)8e{emb0+)jv?iVdr&vvA)aMoMtuzI^x>+g+ z?#A4)I?-`VzxI`otxQ05VU-DYg-|{Pz-%BpP;!rSE~l^5(F0UT5W^)b6m}{Tcz^5{ zQS0;fxV;%6H;!LvJoDnZ&fi9FKY>9YEl;ApMcGcT#>tmkpz}SSwm>h}LcTJ0J<7Dd zo6o!PXGz|II>;y~X2pguG7pUiA3j!|aG2RTI6J8K)UEd80;=7EWu)oU~xV3_?RF z_Ay9^=(H4Lrb?=HevwyhmvUdO0TW`A>0T|B9Dl0t7w8nJzVB}W`+q+7IV>A|^SZkG z$M&=@om(L1*H50G(m!}LZGQ0cg@=s}D{*dTpog}N_{U<>og#vKaC!}z%J-f4{meRP zLXGA#K#DVopUyhXy@PV z*=l%|ySrh9k{>z{5g8wB5bz)`5}jX~eBy>oCv?FS)!;l4Q@h+E1uedDbA9<$-Qqd_ zZz>Tk{m^;Q_gUUD@?Heyq}5xhYjzyM#B~YZmTm2v*mPV8@Bb=hbOTYgX=#4GaxhTC z{tlGQl7Bed;PL~2qefO{l?V_m25-8LV23ZO_FOsFWa;j>_?7M|&j4=%mntr}25kgn zOR)?a;1zvVcOxS;3EV@N`L*7CF;`D+s^=Sd5r5T+(UhvPF4PyD=64`-xWt24Xzd+C zW`kduW)oC`Kf6^1yD*uHfs1O#{h4%H=4M!%D5{MbhBI_Vb^}0EM zwqkC4KR*HYX|XHiC|nqx^$r3fQM)pxJ4Tz<)j5ZKLF9=V z{r%Re%5McYx`w2i64nLtWE1=-dssv5eF-#rBe28fy7;*Rf#DlvCgYP3AvyqIukckqwG~{ zjczmo3ar)%918{j$aw75eQ=^zZ*n7zJE=D8dF5A3<7}P#@c&@%y~CQy*7ji>`v^J+ z8Iht6O-KY42nYx|LoWeCvI#AsbP}3@fW$WHfKn0+RYFmENJtPQKtMo2S`aXSgbpf% zUX&`k1e82hEJoR=JiS4k{gAtJl z$uJBd68?*IOMt6Oc`jXDzmki%jc5JIgWS53fA|UlcW=}t3}x1rBpd6ZahSdQjF4MM zBkh`SX83(j_BPOOqnBNgAMp_(T_HN*1BcbqVy%vc|0wHP zARo4nRn}?~SwMbq6&dI$l4Io3`i$^+vh=Vj?P}s{ zA9(S;K~>_yzkb>OH^+Z9wcdVKY|r>tu`~RJ<>HH1Z%lot`T}e{Z+m+*;6v+miC>bA zY-AXhjom(5+`t7p(+M6wUVd=8`yoO1)F3moX8zS(KQV=tI5^UEGahu#3Z@;L97%Gd zyQO3Kdg_Ut5!ZTb)DyuU8Q8q&QbZ7&9Q&w2?zDcyqW$FEkPCa4N8IswJ+(F})LPTW z^{Pfy+e6Q}!o6S0(jgLvbolfPeyP^nSKtAVz&nvylZ zRVr_EJ03l(G74!|C?Rrk7u{;r8`SRd-N3K*N$z|JI0i}!OOxUjW}EU}`nBBaAF52G zkP*Je2(7f|EBx)L*t)Ul>i%X;qWkK&wZEW}>o;p+k*VtsmqeY`Uc7%d6NGNK+2cID zdM0gX`y#0?w|5YdAO1dkm2N&c%gK~fpb2u98Z{B&<>2GL@5>7Q?pyz-$-0U2eFA`8GW=N&&%9OfkRx#{G zG|QzSF&8C0!k4Q1#ZC??+;}vq#;lw~aYg=ZQZ9w2s()BI9vLdnZxLd0!Pj<1aP%~u z*(Y=ZLF&Ngy8z6lIj-J>9bqUSH{JWCH-hA zk|)w*hU#Ap!J4TP*5G^#RnY#U!S=sC62tPimoQ)zkb~kP(lV=IakpPJ3$#DBSn_hN z>B3jBOQIY#vacth#swqI4&SpD9*{I75F~!zihkL*?_}|s<|6nnO8$ObQx5ZxCt?;o z9B@{4F}gaEXkQ!=GL5}+zxP90gxpU%YQLu&eEXr6=UbTH)AxS6;8NNDdn5mY3N%kW zc!cZ>QIPA2JAWVf#eBLc5{;47)7(&+@BqIj%TTjZ+%08)7qx!{Uo|%hS|1utQsH5& zQa^hYG&rnSum>wv@=}P`A0zg`0FEl+InD0LGNd3`*4a2=NCTkQqs5VlbpaJ1hg=QY zW2vgMHL7*;)K9@d@(eV9yNDXz#7?wm0%4v>a-S3IR<|6foR(llGh=IQ)Qai9P@@u$ zkqvgXCG^=A&ig`0hNOn%FOCaHog4WebJ7e^9NGsv{IbSnH={~bnFRqEK`_pH?$5KG zY3pso(P4cB;`Gk8s=ct)${rtjZJ%G=EQxR7Pys;sKX2yU+RY6qGd2sNyQkREm=SLc ziyO?Y!>4Xr&X&}iLFEfRAC8U{q3!?`6Uu0)G_EY(i(GEOW~9(2u&CVvP8qyoRX1aK z3|3&^${A58P5cZ}0GU=KSNH>RzRvq~QmB0_7p58DnbCY=XY-P1mTIlU zMe-v2Mn-CMN-cTI69@JnY`vZW3HjQirc*TA zRWstD^(q-hmQNH2ci&l#9r4<4;>*WK7SioaRj*l4?9Q%1_CvB_1^@?<>tv}@!ppu- ziyk`bn{W*SH4@Axa|70emB!JPYZd7~ALXl7bob}t*X={4P66m!VBbrrMX+P#&WkDW zJiG@mU1vmr>^Cv42z}n>!bZcgb8C*)$& zOu)RXo!A9n$&wMw1JA8tdm9)68yBZVn^2-;ckX0jASIUC3;1ZIqoAiQA>U=^!8tof zg5mkl${D!%#}pf8U;C>7vH|#c;lN4Sq{xq1Ub2KbDNh}YYIWYTnqhxTa=IC9R4%kq zcs6J-XWli)siV~9=R5#P9~^fqvC-TB2V;A7#rB?CrWr}r4?||B{R66GDTAf}wJ zqJ8hFIhK@0E9b&(3ucNVEiO}cu$@t{*h?utb!gfnBBtB4QFC>eMrb}yabAs%QCH*@ za0gQjxG0ThZ@ZqnemV3e*6~YctT0jEWi1FUE5tFYwmYB0J@5*OL)3sX2z?IlVZRBR zWDt{H>^zaxwqCC8YDZn?^y1pQah>7N>z<|FjNM>#@;jY$^nP^4IF@N#)3>AG>mp+uI$D z!8;ApIw?xBt$6?wiR|9)c07U0NnQ)Fp6GB|XCtQ1ok86&P57KTD6=5On$d$uKOQ@- zXL%6W0RbQ0CQUYiQu^qiLo*_ZLCAz@*NN(Oz14!RVgtk+q>BWOcitwji{-X(ub$?r z%rnT9rBc!f<}+IBZFgfuH)bf)=zDbNN#ZiuTNs=~-<#?L{)JjOTSIG~ts-+Er< zoj$1pIPHh#G^kZxx7s{iKLe~^epoeB{6Mi1Xc>6lq=>ofwsDb~4c)rAI$d*I=Qh8p zn(Mze!9>E4=AThqTsLng^Dd|7-f@1E-~D`C49wR(u79Z~Z`Gv-(HIF{tJr!){k8eT

    Y% zh!S^}D2vyKj-9hNaonhDn^DH8s-hs|Ho-XQvNwVx0s@ucAb$1BeX0qri)AA^K{D)> zS%2REQvV{1!e*O>m*-u*Ji0S}>kPy$Y1y-&#;a!k$gceSst%=Mb>=(${l0A?8bmHu z5nL!l9lxz!cj@(Tj$b@wcVUF9bb_-(qMMePEo2;NaS|nTr1=WBX76kmJm#X&wxp^X zsLb25&A{eb-5YCybpb5-5)G>=G=p_I(wH=+TMS$8-loaRg+o2pZD+zZOc$(t(vu3H z_)1WRB+&M}QFT5qvujtng*RM!pZHGhpw*73>j{F7M!5}U{$b@`#ZY!BpvR8-)gE9P z9#5H9g}2?kn~{;?mU%~$u*Iv=r9}2(?%SHwc7g|_7>F=Yta6q-HLlJ`?h<0gBgB^f z!j==AI;j55#v~H;>OH>be1e7inQ`kk8&hP2Vy>+me^>fuhxYWrar{@YRkPecDM!>G zJ#)73q!t7Am=7Q(teoh}!N%7OjqUWE>T#Zkb90_K*@*=wjX((Coz?Pd5a_@mY4@Td zgy88hxst)19F=7TU7=2mr&~8FT*RuN6fgIc^nMkaG`?YU*nSh=;J=Vni7(2laFJ}A za&R4Y)HO5XV8Jw@WW?(e>0_@L^@sqsC){Ebk*7u0Ul%xE| z%Q+VgpY_`o#M{RWwGB*$IYazumEAEJ!zE_Xw&T;DA_7T@dD5(pE5`44laU%=?_q@EWeeubD23suCOn{YS+GLE43oZ;>L zDpnv+FL}v3=d7_Zm8mr*%j)rs1=EfY|cf_r?8#S}SjxBl_AmlcU83`oD_3 zYk+?hqc%AO%-?uFM*S+*dtrZq&BS4H^`;Mx!1;0LlL5p(?`9%$WY%Bc_vY&3X8Wxm z>f}eZ4^ca-QxGm%=Zef(7~Ia8j9>O_dy*RMx0V4rUSksBLT7+pv&ve{>}{W0n_^$3 zfL>!P7#`JJHDmm<+8oAYOH^hpr$}DJ0e^AauBSsbjP_XF^8{#qqC9#rwY#f z#wpiW&#Ri}^z~^kyBsUx&Q1%jEmrmssO3${Cm@GW%Aqws(>*&Ll_G8RAgu=xLy=~} z;NQYVyE&k)#3}oLCo|X*0}FUSpoU#%mCXKvl-?JFvJqb^-2rAO3n)dR<`i?9{U zd_HS1T`u|ataweR0_=@jE7*b?r}9bG*&(C=1gnPE%!tAi(%7+gr=W0-{Myv&W#QiD zh;mo3jtKq~4BeNX<1Jwd4NMGIguFqSZa@<}^g%BOxhy@lq`02(*@`dx0zsfmnuTii zN+hO!1SHJ`%#gi+1zj~}kqED03eZ8!1Fkz?R(3hDwO?1NF2z^sN+P}4OdMwD#!h}ggWIin~SO3t5r5(T!q8b z{Ct*G$jy8CB>~j-Li+*paz*oDs2fqljXGWP?1L&U<_fPpjNl@7t(;LwpbrC}2$vu% zVw*Uj*3;0-X#xm^9soG;KnI_J&7|nzB=cFJ)15VZ6GV9ZXaO@yt(bT8d7lHPgzRjn z5_C%!Ue?x!Kt&x6~B_ zICo1u3vlM?-GBnU0iQEm_T1*HwE< zE=aOwx?LePujXMS%wl?lqo8}=BzGb5E(3RGSwo=1%yS|gfT1b^u zKaM82$x<&VFOb806$@(9nrL!w0I!*vB27A<{`*dAw!*|b5a`9D0)Jr}kN6hhej^rO zy^5*nh)_?D(j7(4UFiNs)iqme7~LYtoY_@Cs-aBTuIpk598_+lQ&t@l`KI;q;KFo6 zJIXs}{n}w3*d-3%_$Jk`HXl2KofRp4d4*m!(3g}t=KiB0LGwiQaAB2o4Ja+b>RJ7urHHqzNS!p5buz>G&;AzKY2>Zk{sKT!OMT4C0cwJ?$ki&x+Yno04Zm zago0JApQwK^50(O;^9JlYuzil1}r=nq+5$q^qHbmGoq}>7;culx~sOV;m#kEFIGWo zx;6da>8V5tZD&5YQ!0h&nRF}n;bD|svO)W*vB;su(M;$;=U1UGRCO!pVEE~W7rimZ zK_kgXJ_qc%$Fi}7W_M3;nk=ooV@oubvz@TVTdQJ^=g$2h6o`3eGeY%hN0DCaAH_<&3 z&D-tgFD#BuQH*x}h3F<=m+85JGfz;9KM@P;XMh4Jc-y_9K-ZLhf?)84&m0!gyUg#^;X7J zX&s`YX>$v-ibiAV1;1{P;oKbkk|r&6gTz4T=C+%lf;>Ia^;lilQJP40jneSx`KV(a zK5jMaCB(?n5%NcpZh_+UDd^pH5H4Fh(CLnP?&19N_onNjv$=4KU=bfVS4Gd#oOx9( zTPO)fsAY>U*&Te=IIeHnpbP)8>1ZcK_D4z*=YAHkjMCcE&vBrg%Y#~N;VNnFrPoUY>?d==FjTW%}#bm@ck<5U4= z;~F4E$Q|Eag_RcgsUJ2>I-kP~rvvi6rr}y7GbHYK+rJg4+J1B+<4ak??Ymy>-9}ky07AT9q6K}($Ej*6j!Gfk)!EGW#3YsChw}#|CpjzXde*v_moE}KOFy-UEy2G?z8_}(EqWH z%;g%Mtv+x){`z1-(HUIfYWo5ii=AMGLdhkbQWBn0yB>VLcEtS8Z)7J|uQ{9MqD7wT=;Im*a2!DJ3X=7*@Lyv4(Q?f_4v<@ImO7O@9=e0 zITj3%dyduAjG()Qm&C5F?z@ybF=nDRUuFTI6igQPRnKDzVjHqs`VT89IlYMmCHxV1 zh3BR(`RQOh5y~aVy4axCbWt|RtCccnd56+3b8+{3X6wxhwPrvh(=L}@Tjoa9me)_$ zYrF0`FX?rB{zC9INacpr6bNIK$jZpj2UaO&vkQk7Qs=M4zeC)6iOINT{QFEJe?V;! zFA<%7F*}E+^y)<$-*MLHv|od@q9#Hu=h^MC1%rbgRsOYp(~SM+di_Qg1ubZ_;TUhX ziis?3l8r~W8k%Di@hbZB-AGnMcrk`z;ce_U z)dDxo3(h4KlrtTkyA+q|| zYV(5DrtN@RC-G+sqf#?I&JeYmEeV&r^on!b#feKoPmHqH+`~sN%@)eZ1EU zS9P6cc&h7@Vjf&J^WU!V{n>i{@5}}2HadDMGeVZ&$W|JxlPyJZL_v<+$x@I{zsE|H z3>u!Ra`v0Kv7mw7@AG2boF4-Rl>$H$Tttw*%-o~Q?TCkEx!W=<(bP}u`WZM#Qg4X^ z27Bti-NM+xyCw=MU@xxaz_^WhpU!Z*vnV8N zgr))`YHWG&?S0ogd=k5O8)2s0DI#gPZgi%vi z)WOIH)HeP-7KPymy&qK7{!9{ztKk%>iOtRi2?m|i3rWs}B{mpd07Wpryw3g;w2+&t zPpC#uXSDT6s6RJwE#4k1HF&yf%N$NTRTr4 zmP=sZA50yvO>Xds%tT}A~w5E0%}_;Be=C`a$Xn? zim1%?m+x)VTr; uIV+bu92mL&1fK<@M!0h@PI`MtnPzhKZiC*&tE47W?E~52tmO z&X!)K%t#9~ekr<@II{=Nd$!#RYA7O|rF6wCVf0T>8GyiSm>L`UN(3T(!bR_@n<+o< z*zp^y*tZH6&U{|t(~_!!u{!FEaFy*G9;Qv<)XQH6S%p<4)9sRmD=?iZf;e-kLTIg( zd>PBXI7vSV=rR*6*q|iL(8FRAzxMehDA%|aQ6L2_HXe`L-95RNm8ee~mCMR1N(gKj z8VLtU!K2w!dhN}&-*>E-Rw!;1X%D|Hd&vk(9m2x~8O4x;-ThPWC2C;%1eI+*Qk}$~YdVW<44_qqYn>D^hG~V@l`p+-^ul6$( z)`j9aaElO{7gDQawIra+yr+#8TT0aCiEfx_kD2cgFM^fH4ONoL-1v%y_YMwrHIJubzy^hx_IR~vEz$2?% zXpCf$ZUi>aL8NWf1;%acq-wv{rKYWY=6Shg6N0%D+NYW3fa z@4^ID#eUSDyjEXR7`0+Lt%BF-B6<-=XGnZx2dXiS4ePVyZ}yIsS^Rx-HcB_$eO5hi zl{cp8fN;$oP|Lx)#d3W0hHG@XWQt!QTg0Uetime|4!}ns{%G+z#QtaJ)-l$_QRHfL z7t$2wHFn$oKi(-6_+W;w=|pJ+c9k<-nlZigk;7~0u8Vj+ijlD6vdj>nx_r7L2|Jr{ z$&<3~sN6LKZGQ&epVz*-`u1YH@M9yB^tyL-R8**&mrYSoIV3^ z%*|!4ngK}r9muv$)hA}i52xuHx_Pl?r`07K)ZN{f;aEq}@c=(kY6q_1)X@Pr4N%+@$;A%`Xg5 z7i04s%tMu3{^oA+7f;{$WMGZ-Np?FdD|SwSVoi&RQK34arPbE+?#Iq6r3}5tK$12o zPj{YMP|d669fOjM)0cZz{1016h4cp8eb1LGNq?JLh`6PG@c3%Q)e&{2tc9WyQbEkQ zVgXjRckB(PNMc$PLy97{6At<^^4q$FjSA-ZhdaGika3?{tyYmx9)Kj^)AU=)gNEG? z-E+x+gO$8VJIi$lD#}456cnA}l+y28V+%po?x|(eCW{3w^!+Fk#1HM*ldhe-6p1| zo6)O#B`eSt=1h4%;brK=jiBYS+=gkFwc|@ybYISR|F}Z*Y5$?y5Hn1-0+}l#n+7w? zhGT2F!-!MJ^GA<}%_`Sxp%0a$qVniP6!YIiA5=b0UqKaqlDN>it&7{gsWcX>qbDmJ zV5P&dUN^+**m_cWF}bWR%oi@MSb#JeSBXbk05$`!TOQ7+|2e{N|uJulIPZkSL6#?&V(J8WAzN~Vv*fI3ZW%K((&7lf(UY#wj= zNDaWRR9ChN`iajfHQf-ml7wG=O_(96p!1Pkl%ra{24JV6E3y@2L|*Hkod6uY+N!#4BB*Ha0*Wf3bZTpv`O3+S?QsIx-7 zrPsABSHA%9VI*^wD1zFAsV(j1#vRqGX~R06s|gc6x#J&+5pIc(QX3_h53fY7lu9UBy+1CLJBV!#N;y zwGc4FB7&=eT%Up9C$1hFGHZ@Y%7QRCLu2$RVcZy%kFM99sYzv2M(xZ9Q8E5$vptV=%oXg!c|yO z+50UYE*2f@afw(S-=>p%5+~DTM??HpSQx#nAj}42@R-up+ZW2u0MZCcrh->ZQ}!=! zC+p&Xho8#*P2Yva%F15Zk3!IVgDq4Bcg=dVc1JDieHFXE=^uCiWcnv$&2D$O2(ec5 zy|VYcq~#(~esNnLo%jk_cImec-V8 zrcUybV_u{PyP*l!wQIR1c0C=PUWi=5J=o4nC1&2NN~ElD^>d=L)ZK(EC~$!+h35ab zc!_08FDqrRWigsP&WzzGcxxc{W8G9@WaJqS&a|k4ArL{vdXh2f&H3w`5@xO;+Ks|6 zGg%>vo)rW2qp=VK)%Je->M*lb6iQUTQ>O+D5C4AX^#|9NNC;c0R@n|;{y1q!UV1HX zOupR2x_{CW{miX(%}qqs%HC%CuQ0Yh<4SyQB;Vm~e+HlUS6JMi@e{tG;r@Yk6Gfo- zA648XStqYsW?m&>!FC#ii-^rF7eqb&-qi88Q7A#}JH*9Y+2e>pf`x@oes(s6kP=?P z4|X(tjs-YOq#4o{dUZ$r_pF5z*)^1rqA(B$FX`~Kp9bf>5UDjdW{Y~8#Zg+!D)lG# zha0CbI*Gf76i%=2N_Ah5^vWnXthUZ0Dh*8y;xx7W`zeUd=? z)*KnwDh}Fl5MWnjE*WZ?bE(c(FBq5KrU;K+__dZRR{|NQqo02 zYMPh+0PG4E3gPevd7ejxLOr zD=8r*78YikH)}p7Ln%}~O3~YhBLXK{fd;qc|NL2j^jU*O&+$sNbaf*e`lgEPbD(9C zo_RO6+Dt@oH#1#>7^iW2XB0;GRk0C>$94=>MTdqBUhYRLheXR4OkKZ`RKS2rY51kx zUxI+%)QOeFrWmG<5K2>XQ`X|WNU_L5%gi=@Te<4v!;A1sQ9xtx)~?_Gqjm6~j>I3` z82fD6usZCaC5jAiV{QLev7Qk5X;I|izfcr@_^*Zlu-M-?r76TgT(VCux>rB@+jwX` zI=*Kjj3fw6=@>t!W|JTO#OA;z8BfVRbNJOCXXB!N@H!nwxG2Ig8UOI|Z|`bCbnkpXyNDeK}IGGVIb+p!`W*@b#gTU?CYxkI$jgF6t8JTv3a;r_fu@DE9-!6 zG^0E4^b3lpSg9g#?)4DXfkVh6QibY#nUJSd% zZG#4_?UU<^tngJaaQs+cR+}^7?X7dHn4+&tjFGl;gDwI%MLi)f(j4+T?!uymeEf{j z;9NxIij&%$M)K7UJ^->+4lO$lI0?h)V4eC1N~ zG)2n3GB>A1sCl%>LrH;lelQ3)fvibMwD)u&SdI2U+=~r3Y}2N)XpuhrRE6C$?wD|Yk9&!c20Yd2B|D9>nZSX-we|SE0be;-HhxX}%Qd&F z`2@y~b@0&7b$jx>B~Al~w$h+Hk0g}OM0m(TVE8xypq@QD%ll8A=(iIx!4QUtxo~vi zYP^~IekG6;k3OR@Ul{3&*G0~p-$acV=)beoNl@J^%j}_us6$ld1bYig0{sUo!4yeXH18?WHH@y#k~ciA$I=cV)_Z zv*GF{#ItG%r@tO0WsZ6rKhkYe+oHGvg=5ShPGRC zm?f2AO9zG4w*2w86^$J&d11>(Z^~1ntG_#H#4m`6(Q`=kVd^2L- zRp9Y_MpE*Eh0)V`(Jrr8*HELvE>nE)~~QUN6LX-LCciE!Gp@D0$wCI zyRc0ypfw!6Tkx}$CphDydHq7qq}!BgqTe%$}Z~^G>h@dqz&36`^%4H-F0$g#zm-(TzJ@ zSyL>q9C7uJTI$Mk5qjv;%2;l(6CNRgqkME2;iH*nZD;=DLGVI&SXKQHOuE=%)2WY@ zaWv0S$8(MB#>9cWKc=Zg-rd@du&D^mm$EVN{US)+8~0Ug-%6xzMv!;sB?kSm2@h1d*KZqvWj{s$Z9?<{M^TY>`O(&) zpsNLg;DET0d?w35VEnwpbsCK&uhq=h-P6xNbLUee0R={~nJJ$m3sd2JROv|S4>RAB znSjq#qaCR>(3^Er=}5H%pm>TP5d9dXsOVukjy@@T`0twhmguFo*NzH0-kbZJ82Pd< z|HHtIT~Gf;M&HUUj+9Sj1$ReJ_aTN#9*wFC!(?3Q^3`?QrCe#$=jlMiQdC6n$o5a< zQCdMkY@aNu?E~1Qcx_C<@M2<4hGmY0(R9LX;lZ&JC+On}U61u%1 zS~@CERw}aR08)iUeq8THJM4hdd@=H%HhMcYY|pNm=6oo#c!l7He%IqujXdiV4k7d zy;UN6walR0KtCMUtXyU^OKo=IZkmRJT&6BETc$iCgOYH8m$+F9xpkcy+Em)?(IR5z zq(-fy-Ym1iat(L2y6ZHgPD}otk+w3oZ^4ONxuryHMO(Ygt`R- zq9_u57pHtLdJ03WE5bWr{yEjZbzIY`yBO^v8_9xZGQ&ahz$kruYrnq!^jv_!L6mba zp69npsAo#BOt?p`;*3ymy#1h|CJ!qlUu32hgjiZqm0lgt8# zTLy4W>Mm=ky2dgSPT937Tsno;>@Ek0Ez)OJuXEcNSceY#kdoV$MO7AI0V0rN@qW6h znf;EFy&Hy)o9%AfhuyFU+!#Fay9n0avYgSv%Hk>ikeeQtKG#gIK9(TFKB#I(B&w@ikZVvHmY z#EygOe=yHj7OgoM{(6n2i*dP9z>Im^)=1rDH`+lcL~@*(Ew;^AYW z2yP|}3k3=9-%w4_$dQ9Potxmk=qL@ndo=3ty0^!|FC3)7rNQ%c*DkQ7tLg-Jhh-GsetJ!7seAWaY6 zfuP3JGcgLj;tCv9DWrawrirGzghm$bJB(rkzu~} zL?8A={ytN_z+Ztg9>7Q(FVF55UXw+b$e(;FM4}*0EqS5{P+3Etjs#Wgpv5+%b{?jT z1qITGeQNOJz!YBI32K}me2tS^x?(UpbSN5~_&J=?4*yjlb`xY2noqyNp4Z|`8UcD3 zqiWQ}+9!wAg*gALQJ8vyCA;I`Ws;-qz<3eyZUROTikGBz|8QCFz12n+Lg%)%-|L(w z!-t)N17=R&Yn8urjQNTgO6f%z%9YRGu*?Rym;g~fnw@s2AVma zh>H@S%JimgWd$Gk*?LH%Ipr@d9kym)-PaT(3c5f#24}(fny@^wLN+}lWBJbg<+!5d znFTz%eOq#}XsV8hZlS=Nx(iiZyM$@bFn^I1$UDq$4>O5r&Tf zsrt4&SI@v0SXk)(p*=G*kEMegpRaY~;piyz1L%P1Tg5HYz>^1lSv&&G#=N;tc1|ZH z`JRKLan3#9w*}vpR!!)TvR}2p42eWfNpPYQhF0x1EPV1}oKY4gKe(MQMPd2+HA^j` z2pT2C#>>fMPoZ`B0~j?daeaWvmlikYHEm-QcwIhEhs!pb`fMn&B()jOg~4gD#9yBD zxy=rfic7Lq+KQW;Ikf6cy_^3kNBNh?p{-CAw`4@1QcDQKk*!6t-+%wCoN?x7PDans zN`p`l#nm=@phPU-!{2xNXS#=r>$I-;JlgX3+pgy~z$)eGj_z-}q;J<_#s0Ub{||L# z#wbl>zYv(0#oI#TZ53~~+ki{T>}-P{vjYxO@c&q1L8I zuV~(m*34hC_bOh@khh$`yf0_M#&gwAF9ts@s@z$?TOP7k$=2&NCSaQ(b7;BoigQTg z9N$DM$qu*@GzS^Rv``K}B3}{POL^xVH3n^bbE;q_$4J>^0)}i49SwTnhL#^WmLwM&W_L<=o#ADbF`-eu^I}~gxd zZ_Llg%BB@<=TL&~=C82pyR!SIW9$1Zp{gP$Awe;zYx1MP z70-XXnSfEmLY+3QBn(O-6=8T^CC z^iFcl>u;L|D=78z3FVe+S}}E(?i{~;S@zMt<9U6v`v1$#f1hsC`FRJCCuj{I@_Eg! zA!hw8$!*izaFU18gVJP9)l+^8YzuFwqQy1Lkq~aB1@B?G0zW8XIQ*0@oZ@9%pXc>6 zA8rIbkr5^$%U|3Ln#` z)^4Z*A@(r!o0a!pd(^AHDNIbE4qBp_eUPdJx7f)-c`C!s@NIF@FF_RNQw;+BHSk8h$NuRcWl zl-TJqlOLl#JflQRWHXTG>1%l4bYx%9$#Fep#j(yBGj9;e0+#*`J}9cLV?wA%Sq}wT|{cXlscthaTyd6YsK5stGDZyq9XEA zOrn9UhO*AI$bMVAYzDVU_a(;nvtlU&WO`cF?pkNP zqjy5&Kh;2Sv6p|rYtg4rO3?Fts;+fyA}vPb+uZR(`-}7RK-5Hq+i-4~N~pk5Niw$D z0y~8|_;zZ?=xr;F_3+m1T1fk?V-Vpr(_<(-H03VQgU0HA!)~dPc#cgaHv{&-Rci~!g z1bXUW5$p-savdw8kV8?LZcLFf&}c1jXq7bNT5g?8i+Kc-eph0Pq4O1qB#f|~_^wc) zA$AekyI~D&5I7>uH-6PF2#X{hDnW_DxH-}8&R?WZFZ?^;p3<6QWl!l!z&iK3#r>MG^JEiq!~s}U&L?LLn$a$qA%c&(E)KyFK~IY zy{zUF!otR`i6C0uN-M5V@10_Eepv6C&OVa*2*}8^+p%pG1vyNM5xnm=8zY_iVH*W{ zi9|DqI7?Z=)G@7`O89n=(kx`_9Y}SJ(E)acPML1TI$6}edSS5OLbzPY9Jp}Rd`c~# zxU}-a&j^QKHC%eklk|PBO0oeSM)hYE9IMmRBS<8r`u1S)YV6DQNedG#hhynn-4Ud}_qL54 zNF}grFD;BSk`YRKPiYOoslr`rTOWwY3l-Q{b%-jK|8o`tcTz6vns45vjYQYn4ZV`- zWOkUQqkgTq2wISWQ6+SlzKKb(-<@F*cIT79?7&428^gipQC=PpIDC4Z<{LiYe(0;1 zRYg9pMpo@UCybJHBx~&_8m~$FLmQvpcO8iB(;HF6-3LsOTQjF)>msT=YC5A>d!p<& zF)?>;7T@>EWTTgT4lE$D~1%^n%X<$dmXS@}$ zDu#zjP2H}fpbW#WivOlb9-%S=i2gi_?usiD*_H=?WFL?*GY99TkbRB0NycBqvr(gUBdiL%E^_D>fkTz7HP&o!u@xJEuqNu1as-`>D zf-TzuJq@N&MOik^{Qm@QdwTeQ|LD#0n^2y5iHnK-b`{>wc?GwAY{P^{eiegcNsad6 zF2Y^tC2?%H^i+5Kf{QoDCvzxLB!dxZFuExc3REai347Fo=x|pcGSbU*RgJ1G!-w+u zcDtTOpj0&-E6S6S0u?_0!mJUiq^S=T7#P~lX5de2o4^2-?Wi->-bhE5?#FQ96Hzd3 zv8(&{)$51dvG?FIG;Uj{RYZJ|KbYHV|JeT0E#=%X3FE_7N+;IX$A)b4QcwodUuZh> zXRMq{dQVgQ7gTf;M&{UHSE#HX!(%nf5rd*VdqlZ86O*4#Xjg)}tyx%>+A40@oXV|{ zywcv^DVDw#McG_^erGFg?gb1tk{61ME}SM!)SnVE$wjDo?$cUw=Vg7Lmk zaxrs?u1nWk8;xN+yVG?eM0>cmuNBqkEFkD(tYhgHW2nVZeFRU9?i!Ka z3O!K)@-j=Na%TJD2xvP~sMu=0fkM={{9#52JSgSm{IiPFXpAN^cRVyr?OJGi){t$a zI4{U0x6%)aQcUV-&+I!@5(kG`*IUkc#2Vz|t?LG!KQ7Y~#W%XXQc4xtedZii|3rk( zXuJ`HuFObWt94Q@6~9C2%*TQ})s+eQ)c=RQ?+$2U+xiVQj-YZ-5$Q@N6r}{Di5>|^ zH$n(4pwdI{NVTDW6zLr_^n?WIgb3K^O-d-CD!q4*>KhB5bMEz?&-?E8?)Sa-zWg!S zGqcy)d)Dk(vuCfheyhrZz!-od7z8cc-DtEnZYAcF$;xg-PsQloz?hyaIS}9~kTmPm zbG-7Nq+Sh%hfX2ae}|^EPV8a)MqSgJC@>X#^q%ywST$V@`}9XiDmJD?b7^|r@c}7h z&WR@H`VvW*1bM=(QI*KtchgZ|ebcG3dFT6AEvIc)I<8fRhO`SF)C8wFF?PI_4B>P{ z-z>|1bH>a5Zyxu*-`$^Hx_uVDir>}>xw==H+F-pGpBkyJkD8dA9mWIBhRNz5 z2$6R}IA*{=!xU5P6X|)yv*(HLt}Ak(E2erIMIy911*=#Lxt$3$ZdYu?WZ^p@_UdU|TSUJxckZGuiW zuCMR)N{3G|O!MMN2!Mdz8qcbRTg|)HS-d0LRu+Z2?#juZuR~AGnn_To1ELyX^`mgD z*Jp0m{=FpsbcEu&SvW{RM{a%>ZM12-KM4f-mzKPLIxKD9S~bQWlhv7f@S4EEEzW75 zR${5c;QG;U7hjA1ZTGpYJJ0E7+NsKB>(3qC+WkP}?5u8$tzr4N4XUPIvwVPb9Ys9c z8MN==(O(Phe-rWvR7y?ZQzunfXuB)p=KaPj7T#qGbGI8@r???v9ih*pV9e|`yueqW z%068Jej)Lde*|tzVeWm6Z=bjlX(!cfahN!_@1;?JDtNkt_l3m#R^ZHc%KfL$^<>bx z^hOynt$V*QNwOnLxE+(;9s1n)sohrM$Ho5jqwzNpNCNbIrElNCJ9YVL);0`54uhdD zGn&pn#`j?s2EVY+S1*}mB0d6QAc5W^)`ACXLgqhrzUe#`--lipteoTjo%G0!O0!sK zmo41g4&Wc|QfblOcVYgEa3Zi<3iHD?ZhhjGetyX8hWQbd;8_9J+w%cZ!W#V zw?JLRtq!+F3*lEwtFQC8o!oP91z;g7(L&%3I}r>CS`T>Owmfx)R_O(g!S%AsPngX3 z9C}HSs}HS$eFnEYanCx5ILEigUHr^Zw>{V7qP6&m*j#UxD~t)G=78FObtaLC5oZmn z?oq3F0#pOUNH1wP>J(*wt%55b$2VFdl&ghlan?X8(*^GQuuFe>R(`lMX)E^&+}o;> zj@zR7Nyh|kj46qOJ={HkL$4iaVl9R_WhI`s zKF6G6kSjhlVPsd_-Y0ZEKe=M|fQB-rl#D|nFy%UN+7JVr6}($DF;o%0ku!i({Q7r^ z+`aM|Q)Q7;T%)F@siCQvzEAgitA}Alx&{%UNqs zX$uty;q1>|jr$klG!Ur~JyRB0Q|vAVQP0&^!no4I)zmVYU=w|k94N3X z%hFA^c%QG*2>B)y=HuU*`Sy3f{_4b3B_x5A_3e-L6-O#&yVFiP_3PhL#RK1H|KDTL zKnXnD9RTRGTGb8H|9eP04@m{O_vI0r{O2Wc*pJ?WsQN?^^y#=fA1G_rGiTJI@)!m2knS!XeU& zA5Z?S<-Z7LPxCQ-*Yck`pF!u+8wtD#cB)*zQ&I#0FRWjH7gp}ktqMRk-?jXg-hX?m zJ^WqE?|W&|!(A*MRDEH-W=QjQEf4T_M)_j`Ls0J5o*#+)hRaf2y%z)ZSr<`!!({=0 zTfQY8*v}9)>8Fm*g-iHyFhp%@`jo(dz>PD4Kh#`*K1dPr9ro?_T*7z0I}dd-Nwe)r zOv$aDK9EXUx$;CgSZw6VRk5{u%Zz=3ko@1G4`w3M)ns6b0?Ufr#iC3midcyE8`has z``edi^iI@od!;yPrp^?0if(IKwI3FmgbD~bJ8BAYI)G&zFk^Bw!Hvyu3_+Eb(RGQ| zhbVgKY-{&uW#3AEi)??SgF+Y=KRXta>2Xm}8Ua5ZWti)VVV9o$%Vqk{y8xvn|*ZNBK@!Bp3y092(#Rr@7y zp?BBetW3`wmlOS?rNxI~3^;xrk_nC1)g2Q@jJXFR-+MkJQm-41lp=xl;N}W7i46#>I z-PCGE9Z%2IalMLj6o}u<%F4nf)ktQLsl7+0cDNF*;^s>X7?`*e_}ripQOF8`RiK~ACcJR^NBup0AIcT@b}|M z(;xiwM6bi^pyF*Z=2RrB2;PxJUM73!-x&4o^3rI||cFP;CMJgmNaE)0ky2NLwa5wFw#!wdT7D*PyPSmb9| zl*vpBosfn1RsI$m6jY4>XT=(_T@LyuG-9;sG&h_cGr;!S=^!8*Y>`Jcaf1SQLH>uo zBW@`V`70_k8q@xeUIavW`6(lX�ox?B==DlYEX926VF->AIW&pW09&_V#97^sj%(1?c-YU>lBh_uzWQ4gLmcFD3yrqlgy<(e1N*#gnN=PyobqZ5=E)ThXSo8$~_aP{g{IGEFedyLQRiYP$TFdfGpkEm6WG()!QgRSj^Kraata>CG!h zX9^oF%@Sp2*Eo{+l)Kr(c#JX?-Y2p?3x~2qqg+P}9*Ra_BHv|a zKfB9^JYJ=qGnNm5^sSnaOb3Xqkka!TDJ!g#+OO)t-8g!WUSd{O4@-mm+6klC#$HUB zva+s2vGMcZ7+8~&Cc=Xwa-+(|8RP$0R|;*fcF%VAd4s8dt)h6bE8zn8l7D3)P3tsQ zyf<^%Qei(^GJdE2u~>Lt2^C($Ch~k&*}&$ZN`5tw+FEh}Ve)zyW|t(@fQ@J%=(iVJ z7;!Tf4v4Y$K+*Hq(c=&MCeGSG^sd_oMgdUX(aY&$2fb_Tvbdx^tauVCe2HYQBuA(aP%&Xs3Jvw`LdAy&k=LMHo&wX>sQSW25Ab{Q9vixVB&5Nd=5p^| z6Y%1CZ6SSKX+-rCcHd8+8$J(~JXxBfh2UUb-dnx`f`B2$@a71bN+~@g*rHwEtp0%` zrxmPFIXK?flHd_{Wu#eOtSkZQt3o{&X5q&#cG=<_jCAGT$#;!pa5PGQ91?(JL@S4b zIp|;)c=p%pTcx9ZFu3Mh+PDj@&c{PHHM%TQF;ikLYg?K#bauw@&?3R5D8@~5^v%ZV ze0DT`8=ZUHZqicrAR9L`y7e=LVU$dYHGqZuYPHQ7FurP#6`zN9kKNLBL3djFlN{eS zBZe8|WXrfCn#Tc00W+#C56$*`olMjwG*m)7{je@=24VS!AW6@0+EH8S=#^ zS5xD8syD6CHgRA{osUZulX$H@;=0iOmaqN$)`Zut{7P$??x7H_D3dY`m6qWi_-~tT zns$wo*Sd1_#k4W}tE&LCw`8A%81ago)L{>A%+_z4RAYJhsdP=AhBe%JYqA)eC;WmW zCadDS7;`htS+<*$qW5bRyRV$zPxGa4xp8QTcJ0?j(3`IXrwcLXsSVb?1xkZBlG-OwE%jfwOd+SZ z%L*XVZpK!Mu(&pr@HZhl#nE!cx*A%YnTL-bh8O|qzB*QlUfN%pep;Z9gm7hdyKrvdYn1S^;nP z*pKvjRC8i~*pF#i2m! zYEN-wSstmqHlZjrOw$8wt1a!iok1t@ww*;dyKSl)elz4=@71Q@qs2#NvTwBzG?-!@ zCldRJT`vQkf{yJyD`&eB{p?;Y?Rfb{UlM!ywWa@r?T->RAGtGYyD5PRKxLuBe*tHY zn!su!2usRk6E#*vZW4j`8T;y`DAkfzc5N7CW%_ohu%Jh%Nq7T0*A*a11ToApu0NFK z*9*^WZBDrk;0kKg8~cot1(DO8!P8m93PpZt#oroO0>#s&y4h4@wVvdeXFH?1docZG z#*&Vs$Vx|335wVEGfVRyMnJWBC(kCE_3`)>k((7IaSCAR5DE0c`)!i?jlaHjssbu( zWM>~SVnR61c<-%rMnv;lPEIDqZ-%I$&uk9u<>UkaguVeJ9x>Au&54s&$GUs}a%=Cc zbm`<@HcH_wvvj=!DyB|~mf+}~F{gSQdEJz2t?Zt1&8us9Ww?+K9}HL)KTObcqZ>#T z0qr+y;wr7V+}L|9fJ*olw5J4l`h{L-bw0#P_to)x?`88-ZPZ7sR^gZslq4Rt2nBu` zVONKamVCsI6{O*lLbV>>%=)O4m*Y`Cf+|1&xqaMh`-g^f`ngKQEk|_$_%emm9svPQ zdieQ1i2j4u_OERrcYgTkN|B#$kKcUoIhli=bxtCG#WOh|C@?51n)`jYY`_PrC~CsU zuAw?PR^w;VJ7j!M??9R4RJZ*isfc;W_Z*Ln8cF-EQC3#rJ-nxLm}WP>aY=j%CzF}1F*-7vH6F2XTN>r&nR~3(u z$uEX);)=S*cb7Fa-z2hn9kRVG7ZXvO`h`0C0Y$01W0&3;@t}xK|Ap7qhcLHUjij&sAVBCf2E#w-c#O zCJIGmwON>Kzh()ro!aDkffCDu-Wa;Tc;2s{MOpZfwYdPl=3`Z4GVQ}|3wK`xO?Ju7k(C07j z_o5_Z!sG&m?IE57m5@#%2&CqrUDnaq7fwD(H!)&s|4Cs!Xh-AlI{aS+jPPxhMUdm}#iH zjkQCoX1nn5FvgEueg@+gQ88eSME_tj%?maDpo{2sq3g~xQAUf)ENNO&T!>6s-NJhq zrHN@aB}JD^r5=6NvamY#w2*kE*cHW0T^zz39P3;ei9FEO#5=0o4o!!>)14B7J8 zE%D!NE#?Ow69?|lXuV5TF1$Wj_aNRp-NV!<^@-HNmaHt~XfQu%-dZ5aa9vPdFZGy{@zIJGDY4giF6Y8dIzgspTa;h{uaHrw3o-(% zcWKTt+UY)vV$@eL4o9*EGJt!pZTjnq0V}(C%lQW*EIyLQ2+V6mO{zl3(heSOM)eZT zrxTgWMFsdLoccP;jglpjz2^F-yE7_eyY4geQ3}vFr0pDEiT?yram;?+XXBEmU+CDL zM#sEcOhtL_qP38<0J+w`EWxy)4~LpEc1o@$zPa;toSejcWHrhSU8>be|F_Lteqw|Q zg$Aw`d0;CyP6y?U>)OXn;YT-5zWpaw`D(XKg%fXH*f}IB5Z9@SIR}4n@xeFjn!j%T z^!yK!gzo=7iGEnPH^y!pr?Gh#Cnl{-*#8McHL$YO`>adPhC6&Btj~fUBmPb*`odCG z<~@aRT`Zx#{p}q#or%hGQb(;TwJnk$aA{{f7OGJB*{&V-q6!PHq^LGJ0d z{OKr2*&NC`3T8hdM%HhD+^bL%EK%|noUF9ugKFr!g*OVM zC``hUa{WkcFrhwP1;NWm0b_E;7pQzlCG6s+#MFw98Eb7m$hE;MU+5c8(-(|bs8Zx0 z!CK^w1HVS-sFpx8T9<(R@6N@@5haf7}JND*|U!NknFD{11JRLWEv zLQj;*?z$Sf>P078!nm~sP-7hVAwwh8ZY9DDn=+H8)5_!d7Nce8i79>f!A=ai=*F|a zWGRe=nc0ib7_$y|9Agy%T~136$3{QB?Jm3T(Gyv@QiH<4d$LSffmlVe5b6SOddPuP zr(TY@Lgx?^he4R!v1WR#-?W4*vU25mTlv}6>8hy)LC#3+1ljyqt=@Db&&H3!6@Lxw>ewF8cQ@sZM-DF|ufYFe80a|N@Dxw?br5RX2S|KD**M-V@-H2(N=jk09o+&LGo zxFza$9e?Eu-`0d($k`+<*ZEU4zk)o-OGQHAOGfox z9xC2hrIxng$edLhm)~StvO^A$(gN}FB$TKn6$f0?l{Wj~+)6O#DAVQJs`-}O{%Yy( zCf>q3;`tD2p4ev?Z?TBpnMp-W713M4Z5eDFq!0yuxH0mhlh%ek#flxcFVCVlY~o=` z4~|}GO51FrsCWv_H0O7+905zd{Z~6Yb#QQ$-RqH!=M<~l(U#QKpFrrOXu!=e z9b00V@uc^|yIMlWvBybdKJ#&dIUec$BE3Rqa#LS6b%e~d4Qp&7z8a1={Gfi}w>w{- z?Ei{a2SRw&qj|&Q#l^+wn;y+A_pvqe`|c2hr#&-`nO!trWavH^m>Ksl-nx4UAw25Q zT=TgCLnyY>t%3A@9XKfPdNnEK^D}|?PD6d!px|(MPxDI6IzuRbr>hL`=(?Xg=){J| znUBG~nP)Jg&zbos^}=U?m%4zpq|8;g1}6GYy<$`{ z%8(f`EX+SckbZOFEP|!=i9lUQss=H3XgP6`%XL^48Q> z6Ts>44ta$fYb~oJb~GE`9ohRU5u-va4GQ6UD+Bkmdx2HU05zjj;p&x#Z4(|r%5JFC zQw&UxQeiIft1-H=^~9Q&-=*XBB5^{7(HL!jCe`tONKl-~>whZ0e$G+5{pA?Z?Wl49 z-9;_SS1dp(;NU@d!ICjDCPBTSh@CvG^HG=IEF%Iu29^Z^LXDY2R3-6N1|+EbHQC1+ z8)sUHr^0=nbgXDk8mG4YGUSpXdUYfd(KmT>08{Q&CEy?5Gn~v9(I$Bd*BbZ>5D6KF z_n@_|=zrQy#cL_#A_~AY)QMWrLd5L=l$cwcHI)3D6C+l%l`;bWIGdn!o8YP3)~V^Y z$MuCyzPpa#bS3MI;X>%EKe{f9i-jYEZ-v5J90{~^qK_o$j~WSWS{WJnawWyFqdYh1r2A+%jR zf6DEqm)TQ_lJa-&^oAdXACAVC6Q_8BYx5vJ95U_;#CIsrzWPs~yQ80hsstWJ{vTHo zzy5v;zu(J$SfEY~Se8D5BKT-fSU&f3<;%Anv15z!0w1t7mS|iJo@cH$ikoH{FazBj z6WMOj8CYyI-AK+WAG>jg)fmV;@IMv)P7c?9G4}w{#L;|Y zDt?Gkqjk}DOpBGcbu1LX{S$e*I@mo1Vv$p<`UHx*<}p7bb@AMG{rxUq<5-aM4jcWh z>E-Fl=Mx_*m%%j1osb(8?JHhJ8&Q<_-8ZQp0IPMDtlOI8o&P+pRV{%RETd1sHI>K9 zna9Ol*|}+97cl4i)O9FYt>wBhwp4xIb^GS3t;O!7CV`ywX;vpI+Sv-l*{0jQGvqAB zFR`3@v*Te`STlR{(dNpaWhhzl-JuVY0uI-#k83OOk6Mv993>h3L)4}cN~XQ5|C~qd z9}e2TJnMy-KsA<#EW?8a%YdDe83TzrD-$JyNJ8^p7ieG)OK>_X!KE-Eeuu(ur z-f=X^P}X^?WPTTV=STvj6}xGi*5|s%6aK*C zuAAWE_UGP|qJX?C>vjV=pt`31oZtqL?39h*noKV7KX3ugJ$E&TGlz?myYBR^NB)V+ zxBloq#M9$bTz5uq6N|zKW^f?qt8nScNHwU^WLC>n-_O9auqBnt^;koMz*GhTKS=ah zKD}?}aSXy!s!wWieZGJD_2Yk8B!1^f{l!ymw*$1>zxM%ad$boVV8J1VD6{L@)P)i$wjbaj-Drf(9$~m*K_4 znBvv2ZS^^cpXPj=Z?NyL!ZG;_oT^13j_$1@I;w7mc2MsQ!3z__)m%25j80-K0S&2Cow?p!^i)M9JCkGq#9jJJmJ;YaUBBxJ0T^>(-o7a9(EWB=^?n(pdU)odHljOHIXSN6E4JN$5b53 zxn0j;A0iL<$+JQ3PaZ?wrpQ~OeOh+CE+%fVN_nACg%#e8J?c;*BzNAU9wtp-Nb zFb-HA0YNp2tS?>Qrk@iLo*`Vh7IKd_JrEL>8&F1godMH)w9MWHFN3P3p>O+g;0qhUW-p?D8OjPYu!5>A37VcuD1X<8&n38&2Mn*2d#y*mynPXH? zbLwVOr>hh3;(0ZuAd@xL67!%sO&h2hoQ?5MV99Lq2>5e2G%#J>c41k1TnQhHzgGAQ5+D$l7N7rHR&jvwN-WDSP3X(5qUv;qq?6fY*g?5ibfJKm5G zx-f$dcMZ`aHzTo|n@gnB}dBrPNRnK<*-O>rgGfzUX(R5;#}HyrNTavh@6U&9v#?ZZ@z zbZVrFz^lM5Co(YckKIKOtSn*A*M^%JLaT1)OZ^o?qasoa-OfI#&nKU9Fl^9t%T_SI)|6m+22 zSM&S^2@>b(lDv~QAkk7QnFtZRSnHh25Neeo6RNc|e}8=E>s7tL(&p{=ZVcA(iW(ZuIDFFST5;Z>^stt&7Ek+$g;E2| z=kzXGeb`Xi<27lGg}bdQ%N6Ndx~}iEa#Xk-qOZmx&E_^PWHItFWUWi^&VV!Y2+&O` zCVIp>B}VLQeF1@9cP2@%-^Oc{DYyM+pb}4fqV&(_){Vva3xhJ_JdMCKxHll;M*VYzI z1fX6ayQ)@^(#3-COX17j&Wobxmkt zzV~sPdeq5q{wOsm%~$#-F*WV%<#^A82hDIMY+?p&y&Y~l2f@#)y+Tfyg>tqNq971k zvbCAtt#D4Ku=-?5_IY_NWWy7&cnwr~iJr3|%6-jIyH^|SW(A(EOl)qPNA_z&&L?vi z8$H#E#zF+uOve*CHI(SA&baUwUsu?HlS~M2(iDi`K6YEJX3azBt+ZbUY_w6@Z5I#Q zJg-v?^==Zac_(3NFBVy_(mk}bQARPRhJJ=S0i>>y|3B3^O9`S~(#b@68{kx)5J zncAx+NJFV^AV;R!kWw4+KD`MGFlr&0oaqt@` zt>aFKV1gfYS)*?1%ui;t(sBb!rUI~nK~CL$6%vwkn7~=2!D9Wm>&M8x^tLL;WI|_s zEegxaa0Lz|RK1!8y(RDe5m|stzAhk1ug^QG`b@c5laGH|Z!%}uNl~w}T}BBR#~I8; zhD>s#8p!d+!J*JWg6M;zrWfxCs*#)I$s<+ucUPOjZms{)5#<6Gt&|p$lO7SNd+NMW zCZXbhC8i4^O*yc9PrXHpMId-bluf~dFca*#Hxb{kl zG*w`-lR?+mF|amk;7^c$%{?p;JasP$edhO>=7=er1onZtG>P=F7WQkQ z_Z#FB;H$)M;4u2@#s^^kzP{7!TMUw+BOd**Dj}$dP#E#t>)lTL@3FsQin%nkK z{%V?@shk}ym}-nLha+DhUZym@1wHzaa}x_a&x@yn8~p9mu8|Yvj{0`RtDf1IqytW= zbTiJj4V+f7GevW16ED=AV{Fhi)-6iRxIl5Ma{U?x`PaUXOS0EXRtiQ%%BIZ;f^98o z_JvwFlDU=s)75I+>1=Ws2cXUKcXL1-+NxKEnKZa0KJ+>3z3Ra926AtE78B zaTJ&wSZ>&qS`@-y@LmDNbEfxdP^oyla$Vc$<*sP05HZjGNT`;vn{YrSgArK5;O|%{ zAMN_P3uwFVve(33n|H*X_U~OF5=dixwvhf7y1%abkwdf%69!$~mG!439eU@xWzhZe@n* z{`wFUm%SXwPmZs-Y!A-~ zR}V5M5vzS(?jIO{3m7E9t1k$7iK+{_ZgIyA3a(+2-K^7b@r;(MVdTZ4sbHi@-)vo# z#JQ6n4n1Kle;O5eZX~W#ADjBH-@Uv}c`EzFK>iGej#Ws>9Dyy$Sp)=rcdR>uHX&lGMviyvz-j;KL{(Hi8a-1%U-PNm8*_cz@ z$~n%bJFfQU;A|zfqTmKYG$$$X$d%)7uB3`vZM$x?mO&w6=V2EW(fVbBMNUbH1Wpn% zy5tmpmq*dbN+oAw1)i+-Kx(yUE2#QoID3`OR5qtxG?%7jUmb04Ji$Y#T$vjF0UM_0Emm)DBZn7 zhLbkr$b4CL$FgDtfsk9o!vsxCEXvsNYzDqHiM~94=V)=AY_iJ1>f|KT@a!c)6OEKx zBVjCreDh;dH-&Rz`=7`>VNJ~-%ca^rZRedm^+=zqwIP8IouwDY>7uG0uG3}F1xADU z^|Gl}^!ny-@2SgA_*-k-c!Z1h@G_)a5mGPgziBNPzEc&vp!ScomO0KMlf#)_g^tVSH9nm?JGcN6In`N&b2L5S ziQo@%59PX;(n|9Qbht-u=Lp*h*@Ky`Ob9Nsh;13RMcBrUBr%6hiN6_lW7B1OE(a@S zlP=w?-MQaNF+HqHUJ|jhy*>D&)1c~t!n$(8xNlZ3O;Vq?iOJ98;c1RF8u|T< z&F$Q4<4L=z))(CzR>QXTuQJ_SCWhvOcFX>H#7bHvvj;5kGc6`z9%E9#LmD)}FKz?m^Ct_w;@m!fJoqdWB zt@j~Vu(>sZM2QLzmizb3u#~qgL(Qy@;gZfMjb1f|Ly_jW2l^699gwv*N+%&@?Yd1U z)35^THn_rNj@BHwmZGE{30Qw}&%G1WoyVs00KK!ZsS2!I{h&imimqv;DxA*k#pc36 zVJ=R=2sDtE_}(b3`{~NJCr{s+UCPaJ;nxdtB(xR5nLAX?9nquiH$@X^*lRw}TmN@=J?jOj@nN|EZ-~PWFaR>1u7vK6)%)} z_H~biv)Cl`HJM?Nn9}-lCykywZ+QL*WSB53ymjT~Zn*^6cQoaa)1!+**g= z^(&ac4NM1$c%kKhe6_t@+Ng%IS(!dw1c~G_tq2xekX-;4f!24MX0<=aBTbvjBoWd} zPSSs4Njds3Y!cd_!f%{SXznpOX?X-zb%H&Rw;dd0Vl8i_9|eESZU;9yoR<W^70rsZaQg;S-Ci{Vj#4b4pnZL1Q^}YuCAS1W(u>nE>P6Cg3vBfIdDzBi+Btk z%oUbnmEHlBhkE25xUlm#{10``W=K5#08h8M)zFL`bc(w!E{weX3FN}97ZCI@;_woa zbPBjk#Vs{AJv%2;N9b^1b=4IO#txz`OD=nnKi&h(p4yd@ut4mRsrZ{4vftBXlxoLg z+zPH!b1{!yIgULekf$+^U|3V$h~iJl8HIb`^XKaEzlPl&y`p5d0Hw2em21UgqY^D; zfp%^y>8s<9?a(zymKdLx#Qklzt234HQ8q6)i}WVl;=ugAW*y|(5CnQoJy*BB)b_;f zY>}Bi+rXX*?-%;p4*vT8P^I5wma^t^E-An)Z?3S|(G zt?-}w{U};x-|3&?K|Lg|JKW&Xt}9g`yG>0!q@^_6P;^39YCv#XUIuoE81;Z!eTw{gpvlc^2E1pQp;aHKn>})K?zrz*= zlW@>{ciBIWmc&;ndQ3tYM=!|nLP{$z2IkA32@?gE*lcAY(IQ6eUw#9~2nco`;?c z0>ZVTc@0+#BpJKV^9#90cDTb^QMrYN=ERG9820Eg%+beaxQjF!7;AdVK?}aF@*WBu zYOgrtl{MKDD(wC8Hc`%AqwMK_j%TN00kI4S_S&lXg-)3P(HlJ$&r6)@=7&g?eEGaee-5=#Jt z&Z`)qH=Z?rC^szTZaOW3MXWu-mTAN((>{UT`0`b5GdRvjyzu?KXl%4H{y{Z?HO=cy z)7?qAg~58ePoP5v#G~_Ue<+{s+L8Pi!CjNhc7FNBKWwrZ@(HBw!&#R%rPwIb&EgZ~ zcqJpA)>g%DU155*P^z{nEBahd5#klb$6*sNB&@zM9g(OxN#>Mb9I6iscLu^I|0)1lfY-mv!Z()dfWpDY^XX@%rTk$=y4* zjb60%PmQd#Nbl!f8)j_oLi@bP9k8@rb@Vh|9ramOal=OR%FG-|=neXxCjT9RjWjek zhXr0aU+!27=<6L*`2q5jmH4Hf8l*p8Y5&|O&^zvA?j;S+Bo377R$q{_e*y`2@1Q(< zk)1FQb@A7W%RBojx1YZ7-I@VB3pLa|2dKA8*JADu8@)|&_vWOipR7ofgJN)}$kfL3 z5)FmnA|V0!X#!foWo@!cU?HqtVDdpYTwY$Y5?_jz#MqyNL0SZbvY2HHB2n>F`UNZ1 zjXU)EDEak{d`0+mLW51|?~UN>1oi0c(5cwu4aUpbT@~~q42F1P z7R)3aBT4JlxeXF+C7)4>m6{n&cGGn$XGv%}O*6N|@Ds?~_L#oX?Q5;xw1MF&Ndp(+ z5XU<%=LIigKd73iDs+KGJzQd3cex4M1s1)|yD{?NmZQ0tw2*v0bf)KfPZLKDTILQz z-VvmoV=^7I&&-aT;T)f8jKf2x>Ltt)ysKR7&OP8P3f25uxfx3^h z34_sZutkkeWeQa8UbAfZ1j$BqP?{gI_ot{RW|;=I{-!)cwy$twT zh0Ii)pZwJPhTGGV4G`f+R_@Eh)_a6d{XS^~GbBQCdH~SG@ zNw#e2A6_2Z2*r}z8ajv9Pr}VGtH{JTr0wAeL6A}GYWbTYhc~*SDVOE7QJL_b=GD#* z$@ks?1UQmgUFZ9iIUum&D=EPK55)X`Aic3~#-7=(&rRDx}+)M=(NLD zplPfd90-*t^{#qKz06ut>MHz3R~k5~jx9)<#tp5Ux^oT)f5PSsP1|oh+@pF7qW_%^ z2(kD|_2uXu-RGdGuT;Qcz)K8UVOYXenxXs11(`yr|ENz7R0#xuJTe9(1RQ-Z1;kpF z4bWV+*tcC1^{tzxdsNRUT))$O4juVQ_4(*`Iv{fDuP^S0YqUQ#IR1%Blmos`5uZSj z^uXt{H!v}noNT#I^P=)nCePHvg*espHB(?sc)C4#{hqIoZ2Hj0!9|_{Hl;)wUb5V6 z*C!Lzk=w_=qnuf7t^<~g9~werDOWw$+{{ZVfjEOG2Ax>9Qc^v#@coiRjOyT16MDX zTG_JdeNk-rVJdZ{otDMJWovS=} z<-jc@b=|(;5npvUoK}j80Duf{x}`*!a}0k15$$@q+$m=b)_-7iIa^H5W_FUiQ&Ge(FX(RveJ1@8__330!zN*BanbI#wlU9`)jZ;( zP>-L2`uz8daEH1WK}JL!^QwAl!@bK#uXB^*-P?4MvtN<1uQaP;FW9R;j_Sqmqn#u9 zOVsMk0*7@xTehto6UMa%pO?Jwt*+s#aIx8<_G~$HXv~*xN~1wzzu(I6`+2#=yn4GN zeV^9@fM#c6&e$N8%m~iQoIbM4-;omWpIT>%?76972^a#(WA626@^Iwsya zZH~|t#nv2rsBFFLUj%2IHF=$TV8HlGjUPaE88%MUJ%CQlx7_zQMK8ga+w2tFy?N~q zpk1zQdX_>`OsI#9?!#p`8Io#Pu)Om`k74Mok6mzD&5v*yuC8*n;wlG1Cn+s*uVps=LRxOhZU6fxo0XcNEkxFVv2_i3*k1s&{X{u&c29 z9^DD1L>9mdSBjdu$LwIIv>Y+)5C;NG(*y7HLDDnn1A5j$uWt3YItCJCi#^?m=Fkfm zl9&GtY8FmH2=(F{7#>jR`&Uf!R=dN|v`u?Kr_sQPB{d<0Uv-k5pkF{vb$-y{7L&H5kb?BMX6z$_FnOU^R z-+t(dG^vn^5{KB1RmTJp_2w&ou*xA}W4jE=39+v9NbNUQ3+ua;#rNDASCozz;Vo`S z8AlcMp<;<-Ms}i&b&1<8vWmzDh;+=bE>}X_Asm_w#6-?OAX?u|mUJ$6c8bBL)bqU$ zUKs826~Beh?OmXy=mgc$?6r)h&~R#yG>5~6^s9u_G6IRaqDQsolU&1Z^A% zMm1S7VTe80h!!JCJ-RidRh**dYjGWJ^U+RJK}NbfJ`W`^EoXhK`cUJ-DjegIXf)Fq z+YN6vxHRK$efCOiEy3(jNi7|Yz@Tm+4hvf$vLV=b&gDvNE7JEbg*o}i%&R~}>;)w7 zALoR(s+9de5WfIUl*l9!K>Jk_v>-_))3wXf{v(k|PKS>koixr!Xj8557)!B5Pae7w zr7NogNgJ!3i}`-Cm+b4f_s!=6d@5Fo%UXXGtmhD9-?QkM7Sj1l4I!!V(N7IBY@%cGEiEirpNn- zO-!cgw)_}$R++q~T#N$7O#Wp)K$i10<`lAnfM$B1gBB4okm4+h1P-5%n>WcJTU6yYV4zSCTW|o&Eq5 z{YkS>lW2Buz?0x+^oLMNmM_-qjq@z!`0NJ=@CN5e-_UV>L7;HT+H1&NT22^!813qi*j(DHC-*Zc7OAN~vu( zR_QmXSxMF_&cx4~`qteIaTOKPqI-uqHD5TIASuUjcb&D-f}eal+cE~553xQZ*8kHD z-X}1wHnOV(PQlSR4e>_2M;7YCFQkt8$PzOyCcaoTIrGNdvAu;ts5`QA=;9?;C1&2Q zFZTrGc`yxwrx@knV(ahd72s*Wh^-kqOsCHo>(QWrk#mR!&*6oz&Svb4%Zmb)v`7^rW7Bv z)*EMd5Y>$~kPDqPMSG#A{TEimZ`T7)&{ol4P|-w5Ks_PO%FXQX1K!FY`{knh$5_Ru z)}{z+aR}cBtViC&l^D>9zRS zCU3up$apBCqD@wjXDu}o7^n9P+Io2Rgy{ELftSzkKb{=FU@2}_P#`@EgPF=94jBzZ zjx9y@-%y>Ucz2o`|7;}_kzRpJHiB}4kNX6?#E33ud#=|Gah|!U0Tfj2hvWmJN%{21VJ=u7knl5H(v~vx9puR z4=z<}Bx@ZUWlvZhA}D#LLRDg|l%?wjK07iQ?<=Gz?9u>XBhgoG3I==dt+^4c#zvvg z^uE_X9abyr`G$Y|Un&+57~vEVy6cq0t@{|)K$_)A;EL~DH&avR*kr$OS_)l}$?BOW zyWjbjW4vTr4?8eY8aNAcL$-nuUti??&wP>He_rq0jjl-M#EC?31|RKRQc=4GaZaGk zaw}CQwm<&19(?;!uEf)bCPd_Ypg|}Z64qW0D0hQ!@Ua6I#!PeCjg5?Kz|)LSTd3b} zmqpKLzOy?ds^RAiod53TcN0E4wp0Nn0~e8EtNXIi3Ss5MSlxtnn(P9e9;3d9g?%T~ zZ{3p&Zq9c$k9NHzYe3@zmF$g;p~&1LlE1q=y?oZ5WJyVua~1PZ&&x+(W3;CV9=;m5 zTd-qM^av3j8R7_jF?r1u--2VxDvJ$l8-4zVP^J8ZJWLC>PJeF~LQ}pAW=9gg#jsv@ zAvNUffFfZbxpyBF%Fn_v%Lge*@T$KKmYR`T5k0^{F4*OGUm z4gXBvq4P<_`Wr)r?z7Mv-M9Y`x+VJpNYgoTe8jQ_0QJu_o!dK5-k;%cU*<3(@;A@e zT!*wvc@#!BlIGuXrdGi4fBA!tOmJ=tZqaH+8HZ1qy@i6;GMGiu0gI!0^XM0#osLW< z*f{LwsbinJMID_=|B+hFS5S*7(cxKb2b<~_X?l`cySzvea4Yd$_u=q)YPOKiNX1!; zCrlB~tM{}FKGD<_i(*zZDI-jzEZs|tE@c`{8|OmA=U(pFQshwMogis! z@rDV`zz~9x;VlenZOf<+$>~|KhCinkXrnV_KvpC*Z*(NB;@XcA<|_-;N}hK>yB&K~ z8xB8JR%;>ah7Yf@`_r3PA>)4t%{q7Nxpbvn{vU7~^m29=K~ks7gytLkGv*H=Ia?u8 zK)#Pp5XqcyHHg@aS1yjTLnsp(?te5~3`(S-BYw1_46J0QrlI9~VEBbW`YhsAU9h-? zP^wJZBn}%yFl8<3EUHU>H-^K;eI(QNjkIgMgsZ%c5}67&@9cJ>+jkcqn!ue}l=9y( zsS&r%+ty0NC!xGjCZ^Cf#&|t`t@z8A`rGWMo2Dr)ci&d`@q=Q=l@*t5_=Zq-JTyeu z$MQzF64~yU&BX`5g^B{OVv$61dcZ44VwFwlg{=JFdRS<&##;6I;=Q&{xPB%f-G?P$ zqh*~!THfuNKw5IWV|=1rMl7qXl~&%rn9OORjsMb6!^~N43`;+dY~SDj2K#8`u-n!G zhG6Trf5;W`nJVyypOrqI9rM~NrYC5%uMC|-mvBfZRZasJY=iul&L`LaNDZ$)q6#zB zvpW&>GB~MexVF#0dumz{8#44{qvhY7N2H3t7I)ZzE9 zFHQdQ1I$Wo0!8Yl#8PG-f@@k%yu~6us!4tnA8xlJ5qzw#chS?s0*moPbny%~V?S(g zsP2Bp9W66#4pwf|)%R}Ui-}CHbfEBSS`-Op1m(Rd8gwDGVy=JEPwGlwPAM zS+W|lKxXh*u8^-{Wu1O+&^|**M4Lft@&N>yfhpk3yJ(YBR?!R}32O+XWWoDg8hW|! z?ONsdk=I0_{+wU0dW`j8GgEXNx_KNl#40!>V6F1Xs*kMwMObbG7)6NW6Ar=hpzAxQ z$X+hNvoEuSyoaA@>Xy8o$b#A)5>c^MKAb{{$cuWoR~vl%$Qz>qAo(gw@j|=&EnVso zswTBNa(2Y@pNM?@2DyeV+6(7Z0K)sU(tOG^4TZoNl2raK-?_7{EL2tw9+6juUVs8Y z4da9Q={~oZ%g=7C-2LeRqH{NoRyE5H9u zUg;edqu)hfIcvoM!9Yfm^Ewp`7;{k>8L6+C}cQJl_4j-3{T`3#(#;#P?N* zQ5mxZqHDX)NcuQY#=3(Yi0cZtb~i~!z+KcrfLoQm(+H`S+Q8_Qi!)h`3^tP*hFnk) zaxBKnSv&HaKr5njj&#cGbiiQE(=E69%*oSir6l9lt*NF{qjo(l0grZhZ2kN+8QzV9 zpP*+bXckmkJNEc}`M??Hu4#6D-EFawIHmI0Mql8m7u)+0Q#3EO2n*JFpD_US@yRSM zjOJr(h1lpNlN|OF-bC^c0=-`9$n(`(Nf0{TZx-(u?4~6UaI4~{ zSW?o~*HepO9sAPL9^WPNdEiQ^__hOMwp3=#quYa1!K$RMr?(i`zTsrdGXw+jP05Q99TC_0z>^W z%4aEk{ia^1LycoOoCyLn*oED8>IDD_=d&-noEy9V@;n*Ec$+hZoZrCj77Z8HMMm6UDe(>Mm{A_L!XHo6`^0+k^LfVbG9@S`%`B z$*6pL{AVEDUd?S!y>QgZ!D znZY9hGboyh>Fg`Q`ytil%5pNXa2%mgP5rs@;=Hmz?-&i`jj4Z*Ln)?I)G^aqx#9%7 z@=q1~I@vFd319Ny##~DX zOs^nV7y82)ZB)`glH=f?TK$bL0$^$SG+CpYCoBBfmERQoTPp#u)Oxk-tpDCcwh_wE zGg8TplVQB2S&-)Pb2N{ub=+Zsw-UXX_QKb=#hQ5osolGQLg5r1+%Xdq746r!_Dn>8 z^U4BX$@^3Td~KXFSDVZA7Z|GmSOA--#n+Yw${Ys;TM2*#I7S||Y-8UcKJNV(N7Ji$wEpon~ zjY9OS;G25)YPovv3A}2|+^Uv9BVkvbE0lhMnl{{DIkMR-Ycm)-HeA8RMO>%4^z+M2 zW(9gBR9L!qqZ&Pzy-uj!WcGjKP~cPFz-YhLju&4%?3-BuZxI5Ry57cKnFWYCwAt*} z?Z;Qhga87HqhEvFwQ*nk_&>}FOwa~2X{U4ee>2KmqRX$6ZmJ)huiHY*?*B4{myp2@g&?b>I3k++-73Wy`% zyM93Ja3UXY-OXkN8i~eXV>I)sasmNnMs7CyX7iE?+QiJwW(9h+wYJk#t&L9H6$YdR z-N01w?{yJ?9tH>m9T14nXCiO=6BRkBahT!Y&w|A1Ev0|50XY9jG%)#)4>3PkrG5gpZbg*y1e|?av&t_o7=-aZdRS$TW)Rkj#4=qZbfTZ@ri+ zMb&odqcCV=mgD}SjsbdZAkVK9M|0fSnibUdti5U zy4)DjjPdG*M3O}bQ*rJOp)#j7;q1~0C@k^%F*F471Kmw-*^Seu)7O|nXAj~mEF)RO-oa- ztd2E4MB9D$&Xj+`VDry#3LZt{%H{-@f@u*X=uqI}-nz_pO=nian=P0SBFPim?M2~Ku@6?Eq$L6g zD=(i$opXxk9D}&&p+{})nBGyJBrm)#-G>dmSjk5m92^l&+EyL~TZ)gcIw|i_RGEH| z-qnScWd}z<;@XXbw)EVh0pTFSM-~n=s<{{Sdlz?mF>7&2wd#bmz=vCm``_nHzJZJ7 zO-ilXw4vE4izemRfT7-rWu|g*(398W#e|#;R~xV_KSpQrm9`r-*0IRsz??@;LV8oH zVqWYXgZh|Ekdy(EUBtefc^98^AM|Lx>0^6cyT-1qMsW%@y*M-nM^jYKPg{Wm4i7(T zJ#fEU*Z#qqgoK!@>Bmxvn(ikUn`hcnqS70qNL79S9zzq>XQ$sOAF#jnx>q4m$x$|Q z+?Hdt8bLf{JQ_V3Q1H}Ffz2cvWr*10gV0R|=Sc^7Zq?Qp@SO9nqW)S@zb#X(E%&yk zH=HzQ+cf0WzAY+9lnoI(#nvO~yDdveI&7C}iL-{v8tCNh z7sN{PB^xMdWy~?oY;V1!(gwfM_y)45d7uLK5OpeV$=ugmmLa+O{?HDJ%$`2^g1nd* zShb}EFAAjKuuvd3YKqPg=R2;6<6ytHZi_E!*pqK_2}F9hQQWQS@d05uYL06QaDdD# z#T43?H>^df>QgwFnkAzV8i%_PGfg^cDvBEDrBf%Y#FeJP>nU?Fkc0fCR8UxWEU`(( z6@f-m6W|Zs#ND7lz+8s(S62@W*1Z5c^rraJ>Km2SrkCS%e>{nxdve5PZEDA56|F+( zR@%pk3x2O5l`NCPbMB!@iB2hR#!O9WWp&&T=u&F*g5_et_c|whX-gK21f6U%>Ub_O z8Jwzr9PS$3Cx6qiw^7HRQZ|%Twxrrz%%#zkC#hrEk-Ho~Bp$A@6BD96Y-RW;E=2lw z-Q=JH*U@jh13Rvq>+Llk!0OWSh*x!_BBKY>R^kpHIdpjNG3SG5lnvV6E|;`myxl0W zyFL%8WBncVJ`6U%USJ;$e9>xUGUeepW+#icc!k&0%1q-;C1FA&gSXEg9KB~}twNZy zIOGLC^F*>?Ta2PJs3$`Ua`}nSPV-e|Xyu^r)Uo%iLFyORI_V0y$`YI>Eq`jXdeq`* z;L)A|mCr|*8N04?Dns)mzDsJ+E^b&y?qD+#Fh=Ns%E zE^fH}cchMzifE1q|srp$8Fyg#fai&3kExD`uJUhv($jZC*p0WhEMtN zEY5)yUV#}{{#P|jIq`#GSmW*E@q0JQ+3z&F&fhwBv#8F877GbeXS^@{N1`+&(NM#f z`v9FUvDA6;H7N+1pPK?9GJ&p;9dqv)IgXWuIpvb)N)Pqr4!MsRSJdi!(Wy^8wf6^rz5~@7>!=)@3$)*QI zEy?2i-Vta^JK(JjB2|-;2F8r5)Jn8{xgK|9On#m0Ts?bU{-lC~Te2-PbJ}S__xV26 z`edZqGESYn%?p>%nthC>XUAn zgTi@Q%u^EMm*W_S^h1sX6I?8blQ_x#koG`0H?u<{r$kk@!=ij?@g?|Vu8oew>flqG zDTZ^`2<@5v;XMdZrQCvh!}N0#UTRWNR=TZuoq_!Ab%f2hHm(>ICko zhr8KstMyV_C~#Zguo->iQm1~p>+dqV^NBa(3|Eb5Oa>t0Na-p2V>=Oo@OC1^TSn^$O+EY}~X2Fl4bjGG+ zyWH?+!G^|Q&LUvoLKWN6-XnAMS+QIsIA!V=-S1@4WNY=6K9_)K9jUI_j~BcI6eJAf;I1vSnnLjD@U^#Pq0fS)CxofK$<|i95!U6t*pz(tpmzow5k9U6E?w&) zZdKvtEHV!IAPLP?r6#|n++#Ar?SriwWZ*ZlNtiJl)4*C3W>fleC}p*e@|;wB6Z9#RiN7C8G_#|OnaYVtJxBvGx6{V&W(?+B?x*PF1(8IfP#Swt z;F!D^rdLi;+o3MJ+{#IMyl(B!Jh%6!z~58rQ5V=g%R6Xa<5RO(eVrlu!y$#`KH_CQED0I2Lqf9z8^`P3HGs!5HY zOdfRtbHO=9lCRR6D)g1g{dcs#3&>5;_Mq6lxok}*G4NPEepp8CnH|Uc$8ARQ47g`z zX?!8FoqREfo*Q^%f9EWYcCij(Wvz`2`9mlsPY7%C*^rWx@#bS}u1)!GVB}NmC5!j0 zQ*1ji3|CJiXX1ScUjA& zHJZVjE~#qvk$;mJTvVd0E-U*uSz7336wtNpo_6mn+=|9E^LyuB6SXMg{0 z{6f}^n>MUMptKnN`5U8k=VJ-{TaU}Q4t5Hn$TbJiMY@8 zI0H&);z)P?3aFNbe@F53?ktL_0aK?Lj-jcGS&5aIkbjgwCMz#RicqG66EITY{VhU!=rms zCspuPS { expect(onSubmit.mock.calls[0][0].trafficClass).toBe("opportunistic"); }); + it("submits Ultrafast service tier", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn().mockResolvedValue(undefined); + + renderWithProviders( + , + ); + + await user.type(screen.getByLabelText("Name"), "Ultrafast key"); + await user.click(screen.getByRole("combobox", { name: /enforced service tier/i })); + await user.click(await screen.findByRole("option", { name: "Ultrafast" })); + await user.click(screen.getByRole("button", { name: "Create" })); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledTimes(1); + }); + + expect(onSubmit.mock.calls[0][0].enforcedServiceTier).toBe("ultrafast"); + }); + it("renders and submits a transport policy override", async () => { const user = userEvent.setup(); const onSubmit = vi.fn().mockResolvedValue(undefined); diff --git a/frontend/src/features/api-keys/components/api-key-create-dialog.tsx b/frontend/src/features/api-keys/components/api-key-create-dialog.tsx index e6589988dc..c4376d00b7 100644 --- a/frontend/src/features/api-keys/components/api-key-create-dialog.tsx +++ b/frontend/src/features/api-keys/components/api-key-create-dialog.tsx @@ -260,6 +260,7 @@ function ApiKeyCreateForm({ busy, onClose, onSubmit }: ApiKeyCreateFormProps) { {t("common.serviceTier.default")} {t("common.serviceTier.priority")} {t("common.serviceTier.flex")} + {t("common.serviceTier.ultrafast")}

    diff --git a/frontend/src/features/api-keys/components/api-key-edit-dialog.test.tsx b/frontend/src/features/api-keys/components/api-key-edit-dialog.test.tsx index bc8ddd7929..df842ed191 100644 --- a/frontend/src/features/api-keys/components/api-key-edit-dialog.test.tsx +++ b/frontend/src/features/api-keys/components/api-key-edit-dialog.test.tsx @@ -527,6 +527,20 @@ describe("ApiKeyEditDialog", () => { const trafficClassSelect = screen.getByRole("combobox", { name: /traffic class/i }); expect(trafficClassSelect).toHaveTextContent("Opportunistic"); }); + + it("shows the stored Ultrafast service tier", () => { + renderWithProviders( + , + ); + + expect(screen.getByRole("combobox", { name: /enforced service tier/i })).toHaveTextContent("Ultrafast"); + }); }); describe("hasLimitRuleChanges", () => { diff --git a/frontend/src/features/api-keys/components/api-key-edit-dialog.tsx b/frontend/src/features/api-keys/components/api-key-edit-dialog.tsx index 3885d2d15c..f405931770 100644 --- a/frontend/src/features/api-keys/components/api-key-edit-dialog.tsx +++ b/frontend/src/features/api-keys/components/api-key-edit-dialog.tsx @@ -320,9 +320,11 @@ function ApiKeyEditForm({ apiKey, busy, onSubmit, onClose }: ApiKeyEditFormProps
    -
    {t("apiKeys.form.enforcedServiceTier")}
    +
    diff --git a/frontend/src/features/api-keys/schemas.test.ts b/frontend/src/features/api-keys/schemas.test.ts index b87465d201..10b77f3870 100644 --- a/frontend/src/features/api-keys/schemas.test.ts +++ b/frontend/src/features/api-keys/schemas.test.ts @@ -189,6 +189,15 @@ describe("ApiKeyCreateRequestSchema", () => { expect(parsed.enforcedReasoningEffort).toBe("ultra"); }); + it("accepts Ultrafast service tier in create payload", () => { + const parsed = ApiKeyCreateRequestSchema.parse({ + name: "Ultrafast key", + enforcedServiceTier: "ultrafast", + }); + + expect(parsed.enforcedServiceTier).toBe("ultrafast"); + }); + it("accepts a non-empty allowed reasoning effort list", () => { const parsed = ApiKeyCreateRequestSchema.parse({ name: "Selectable reasoning key", @@ -277,6 +286,14 @@ describe("ApiKeyUpdateRequestSchema", () => { expect(parsed.trafficClass).toBe("opportunistic"); }); + + it("accepts Ultrafast service tier in update payload", () => { + const parsed = ApiKeyUpdateRequestSchema.parse({ + enforcedServiceTier: "ultrafast", + }); + + expect(parsed.enforcedServiceTier).toBe("ultrafast"); + }); }); describe("LimitRuleCreateSchema", () => { diff --git a/frontend/src/features/api-keys/schemas.ts b/frontend/src/features/api-keys/schemas.ts index a192e37864..2bbd8b91b0 100644 --- a/frontend/src/features/api-keys/schemas.ts +++ b/frontend/src/features/api-keys/schemas.ts @@ -30,7 +30,7 @@ const ApiKeyUsageSummarySchema = z.object({ totalCostUsd: z.number().nonnegative().default(0), }); -const SERVICE_TIERS = ["auto", "default", "priority", "flex"] as const; +const SERVICE_TIERS = ["auto", "default", "priority", "flex", "ultrafast"] as const; export type ServiceTierType = (typeof SERVICE_TIERS)[number]; export const TRAFFIC_CLASSES = ["foreground", "opportunistic"] as const; diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 518306ccf3..41a81f801d 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -559,6 +559,7 @@ "common.serviceTier.default": "Default", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "Priority", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "Active", "common.states.disabled": "Disabled", "common.states.enabled": "Enabled", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index adac30f799..b5d482b8c9 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -559,6 +559,7 @@ "common.serviceTier.default": "Default", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "Priority", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "활성", "common.states.disabled": "꺼짐", "common.states.enabled": "켜짐", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index e76c857631..60d781635a 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -559,6 +559,7 @@ "common.serviceTier.default": "默认", "common.serviceTier.flex": "Flex", "common.serviceTier.priority": "优先", + "common.serviceTier.ultrafast": "Ultrafast", "common.states.active": "活跃", "common.states.disabled": "已禁用", "common.states.enabled": "已启用", diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml new file mode 100644 index 0000000000..4af864176c --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-14 diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md new file mode 100644 index 0000000000..8edd2e5838 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/design.md @@ -0,0 +1,35 @@ +## Context + +The Responses request models, upstream transports, request logs, and model registry already carry service tiers as normalized strings. They therefore preserve `ultrafast` without a transport change and can route it using live per-account catalog metadata. The remaining hard-coded allowlists are the API-key CRUD contract and dashboard controls. + +OpenAI documents `ultrafast` as an access-controlled processing tier currently available for `gpt-5.6-sol`. Entitlement must therefore come from each account's live upstream catalog instead of a static plan or bootstrap assumption. + +## Goals / Non-Goals + +**Goals:** + +- Make `ultrafast` a supported canonical API-key service tier. +- Expose the tier through the existing dashboard API-key controls. +- Preserve existing entitlement-aware account routing and response-tier logging. +- Add focused regression coverage and user-facing compatibility notes. + +**Non-Goals:** + +- Invent an `ultrafast` model-name alias. +- Advertise Ultrafast from bootstrap metadata or grant it to a plan statically. +- Add a setting, dependency, or database migration. +- Guess a distinct Ultrafast token price that OpenAI has not published. + +## Decisions + +1. Add `ultrafast` only to the existing backend and frontend API-key tier allowlists. The request models and transports already pass it through, so adding another normalization layer would duplicate working behavior. +2. Keep `ultrafast` canonical. Unlike the legacy `fast` alias, it is an upstream wire value and must not normalize to `priority`. +3. Reuse live model-catalog routing. An explicit or enforced Ultrafast request can select only accounts whose catalog advertises that tier; the existing enforced-tier fallback still removes it for models that do not advertise it. +4. Do not add Ultrafast to the bundled model catalog. Static metadata cannot prove access to an access-controlled preview and would expose a tier that an imported account may not hold. +5. Keep pricing unchanged. No distinct public Ultrafast token price is available in the official OpenAI documentation, so this change does not introduce a speculative multiplier. + +## Risks / Trade-offs + +- [An entitled account's catalog does not advertise `ultrafast`] → The existing explicit-tier routing error remains visible instead of silently selecting an ineligible account. +- [OpenAI later publishes distinct Ultrafast pricing] → Add the published rates in a focused pricing change before claiming separate cost accuracy. +- [Dashboard-visible option requires review evidence] → Include before and after screenshots in the PR body as required by the simplicity gates. diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md new file mode 100644 index 0000000000..cc3ec42afe --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/proposal.md @@ -0,0 +1,27 @@ +## Why + +OpenAI introduced an access-controlled Ultrafast processing tier for `gpt-5.6-sol`. codex-lb already preserves unknown request tier strings, but its API-key policy and dashboard reject `ultrafast`, leaving the feature incomplete and untested. + +## What Changes + +- Accept and persist `ultrafast` as an API-key-enforced service tier. +- Expose Ultrafast in the API key create and edit controls. +- Preserve and forward the canonical `ultrafast` value through Responses-compatible routes. +- Use live upstream model-catalog entitlement data to route Ultrafast requests only to advertising accounts. +- Document the upstream availability constraint and add focused regression coverage. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `api-keys`: allow dashboard API keys to enforce the canonical `ultrafast` tier. +- `responses-api-compat`: define pass-through behavior for explicit and enforced Ultrafast requests. +- `model-catalog-compat`: define entitlement-aware account routing for the access-controlled tier. + +## Impact + +The change affects API-key request validation and normalization, dashboard API-key forms and translations, Responses compatibility documentation, model-catalog routing tests, and focused backend/frontend tests. It adds no dependency, setting, database migration, or bootstrap entitlement metadata. diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md new file mode 100644 index 0000000000..591b8408d6 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/api-keys/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: API keys can enforce the Ultrafast service tier + +The dashboard API key CRUD surface MUST accept and persist `ultrafast` as a canonical enforced service tier. The service MUST return the same canonical value and MUST NOT normalize it to `priority`. + +#### Scenario: Create an API key with Ultrafast enforcement + +- **WHEN** a dashboard client creates an API key with `enforcedServiceTier: "ultrafast"` +- **THEN** the request is accepted +- **AND** the persisted and returned enforced service tier is `ultrafast` + +#### Scenario: Enforce Ultrafast on an advertising model + +- **GIVEN** an account model advertises the `ultrafast` service tier +- **WHEN** a request uses an API key whose enforced service tier is `ultrafast` +- **THEN** the upstream request carries `service_tier: "ultrafast"` diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..b69a2ced33 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/model-catalog-compat/spec.md @@ -0,0 +1,17 @@ +## ADDED Requirements + +### Requirement: Ultrafast routing follows live account entitlement + +The system MUST treat `ultrafast` as an access-controlled service tier and MUST derive account eligibility from live or retained per-account upstream catalog metadata. The bundled bootstrap catalog MUST NOT invent Ultrafast entitlement. + +#### Scenario: Only an advertising account is eligible + +- **GIVEN** two accounts advertise `gpt-5.6-sol` +- **AND** only one account advertises the `ultrafast` service tier +- **WHEN** a request explicitly asks for `service_tier: "ultrafast"` +- **THEN** account selection considers only the advertising account + +#### Scenario: Bootstrap metadata does not grant preview access + +- **WHEN** no live or retained account catalog advertises `ultrafast` +- **THEN** bootstrap model metadata does not expose or grant that tier diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..c2bace9663 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/specs/responses-api-compat/spec.md @@ -0,0 +1,15 @@ +## ADDED Requirements + +### Requirement: Responses routes preserve the Ultrafast service tier + +Responses-compatible routes MUST accept the canonical `ultrafast` service tier and MUST forward it unchanged. When upstream reports the actual response tier, request logging MUST preserve `ultrafast` using the existing requested, actual, and billable tier contract. + +#### Scenario: Explicit Ultrafast request is forwarded + +- **WHEN** a client sends a Responses request with `service_tier: "ultrafast"` +- **THEN** the forwarded upstream payload contains `service_tier: "ultrafast"` + +#### Scenario: Upstream confirms Ultrafast processing + +- **WHEN** upstream completes a request with `response.service_tier: "ultrafast"` +- **THEN** the actual and billable request-log tiers are `ultrafast` diff --git a/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md new file mode 100644 index 0000000000..bf283c6da4 --- /dev/null +++ b/openspec/changes/archive/2026-08-14-support-ultrafast-service-tier/tasks.md @@ -0,0 +1,15 @@ +## 1. Backend support + +- [x] 1.1 Accept and persist canonical `ultrafast` API-key enforcement values +- [x] 1.2 Add focused request, API-key, catalog-routing, and logging regression coverage + +## 2. Dashboard and documentation + +- [x] 2.1 Add Ultrafast to dashboard schemas, create/edit controls, and translations +- [x] 2.2 Add frontend schema and interaction coverage for the new option +- [x] 2.3 Document official availability, entitlement behavior, and a concrete request example + +## 3. Verification + +- [x] 3.1 Validate OpenSpec artifacts and run focused backend/frontend checks +- [x] 3.2 Run the repository local CI gate and capture dashboard before/after evidence diff --git a/openspec/specs/api-keys/spec.md b/openspec/specs/api-keys/spec.md index 2127be5b61..274490f613 100644 --- a/openspec/specs/api-keys/spec.md +++ b/openspec/specs/api-keys/spec.md @@ -1403,3 +1403,19 @@ reclamation does. - **WHEN** stale usage-reservation reclamation runs - **THEN** the reservation stays `reserved` +### Requirement: API keys can enforce the Ultrafast service tier + +The dashboard API key CRUD surface MUST accept and persist `ultrafast` as a canonical enforced service tier. The service MUST return the same canonical value and MUST NOT normalize it to `priority`. + +#### Scenario: Create an API key with Ultrafast enforcement + +- **WHEN** a dashboard client creates an API key with `enforcedServiceTier: "ultrafast"` +- **THEN** the request is accepted +- **AND** the persisted and returned enforced service tier is `ultrafast` + +#### Scenario: Enforce Ultrafast on an advertising model + +- **GIVEN** an account model advertises the `ultrafast` service tier +- **WHEN** a request uses an API key whose enforced service tier is `ultrafast` +- **THEN** the upstream request carries `service_tier: "ultrafast"` + diff --git a/openspec/specs/model-catalog-compat/spec.md b/openspec/specs/model-catalog-compat/spec.md index 8cbc2cc3ba..4c5141e9c7 100644 --- a/openspec/specs/model-catalog-compat/spec.md +++ b/openspec/specs/model-catalog-compat/spec.md @@ -1057,3 +1057,18 @@ The model catalog builders for `GET /v1/models` and `GET /backend-api/codex/mode acquisition - **THEN** the reservation is released before cancellation propagates +### Requirement: Ultrafast routing follows live account entitlement + +The system MUST treat `ultrafast` as an access-controlled service tier and MUST derive account eligibility from live or retained per-account upstream catalog metadata. The bundled bootstrap catalog MUST NOT invent Ultrafast entitlement. + +#### Scenario: Only an advertising account is eligible + +- **GIVEN** two accounts advertise `gpt-5.6-sol` +- **AND** only one account advertises the `ultrafast` service tier +- **WHEN** a request explicitly asks for `service_tier: "ultrafast"` +- **THEN** account selection considers only the advertising account + +#### Scenario: Bootstrap metadata does not grant preview access + +- **WHEN** no live or retained account catalog advertises `ultrafast` +- **THEN** bootstrap model metadata does not expose or grant that tier diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index 6f96c7ccc7..4a82968a8e 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -75,6 +75,33 @@ Responses request with: Clients that expose Fast Mode as `fast` may keep using that spelling; codex-lb normalizes it to `priority` before forwarding. +### Ultrafast Processing + +The [OpenAI Responses API reference](https://developers.openai.com/api/reference/resources/responses/methods/create) +documents `ultrafast` as an access-controlled processing tier currently +available for `gpt-5.6-sol`. codex-lb forwards this canonical value unchanged; +it does not grant Ultrafast access by itself. + +Account eligibility comes from live or retained per-account upstream catalog +metadata. The bundled bootstrap catalog deliberately does not advertise +Ultrafast. If no account advertises the tier, an explicit Ultrafast request +cannot select an eligible account; API-key enforcement follows the existing +model-capability fallback when the model itself does not advertise the tier. + +Send a Responses request with: + +```json +{ + "model": "gpt-5.6-sol", + "input": "Summarize the change.", + "service_tier": "ultrafast" +} +``` + +After completion, verify that the response reports +`service_tier: "ultrafast"`. Request logs retain `ultrafast` in the requested, +actual, and effective billable tier fields when upstream confirms it. + ### Operator Fast Mode prohibition Operators can enable the Routing setting `prohibitFastMode` when qualified diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index a580853043..a56410d4de 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -5299,3 +5299,16 @@ only an inactive `unknown` operation may enter a fresh recovery attempt. - **WHEN** a duplicate request finds a submitted operation still referenced by another pending request - **THEN** the proxy refuses a second dispatch and preserves the existing spool +### Requirement: Responses routes preserve the Ultrafast service tier + +Responses-compatible routes MUST accept the canonical `ultrafast` service tier and MUST forward it unchanged. When upstream reports the actual response tier, request logging MUST preserve `ultrafast` using the existing requested, actual, and billable tier contract. + +#### Scenario: Explicit Ultrafast request is forwarded + +- **WHEN** a client sends a Responses request with `service_tier: "ultrafast"` +- **THEN** the forwarded upstream payload contains `service_tier: "ultrafast"` + +#### Scenario: Upstream confirms Ultrafast processing + +- **WHEN** upstream completes a request with `response.service_tier: "ultrafast"` +- **THEN** the actual and billable request-log tiers are `ultrafast` diff --git a/tests/integration/test_api_keys_api.py b/tests/integration/test_api_keys_api.py index 2b976c34e3..4508b5dbe0 100644 --- a/tests/integration/test_api_keys_api.py +++ b/tests/integration/test_api_keys_api.py @@ -4,6 +4,7 @@ import base64 import contextlib import json +from dataclasses import replace from datetime import timedelta from types import SimpleNamespace from typing import cast @@ -778,10 +779,14 @@ async def fake_stream(payload, _headers, _access_token, _account_id, base_url=No @pytest.mark.asyncio -async def test_api_key_enforces_service_tier_for_responses(async_client, monkeypatch): - await _populate_test_registry() - model_ids = sorted(_TEST_MODELS) - forced_model = model_ids[0] +@pytest.mark.parametrize( + ("enforced_service_tier", "expected_service_tier"), + [("fast", "priority"), ("ULTRAFAST", "ultrafast")], +) +async def test_api_key_enforces_service_tier_for_responses( + async_client, monkeypatch, enforced_service_tier, expected_service_tier +): + forced_model = "gpt-5.6-sol" enable = await async_client.put( "/api/settings", @@ -794,27 +799,47 @@ async def test_api_key_enforces_service_tier_for_responses(async_client, monkeyp ) assert enable.status_code == 200 + account_id = await _import_account( + async_client, + f"acc_enforced_{expected_service_tier}_service_tier", + f"enforced-{expected_service_tier}-service-tier@example.com", + ) + advertising_model = replace( + _make_upstream_model(forced_model), + raw={"service_tiers": [{"slug": expected_service_tier}]}, + ) + await get_model_registry().update( + {"pro": [advertising_model]}, + per_account_results={account_id: ("pro", [advertising_model])}, + active_account_plans={account_id: "pro"}, + ) + created = await async_client.post( "/api/api-keys/", json={ "name": "enforced-service-tier", "allowedModels": [forced_model], "enforcedModel": forced_model, - "enforcedServiceTier": "fast", + "enforcedServiceTier": enforced_service_tier, }, ) assert created.status_code == 200 key = created.json()["key"] - assert created.json()["enforcedServiceTier"] == "priority" - - await _import_account(async_client, "acc_enforced_service_tier", "enforced-service-tier@example.com") + assert created.json()["enforcedServiceTier"] == expected_service_tier seen: dict[str, str | None] = {} async def fake_stream(payload, _headers, _access_token, _account_id, base_url=None, raise_for_status=False): seen["service_tier"] = payload.service_tier usage = {"input_tokens": 3, "output_tokens": 2} - event = {"type": "response.completed", "response": {"id": "resp_enforced_service_tier", "usage": usage}} + event = { + "type": "response.completed", + "response": { + "id": "resp_enforced_service_tier", + "service_tier": expected_service_tier, + "usage": usage, + }, + } yield f"data: {json.dumps(event)}\n\n" monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) @@ -834,7 +859,15 @@ async def fake_stream(payload, _headers, _access_token, _account_id, base_url=No assert response.status_code == 200 _ = [line async for line in response.aiter_lines() if line] - assert seen["service_tier"] == "priority" + assert seen["service_tier"] == expected_service_tier + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).order_by(RequestLog.requested_at.desc())) + latest_log = result.scalars().first() + assert latest_log is not None + assert latest_log.requested_service_tier == expected_service_tier + assert latest_log.actual_service_tier == expected_service_tier + assert latest_log.service_tier == expected_service_tier @pytest.mark.asyncio diff --git a/tests/unit/test_api_keys_service.py b/tests/unit/test_api_keys_service.py index 63fc9d9e1d..9a33fd47f1 100644 --- a/tests/unit/test_api_keys_service.py +++ b/tests/unit/test_api_keys_service.py @@ -848,6 +848,23 @@ async def test_create_key_normalizes_fast_service_tier_alias() -> None: assert created.enforced_service_tier == "priority" +@pytest.mark.asyncio +async def test_create_key_preserves_ultrafast_service_tier() -> None: + repo = _FakeApiKeysRepository() + service = ApiKeysService(repo) + + created = await service.create_key( + ApiKeyCreateData( + name="ultrafast-service-tier-policy", + allowed_models=None, + enforced_service_tier=" ULTRAFAST ", + expires_at=None, + ) + ) + + assert created.enforced_service_tier == "ultrafast" + + @pytest.mark.asyncio async def test_update_key_normalizes_service_tier_alias() -> None: repo = _FakeApiKeysRepository() diff --git a/tests/unit/test_model_registry.py b/tests/unit/test_model_registry.py index 2cc88b077e..fa3ed54477 100644 --- a/tests/unit/test_model_registry.py +++ b/tests/unit/test_model_registry.py @@ -253,6 +253,7 @@ def test_bootstrap_models_include_representative_upstream_metadata(): "ultra", ] assert sol.raw["additional_speed_tiers"] == ["fast"] + assert "ultrafast" not in str(sol.raw["service_tiers"]) terra = models["gpt-5.6-terra"] assert terra.display_name == "GPT-5.6-Terra" diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 36d4e91af9..04e8cd6e60 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -175,17 +175,18 @@ def test_strip_unsupported_fields_namespace_flag_controls_replayed_calls(namespa assert stripped["input"] == [{"type": "function_call"}] -def test_responses_preserves_service_tier(): +@pytest.mark.parametrize("service_tier", ["priority", "ultrafast"]) +def test_responses_preserves_service_tier(service_tier: str): payload = { "model": "gpt-5.1", "instructions": "hi", "input": [], - "service_tier": "priority", + "service_tier": service_tier, } request = ResponsesRequest.model_validate(payload) dumped = request.to_payload() - assert dumped["service_tier"] == "priority" + assert dumped["service_tier"] == service_tier def test_responses_normalizes_fast_service_tier_to_priority_for_upstream(): @@ -579,16 +580,17 @@ def test_openai_compatible_top_level_verbosity_is_normalized(): assert "verbosity" not in dumped -def test_v1_responses_preserves_service_tier(): +@pytest.mark.parametrize("service_tier", ["priority", "ultrafast"]) +def test_v1_responses_preserves_service_tier(service_tier: str): payload = { "model": "gpt-5.1", "input": "hello", - "service_tier": "priority", + "service_tier": service_tier, } request = V1ResponsesRequest.model_validate(payload).to_responses_request() dumped = request.to_payload() - assert dumped["service_tier"] == "priority" + assert dumped["service_tier"] == service_tier def test_v1_responses_normalizes_fast_service_tier_to_priority_for_upstream(): diff --git a/tests/unit/test_proxy_load_balancer_refresh.py b/tests/unit/test_proxy_load_balancer_refresh.py index f82d9b5fa8..6166c2e824 100644 --- a/tests/unit/test_proxy_load_balancer_refresh.py +++ b/tests/unit/test_proxy_load_balancer_refresh.py @@ -1237,26 +1237,26 @@ async def test_select_account_filters_requested_service_tier_plans(monkeypatch) @pytest.mark.asyncio async def test_select_account_filters_requested_service_tier_accounts(monkeypatch) -> None: - no_fast = _make_account("acc-tier-pro-default", "tier-pro-default@example.com") - no_fast.plan_type = "pro" - fast = _make_account("acc-tier-pro-fast", "tier-pro-fast@example.com") - fast.plan_type = "pro" + no_ultrafast = _make_account("acc-tier-pro-default", "tier-pro-default@example.com") + no_ultrafast.plan_type = "pro" + ultrafast = _make_account("acc-tier-pro-ultrafast", "tier-pro-ultrafast@example.com") + ultrafast.plan_type = "pro" now = utcnow() now_epoch = int(now.replace(tzinfo=timezone.utc).timestamp()) usage_repo = StubUsageRepository( primary={ - no_fast.id: UsageHistory( + no_ultrafast.id: UsageHistory( id=63, - account_id=no_fast.id, + account_id=no_ultrafast.id, recorded_at=now, window="primary", used_percent=1.0, reset_at=now_epoch + 300, window_minutes=5, ), - fast.id: UsageHistory( + ultrafast.id: UsageHistory( id=64, - account_id=fast.id, + account_id=ultrafast.id, recorded_at=now, window="primary", used_percent=2.0, @@ -1272,7 +1272,7 @@ async def test_select_account_filters_requested_service_tier_accounts(monkeypatc lambda: SimpleNamespace( plan_types_for_model=lambda _model: frozenset({"pro"}), account_ids_for_model_service_tier=lambda _model, tier: ( - frozenset({fast.id}) if tier == "priority" else None + frozenset({ultrafast.id}) if tier == "ultrafast" else None ), plan_types_for_model_service_tier=lambda _model, _tier: frozenset({"pro"}), ), @@ -1280,15 +1280,15 @@ async def test_select_account_filters_requested_service_tier_accounts(monkeypatc balancer = LoadBalancer( lambda: _repo_factory( - StubAccountsRepository([no_fast, fast]), + StubAccountsRepository([no_ultrafast, ultrafast]), usage_repo, StubStickySessionsRepository(), ) ) - selection = await balancer.select_account(model="gpt-5.5", service_tier="priority") + selection = await balancer.select_account(model="gpt-5.6-sol", service_tier="ultrafast") assert selection.account is not None - assert selection.account.id == fast.id + assert selection.account.id == ultrafast.id @pytest.mark.asyncio From 8e7589e4286869b9030eb1b19726297c63852ed3 Mon Sep 17 00:00:00 2001 From: Chao Xu Date: Tue, 18 Aug 2026 17:51:24 +0800 Subject: [PATCH 078/117] feat(ui): surface reasoning token usage (#1801) * feat(ui): surface reasoning token usage * docs: document reasoning token usage * test(reports): adapt API-key-filter CSV fixture to required reasoningTokens field #1728 landed on main after this branch was cut and its daily-row fixture predates the reasoningTokens schema field; align the mock and CSV expectation with the new column. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- .all-contributorsrc | 10 + README.md | 1 + app/modules/reports/repository.py | 19 +- app/modules/reports/schemas.py | 3 + app/modules/reports/service.py | 3 + docs/usage-reporting.md | 26 +++ .../reports-date-range-flow.test.tsx | 2 + .../components/recent-requests-table.test.tsx | 70 +++++++ .../components/recent-requests-table.tsx | 14 ++ .../components/cost-per-day-chart.test.tsx | 3 + .../components/daily-detail-table.test.tsx | 121 ++++++++++- .../reports/components/daily-detail-table.tsx | 129 ++++++------ .../components/queue-wait-chart.test.tsx | 1 + .../reports/components/reports-page.test.tsx | 5 +- .../components/reports-summary-cards.test.tsx | 89 +++++++- .../components/reports-summary-cards.tsx | 13 +- .../components/tokens-per-day-chart.test.tsx | 3 + frontend/src/features/reports/daily-series.ts | 1 + .../reports/hooks/use-reports.test.tsx | 2 + frontend/src/features/reports/schemas.test.ts | 49 ++++- frontend/src/features/reports/schemas.ts | 3 + frontend/src/i18n/locales/en.json | 5 + frontend/src/i18n/locales/ko.json | 5 + frontend/src/i18n/locales/zh-CN.json | 5 + mkdocs.yml | 1 + .../surface-reasoning-token-usage/design.md | 38 ++++ .../surface-reasoning-token-usage/proposal.md | 24 +++ .../specs/frontend-architecture/spec.md | 64 ++++++ .../surface-reasoning-token-usage/tasks.md | 21 ++ tests/integration/test_reports_api.py | 197 ++++++++++++++++++ tests/unit/test_reports_service.py | 11 + 31 files changed, 856 insertions(+), 82 deletions(-) create mode 100644 docs/usage-reporting.md create mode 100644 openspec/changes/surface-reasoning-token-usage/design.md create mode 100644 openspec/changes/surface-reasoning-token-usage/proposal.md create mode 100644 openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/surface-reasoning-token-usage/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index fd91ee96ea..e03359b272 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1281,6 +1281,16 @@ "contributions": [ "code" ] + }, + { + "login": "chaoxu", + "name": "Chao Xu", + "avatar_url": "https://avatars.githubusercontent.com/u/18860?v=4", + "profile": "https://chaoxu.prof/", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 7c057f0324..5c3b2845a9 100644 --- a/README.md +++ b/README.md @@ -291,6 +291,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e     BrenticusMaximus
    BrenticusMaximus
    💻 ⚠️ Sakthimaran
    Sakthimaran
    💻 ⚠️ Evan
    Evan
    💻 + Chao Xu
    Chao Xu
    💻 ⚠️ diff --git a/app/modules/reports/repository.py b/app/modules/reports/repository.py index 89b376a1ff..ffadccaf5e 100644 --- a/app/modules/reports/repository.py +++ b/app/modules/reports/repository.py @@ -34,6 +34,7 @@ class DailyReportAggregateRow: requests: int input_tokens: int output_tokens: int + reasoning_tokens: int | None cached_input_tokens: int cost_usd: float active_accounts: int @@ -50,6 +51,8 @@ class SummaryAggregateRow: total_cost_usd: float total_input_tokens: int total_output_tokens: int + total_reasoning_tokens: int + reasoning_usage_known_requests: int total_cached_tokens: int total_requests: int total_errors: int @@ -154,6 +157,7 @@ async def aggregate_daily_rows( requests=int(row.requests or 0), input_tokens=int(row.input_tokens or 0), output_tokens=int(row.output_tokens or 0), + reasoning_tokens=int(row.reasoning_tokens) if row.reasoning_tokens is not None else None, cached_input_tokens=int(row.cached_input_tokens or 0), cost_usd=float(row.cost_usd or 0.0), active_accounts=int(row.active_accounts or 0), @@ -190,7 +194,12 @@ async def aggregate_summary( columns = [ func.coalesce(func.sum(RequestLog.cost_usd), 0.0).label("total_cost_usd"), func.coalesce(func.sum(RequestLog.input_tokens), 0).label("total_input_tokens"), - func.coalesce(func.sum(RequestLog.output_tokens), 0).label("total_output_tokens"), + func.coalesce( + func.sum(func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)), + 0, + ).label("total_output_tokens"), + func.coalesce(func.sum(RequestLog.reasoning_tokens), 0).label("total_reasoning_tokens"), + func.count(RequestLog.reasoning_tokens).label("reasoning_usage_known_requests"), func.coalesce(func.sum(RequestLog.cached_input_tokens), 0).label("total_cached_tokens"), func.count().label("total_requests"), func.coalesce( @@ -223,6 +232,8 @@ async def aggregate_summary( total_cost_usd=float(row.total_cost_usd), total_input_tokens=int(row.total_input_tokens), total_output_tokens=int(row.total_output_tokens), + total_reasoning_tokens=int(row.total_reasoning_tokens), + reasoning_usage_known_requests=int(row.reasoning_usage_known_requests), total_cached_tokens=int(row.total_cached_tokens), total_requests=int(row.total_requests), total_errors=int(row.total_errors), @@ -608,7 +619,11 @@ def _daily_rows_stmt( day_ranges_cte.c.report_date, func.count(RequestLog.id).label("requests"), func.coalesce(func.sum(RequestLog.input_tokens), 0).label("input_tokens"), - func.coalesce(func.sum(RequestLog.output_tokens), 0).label("output_tokens"), + func.coalesce( + func.sum(func.coalesce(RequestLog.output_tokens, RequestLog.reasoning_tokens, 0)), + 0, + ).label("output_tokens"), + func.sum(RequestLog.reasoning_tokens).label("reasoning_tokens"), func.coalesce(func.sum(RequestLog.cached_input_tokens), 0).label("cached_input_tokens"), func.coalesce(func.sum(RequestLog.cost_usd), 0.0).label("cost_usd"), func.count(func.distinct(RequestLog.account_id)).label("active_accounts"), diff --git a/app/modules/reports/schemas.py b/app/modules/reports/schemas.py index 2bcafd60d6..de10ba0a00 100644 --- a/app/modules/reports/schemas.py +++ b/app/modules/reports/schemas.py @@ -10,6 +10,7 @@ class DailyReportRow(DashboardModel): requests: int input_tokens: int output_tokens: int + reasoning_tokens: int | None cached_input_tokens: int cost_usd: float active_accounts: int @@ -46,6 +47,8 @@ class ReportSummary(DashboardModel): total_cost_usd: float total_input_tokens: int total_output_tokens: int + total_reasoning_tokens: int + reasoning_usage_known_requests: int total_cached_tokens: int total_requests: int total_errors: int diff --git a/app/modules/reports/service.py b/app/modules/reports/service.py index a58cd6fdfb..cfff634326 100644 --- a/app/modules/reports/service.py +++ b/app/modules/reports/service.py @@ -83,6 +83,7 @@ async def get_reports( requests=row.requests, input_tokens=row.input_tokens, output_tokens=row.output_tokens, + reasoning_tokens=row.reasoning_tokens, cached_input_tokens=row.cached_input_tokens, cost_usd=round(row.cost_usd, 4), active_accounts=row.active_accounts, @@ -126,6 +127,8 @@ async def get_reports( total_cost_usd=round(summary.total_cost_usd, 4), total_input_tokens=summary.total_input_tokens, total_output_tokens=summary.total_output_tokens, + total_reasoning_tokens=summary.total_reasoning_tokens, + reasoning_usage_known_requests=summary.reasoning_usage_known_requests, total_cached_tokens=summary.total_cached_tokens, total_requests=summary.total_requests, total_errors=summary.total_errors, diff --git a/docs/usage-reporting.md b/docs/usage-reporting.md new file mode 100644 index 0000000000..5b2199612e --- /dev/null +++ b/docs/usage-reporting.md @@ -0,0 +1,26 @@ +# Usage Reporting + +codex-lb records the token counts reported in the terminal Responses API event. It does not retokenize prompts or responses, and it does not estimate hidden reasoning usage. + +For direct Codex traffic over HTTP or WebSocket, the reported buckets have these relationships: + +- `input_tokens` includes the full input count; cached input is reported as a subset. +- `output_tokens` includes all generated output, including reasoning tokens. +- `reasoning_tokens` is the reported reasoning subset of `output_tokens`. +- Total tokens are `input_tokens + output_tokens`. Do not add cached input or reasoning tokens again. + +## Dashboard + +The **Request Logs** token cell shows total tokens, with reported cached-input and reasoning counts underneath. Open **Details** to see the exact reported reasoning count and its relationship to output tokens. + +The **Reports** page shows the reported reasoning total for the selected date range and filters. Its coverage count states how many requests supplied a reasoning value. The daily breakdown and CSV export include the same reasoning field. + +## Missing Usage + +A reported zero remains `0`. A missing value remains unknown and appears as `—` in the daily report or is omitted from request details. codex-lb does not turn missing usage into zero. + +Reasoning usage may be missing when the upstream terminal event does not include it, when a stream ends before that event arrives, or for older request-log rows. The dashboard does not backfill those rows. Custom OpenAI-compatible model sources do not currently feed reasoning details into this reporting path. + +--- + +*Spec: [frontend-architecture](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/frontend-architecture)* diff --git a/frontend/src/__integration__/reports-date-range-flow.test.tsx b/frontend/src/__integration__/reports-date-range-flow.test.tsx index 01e62aa511..bc30e5b8f5 100644 --- a/frontend/src/__integration__/reports-date-range-flow.test.tsx +++ b/frontend/src/__integration__/reports-date-range-flow.test.tsx @@ -14,6 +14,8 @@ const EMPTY_REPORT: ReportsResponse = { totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, totalCancelled: 0, diff --git a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx index 88bd0f8083..eb91bb6c19 100644 --- a/frontend/src/features/dashboard/components/recent-requests-table.test.tsx +++ b/frontend/src/features/dashboard/components/recent-requests-table.test.tsx @@ -458,6 +458,76 @@ describe("RecentRequestsTable", () => { expect(within(row as HTMLElement).getByText("200.0")).toBeInTheDocument(); }); + it("shows reasoning as secondary token metadata and an included-output detail", () => { + render( + , + ); + + expect(screen.getByText("1.2K")).toBeInTheDocument(); + expect(screen.getByText("80 reasoning")).toBeInTheDocument(); + + const dialog = openRequestDetails(); + const reasoningLabel = within(dialog).getByText( + "Reasoning tokens (included in output)", + ); + expect(reasoningLabel.parentElement?.parentElement).toHaveTextContent("80"); + }); + + it("renders a known zero reasoning count", () => { + render( + , + ); + + expect(screen.getByText("0 reasoning")).toBeInTheDocument(); + const dialog = openRequestDetails(); + const reasoningLabel = within(dialog).getByText( + "Reasoning tokens (included in output)", + ); + expect(reasoningLabel.parentElement?.parentElement).toHaveTextContent("0"); + }); + + it("omits unknown reasoning usage instead of estimating it", () => { + render( + , + ); + + expect(screen.queryByText(/reasoning/i)).not.toBeInTheDocument(); + const dialog = openRequestDetails(); + expect( + within(dialog).queryByText("Reasoning tokens (included in output)"), + ).not.toBeInTheDocument(); + }); + it("does not calculate TPS from fallback output tokens", () => { render( )} + {request.reasoningTokens != null ? ( +
    + {t("dashboard.requests.reasoningTokensShort", { + count: formatCompactNumber(request.reasoningTokens), + })} +
    + ) : null}
    : null} {isColumnVisible("cost") ? @@ -595,6 +602,13 @@ export function RecentRequestsTable({ + {selectedRequest?.reasoningTokens != null ? ( + + ) : null}
    diff --git a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx index 26f5b0c490..1bc4405b45 100644 --- a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx @@ -41,6 +41,7 @@ describe("CostPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, @@ -66,6 +67,7 @@ describe("CostPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, @@ -78,6 +80,7 @@ describe("CostPerDayChart", () => { conversations: 0, inputTokens: 6_800_000, outputTokens: 73_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, diff --git a/frontend/src/features/reports/components/daily-detail-table.test.tsx b/frontend/src/features/reports/components/daily-detail-table.test.tsx index 87ace864ca..61fa83961c 100644 --- a/frontend/src/features/reports/components/daily-detail-table.test.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.test.tsx @@ -11,8 +11,12 @@ import { type DailyDetailTableProps, } from "./daily-detail-table"; -type DailyDetailTableFixtureRow = Omit & { +type DailyDetailTableFixtureRow = Omit< + DailyReportRow, + "cancelledCount" | "reasoningTokens" +> & { cancelledCount?: number; + reasoningTokens?: number | null; }; function DailyDetailTable({ @@ -22,7 +26,7 @@ function DailyDetailTable({ return ( ({ cancelledCount: 0, ...row }))} + data={data.map((row) => ({ cancelledCount: 0, reasoningTokens: 0, ...row }))} /> ); } @@ -127,6 +131,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 100, outputTokens: 20, + reasoningTokens: 12, cachedInputTokens: 0, costUsd: 1, activeAccounts: 1, @@ -137,6 +142,8 @@ describe("DailyDetailTable", () => { expect(Reflect.get(rows[0] ?? {}, "cancelledCount")).toBe(2); expect(Reflect.get(rows[1] ?? {}, "cancelledCount")).toBe(0); + expect(rows[0]?.reasoningTokens).toBe(12); + expect(rows[1]?.reasoningTokens).toBe(0); expect(rows[0]?.requests).toBe(4); expect(rows[0]?.errorCount).toBe(1); }); @@ -330,6 +337,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 100, outputTokens: 20, + reasoningTokens: 12, cachedInputTokens: 1, costUsd: 1, activeAccounts: 3, @@ -347,18 +355,102 @@ describe("DailyDetailTable", () => { expect(revokeObjectURL).toHaveBeenCalledWith("blob:daily-breakdown"); await expect(blobText()).resolves.toBe( [ - "Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors", - "2026-06-05,4,0,100,20,1,1.0000,3,2,1", - "2026-06-06,0,0,0,0,0,0.0000,0,0,0", + "Date,Requests,Conversations,Input Tokens,Output Tokens,Reported Reasoning Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors", + "2026-06-05,4,0,100,20,12,1,1.0000,3,2,1", + "2026-06-06,0,0,0,0,0,0,0.0000,0,0,0", ].join("\n"), ); }); + it("renders and exports unknown reasoning separately from known zero and sorts unknown last", async () => { + const user = userEvent.setup(); + const blobText = vi.fn(async () => ""); + vi.spyOn(URL, "createObjectURL").mockImplementation((blob) => { + if (!(blob instanceof Blob)) { + throw new TypeError("expected Blob export payload"); + } + blobText.mockImplementation(() => blob.text()); + return "blob:nullable-reasoning"; + }); + vi.spyOn(URL, "revokeObjectURL").mockImplementation(() => {}); + vi.spyOn(HTMLAnchorElement.prototype, "click").mockImplementation(() => {}); + + render( + , + ); + + const unknownCells = screen + .getByTestId("daily-breakdown-row-2026-06-05") + .querySelectorAll("td"); + const zeroCells = screen + .getByTestId("daily-breakdown-row-2026-06-06") + .querySelectorAll("td"); + expect(unknownCells[5]?.textContent?.trim()).toBe("—"); + expect(zeroCells[5]?.textContent?.trim()).toBe("0"); + + await user.click(screen.getByRole("button", { name: "Reported Reasoning Tokens" })); + expect( + screen.getAllByTestId(/daily-breakdown-row-/).map((row) => row.dataset.testid), + ).toEqual([ + "daily-breakdown-row-2026-06-06", + "daily-breakdown-row-2026-06-07", + "daily-breakdown-row-2026-06-05", + ]); + + await user.click(screen.getByRole("button", { name: /csv/i })); + const csvLines = (await blobText()).split("\n"); + expect(csvLines[1]?.split(",")[5]).toBe(""); + expect(csvLines[2]?.split(",")[5]).toBe("0"); + expect(csvLines[3]?.split(",")[5]).toBe("5"); + }); + it.each([ ["Day", "daily-breakdown-row-2026-06-05"], ["Reqs", "daily-breakdown-row-2026-06-06"], ["Input Tokens", "daily-breakdown-row-2026-06-05"], ["Output Tokens", "daily-breakdown-row-2026-06-05"], + ["Reported Reasoning Tokens", "daily-breakdown-row-2026-06-06"], ["Cost", "daily-breakdown-row-2026-06-05"], ["Accounts", "daily-breakdown-row-2026-06-06"], ])("sorts by %s when its header is clicked", async (headerLabel, expectedFirstRow) => { @@ -376,6 +468,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 100, outputTokens: 20, + reasoningTokens: 5, cachedInputTokens: 0, costUsd: 1, activeAccounts: 3, @@ -387,6 +480,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 200, outputTokens: 30, + reasoningTokens: 1, cachedInputTokens: 0, costUsd: 2, activeAccounts: 1, @@ -398,6 +492,7 @@ describe("DailyDetailTable", () => { conversations: 0, inputTokens: 300, outputTokens: 40, + reasoningTokens: 3, cachedInputTokens: 0, costUsd: 3, activeAccounts: 2, @@ -583,15 +678,15 @@ describe("DailyDetailTable", () => { const headerRow = screen.getAllByRole("row")[0]; const headerCells = Array.from(headerRow?.querySelectorAll("th") ?? []); const labels = headerCells.map((c) => c.textContent?.trim() ?? ""); - expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Cost", "Accounts", "Cancelled", "Errors"]); + expect(labels).toEqual(["Day", "Reqs", "Conversations", "Input Tokens", "Output Tokens", "Reported Reasoning Tokens", "Cost", "Accounts", "Cancelled", "Errors"]); // CSV: full header + first data row with Conversations between Requests and Input Tokens await user.click(screen.getByRole("button", { name: /csv/i })); const csv = await blobText(); const csvLines = csv.split("\n"); - expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors"); + expect(csvLines[0]).toBe("Date,Requests,Conversations,Input Tokens,Output Tokens,Reported Reasoning Tokens,Cached Tokens,Cost USD,Active Accounts,Cancelled,Errors"); // First data row in CSV (chronological: 06-05 first, conversations=1) - expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,1\.0000,1,0,0/); + expect(csvLines[1]).toMatch(/2026-06-05,8,1,100,20,0,0,1\.0000,1,0,0/); }); it("zero-filled gap rows have conversations=0 in column 2", () => { @@ -616,7 +711,7 @@ describe("DailyDetailTable", () => { expect(dataCells[2]?.textContent?.trim()).toBe("3"); }); - it("both header and body tables share min-width for mobile overflow", () => { + it("keeps headers and rows in one horizontally scrollable table", () => { render( { />, ); - const tables = document.querySelectorAll("table.min-w-\\[900px\\]"); - expect(tables.length).toBe(2); + const scrollContainer = screen.getByTestId("daily-breakdown-scroll-body"); + const tables = scrollContainer.querySelectorAll("table.min-w-\\[1000px\\]"); + expect(scrollContainer).toHaveClass("overflow-x-auto", "overflow-y-auto"); + expect(tables).toHaveLength(1); + expect(tables[0]?.querySelector("thead")).toBeInTheDocument(); + expect(tables[0]?.querySelector("tbody")).toBeInTheDocument(); }); }); diff --git a/frontend/src/features/reports/components/daily-detail-table.tsx b/frontend/src/features/reports/components/daily-detail-table.tsx index 21dd78fd7e..4d539db94d 100644 --- a/frontend/src/features/reports/components/daily-detail-table.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.tsx @@ -16,7 +16,7 @@ export type DailyDetailTableProps = { const DAILY_BREAKDOWN_SCROLL_HEIGHT_CLASS = "max-h-[17.5rem]"; -type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "costUsd" | "activeAccounts" | "cancelledCount" | "errorCount"; +type SortKey = "date" | "requests" | "conversations" | "inputTokens" | "outputTokens" | "reasoningTokens" | "costUsd" | "activeAccounts" | "cancelledCount" | "errorCount"; type SortDirection = "asc" | "desc"; function formatTokens(v: number): string { @@ -58,10 +58,13 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP {t("reports.dailyBreakdown.csv")}
    -
    - +
    +
    - + toggleSort("outputTokens")} /> + toggleSort("reasoningTokens")} + /> + + {rows.map((row) => ( + + + + + + + + + + + + + ))} +
    + {formatReportBucketDate(row.date, dateDisplayFormat)} + {row.requests} + {row.conversations} + + {formatTokens(row.inputTokens)}{" "} + + ({formatTokens(row.cachedInputTokens)}) + + + {formatTokens(row.outputTokens)} + + {row.reasoningTokens == null ? "—" : formatTokens(row.reasoningTokens)} + + ${row.costUsd.toFixed(2)} + + {row.activeAccounts} + + {row.cancelledCount} + {row.errorCount}
    -
    - - - - {rows.map((row) => ( - - - - - - - - - - - - ))} - -
    - {formatReportBucketDate(row.date, dateDisplayFormat)} - - {row.requests} - - {row.conversations} - - {formatTokens(row.inputTokens)}{" "} - - ({formatTokens(row.cachedInputTokens)}) - - - {formatTokens(row.outputTokens)} - - ${row.costUsd.toFixed(2)} - - {row.activeAccounts} - - {row.cancelledCount} - - {row.errorCount} -
    -
    ); @@ -220,14 +220,15 @@ function ColumnGroup() { return ( + - - - - - - + + + + + + ); } @@ -240,6 +241,15 @@ function sortRows( const leftValue = left[sort.key]; const rightValue = right[sort.key]; + if (leftValue == null && rightValue == null) { + return 0; + } + if (leftValue == null) { + return 1; + } + if (rightValue == null) { + return -1; + } if (leftValue < rightValue) { return sort.direction === "asc" ? -1 : 1; } @@ -259,6 +269,7 @@ function exportCSV(rows: DailyReportRow[], t: TFunction) { t("reports.dailyBreakdown.csvColumns.conversations"), t("reports.dailyBreakdown.csvColumns.inputTokens"), t("reports.dailyBreakdown.csvColumns.outputTokens"), + t("reports.dailyBreakdown.csvColumns.reasoningTokens"), t("reports.dailyBreakdown.csvColumns.cachedTokens"), t("reports.dailyBreakdown.csvColumns.costUsd"), t("reports.dailyBreakdown.csvColumns.activeAccounts"), @@ -266,7 +277,7 @@ function exportCSV(rows: DailyReportRow[], t: TFunction) { t("reports.dailyBreakdown.csvColumns.errors"), ]; const lines = rows.map((r) => - [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.cancelledCount, r.errorCount].join(","), + [r.date, r.requests, r.conversations, r.inputTokens, r.outputTokens, r.reasoningTokens ?? "", r.cachedInputTokens, r.costUsd.toFixed(4), r.activeAccounts, r.cancelledCount, r.errorCount].join(","), ); const csv = [headers.join(","), ...lines].join("\n"); const blob = new Blob([csv], { type: "text/csv;charset=utf-8;" }); diff --git a/frontend/src/features/reports/components/queue-wait-chart.test.tsx b/frontend/src/features/reports/components/queue-wait-chart.test.tsx index 91ba329ccc..7965ccf0e1 100644 --- a/frontend/src/features/reports/components/queue-wait-chart.test.tsx +++ b/frontend/src/features/reports/components/queue-wait-chart.test.tsx @@ -29,6 +29,7 @@ const BASE_ROW = { conversations: 0, inputTokens: 1_000, outputTokens: 100, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 0.5, activeAccounts: 1, diff --git a/frontend/src/features/reports/components/reports-page.test.tsx b/frontend/src/features/reports/components/reports-page.test.tsx index 26512ce755..2136291385 100644 --- a/frontend/src/features/reports/components/reports-page.test.tsx +++ b/frontend/src/features/reports/components/reports-page.test.tsx @@ -49,6 +49,8 @@ const EMPTY_REPORT: ReportsResponse = { totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, totalCancelled: 0, @@ -873,6 +875,7 @@ describe("ReportsPage", () => { conversations: 2, inputTokens: 1000, outputTokens: 200, + reasoningTokens: 0, cachedInputTokens: 50, costUsd: 0.15, activeAccounts: 1, @@ -904,6 +907,6 @@ describe("ReportsPage", () => { expect(createObjectURL).toHaveBeenCalledOnce(); const csvContent = await blobText(); - expect(csvContent).toContain("2030-01-15,42,2,1000,200,50,0.1500,1,0,0"); + expect(csvContent).toContain("2030-01-15,42,2,1000,200,0,50,0.1500,1,0,0"); }); }); diff --git a/frontend/src/features/reports/components/reports-summary-cards.test.tsx b/frontend/src/features/reports/components/reports-summary-cards.test.tsx index d2eb22f71c..dc80664d22 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.test.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.test.tsx @@ -8,8 +8,13 @@ import { type ReportsSummaryCardsProps, } from "./reports-summary-cards"; -type ReportsSummaryFixture = Omit & { +type ReportsSummaryFixture = Omit< + ReportSummary, + "totalCancelled" | "totalReasoningTokens" | "reasoningUsageKnownRequests" +> & { totalCancelled?: number; + totalReasoningTokens?: number; + reasoningUsageKnownRequests?: number; }; function ReportsSummaryCards({ @@ -19,7 +24,12 @@ function ReportsSummaryCards({ return ( ); } @@ -32,6 +42,8 @@ describe("ReportsSummaryCards", () => { totalCostUsd: 15, totalInputTokens: 1_600_000_000, totalOutputTokens: 13_000_000, + totalReasoningTokens: 8_000_000, + reasoningUsageKnownRequests: 1400, totalCachedTokens: 990_000_000, totalRequests: 1500, totalErrors: 0, @@ -70,11 +82,50 @@ describe("ReportsSummaryCards", () => { ); expect( - within(tokensCard).getByText("Input 1.6B · Cache 990M · Output 13.0M"), + within(tokensCard).getByText( + "Input 1.6B · Cache 990M · Output 13.0M", + ), ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 8.0M (included in output) · 1400/1500 requests")).toBeInTheDocument(); + expect(tokensCard.parentElement).toHaveClass("lg:grid-cols-3", "xl:grid-cols-6"); expect(within(requestsCard).getByText("avg 500/day · 3 accounts")).toBeInTheDocument(); }); + it("shows reported reasoning coverage without adding reasoning to the token total", () => { + render( + , + ); + + const tokensCard = screen.getByTestId("report-summary-card-tokens"); + expect(within(tokensCard).getByText("140")).toBeInTheDocument(); + expect( + within(tokensCard).getByText( + "Input 100 · Cache 10 · Output 40", + ), + ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 30 (included in output) · 2/4 requests")).toBeInTheDocument(); + expect(within(tokensCard).queryByText("170")).not.toBeInTheDocument(); + }); + it("hides comparison badges when unavailable or previous totals are zero", () => { const { rerender } = render( { const requestsCard = screen.getByTestId("report-summary-card-requests"); expect(within(tokensCard).getByText("100.0B")).toBeInTheDocument(); - expect(within(tokensCard).getByText("Input 100.0B · Cache 0 · Output 0")).toBeInTheDocument(); + expect( + within(tokensCard).getByText( + "Input 100.0B · Cache 0 · Output 0", + ), + ).toBeInTheDocument(); + expect(within(tokensCard).getByText("Reported reasoning 0 (included in output) · 0/100000 requests")).toBeInTheDocument(); expect(within(requestsCard).getByText("100.0K")).toBeInTheDocument(); }); + + it("hides reasoning coverage when there are no requests", () => { + render( + , + ); + + expect(screen.queryByText(/Reported reasoning/)).not.toBeInTheDocument(); + }); }); diff --git a/frontend/src/features/reports/components/reports-summary-cards.tsx b/frontend/src/features/reports/components/reports-summary-cards.tsx index 9919936f3c..09807f38e4 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.tsx @@ -33,6 +33,14 @@ export function ReportsSummaryCards({ summary, comparison }: ReportsSummaryCards cache: formatNumber(summary.totalCachedTokens), output: formatNumber(summary.totalOutputTokens), }), + secondarySub: + summary.totalRequests > 0 + ? t("reports.summary.reasoningSub", { + reasoning: formatNumber(summary.totalReasoningTokens), + known: summary.reasoningUsageKnownRequests, + total: summary.totalRequests, + }) + : undefined, comparison: buildComparison( summary.totalInputTokens + summary.totalOutputTokens, comparison.previous.totalTokens, @@ -67,7 +75,7 @@ export function ReportsSummaryCards({ summary, comparison }: ReportsSummaryCards ]; return ( -
    +
    {cards.map((card) => (
    {card.sub ?
    {card.sub}
    : null} + {"secondarySub" in card && card.secondarySub ? ( +
    {card.secondarySub}
    + ) : null}
    ))}
    diff --git a/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx b/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx index d4b21071e2..2906898de2 100644 --- a/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/tokens-per-day-chart.test.tsx @@ -41,6 +41,7 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, @@ -66,6 +67,7 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 5_400_000, outputTokens: 59_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 3.77, activeAccounts: 2, @@ -78,6 +80,7 @@ describe("TokensPerDayChart", () => { conversations: 0, inputTokens: 6_800_000, outputTokens: 73_000, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 4.54, activeAccounts: 2, diff --git a/frontend/src/features/reports/daily-series.ts b/frontend/src/features/reports/daily-series.ts index 999d9166b3..f76723e8c5 100644 --- a/frontend/src/features/reports/daily-series.ts +++ b/frontend/src/features/reports/daily-series.ts @@ -41,6 +41,7 @@ function createZeroRow(date: string): DailyReportRow { conversations: 0, inputTokens: 0, outputTokens: 0, + reasoningTokens: 0, cachedInputTokens: 0, costUsd: 0, activeAccounts: 0, diff --git a/frontend/src/features/reports/hooks/use-reports.test.tsx b/frontend/src/features/reports/hooks/use-reports.test.tsx index e6661494fb..01c6a01495 100644 --- a/frontend/src/features/reports/hooks/use-reports.test.tsx +++ b/frontend/src/features/reports/hooks/use-reports.test.tsx @@ -12,6 +12,8 @@ vi.mock("@/lib/api-client", () => ({ totalCostUsd: 0, totalInputTokens: 0, totalOutputTokens: 0, + totalReasoningTokens: 0, + reasoningUsageKnownRequests: 0, totalCachedTokens: 0, totalRequests: 0, totalErrors: 0, diff --git a/frontend/src/features/reports/schemas.test.ts b/frontend/src/features/reports/schemas.test.ts index 0d7e9d9ab9..464ec0932b 100644 --- a/frontend/src/features/reports/schemas.test.ts +++ b/frontend/src/features/reports/schemas.test.ts @@ -6,12 +6,13 @@ function validReportsPayload() { return { summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 4, totalCancelled: 2, totalErrors: 1, totalConversations: 7, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 2, errorCount: 1 }], + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 2, errorCount: 1 }], byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], byAccount: [], @@ -19,16 +20,17 @@ function validReportsPayload() { } describe("ReportsResponseSchema", () => { - it("preserves conversation and cancellation totals from the reports payload", () => { + it("preserves conversation, cancellation, and reasoning totals from the reports payload", () => { const parsed = ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 4, totalErrors: 1, totalCancelled: 2, totalConversations: 7, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 1, cancelledCount: 2 }], + daily: [{ date: "2026-06-05", requests: 4, conversations: 3, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, errorCount: 1, cancelledCount: 2 }], byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 4, percentage: 100 }], byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 4, percentage: 100 }], byAccount: [], @@ -37,10 +39,13 @@ describe("ReportsResponseSchema", () => { expect(parsed.summary.totalErrors).toBe(1); expect.soft(Reflect.get(parsed.summary, "totalCancelled")).toBe(2); expect(parsed.summary.totalConversations).toBe(7); + expect(parsed.summary.totalReasoningTokens).toBe(70); + expect(parsed.summary.reasoningUsageKnownRequests).toBe(3); expect(parsed.daily[0]?.requests).toBe(4); expect(parsed.daily[0]?.errorCount).toBe(1); expect.soft(Reflect.get(parsed.daily[0] ?? {}, "cancelledCount")).toBe(2); expect(parsed.daily[0]?.conversations).toBe(3); + expect(parsed.daily[0]?.reasoningTokens).toBe(35); }); it("rejects omitted totalCancelled on summary", () => { @@ -57,16 +62,42 @@ describe("ReportsResponseSchema", () => { expect(() => ReportsResponseSchema.parse(payload)).toThrow(/cancelledCount/i); }); + it("rejects omitted reasoning totals and coverage on summary", () => { + const missingTotal = validReportsPayload(); + Reflect.deleteProperty(missingTotal.summary, "totalReasoningTokens"); + expect(() => ReportsResponseSchema.parse(missingTotal)).toThrow(/totalReasoningTokens/i); + + const missingCoverage = validReportsPayload(); + Reflect.deleteProperty(missingCoverage.summary, "reasoningUsageKnownRequests"); + expect(() => ReportsResponseSchema.parse(missingCoverage)).toThrow(/reasoningUsageKnownRequests/i); + }); + + it("rejects omitted reasoningTokens on daily rows", () => { + const payload = validReportsPayload(); + Reflect.deleteProperty(payload.daily[0] ?? {}, "reasoningTokens"); + + expect(() => ReportsResponseSchema.parse(payload)).toThrow(/reasoningTokens/i); + }); + + it("preserves null reasoningTokens as unknown on daily rows", () => { + const payload = validReportsPayload(); + Reflect.set(payload.daily[0] ?? {}, "reasoningTokens", null); + + const parsed = ReportsResponseSchema.parse(payload); + expect(parsed.daily[0]?.reasoningTokens).toBeNull(); + }); + it("rejects omitted totalConversations on summary", () => { expect(() => ReportsResponseSchema.parse({ summary: { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, totalErrors: 1, activeAccounts: 3, avgCostPerDay: 4.17, avgRequestsPerDay: 8.33, }, comparison: { canCompare: true, previous: { totalCostUsd: 10, totalTokens: 400, totalRequests: 20 } }, - daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 0, errorCount: 0 }], + daily: [{ date: "2026-06-05", requests: 10, conversations: 0, inputTokens: 100, outputTokens: 50, reasoningTokens: 35, cachedInputTokens: 0, costUsd: 1, activeAccounts: 2, cancelledCount: 0, errorCount: 0 }], byModel: [{ model: "gpt-5.1", costUsd: 12.5, requests: 25, percentage: 100 }], byUseragent: [{ useragent: "claude-code", costUsd: 12.5, requests: 25, percentage: 100 }], byAccount: [], @@ -81,6 +112,8 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, @@ -133,6 +166,8 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, @@ -157,6 +192,8 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, @@ -184,6 +221,8 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, @@ -222,6 +261,8 @@ describe("ReportsResponseSchema", () => { totalCostUsd: 12.5, totalInputTokens: 300, totalOutputTokens: 200, + totalReasoningTokens: 70, + reasoningUsageKnownRequests: 3, totalCachedTokens: 0, totalRequests: 25, totalCancelled: 0, diff --git a/frontend/src/features/reports/schemas.ts b/frontend/src/features/reports/schemas.ts index 33e5b65397..07dc9a7fed 100644 --- a/frontend/src/features/reports/schemas.ts +++ b/frontend/src/features/reports/schemas.ts @@ -6,6 +6,7 @@ const DailyReportRowSchema = z.object({ conversations: z.number(), inputTokens: z.number(), outputTokens: z.number(), + reasoningTokens: z.number().nullable(), cachedInputTokens: z.number(), costUsd: z.number(), activeAccounts: z.number(), @@ -41,6 +42,8 @@ const ReportSummarySchema = z.object({ totalCostUsd: z.number(), totalInputTokens: z.number(), totalOutputTokens: z.number(), + totalReasoningTokens: z.number(), + reasoningUsageKnownRequests: z.number(), totalCachedTokens: z.number(), totalRequests: z.number(), totalCancelled: z.number(), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 41a81f801d..84c7efbf55 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -689,6 +689,7 @@ "dashboard.requestDetails.fullError": "Full Error", "dashboard.requestDetails.noErrorDetail": "No error detail recorded.", "dashboard.requestDetails.queue": "Queue", + "dashboard.requestDetails.reasoningTokensIncluded": "Reasoning tokens (included in output)", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "Request kind", "dashboard.requestDetails.routeEndpoint": "Proxy endpoint", @@ -721,6 +722,7 @@ "dashboard.requests.emptyFilteredTitle": "No matching requests", "dashboard.requests.emptyTitle": "No requests yet", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.reasoningTokensShort": "{{count}} reasoning", "dashboard.requests.resizeColumn": "Resize {{column}} column", "dashboard.requests.title": "Request Logs", "dashboard.requests.unassigned": "Unassigned", @@ -974,6 +976,7 @@ "reports.dailyBreakdown.columns.errors": "Errors", "reports.dailyBreakdown.columns.inputTokens": "Input Tokens", "reports.dailyBreakdown.columns.outputTokens": "Output Tokens", + "reports.dailyBreakdown.columns.reasoningTokens": "Reported Reasoning Tokens", "reports.dailyBreakdown.columns.reqs": "Reqs", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "Active Accounts", @@ -986,6 +989,7 @@ "reports.dailyBreakdown.csvColumns.errors": "Errors", "reports.dailyBreakdown.csvColumns.inputTokens": "Input Tokens", "reports.dailyBreakdown.csvColumns.outputTokens": "Output Tokens", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "Reported Reasoning Tokens", "reports.dailyBreakdown.csvColumns.requests": "Requests", "reports.dailyBreakdown.title": "Daily Breakdown", "reports.distribution.byModel": "Distribution by Model", @@ -1011,6 +1015,7 @@ "reports.summary.requestsSub": "avg {{requests}}/day · {{accounts}} accounts", "reports.summary.tokens": "Tokens", "reports.summary.tokensSub": "Input {{input}} · Cache {{cache}} · Output {{output}}", + "reports.summary.reasoningSub": "Reported reasoning {{reasoning}} (included in output) · {{known}}/{{total}} requests", "reports.summary.totalCost": "Total Cost", "reports.summary.conversations": "Active Conversations", "settings.page.title": "Settings", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index b5d482b8c9..1b763ad5f5 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -689,6 +689,7 @@ "dashboard.requestDetails.fullError": "전체 오류", "dashboard.requestDetails.noErrorDetail": "기록된 오류 상세가 없습니다.", "dashboard.requestDetails.queue": "Queue", + "dashboard.requestDetails.reasoningTokensIncluded": "추론 token (Output token에 포함)", "dashboard.requestDetails.requestId": "Request ID", "dashboard.requestDetails.requestKind": "요청 종류", "dashboard.requestDetails.routeEndpoint": "프록시 엔드포인트", @@ -721,6 +722,7 @@ "dashboard.requests.emptyFilteredTitle": "일치하는 요청이 없습니다", "dashboard.requests.emptyTitle": "아직 요청이 없습니다", "dashboard.requests.requestedTier": "Requested {{tier}}", + "dashboard.requests.reasoningTokensShort": "추론 {{count}}", "dashboard.requests.resizeColumn": "{{column}} 열 크기 조절", "dashboard.requests.title": "요청 로그", "dashboard.requests.unassigned": "미할당", @@ -974,6 +976,7 @@ "reports.dailyBreakdown.columns.errors": "오류", "reports.dailyBreakdown.columns.inputTokens": "Input token", "reports.dailyBreakdown.columns.outputTokens": "Output token", + "reports.dailyBreakdown.columns.reasoningTokens": "보고된 추론 token", "reports.dailyBreakdown.columns.reqs": "요청", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "활성 Accounts", @@ -984,6 +987,7 @@ "reports.dailyBreakdown.csvColumns.errors": "오류", "reports.dailyBreakdown.csvColumns.inputTokens": "Input token", "reports.dailyBreakdown.csvColumns.outputTokens": "Output token", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "보고된 추론 token", "reports.dailyBreakdown.csvColumns.requests": "요청", "reports.dailyBreakdown.title": "일별 상세", "reports.distribution.byModel": "Model별 분포", @@ -1010,6 +1014,7 @@ "reports.summary.requestsSub": "평균 {{requests}}/일 · {{accounts}} Accounts", "reports.summary.tokens": "Token", "reports.summary.tokensSub": "Input {{input}} · Cache {{cache}} · Output {{output}}", + "reports.summary.reasoningSub": "보고된 추론 {{reasoning}} (Output 일부) · {{known}}/{{total}}개 요청", "reports.summary.totalCost": "총 비용", "reports.dailyBreakdown.columns.conversations": "대화", "reports.dailyBreakdown.csvColumns.conversations": "대화", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 60d781635a..224c87c2cc 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -689,6 +689,7 @@ "dashboard.requestDetails.fullError": "完整错误", "dashboard.requestDetails.noErrorDetail": "未记录错误详情。", "dashboard.requestDetails.queue": "队列", + "dashboard.requestDetails.reasoningTokensIncluded": "推理 token（包含在输出 token 中）", "dashboard.requestDetails.requestId": "请求 ID", "dashboard.requestDetails.requestKind": "请求类型", "dashboard.requestDetails.routeEndpoint": "代理端点", @@ -721,6 +722,7 @@ "dashboard.requests.emptyFilteredTitle": "没有匹配的请求", "dashboard.requests.emptyTitle": "暂无请求", "dashboard.requests.requestedTier": "请求 {{tier}}", + "dashboard.requests.reasoningTokensShort": "推理 {{count}}", "dashboard.requests.resizeColumn": "调整“{{column}}”列宽", "dashboard.requests.title": "请求日志", "dashboard.requests.unassigned": "未分配", @@ -974,6 +976,7 @@ "reports.dailyBreakdown.columns.errors": "错误", "reports.dailyBreakdown.columns.inputTokens": "输入 token", "reports.dailyBreakdown.columns.outputTokens": "输出 token", + "reports.dailyBreakdown.columns.reasoningTokens": "已报告推理 token", "reports.dailyBreakdown.columns.reqs": "请求", "reports.dailyBreakdown.csv": "CSV", "reports.dailyBreakdown.csvColumns.activeAccounts": "活跃账户", @@ -984,6 +987,7 @@ "reports.dailyBreakdown.csvColumns.errors": "错误", "reports.dailyBreakdown.csvColumns.inputTokens": "输入 token", "reports.dailyBreakdown.csvColumns.outputTokens": "输出 token", + "reports.dailyBreakdown.csvColumns.reasoningTokens": "已报告推理 token", "reports.dailyBreakdown.csvColumns.requests": "请求", "reports.dailyBreakdown.title": "每日明细", "reports.distribution.byModel": "按 Model 分布", @@ -1010,6 +1014,7 @@ "reports.summary.requestsSub": "平均 {{requests}}/天 · {{accounts}} 个账户", "reports.summary.tokens": "Token", "reports.summary.tokensSub": "输入 {{input}} · 缓存 {{cache}} · 输出 {{output}}", + "reports.summary.reasoningSub": "已报告推理 {{reasoning}}（输出的一部分）· {{known}}/{{total}} 个请求", "reports.summary.totalCost": "总费用", "reports.dailyBreakdown.columns.conversations": "对话", "reports.dailyBreakdown.csvColumns.conversations": "对话", diff --git a/mkdocs.yml b/mkdocs.yml index 62fe5737cf..7f77b01a23 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -59,6 +59,7 @@ nav: - Client Setup: client-setup.md - Live Voice: live-voice.md - Conversations: conversations.md + - Usage Reporting: usage-reporting.md - Configuration: configuration.md - Anonymous Telemetry: telemetry.md - Authentication: authentication.md diff --git a/openspec/changes/surface-reasoning-token-usage/design.md b/openspec/changes/surface-reasoning-token-usage/design.md new file mode 100644 index 0000000000..959782e813 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/design.md @@ -0,0 +1,38 @@ +## Context + +The Responses API reports exact reasoning usage at `usage.output_tokens_details.reasoning_tokens`. codex-lb already parses that terminal usage object into `request_logs.reasoning_tokens` for direct Codex subscription traffic over HTTP and WebSocket and exposes it as `reasoningTokens` from the request-log API. The request table and reports page omit the value, while reports aggregate only input, cached-input, and inclusive output totals. + +## Goals / Non-Goals + +**Goals:** + +- Make per-request reasoning usage visible without a database query. +- Aggregate reported reasoning usage over the same date and account/model/user-agent filters as the existing reports totals, with summary coverage for requests whose count is known. +- State the subset relationship in the UI contract so reasoning is never added to output a second time. + +**Non-Goals:** + +- Estimate reasoning usage from reasoning summaries or visible text. +- Change token limits, pricing, cost calculation, or API-key quota enforcement. +- Backfill responses whose upstream terminal event did not provide usage. +- Extend reasoning-detail capture for custom OpenAI-compatible model sources; their forwarding parser is a separate protocol change. + +## Decisions + +- Use the upstream-provided count already stored in `request_logs.reasoning_tokens`; no tokenizer or heuristic is introduced. +- Keep `outputTokens` inclusive of reasoning tokens. `reasoningTokens` is an additive breakdown field, not another component of total tokens. +- Add nullable `reasoningTokens` to each reports daily row, plus `totalReasoningTokens` and `reasoningUsageKnownRequests` to the reports summary. An all-unknown day remains null, while a known zero remains zero. The previous-window token comparison remains input plus inclusive output, using the existing reasoning-only fallback when an older row lacks an output total. +- Render the reasoning subset as secondary request-row metadata and as an explicit request-detail field. Reports render it in the token summary, daily table, and CSV. + +## Risks / Trade-offs + +- [Risk] Operators add reasoning to output and overstate usage. The spec and UI copy identify reasoning as included in output, and total-token calculations continue to use input plus inclusive output without adding reasoning again. +- [Risk] Legacy, cancelled, or interrupted rows may have no terminal reasoning count. Request history leaves the value unknown, an all-unknown daily aggregate remains null, reports label the aggregate as reported reasoning, and summary coverage states how many requests supplied a count. Missing values are excluded rather than inferred as known zero. + +## Migration Plan + +Ship the additive API and dashboard fields together. Rollback removes the new fields and rendering; the existing `request_logs.reasoning_tokens` data remains intact. + +## Open Questions + +None. diff --git a/openspec/changes/surface-reasoning-token-usage/proposal.md b/openspec/changes/surface-reasoning-token-usage/proposal.md new file mode 100644 index 0000000000..0099bad190 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/proposal.md @@ -0,0 +1,24 @@ +## Why + +codex-lb already persists the upstream reasoning-token count reported for completed direct Codex subscription responses, but the dashboard only renders total, cached-input, and output token totals. Operators cannot see the reasoning subset in request history or aggregate it over a report window without querying the database directly. + +## What Changes + +- Show the persisted reasoning-token count in request-log rows and request details. +- Add reported reasoning-token totals and summary coverage to the reports response. +- Render reported reasoning totals in the reports summary, daily breakdown, and CSV export. +- Keep reasoning tokens as a subset of output tokens so existing total-token and cost calculations do not double-count them. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `frontend-architecture`: Request history and reports expose the reasoning-token subset already recorded by the proxy. + +## Impact + +Reports API aggregation, dashboard schemas and components, localized labels, user-facing usage-reporting documentation, and focused backend/frontend tests. No database migration, configuration, routing, pricing, or proxy-protocol change. diff --git a/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md b/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..82d221ca47 --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/specs/frontend-architecture/spec.md @@ -0,0 +1,64 @@ +## ADDED Requirements + +### Requirement: Request logs surface reasoning-token usage + +The dashboard request-log API and UI MUST preserve and render the upstream-provided reasoning-token count separately from the inclusive output-token count. The UI MUST treat reasoning tokens as a subset of output tokens and MUST NOT add them to total-token or cost calculations. + +#### Scenario: Request row shows an available reasoning count + +- **GIVEN** a request-log row has `outputTokens=200` and `reasoningTokens=80` +- **WHEN** the dashboard renders the recent-requests table +- **THEN** the token cell shows 80 reasoning tokens as secondary metadata +- **AND** the row's total-token value remains input tokens plus 200 output tokens + +#### Scenario: Request detail identifies the reasoning subset + +- **GIVEN** a request-log row has a persisted reasoning-token count +- **WHEN** the operator opens `View Details` +- **THEN** the dialog renders the exact reasoning-token count +- **AND** its label identifies the count as included in output tokens + +#### Scenario: Missing reasoning usage is not estimated + +- **GIVEN** a request-log row has `reasoningTokens=null` +- **WHEN** the dashboard renders the row and its details +- **THEN** the dashboard does not derive a reasoning count from output text, reasoning summaries, or total output tokens + +### Requirement: Reports expose reasoning-token totals + +`GET /api/reports` MUST expose the sum of reported reasoning counts as `totalReasoningTokens` in its summary and nullable `reasoningTokens` in each daily row, using the same date, account, model, and user-agent filters as the existing token totals. The summary MUST expose `reasoningUsageKnownRequests`, counting rows whose upstream reasoning count is known, including known zeroes. The reports UI MUST label the aggregate as reported reasoning, show its known-request coverage, render it in the daily breakdown, and export it in the daily CSV. A daily row with requests but no reported reasoning counts MUST preserve `reasoningTokens=null`; known zero MUST remain zero. Existing total-token comparisons MUST remain input tokens plus inclusive output tokens, with a stored reasoning count serving as the output fallback when an older row has no output total. + +#### Scenario: Reports aggregate reasoning usage + +- **GIVEN** eligible request logs in a report window contain reasoning-token counts of 30 and 70 and one request with an unknown count +- **WHEN** an operator requests that report window +- **THEN** `summary.totalReasoningTokens` is 100 +- **AND** `summary.reasoningUsageKnownRequests` is 2 +- **AND** each daily row's `reasoningTokens` is the sum for that local-calendar day + +#### Scenario: Reports identify reasoning as an output subset + +- **GIVEN** a report summary has 1,000 input tokens, 400 output tokens, and 250 reasoning tokens +- **WHEN** the dashboard renders the token summary +- **THEN** the total remains 1,400 tokens +- **AND** the summary identifies 250 reasoning tokens as included in the 400 output tokens + +#### Scenario: Daily CSV exports reasoning tokens + +- **WHEN** an operator exports the reports daily breakdown +- **THEN** the CSV contains a Reported Reasoning Tokens column +- **AND** every row contains that day's reasoning-token aggregate + +#### Scenario: A daily aggregate has no reported reasoning usage + +- **GIVEN** a report day contains requests whose reasoning-token counts are all unknown +- **WHEN** the reports API and dashboard render that day +- **THEN** the daily `reasoningTokens` value remains null +- **AND** the table and CSV do not present it as a known zero + +#### Scenario: A legacy row has reasoning usage but no output total + +- **GIVEN** an eligible request log has `outputTokens=null` and `reasoningTokens=40` +- **WHEN** the report aggregates inclusive output tokens +- **THEN** the row contributes 40 output tokens and 40 reported reasoning tokens +- **AND** reasoning is not added to that output total a second time diff --git a/openspec/changes/surface-reasoning-token-usage/tasks.md b/openspec/changes/surface-reasoning-token-usage/tasks.md new file mode 100644 index 0000000000..13b6dbf16d --- /dev/null +++ b/openspec/changes/surface-reasoning-token-usage/tasks.md @@ -0,0 +1,21 @@ +## 1. Reports contract + +- [x] 1.1 Aggregate reasoning tokens in reports summary and daily rows +- [x] 1.2 Expose `totalReasoningTokens`, `reasoningUsageKnownRequests`, and daily `reasoningTokens` from the reports API +- [x] 1.3 Add backend regression coverage for filtered and unfiltered report windows +- [x] 1.4 Preserve reasoning-only output fallback and nullable all-unknown daily reasoning aggregates + +## 2. Dashboard presentation + +- [x] 2.1 Render reasoning usage in request rows and request details +- [x] 2.2 Render reported reasoning totals and summary coverage in reports +- [x] 2.3 Include reasoning tokens in the daily CSV export +- [x] 2.4 Add English, Korean, and Simplified Chinese labels +- [x] 2.5 Render and export all-unknown daily reasoning usage distinctly from known zero +- [x] 2.6 Document token-bucket semantics, dashboard surfaces, and missing-usage behavior in the docs site + +## 3. Validation + +- [x] 3.1 Run focused backend and frontend tests +- [x] 3.2 Run backend lint/type checks and frontend typecheck/build +- [x] 3.3 Validate OpenSpec diff --git a/tests/integration/test_reports_api.py b/tests/integration/test_reports_api.py index 30b7c53490..80b92db9a7 100644 --- a/tests/integration/test_reports_api.py +++ b/tests/integration/test_reports_api.py @@ -84,6 +84,7 @@ async def test_reports_api_returns_null_account_bucket(async_client, db_setup): "requests": 2, "inputTokens": 15, "outputTokens": 5, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -105,6 +106,194 @@ async def test_reports_api_returns_null_account_bucket(async_client, db_setup): ] +async def test_reports_api_aggregates_reasoning_tokens_for_unfiltered_window(async_client, db_setup): + async with SessionLocal() as session: + session.add(_make_account("acc_reports_reasoning", "reports-reasoning@example.com")) + session.add_all( + [ + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-day-1", + requested_at=datetime(2026, 6, 1, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=10, + output_tokens=40, + reasoning_tokens=30, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-missing", + requested_at=datetime(2026, 6, 1, 11, 0), + model="gpt-5.1", + status="success", + input_tokens=20, + output_tokens=20, + reasoning_tokens=None, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-zero", + requested_at=datetime(2026, 6, 1, 12, 0), + model="gpt-5.1", + status="success", + input_tokens=5, + output_tokens=10, + reasoning_tokens=0, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-day-2", + requested_at=datetime(2026, 6, 2, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=30, + output_tokens=80, + reasoning_tokens=70, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-only-output-fallback", + requested_at=datetime(2026, 6, 2, 11, 0), + model="gpt-5.1", + status="success", + input_tokens=5, + output_tokens=None, + reasoning_tokens=5, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-all-unknown-day", + requested_at=datetime(2026, 6, 3, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=10, + output_tokens=20, + reasoning_tokens=None, + ), + RequestLog( + account_id="acc_reports_reasoning", + request_id="report-reasoning-outside-window", + requested_at=datetime(2026, 6, 4, 10, 0), + model="gpt-5.1", + status="success", + input_tokens=100, + output_tokens=1000, + reasoning_tokens=900, + ), + ] + ) + await session.commit() + + response = await async_client.get( + "/api/reports", + params={"start_date": "2026-06-01", "end_date": "2026-06-03"}, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["summary"]["totalReasoningTokens"] == 105 + assert payload["summary"]["reasoningUsageKnownRequests"] == 4 + assert payload["summary"]["totalOutputTokens"] == 175 + assert [(row["date"], row["outputTokens"], row["reasoningTokens"]) for row in payload["daily"]] == [ + ("2026-06-01", 70, 30), + ("2026-06-02", 85, 75), + ("2026-06-03", 20, None), + ] + + +async def test_reports_api_reasoning_tokens_honor_filters_without_double_counting_comparison( + async_client, + db_setup, +): + async with SessionLocal() as session: + session.add_all( + [ + _make_account("acc_reports_reasoning_filter", "reports-reasoning-filter@example.com"), + _make_account("acc_reports_reasoning_other", "reports-reasoning-other@example.com"), + ] + ) + session.add_all( + [ + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-previous", + requested_at=datetime(2026, 5, 31, 10, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=50, + output_tokens=None, + reasoning_tokens=70, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-selected", + requested_at=datetime(2026, 6, 1, 10, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=40, + reasoning_tokens=30, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-other-model", + requested_at=datetime(2026, 6, 1, 11, 0), + model="gpt-5.2", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=110, + reasoning_tokens=100, + ), + RequestLog( + account_id="acc_reports_reasoning_other", + request_id="report-reasoning-filter-other-account", + requested_at=datetime(2026, 6, 1, 12, 0), + model="gpt-5.1", + useragent_group="opencode", + status="success", + input_tokens=10, + output_tokens=210, + reasoning_tokens=200, + ), + RequestLog( + account_id="acc_reports_reasoning_filter", + request_id="report-reasoning-filter-other-useragent", + requested_at=datetime(2026, 6, 1, 13, 0), + model="gpt-5.1", + useragent_group="CodexCLI", + status="success", + input_tokens=10, + output_tokens=310, + reasoning_tokens=300, + ), + ] + ) + await session.commit() + + response = await async_client.get( + "/api/reports", + params={ + "start_date": "2026-06-01", + "end_date": "2026-06-01", + "account_id": "acc_reports_reasoning_filter", + "model": "gpt-5.1", + "useragent_group": "opencode", + }, + ) + + assert response.status_code == 200 + payload = response.json() + assert payload["summary"]["totalReasoningTokens"] == 30 + assert payload["summary"]["reasoningUsageKnownRequests"] == 1 + assert payload["summary"]["totalOutputTokens"] == 40 + assert payload["daily"][0]["reasoningTokens"] == 30 + assert payload["comparison"]["previous"]["totalTokens"] == 120 + + async def test_reports_api_returns_distinct_nonblank_conversation_counts(async_client, db_setup): async with SessionLocal() as session: session.add(_make_account("acc_reports_conversations", "reports-conversations@example.com")) @@ -306,6 +495,7 @@ async def test_reports_api_includes_preserved_deleted_account_history(async_clie "requests": 1, "inputTokens": 13, "outputTokens": 7, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -450,6 +640,7 @@ async def test_reports_api_interprets_dates_in_requested_timezone(async_client, "requests": 2, "inputTokens": 5, "outputTokens": 2, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -701,6 +892,7 @@ async def test_reports_api_default_range_uses_last_seven_calendar_days_in_reques "requests": 1, "inputTokens": 5, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -716,6 +908,7 @@ async def test_reports_api_default_range_uses_last_seven_calendar_days_in_reques "requests": 1, "inputTokens": 5, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -806,6 +999,7 @@ async def test_reports_api_uses_dst_aware_boundaries_for_requested_timezone(asyn "requests": 2, "inputTokens": 5, "outputTokens": 2, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1578,6 +1772,7 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1593,6 +1788,7 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, @@ -1608,6 +1804,7 @@ async def test_reports_api_summary_uses_sql_range_totals_not_rounded_daily_rows( "requests": 1, "inputTokens": 1, "outputTokens": 1, + "reasoningTokens": None, "medianTtftMs": 0.0, "medianTps": 0.0, "medianQueueMs": 0.0, diff --git a/tests/unit/test_reports_service.py b/tests/unit/test_reports_service.py index fdbae625d5..ebfe92b596 100644 --- a/tests/unit/test_reports_service.py +++ b/tests/unit/test_reports_service.py @@ -53,6 +53,8 @@ async def test_get_reports_averages_use_inclusive_local_calendar_days( total_cost_usd=60.0, total_input_tokens=0, total_output_tokens=0, + total_reasoning_tokens=0, + reasoning_usage_known_requests=0, total_cached_tokens=0, total_requests=30, conversation_count=0, @@ -144,6 +146,8 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( total_cost_usd=1.2, total_input_tokens=12, total_output_tokens=6, + total_reasoning_tokens=4, + reasoning_usage_known_requests=2, total_cached_tokens=2, total_requests=2, conversation_count=1, @@ -155,6 +159,8 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( total_cost_usd=0.4, total_input_tokens=4, total_output_tokens=2, + total_reasoning_tokens=2, + reasoning_usage_known_requests=1, total_cached_tokens=0, total_requests=1, conversation_count=0, @@ -172,6 +178,7 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( conversation_count=1, input_tokens=12, output_tokens=6, + reasoning_tokens=None, cached_input_tokens=2, cost_usd=1.2, active_accounts=1, @@ -247,8 +254,12 @@ async def test_get_reports_serializes_conversation_and_breakdown_request_counts( assert result.daily[0].conversations == 1 assert result.daily[0].median_tps == 78.9 assert result.daily[0].median_queue_ms == 45.68 + assert result.daily[0].reasoning_tokens is None assert result.by_model[0].model == "gpt-5.1" assert result.summary.total_conversations == 1 + assert result.summary.total_reasoning_tokens == 4 + assert result.summary.reasoning_usage_known_requests == 2 + assert result.comparison.previous.total_tokens == 6 assert result.by_model[0].requests == 2 assert result.by_useragent[0].useragent == "opencode" assert result.by_useragent[0].requests == 2 From 1add1041b61e7b20ea104a91dd6331f03db904c7 Mon Sep 17 00:00:00 2001 From: zenasharp Date: Wed, 19 Aug 2026 16:34:16 +0700 Subject: [PATCH 079/117] fix(models): raise GPT-5.6 bootstrap max_context_window to 872k (#1813) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified: upstream openai/codex commit 2eee483e ships context_window=272000/max_context_window=872000 for all GPT-5.6 slugs with the clamp in model_info.rs; negative control (rust-v0.148.0-alpha.21 still 272000) holds. Local codex review + adversarial triage both flag one P2: the openspec deltas are NOT order-independent as claimed — fix-gpt56-context-window must be archived BEFORE raise-gpt56-max-context-window; handling the archive in a follow-up. --- .all-contributorsrc | 11 +++ README.md | 3 + app/core/openai/model_registry.py | 14 +++- docs/client-setup.md | 27 ++++++- .../proposal.md | 54 +++++++++++++ .../specs/model-catalog-compat/spec.md | 77 +++++++++++++++++++ .../raise-gpt56-max-context-window/tasks.md | 49 ++++++++++++ tests/integration/test_v1_models.py | 20 ++++- tests/unit/test_model_registry.py | 13 +++- 9 files changed, 262 insertions(+), 6 deletions(-) create mode 100644 openspec/changes/raise-gpt56-max-context-window/proposal.md create mode 100644 openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md create mode 100644 openspec/changes/raise-gpt56-max-context-window/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index e03359b272..02f9cd703a 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1291,6 +1291,17 @@ "code", "test" ] + }, + { + "login": "zenasharp", + "name": "zenasharp", + "avatar_url": "https://avatars.githubusercontent.com/u/170236008?v=4", + "profile": "https://github.com/zenasharp", + "contributions": [ + "code", + "test", + "doc" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 5c3b2845a9..4c92ae4da5 100644 --- a/README.md +++ b/README.md @@ -293,6 +293,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e Evan
    Evan
    💻 Chao Xu
    Chao Xu
    💻 ⚠️ + + zenasharp
    zenasharp
    💻 ⚠️ 📖 + diff --git a/app/core/openai/model_registry.py b/app/core/openai/model_registry.py index 2cb5728731..3a0c3d91f7 100644 --- a/app/core/openai/model_registry.py +++ b/app/core/openai/model_registry.py @@ -220,7 +220,9 @@ def _gpt56_raw( ) -> dict[str, JsonValue]: """Raw catalog fields for the GPT-5.6 family, mirroring the upstream bundled catalog (codex-rs/models-manager/models.json at rust-v0.145.0) - field-for-field. The ~16.5 KB ``base_instructions`` string and the + field-for-field, with one tracked exception: ``max_context_window``, which + upstream later raised from 272,000 to 872,000 (see the field comment + below). The ~16.5 KB ``base_instructions`` string and the personality-templated ``model_messages`` object are deliberately not bundled; the live upstream registry supplies them on the first refresh. """ @@ -237,6 +239,16 @@ def _gpt56_raw( "use_responses_lite": True, "include_skills_usage_instructions": False, "auto_review_model_override": None, + # Upstream raised only the ceiling: ``max_context_window`` 272000 -> + # 872000 with ``context_window`` unchanged at 272000. ``_bootstrap_model`` + # synthesizes ``max_context_window == context_window``, so the family + # ceiling has to override it here, the same decoupling ``gpt-5.4`` uses. + # Pinned evidence: openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 -- "Raise the GPT-5.6 maximum + # context window" (openai/codex#39102). Not yet in a ``rust-v*`` release + # tag; ``rust-v0.148.0-alpha.21`` still ships 272000. Re-pin to the tag + # once one carries it. + "max_context_window": 872_000, "auto_compact_token_limit": None, "comp_hash": "3000", "reasoning_summary_format": "experimental", diff --git a/docs/client-setup.md b/docs/client-setup.md index eb479336df..2ee547810e 100644 --- a/docs/client-setup.md +++ b/docs/client-setup.md @@ -4,7 +4,7 @@ Point any OpenAI-compatible client at codex-lb. If [API key auth](api-keys.md) i Model availability is discovered from the upstream Codex model catalog and can vary by account plan, workspace, rollout, and upstream deprecation state. Prefer the live `GET /v1/models` or `GET /backend-api/codex/models` response over a copied static table when configuring clients or API-key model allowlists. -The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all 272k context. `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. +The examples below use the current frontier lineup: **`gpt-5.6-sol`** (strongest), **`gpt-5.6-terra`** (balanced), and **`gpt-5.6-luna`** (fast) — all with a 272k default input budget and an 872k upstream maximum ([opt-in, Codex CLI only](#opting-into-the-872k-context-window)). `gpt-5.5` and `gpt-5.4` are still served for older pinned clients; retired slugs such as `gpt-5.3-codex`, `gpt-5.3-codex-spark`, and `gpt-5.1-codex-mini` were dropped from the upstream bundled catalog and should no longer be used in new configs. | Client | Endpoint | Config | |--------|----------|--------| @@ -31,6 +31,31 @@ supports_websockets = true requires_openai_auth = true # required for codex app ``` +### Opting into the 872k context window + +GPT-5.6 ships a 272,000-token default input budget with an 872,000-token +maximum. codex-lb advertises both — `context_window` and `max_context_window` +on `GET /backend-api/codex/models` — and the Codex CLI stays on the default +until you raise it in `~/.codex/config.toml` (top level, before any +`[section]` header): + +```toml +model_context_window = 872000 +``` + +- Values above `max_context_window` are clamped to it: `model_context_window = + 1000000` resolves to 872,000 and does not unlock a 1M window. +- Leave `model_auto_compact_token_limit` unset. Codex auto-compacts at 90% of + the resolved window — 784,800 tokens here — and clamps any larger configured + value down to that, so setting `900000` is a no-op. Set it only to compact + *earlier*. +- Cost: input beyond the 272,000-token threshold is metered at the upstream + long-context rate. That threshold is why 272,000 stays the default. + +These keys are Codex-CLI-only. The OpenCode / OpenClaw / SDK examples below +stay at 272000 because `/v1/models` reports the default input budget, not the +ceiling. + ### Daybreak Blue profile (Trusted Access) Use a separate provider for authorized defensive cybersecurity work. The diff --git a/openspec/changes/raise-gpt56-max-context-window/proposal.md b/openspec/changes/raise-gpt56-max-context-window/proposal.md new file mode 100644 index 0000000000..ba518e90ea --- /dev/null +++ b/openspec/changes/raise-gpt56-max-context-window/proposal.md @@ -0,0 +1,54 @@ +## Why + +Upstream raised the GPT-5.6 maximum context window from 272,000 to 872,000 +tokens while leaving the default input budget at 272,000 +(`codex-rs/models-manager/models.json`, openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14`, "Raise the GPT-5.6 maximum context +window", openai/codex#39102). codex-lb's bootstrap catalog synthesizes +`max_context_window` as a copy of `context_window` +(`app/core/openai/model_registry.py`), so it advertises a 272,000 ceiling for +Sol, Terra, and Luna. A Codex client pointed at codex-lb before the first live +registry refresh therefore has its `model_context_window` opt-in clamped to +272,000 and cannot reach the window upstream actually serves. + +## What Changes + +- Decouple `max_context_window` from `context_window` for the GPT-5.6 + bootstrap family: `max_context_window` becomes 872,000; `context_window` + stays 272,000. +- Keep the GPT-5.6 base provenance pinned at Codex `rust-v0.145.0`, with a + single tracked exception for `max_context_window`, pinned to the upstream + commit that raised it (no `rust-v*` release tag carries it yet as of + `rust-v0.148.0-alpha.21`). +- Document the Codex CLI opt-in, including the clamp semantics that make + `model_context_window = 1000000` resolve to 872,000 and + `model_auto_compact_token_limit = 900000` a no-op (Codex clamps the + auto-compact limit to 90% of the resolved window). + +## Non-goals + +- `context_window` stays 272,000. It is the tuned default input budget and the + upstream long-context pricing threshold. +- The other post-`rust-v0.145.0` upstream deltas to these entries + (`include_apps_usage_instructions`, `include_plugin_usage_instructions`, the + `base_instructions` relocation to `prompt.md`, `supports_parallel_tool_calls` + now serde-defaulted) are out of scope and need their own compatibility + review. +- No clamp or override logic changes; `/v1` input budget fields keep reporting + the default input budget (see PR #1808 for override plumbing work). + +## Impact + +- No schema, route, or database migration change. +- Before a live registry refresh, `GET /backend-api/codex/models` changes the + GPT-5.6 advertised `max_context_window` from 272,000 to 872,000 tokens; + `context_window` is unchanged. +- `GET /v1/models` is unchanged: it reports the default input budget and does + not promote `raw["max_context_window"]`. +- Operator `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entries and persisted + registry snapshots continue to take precedence over bootstrap values. +- The delta restates the `model-catalog-compat` GPT-5.6 requirement in full, + so it is order-insensitive with respect to the still-unarchived + `fix-gpt56-context-window` delta (issue #1714): applied before or after it, + the merged requirement text is identical. +- Revert cost is one line if upstream reverts before tagging a release. diff --git a/openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..fa31dc46f7 --- /dev/null +++ b/openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md @@ -0,0 +1,77 @@ +## MODIFIED Requirements + +### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog + +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves, with one tracked +exception: `max_context_window`, which upstream raised from `272000` to +`872000` in openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14` (openai/codex#39102) and which no +`rust-v*` release tag carries as of `rust-v0.148.0-alpha.21`. In particular +each entry MUST carry: `context_window` of `272000` and `max_context_window` +of `872000`; `minimal_client_version` `"0.144.0"`; `tool_mode` +`"code_mode_only"`; `use_responses_lite` `true`; `apply_patch_tool_type` +`"freeform"`; `web_search_tool_type` `"text_and_image"`; +`supports_image_detail_original` `true`; `truncation_policy` `{ "mode": +"tokens", "limit": 10000 }`; `comp_hash` `"3000"`; `reasoning_summary_format` +`"experimental"`; `default_reasoning_summary` `"none"`; +`include_skills_usage_instructions` `false`; `experimental_supported_tools` +`[]` (a field the Codex client's deserializer requires); `supports_search_tool` +`true`; `additional_speed_tiers` `["fast"]`; the `priority`/`Fast` service tier +entry; `shell_type` `"shell_command"`; `prefer_websockets` `true`; and the +21-plan `available_in_plans` list upstream advertises (including `edu_plus`, +`edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST +be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and +reasoning-level descriptions MUST be the verbatim upstream strings. + +`context_window` is the default input budget and `max_context_window` is the +ceiling a client may opt into; the two MUST NOT be collapsed into one value +for these entries. + +The ~16.5 KB upstream `base_instructions` prompt and the personality-templated +`model_messages` object are deliberately NOT bundled in the bootstrap catalog; +the first successful live registry refresh supplies them. This is the only +sanctioned divergence from the upstream GPT-5.6 entries beyond the +`max_context_window` exception above. + +#### Scenario: GPT-5.6 bootstrap entries advertise the raised upstream ceiling + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` +- **AND** each reports `max_context_window=872000` + +#### Scenario: OpenAI-compatible metadata keeps the default input budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /v1/models` +- **THEN** each GPT-5.6 entry reports `context_window=272000` and + `input_context_window=272000` +- **AND** the raised Codex-native ceiling is not promoted into the + OpenAI-compatible input budget fields + +#### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` +- **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna + +#### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` +- **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` +- **AND** only `gpt-5.6-sol` carries a non-null `availability_nux` message diff --git a/openspec/changes/raise-gpt56-max-context-window/tasks.md b/openspec/changes/raise-gpt56-max-context-window/tasks.md new file mode 100644 index 0000000000..9c9ff4efb5 --- /dev/null +++ b/openspec/changes/raise-gpt56-max-context-window/tasks.md @@ -0,0 +1,49 @@ +## 1. Bootstrap catalog + +- [x] 1.1 Add `max_context_window: 872_000` to `_gpt56_raw()` so it overrides + the `_bootstrap_model` synthesis for all three GPT-5.6 slugs at once. +- [x] 1.2 Leave `context_window` at 272,000 for Sol, Terra, and Luna. +- [x] 1.3 Amend the `_gpt56_raw()` docstring to record `max_context_window` as + the one tracked divergence from the `rust-v0.145.0` base pin, citing the + upstream commit. + +## 2. Regression coverage + +- [x] 2.1 Assert `max_context_window == 872_000` and `context_window == + 272_000` for every GPT-5.6 entry in the shared unit-test loop. +- [x] 2.2 Assert the same pair through `GET /backend-api/codex/models` in the + shared integration-test loop. +- [x] 2.3 Assert `max_context_window > context_window` in both loops so a + future re-unification of the two fields fails loudly. +- [x] 2.4 Assert `GET /v1/models` still reports 272,000 for the GPT-5.6 input + budget fields on the bootstrap path. +- [x] 2.5 Re-pin both evidence comments to `rust-v0.145.0` plus the upstream + commit for `max_context_window`. + +## 3. Documentation + +- [x] 3.1 Distinguish the 272k default budget from the 872k maximum in the + client-setup model lineup summary. +- [x] 3.2 Document the Codex CLI opt-in with correct clamp semantics: values + above `max_context_window` are clamped, and the auto-compact limit + resolves to 90% of the window, so larger values are no-ops. +- [x] 3.3 Leave the OpenCode and OpenClaw examples at 272,000, matching what + `/v1/models` advertises. + +## 4. Specification + +- [x] 4.1 Add a `model-catalog-compat` delta requiring `context_window` + 272,000 and `max_context_window` 872,000 for the GPT-5.6 bootstrap + entries. +- [x] 4.2 Restate the requirement and surviving scenarios in full so the delta + is order-insensitive against the unarchived `fix-gpt56-context-window` + delta. +- [x] 4.3 Carry GIVEN clauses on context-budget scenarios excluding refreshed + snapshots, persisted snapshots, and operator context-window overrides. + +## 5. Validation + +- [x] 5.1 `openspec validate raise-gpt56-max-context-window --strict` +- [x] 5.2 `openspec validate --specs` +- [x] 5.3 Focused unit + integration model-catalog tests, ruff, and + `mkdocs build --strict`. diff --git a/tests/integration/test_v1_models.py b/tests/integration/test_v1_models.py index be1d673a62..f04cdf512b 100644 --- a/tests/integration/test_v1_models.py +++ b/tests/integration/test_v1_models.py @@ -261,6 +261,18 @@ async def test_v1_models_uses_bootstrap_models_when_registry_not_populated(async assert ids == BOOTSTRAP_MODEL_SLUGS assert "gpt-5.5-pro" not in ids + # The raised GPT-5.6 ceiling is a Codex-native field. /v1 input budgets + # stay on ``context_window`` so OpenAI-compatible clients keep packing to + # the 272k default instead of the 872k ``max_context_window`` ceiling. + entries = {item["id"]: item for item in payload["data"]} + for slug in ("gpt-5.6-sol", "gpt-5.6-terra", "gpt-5.6-luna"): + entry = entries[slug] + assert entry["metadata"]["context_window"] == 272_000 + assert entry["metadata"]["input_context_window"] == 272_000 + assert entry["capabilities"]["context_length"] == 272_000 + assert entry["context_length"] == 272_000 + assert entry["contextLength"] == 272_000 + @pytest.mark.asyncio async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_client): @@ -315,7 +327,10 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien } # Reproducible upstream catalog evidence: - # codex-rs/models-manager/models.json at rust-v0.145.0. + # codex-rs/models-manager/models.json at rust-v0.145.0, except + # ``max_context_window``: raised to 872000 in openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 (openai/codex#39102), which no + # rust-v* release tag carries yet. for gpt56 in (sol, terra, luna): assert gpt56["minimal_client_version"] == "0.144.0" assert gpt56["context_window"] == 272_000 @@ -328,7 +343,8 @@ async def test_backend_codex_models_uses_bootstrap_upstream_metadata(async_clien assert gpt56["reasoning_summary_format"] == "experimental" assert gpt56["comp_hash"] == "3000" assert gpt56["experimental_supported_tools"] == [] - assert gpt56["max_context_window"] == 272_000 + assert gpt56["max_context_window"] == 872_000 + assert gpt56["max_context_window"] > gpt56["context_window"] assert gpt56["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] diff --git a/tests/unit/test_model_registry.py b/tests/unit/test_model_registry.py index fa3ed54477..2d15149a43 100644 --- a/tests/unit/test_model_registry.py +++ b/tests/unit/test_model_registry.py @@ -273,7 +273,10 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert [level.effort for level in luna.supported_reasoning_levels] == ["low", "medium", "high", "xhigh", "max"] # Reproducible upstream catalog evidence: - # codex-rs/models-manager/models.json at rust-v0.145.0. + # codex-rs/models-manager/models.json at rust-v0.145.0, except + # ``max_context_window``: raised to 872000 in openai/codex commit + # 2eee483e49f88b868f67364134a658b3298e6c14 (openai/codex#39102), which no + # rust-v* release tag carries yet. for gpt56 in (sol, terra, luna): assert gpt56.minimal_client_version == "0.144.0" assert gpt56.context_window == 272_000 @@ -289,7 +292,13 @@ def test_bootstrap_models_include_representative_upstream_metadata(): assert gpt56.raw["include_skills_usage_instructions"] is False assert gpt56.raw["experimental_supported_tools"] == [] assert gpt56.raw["supports_search_tool"] is True - assert gpt56.raw["max_context_window"] == 272_000 + # The upstream ceiling is decoupled from the default input budget, so + # the ``_bootstrap_model`` synthesis (max == context_window) must not + # win for these entries. + max_context_window = gpt56.raw["max_context_window"] + assert isinstance(max_context_window, int) + assert max_context_window == 872_000 + assert max_context_window > gpt56.context_window assert gpt56.raw["service_tiers"] == [ {"id": "priority", "name": "Fast", "description": "1.5x speed, increased usage"} ] From 812265d11c0faa470da1db1b689442afa2b93869 Mon Sep 17 00:00:00 2001 From: yshishenya Date: Wed, 19 Aug 2026 12:42:58 +0300 Subject: [PATCH 080/117] fix(proxy): drop malformed compact item ids (#1815) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triage verification: 20-agent adversarial review (merge_ready, 2/2 verifiers agree) + local codex review escalated 2 P1s + 1 P2, all three refuted on adjudication with code evidence — (1) the compaction-id-fidelity design doc's ciphertext binding is scoped to genuine cmp_ ids, which this PR preserves byte-for-byte, and the doc itself endorses no-ID emission for invalid ids; preserving a msg_ id is a guaranteed upstream 400 today, so id-drop is strictly better-or-equal in every branch; (2) the stale-added-before-unindexed-done selection is mechanically accurate but requires an internally inconsistent stream no observed upstream emits, and main behaves strictly worse under it (optional hardening: prefer unindexed done records — not blocking); (3) pending-delta vs main-spec divergence is the designed pre-archive steady state. CI fully green on head 0d432e57. --- app/core/clients/proxy.py | 40 ++++++++++----- app/core/openai/models.py | 9 ++++ app/modules/proxy/api.py | 5 +- .../proposal.md | 3 ++ .../specs/responses-api-compat/spec.md | 18 +++++-- .../tasks.md | 1 + tests/integration/test_proxy_compact.py | 14 ++--- tests/unit/test_codex_upstream_paths.py | 51 ++++++++++++++++++- .../unit/test_proxy_api_responses_contract.py | 20 ++++++++ 9 files changed, 136 insertions(+), 25 deletions(-) diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index d0e849df8a..9b602ba5fd 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -58,7 +58,7 @@ ) from app.core.openai.exceptions import ClientPayloadError from app.core.openai.model_registry import get_model_registry -from app.core.openai.models import CompactResponsePayload, OpenAIError +from app.core.openai.models import CompactResponsePayload, OpenAIError, normalize_compaction_item_id from app.core.openai.parsing import ( classify_event_type, parse_compact_response_payload, @@ -1245,6 +1245,7 @@ async def _compact_response_payload_from_sse( ) -> JsonValue: last_payload: dict[str, JsonValue] | None = None output_items: dict[int, dict[str, JsonValue]] = {} + unindexed_output_items: list[dict[str, JsonValue]] = [] async for event_block in _iter_sse_events(resp, idle_timeout_seconds, max_event_bytes): payload = parse_sse_data_json(event_block) if payload is None: @@ -1254,15 +1255,26 @@ async def _compact_response_payload_from_sse( if event_type in {"response.output_item.added", "response.output_item.done"}: output_index = payload.get("output_index") item = payload.get("item") - if isinstance(output_index, int) and isinstance(item, dict): + if not isinstance(item, dict): + continue + if isinstance(output_index, int): output_items[output_index] = dict(item) + elif event_type == "response.output_item.done": + # Some compatible upstream responses omit output_index on the + # terminal item even though response.completed has no output. + unindexed_output_items.append(dict(item)) if event_type == "response.completed": response = payload.get("response") if isinstance(response, dict): existing_output = response.get("output") - if output_items and not (isinstance(existing_output, list) and existing_output): + if (output_items or unindexed_output_items) and not ( + isinstance(existing_output, list) and existing_output + ): merged_response = dict(response) - merged_response["output"] = [item for _, item in sorted(output_items.items())] + merged_response["output"] = [ + *[item for _, item in sorted(output_items.items())], + *unindexed_output_items, + ] return merged_response return response raise ValueError("response.completed event missing response object") @@ -1355,10 +1367,12 @@ def _compact_output_item_from_message(item: Mapping[str, JsonValue]) -> dict[str "type": "compaction", "encrypted_content": text, } - for key in ("id", "status"): - value = item.get(key) - if isinstance(value, str) and value.strip(): - normalized[key] = value + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: + normalized["id"] = item_id + status = item.get("status") + if isinstance(status, str) and status.strip(): + normalized["status"] = status return normalized @@ -1392,10 +1406,12 @@ def _normalize_compact_output_item(item: Mapping[str, JsonValue]) -> dict[str, J "type": "compaction", "encrypted_content": encrypted_content, } - for key in ("id", "status"): - value = item.get(key) - if isinstance(value, str) and value.strip(): - normalized[key] = value + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: + normalized["id"] = item_id + status = item.get("status") + if isinstance(status, str) and status.strip(): + normalized["status"] = status return normalized diff --git a/app/core/openai/models.py b/app/core/openai/models.py index 4bf025cc25..b6d73b9c66 100644 --- a/app/core/openai/models.py +++ b/app/core/openai/models.py @@ -142,5 +142,14 @@ def _normalize_usage(cls, value: ModelLikeInput | None) -> ResponseUsage | None: return _normalize_model_value(ResponseUsage, value) +def normalize_compaction_item_id(item_id: object) -> str | None: + """Return a valid compaction ID without changing opaque upstream identity.""" + if not isinstance(item_id, str): + return None + if item_id.startswith("cmp_"): + return item_id + return None + + OpenAIResponseResult: TypeAlias = OpenAIResponsePayload | OpenAIErrorEnvelope CompactResponseResult: TypeAlias = CompactResponsePayload | OpenAIErrorEnvelope diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index b0a48f1502..2056b4dfeb 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -123,6 +123,7 @@ OpenAIError, OpenAIResponsePayload, OpenAIResponseResult, + normalize_compaction_item_id, ) from app.core.openai.models import ( OpenAIErrorEnvelope as OpenAIErrorEnvelopeModel, @@ -6201,8 +6202,8 @@ def _normalize_compaction_output_item(item: Mapping[str, JsonValue]) -> dict[str "type": "compaction", "encrypted_content": encrypted_content, } - item_id = item.get("id") - if isinstance(item_id, str) and item_id.strip(): + item_id = normalize_compaction_item_id(item.get("id")) + if item_id is not None: normalized["id"] = item_id status = item.get("status") if isinstance(status, str) and status.strip(): diff --git a/openspec/changes/document-compact-trigger-proxy-contract/proposal.md b/openspec/changes/document-compact-trigger-proxy-contract/proposal.md index 6f8390f8e7..665e9a6e2f 100644 --- a/openspec/changes/document-compact-trigger-proxy-contract/proposal.md +++ b/openspec/changes/document-compact-trigger-proxy-contract/proposal.md @@ -18,6 +18,9 @@ implementation detail that contradicts the existing context notes. - Document that Codex compact transport uses streamed `POST /backend-api/codex/responses` with `stream=true` and `store=false`, and reconstructs the compact response from the terminal SSE lifecycle. +- Document that legacy message-shaped compact output is converted to a + `compaction` item while only valid opaque `cmp_` IDs are preserved; malformed + IDs are omitted rather than rewritten. - Document that the standalone Codex `/backend-api/codex/responses/compact` route remains a compatibility endpoint, while `/v1/responses/compact` preserves duplicate-trigger normalization for existing OpenAI-compatible diff --git a/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md b/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md index b71c3ec554..a2ed56a334 100644 --- a/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md +++ b/openspec/changes/document-compact-trigger-proxy-contract/specs/responses-api-compat/spec.md @@ -4,11 +4,11 @@ When `POST /backend-api/codex/responses` receives a request whose top-level `input` array contains exactly one `{"type":"compaction_trigger"}` item as its final element, the proxy SHALL remove that trigger before calling upstream compaction handling and SHALL emit a raw SSE stream that contains exactly one compaction output item. The internal compact request built for that flow MUST contain exactly one terminal `compaction_trigger` item on the compact wire, and the proxy MUST reject duplicate or non-terminal top-level `compaction_trigger` placement locally with HTTP 400 `invalid_request_error` before any upstream compact handling. -The stream MUST emit `response.created`, `response.output_item.added`, `response.output_item.done`, and `response.completed` in that order with monotonically increasing sequence numbers. The added event MUST expose the selected compaction item as in progress. The done event and terminal completed response MUST carry the same terminal `compaction` item. When the selected encrypted upstream compaction item carries a non-empty `id` or `status`, the synthetic stream MUST preserve those values with its `encrypted_content`; it MUST NOT generate a replacement item ID. +The stream MUST emit `response.created`, `response.output_item.added`, `response.output_item.done`, and `response.completed` in that order with monotonically increasing sequence numbers. The added event MUST expose the selected compaction item as in progress. The done event and terminal completed response MUST carry the same terminal `compaction` item. When the selected encrypted upstream compaction item carries a valid `cmp_` ID or status, the synthetic stream MUST preserve those values with its `encrypted_content`; it MUST NOT generate or rewrite a replacement item ID. A malformed, empty, or non-`cmp_` ID MUST be omitted while the opaque encrypted content remains unchanged. Codex compact flows SHALL send the upstream compact request to `POST /backend-api/codex/responses` with `stream=true` and `store=false`, accept the upstream SSE response, and reconstruct one normalized compact response item from the terminal response lifecycle; they MUST NOT require the legacy `/backend-api/codex/responses/compact` upstream route to be available. -For Codex-affinity standalone compact requests, `POST /backend-api/codex/responses/compact` SHALL remain available as a compatibility endpoint with its subscription-backed compact routing contract, and SHALL normalize an upstream remote-compaction-v2 response that includes historical message output plus a compaction summary into the single compact output item required by Codex clients. A non-empty upstream compaction item `id` or `status` MUST be preserved in that normalized output item. +For Codex-affinity standalone compact requests, `POST /backend-api/codex/responses/compact` SHALL remain available as a compatibility endpoint with its subscription-backed compact routing contract, and SHALL normalize an upstream remote-compaction-v2 response that includes historical message output plus a compaction summary into the single compact output item required by Codex clients. A valid upstream `cmp_` compaction item `id` and any non-empty `status` MUST be preserved in that normalized output item. An empty, non-string, or non-`cmp_` ID MUST be omitted rather than rewritten; encrypted content MUST remain unchanged. OpenAI-style `/v1/responses/compact` is otherwise unchanged by this requirement; when it receives duplicate top-level `compaction_trigger` items, codex-lb preserves the existing compatibility behavior and the forwarded compact input contains one terminal trigger. @@ -54,6 +54,18 @@ OpenAI-style `/v1/responses/compact` is otherwise unchanged by this requirement; - **AND** it does not require the legacy `/backend-api/codex/responses/compact` upstream route to be available +#### Scenario: Legacy message-shaped compact output does not get a rewritten item ID + +- **WHEN** the upstream compact response exposes the encrypted compact payload + as a legacy `message` item with a non-empty ID that does not begin with `cmp_` +- **THEN** the proxy converts that item to `type="compaction"` and omits the + malformed ID +- **AND** the proxy preserves the encrypted content unchanged +- **AND** an existing ID that begins with `cmp_` is preserved byte-for-byte +- **AND** the proxy does not synthesize a `cmp_msg_...` ID +- **AND** ordinary message items outside the compact-output conversion remain + unchanged + #### Scenario: Standalone Codex compact remains a compatibility endpoint - **WHEN** a client calls `POST /backend-api/codex/responses/compact` @@ -66,7 +78,7 @@ OpenAI-style `/v1/responses/compact` is otherwise unchanged by this requirement; - **WHEN** a Codex-affinity `POST /backend-api/codex/responses/compact` request receives upstream output that contains historical message items and one compaction summary item - **THEN** the JSON response body contains exactly one `output` item for that compaction summary -- **AND** the normalized item preserves the compaction summary's non-empty upstream ID and status +- **AND** the normalized item preserves the compaction summary's valid `cmp_`-prefixed upstream ID and status - **AND** it does not expose historical message items as standalone compact output #### Scenario: OpenAI-compatible compact normalizes duplicate triggers diff --git a/openspec/changes/document-compact-trigger-proxy-contract/tasks.md b/openspec/changes/document-compact-trigger-proxy-contract/tasks.md index bf16a98182..65205563d9 100644 --- a/openspec/changes/document-compact-trigger-proxy-contract/tasks.md +++ b/openspec/changes/document-compact-trigger-proxy-contract/tasks.md @@ -7,6 +7,7 @@ - [x] 1.3 Record the streamed `/backend-api/codex/responses` compact transport, the standalone Codex compatibility endpoint, and the `/v1` normalization asymmetry. +- [x] 1.4 Record the legacy message-shaped compact ID filtering contract. ## 2. Validate the change diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 3143e46e38..6bb285ca26 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -794,7 +794,6 @@ async def lease_session(session_override=None): assert body["id"] == "resp_compact_summary_1" assert body["output"] == [ { - "id": "msg_compact_summary_1", "type": "compaction", "status": "completed", "encrypted_content": "enc_compact_summary_1", @@ -1629,15 +1628,18 @@ async def test_proxy_compact_output_round_trips_into_followup_responses_without_ "object": "response.compaction", "output": [ { - "type": "message", + "type": "compaction", "id": "msg_compact_round_trip", - "role": "assistant", - "content": [{"type": "output_text", "text": "preserve me exactly"}], + "encrypted_content": "preserve me exactly", }, {"type": "reasoning", "encrypted_content": "enc_round_trip_state"}, ], "retained_items": [{"type": "item_reference", "id": "msg_original_round_trip"}], } + expected_compact_window = { + **compact_window, + "output": [{"type": "compaction", "encrypted_content": "preserve me exactly"}], + } seen_inputs: list[object] = [] async def fake_compact(payload, headers, access_token, account_id): @@ -1653,7 +1655,7 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, compact_payload = {"model": "gpt-5.1", "instructions": "compact", "input": []} compact_response = await async_client.post("/backend-api/codex/responses/compact", json=compact_payload) assert compact_response.status_code == 200 - assert compact_response.json() == compact_window + assert compact_response.json() == expected_compact_window stream_payload = { "model": "gpt-5.1", @@ -1664,7 +1666,7 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, response = await async_client.post("/backend-api/codex/responses", json=stream_payload) assert response.status_code == 200 - assert seen_inputs == [compact_window["output"]] + assert seen_inputs == [expected_compact_window["output"]] _NEUTRAL_FULL_RESEND_INPUT: list[dict[str, object]] = [ diff --git a/tests/unit/test_codex_upstream_paths.py b/tests/unit/test_codex_upstream_paths.py index 5b2ad5b20a..7c1ca372ec 100644 --- a/tests/unit/test_codex_upstream_paths.py +++ b/tests/unit/test_codex_upstream_paths.py @@ -96,12 +96,32 @@ async def iter_chunked(self, size: int): ) +class _CompactStreamWithoutOutputIndexContent: + async def iter_chunked(self, size: int): + del size + yield ( + b'data: {"type":"response.output_item.done",' + b'"item":{"id":"msg_compact_without_index","type":"message",' + b'"status":"completed","content":[{"type":"output_text",' + b'"text":"enc_compact_without_index"}]}}\n\n' + b'data: {"type":"response.completed","response":' + b'{"object":"response","id":"resp_compact_without_index",' + b'"status":"completed","output":[]}}\n\n' + ) + + class _CompactStreamResponse: status_code = 200 headers: dict[str, str] = {} content = _CompactStreamContent() +class _CompactStreamWithoutOutputIndexResponse: + status_code = 200 + headers: dict[str, str] = {} + content = _CompactStreamWithoutOutputIndexContent() + + class _BufferedCompactStreamResponse: status = 200 status_code = 200 @@ -425,7 +445,7 @@ async def test_compact_responses_uses_codex_client_when_route_is_resolved(route: assert response.id == "resp_compact_1" assert response.model_extra is not None assert response.model_extra["output"] == [ - {"id": "msg_compact_1", "type": "compaction", "status": "completed", "encrypted_content": "enc_compact_1"} + {"type": "compaction", "status": "completed", "encrypted_content": "enc_compact_1"} ] assert client.calls[0]["url"].endswith("/backend-api/codex/responses") assert client.calls[0]["route"] is route @@ -436,6 +456,33 @@ async def test_compact_responses_uses_codex_client_when_route_is_resolved(route: assert trace.endpoint_id == "ep_1" +@pytest.mark.asyncio +async def test_compact_responses_recovers_terminal_item_without_output_index( + route: ResolvedUpstreamRoute, +) -> None: + client = _RouteMetadataCodexClient(_CompactStreamWithoutOutputIndexResponse()) + payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") + + response = await compact_responses( + payload, + {"user-agent": "codex"}, + "access", + "chatgpt_account", + session=cast(Any, object()), + route=route, + codex_client=cast(Any, client), + ) + + assert response.model_extra is not None + assert response.model_extra["output"] == [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "enc_compact_without_index", + } + ] + + @pytest.mark.asyncio async def test_compact_responses_routed_buffered_sse_keeps_compact_protocol(route: ResolvedUpstreamRoute) -> None: client = _RouteMetadataCodexClient(_BufferedCompactStreamResponse()) @@ -504,7 +551,7 @@ async def test_compact_responses_message_fallback_selects_last_message( assert response.id == "resp_compact_messages" assert response.model_extra is not None assert response.model_extra["output"] == [ - {"id": "msg_summary", "type": "compaction", "status": "completed", "encrypted_content": "enc_summary"} + {"type": "compaction", "status": "completed", "encrypted_content": "enc_summary"} ] diff --git a/tests/unit/test_proxy_api_responses_contract.py b/tests/unit/test_proxy_api_responses_contract.py index 7ba651d6f5..9ba8426fb8 100644 --- a/tests/unit/test_proxy_api_responses_contract.py +++ b/tests/unit/test_proxy_api_responses_contract.py @@ -423,6 +423,26 @@ def test_compact_response_output_item_preserves_summary_item_id() -> None: } +def test_compact_response_output_item_drops_invalid_id_prefix() -> None: + payload = CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "output": [ + { + "id": "msg_compact_context", + "type": "compaction", + "encrypted_content": "COMPACT_CONTEXT", + } + ], + } + ) + + assert proxy_api_module._compact_response_output_item(payload) == { + "type": "compaction", + "encrypted_content": "COMPACT_CONTEXT", + } + + def test_compact_response_id_generates_unique_fallback(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(proxy_api_module, "get_request_id", lambda: None) payload = CompactResponsePayload.model_validate({"object": "response.compaction"}) From 3af2dff07e35305b8371acd104cb3d124f0a70e7 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 19 Aug 2026 20:07:00 +0900 Subject: [PATCH 081/117] chore(openspec): archive GPT-5.6 context-window changes in dependency order (#1819) Archive fix-gpt56-context-window before raise-gpt56-max-context-window per the ordering constraint both #1813 review passes flagged: the deltas restate the same requirement, so archiving the raise first would have regressed max_context_window to 272000 against the shipped code. The raise delta needed a refresh before archiving (its MODIFIED block was authored while fix-gpt56-context-window was still unarchived): the retained 'corrected upstream context budget' scenario is restored as a budget-only assertion so the archiver does not treat the ceiling scenario as silently dropping it. Final spec validates strict (57/57) and carries context_window=272000 / max_context_window=872000. Co-authored-by: Claude Fable 5 --- .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 0 .../tasks.md | 0 .../proposal.md | 0 .../specs/model-catalog-compat/spec.md | 9 +++ .../tasks.md | 0 openspec/specs/model-catalog-compat/spec.md | 77 +++++++++++++++---- 7 files changed, 70 insertions(+), 16 deletions(-) rename openspec/changes/{fix-gpt56-context-window => archive/2026-08-19-fix-gpt56-context-window}/proposal.md (100%) rename openspec/changes/{fix-gpt56-context-window => archive/2026-08-19-fix-gpt56-context-window}/specs/model-catalog-compat/spec.md (100%) rename openspec/changes/{fix-gpt56-context-window => archive/2026-08-19-fix-gpt56-context-window}/tasks.md (100%) rename openspec/changes/{raise-gpt56-max-context-window => archive/2026-08-19-raise-gpt56-max-context-window}/proposal.md (100%) rename openspec/changes/{raise-gpt56-max-context-window => archive/2026-08-19-raise-gpt56-max-context-window}/specs/model-catalog-compat/spec.md (91%) rename openspec/changes/{raise-gpt56-max-context-window => archive/2026-08-19-raise-gpt56-max-context-window}/tasks.md (100%) diff --git a/openspec/changes/fix-gpt56-context-window/proposal.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/proposal.md similarity index 100% rename from openspec/changes/fix-gpt56-context-window/proposal.md rename to openspec/changes/archive/2026-08-19-fix-gpt56-context-window/proposal.md diff --git a/openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/specs/model-catalog-compat/spec.md similarity index 100% rename from openspec/changes/fix-gpt56-context-window/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-19-fix-gpt56-context-window/specs/model-catalog-compat/spec.md diff --git a/openspec/changes/fix-gpt56-context-window/tasks.md b/openspec/changes/archive/2026-08-19-fix-gpt56-context-window/tasks.md similarity index 100% rename from openspec/changes/fix-gpt56-context-window/tasks.md rename to openspec/changes/archive/2026-08-19-fix-gpt56-context-window/tasks.md diff --git a/openspec/changes/raise-gpt56-max-context-window/proposal.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/proposal.md similarity index 100% rename from openspec/changes/raise-gpt56-max-context-window/proposal.md rename to openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/proposal.md diff --git a/openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md similarity index 91% rename from openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md rename to openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md index fa31dc46f7..0d460ab1eb 100644 --- a/openspec/changes/raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md +++ b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/specs/model-catalog-compat/spec.md @@ -38,6 +38,15 @@ the first successful live registry refresh supplies them. This is the only sanctioned divergence from the upstream GPT-5.6 entries beyond the `max_context_window` exception above. +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` + #### Scenario: GPT-5.6 bootstrap entries advertise the raised upstream ceiling - **GIVEN** the model registry has no refreshed upstream snapshot diff --git a/openspec/changes/raise-gpt56-max-context-window/tasks.md b/openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/tasks.md similarity index 100% rename from openspec/changes/raise-gpt56-max-context-window/tasks.md rename to openspec/changes/archive/2026-08-19-raise-gpt56-max-context-window/tasks.md diff --git a/openspec/specs/model-catalog-compat/spec.md b/openspec/specs/model-catalog-compat/spec.md index 4c5141e9c7..65417be830 100644 --- a/openspec/specs/model-catalog-compat/spec.md +++ b/openspec/specs/model-catalog-compat/spec.md @@ -169,33 +169,76 @@ When serving `GET /v1/models`, the system SHALL preserve upstream speed-tier met ### Requirement: GPT-5.6 bootstrap metadata matches the upstream bundled catalog -The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`) MUST mirror the upstream bundled catalog (`codex-rs/models-manager/models.json` at codex release rust-v0.144.1) field-for-field for every metadata field codex-lb serves. In particular each -entry MUST carry: `context_window` and `max_context_window` of `372000`; -`minimal_client_version` `"0.144.0"`; `tool_mode` `"code_mode_only"`; -`use_responses_lite` `true`; `apply_patch_tool_type` `"freeform"`; -`web_search_tool_type` `"text_and_image"`; `supports_image_detail_original` -`true`; `truncation_policy` `{"mode": "tokens", "limit": 10000}`; -`comp_hash` `"3000"`; `reasoning_summary_format` `"experimental"`; -`default_reasoning_summary` `"none"`; `include_skills_usage_instructions` -`false`; `experimental_supported_tools` `[]` (a field the Codex client's -deserializer requires); `supports_search_tool` `true`; `additional_speed_tiers` -`["fast"]`; the `priority`/`Fast` service tier entry; `shell_type` -`"shell_command"`; `prefer_websockets` `true`; and the 21-plan -`available_in_plans` list upstream advertises (including `edu_plus`, +The GPT-5.6 bootstrap catalog entries (`gpt-5.6-sol`, `gpt-5.6-terra`, +`gpt-5.6-luna`) MUST mirror the upstream bundled catalog +(`codex-rs/models-manager/models.json` at Codex release `rust-v0.145.0`) +field-for-field for every metadata field codex-lb serves, with one tracked +exception: `max_context_window`, which upstream raised from `272000` to +`872000` in openai/codex commit +`2eee483e49f88b868f67364134a658b3298e6c14` (openai/codex#39102) and which no +`rust-v*` release tag carries as of `rust-v0.148.0-alpha.21`. In particular +each entry MUST carry: `context_window` of `272000` and `max_context_window` +of `872000`; `minimal_client_version` `"0.144.0"`; `tool_mode` +`"code_mode_only"`; `use_responses_lite` `true`; `apply_patch_tool_type` +`"freeform"`; `web_search_tool_type` `"text_and_image"`; +`supports_image_detail_original` `true`; `truncation_policy` `{ "mode": +"tokens", "limit": 10000 }`; `comp_hash` `"3000"`; `reasoning_summary_format` +`"experimental"`; `default_reasoning_summary` `"none"`; +`include_skills_usage_instructions` `false`; `experimental_supported_tools` +`[]` (a field the Codex client's deserializer requires); `supports_search_tool` +`true`; `additional_speed_tiers` `["fast"]`; the `priority`/`Fast` service tier +entry; `shell_type` `"shell_command"`; `prefer_websockets` `true`; and the +21-plan `available_in_plans` list upstream advertises (including `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci`). `multi_agent_version` MUST be `"v2"` for Sol and Terra and `"v1"` for Luna. Sol MUST carry the upstream -`availability_nux` message while Terra and Luna carry `null`. Default -reasoning levels MUST be `low` for Sol and `medium` for Terra and Luna, and +`availability_nux` message while Terra and Luna carry `null`. Default reasoning +levels MUST be `low` for Sol and `medium` for Terra and Luna, and reasoning-level descriptions MUST be the verbatim upstream strings. +`context_window` is the default input budget and `max_context_window` is the +ceiling a client may opt into; the two MUST NOT be collapsed into one value +for these entries. + The ~16.5 KB upstream `base_instructions` prompt and the personality-templated `model_messages` object are deliberately NOT bundled in the bootstrap catalog; the first successful live registry refresh supplies them. This is the only -sanctioned divergence from the upstream GPT-5.6 entries. +sanctioned divergence from the upstream GPT-5.6 entries beyond the +`max_context_window` exception above. + +#### Scenario: GPT-5.6 bootstrap entries retain the corrected upstream context budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` + +#### Scenario: GPT-5.6 bootstrap entries advertise the raised upstream ceiling + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /backend-api/codex/models` +- **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` report + `context_window=272000` +- **AND** each reports `max_context_window=872000` + +#### Scenario: OpenAI-compatible metadata keeps the default input budget + +- **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded +- **AND** no `CODEX_LB_MODEL_CONTEXT_WINDOW_OVERRIDES` entry applies to these slugs +- **WHEN** a client calls `GET /v1/models` +- **THEN** each GPT-5.6 entry reports `context_window=272000` and + `input_context_window=272000` +- **AND** the raised Codex-native ceiling is not promoted into the + OpenAI-compatible input budget fields #### Scenario: GPT-5.6 entries expose upstream tool and multi-agent metadata - **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded - **WHEN** a client calls `GET /backend-api/codex/models` - **THEN** `gpt-5.6-sol`, `gpt-5.6-terra`, and `gpt-5.6-luna` carry `tool_mode: "code_mode_only"`, `use_responses_lite: true`, `experimental_supported_tools: []`, and `minimal_client_version: "0.144.0"` - **AND** `multi_agent_version` is `"v2"` for Sol and Terra and `"v1"` for Luna @@ -203,6 +246,7 @@ sanctioned divergence from the upstream GPT-5.6 entries. #### Scenario: GPT-5.6 entries expose upstream reasoning-summary and plan metadata - **GIVEN** the model registry has no refreshed upstream snapshot +- **AND** no persisted snapshot is loaded - **WHEN** a client calls `GET /backend-api/codex/models` - **THEN** each GPT-5.6 entry carries `default_reasoning_summary: "none"`, `reasoning_summary_format: "experimental"`, and `comp_hash: "3000"` - **AND** each GPT-5.6 entry's `available_in_plans` includes `edu_plus`, `edu_pro`, `enterprise_cbp_automation`, and `sci` @@ -1072,3 +1116,4 @@ The system MUST treat `ultrafast` as an access-controlled service tier and MUST - **WHEN** no live or retained account catalog advertises `ultrafast` - **THEN** bootstrap model metadata does not expose or grant that tier + From eeab46a5edf5be16ff2915edc21a7f6a9a424717 Mon Sep 17 00:00:00 2001 From: Rob K <12484127+rknightion@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:27:21 +0100 Subject: [PATCH 082/117] fix(proxy): classify parameterless previous response errors (#1818) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Triage verification: 20-agent adversarial review (merge_ready, 2/2 verifiers agree) + clean local codex review. Empirically confirmed the two-part fix: (1) strict exact-match widening of is_previous_response_not_found_error for the code-less terse upstream frame, (2) _normalize_error_code at the one WS rewrite call site whose six siblings already normalize — PR tests fail on main head and pass on PR head. Chosen over the overlapping #1817 (hard conflict in errors.py): #1818 is a strict superset — the /v1/responses masking test fails on #1817's head for the observed frame. Known P3 follow-up: bridge-local recovery gate (http_bridge/helpers.py ~2773) still reads raw error codes without normalization; same gap pre-exists on main. --- app/core/errors.py | 17 ++++++- .../proxy/_service/websocket/helpers.py | 5 +- .../.openspec.yaml | 2 + .../design.md | 51 +++++++++++++++++++ .../proposal.md | 27 ++++++++++ .../specs/responses-api-compat/spec.md | 31 +++++++++++ .../tasks.md | 20 ++++++++ .../specs/responses-api-compat/context.md | 15 ++++-- openspec/specs/responses-api-compat/spec.md | 30 +++++++++++ .../test_proxy_websocket_responses.py | 32 +++++++----- tests/unit/test_openai_errors.py | 35 +++++++++++++ 11 files changed, 247 insertions(+), 18 deletions(-) create mode 100644 openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md create mode 100644 openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md create mode 100644 openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md diff --git a/app/core/errors.py b/app/core/errors.py index 395470dbef..75dfafab38 100644 --- a/app/core/errors.py +++ b/app/core/errors.py @@ -79,6 +79,15 @@ def is_previous_response_not_found_message(message: str | None) -> bool: return "previous response" in normalized and "not found" in normalized +def _is_invalid_previous_response_id_message(message: str | None) -> bool: + if message is None: + return False + normalized = " ".join(message.lower().split()) + if normalized.endswith("."): + normalized = normalized[:-1] + return normalized == "invalid `previous_response_id`" + + def previous_response_id_from_not_found_message(message: str | None) -> str | None: if message is None: return None @@ -102,9 +111,13 @@ def is_previous_response_not_found_error( ) -> bool: if code == PREVIOUS_RESPONSE_NOT_FOUND_CODE: return True - if code != "invalid_request_error" or param != "previous_response_id": + if code != "invalid_request_error": + return False + if param is None: + return _is_invalid_previous_response_id_message(message) + if param != "previous_response_id": return False - return is_previous_response_not_found_message(message) + return is_previous_response_not_found_message(message) or _is_invalid_previous_response_id_message(message) def response_failed_event( diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 0cfc86b406..349895c2f3 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -1118,7 +1118,10 @@ def _maybe_rewrite_websocket_previous_response_not_found_event( upstream_control: _WebSocketUpstreamControl, original_text: str, ) -> tuple[OpenAIEvent | None, dict[str, JsonValue] | None, str | None, str]: - error_code = _websocket_event_error_code(event_type, payload) + error_code = _normalize_error_code( + _websocket_event_error_code(event_type, payload), + _websocket_event_error_type(event_type, payload), + ) error_param = _websocket_event_error_param(event_type, payload) error_message = _websocket_event_error_message(event_type, payload) should_rewrite = _facade()._is_previous_response_not_found_error( diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md new file mode 100644 index 0000000000..9bfff7fd5f --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/design.md @@ -0,0 +1,51 @@ +## Context + +See [proposal.md](proposal.md) for the production incident. The shared classifier currently accepts canonical `code = "previous_response_not_found"`, or `code = "invalid_request_error"` only when `param = "previous_response_id"` and the message says the response was not found. The observed upstream frame has neither `code` nor `param`; normalization yields `code = "invalid_request_error"`, and its new ``Invalid `previous_response_id`.`` wording fails the message test. + +Every downstream recovery mechanism already depends on this classifier. Direct WebSocket full resends retain a safe request body without the anchor and can replay transparently. Delta-only Codex-native requests receive a sanitized canonical code that the client uses to resend full local history; public `/v1` traffic receives generic `stream_incomplete` masking. The classifier miss bypasses all of those paths. + +The incident data also contained upstream WebSocket interruptions, downstream disconnects, and connection-limit rotations. Those events explain why otherwise recent response ids can become unusable across connection boundaries, but they do not justify changing cleanup, retry, or transport policy here. The cleanup-budget and phase-attribution changes from upstream PRs #1723 and #1726 are already present on the affected deployment. + +## Goals / Non-Goals + +**Goals:** + +- Recognize the exact newly observed stale-anchor envelope at the shared classification boundary. +- Preserve the existing safety gates that decide between transparent replay, client-assisted full resend, and fail-closed masking. +- Keep false-positive risk bounded with explicit code, parameter, and exact-message checks. + +**Non-Goals:** + +- Do not retry delta-only input without conversation history. +- Do not change WebSocket cleanup budgets, connection lifetime, account routing, health penalties, or retry-circuit policy. +- Do not infer that every generic invalid request is a stale anchor. + +## Decisions + +### Extend the shared semantic classifier + +Add a normalized-message predicate for ``Invalid `previous_response_id``` with zero or one trailing period, and accept it only when the normalized error code is `invalid_request_error` and `param` is absent or already names `previous_response_id`. Reject other trailing punctuation and every different named parameter. Normalize `error.type` at the WebSocket rewrite helper just as its detection and retry-decision callers already do; without that consistency, the first classifier can recognize a code-less frame while the later rewrite still relays it raw. This keeps nested and top-level WebSocket consumers, the HTTP bridge, and compact/error sanitizers on one source of truth. + +Alternative: special-case the raw frame inside the WebSocket relay. Rejected because it would duplicate semantics, miss other existing classifier consumers, and make nested versus top-level envelopes diverge. + +### Reuse existing recovery policy unchanged + +Once classified, the event follows the existing `previous_response_not_found` paths. A self-contained full resend can be replayed without the anchor; a delta-only request cannot. Codex-native clients receive the canonical sanitized code for their controlled full-history retry, while public clients retain generic masking. + +Alternative: drop `previous_response_id` and retry every request. Rejected because the observed first and third failures carried only tool-call output deltas; replaying those without history would silently detach the tool result from its conversation. + +### Treat connection churn as evidence, not patch scope + +The rejected ids were successful on the same account and session 9–17 seconds earlier. That makes long-term retention and cross-account explanations less likely, but does not rule out short retention or reconnect invalidation. The deployment also recorded connection churn, which may make the stale-anchor condition more frequent, but the classifier repair remains correct whether the anchor was invalidated by a reconnect, upstream retention, or another server-side lifecycle boundary. + +Alternative: combine this patch with #1711 transport changes. Rejected because #1711's cleanup warning is observational, its focused fixes are already merged, and the overnight data does not prove one new transport mutation that would eliminate all three failures. + +## Risks / Trade-offs + +- [Upstream reuses the exact message for malformed client ids] → The same recovery remains safe: transparent replay is still gated on a self-contained body, while delta-only clients receive a sanitized request to resend full history. +- [Over-classifying unrelated invalid requests] → Require `invalid_request_error`, reject any different named parameter, and match only the observed message after case/whitespace normalization and optional terminal punctuation. +- [Shared classifier changes non-WebSocket consumers] → Those consumers already treat unusable `previous_response_id` as continuity loss; focused tests cover the classifier plus Codex-native and public route behavior. + +## Migration Plan + +No data or configuration migration is required. Deploy as an application patch; rollback restores the prior raw-400 behavior without changing persisted state. diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md new file mode 100644 index 0000000000..7b45bb1739 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/proposal.md @@ -0,0 +1,27 @@ +## Why + +The ChatGPT-backed Codex WebSocket now emits stale-anchor failures as `invalid_request_error` with no `code` or `param` and the message ``Invalid `previous_response_id`.``. codex-lb does not recognize that observed shape, so it relays the raw 400 instead of entering its existing safe replay or sanitized client-recovery path. + +Production evidence on current upstream `main` recorded three affected Codex sessions in one overnight window. In every case the rejected anchor was a successful response from the same session and account only 9–17 seconds earlier, making this an active compatibility gap rather than an old retained response or account-routing mismatch. + +## What Changes + +- Classify the exact observed parameterless `invalid_request_error` message as a previous-response continuity miss. +- Reuse the existing WebSocket recovery contract: transparently replay self-contained full resends without the anchor, surface sanitized canonical `previous_response_not_found` to Codex-native delta clients, and retain generic masking for public `/v1` clients. +- Preserve classification boundaries for unrelated invalid-request errors and errors naming a different parameter. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Recognize the parameterless invalid-previous-response error shape emitted by the upstream Codex WebSocket and route it through existing stale-anchor recovery and masking. + +## Impact + +- Shared OpenAI error classification in `app/core/errors.py`. +- Direct Responses WebSocket behavior on `/backend-api/codex/responses` and `/v1/responses` through their existing recovery policies. +- Route-level and classifier regression coverage; no API, schema, migration, dependency, configuration, or dashboard changes. diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..ca1560ac13 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/specs/responses-api-compat/spec.md @@ -0,0 +1,31 @@ +## ADDED Requirements + +### Requirement: Parameterless invalid previous-response errors use continuity recovery + +When an upstream Responses WebSocket rejects an anchored request with `type = "invalid_request_error"`, no `code` or `param`, and the normalized message ``Invalid `previous_response_id``` with or without one trailing period, the service MUST classify the frame as a previous-response continuity miss. It MUST apply the same replay, masking, ownership, and account-health rules as the canonical `previous_response_not_found` error and MUST NOT relay the raw invalid-request frame downstream. A different named parameter or any other trailing punctuation MUST NOT match this error shape. + +#### Scenario: Codex-native delta continuation receives the canonical recovery signal + +- **GIVEN** a Codex-native `/backend-api/codex/responses` request carries `previous_response_id` and delta-only tool output that cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the downstream client receives a sanitized error with `code = "previous_response_not_found"` +- **AND** the raw upstream envelope and previous response id are not exposed + +#### Scenario: Self-contained full resend is replayed without the rejected anchor + +- **GIVEN** an anchored direct WebSocket request retains a self-contained full-resend body that is safe to replay without `previous_response_id` +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the service reconnects and replays the retained body without `previous_response_id` +- **AND** the raw upstream error is not sent downstream + +#### Scenario: Public WebSocket retains generic continuity masking + +- **GIVEN** a public `/v1/responses` WebSocket request carries `previous_response_id` but cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error +- **THEN** the downstream client receives the existing sanitized `stream_incomplete` continuity failure +- **AND** neither `previous_response_not_found` nor the raw upstream envelope is exposed + +#### Scenario: Unrelated invalid requests retain their original classification + +- **WHEN** upstream returns `invalid_request_error` with a different message or names a parameter other than `previous_response_id` +- **THEN** the service MUST NOT classify that error as a previous-response continuity miss diff --git a/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md new file mode 100644 index 0000000000..8f05dc0f04 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-classify-invalid-previous-response-id/tasks.md @@ -0,0 +1,20 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a Codex-native route regression using the exact production frame (`invalid_request_error`, no `code`/`param`, ``Invalid `previous_response_id`.``) and verify it fails by exposing the raw 400 before implementation. + +## 2. Classification Fix + +- [x] 2.1 Extend the shared previous-response classifier with the exact parameterless upstream message, normalize the code-less nested frame consistently at the rewrite call, and reject a different named parameter or unrelated invalid-request message. +- [x] 2.2 Verify the route regression passes and the existing canonical stale-anchor recovery tests remain green. + +## 3. Compatibility Boundaries + +- [x] 3.1 Cover the exact observed frame in the self-contained full-resend replay path and confirm the replay drops `previous_response_id`. +- [x] 3.2 Cover the exact observed frame on public `/v1/responses` and confirm it retains generic `stream_incomplete` masking. +- [x] 3.3 Add focused classifier cases for the observed shape and false-positive boundaries. +- [x] 3.4 Add the stable failure-mode and recovery example to the existing `responses-api-compat` context documentation. + +## 4. Verification + +- [x] 4.1 Run focused OpenAI error and direct WebSocket route tests, then the relevant proxy architecture and formatting/lint/type checks. +- [x] 4.2 Run strict OpenSpec validation, the repository's proportionate final gate, and review the final diff for unrelated changes. diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index 4a82968a8e..e2a75c6ca4 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -31,7 +31,7 @@ See `openspec/specs/responses-api-compat/spec.md` for normative requirements. - Compact transport may use bounded same-contract retries only for safe pre-body transport failures and `401 -> refresh -> retry`. - `/v1/responses/compact` is supported only when the upstream implements it. - `prompt_cache_key` affinity on OpenAI-style routes is intentionally bounded by a dashboard-managed freshness window, unlike durable backend `session_id` or dashboard sticky-thread routing. -- Codex-native direct websocket `/backend-api/codex/responses` treats upstream `previous_response_id` as an ephemeral anchor. If that anchor goes stale, the proxy must mask raw `previous_response_not_found` details and emit a sanitized `codex_previous_response_stale` classifier so compatible Codex clients can soft-reset and retry without `previous_response_id`. +- Codex-native direct websocket `/backend-api/codex/responses` treats upstream `previous_response_id` as an ephemeral anchor. If that anchor goes stale, the proxy masks raw upstream details and emits the sanitized canonical `previous_response_not_found` classifier so compatible Codex clients can retry with full local history and no `previous_response_id`. The upstream Codex socket has emitted this condition both with the canonical code and as a parameterless `invalid_request_error` carrying ``Invalid `previous_response_id`.``; both shapes use the same recovery policy. - Upstream Responses WebSockets use transport ping/pong control frames to detect a black-holed connection without confusing valid application-event silence with an idle turn. Direct and routed connections reuse `proxy_downstream_websocket_idle_timeout_seconds` for this zero-config liveness budget. - A post-send liveness timeout is delivery-ambiguous. It remains account-neutral, is never transparently replayed, and retires the affected upstream socket so a client retry opens a fresh route without risking duplicated model work or tool side effects. - An HTTP SSE first-event `stream_idle_timeout` is also account-neutral for health writes. The request may still exclude that account and fail over, but idle silence must not increment `error_count` or move the account into probe/drain. @@ -145,7 +145,7 @@ when upstream reports a different actual tier. - **HTTP bridge session closes or expires:** The next compatible HTTP `/v1/responses` or `/backend-api/codex/responses` request recreates a fresh upstream websocket bridge session; continuity is guaranteed only within the lifetime of one active bridged session. - **Multi-instance routing without bridge owner policy:** if operators do not configure a bridge ring or front-door affinity, continuity can still fragment across replicas. With a configured bridge ring, hard continuity keys landing on a non-owner replica are proxy-forwarded to the owner replica; the proxy fails closed only when the owner endpoint or ring membership cannot be resolved or the forward signature fails authentication. Gateway-safe prompt-cache requests may accept locality misses and continue locally instead of forwarding. - **Codex websocket reconnects:** Reconnect continuity now depends on the client replaying the accepted `x-codex-turn-state`; generated turn-state is emitted on accept for backend Codex routes and echoed back when the client already supplies one. -- **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are surfaced as `response.failed` / `codex_previous_response_stale` without the raw upstream code or missing `resp_...` id; OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. +- **Codex websocket stale previous-response anchors:** Direct backend Codex websocket stale-anchor failures are either replayed transparently from a self-contained full resend or surfaced as a sanitized `response.failed` whose `response.error.code` is `previous_response_not_found`; the error omits `param`, the raw upstream envelope, and the missing `resp_...` id. A connect-time failure uses the same code directly at `error.code`. This includes the parameterless upstream message ``Invalid `previous_response_id`.``. OpenAI-compatible `/v1/responses` websocket clients continue to receive generic `stream_incomplete` masking. - **Websocket handshake forbidden/not-found:** Auto transport now fails loud on `403` / `404` instead of silently hiding the websocket regression behind HTTP fallback. - **Upstream websocket stops answering pings:** Pending direct-WebSocket and HTTP-bridge work fails with `upstream_websocket_liveness_timeout`; the account remains healthy and the request is not replayed because upstream acceptance is unknown. - **Repeated eventless bridge failures:** Two consecutive request-affecting pre-response failures can open the hard-key cooldown. A successful terminal response clears the state; an idle close followed by one real timeout remains only one strike. @@ -186,6 +186,15 @@ the next request times out before `response.created`. The idle close is logged but contributes no failure; the timeout is the first strike. Only another consecutive eventless pending failure may open the repeated-failure cooldown. +Stale-anchor recovery example: a reconnect sends a tool-output delta with a +recent `previous_response_id`, and upstream answers +``{"type":"error","status":400,"error":{"type":"invalid_request_error","message":"Invalid `previous_response_id`."}}``. +Because the delta cannot stand alone, codex-lb returns a sanitized +`previous_response_not_found` signal on the Codex-native route so the client can +retry once with full local history. If the original request already contained a +self-contained full resend, codex-lb instead reconnects and replays that body +without the rejected anchor. + ## Known Client Integrations (Reference) Third-party agents that consume the `/v1` Responses surface documented by this @@ -216,5 +225,5 @@ OpenSpec change first. - Post-deploy: monitor `no_accounts`, `stream_incomplete`, and `upstream_unavailable`. - Post-deploy: monitor `upstream_websocket_liveness_timeout`; recurring failures indicate a host route, VPN, proxy, or intermediary that black-holes established WebSockets. - Post-deploy: correlate retry-circuit `opened`, `half_open`, and `reset` events with bridge `pending` and `response_events_seen` diagnostics. An idle `pending=0` retirement must not precede an immediate two-failure cooldown. -- Post-deploy: monitor `codex_previous_response_stale` on `/backend-api/codex/responses`; recurring spikes mean clients are still relying on stale upstream anchors and should perform the documented full-context retry without `previous_response_id`. +- Post-deploy: monitor `previous_response_not_found` on `/backend-api/codex/responses`; recurring spikes show repeated continuity failures, which may come from malformed client identifiers, server-side invalidation, or connection lifecycle. Clients should perform the documented full-context retry without `previous_response_id`. Investigate socket-lifecycle remediation only when a separate close-reason, reconnect, or transport diagnostic correlates with the failures. - Websocket/Codex CLI tier verification runbook: `openspec/specs/responses-api-compat/ops.md` diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index a56410d4de..b201e659e9 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -426,6 +426,36 @@ When a direct WebSocket `response.create` request includes both `previous_respon - **THEN** the service MUST NOT replay that payload as a fresh turn without `previous_response_id` - **AND** the downstream client receives a retryable continuity failure rather than a fabricated fresh turn +### Requirement: Parameterless invalid previous-response errors use continuity recovery + +When an upstream Responses WebSocket rejects an anchored request with `type = "invalid_request_error"`, no `code` or `param`, and the normalized message ``Invalid `previous_response_id``` with or without one trailing period, the service MUST classify the frame as a previous-response continuity miss. It MUST apply the same replay, masking, ownership, and account-health rules as the canonical `previous_response_not_found` error and MUST NOT relay the raw invalid-request frame downstream. A different named parameter or any other trailing punctuation MUST NOT match this error shape. + +#### Scenario: Codex-native delta continuation receives the canonical recovery signal + +- **GIVEN** a Codex-native `/backend-api/codex/responses` request carries `previous_response_id` and delta-only tool output that cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the downstream client receives a sanitized error with `code = "previous_response_not_found"` +- **AND** the raw upstream envelope and previous response id are not exposed + +#### Scenario: Self-contained full resend is replayed without the rejected anchor + +- **GIVEN** an anchored direct WebSocket request retains a self-contained full-resend body that is safe to replay without `previous_response_id` +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error before `response.created` +- **THEN** the service reconnects and replays the retained body without `previous_response_id` +- **AND** the raw upstream error is not sent downstream + +#### Scenario: Public WebSocket retains generic continuity masking + +- **GIVEN** a public `/v1/responses` WebSocket request carries `previous_response_id` but cannot be replayed safely without its anchor +- **WHEN** upstream returns the parameterless ``Invalid `previous_response_id`.`` error +- **THEN** the downstream client receives the existing sanitized `stream_incomplete` continuity failure +- **AND** neither `previous_response_not_found` nor the raw upstream envelope is exposed + +#### Scenario: Unrelated invalid requests retain their original classification + +- **WHEN** upstream returns `invalid_request_error` with a different message or names a parameter other than `previous_response_id` +- **THEN** the service MUST NOT classify that error as a previous-response continuity miss + ### Requirement: Public Responses errors mask previous-response misses Public Responses endpoints MUST NOT return an OpenAI-shaped `previous_response_not_found` error to clients. If a lower layer still raises or collects that error, the API layer MUST rewrite it to a retryable `stream_incomplete` continuity failure and remove the missing response id from the public payload. diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index 69d1a61195..70c1e2e265 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -6008,9 +6008,7 @@ def test_responses_websocket_replays_client_full_resend_previous_response_miss_w "status": 400, "error": { "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_ws_prev_anchor' not found.", - "param": "previous_response_id", + "message": "Invalid `previous_response_id`.", }, }, separators=(",", ":"), @@ -6192,9 +6190,7 @@ def test_v1_responses_websocket_masks_invalid_request_previous_response_not_foun "status": 400, "error": { "type": "invalid_request_error", - "code": "invalid_request_error", - "message": ("Previous response with id 'resp_ws_prev_anchor' not found."), - "param": "previous_response_id", + "message": "Invalid `previous_response_id`.", }, }, separators=(",", ":"), @@ -6445,9 +6441,26 @@ async def fake_try_open_websocket_connect_attempt( _assert_previous_response_not_found_error(event["error"]) +@pytest.mark.parametrize( + "upstream_error", + [ + { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": "Previous response with id 'resp_ws_prev_anchor' not found.", + "param": "previous_response_id", + }, + { + "type": "invalid_request_error", + "message": "Invalid `previous_response_id`.", + }, + ], + ids=["canonical-not-found", "parameterless-invalid-id"], +) def test_backend_responses_websocket_masks_short_previous_response_not_found_without_retry( app_instance, monkeypatch, + upstream_error, ): first_upstream = _SequencedUpstreamWebSocket( [], @@ -6481,12 +6494,7 @@ def test_backend_responses_websocket_masks_short_previous_response_not_found_wit { "type": "error", "status": 400, - "error": { - "type": "invalid_request_error", - "code": "previous_response_not_found", - "message": "Previous response with id 'resp_ws_prev_anchor' not found.", - "param": "previous_response_id", - }, + "error": upstream_error, }, separators=(",", ":"), ), diff --git a/tests/unit/test_openai_errors.py b/tests/unit/test_openai_errors.py index 70ec2d136f..ac0dc547dd 100644 --- a/tests/unit/test_openai_errors.py +++ b/tests/unit/test_openai_errors.py @@ -52,11 +52,46 @@ def test_previous_response_not_found_classifier_covers_openai_shapes(): param="previous_response_id", message='Previous response with id "resp_abc" not found.', ) + assert is_previous_response_not_found_error( + code="invalid_request_error", + param=None, + message="Invalid `previous_response_id`.", + ) + assert is_previous_response_not_found_error( + code="invalid_request_error", + param=None, + message="Invalid `previous_response_id`", + ) + assert is_previous_response_not_found_error( + code="invalid_request_error", + param="previous_response_id", + message="Invalid `previous_response_id`.", + ) assert not is_previous_response_not_found_error( code="invalid_request_error", param="input", message='Previous response with id "resp_abc" not found.', ) + assert not is_previous_response_not_found_error( + code="invalid_request_error", + param="input", + message="Invalid `previous_response_id`.", + ) + assert not is_previous_response_not_found_error( + code="invalid_request_error", + param=None, + message="Invalid request payload.", + ) + assert not is_previous_response_not_found_error( + code="invalid_request_error", + param=None, + message="Invalid `previous_response_id`...", + ) + assert not is_previous_response_not_found_error( + code=None, + param=None, + message="Invalid `previous_response_id`.", + ) def test_previous_response_id_from_not_found_message_extracts_anchor(): From f839952ed0920085af1d609677f60f11d86ae75f Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 19 Aug 2026 20:50:07 +0900 Subject: [PATCH 083/117] chore: release v1.24.0-beta.2 (#1810) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 ++-- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index b6ae45e81c..db51e06b8b 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.24.0-beta.1" # x-release-please-version +__version__ = "1.24.0-beta.2" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index 7a7ccf4169..ab62108e2a 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.24.0-beta.1 -appVersion: 1.24.0-beta.1 +version: 1.24.0-beta.2 +appVersion: 1.24.0-beta.2 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index 26e2cba030..c716ae84ac 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.24.0-beta.1", + "version": "1.24.0-beta.2", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 2ddaa60269..e7f19aab46 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.24.0-beta.1" +version = "1.24.0-beta.2" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 2f40f62031..dfd12b2280 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0b1" +version = "1.24.0-beta.2" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From 3381938aa278d7f3cd371bdd76c7914856586bf8 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 20 Aug 2026 07:00:06 +0200 Subject: [PATCH 084/117] fix(proxy): bind account-bound retries to dispatch owner (#1829) * fix(proxy): bind account-bound retries to dispatch owner * fix(proxy): defer replay owner until dispatch --- .../_service/http_bridge/request_submit.py | 63 +- .../_service/http_bridge/service_stubs.py | 4 + app/modules/proxy/_service/streaming/retry.py | 59 +- app/modules/proxy/_service/support.py | 3 + .../proxy/_service/websocket/helpers.py | 109 ++- app/modules/proxy/_service/websocket/mixin.py | 82 ++- app/modules/proxy/service.py | 1 + .../.openspec.yaml | 2 + .../context.md | 54 ++ .../design.md | 110 +++ .../proposal.md | 46 ++ .../specs/responses-api-compat/spec.md | 92 +++ .../tasks.md | 29 + .../specs/responses-api-compat/context.md | 30 + openspec/specs/responses-api-compat/spec.md | 91 +++ .../test_proxy_websocket_responses.py | 26 +- tests/unit/test_proxy_utils.py | 678 ++++++++++++++++++ 17 files changed, 1410 insertions(+), 69 deletions(-) create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 29823ca125..364179fbf4 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -122,6 +122,7 @@ _upstream_response_create_max_bytes, _websocket_auth_failure_permanent_code, _websocket_auth_failure_requires_reauth, + _websocket_request_text_is_account_neutral_fresh_replay, ) from app.modules.proxy._service.observability import ( _hash_identifier as _hash_identifier, @@ -3094,6 +3095,7 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: ) if request_state.replay_count >= 1 and not additional_clean_close_retry: return False + account_bound_replay = False if request_state.previous_response_id is not None: require_preferred_reconnect = False if account_neutral_recovery: @@ -3125,11 +3127,26 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: # Account-scoped uploaded files cannot be replayed on a # different owner. Keep the preferred account mandatory for # both silent recovery and clean-close recovery. - require_preferred_reconnect = account_neutral_recovery or request_state.file_required_preferred_account + candidate_text = ( + request_state.fresh_upstream_request_text + if request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text + else request_state.request_text + ) + # The send boundary decorates durable operations with + # codex_lb_operation_id after selection. Keep that operation + # identity on its owner unless a dedicated rebind path has + # already replaced the operation ID. + candidate_portable = request_state.operation_id is None and ( + _websocket_request_text_is_account_neutral_fresh_replay(candidate_text) + ) request_text = _prepare_websocket_request_state_for_visible_output_replay(request_state) - if request_text is None: + if request_text is None or request_text != candidate_text: return False - if account_neutral_recovery: + account_bound_replay = not candidate_portable + require_preferred_reconnect = ( + account_neutral_recovery or account_bound_replay or request_state.file_required_preferred_account + ) + if account_neutral_recovery or account_bound_replay: request_state.preferred_account_id = session.account.id elif not request_state.file_required_preferred_account: if hard_owner_bound and not model_fallback_replay and not fresh_hard_request_account_switch_allowed: @@ -3210,7 +3227,7 @@ def request_is_retryable(request_state: _WebSocketRequestState) -> bool: await self._reconnect_http_bridge_session( session, request_state=request_state, - require_same_account=account_neutral_recovery, + require_same_account=account_neutral_recovery or account_bound_replay, require_preferred_account=True, **reconnect_reader_kwargs, ) @@ -3319,7 +3336,22 @@ async def _retry_http_bridge_precreated_auth_request( error_message: str | None, ) -> Literal["not_replayable", "retried", "failed"]: permanent_failure_code = _websocket_auth_failure_permanent_code(error_message) - request_text = _prepare_websocket_request_state_for_auth_replay(request_state) + bound_to_current_account = request_state.replay_required_account_id == session.account.id + if bound_to_current_account and ( + _websocket_auth_failure_requires_reauth(error_message) + or request_state.auth_replay_counts_by_account.get(session.account.id, 0) > 0 + ): + failure_code = permanent_failure_code or _WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE + await self._load_balancer.mark_permanent_failure(session.account, failure_code) + setattr(request_state, "account_health_error_handled", True) + request_state.force_refresh_account_id = None + request_state.preferred_account_id = None + request_state.excluded_account_ids.add(session.account.id) + return "not_replayable" + request_text = _prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id=session.account.id, + ) if request_text is None: await self._load_balancer.mark_permanent_failure(session.account, permanent_failure_code) setattr(request_state, "account_health_error_handled", True) @@ -3368,10 +3400,14 @@ async def _retry_http_bridge_precreated_auth_request( await self._reconnect_http_bridge_session( session, request_state=request_state, - require_same_account=is_http_bridge_account_neutral_replay( - kind=session.key.affinity_kind, - key=session.key.affinity_key, + require_same_account=( + bound_to_current_account + or is_http_bridge_account_neutral_replay( + kind=session.key.affinity_kind, + key=session.key.affinity_key, + ) ), + require_preferred_account=bound_to_current_account, ) request_text = self._http_bridge_text_with_account_installation_id(session, request_state, request_text) await _send_http_bridge_request_text_with_archive_id(session, request_state, request_text) @@ -3413,13 +3449,13 @@ async def _retry_http_bridge_security_work_request( key=session.key.affinity_key, ): return False - retry_text = request_state.request_text - if not retry_text: - return False if request_state.file_required_preferred_account: return False if not _websocket_request_can_replay_before_visible_output(request_state): return False + retry_text = _prepare_websocket_request_state_for_account_switch(request_state) + if retry_text is None: + return False owner_account_id = session.account.id previous_replay_count = request_state.replay_count @@ -3436,11 +3472,6 @@ async def _retry_http_bridge_security_work_request( session.turn_state_alias_registration_generations ) previous_session_headers = session.headers - if request_state.previous_response_id is not None: - retry_text = _prepare_websocket_request_state_for_account_switch(request_state) - if retry_text is None: - return False - request_state.preferred_account_id = None request_state.excluded_account_ids.add(owner_account_id) request_state.affinity_policy = replace( diff --git a/app/modules/proxy/_service/http_bridge/service_stubs.py b/app/modules/proxy/_service/http_bridge/service_stubs.py index 3db2e16c15..86ed81c9a3 100644 --- a/app/modules/proxy/_service/http_bridge/service_stubs.py +++ b/app/modules/proxy/_service/http_bridge/service_stubs.py @@ -452,6 +452,10 @@ def _prepare_websocket_request_state_for_account_switch(*args: Any, **kwargs: An return _service_global("_prepare_websocket_request_state_for_account_switch")(*args, **kwargs) +def _websocket_request_text_is_account_neutral_fresh_replay(*args: Any, **kwargs: Any) -> Any: + return _service_global("_websocket_request_text_is_account_neutral_fresh_replay")(*args, **kwargs) + + def _matching_websocket_request_states_for_previous_response_error(*args: Any, **kwargs: Any) -> Any: return _service_global("_matching_websocket_request_states_for_previous_response_error")(*args, **kwargs) diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index 0b28853a44..c0d2c43abe 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -79,6 +79,7 @@ is_upstream_model_capacity_error, ) from app.modules.proxy.load_balancer import AccountLease, AccountSelection +from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response _REQUEST_TRANSPORT_HTTP = "http" @@ -176,7 +177,10 @@ def _verified_cross_transport_fresh_replay( stored_fingerprint=continuity_state.last_completed_input_prefix_fingerprint, ): return None - return payload.model_copy(update={"previous_response_id": None}) + fresh_payload = payload.model_copy(update={"previous_response_id": None}) + if not responses_payload_is_account_neutral_fresh_replay(fresh_payload.to_replay_safety_payload()): + return None + return fresh_payload def _effective_http_downstream_transport_policy( @@ -396,6 +400,7 @@ async def _stream_with_retry( deferred_capacity_account: Account | None = None deferred_capacity_lease: AccountLease | None = None preferred_account_id: str | None = None + payload_replay_required_account_id: str | None = None file_preferred_account_id: str | None = rewritten_file_account_id require_preferred_account = False last_retryable_stream_error: _RetryableStreamError | None = None @@ -577,18 +582,39 @@ async def _settle_process_network_budget_exhaustion( ) settled = await _settle_stream_usage_before_pending_penalty(settlement) + def _authorize_payload_dispatch(account: Account) -> bool: + required_account_id = payload_replay_required_account_id + if required_account_id is not None and required_account_id != account.id: + raise ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) + return required_account_id is None and not responses_payload_is_account_neutral_fresh_replay( + payload.to_replay_safety_payload() + ) + def _move_verified_fresh_replay_from_owner(*, account_id: str, outcome: str) -> bool: # Only a proxy-injected owner anchor with locally verified full # input may move; the failed owner stays excluded so sticky # selection cannot immediately loop back to it. - nonlocal affinity, payload, preferred_account_id, require_preferred_account, verified_fresh_replay_payload + nonlocal affinity, payload, payload_replay_required_account_id + nonlocal preferred_account_id, require_preferred_account, verified_fresh_replay_payload if not ( require_preferred_account and preferred_account_id == account_id and verified_fresh_replay_payload is not None ): return False + if not responses_payload_is_account_neutral_fresh_replay( + verified_fresh_replay_payload.to_replay_safety_payload() + ): + return False payload = verified_fresh_replay_payload + payload_replay_required_account_id = None verified_fresh_replay_payload = None excluded_account_ids.add(account_id) preferred_account_id = None @@ -1037,6 +1063,13 @@ async def _retry_account_model_rejection( yield format_sse_event(_facade()._proxy_request_timeout_event(request_id)) return while True: + effective_preferred_account_id = resolve_required_account_id( + ("continuation", preferred_account_id), + ("dispatched payload", payload_replay_required_account_id), + ) + effective_require_preferred_account = ( + require_preferred_account or payload_replay_required_account_id is not None + ) try: selection = await proxy._select_account_with_budget_compatible( deadline, @@ -1050,7 +1083,7 @@ async def _retry_account_model_rejection( model=payload.model, service_tier=payload.service_tier, exclude_account_ids=excluded_account_ids, - preferred_account_id=preferred_account_id, + preferred_account_id=effective_preferred_account_id, require_security_work_authorized=require_security_work_authorized, lease_kind="stream", estimated_lease_tokens=estimated_lease_tokens, @@ -1058,7 +1091,7 @@ async def _retry_account_model_rejection( # verified-fresh replay branch below removes its # anchor before it permits cross-account movement. fallback_on_preferred_account_unavailable=not ( - require_preferred_account or file_required_preferred_account + effective_require_preferred_account or file_required_preferred_account ), ) except ProxyResponseError as exc: @@ -1846,6 +1879,7 @@ async def _retry_account_model_rejection( ) try: settlement = _StreamSettlement() + register_payload_owner = _authorize_payload_dispatch(account) inner_stream = proxy._stream_once( account, payload, @@ -1887,8 +1921,21 @@ async def _retry_account_model_rejection( enforce_openai_sdk_contract=enforce_openai_sdk_contract, ) try: - async for line in inner_stream: - yield line + try: + async for line in inner_stream: + if register_payload_owner: + payload_replay_required_account_id = account.id + register_payload_owner = False + yield line + if register_payload_owner: + payload_replay_required_account_id = account.id + except BaseException as exc: + if register_payload_owner and not ( + isinstance(exc, ProxyResponseError) + and is_confirmed_pre_dispatch_transport_error(exc) + ): + payload_replay_required_account_id = account.id + raise finally: close_task = asyncio.create_task( inner_stream.aclose(), diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index fd11e00f20..8a79957e72 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -1066,6 +1066,9 @@ class _WebSocketRequestState: fresh_upstream_request_responses_lite_model: str | None = None request_stage: str = "first_turn" preferred_account_id: str | None = None + # Once an account-bound body has been dispatched, retries remain pinned to + # that owner even when stale-anchor recovery removes previous_response_id. + replay_required_account_id: str | None = None require_security_work_authorized: bool = False durable_capability_lineage_required: bool = False file_required_preferred_account: bool = False diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 349895c2f3..83858cc645 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -339,6 +339,7 @@ from app.modules.proxy.http_bridge_forwarding import ( OwnerForwardRelayFailure as OwnerForwardRelayFailure, ) +from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay def _facade() -> Any: @@ -459,37 +460,75 @@ def _websocket_owner_switch_has_other_pending_requests( return any(pending is not request_state for pending in pending_requests) -def _prepare_websocket_request_state_for_account_switch( +def _websocket_request_text_is_account_neutral_fresh_replay(request_text: str | None) -> bool: + if not isinstance(request_text, str): + return False + try: + payload = json.loads(request_text) + except json.JSONDecodeError: + return False + if not isinstance(payload, dict): + return False + event_type = payload.get("type") + if event_type is not None and event_type != "response.create": + return False + payload.pop("type", None) + return responses_payload_is_account_neutral_fresh_replay(cast(dict[str, JsonValue], payload)) + + +def _bind_websocket_request_dispatch_owner( + request_state: "_WebSocketRequestState", + *, + account_id: str, + exact_request_text: str, +) -> bool: + required_account_id = request_state.replay_required_account_id + if _websocket_request_text_is_account_neutral_fresh_replay(exact_request_text): + return required_account_id is None or required_account_id == account_id + if required_account_id is not None and required_account_id != account_id: + return False + request_state.preferred_account_id = account_id + request_state.replay_required_account_id = account_id + return True + + +def _install_verified_fresh_replay( request_state: "_WebSocketRequestState", + *, + require_proxy_injected_previous_response_id: bool = True, + require_account_neutral: bool = True, ) -> str | None: - """Return an unsent request body only when moving accounts is proven safe.""" - if request_state.previous_response_id is None: - return request_state.request_text - if not ( - request_state.proxy_injected_previous_response_id - and request_state.fresh_upstream_request_is_retry_safe - and request_state.fresh_upstream_request_text - ): + if not (request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text): return None - try: - fresh_payload = json.loads(request_state.fresh_upstream_request_text) - except (TypeError, json.JSONDecodeError): + if require_proxy_injected_previous_response_id and not request_state.proxy_injected_previous_response_id: return None - fresh_input = fresh_payload.get("input") - if extract_input_file_ids(fresh_input): - # A retained full body can be replay-safe for text continuity while - # still naming an account-scoped uploaded file. Keep its injected - # anchor instead of moving that file reference to another account. + fresh_request_text = request_state.fresh_upstream_request_text + account_neutral = _websocket_request_text_is_account_neutral_fresh_replay(fresh_request_text) + if require_account_neutral and not account_neutral: return None - - request_state.request_text = request_state.fresh_upstream_request_text + replay_required_account_id = request_state.replay_required_account_id or request_state.preferred_account_id + if not account_neutral and replay_required_account_id is None: + return None + request_state.request_text = fresh_request_text request_state.previous_response_id = None request_state.preferred_account_id = None + request_state.replay_required_account_id = None if account_neutral else replay_required_account_id request_state.proxy_injected_previous_response_id = False request_state.fresh_upstream_request_is_retry_safe = False request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model _refresh_websocket_request_input_fingerprint_from_text(request_state) - return request_state.request_text + return fresh_request_text + + +def _prepare_websocket_request_state_for_account_switch( + request_state: "_WebSocketRequestState", +) -> str | None: + """Return an unsent request body only when moving accounts is proven safe.""" + if request_state.previous_response_id is None: + if not _websocket_request_text_is_account_neutral_fresh_replay(request_state.request_text): + return None + return request_state.request_text + return _install_verified_fresh_replay(request_state) def _websocket_continuity_anchor_for_payload( @@ -900,35 +939,43 @@ def _websocket_auth_request_can_switch_account(request_state: _WebSocketRequestS if request_state.file_required_preferred_account: return False if request_state.previous_response_id is None: - return True + return request_state.request_text is None or _websocket_request_text_is_account_neutral_fresh_replay( + request_state.request_text + ) if not ( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text ): return False - return not _websocket_fresh_request_blocks_account_switch(request_state) + return _websocket_request_text_is_account_neutral_fresh_replay( + request_state.fresh_upstream_request_text + ) and not _websocket_fresh_request_blocks_account_switch(request_state) def _prepare_websocket_request_state_for_auth_replay( request_state: _WebSocketRequestState, + *, + current_account_id: str | None = None, ) -> str | None: if request_state.last_downstream_sequence_number is not None: return None - if not _websocket_auth_request_can_switch_account(request_state): + can_switch_account = _websocket_auth_request_can_switch_account(request_state) + can_retry_bound_owner = ( + request_state.auth_replay_count == 0 + and current_account_id is not None + and request_state.replay_required_account_id == current_account_id + and isinstance(request_state.request_text, str) + ) + if not can_switch_account and not can_retry_bound_owner: return None - if ( + if can_switch_account and ( request_state.proxy_injected_previous_response_id and request_state.fresh_upstream_request_is_retry_safe and request_state.fresh_upstream_request_text ): - request_state.request_text = request_state.fresh_upstream_request_text - request_state.previous_response_id = None - request_state.preferred_account_id = None - request_state.proxy_injected_previous_response_id = False - request_state.fresh_upstream_request_is_retry_safe = False - request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model - _refresh_websocket_request_input_fingerprint_from_text(request_state) + if _install_verified_fresh_replay(request_state) is None: + return None request_text = request_state.request_text if not isinstance(request_text, str): return None diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index b1e948dab7..1c268f7ea9 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -383,8 +383,10 @@ from app.modules.proxy._service.websocket.helpers import ( _app_error_to_websocket_event, _assign_websocket_response_id, + _bind_websocket_request_dispatch_owner, _find_websocket_request_state_by_response_id, _forget_websocket_stale_previous_response, + _install_verified_fresh_replay, _is_websocket_response_create, _is_websocket_stale_previous_response, _match_websocket_request_state_for_anonymous_event, @@ -2575,6 +2577,19 @@ def take_reader_replay_request_state() -> _WebSocketRequestState | None: if text_data is not None: archive_request_id = None if request_state is None else request_state.archive_request_id if request_state is not None and payload is not None and _is_websocket_response_create(payload): + if account is None or not _bind_websocket_request_dispatch_owner( + request_state, + account_id=account.id, + exact_request_text=text_data, + ): + raise ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) request_state.response_create_sent_at = time.monotonic() with _websocket_archive_request_context(archive_request_id): await upstream.send_text(text_data) @@ -3441,13 +3456,18 @@ async def _record_or_defer_confirmed_route_backoff(account: Account) -> None: for attempt in range(max_attempts): is_retry = attempt > 0 forced_refresh_account_id = request_state.force_refresh_account_id - preferred_account_id = forced_refresh_account_id or request_state.preferred_account_id + preferred_account_id = ( + request_state.replay_required_account_id + or forced_refresh_account_id + or request_state.preferred_account_id + ) turn_state_owner_required = ( request_state.affinity_policy.codex_session_source == "turn_state" and request_state.preferred_account_id is not None ) require_preferred_account = ( (request_state.previous_response_id is not None and request_state.preferred_account_id is not None) + or request_state.replay_required_account_id is not None or request_state.file_required_preferred_account or turn_state_owner_required ) @@ -3761,6 +3781,14 @@ async def _heartbeat(remaining_seconds: float) -> None: break account = selection.account + if ( + account is not None + and request_state.replay_required_account_id is None + and request_state.request_text is not None + and not _facade()._websocket_request_text_is_account_neutral_fresh_replay(request_state.request_text) + ): + request_state.preferred_account_id = account.id + request_state.replay_required_account_id = account.id if ( account is not None and require_preferred_account @@ -5544,18 +5572,21 @@ async def _process_upstream_websocket_text( # transparently retried. retry_error_code = None else: - upstream_control.reconnect_requested = True - request_state.request_text = request_state.fresh_upstream_request_text - request_state.previous_response_id = None - request_state.proxy_injected_previous_response_id = False - request_state.fresh_upstream_request_is_retry_safe = False - request_state.responses_lite_model = request_state.fresh_upstream_request_responses_lite_model - request_state.replay_count += 1 - request_state.awaiting_response_created = True - request_state.response_id = None - _clear_websocket_request_error_overrides(request_state) - upstream_control.suppress_downstream_event = True - upstream_control.replay_request_state = request_state + replay_text = _install_verified_fresh_replay( + request_state, + require_proxy_injected_previous_response_id=False, + require_account_neutral=False, + ) + if replay_text is None: + retry_error_code = None + else: + upstream_control.reconnect_requested = True + request_state.replay_count += 1 + request_state.awaiting_response_created = True + request_state.response_id = None + _clear_websocket_request_error_overrides(request_state) + upstream_control.suppress_downstream_event = True + upstream_control.replay_request_state = request_state else: upstream_control.reconnect_requested = True request_state.replay_count += 1 @@ -5678,10 +5709,31 @@ async def _handle_precreated_websocket_auth_failure( ) -> bool: proxy = cast(_WebSocketServiceProtocol, self) _ = proxy - if _prepare_websocket_request_state_for_auth_replay(request_state) is None: + bound_to_current_account = request_state.replay_required_account_id == account.id + requires_reauth = _websocket_auth_failure_requires_reauth(error_message) + if bound_to_current_account and ( + requires_reauth or request_state.auth_replay_counts_by_account.get(account.id, 0) > 0 + ): + failure_code = ( + _facade()._WEBSOCKET_SESSION_EXPIRED_FAILURE_CODE + if requires_reauth + else _facade()._WEBSOCKET_AUTH_INVALIDATED_FAILURE_CODE + ) + await proxy._load_balancer.mark_permanent_failure(account, failure_code) + request_state.force_refresh_account_id = None + request_state.preferred_account_id = None + request_state.excluded_account_ids.add(account.id) + return False + if ( + _prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id=account.id, + ) + is None + ): return False - if _websocket_auth_failure_requires_reauth(error_message): + if requires_reauth: failure_code = _facade()._WEBSOCKET_SESSION_EXPIRED_FAILURE_CODE elif request_state.auth_replay_counts_by_account.get(account.id, 0) == 0: request_state.auth_replay_counts_by_account[account.id] = 1 diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 898b41d88e..cf6fcbafd0 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -704,6 +704,7 @@ _websocket_precreated_auth_error_code, # noqa: F401 _websocket_precreated_retry_error_code, # noqa: F401 _websocket_receive_timeout_for_pending_requests, # noqa: F401 + _websocket_request_text_is_account_neutral_fresh_replay, # noqa: F401 _websocket_response_id, # noqa: F401 _websocket_top_level_error_payload, # noqa: F401 _wrapped_websocket_error_event, # noqa: F401 diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md new file mode 100644 index 0000000000..08de111df6 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/context.md @@ -0,0 +1,54 @@ +# Previous-response replay owner fencing + +## Purpose + +This change distinguishes continuation-anchor recovery from payload +portability. A retry may safely remove a stale anchor yet still be forbidden +from changing accounts because retained request items remain account-scoped. + +## Example + +Account A first receives: + +```json +{ + "previous_response_id": "resp_owner", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound-ciphertext" + } + ] +} +``` + +If a pre-visible failure triggers stale-anchor recovery, the proxy may remove +`previous_response_id` only as part of a verified replay. Because the retained +encrypted reasoning is not account-neutral, the replacement remains bound to +account A. Account B must never receive it. + +An ordinary fresh request containing only portable user input can pass the +canonical predicate and may use normal account selection. + +A selected account is not recorded as owner when transport evidence proves the +request failed before dispatch. The body may then make its first real dispatch +on another eligible account. Ambiguous failures remain pinned. + +HTTP bridge operation IDs are proxy-owned but still identify an in-flight +operation. A bridge retry carrying an existing operation ID remains on its +current account unless the operation is explicitly rebound before selection. + +## Operational Notes + +- Owner-unavailable failures are internal retry decisions; they do not add a + setting or require operator action. +- Existing file ownership remains an independent strict pin. +- Verified fresh-body installation clears the old dispatch owner atomically + with replacing the request body. +- A bound request may perform one forced authentication refresh on the same + owner; it does not become eligible for cross-account auth failover. +- API-key reservation settlement still completes before deferred account-health + writes. +- The change covers HTTP streaming, HTTP bridge, and direct WebSocket paths so + operators do not observe transport-dependent account ownership. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md new file mode 100644 index 0000000000..2b0ab9f16c --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/design.md @@ -0,0 +1,110 @@ +## Context + +Responses requests can carry both a server-side continuation anchor and +client-retained material. Removing a stale `previous_response_id` does not make +the remaining body portable: encrypted reasoning, account-scoped probe items, +and other retained state can still belong to the account that first received +the request. + +The selector already supports strict required-account routing for known +previous-response and file owners. The missing state is payload dispatch +provenance: after a pre-visible retry excludes an account, later selection can +no longer tell that the retained body was already dispatched there. + +## Goals / Non-Goals + +**Goals:** + +- Bind nonportable payloads to their first dispatch account. +- Enforce the binding consistently in HTTP streaming, HTTP bridge, and direct + WebSocket retry paths. +- Allow cross-account replay only after exact-wire verification proves the + resulting request is an account-neutral fresh replay. +- Preserve existing settlement, health-write, and file-owner invariants. + +**Non-Goals:** + +- Changing stale previous-response error classification from PR #1818. +- Preserving or reshaping unrelated bare/raw upstream error fields. +- Changing public API envelopes, retry counts, quota accounting, or settings. +- Making encrypted reasoning or account-scoped probe items portable. + +## Decisions + +### Use the canonical portability predicate + +Every candidate body is evaluated with +`responses_payload_is_account_neutral_fresh_replay`. Ad hoc checks for files or +`previous_response_id` are insufficient because account scope can live in +retained input items. + +Alternative: extend each transport's file checks. Rejected because it +duplicates an incomplete allowlist and already failed to catch encrypted +reasoning. + +### Bind on first nonportable dispatch + +A request-local dispatch-owner ID is authorized before the first nonportable +payload is sent and persisted after the first upstream event or normal stream +completion. Ambiguous/post-dispatch failures also preserve that owner, while a +positively confirmed pre-dispatch transport failure does not create one. Every +later selection treats a persisted owner like any other strict continuity +requirement. + +Alternative: infer ownership from the current preferred account. Rejected +because retry branches intentionally clear or replace preference state. + +### Clear ownership only after verified neutral replay + +Verified stale-anchor recovery may replace the wire body with a reconstructed +fresh request. The dispatch binding is cleared only when that exact replacement +passes the canonical account-neutral predicate. Body replacement and +owner-fence clearing occur in one transition so a retry cannot observe mixed +state. A verified nonneutral replacement may be installed for a same-owner +retry, but that transition preserves the existing dispatch-owner fence. + +Alternative: clear ownership whenever the anchor is removed. Rejected because +the reproduced defect retained owner-bound ciphertext after anchor removal. + +### Treat proxy-owned operation metadata as account-bound + +HTTP bridge sends may add `codex_lb_operation_id` after request preparation. +Until a dedicated rebind path replaces that operation identity, selection +treats the request as nonportable and requires the current account. + +Alternative: remove the proxy-owned field before portability checks. Rejected +because normalization would authorize a different account while preserving the +same operation identity on the final wire request. + +### Fail closed across transport-specific recovery + +Trusted Access migration/degradation, owner exclusion, bridge reconnect, and +WebSocket account switching may not bypass payload ownership. If the owner +cannot satisfy the retry, the proxy returns the stable owner-unavailable error +without dispatching the retained body elsewhere. + +A generic authentication failure is split into two decisions: one forced token +refresh may replay a bound body on the same owner, while owner exclusion or +cross-account migration still requires an atomically installed neutral body. +Permanent authentication failure remains terminal for a bound body. + +## Risks / Trade-offs + +- **Fewer automatic retries for account-bound bodies** → This is intentional; + confidentiality and continuation correctness outrank cross-account fallback. +- **False nonportability** → The canonical predicate is an explicit allowlist, + so unknown retained item types fail closed. +- **Transport drift** → Shared helpers plus focused HTTP, bridge, and WebSocket + regressions keep the invariant aligned. +- **Settlement regression** → The change does not move reservation settlement + or deferred health writes; existing settlement tests remain mandatory. + +## Migration Plan + +No data or configuration migration is required. Deploy the proxy code normally. +Rollback is a code rollback; no persisted format changes. + +## Open Questions + +None. Current-main runtime probes reproduce the cross-account dispatch and the +existing selector already provides the strict owner-routing primitive. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md new file mode 100644 index 0000000000..8ad755213e --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/proposal.md @@ -0,0 +1,46 @@ +## Why + +A pre-visible Responses retry can remove a `previous_response_id` anchor while +retaining account-scoped request material, exclude the original account, and +dispatch the retained payload to another account. Encrypted reasoning was +reproduced crossing accounts through the HTTP stream path; the same missing +dispatch provenance affects HTTP bridge and direct WebSocket retries. + +PR #1818 fixed parameterless stale-response classification but intentionally +did not add payload-owner fencing. The remaining defect violates account +ownership even when session/file continuity and API-key settlement work as +designed. + +## What Changes + +- Classify exact-wire replay candidates with the canonical + account-neutral-fresh-replay predicate. +- Bind every nonportable Responses payload to its first dispatch account. +- Merge payload ownership with previous-response and file ownership during + every HTTP stream, HTTP bridge, and direct WebSocket selection. +- Fail closed rather than excluding the owner or moving retained + account-scoped material during Trusted Access migration/degradation. +- Clear payload ownership only after verified anchor removal produces a + canonical account-neutral fresh replay. +- Keep proxy-owned operation metadata on its current account unless a + dedicated operation-rebind path replaces that identity before selection. +- Preserve existing file pinning, API-key settlement ordering, error + classification, and raw error-envelope behavior. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: require account-bound retry payloads to remain on + their dispatch owner across all Responses transports. + +## Impact + +- **Affected code:** Responses replay safety, HTTP streaming retries, HTTP + bridge reconnects, and direct WebSocket account switching. +- **Affected tests:** proxy streaming utilities and WebSocket Responses + integration tests. +- **API/schema changes:** none. +- **Configuration changes:** none. +- **Security impact:** prevents account-scoped request material from crossing + account boundaries during internal retries. diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..e09e31f394 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/specs/responses-api-compat/spec.md @@ -0,0 +1,92 @@ +## ADDED Requirements + +### Requirement: Account-bound retries remain on their dispatch owner + +The proxy MUST bind a Responses request body that is not a canonical +account-neutral fresh replay to the account that first receives that exact +body. Every later selection for that request MUST treat the dispatch owner as a +strict required account across HTTP streaming, HTTP bridge, and direct +WebSocket transports. + +The proxy MUST NOT exclude the dispatch owner and send the retained body to a +different account during stale-anchor recovery, retryable account failure, +Trusted Access migration or degradation, bridge reconnect, or WebSocket account +switching. If the required owner is unavailable, the proxy MUST fail closed +without dispatching the retained body to another account. + +The proxy MAY perform one forced authentication refresh and replay a retained +account-bound body on the same dispatch owner. It MUST NOT use that refresh to +exclude the owner or migrate the body to another account, and a permanent +authentication failure MUST remain terminal for the bound body. + +The proxy MAY clear the dispatch-owner binding only after verified recovery +replaces the exact wire body and the replacement passes the canonical +account-neutral-fresh-replay predicate. Removing `previous_response_id` alone +MUST NOT make retained account-scoped input portable. + +Proxy-owned operation metadata that will be added at the send boundary MUST +remain bound to the current account unless an explicit operation-rebind path +replaces that identity before account selection. Installing a verified fresh +body and clearing its dispatch-owner binding MUST occur as one state +transition. + +#### Scenario: Encrypted reasoning remains on its first dispatch account + +- **GIVEN** account A first receives a Responses request containing encrypted + reasoning or another account-scoped retained item +- **WHEN** a pre-visible retry excludes account A or requests a differently + authorized account +- **THEN** the proxy does not dispatch the retained body to account B +- **AND** the retry fails closed when account A is unavailable + +#### Scenario: Verified account-neutral fresh replay may change accounts + +- **GIVEN** verified recovery removes a stale continuation anchor +- **AND** the exact replacement body contains only canonical account-neutral + fresh input +- **WHEN** normal retry selection chooses account B +- **THEN** the proxy may dispatch the replacement body to account B + +#### Scenario: Confirmed pre-dispatch failure does not create an owner + +- **GIVEN** account A is selected for a nonportable Responses body +- **WHEN** transport evidence confirms the request failed before any upstream + bytes were dispatched +- **THEN** the proxy does not record account A as the dispatch owner +- **AND** normal retry selection may dispatch the body first on account B + +#### Scenario: HTTP bridge preserves payload ownership + +- **GIVEN** an HTTP bridge request has already dispatched a nonportable body to + account A +- **WHEN** pre-created recovery or reconnect selection excludes account A +- **THEN** the bridge does not submit that body on account B + +#### Scenario: Direct WebSocket preserves payload ownership + +- **GIVEN** a direct WebSocket request has already dispatched a nonportable body + to account A +- **WHEN** retry handling prepares an account switch +- **THEN** the proxy rejects the switch unless the exact replacement body is a + canonical account-neutral fresh replay + +#### Scenario: Bound authentication refresh stays on the owner + +- **GIVEN** a nonportable body is bound to account A +- **WHEN** account A reports a refreshable authentication failure before + visible output +- **THEN** the proxy may refresh and replay once on account A +- **AND** it does not dispatch the retained body to account B + +#### Scenario: HTTP bridge operation identity remains on its owner + +- **GIVEN** an HTTP bridge retry retains a proxy-owned operation identity +- **AND** no explicit operation rebind has replaced that identity +- **WHEN** retry selection evaluates another account +- **THEN** the bridge requires the current operation owner + +#### Scenario: Existing settlement ordering is unchanged + +- **GIVEN** an API-key reservation requires settlement during the failed retry +- **WHEN** account health is updated +- **THEN** required settlement still completes before deferred health writes diff --git a/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md new file mode 100644 index 0000000000..53d6b3264a --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-previous-response-replays-to-dispatch-owner/tasks.md @@ -0,0 +1,29 @@ +## 1. Regression coverage + +- [x] 1.1 Add a deterministic HTTP streaming regression proving account-bound + encrypted reasoning never dispatches to a Trusted Access replacement. +- [x] 1.2 Add direct WebSocket regressions for unanchored and verified-fresh + account-bound request bodies. +- [x] 1.3 Add HTTP bridge coverage for owner exclusion and account-neutral + replacement controls. +- [x] 1.4 Add a confirmed pre-dispatch regression proving owner registration + waits for actual upstream dispatch. + +## 2. Owner-fencing implementation + +- [x] 2.1 Route every replay candidate through the canonical account-neutral + fresh-replay predicate. +- [x] 2.2 Bind nonportable HTTP stream payloads to their first dispatch owner + and require that owner during later selections. +- [x] 2.3 Enforce the same binding in HTTP bridge and direct WebSocket account + switching without changing settlement ordering. + +## 3. Verification and publication + +- [x] 3.1 Capture genuine focused RED, implement the minimal owner fence, and + run focused HTTP/bridge/WebSocket tests GREEN. +- [x] 3.2 Run diagnostics, Ruff, typecheck, architecture gates, full affected + tests, and strict affected OpenSpec validation. +- [x] 3.3 Execute an isolated real-surface account-switch scenario proving no + cross-account dispatch and an account-neutral control. +- [x] 3.4 Complete independent review and sync the verified change for archive. diff --git a/openspec/specs/responses-api-compat/context.md b/openspec/specs/responses-api-compat/context.md index e2a75c6ca4..b9e4547395 100644 --- a/openspec/specs/responses-api-compat/context.md +++ b/openspec/specs/responses-api-compat/context.md @@ -195,6 +195,36 @@ retry once with full local history. If the original request already contained a self-contained full resend, codex-lb instead reconnects and replays that body without the rejected anchor. +## Previous-response replay owner fencing + +Removing a stale continuation anchor does not make every retained body +portable. Encrypted reasoning, account-scoped items, file references, and +durable bridge operation identities remain owned by the account that first +received them. The proxy records that dispatch owner and requires it on later +HTTP streaming, HTTP bridge, and direct WebSocket selections. + +For example, if account A first receives encrypted reasoning and then returns a +pre-visible Trusted Access or authentication failure, account B must never +receive the retained ciphertext. One forced token refresh may replay the body +on account A; permanent failure or owner unavailability fails closed. + +Verified recovery installs a replacement body and updates owner state +atomically. A canonical account-neutral replacement clears the owner and may +use normal failover. A verified nonneutral replacement, including a +Responses-Lite full resend, may replay only on the same owner and preserves the +fence. + +HTTP bridge tracing archive IDs do not pin neutral requests. A real durable +`operation_id` does pin the request until an explicit operation-rebind path +replaces that identity. Existing file pins and API-key settlement-before-health +ordering remain independent invariants. + +Streaming selection authorizes owner compatibility before opening upstream, but +persists a new owner only after dispatch is observed. A transport failure that +is positively classified as pre-dispatch therefore leaves the body unowned and +eligible for its first real dispatch on another account. Ambiguous failures +remain owner-bound. + ## Known Client Integrations (Reference) Third-party agents that consume the `/v1` Responses surface documented by this diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index b201e659e9..3bf0552710 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -5342,3 +5342,94 @@ Responses-compatible routes MUST accept the canonical `ultrafast` service tier a - **WHEN** upstream completes a request with `response.service_tier: "ultrafast"` - **THEN** the actual and billable request-log tiers are `ultrafast` + +### Requirement: Account-bound retries remain on their dispatch owner + +The proxy MUST bind a Responses request body that is not a canonical +account-neutral fresh replay to the account that first receives that exact +body. Every later selection for that request MUST treat the dispatch owner as a +strict required account across HTTP streaming, HTTP bridge, and direct +WebSocket transports. + +The proxy MUST NOT exclude the dispatch owner and send the retained body to a +different account during stale-anchor recovery, retryable account failure, +Trusted Access migration or degradation, bridge reconnect, or WebSocket account +switching. If the required owner is unavailable, the proxy MUST fail closed +without dispatching the retained body to another account. + +The proxy MAY perform one forced authentication refresh and replay a retained +account-bound body on the same dispatch owner. It MUST NOT use that refresh to +exclude the owner or migrate the body to another account, and a permanent +authentication failure MUST remain terminal for the bound body. + +The proxy MAY clear the dispatch-owner binding only after verified recovery +replaces the exact wire body and the replacement passes the canonical +account-neutral-fresh-replay predicate. Removing `previous_response_id` alone +MUST NOT make retained account-scoped input portable. + +Proxy-owned operation metadata that will be added at the send boundary MUST +remain bound to the current account unless an explicit operation-rebind path +replaces that identity before account selection. Installing a verified fresh +body and clearing its dispatch-owner binding MUST occur as one state +transition. + +#### Scenario: Encrypted reasoning remains on its first dispatch account + +- **GIVEN** account A first receives a Responses request containing encrypted + reasoning or another account-scoped retained item +- **WHEN** a pre-visible retry excludes account A or requests a differently + authorized account +- **THEN** the proxy does not dispatch the retained body to account B +- **AND** the retry fails closed when account A is unavailable + +#### Scenario: Verified account-neutral fresh replay may change accounts + +- **GIVEN** verified recovery removes a stale continuation anchor +- **AND** the exact replacement body contains only canonical account-neutral + fresh input +- **WHEN** normal retry selection chooses account B +- **THEN** the proxy may dispatch the replacement body to account B + +#### Scenario: Confirmed pre-dispatch failure does not create an owner + +- **GIVEN** account A is selected for a nonportable Responses body +- **WHEN** transport evidence confirms the request failed before any upstream + bytes were dispatched +- **THEN** the proxy does not record account A as the dispatch owner +- **AND** normal retry selection may dispatch the body first on account B + +#### Scenario: HTTP bridge preserves payload ownership + +- **GIVEN** an HTTP bridge request has already dispatched a nonportable body to + account A +- **WHEN** pre-created recovery or reconnect selection excludes account A +- **THEN** the bridge does not submit that body on account B + +#### Scenario: Direct WebSocket preserves payload ownership + +- **GIVEN** a direct WebSocket request has already dispatched a nonportable body + to account A +- **WHEN** retry handling prepares an account switch +- **THEN** the proxy rejects the switch unless the exact replacement body is a + canonical account-neutral fresh replay + +#### Scenario: Bound authentication refresh stays on the owner + +- **GIVEN** a nonportable body is bound to account A +- **WHEN** account A reports a refreshable authentication failure before + visible output +- **THEN** the proxy may refresh and replay once on account A +- **AND** it does not dispatch the retained body to account B + +#### Scenario: HTTP bridge operation identity remains on its owner + +- **GIVEN** an HTTP bridge retry retains a proxy-owned operation identity +- **AND** no explicit operation rebind has replaced that identity +- **WHEN** retry selection evaluates another account +- **THEN** the bridge requires the current operation owner + +#### Scenario: Existing settlement ordering is unchanged + +- **GIVEN** an API-key reservation requires settlement during the failed retry +- **WHEN** account health is updated +- **THEN** required settlement still completes before deferred health writes diff --git a/tests/integration/test_proxy_websocket_responses.py b/tests/integration/test_proxy_websocket_responses.py index 70c1e2e265..16c5ee2e1e 100644 --- a/tests/integration/test_proxy_websocket_responses.py +++ b/tests/integration/test_proxy_websocket_responses.py @@ -4409,6 +4409,8 @@ def test_v1_responses_websocket_reuses_upstream_for_sequential_requests(app_inst ], ) connect_calls: list[dict[str, object]] = [] + dispatch_owner_snapshots: list[tuple[str | None, str | None]] = [] + original_bind_dispatch_owner = websocket_mixin_module._bind_websocket_request_dispatch_owner class _FakeSettingsCache: async def get(self): @@ -4449,7 +4451,18 @@ async def fake_connect_proxy_websocket( "model": model, } ) - return SimpleNamespace(id=f"acct_ws_proxy_{len(connect_calls)}"), first_upstream + return SimpleNamespace(id="acct_ws_proxy_owner"), first_upstream + + def capture_dispatch_owner(*args, **kwargs): + bound = original_bind_dispatch_owner(*args, **kwargs) + request_state = args[0] if args else kwargs["request_state"] + dispatch_owner_snapshots.append( + ( + request_state.preferred_account_id, + request_state.replay_required_account_id, + ) + ) + return bound async def fake_write_request_log(self, **kwargs): del self, kwargs @@ -4459,6 +4472,11 @@ async def fake_write_request_log(self, **kwargs): monkeypatch.setattr(proxy_module, "get_settings_cache", lambda: _FakeSettingsCache()) monkeypatch.setattr(proxy_module.ProxyService, "_connect_proxy_websocket", fake_connect_proxy_websocket) monkeypatch.setattr(proxy_module.ProxyService, "_write_request_log", fake_write_request_log) + monkeypatch.setattr( + websocket_mixin_module, + "_bind_websocket_request_dispatch_owner", + capture_dispatch_owner, + ) first_request = { "type": "response.create", @@ -4471,6 +4489,7 @@ async def fake_write_request_log(self, **kwargs): "type": "response.create", "model": "gpt-5.5", "input": "second", + "account_bound_probe": "owner-bound", "promptCacheKey": "thread_b", "stream": True, } @@ -4489,6 +4508,10 @@ async def fake_write_request_log(self, **kwargs): assert connect_calls[0]["sticky_key"] == "thread_a" assert connect_calls[0]["sticky_kind"] == proxy_module.StickySessionKind.PROMPT_CACHE assert connect_calls[0]["model"] == "gpt-5.4" + assert dispatch_owner_snapshots == [ + (None, None), + ("acct_ws_proxy_owner", "acct_ws_proxy_owner"), + ] _assert_upstream_payloads( first_upstream.sent_text, [ @@ -4505,6 +4528,7 @@ async def fake_write_request_log(self, **kwargs): "model": "gpt-5.5", "instructions": "", "input": [{"role": "user", "content": [{"type": "input_text", "text": "second"}]}], + "account_bound_probe": "owner-bound", "store": False, "include": [], "prompt_cache_key": "thread_b", diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index b8d03ed5a2..665eff1219 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -479,6 +479,150 @@ def test_websocket_account_switch_keeps_anchor_when_fresh_replay_references_file assert request_state.preferred_account_id == "acc_file_owner" +def test_websocket_account_switch_blocks_unanchored_account_bound_request(): + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_account_bound", + model="gpt-5.6-sol", + service_tier="priority", + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + preferred_account_id="acc_owner", + ) + + assert websocket_mixin._prepare_websocket_request_state_for_account_switch(request_state) is None + assert request_state.preferred_account_id == "acc_owner" + + +def test_websocket_account_switch_blocks_account_bound_fresh_replay(): + fresh_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"role":"user","content":"hello"},' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"},' + '{"type":"function_call","name":"lookup","call_id":"call_1","arguments":"{}"},' + '{"type":"function_call_output","call_id":"call_1","output":"ok"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_account_bound_fresh", + model="gpt-5.6-sol", + service_tier="priority", + reasoning_effort="high", + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","previous_response_id":"resp_proxy"}', + previous_response_id="resp_proxy", + preferred_account_id="acc_owner", + proxy_injected_previous_response_id=True, + fresh_upstream_request_is_retry_safe=True, + fresh_upstream_request_text=fresh_text, + ) + + assert websocket_mixin._prepare_websocket_request_state_for_account_switch(request_state) is None + assert request_state.previous_response_id == "resp_proxy" + assert request_state.preferred_account_id == "acc_owner" + + +def test_websocket_dispatch_owner_rejects_account_bound_socket_reuse(): + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_dispatch_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text=request_text, + ) + + assert websocket_mixin._bind_websocket_request_dispatch_owner( + request_state, + account_id="acc_owner", + exact_request_text=request_text, + ) + assert not websocket_mixin._bind_websocket_request_dispatch_owner( + request_state, + account_id="acc_other", + exact_request_text=request_text, + ) + assert request_state.preferred_account_id == "acc_owner" + assert request_state.replay_required_account_id == "acc_owner" + + +def test_websocket_verified_fresh_replay_clears_dispatch_owner_atomically(): + fresh_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"portable user input"}' + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_verified_fresh_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text='{"type":"response.create","previous_response_id":"resp_proxy"}', + previous_response_id="resp_proxy", + preferred_account_id="acc_owner", + replay_required_account_id="acc_owner", + proxy_injected_previous_response_id=True, + fresh_upstream_request_is_retry_safe=True, + fresh_upstream_request_text=fresh_text, + ) + + assert websocket_mixin._install_verified_fresh_replay(request_state) == fresh_text + assert request_state.request_text == fresh_text + assert request_state.previous_response_id is None + assert request_state.preferred_account_id is None + assert request_state.replay_required_account_id is None + + +def test_websocket_bound_auth_replay_allows_one_same_owner_refresh(): + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="req_ws_bound_auth_refresh", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + request_text=request_text, + preferred_account_id="acc_owner", + replay_required_account_id="acc_owner", + ) + + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_other", + ) + is None + ) + assert request_state.auth_replay_count == 0 + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_owner", + ) + == request_text + ) + assert request_state.auth_replay_count == 1 + assert request_state.replay_required_account_id == "acc_owner" + assert ( + websocket_mixin._prepare_websocket_request_state_for_auth_replay( + request_state, + current_account_id="acc_owner", + ) + is None + ) + + def test_websocket_owner_switch_detects_other_pending_request() -> None: current = proxy_service._WebSocketRequestState( request_id="req_owner_switch", @@ -17248,6 +17392,134 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert authorized_lease in released_leases +@pytest.mark.asyncio +async def test_stream_responses_account_bound_pre_dispatch_failure_retries_other_account(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + first_account = _make_account("acc_pre_dispatch_bound_first") + second_account = _make_account("acc_pre_dispatch_bound_second") + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=first_account, error_message=None), + AccountSelection(account=second_account, error_message=None), + ] + ) + attempted_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, base_url, raise_for_status + attempted_account_ids.append(account_id) + if account_id == first_account.chatgpt_account_id: + raise _pre_dispatch_proxy_connect_error("first bound account proxy route unavailable") + yield 'data: {"type":"response.completed","response":{"id":"resp_bound_fallback"}}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(side_effect=lambda account, **kwargs: account)) + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "", + "input": [ + { + "type": "reasoning", + "id": "rs_pre_dispatch_owner", + "encrypted_content": "owner-bound", + } + ], + "stream": True, + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-pre-dispatch"})] + + assert select_account.await_count == 2 + assert attempted_account_ids == [ + first_account.chatgpt_account_id, + second_account.chatgpt_account_id, + ] + assert any("resp_bound_fallback" in chunk for chunk in chunks) + + +@pytest.mark.asyncio +async def test_stream_responses_refuses_account_bound_security_work_retry(monkeypatch): + settings = _make_proxy_settings() + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + regular_account = _make_account("acc_regular_security_account_bound") + authorized_account = _make_account("acc_authorized_security_account_bound") + authorized_account.security_work_authorized = True + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=regular_account, error_message=None), + AccountSelection(account=authorized_account, error_message=None), + ] + ) + cyber_message = ( + "This chat was flagged for possible cybersecurity risk. " + "If this seems wrong, try rephrasing your request. " + "To get authorized for security work, join the Trusted Access for Cyber program. " + "https://chatgpt.com/cyber" + ) + dispatched_account_ids: list[str] = [] + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, base_url, raise_for_status + dispatched_account_ids.append(account_id) + if account_id == regular_account.chatgpt_account_id: + yield ( + "data: " + + json.dumps( + { + "type": "response.failed", + "response": { + "id": "resp_cyber_account_bound", + "error": { + "code": "invalid_request_error", + "type": "invalid_request_error", + "message": cyber_message, + }, + }, + } + ) + + "\n\n" + ) + return + yield 'data: {"type":"response.completed","response":{"id":"resp_cross_account"}}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings_cache", lambda: _SettingsCache(settings)) + monkeypatch.setattr(proxy_service, "get_settings", lambda: settings) + monkeypatch.setattr(service._load_balancer, "select_account", select_account) + monkeypatch.setattr(service._load_balancer, "record_error", AsyncMock()) + monkeypatch.setattr(service._load_balancer, "record_success", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(side_effect=lambda account, **kwargs: account)) + monkeypatch.setattr(proxy_service, "core_stream_responses", fake_stream) + + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.1", + "instructions": "check api", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + } + ], + "stream": True, + } + ) + + chunks = [chunk async for chunk in service.stream_responses(payload, {"session_id": "sid-stream"})] + + assert dispatched_account_ids == [regular_account.chatgpt_account_id] + assert all("resp_cross_account" not in chunk for chunk in chunks) + + @pytest.mark.asyncio async def test_stream_responses_treats_missing_security_work_pool_as_optional(monkeypatch): settings = _make_proxy_settings() @@ -17737,6 +18009,63 @@ async def fake_reconnect_http_bridge_session( assert request_state.event_queue.empty() +@pytest.mark.asyncio +async def test_http_bridge_refuses_account_bound_security_work_retry(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + regular_account = _make_account("acc_bridge_security_account_bound") + request_text = json.dumps( + { + "type": "response.create", + "model": "gpt-5.1", + "input": [ + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + } + ], + }, + separators=(",", ":"), + ) + request_state = proxy_service._WebSocketRequestState( + request_id="bridge_req_security_account_bound", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=1.0, + awaiting_response_created=True, + event_queue=asyncio.Queue(), + transport="http", + request_text=request_text, + preferred_account_id=regular_account.id, + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("turn_state_header", "turn-security-owner", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.1", + account=regular_account, + upstream=AsyncMock(), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=1.0, + idle_ttl_seconds=300.0, + ) + reconnect = AsyncMock() + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_security_work_request(session, request_state) is False + + reconnect.assert_not_awaited() + assert request_state.preferred_account_id == regular_account.id + assert request_state.excluded_account_ids == set() + assert request_state.request_text == request_text + + @pytest.mark.parametrize( ("item_type", "expected_deferred"), [ @@ -18181,6 +18510,83 @@ async def test_http_bridge_recovery_auth_reconnect_failure_preserves_original_au assert await request_state.event_queue.get() is None +@pytest.mark.asyncio +async def test_retry_http_bridge_bound_auth_refresh_never_sends_on_other_account(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + owner = _make_account("acc_bridge_bound_auth_owner") + other = _make_account("acc_bridge_bound_auth_other") + request_text = ( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ) + request_state = proxy_service._WebSocketRequestState( + request_id="bridge_bound_auth_refresh", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=request_text, + preferred_account_id=owner.id, + replay_required_account_id=owner.id, + ) + owner_upstream = AsyncMock() + other_upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-bound-auth", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=owner, + upstream=owner_upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + + async def adversarial_reconnect( + reconnect_session, + *, + request_state, + require_same_account=False, + require_preferred_account=False, + **kwargs, + ): + del request_state, kwargs + if not (require_same_account and require_preferred_account): + reconnect_session.account = other + reconnect_session.upstream = other_upstream + return + raise proxy_module.ProxyResponseError( + 502, + openai_error( + "previous_response_owner_unavailable", + "Request payload owner account is unavailable; retry later.", + error_type="server_error", + ), + ) + + monkeypatch.setattr(service, "_reconnect_http_bridge_session", adversarial_reconnect) + + result = await service._retry_http_bridge_precreated_auth_request( + session, + request_state, + error_message="Authentication failed", + ) + + assert result == "failed" + assert session.account is owner + owner_upstream.send_text.assert_not_awaited() + other_upstream.send_text.assert_not_awaited() + assert request_state.replay_required_account_id == owner.id + + @pytest.mark.asyncio async def test_http_bridge_keeps_previous_response_pinned_security_work_error(monkeypatch): request_logs = _RequestLogsRecorder() @@ -20093,6 +20499,68 @@ async def select_account(deadline: float, **kwargs: object) -> AccountSelection: assert request_logs.calls[0]["account_id"] == account_owner.id +@pytest.mark.asyncio +async def test_connect_proxy_websocket_account_bound_replay_stays_on_owner(monkeypatch): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + account_owner = _make_account("acc_ws_replay_owner") + account_other = _make_account("acc_ws_replay_other") + select_account = AsyncMock( + side_effect=[ + AccountSelection(account=account_owner, error_message=None), + AccountSelection(account=account_other, error_message=None), + ] + ) + handshake_error = proxy_module.ProxyResponseError( + 429, + openai_error("usage_limit_reached", "usage limit reached"), + ) + monkeypatch.setattr(service, "_select_account_with_budget", select_account) + monkeypatch.setattr(service._load_balancer, "mark_rate_limit", AsyncMock()) + monkeypatch.setattr(service, "_ensure_fresh", AsyncMock(return_value=account_owner)) + open_upstream = AsyncMock(side_effect=[handshake_error]) + monkeypatch.setattr(service, "_open_upstream_websocket", open_upstream) + monkeypatch.setattr(service, "_release_websocket_reservation", AsyncMock()) + + request_state = proxy_service._WebSocketRequestState( + request_id="ws_req_account_bound_replay", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + request_text=( + '{"type":"response.create","model":"gpt-5.1","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + ) + websocket_send = AsyncMock() + + selected_account, selected_upstream = await service._connect_proxy_websocket( + {}, + sticky_key=None, + sticky_kind=None, + prefer_earlier_reset=False, + prefer_earlier_reset_window="secondary", + routing_strategy="usage_weighted", + model="gpt-5.1", + request_state=request_state, + api_key=None, + client_send_lock=anyio.Lock(), + websocket=cast(WebSocket, SimpleNamespace(send_text=websocket_send)), + ) + + assert selected_account is None + assert selected_upstream is None + assert select_account.await_count == 2 + assert select_account.await_args_list[1].kwargs["preferred_account_id"] == account_owner.id + open_upstream.assert_awaited_once() + websocket_send_args = websocket_send.await_args + assert websocket_send_args is not None + sent_payload = json.loads(websocket_send_args.args[0]) + assert sent_payload["error"]["code"] == "previous_response_owner_unavailable" + + @pytest.mark.asyncio async def test_connect_proxy_websocket_surfaces_local_connect_overload_without_penalizing_account(monkeypatch): settings = _make_proxy_settings() @@ -36273,6 +36741,51 @@ def test_cross_transport_fresh_replay_requires_matching_ws_continuity_prefix(): assert fresh.input == full_input +def test_cross_transport_fresh_replay_rejects_account_bound_payload(): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + first_input: list[JsonValue] = [ + {"role": "user", "content": [{"type": "input_text", "text": "call echo"}]}, + ] + full_input: list[JsonValue] = [ + *first_input, + { + "type": "reasoning", + "id": "rs_owner", + "encrypted_content": "owner-bound", + }, + { + "type": "function_call", + "name": "echo", + "call_id": "call_1", + "arguments": '{"value":"ok"}', + }, + {"type": "function_call_output", "call_id": "call_1", "output": "ok"}, + ] + service._websocket_continuity_index[("turn_generated_by_ws", None)] = proxy_service._WebSocketContinuityState( + last_completed_response_id="resp_ws_owner", + last_completed_input_count=len(first_input), + last_completed_input_prefix_fingerprint=proxy_service._fingerprint_input_items(first_input), + ) + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "test", + "previous_response_id": "resp_ws_owner", + "input": full_input, + } + ) + + assert ( + streaming_retry_module._verified_cross_transport_fresh_replay( + cast(Any, service), + payload=payload, + headers={"x-codex-session-id": "sid-cross-transport"}, + api_key=None, + ) + is None + ) + + def test_cross_transport_fresh_replay_rejects_unverified_client_full_resend(): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) payload = ResponsesRequest.model_validate( @@ -44793,6 +45306,171 @@ async def test_retry_http_bridge_precreated_request_migrates_only_safe_initial_t upstream.send_text.assert_awaited_once_with(request_state.request_text) +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_keeps_account_bound_body_on_owner(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_account_bound") + request_state = proxy_service._WebSocketRequestState( + request_id="req_bridge_account_bound", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=( + '{"type":"response.create","model":"gpt-5.6-sol","input":[' + '{"type":"reasoning","id":"rs_owner","encrypted_content":"owner-bound"}]}' + ), + preferred_account_id=account.id, + ) + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-account-bound", None), + headers={"x-codex-turn-state": "turn_state_owner"}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + upstream_turn_state="turn_state_owner", + downstream_turn_state="turn_state_owner", + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + require_same_account=True, + require_preferred_account=True, + ) + assert request_state.preferred_account_id == account.id + assert request_state.excluded_account_ids == set() + assert session.upstream_turn_state == "turn_state_owner" + upstream.send_text.assert_awaited_once_with(request_state.request_text) + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_keeps_operation_id_on_owner(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_operation_owner") + request_text = '{"type":"response.create","model":"gpt-5.6-sol","input":"portable user input"}' + request_state = proxy_service._WebSocketRequestState( + request_id="req_bridge_operation_owner", + model="gpt-5.6-sol", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + awaiting_response_created=True, + transport="http", + request_text=request_text, + preferred_account_id=account.id, + archive_request_id="archive_bridge_operation_owner", + operation_id="op_bridge_owner", + ) + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-operation-owner", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + require_same_account=True, + require_preferred_account=True, + ) + assert request_state.excluded_account_ids == set() + assert request_state.operation_id == "op_bridge_owner" + upstream.send_text.assert_awaited_once() + send_args = upstream.send_text.await_args + assert send_args is not None + assert json.loads(send_args.args[0]) == { + "type": "response.create", + "model": "gpt-5.6-sol", + "input": "portable user input", + "client_metadata": {"codex_lb_operation_id": "op_bridge_owner"}, + } + + +@pytest.mark.asyncio +async def test_retry_http_bridge_precreated_request_allows_prepared_neutral_archive(monkeypatch): + service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) + account = _make_account("acc_bridge_prepared_neutral") + payload = ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "", + "input": "portable user input", + "stream": True, + } + ) + request_state, request_text = service._prepare_response_bridge_request_state( + payload, + api_key=None, + api_key_reservation=None, + include_type_field=True, + attach_event_queue=False, + transport=proxy_service._REQUEST_TRANSPORT_HTTP, + client_metadata=None, + ) + request_state.preferred_account_id = account.id + upstream = AsyncMock() + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-prepared-neutral", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.6-sol", + account=account, + upstream=upstream, + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + reconnect = AsyncMock(return_value=None) + monkeypatch.setattr(service, "_reconnect_http_bridge_session", reconnect) + + assert request_state.archive_request_id is not None + assert request_state.operation_id is None + assert await service._retry_http_bridge_precreated_request(session) is True + + reconnect.assert_awaited_once_with( + session, + request_state=request_state, + ) + upstream.send_text.assert_awaited_once_with(request_text) + + @pytest.mark.asyncio async def test_retry_http_bridge_precreated_request_keeps_hard_session_owner_bound(monkeypatch): service = proxy_service.ProxyService(_repo_factory(_RequestLogsRecorder())) From 6ba083d7df4f82c2d2e6aa08e9a1e1fb47b59faa Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 20 Aug 2026 07:04:16 +0200 Subject: [PATCH 085/117] fix(proxy): reject truncated chat completion streams (#1833) * fix(proxy): reject truncated chat completion streams * fix(proxy): preserve terminal stream errors * fix(proxy): normalize empty stream errors --- app/core/openai/chat_responses.py | 33 +++++-- .../.openspec.yaml | 2 + .../design.md | 79 +++++++++++++++++ .../proposal.md | 34 +++++++ .../specs/chat-completions-compat/spec.md | 38 ++++++++ .../tasks.md | 23 +++++ .../test_proxy_chat_completions.py | 88 +++++++++++++++++++ tests/unit/test_chat_response_mapping.py | 57 ++++++++++++ 8 files changed, 349 insertions(+), 5 deletions(-) create mode 100644 openspec/changes/reject-truncated-chat-completions/.openspec.yaml create mode 100644 openspec/changes/reject-truncated-chat-completions/design.md create mode 100644 openspec/changes/reject-truncated-chat-completions/proposal.md create mode 100644 openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md create mode 100644 openspec/changes/reject-truncated-chat-completions/tasks.md diff --git a/app/core/openai/chat_responses.py b/app/core/openai/chat_responses.py index 73e48e21ae..78ad429dff 100644 --- a/app/core/openai/chat_responses.py +++ b/app/core/openai/chat_responses.py @@ -353,17 +353,19 @@ def iter_chat_chunks( response = payload.get("response") if isinstance(response, dict): maybe_error = response.get("error") - if isinstance(maybe_error, dict): + if isinstance(maybe_error, dict) and maybe_error: error = maybe_error else: maybe_error = payload.get("error") - if isinstance(maybe_error, dict): + if isinstance(maybe_error, dict) and maybe_error: error = maybe_error if error is not None: error_payload: dict[str, JsonValue] = {"error": error} - yield _dump_sse(error_payload) - yield "data: [DONE]\n\n" - return + else: + error_payload = _default_error_envelope().model_dump(mode="json", exclude_none=True) + yield _dump_sse(error_payload) + yield "data: [DONE]\n\n" + return if event_type in ("response.completed", "response.incomplete"): for tool_state in state.tool_calls: stream_delta = tool_state.build_stream_delta() @@ -448,6 +450,9 @@ async def stream_chat_chunks( if chunk.strip() == "data: [DONE]": terminal_chunk_sent = True break + if not terminal_chunk_sent: + yield _dump_sse(_upstream_stream_truncated_error_payload()) + yield "data: [DONE]\n\n" async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> ChatCompletionResult: @@ -460,6 +465,7 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha tool_index = ToolCallIndex() tool_calls: list[ToolCallState] = [] terminal_error: ChatCompletionResult | None = None + terminal_event_seen = False async for line in stream: payload = _parse_data(line) @@ -496,6 +502,7 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha if terminal_error is not None: continue if event_type in ("response.completed", "response.incomplete"): + terminal_event_seen = True response = payload.get("response") if isinstance(response, dict): response_id_value = response.get("id") @@ -507,6 +514,8 @@ async def collect_chat_completion(stream: AsyncIterator[str], model: str) -> Cha if terminal_error is not None: return terminal_error + if not terminal_event_seen: + return _upstream_stream_truncated_error() message_content: str | None = "".join(content_parts) message_refusal = "".join(refusal_parts) or None @@ -582,6 +591,20 @@ def _dump_sse(payload: dict[str, JsonValue]) -> str: return format_sse_data(payload) +def _upstream_stream_truncated_error_payload() -> dict[str, JsonValue]: + return { + "error": { + "message": "Responses stream ended before a terminal event", + "type": "server_error", + "code": "upstream_stream_truncated", + } + } + + +def _upstream_stream_truncated_error() -> OpenAIErrorEnvelope: + return OpenAIErrorEnvelope.model_validate(_upstream_stream_truncated_error_payload()) + + def _finish_reason_from_incomplete(response: JsonValue | None) -> str: response_mapping = _as_mapping(response) if response_mapping is None: diff --git a/openspec/changes/reject-truncated-chat-completions/.openspec.yaml b/openspec/changes/reject-truncated-chat-completions/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/reject-truncated-chat-completions/design.md b/openspec/changes/reject-truncated-chat-completions/design.md new file mode 100644 index 0000000000..b493ebc75f --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/design.md @@ -0,0 +1,79 @@ +## Context + +`POST /v1/chat/completions` always asks the upstream Responses client for an SSE +iterator. The adapter converts that iterator either into Chat Completions SSE or +into a collected JSON response. Explicit terminal events are already mapped, +but natural iterator exhaustion is not tracked. This leaves the streaming +protocol unterminated and lets the collected path synthesize +`finish_reason=stop`. + +The public `/v1/responses` path already uses `upstream_stream_truncated`, +`server_error`, and HTTP 502 for EOF before a terminal event. The Chat adapter +must match those machine-consumed semantics without depending on the proxy API +module. + +## Goals / Non-Goals + +**Goals:** + +- Detect whether a terminal upstream Responses event was observed. +- Use the canonical `upstream_stream_truncated` code and `server_error` type. +- Finish streaming Chat errors with `data: [DONE]`. +- Keep explicit completion, incomplete, failure, and generator-cleanup behavior + unchanged. + +**Non-Goals:** + +- Change public `/v1/responses` normalization. +- Change retry, account selection, keepalive, or reservation policy. +- Convert explicit `response.incomplete` into a transport error. +- Refactor unrelated Chat payload or tool-call mapping. + +## Decisions + +### Decision: detect truncation at the Chat adapter boundary + +The adapter is the first layer that knows whether it observed a Chat-relevant +terminal Responses event. `stream_chat_chunks` will synthesize the error chunk +and `[DONE]` only when its mapped iterator exhausts without a terminal marker. +`collect_chat_completion` will return the equivalent error envelope before it +assembles a `ChatCompletion`. + +This keeps the behavior correct for both the subscription route and any other +caller of the adapter without changing the public Responses pipeline. + +### Decision: preserve canonical machine semantics + +The synthesized error uses: + +- code: `upstream_stream_truncated` +- type: `server_error` +- message: `Responses stream ended before a terminal event` + +The existing route-level `_status_for_error` fallback maps that envelope to +HTTP 502. No Chat-specific status policy is added. + +### Decision: leave explicit incomplete events successful + +`response.incomplete` is a terminal event with a meaningful finish reason such +as `length` or `content_filter`. It remains a Chat completion. Only EOF without +any terminal event is classified as transport truncation. + +## Risks / Trade-offs + +- A caller that previously relied on partial EOF content will now receive a + retriable error. That is intentional because presenting partial output as + complete is contract-breaking and suppresses retries. +- Streaming may already have emitted partial content before the error. The + terminal error chunk and `[DONE]` make that state explicit without retracting + bytes already delivered. + +## Test Strategy + +- Unit-test streaming delta-then-EOF for error + `[DONE]`. +- Unit-test collected delta-then-EOF for the canonical error envelope. +- Route-test non-streaming HTTP status and error code. +- Manually drive streaming and non-streaming ASGI requests with an inert + delta-then-EOF upstream. +- Keep existing explicit completion, incomplete, error, usage, tool-call, and + generator-close tests green. diff --git a/openspec/changes/reject-truncated-chat-completions/proposal.md b/openspec/changes/reject-truncated-chat-completions/proposal.md new file mode 100644 index 0000000000..97ec561b1f --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/proposal.md @@ -0,0 +1,34 @@ +## Why + +The Chat Completions adapter currently treats an upstream Responses iterator +that reaches EOF without a terminal event as success. Streaming callers receive +content without the required `data: [DONE]` marker, while non-streaming callers +receive a successful `chat.completion` whose partial text has +`finish_reason=stop`. The public Responses adapter already classifies the same +condition as `upstream_stream_truncated`. + +## What Changes + +- Detect upstream EOF before any `response.completed`, + `response.incomplete`, `response.failed`, or `error` event. +- Emit an OpenAI error chunk followed by `data: [DONE]` for streaming Chat + Completions. +- Return an OpenAI error envelope that maps to HTTP 502 for non-streaming Chat + Completions. +- Preserve explicit terminal/error handling, usage/tool-call finalization, and + upstream generator cleanup. + +## Capabilities + +### Modified Capabilities + +- `chat-completions-compat`: define deterministic truncation behavior for + streaming and collected Chat Completions. + +## Impact + +- Affected code: `app/core/openai/chat_responses.py` +- Affected route: `POST /v1/chat/completions` +- Affected tests: Chat response mapping and proxy Chat Completions integration +- Compatibility: malformed upstream termination changes from false success to a + stable OpenAI server-error envelope diff --git a/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md b/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md new file mode 100644 index 0000000000..c9b7a66737 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/specs/chat-completions-compat/spec.md @@ -0,0 +1,38 @@ +## ADDED Requirements + +### Requirement: Chat Completions reject truncated upstream Responses streams + +`POST /v1/chat/completions` MUST classify upstream Responses iterator +exhaustion before a terminal `response.completed`, `response.incomplete`, +`response.failed`, or `error` event as `upstream_stream_truncated`. The error +MUST use OpenAI error type `server_error`. Partial content received before the +exhaustion MUST NOT be presented as a successfully completed non-streaming Chat +Completion. + +#### Scenario: Streaming upstream EOF emits error and done + +- **WHEN** a streaming Chat Completions request receives zero or more + non-terminal upstream Responses events +- **AND** the upstream iterator reaches EOF before a terminal event +- **THEN** the proxy MUST emit an OpenAI error chunk with code + `upstream_stream_truncated` +- **AND** the proxy MUST terminate the stream with `data: [DONE]` + +#### Scenario: Collected upstream EOF returns an error envelope + +- **WHEN** a non-streaming Chat Completions request receives zero or more + non-terminal upstream Responses events +- **AND** the upstream iterator reaches EOF before a terminal event +- **THEN** the proxy MUST return HTTP 502 +- **AND** the response body MUST be an OpenAI error envelope with code + `upstream_stream_truncated` and type `server_error` +- **AND** the proxy MUST NOT return a `chat.completion` success object + +#### Scenario: Explicit terminal events retain existing behavior + +- **WHEN** the upstream iterator emits `response.completed`, + `response.incomplete`, `response.failed`, or `error` +- **THEN** the proxy MUST preserve the existing Chat Completions mapping for + that event +- **AND** the proxy MUST preserve existing usage, tool-call, and upstream + generator cleanup behavior diff --git a/openspec/changes/reject-truncated-chat-completions/tasks.md b/openspec/changes/reject-truncated-chat-completions/tasks.md new file mode 100644 index 0000000000..6fbf870873 --- /dev/null +++ b/openspec/changes/reject-truncated-chat-completions/tasks.md @@ -0,0 +1,23 @@ +## 1. Specification + +- [x] 1.1 Define streaming and non-streaming EOF truncation requirements. +- [x] 1.2 Validate the scoped OpenSpec change. + +## 2. Regression Coverage + +- [x] 2.1 Add a streaming adapter regression for error chunk plus `[DONE]`. +- [x] 2.2 Add a collected adapter regression for the canonical error envelope. +- [x] 2.3 Add a non-streaming route regression for HTTP 502 and + `upstream_stream_truncated`. + +## 3. Implementation + +- [x] 3.1 Track terminal event observation in the Chat adapter. +- [x] 3.2 Synthesize canonical truncation errors without changing explicit + terminal behavior. + +## 4. Verification + +- [x] 4.1 Run focused Chat adapter and route tests. +- [x] 4.2 Manually verify streaming and non-streaming HTTP surfaces. +- [x] 4.3 Run lint, type, diagnostic, and strict OpenSpec gates. diff --git a/tests/integration/test_proxy_chat_completions.py b/tests/integration/test_proxy_chat_completions.py index 3dff038c07..f50724af29 100644 --- a/tests/integration/test_proxy_chat_completions.py +++ b/tests/integration/test_proxy_chat_completions.py @@ -60,6 +60,61 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert any("chat.completion.chunk" in line for line in lines) +@pytest.mark.asyncio +async def test_v1_chat_completions_stream_truncated_eof_emits_error_and_done(async_client, monkeypatch): + # #given + email = "chat-stream-truncated@example.com" + raw_account_id = "acc_chat_stream_truncated" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + # #when + payload = {"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}], "stream": True} + async with async_client.stream("POST", "/v1/chat/completions", json=payload) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line] + + # #then + assert json.loads(lines[-2][len("data: ") :])["error"]["code"] == "upstream_stream_truncated" + assert lines[-1] == "data: [DONE]" + + +@pytest.mark.asyncio +async def test_v1_chat_completions_stream_terminal_error_without_payload_uses_default_error( + async_client, + monkeypatch, +): + email = "chat-stream-terminal-error@example.com" + raw_account_id = "acc_chat_stream_terminal_error" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"partial"}\n\n' + yield 'data: {"type":"response.failed","response":{"id":"r1","status":"failed"}}\n\n' + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + payload = {"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}], "stream": True} + async with async_client.stream("POST", "/v1/chat/completions", json=payload) as response: + assert response.status_code == 200 + lines = [line async for line in response.aiter_lines() if line] + + assert json.loads(lines[-2][len("data: ") :])["error"]["code"] == "upstream_error" + assert lines[-1] == "data: [DONE]" + + @pytest.mark.asyncio async def test_v1_chat_completions_omits_synthesized_tools(async_client, monkeypatch): email = "chatnotools@example.com" @@ -211,6 +266,39 @@ async def fake_stream(payload, headers, access_token, account_id, base_url=None, assert body["object"] == "chat.completion" +@pytest.mark.asyncio +async def test_v1_chat_completions_non_stream_truncated_eof_returns_502(async_client, monkeypatch): + # #given + email = "chat-nonstr-truncated@example.com" + raw_account_id = "acc_chat_nonstr_truncated" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + imported = await async_client.post("/api/accounts/import", files=files) + assert imported.status_code == 200 + + async def passthrough_probe(stream, **_kwargs): + return stream, None + + async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False): + del payload, headers, access_token, account_id, base_url, raise_for_status + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + monkeypatch.setattr(proxy_api, "_probe_chat_stream_startup_error", passthrough_probe) + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + + # #when + response = await async_client.post( + "/v1/chat/completions", + json={"model": "gpt-5.2", "messages": [{"role": "user", "content": "hi"}]}, + ) + + # #then + assert response.status_code == 502 + body = response.json() + assert body["error"]["code"] == "upstream_stream_truncated" + assert body["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_v1_chat_completions_non_stream_rate_limit_closes_stream_and_returns_429(async_client, monkeypatch): # #given diff --git a/tests/unit/test_chat_response_mapping.py b/tests/unit/test_chat_response_mapping.py index e08063b20a..0c32197214 100644 --- a/tests/unit/test_chat_response_mapping.py +++ b/tests/unit/test_chat_response_mapping.py @@ -74,6 +74,47 @@ def test_error_event_emits_done_chunk(): assert chunks[-1].strip() == "data: [DONE]" +@pytest.mark.parametrize( + "event_line", + [ + 'data: {"type":"response.failed","response":{"id":"r1","status":"failed"}}\n\n', + 'data: {"type":"response.failed","response":{"error":{}}}\n\n', + 'data: {"type":"error"}\n\n', + 'data: {"type":"error","error":{}}\n\n', + ], +) +@pytest.mark.asyncio +async def test_stream_chat_chunks_preserves_terminal_error_without_payload(event_line: str): + async def _stream(): + yield event_line + + chunks = [chunk async for chunk in stream_chat_chunks(_stream(), model="gpt-5.2")] + + error_payload = json.loads(chunks[-2][5:].strip()) + assert error_payload["error"] == { + "message": "Upstream error", + "type": "server_error", + "code": "upstream_error", + } + assert chunks[-1].strip() == "data: [DONE]" + + +@pytest.mark.asyncio +async def test_stream_chat_chunks_emits_error_and_done_when_upstream_ends_without_terminal_event(): + # #given + async def _stream(): + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + # #when + chunks = [chunk async for chunk in stream_chat_chunks(_stream(), model="gpt-5.2")] + + # #then + assert chunks[-1].strip() == "data: [DONE]" + error_chunk = json.loads(chunks[-2][5:].strip()) + assert error_chunk["error"]["code"] == "upstream_stream_truncated" + assert error_chunk["error"]["type"] == "server_error" + + @pytest.mark.asyncio async def test_collect_completion_parses_event_prefixed_sse_block(): lines = [ @@ -94,6 +135,22 @@ async def _stream(): assert result.error.code == "no_accounts" +@pytest.mark.asyncio +async def test_collect_chat_completion_returns_error_when_upstream_ends_without_terminal_event(): + # #given + async def _stream(): + yield 'data: {"type":"response.output_text.delta","delta":"hi"}\n\n' + + # #when + result = await collect_chat_completion(_stream(), model="gpt-5.2") + + # #then + assert isinstance(result, OpenAIErrorEnvelope) + assert result.error is not None + assert result.error.code == "upstream_stream_truncated" + assert result.error.type == "server_error" + + def test_tool_call_delta_is_emitted(): lines = [ ( From bd67c640012692786f6beddd3d61c67cea759c47 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 20 Aug 2026 07:05:37 +0200 Subject: [PATCH 086/117] fix(proxy): retain image reservation recovery ownership (#1822) * fix(proxy): retain image reservation recovery ownership * fix(proxy): preserve explicit zero image usage --- app/modules/proxy/_service/api_key_usage.py | 28 +++ app/modules/proxy/api.py | 59 ++--- .../.openspec.yaml | 2 + .../design.md | 89 +++++++ .../proposal.md | 38 +++ .../specs/images-api-compat/spec.md | 35 +++ .../tasks.md | 16 ++ openspec/specs/images-api-compat/spec.md | 22 +- tests/integration/test_proxy_images.py | 227 +++++++++++++++++- tests/unit/test_proxy_utils.py | 164 +++++++++++++ 10 files changed, 633 insertions(+), 47 deletions(-) create mode 100644 openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md create mode 100644 openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md create mode 100644 openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md create mode 100644 openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index e1b3d6b3bd..391579d29e 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -359,6 +359,34 @@ async def _settle_compact_api_key_usage( finally: _signal_propagated_responses_service_cleanup_ready() + async def settle_image_api_key_usage( + self, + api_key: ApiKeyData | None, + reservation: ApiKeyUsageReservationData | None, + *, + model: str, + input_tokens: int | None, + output_tokens: int | None, + cached_input_tokens: int | None, + request_id: str, + ) -> bool: + """Transfer captured image usage to tracked reservation settlement.""" + has_usage = input_tokens is not None or output_tokens is not None + settlement = _StreamSettlement( + status="success" if has_usage else "failed", + model=model, + input_tokens=int(input_tokens or 0) if has_usage else None, + output_tokens=int(output_tokens or 0) if has_usage else None, + cached_input_tokens=int(cached_input_tokens or 0) if has_usage else None, + service_tier=None, + ) + return await self._settle_stream_api_key_usage( + api_key, + reservation, + settlement, + request_id=request_id, + ) + async def _settle_stream_api_key_usage( self, api_key: ApiKeyData | None, diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 2056b4dfeb..ef95d04bbb 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -3179,6 +3179,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3232,6 +3234,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3474,6 +3478,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -3527,6 +3533,8 @@ async def _stream_with_log_rewrite() -> AsyncIterator[bytes]: _output = captured.get("image_output_tokens") _cached = captured.get("image_cached_input_tokens") await _finalize_image_reservation( + context.service, + api_key, reservation, model=public_model, input_tokens=_input if isinstance(_input, int) else None, @@ -7651,6 +7659,8 @@ async def _release_reservation_best_effort( async def _finalize_image_reservation( + service: proxy_service_module.ProxyService, + api_key: ApiKeyData | None, reservation: ApiKeyUsageReservationData | None, *, model: str, @@ -7658,47 +7668,18 @@ async def _finalize_image_reservation( output_tokens: int | None, cached_input_tokens: int | None = None, ) -> None: - """Finalize the API-key usage reservation for a ``/v1/images/*`` call. - - The image adapter bypasses the standard stream settlement (``stream_responses`` - is invoked with ``api_key_reservation=None``) because the ``image_generation`` - tool path typically leaves ``response.usage`` empty; charging from - ``tool_usage.image_gen`` is the only source of truth. This helper - finalizes the reservation with the captured image tokens when present, - otherwise releases it. Calling this exactly once per request prevents - the double-billing scenario where both the standard settlement and - the post-hoc image record_usage path increment limits. - - Persistence errors are caught and logged so a transient DB/session - failure during the tail accounting cannot turn a successfully - generated image into a user-facing 500 (non-streaming) or an - abrupt stream termination (streaming). This mirrors the - best-effort accounting policy used by - ``ProxyService._settle_stream_api_key_usage``. - """ + """Transfer image-token settlement to tracked persistence ownership.""" if reservation is None: return - try: - if not input_tokens and not output_tokens: - await _release_reservation(reservation) - return - async with get_background_session() as session: - service = ApiKeysService(ApiKeysRepository(session)) - await service.finalize_usage_reservation( - reservation.reservation_id, - model=model, - input_tokens=int(input_tokens or 0), - output_tokens=int(output_tokens or 0), - cached_input_tokens=int(cached_input_tokens or 0), - service_tier=None, - ) - except Exception: - logger.warning( - "failed to finalize image reservation reservation_id=%s model=%s", - reservation.reservation_id, - model, - exc_info=True, - ) + await service.settle_image_api_key_usage( + api_key, + reservation, + model=model, + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_input_tokens=cached_input_tokens, + request_id=get_request_id() or reservation.reservation_id, + ) async def _settle_source_reservation( diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md new file mode 100644 index 0000000000..a7c1d395f9 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/design.md @@ -0,0 +1,89 @@ +## Context + +Image generation and edit reserve limited API-key quota before invoking the +internal Responses pipeline. They intentionally pass no reservation into that +pipeline because image usage comes from `tool_usage.image_gen`, not +`response.usage`. The image adapter therefore owns final settlement. + +The current adapter performs finalization inline after it has already produced +the public image result. A persistence failure rolls the reservation back to +`reserved`; the adapter logs and returns, so no request-scoped owner remains. +The standard Responses settlement path already provides detached task tracking, +cancellation handoff, retrying release, bounded repository concurrency, and +graceful persistence drain. + +## Goals / Non-Goals + +**Goals** + +- Transfer image reservation ownership exactly once to the existing tracked + settlement machinery. +- Finalize captured image tokens when persistence succeeds. +- Preserve the completed public response while failed or cancelled settlement + transfers ownership to the existing retrying release fallback. +- Keep generation/edit and streaming/non-streaming behavior aligned. + +**Non-Goals** + +- Define image-only retries of authoritative token finalization. +- Change repository states, retry timings, concurrency limits, stale-reset + policy, database schema, settings, or external response shapes. +- Give the internal Responses stream a second settlement owner. +- Broaden pre-terminal image cancellation cleanup. + +## Decisions + +### Reuse tracked stream settlement ownership + +Add one image-facing adapter on the API-key usage mixin. The adapter constructs +the existing settlement value from the public image model, captured image +tokens, API-key data, reservation, service tier `None`, and request id, then +delegates to the existing tracked settlement entrypoint. + +This keeps task registration, cancellation callbacks, retrying release, and +persistence drain in one implementation rather than copying lifecycle logic +into the route module. + +### Preserve image-token authority + +When at least one captured image token field is usable, the adapter records a +successful settlement and normalizes missing token fields to zero. When no +captured image usage is usable, it selects the existing non-success settlement +path so the reservation releases instead of recording fabricated usage. + +The internal Responses call continues receiving `api_key_reservation=None`. + +### Transfer ownership before returning the completed result + +All four image completion paths call the same adapter exactly once. The adapter +returns after the settlement task is registered; the public response does not +wait for persistence. If tracked finalization fails or is cancelled, its done +callback transfers ownership synchronously to the retrying release task. + +Exactly-once refers to the terminal database mutation. Retried release attempts +remain safe because repository transitions claim only a still-reserved row. + +## Risks / Trade-offs + +- A failed finalization falls back to release, so successful image usage can be + omitted under persistence failure. This matches existing standard stream + policy and is preferable to keeping quota ownerless. Retrying authoritative + finalization is a broader accounting-policy change and remains separate. +- Reusing a private settlement value couples the adapter to existing settlement + internals. Keeping construction inside the mixin limits that coupling and + avoids route-level task lifecycle duplication. +- Permanent release failure leaves quota conservatively reserved, but the task + remains visible to persistence drain and the stale reaper remains a final + process-restart fallback. + +## Migration Plan + +No migration or rollout setting is required. Existing terminal reservations are +unchanged; new image completions use tracked settlement after deployment. + +Rollback restores inline image finalization behavior without data conversion. + +## Open Questions + +None for this change. Stronger retries of authoritative image finalization are +explicit follow-up scope. diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md new file mode 100644 index 0000000000..65be5b7be2 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/proposal.md @@ -0,0 +1,38 @@ +## Why + +Image generation and edit routes reserve limited API-key quota but deliberately +exclude that reservation from the internal Responses stream settlement path. +Their image-specific finalizer currently logs and abandons the reservation when +persistence fails, leaving quota charged until stale cleanup and leaving +graceful persistence drain unaware of the unresolved work. + +## What Changes + +- Transfer image reservation settlement to the existing tracked, + cancellation-safe stream settlement machinery while preserving captured + `tool_usage.image_gen` tokens as the authoritative usage source. +- Preserve successful public Images JSON and SSE responses when settlement + fails or is cancelled. +- Transfer failed or cancelled finalization to the existing tracked, + retrying release fallback so persistence drain remains aware of unresolved + ownership. +- Keep the internal Responses stream reservation-free to prevent duplicate + settlement across image and standard response paths. + +## Capabilities + +### Modified Capabilities + +- `images-api-compat`: require successful image generation and edit paths to + retain tracked reservation ownership through finalization or fallback release. + +## Impact + +- Affects the image generation/edit settlement handoff in + `app/modules/proxy/api.py` and the reusable API-key settlement seam in + `app/modules/proxy/_service/api_key_usage.py`. +- Adds event-driven integration coverage for finalization failure, + cancellation, release retry, persistence drain, and all four image response + modes. +- Does not change database schema, API-key repository transitions, retry + constants, scheduler policy, external response schemas, or settings. diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md new file mode 100644 index 0000000000..d1eedf4554 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/specs/images-api-compat/spec.md @@ -0,0 +1,35 @@ +## MODIFIED Requirements + +### Requirement: Image routes participate in usage accounting and policy + +The system SHALL apply API-key allowed-model policy and model-scoped usage +limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the +effective model. The system SHALL record the publicly-requested `gpt-image-*` +value (not the internal host model) in the request log's `model` column once the +upstream response id becomes known. A successful image generation or edit that +owns a limited API-key reservation SHALL transfer that reservation exactly once +to persistence-drained settlement using captured `tool_usage.image_gen` tokens, +while the internal Responses stream SHALL NOT receive a second settlement +owner. Failed or cancelled finalization SHALL preserve the completed public +image response and transfer ownership to the tracked retrying release fallback. + +#### Scenario: API key allowed-model policy blocks gpt-image-2 + +- **WHEN** an API key's `allowed_models` list does not include `gpt-image-2` +- **THEN** requests to `/v1/images/generations` or `/v1/images/edits` with `model=gpt-image-2` return 403 `model_not_allowed` + +#### Scenario: Request log surfaces the publicly requested image model + +- **WHEN** an `/v1/images/*` request completes successfully against an internal host Responses model (for example `gpt-5.5`) +- **THEN** the resulting `request_logs` row has `model` equal to the publicly requested value (for example `gpt-image-2`) so dashboards and usage views surface the user-visible model rather than the internal host model + +#### Scenario: Failed image-token settlement retains tracked release ownership + +- **GIVEN** a limited API key owns a reservation for a successful image generation or edit request +- **AND** the internal Responses stream receives no API-key reservation +- **AND** the image adapter captures authoritative `tool_usage.image_gen` tokens +- **WHEN** tracked finalization fails or is cancelled while the reservation remains `reserved` +- **THEN** the completed public Images JSON response or SSE completion remains available +- **AND** settlement ownership transfers to a persistence-drained fallback release task +- **AND** transient release failures keep that task tracked and retrying until release succeeds or graceful persistence drain reports timeout +- **AND** a successful fallback restores pre-reserved quota exactly once without recording `response.usage` or starting a second image settlement diff --git a/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md new file mode 100644 index 0000000000..351b6d5c85 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-fix-images-finalize-reservation-release/tasks.md @@ -0,0 +1,16 @@ +## 1. Regression Coverage + +- [x] 1.1 Replace the unkeyed finalization-failure case with a limited-key integration that proves the completed image response transfers an unresolved reservation to tracked release ownership +- [x] 1.2 Add event-driven coverage for settlement cancellation and retrying release while persistence drain remains pending +- [x] 1.3 Cover generation and edit, streaming and non-streaming, to prove exactly one image settlement handoff and no internal Responses reservation owner + +## 2. Tracked Image Settlement + +- [x] 2.1 Add an image-facing API-key usage adapter that delegates captured image tokens to the existing tracked stream settlement lifecycle +- [x] 2.2 Route all four image completion paths through the adapter while preserving public response availability and public model attribution + +## 3. Verification + +- [x] 3.1 Run focused image and settlement tests, Ruff, type checking, proxy architecture checks, and strict affected OpenSpec validation +- [x] 3.2 Exercise the isolated HTTP image surface with a limited key, gated release retry, persistence drain, and real SQLite state assertions +- [x] 3.3 Verify implementation against this change, synchronize the delta, and archive the verified OpenSpec change diff --git a/openspec/specs/images-api-compat/spec.md b/openspec/specs/images-api-compat/spec.md index 2506e09c6b..4db6021a62 100644 --- a/openspec/specs/images-api-compat/spec.md +++ b/openspec/specs/images-api-compat/spec.md @@ -89,7 +89,16 @@ When a client requests `stream=true` on `/v1/images/generations` or `/v1/images/ ### Requirement: Image routes participate in usage accounting and policy -The system SHALL apply API-key allowed-model policy and model-scoped usage limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the effective model. The system SHALL record the publicly-requested `gpt-image-*` value (not the internal host model) in the request log's `model` column once the upstream response id becomes known. +The system SHALL apply API-key allowed-model policy and model-scoped usage +limits to `/v1/images/*` using the publicly-requested `gpt-image-*` value as the +effective model. The system SHALL record the publicly-requested `gpt-image-*` +value (not the internal host model) in the request log's `model` column once the +upstream response id becomes known. A successful image generation or edit that +owns a limited API-key reservation SHALL transfer that reservation exactly once +to persistence-drained settlement using captured `tool_usage.image_gen` tokens, +while the internal Responses stream SHALL NOT receive a second settlement +owner. Failed or cancelled finalization SHALL preserve the completed public +image response and transfer ownership to the tracked retrying release fallback. #### Scenario: API key allowed-model policy blocks gpt-image-2 @@ -101,6 +110,17 @@ The system SHALL apply API-key allowed-model policy and model-scoped usage limit - **WHEN** an `/v1/images/*` request completes successfully against an internal host Responses model (for example `gpt-5.5`) - **THEN** the resulting `request_logs` row has `model` equal to the publicly requested value (for example `gpt-image-2`) so dashboards and usage views surface the user-visible model rather than the internal host model +#### Scenario: Failed image-token settlement retains tracked release ownership + +- **GIVEN** a limited API key owns a reservation for a successful image generation or edit request +- **AND** the internal Responses stream receives no API-key reservation +- **AND** the image adapter captures authoritative `tool_usage.image_gen` tokens +- **WHEN** tracked finalization fails or is cancelled while the reservation remains `reserved` +- **THEN** the completed public Images JSON response or SSE completion remains available +- **AND** settlement ownership transfers to a persistence-drained fallback release task +- **AND** transient release failures keep that task tracked and retrying until release succeeds or graceful persistence drain reports timeout +- **AND** a successful fallback restores pre-reserved quota exactly once without recording `response.usage` or starting a second image settlement + ### Requirement: Image routes expose bounded operational observability The system SHALL emit structured route-completion logs and Prometheus metrics for `/v1/images/generations` and `/v1/images/edits`. Observability labels MUST be bounded to route, effective public model, stream flag, HTTP status, and outcome, and MUST NOT include prompts, image bytes, file names, access tokens, or raw upstream payloads. diff --git a/tests/integration/test_proxy_images.py b/tests/integration/test_proxy_images.py index 494edccc1e..aef0de9119 100644 --- a/tests/integration/test_proxy_images.py +++ b/tests/integration/test_proxy_images.py @@ -9,6 +9,7 @@ from __future__ import annotations +import asyncio import base64 import json import logging @@ -17,6 +18,7 @@ import pytest from httpx import AsyncByteStream +from sqlalchemy import select from starlette.datastructures import UploadFile from starlette.responses import JSONResponse @@ -25,7 +27,9 @@ from app.core.config.settings import Settings from app.core.exceptions import ProxyModelNotAllowed, ProxyRateLimitError from app.core.multipart import MultipartPolicy -from app.db.models import DashboardSettings +from app.db.models import ApiKeyUsageReservation, DashboardSettings +from app.db.session import SessionLocal +from app.modules.api_keys.repository import ApiKeysRepository pytestmark = pytest.mark.integration @@ -1608,12 +1612,30 @@ async def fake_ensure_fresh(self, account, **kwargs): @pytest.mark.asyncio -async def test_images_generations_succeeds_when_reservation_finalize_fails(async_client, monkeypatch): - """A successful image generation must NOT 500 when the post-hoc - API-key reservation finalize raises (e.g. transient DB failure). - The accounting failure is swallowed and logged; the client still - receives the image envelope. - """ +async def test_images_generations_finalize_failure_tracks_release_recovery( + async_client, + monkeypatch, +): + """A successful image keeps tracked ownership after finalization fails.""" + await _enable_api_key_auth(async_client) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "images-finalize-release-recovery", + "limits": [ + { + "limitType": "total_tokens", + "limitWindow": "weekly", + "maxValue": 1_000_000, + }, + ], + }, + ) + assert created.status_code == 200, created.text + key_payload = created.json() + api_key = key_payload["key"] + api_key_id = key_payload["id"] + await _import_account(async_client, "acc_images_finalize_fail", "img-fin-fail@example.com") async def fake_stream(payload, headers, access_token, account_id, base_url=None, raise_for_status=False, **kwargs): @@ -1650,14 +1672,23 @@ async def fake_ensure_fresh(self, account, **kwargs): # Patch finalize to blow up so we can confirm the route still 200s. from app.modules.api_keys.service import ApiKeysService + release_completed = asyncio.Event() + original_release = ApiKeysService.release_usage_reservation + async def fake_finalize(self, *args, **kwargs): del self, args, kwargs raise RuntimeError("simulated DB failure during finalize") + async def tracked_release(self, reservation_id): + await original_release(self, reservation_id) + release_completed.set() + monkeypatch.setattr(ApiKeysService, "finalize_usage_reservation", fake_finalize) + monkeypatch.setattr(ApiKeysService, "release_usage_reservation", tracked_release) response = await async_client.post( "/v1/images/generations", + headers={"Authorization": f"Bearer {api_key}"}, json={ "model": "gpt-image-2", "prompt": "x", @@ -1669,3 +1700,185 @@ async def fake_finalize(self, *args, **kwargs): assert response.status_code == 200, response.text body = response.json() assert body["data"] == [{"b64_json": "B64_FINFAIL"}] + await asyncio.wait_for(release_completed.wait(), timeout=1.0) + + async with SessionLocal() as session: + reservations = ( + ( + await session.execute( + select(ApiKeyUsageReservation).where(ApiKeyUsageReservation.api_key_id == api_key_id) + ) + ) + .scalars() + .all() + ) + assert [reservation.status for reservation in reservations] == ["released"] + + limits = await ApiKeysRepository(session).get_limits_by_key(api_key_id) + assert len(limits) == 1 + assert limits[0].current_value == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route", "stream"), + [ + ("generations", False), + ("generations", True), + ("edits", False), + ("edits", True), + ], +) +async def test_image_routes_handoff_captured_usage_exactly_once( + async_client, + monkeypatch, + route, + stream, +): + await _enable_api_key_auth(async_client) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": f"images-{route}-{'stream' if stream else 'json'}-handoff", + "limits": [ + { + "limitType": "total_tokens", + "limitWindow": "weekly", + "maxValue": 1_000_000, + }, + ], + }, + ) + assert created.status_code == 200, created.text + api_key = created.json()["key"] + + await _import_account( + async_client, + f"acc_images_{route}_{stream}", + f"img-{route}-{stream}@example.com", + ) + + async def fake_stream( + payload, + headers, + access_token, + account_id, + base_url=None, + raise_for_status=False, + **kwargs, + ): + del payload, headers, access_token, account_id, base_url, raise_for_status, kwargs + yield _sse( + { + "type": "response.output_item.done", + "output_index": 0, + "item": { + "type": "image_generation_call", + "id": f"ig_{route}_{stream}", + "status": "completed", + "result": "B64_HANDOFF", + }, + } + ) + yield _sse( + { + "type": "response.completed", + "response": { + "id": f"resp_{route}_{stream}", + "tool_usage": { + "image_gen": { + "input_tokens": 3, + "output_tokens": 4, + } + }, + }, + } + ) + + async def fake_ensure_fresh(self, account, **kwargs): + del self, kwargs + return account + + internal_reservations: list[object] = [] + original_stream_responses = proxy_module.ProxyService.stream_responses + + async def tracked_stream_responses(self, *args, **kwargs): + internal_reservations.append(kwargs.get("api_key_reservation")) + async for chunk in original_stream_responses(self, *args, **kwargs): + yield chunk + + handoffs: list[dict[str, object]] = [] + original_settle_image = proxy_module.ProxyService.settle_image_api_key_usage + + async def tracked_settle_image( + self, + api_key_arg, + reservation_arg, + **kwargs, + ): + handoffs.append( + { + "api_key": api_key_arg, + "reservation": reservation_arg, + **kwargs, + } + ) + return await original_settle_image( + self, + api_key_arg, + reservation_arg, + **kwargs, + ) + + monkeypatch.setattr(proxy_module, "core_stream_responses", fake_stream) + monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh) + monkeypatch.setattr(proxy_module.ProxyService, "stream_responses", tracked_stream_responses) + monkeypatch.setattr( + proxy_module.ProxyService, + "settle_image_api_key_usage", + tracked_settle_image, + ) + + headers = {"Authorization": f"Bearer {api_key}"} + if route == "generations": + response = await async_client.post( + "/v1/images/generations", + headers=headers, + json={ + "model": "gpt-image-2", + "prompt": "handoff", + "stream": stream, + "size": "1024x1024", + "quality": "low", + }, + ) + else: + response = await async_client.post( + "/v1/images/edits", + headers=headers, + data={ + "model": "gpt-image-2", + "prompt": "handoff", + "stream": str(stream).lower(), + "size": "1024x1024", + "quality": "low", + }, + files={ + "image": ( + "source.png", + b"\x89PNG\r\n\x1a\n" + b"\x00" * 16, + "image/png", + ), + }, + ) + + assert response.status_code == 200, response.text + assert internal_reservations == [None] + assert len(handoffs) == 1 + handoff = handoffs[0] + assert handoff["api_key"] is not None + assert handoff["reservation"] is not None + assert handoff["model"] == "gpt-image-2" + assert handoff["input_tokens"] == 3 + assert handoff["output_tokens"] == 4 + assert handoff["cached_input_tokens"] is None diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 665eff1219..0b20f804d3 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -30418,6 +30418,170 @@ async def release_usage_reservation(self, reservation_id: str) -> None: assert released == ["resv_stream_failed_background"] +@pytest.mark.asyncio +async def test_stream_api_key_cancelled_settlement_transfers_to_release(monkeypatch): + finalize_started = asyncio.Event() + release_completed = asyncio.Event() + repo = SimpleNamespace(api_keys=object()) + + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield repo + + class FakeApiKeysService: + def __init__(self, api_keys_repository: object) -> None: + assert api_keys_repository is repo.api_keys + + async def finalize_usage_reservation(self, reservation_id: str, **kwargs: object) -> None: + del reservation_id, kwargs + finalize_started.set() + await asyncio.Event().wait() + + async def release_usage_reservation(self, reservation_id: str) -> None: + assert reservation_id == "resv_image_cancel" + release_completed.set() + + monkeypatch.setattr(proxy_service, "ApiKeysService", FakeApiKeysService) + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_image_cancel", + name="image cancel", + key_prefix="sk-image-cancel", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_image_cancel", + key_id=api_key.id, + model="gpt-image-2", + ) + settlement = proxy_service._StreamSettlement( + status="success", + model="gpt-image-2", + input_tokens=3, + output_tokens=4, + ) + + assert await service._settle_stream_api_key_usage( + api_key, + reservation, + settlement, + request_id="req_image_cancel", + ) + await asyncio.wait_for(finalize_started.wait(), timeout=1.0) + + settlement_task = next(iter(service._background_cleanup_tasks)) + settlement_task.cancel() + with pytest.raises(asyncio.CancelledError): + await settlement_task + + await asyncio.wait_for(release_completed.wait(), timeout=1.0) + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert service._background_cleanup_tasks == set() + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("input_tokens", "output_tokens"), [(3, 4), (0, 0)]) +async def test_image_api_key_settlement_maps_captured_usage_once( + monkeypatch, + input_tokens, + output_tokens, +): + repo = SimpleNamespace(api_keys=object()) + + @asynccontextmanager + async def repo_factory() -> AsyncIterator[SimpleNamespace]: + yield repo + + service = proxy_service.ProxyService(cast(proxy_service.ProxyRepoFactory, repo_factory)) + api_key = ApiKeyData( + id="key_image_handoff", + name="image handoff", + key_prefix="sk-image-handoff", + allowed_models=None, + enforced_model=None, + enforced_reasoning_effort=None, + enforced_service_tier=None, + expires_at=None, + is_active=True, + created_at=utcnow(), + last_used_at=None, + ) + reservation = proxy_service.ApiKeyUsageReservationData( + reservation_id="resv_image_handoff", + key_id=api_key.id, + model="gpt-image-2", + ) + captured: list[ + tuple[ + ApiKeyData | None, + proxy_service.ApiKeyUsageReservationData | None, + proxy_service._StreamSettlement, + str, + bool, + ] + ] = [] + + async def settle_spy( + api_key_arg: ApiKeyData | None, + reservation_arg: proxy_service.ApiKeyUsageReservationData | None, + settlement: proxy_service._StreamSettlement, + request_id: str, + *, + wait_for_settlement: bool = False, + ) -> bool: + settlement.usage_settlement_transferred = True + captured.append( + ( + api_key_arg, + reservation_arg, + settlement, + request_id, + wait_for_settlement, + ) + ) + return True + + monkeypatch.setattr(service, "_settle_stream_api_key_usage", settle_spy) + + assert await service.settle_image_api_key_usage( + api_key, + reservation, + model="gpt-image-2", + input_tokens=input_tokens, + output_tokens=output_tokens, + cached_input_tokens=None, + request_id="req_image_handoff", + ) + + assert len(captured) == 1 + ( + captured_api_key, + captured_reservation, + settlement, + request_id, + wait_for_settlement, + ) = captured[0] + assert captured_api_key is api_key + assert captured_reservation is reservation + assert settlement.status == "success" + assert settlement.model == "gpt-image-2" + assert settlement.input_tokens == input_tokens + assert settlement.output_tokens == output_tokens + assert settlement.cached_input_tokens == 0 + assert settlement.service_tier is None + assert settlement.usage_settlement_transferred + assert request_id == "req_image_handoff" + assert wait_for_settlement is False + + @pytest.mark.asyncio async def test_stream_api_key_release_retries_bound_concurrent_repository_attempts(monkeypatch): retry_concurrency = proxy_service._STREAM_API_KEY_RELEASE_RETRY_MAX_CONCURRENCY From 68892e7afff21cff8910e2ee5456317eff6e231b Mon Sep 17 00:00:00 2001 From: HulianBuligon Date: Thu, 20 Aug 2026 02:07:03 -0300 Subject: [PATCH 087/117] fix(warmup): warm paid-to-free transitions (#1825) * fix(warmup): warm paid-to-free transitions * fix(warmup): reject exhausted transition samples --------- Co-authored-by: Hulian Felipe Muller Buligon --- app/core/usage/refresh_scheduler.py | 4 + app/modules/limit_warmup/service.py | 39 +++++- .../warm-free-plan-transition/design.md | 76 +++++++++++ .../warm-free-plan-transition/proposal.md | 35 +++++ .../specs/usage-refresh-policy/spec.md | 56 ++++++++ .../warm-free-plan-transition/tasks.md | 19 +++ .../test_usage_refresh_scheduler_scope.py | 86 ++++++++++++ tests/unit/test_limit_warmup.py | 129 ++++++++++++++++++ 8 files changed, 443 insertions(+), 1 deletion(-) create mode 100644 openspec/changes/warm-free-plan-transition/design.md create mode 100644 openspec/changes/warm-free-plan-transition/proposal.md create mode 100644 openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md create mode 100644 openspec/changes/warm-free-plan-transition/tasks.md diff --git a/app/core/usage/refresh_scheduler.py b/app/core/usage/refresh_scheduler.py index 0bbd27c14a..a6133d836f 100644 --- a/app/core/usage/refresh_scheduler.py +++ b/app/core/usage/refresh_scheduler.py @@ -197,6 +197,9 @@ async def _refresh_as_leader(self) -> float: selected_account, cycle_complete = self._select_next_account(accounts) if selected_account is not None: selected_account_ids = [selected_account.id] + previous_plan_types = { + selected_account.id: normalize_account_plan_type(selected_account.plan_type) + } before_primary = await usage_repo.latest_by_account( window="primary", account_ids=selected_account_ids, @@ -283,6 +286,7 @@ async def _refresh_as_leader(self) -> float: monthly_entries=warmup_after_monthly, secondary_entries=after_secondary, ), + previous_plan_types=previous_plan_types, refresh_started_at=refresh_started_at, usage_refresh_interval_seconds=self.interval_seconds, ) diff --git a/app/modules/limit_warmup/service.py b/app/modules/limit_warmup/service.py index 71cf52dfd4..f490f96b3b 100644 --- a/app/modules/limit_warmup/service.py +++ b/app/modules/limit_warmup/service.py @@ -16,7 +16,7 @@ from app.core.openai.models import OpenAIError, ResponseUsage from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ResponsesRequest -from app.core.plan_types import account_plan_matches_allowed +from app.core.plan_types import account_plan_matches_allowed, normalize_account_plan_type from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.usage.pricing import get_pricing_for_model from app.core.utils.time import naive_utc_to_epoch, utcnow @@ -342,6 +342,7 @@ async def run_after_usage_refresh( before_secondary: dict[str, UsageHistory], after_primary: dict[str, UsageHistory], after_secondary: dict[str, UsageHistory], + previous_plan_types: dict[str, str | None] | None = None, refresh_started_at: datetime | None = None, usage_refresh_interval_seconds: int = _STAGGER_SLOT_GRACE_SECONDS, ) -> None: @@ -388,6 +389,14 @@ async def run_after_usage_refresh( after_secondary=after_secondary, min_available_percent=settings.limit_warmup_min_available_percent, ) + if candidate is None and window == "secondary": + candidate = _build_paid_to_free_transition_candidate( + account=account, + previous_plan_type=(previous_plan_types or {}).get(account.id), + after_secondary=after_secondary, + refresh_started_at=refresh_started_at, + min_available_percent=settings.limit_warmup_min_available_percent, + ) if ( candidate is None and _account_is_safe_candidate(account) @@ -745,6 +754,34 @@ def usage_reset_confirmed(*, before: UsageHistory | None, after: UsageHistory | return True +def _build_paid_to_free_transition_candidate( + *, + account: Account, + previous_plan_type: str | None, + after_secondary: dict[str, UsageHistory], + refresh_started_at: datetime | None, + min_available_percent: float, +) -> _WarmupCandidate | None: + normalized_previous_plan = normalize_account_plan_type(previous_plan_type) + if normalized_previous_plan is None or normalized_previous_plan == "free": + return None + if normalize_account_plan_type(account.plan_type) != "free": + return None + if refresh_started_at is None: + return None + after = after_secondary.get(account.id) + if after is None or after.window != "monthly" or after.reset_at is None: + return None + if after.recorded_at < refresh_started_at: + return None + if after.used_percent >= 100.0: + return None + available_percent = 100.0 - after.used_percent + if min_available_percent < 100.0 and available_percent < min_available_percent: + return None + return _WarmupCandidate(reset_at=after.reset_at, window="monthly") + + def _build_staggered_idle_candidate( *, account: Account, diff --git a/openspec/changes/warm-free-plan-transition/design.md b/openspec/changes/warm-free-plan-transition/design.md new file mode 100644 index 0000000000..77c51f1cb2 --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/design.md @@ -0,0 +1,76 @@ +## Context + +See `proposal.md` for motivation. The usage updater mutates and synchronizes the +selected account only after its existing paid-to-Free confirmation policy is +satisfied. The warm-up service currently sees only the post-refresh account and +requires matching canonical before/after windows, so it cannot distinguish a +confirmed plan transition from an account that was already Free. + +The existing `usage_reset_confirmed` guard protects ordinary reset detection +from cross-window comparisons and timestamp drift. The transition path must not +weaken that guard. + +## Goals / Non-Goals + +**Goals:** + +- Carry enough refresh-scoped evidence to identify a confirmed paid-to-Free + transition without introducing new persistent state. +- Require the monthly candidate to have been written by the same refresh and + to pass the existing availability, account, and global opt-in gates. +- Reuse the existing monthly warm-up identity and atomic claim. + +**Non-Goals:** + +- Changing paid-to-Free confirmation or ordinary same-window reset detection. +- Adding settings, schema, migrations, retry queues, or periodic backfill. +- Sending warm-up traffic to inactive or non-opted-in accounts. + +## Decisions + +### Snapshot the selected account plan before refresh + +The scheduler will preserve the selected account's normalized pre-refresh plan +and pass it to warm-up evaluation after reloading the account. A transition is +eligible only when the snapshot is a recognized paid plan and the persisted +post-refresh plan is `free`. + +Alternative considered: infer a transition from `secondary` to `monthly` usage +rows. That would incorrectly classify already-Free accounts whose first monthly +sample arrives after stale secondary history. + +### Require a monthly sample written during the same refresh + +The fallback candidate will accept only the selected long-window row when its +canonical window is `monthly`, it has a reset deadline, and its `recorded_at` is +at or after the refresh start. It will apply the existing minimum-availability +gate before returning a candidate. + +Alternative considered: use the latest persisted monthly row regardless of +age. That could warm stale quota after an unrelated plan metadata update. + +### Keep the transition as a fallback to normal reset detection + +The service will first evaluate the existing same-window reset candidate. Only +when that returns no candidate for the configured long window will it evaluate +the paid-to-Free transition. The resulting candidate uses `window="monthly"` +and the monthly `reset_at`, so the existing atomic attempt claim provides +deduplication. + +Alternative considered: alter `usage_reset_confirmed` to allow cross-window +transitions. That would weaken a safety guard used by status recovery and +ordinary warm-up paths. + +## Risks / Trade-offs + +- [A process exits after persisting plan and usage but before warm-up] → The + transition can be missed, matching the current event-triggered reset path; + avoid new persistence until stronger delivery semantics are required. +- [A future updater mutates plan before confirmation] → Keep regression coverage + at scheduler/service boundaries and rely on the updater's existing durable + two-observation confirmation contract. + +## Migration Plan + +No data migration is required. Deploy the code normally; rollback restores the +previous behavior without changing stored warm-up attempts or usage history. diff --git a/openspec/changes/warm-free-plan-transition/proposal.md b/openspec/changes/warm-free-plan-transition/proposal.md new file mode 100644 index 0000000000..807336756f --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/proposal.md @@ -0,0 +1,35 @@ +## Why + +A confirmed paid-to-Free plan change can replace the account's prior paid quota +window with a newly available monthly window. The existing same-window safety +guard correctly rejects arbitrary cross-window comparisons, but it also skips +the opted-in warm-up for this confirmed plan transition. + +## What Changes + +- Preserve the selected account's plan type across one background refresh. +- Treat a confirmed paid-to-Free transition that writes a fresh available + monthly sample as a long-window warm-up candidate. +- Keep ordinary reset detection restricted to matching canonical windows and + keep single, unconfirmed Free observations ineligible. +- Add regressions for the consumer-visible warm-up attempt and the safety + boundaries around unchanged plans and stale monthly history. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `usage-refresh-policy`: Allow an opted-in long-window warm-up after a + confirmed paid-to-Free transition opens a fresh monthly quota window. + +## Impact + +- Affected code: `app/core/usage/refresh_scheduler.py` and + `app/modules/limit_warmup/service.py`. +- Affected tests: focused scheduler and limit warm-up tests. +- No API, schema, migration, setting, dependency, dashboard, or deployment + change. diff --git a/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md b/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md new file mode 100644 index 0000000000..73167cfcd8 --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/specs/usage-refresh-policy/spec.md @@ -0,0 +1,56 @@ +## ADDED Requirements + +### Requirement: Confirmed paid-to-Free transitions warm the new monthly window + +When background usage refresh confirms that an opted-in active account changed +from a recognized paid plan to `free`, and that confirming refresh writes a +fresh monthly usage sample with a reset deadline and enough available quota for +the configured warm-up threshold, the system SHALL attempt one long-window +warm-up for that monthly quota window. Eligibility MUST NOT depend on the usage +percentage reported before the plan change. + +The plan-transition exception SHALL apply only to an actual paid-to-Free change +confirmed by the refresh that wrote the monthly sample. It MUST NOT apply to a +single unconfirmed Free observation, an account that was already Free, or a +monthly sample left over from an earlier refresh. Ordinary same-window reset +detection MUST remain unchanged. The durable warm-up identity SHALL remain the +account, canonical `monthly` window, and monthly reset deadline. The confirming +monthly sample MUST report `used_percent < 100`; the configured minimum- +available threshold MAY impose a stricter lower usage limit. + +#### Scenario: Confirmed paid-to-Free transition warms fresh monthly quota + +- **GIVEN** an active opted-in account whose stored plan is a recognized paid plan +- **WHEN** background usage refresh confirms its transition to `free` +- **AND** that confirming refresh writes a monthly sample with a reset deadline and enough available quota +- **THEN** the system attempts one warm-up identified by the account, `monthly` window, and monthly reset deadline + +#### Scenario: Previous usage percentage does not gate plan-transition warm-up + +- **GIVEN** an active opted-in paid account whose previous selected quota sample was not exhausted +- **WHEN** background usage refresh confirms its transition to `free` and writes an eligible fresh monthly sample +- **THEN** the system attempts the monthly warm-up regardless of the previous usage percentage + +#### Scenario: One unconfirmed Free observation does not warm + +- **GIVEN** an active opted-in account whose stored plan is a recognized paid plan +- **WHEN** one background usage refresh reports `free` without satisfying downgrade confirmation +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Already-Free account does not use the plan-transition exception + +- **GIVEN** an active opted-in account whose stored plan was already `free` +- **WHEN** background usage refresh writes its first monthly sample without confirming a plan change +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Stale monthly history does not warm after a plan change + +- **GIVEN** an active opted-in account whose transition from a paid plan to `free` is confirmed +- **WHEN** the latest monthly sample predates the confirming refresh +- **THEN** no plan-transition warm-up is attempted + +#### Scenario: Existing durable identity deduplicates the transition warm-up + +- **GIVEN** a warm-up attempt already exists for an account, `monthly` window, and monthly reset deadline +- **WHEN** the same confirmed paid-to-Free transition is evaluated again +- **THEN** no second warm-up request is sent for that durable identity diff --git a/openspec/changes/warm-free-plan-transition/tasks.md b/openspec/changes/warm-free-plan-transition/tasks.md new file mode 100644 index 0000000000..6aac835ede --- /dev/null +++ b/openspec/changes/warm-free-plan-transition/tasks.md @@ -0,0 +1,19 @@ +## 1. Refresh-scoped transition evidence + +- [x] 1.1 Snapshot the selected account's plan before background usage refresh. +- [x] 1.2 Pass the pre-refresh plan map and refresh timestamp into long-window warm-up evaluation. + +## 2. Monthly transition candidate + +- [x] 2.1 Add a paid-to-Free fallback candidate that requires a fresh available monthly sample. +- [x] 2.2 Preserve ordinary same-window reset detection and the existing durable monthly claim. + +## 3. Regression coverage + +- [x] 3.1 Prove a confirmed paid-to-Free scheduler refresh sends one monthly warm-up regardless of prior usage. +- [x] 3.2 Cover unconfirmed or unchanged Free plans, stale monthly history, availability gating, and deduplication. + +## 4. Validation + +- [x] 4.1 Run focused scheduler and limit warm-up tests. +- [x] 4.2 Run Ruff format/check, Ty, strict OpenSpec validation, and diff hygiene checks. diff --git a/tests/integration/test_usage_refresh_scheduler_scope.py b/tests/integration/test_usage_refresh_scheduler_scope.py index cb734c0724..8b9ae51a4a 100644 --- a/tests/integration/test_usage_refresh_scheduler_scope.py +++ b/tests/integration/test_usage_refresh_scheduler_scope.py @@ -170,6 +170,7 @@ async def run_after_usage_refresh(self, **kwargs: object) -> None: selected.id, unrelated.id, } + assert warmup_calls[0]["previous_plan_types"] == {selected.id: "plus"} for snapshot_name in ("before_primary", "before_secondary", "after_primary", "after_secondary"): assert set(cast("dict[str, UsageHistory]", warmup_calls[0][snapshot_name])) <= {selected.id} @@ -298,6 +299,91 @@ async def send( assert (attempt.window, attempt.reset_at, attempt.status) == ("monthly", after_reset_at, "succeeded") +@pytest.mark.asyncio +async def test_scheduler_warms_confirmed_paid_to_free_plan_transition( + db_setup, + monkeypatch: pytest.MonkeyPatch, +) -> None: + del db_setup + account = _account("acc_paid_to_free", status=AccountStatus.ACTIVE) + prior_reset_at = int(time.time()) + 7 * 24 * 60 * 60 + monthly_reset_at = int(time.time()) + 30 * 24 * 60 * 60 + + async with SessionLocal() as session: + await AccountsRepository(session).upsert(account) + await UsageRepository(session).add_entry( + account.id, + 100.0, + window="secondary", + recorded_at=utcnow(), + reset_at=prior_reset_at, + window_minutes=10_080, + ) + await SettingsRepository(session).update( + limit_warmup_enabled=True, + limit_warmup_windows="secondary", + limit_warmup_model="gpt-5.1-codex-mini", + ) + + class _Leader: + async def run_if_leader(self, fn: Callable[[], Awaitable[object]]) -> object: + return await fn() + + class _Updater: + async def refresh_accounts( + self, + accounts: list[Account], + latest_usage: dict[str, UsageHistory], + ) -> bool: + assert [candidate.id for candidate in accounts] == [account.id] + assert accounts[0].plan_type == "plus" + accounts[0].plan_type = "free" + async with SessionLocal() as session: + persisted = await AccountsRepository(session).get_by_id(account.id) + assert persisted is not None + persisted.plan_type = "free" + await session.commit() + await UsageRepository(session).add_entry( + account.id, + 0.0, + window="monthly", + recorded_at=utcnow(), + reset_at=monthly_reset_at, + window_minutes=43_200, + ) + return True + + class _Sender: + def __init__(self) -> None: + self.calls: list[tuple[str, str]] = [] + + async def send( + self, + target: Account, + *, + model: str, + prompt: str, + ) -> LimitWarmupSendResult: + self.calls.append((target.id, model)) + return LimitWarmupSendResult(request_id="warmup-plan-transition", success=True, latency_ms=12) + + sender = _Sender() + monkeypatch.setattr(refresh_scheduler_module, "_get_leader_election", lambda: _Leader()) + monkeypatch.setattr(refresh_scheduler_module, "build_background_usage_updater", lambda: _Updater()) + monkeypatch.setattr(refresh_scheduler_module, "StreamingLimitWarmupSender", lambda *_args, **_kwargs: sender) + + scheduler = refresh_scheduler_module.UsageRefreshScheduler(interval_seconds=60, enabled=True) + + assert await scheduler._refresh_once() == 60.0 + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + async with SessionLocal() as session: + persisted_account = await AccountsRepository(session).get_by_id(account.id) + attempt = (await LimitWarmupRepository(session).latest_by_account([account.id]))[account.id] + assert persisted_account is not None + assert persisted_account.plan_type == "free" + assert (attempt.window, attempt.reset_at, attempt.status) == ("monthly", monthly_reset_at, "succeeded") + + @pytest.mark.asyncio @pytest.mark.parametrize("existing_attempt", [False, True], ids=["warmup-new", "warmup-deduped"]) @pytest.mark.parametrize( diff --git a/tests/unit/test_limit_warmup.py b/tests/unit/test_limit_warmup.py index cc1e3bd8b0..0f45c0afa4 100644 --- a/tests/unit/test_limit_warmup.py +++ b/tests/unit/test_limit_warmup.py @@ -1110,6 +1110,135 @@ async def test_monthly_free_quota_reset_warms_and_records_monthly_window() -> No assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("monthly", 2000, "succeeded")] +@pytest.mark.asyncio +async def test_confirmed_paid_to_free_transition_warms_fresh_monthly_window() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = "free" + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + monthly_reset_at = int(refresh_started_at.replace(tzinfo=timezone.utc).timestamp()) + 43_200 * 60 + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(limit_warmup_windows="secondary"), + before_primary={}, + before_secondary={account.id: _usage(account.id, used_percent=37, reset_at=10_000, window="secondary")}, + after_primary={}, + after_secondary={ + account.id: _usage( + account.id, + used_percent=0, + reset_at=monthly_reset_at, + window="monthly", + recorded_at=refresh_started_at, + ) + }, + previous_plan_types={account.id: "plus"}, + refresh_started_at=refresh_started_at, + ) + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at, row.status) for row in repo.rows] == [("monthly", monthly_reset_at, "succeeded")] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("previous_plan_type", "current_plan_type", "sample_age_seconds", "used_percent", "minimum_available"), + [ + ("free", "free", 0, 0.0, 100.0), + ("plus", "plus", 0, 0.0, 100.0), + ("plus", "free", -1, 0.0, 100.0), + ("plus", "free", 0, 2.0, 99.0), + ("plus", "free", 0, 100.0, 100.0), + ], + ids=[ + "already-free", + "unconfirmed", + "stale-monthly", + "below-availability-gate", + "exhausted-monthly-at-default-gate", + ], +) +async def test_paid_to_free_transition_candidate_rejects_unsafe_evidence( + previous_plan_type: str, + current_plan_type: str, + sample_age_seconds: int, + used_percent: float, + minimum_available: float, +) -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = current_plan_type + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + recorded_at = refresh_started_at + timedelta(seconds=sample_age_seconds) + + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings( + limit_warmup_windows="secondary", + limit_warmup_min_available_percent=minimum_available, + ), + before_primary={}, + before_secondary={}, + after_primary={}, + after_secondary={ + account.id: _usage( + account.id, + used_percent=used_percent, + reset_at=2_000_000_000, + window="monthly", + recorded_at=recorded_at, + ) + }, + previous_plan_types={account.id: previous_plan_type}, + refresh_started_at=refresh_started_at, + ) + + assert sender.calls == [] + assert repo.rows == [] + + +@pytest.mark.asyncio +async def test_paid_to_free_transition_warmup_is_deduplicated_by_monthly_reset() -> None: + repo = FakeWarmupRepo() + sender = FakeSender() + service = LimitWarmupService(repo, FakeRequestLogsRepo(), sender=sender) + account = _account() + account.plan_type = "free" + refresh_started_at = datetime(2026, 8, 18, 18, 8, tzinfo=timezone.utc).replace(tzinfo=None) + after_secondary = { + account.id: _usage( + account.id, + used_percent=0, + reset_at=2_000_000_000, + window="monthly", + recorded_at=refresh_started_at, + ) + } + + async def run_once() -> None: + await service.run_after_usage_refresh( + accounts=[account], + settings=_settings(limit_warmup_windows="secondary"), + before_primary={}, + before_secondary={}, + after_primary={}, + after_secondary=after_secondary, + previous_plan_types={account.id: "pro"}, + refresh_started_at=refresh_started_at, + ) + + await run_once() + await run_once() + + assert sender.calls == [(account.id, "gpt-5.1-codex-mini")] + assert [(row.window, row.reset_at) for row in repo.rows] == [("monthly", 2_000_000_000)] + + @pytest.mark.asyncio async def test_long_window_warmup_ignores_cross_window_transition() -> None: repo = FakeWarmupRepo() From 4d0f0ffc64df11a6800397497512abeb5478bce2 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 09:08:44 +0400 Subject: [PATCH 088/117] feat(model-sources): embeddings source capability (#1776) Add POST /v1/embeddings routed to OpenAI-compatible model sources, mirroring the audio-transcriptions capability shape (#261): - model_sources.supports_embeddings column (additive migration) + schema, service, repository finder, and dashboard capability toggle/badge - forwarding: forward_embeddings posts the client payload to the source's /embeddings, passes the response through, and reads usage from prompt/total tokens (embeddings report no completion tokens) - proxy: /v1/embeddings validates model+input, passes other OpenAI params (encoding_format, dimensions, user, ...) through verbatim, enforces the same API-key model access/limits, settles usage, and writes the same model_source request_logs row as chat/audio (previously /v1/embeddings returned 405 and nothing was ever logged) - unknown model returns OpenAI-shaped model_not_found (embeddings have no subscription-backed fallback); upstream errors pass through and log; usage-less responses fail closed for limited keys Integration tests cover the happy path (payload passthrough + ledger row), unknown model, upstream error logging, and the limited-key fail-closed path. --- .../middleware/required_capability_http.py | 1 + ...0816_000000_add_model_source_embeddings.py | 50 ++++ app/db/models.py | 6 + app/modules/model_sources/forwarding.py | 52 +++++ app/modules/model_sources/repository.py | 25 ++ app/modules/model_sources/schemas.py | 3 + app/modules/model_sources/service.py | 4 + app/modules/proxy/api.py | 154 +++++++++++- .../components/model-source-create-dialog.tsx | 1 + .../model-source-edit-dialog.test.tsx | 26 +++ .../components/model-source-edit-dialog.tsx | 1 + .../components/model-source-form-fields.tsx | 1 + .../components/model-source-form.ts | 3 + .../components/model-source-multi-select.tsx | 1 + .../components/model-sources-settings.tsx | 1 + .../features/model-sources/schemas.test.ts | 27 ++- .../src/features/model-sources/schemas.ts | 3 + frontend/src/i18n/locales/en.json | 1 + frontend/src/i18n/locales/ko.json | 1 + frontend/src/i18n/locales/zh-CN.json | 1 + frontend/src/test/mocks/factories.ts | 1 + frontend/src/test/mocks/handlers.ts | 6 + .../.openspec.yaml | 2 + .../add-model-source-embeddings/proposal.md | 48 ++++ .../specs/model-source-routing/spec.md | 108 +++++++++ .../add-model-source-embeddings/tasks.md | 31 +++ .../test_daybreak_capability_routes.py | 9 + .../integration/test_model_source_routing.py | 221 ++++++++++++++++++ tests/unit/test_db_migrate.py | 6 +- 29 files changed, 786 insertions(+), 8 deletions(-) create mode 100644 app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py create mode 100644 openspec/changes/add-model-source-embeddings/.openspec.yaml create mode 100644 openspec/changes/add-model-source-embeddings/proposal.md create mode 100644 openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md create mode 100644 openspec/changes/add-model-source-embeddings/tasks.md diff --git a/app/core/middleware/required_capability_http.py b/app/core/middleware/required_capability_http.py index 36ca7bffda..ab7229a034 100644 --- a/app/core/middleware/required_capability_http.py +++ b/app/core/middleware/required_capability_http.py @@ -30,6 +30,7 @@ "/v1/responses", "/v1/responses/compact", "/v1/chat/completions", + "/v1/embeddings", "/v1/images/generations", "/v1/reset-credit", "/v1/warmup", diff --git a/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py b/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py new file mode 100644 index 0000000000..3619b2d0f1 --- /dev/null +++ b/app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py @@ -0,0 +1,50 @@ +"""add model source embeddings capability + +Revision ID: 20260816_000000_add_model_source_embeddings +Revises: 20260806_030000_add_api_key_allowed_reasoning_efforts +Create Date: 2026-08-16 00:00:00.000000 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260816_000000_add_model_source_embeddings" +down_revision = "20260806_030000_add_api_key_allowed_reasoning_efforts" +branch_labels = None +depends_on = None + + +def _has_table(connection: Connection, table_name: str) -> bool: + return sa.inspect(connection).has_table(table_name) + + +def _columns(connection: Connection, table_name: str) -> set[str]: + if not _has_table(connection, table_name): + return set() + return {column["name"] for column in sa.inspect(connection).get_columns(table_name)} + + +def upgrade() -> None: + bind = op.get_bind() + model_source_columns = _columns(bind, "model_sources") + if model_source_columns and "supports_embeddings" not in model_source_columns: + with op.batch_alter_table("model_sources") as batch_op: + batch_op.add_column( + sa.Column( + "supports_embeddings", + sa.Boolean(), + server_default=sa.false(), + nullable=False, + ) + ) + + +def downgrade() -> None: + bind = op.get_bind() + model_source_columns = _columns(bind, "model_sources") + if "supports_embeddings" in model_source_columns: + with op.batch_alter_table("model_sources") as batch_op: + batch_op.drop_column("supports_embeddings") diff --git a/app/db/models.py b/app/db/models.py index f5634cdb8a..a1da492f80 100644 --- a/app/db/models.py +++ b/app/db/models.py @@ -1275,6 +1275,12 @@ class ModelSource(Base): server_default=false(), nullable=False, ) + supports_embeddings: Mapped[bool] = mapped_column( + Boolean, + default=False, + server_default=false(), + nullable=False, + ) timeout_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True) max_concurrency: Mapped[int | None] = mapped_column(Integer, nullable=True) created_at: Mapped[datetime] = mapped_column(DateTime, server_default=func.now(), nullable=False) diff --git a/app/modules/model_sources/forwarding.py b/app/modules/model_sources/forwarding.py index 8fa2ed496c..727eeb67d6 100644 --- a/app/modules/model_sources/forwarding.py +++ b/app/modules/model_sources/forwarding.py @@ -83,6 +83,13 @@ class SourceAudioTranscription: upstream_status_code: int +@dataclass(frozen=True, slots=True) +class SourceEmbeddings: + payload: dict[str, JsonValue] + usage: SourceUsage | None + upstream_status_code: int + + @dataclass(frozen=True, slots=True) class SourceChatStream: body: AsyncIterator[bytes] @@ -298,6 +305,43 @@ async def forward_audio_transcription( raise _unreachable_error(exc) from exc +async def forward_embeddings( + source: ModelSource, + payload: dict[str, JsonValue], + *, + encryptor: TokenEncryptor | None = None, +) -> SourceEmbeddings: + try: + async with lease_http_session() as session: + timeout = aiohttp.ClientTimeout(total=_source_timeout_seconds(source)) + async with session.post( + _source_url(source, "/embeddings"), + headers=_source_headers(source, encryptor=encryptor), + json=payload, + timeout=timeout, + ) as response: + data = await _response_json(response) + if response.status >= 400: + raise ModelSourceForwardingError( + status_code=response.status, + payload=_redact_source_error_payload( + _error_payload(data), + source, + encryptor=encryptor, + ), + upstream_status_code=response.status, + ) + if data is None: + raise _invalid_upstream_response_error(response.status) + return SourceEmbeddings( + payload=data, + usage=_usage_from_embeddings_payload(data), + upstream_status_code=response.status, + ) + except (aiohttp.ClientError, TimeoutError) as exc: + raise _unreachable_error(exc) from exc + + async def stream_responses( source: ModelSource, payload: dict[str, JsonValue], @@ -537,6 +581,14 @@ def _usage_from_responses_payload(payload: Mapping[str, JsonValue]) -> SourceUsa return _usage_from_responses_mapping(usage) +def _usage_from_embeddings_payload(payload: Mapping[str, JsonValue]) -> SourceUsage | None: + """Embeddings responses report prompt/total tokens and no completion tokens.""" + usage = payload.get("usage") + if not is_json_mapping(usage): + return None + return _usage_from_mapping(usage) or _usage_from_total_tokens_mapping(usage) + + def _usage_from_audio_body(body: bytes, content_type: str | None) -> SourceUsage | None: if not _is_json_content_type(content_type): return None diff --git a/app/modules/model_sources/repository.py b/app/modules/model_sources/repository.py index 38fb42238b..19d39febcd 100644 --- a/app/modules/model_sources/repository.py +++ b/app/modules/model_sources/repository.py @@ -113,6 +113,31 @@ async def find_audio_transcriptions_source_for_model( result = await self._session.execute(stmt) return result.scalar_one_or_none() + async def find_embeddings_source_for_model( + self, + model: str, + *, + allowed_source_ids: set[str] | None = None, + ) -> ModelSource | None: + stmt = ( + select(ModelSource) + .options(selectinload(ModelSource.models)) + .join(ModelSourceModel, ModelSourceModel.source_id == ModelSource.id) + .where(ModelSource.kind == "openai_compatible") + .where(ModelSource.is_enabled.is_(True)) + .where(ModelSource.supports_embeddings.is_(True)) + .where(ModelSourceModel.model == model) + .where(ModelSourceModel.is_enabled.is_(True)) + .order_by(ModelSource.name, ModelSource.id) + .limit(1) + ) + if allowed_source_ids is not None: + if not allowed_source_ids: + return None + stmt = stmt.where(ModelSource.id.in_(allowed_source_ids)) + result = await self._session.execute(stmt) + return result.scalar_one_or_none() + async def create(self, row: ModelSource, *, commit: bool = True) -> ModelSource: self._session.add(row) if commit: diff --git a/app/modules/model_sources/schemas.py b/app/modules/model_sources/schemas.py index b556c4d91e..1643f40191 100644 --- a/app/modules/model_sources/schemas.py +++ b/app/modules/model_sources/schemas.py @@ -37,6 +37,7 @@ class ModelSourceCreateRequest(DashboardModel): supports_chat_completions: bool = True supports_responses: bool = False supports_audio_transcriptions: bool = False + supports_embeddings: bool = False timeout_seconds: int | None = Field(default=None, ge=1) max_concurrency: int | None = Field(default=None, ge=1) models: list[ModelSourceModelInput] = Field(default_factory=list) @@ -50,6 +51,7 @@ class ModelSourceUpdateRequest(DashboardModel): supports_chat_completions: bool | None = None supports_responses: bool | None = None supports_audio_transcriptions: bool | None = None + supports_embeddings: bool | None = None timeout_seconds: int | None = Field(default=None, ge=1) max_concurrency: int | None = Field(default=None, ge=1) models: list[ModelSourceModelInput] | None = None @@ -65,6 +67,7 @@ class ModelSourceResponse(DashboardModel): supports_chat_completions: bool supports_responses: bool supports_audio_transcriptions: bool + supports_embeddings: bool timeout_seconds: int | None max_concurrency: int | None created_at: datetime diff --git a/app/modules/model_sources/service.py b/app/modules/model_sources/service.py index a0420d0d29..b9fd89e858 100644 --- a/app/modules/model_sources/service.py +++ b/app/modules/model_sources/service.py @@ -57,6 +57,7 @@ async def create_source(self, payload: ModelSourceCreateRequest) -> ModelSourceR supports_chat_completions=payload.supports_chat_completions, supports_responses=payload.supports_responses, supports_audio_transcriptions=payload.supports_audio_transcriptions, + supports_embeddings=payload.supports_embeddings, timeout_seconds=payload.timeout_seconds, max_concurrency=payload.max_concurrency, models=model_rows, @@ -88,6 +89,8 @@ async def update_source(self, source_id: str, payload: ModelSourceUpdateRequest) row.supports_responses = payload.supports_responses if "supports_audio_transcriptions" in fields and payload.supports_audio_transcriptions is not None: row.supports_audio_transcriptions = payload.supports_audio_transcriptions + if "supports_embeddings" in fields and payload.supports_embeddings is not None: + row.supports_embeddings = payload.supports_embeddings if "timeout_seconds" in fields: row.timeout_seconds = payload.timeout_seconds if "max_concurrency" in fields: @@ -224,6 +227,7 @@ def _to_response(row: ModelSource) -> ModelSourceResponse: supports_chat_completions=row.supports_chat_completions, supports_responses=row.supports_responses, supports_audio_transcriptions=row.supports_audio_transcriptions, + supports_embeddings=row.supports_embeddings, timeout_seconds=row.timeout_seconds, max_concurrency=row.max_concurrency, created_at=row.created_at, diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index ef95d04bbb..1298966416 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -26,7 +26,7 @@ WebSocket, ) from fastapi.responses import JSONResponse, StreamingResponse -from pydantic import ValidationError +from pydantic import BaseModel, ConfigDict, ValidationError from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession from starlette.convertors import Convertor, register_url_convertor @@ -194,6 +194,9 @@ from app.modules.model_sources.forwarding import ( forward_audio_transcription as forward_source_audio_transcription, ) +from app.modules.model_sources.forwarding import ( + forward_embeddings as forward_source_embeddings, +) from app.modules.model_sources.forwarding import ( forward_responses as forward_source_responses, ) @@ -2509,6 +2512,57 @@ async def v1_audio_transcriptions( ) +class V1EmbeddingsRequest(BaseModel): + """OpenAI-compatible embeddings request. + + Only ``model`` and ``input`` are validated; other OpenAI params + (``encoding_format``, ``dimensions``, ``user``, …) pass through to the + model source verbatim. + """ + + model_config = ConfigDict(extra="allow") + + model: str + input: str | list[str] | list[int] | list[list[int]] + + +@v1_router.post("/embeddings") +async def v1_embeddings( + request: Request, + payload: V1EmbeddingsRequest = Body(...), + context: ProxyContext = Depends(get_proxy_context), + api_key: ApiKeyData | None = Security(validate_proxy_api_key), +) -> Response: + capability_transport_denial = await _required_capability_http_transport_denial(request, api_key) + if capability_transport_denial is not None: + return capability_transport_denial + model = payload.model + rate_limit_headers = await _rate_limit_headers_for_request(context, api_key) + source = await _select_embeddings_model_source(model, api_key) + if source is None: + # Embeddings have no subscription-backed fallback: only configured + # model sources can serve them. + return _logged_error_json_response( + request, + status_code=404, + content=openai_error( + "model_not_found", + f"The model '{model}' does not exist or no enabled model source supports embeddings for it", + error_type="invalid_request_error", + ), + headers=rate_limit_headers, + ) + validate_model_access(api_key, model) + return await _source_embeddings_response( + request=request, + model=model, + payload=payload, + source=source, + api_key=api_key, + rate_limit_headers=rate_limit_headers, + ) + + @router.post( "/images/generations", response_model=None, @@ -4333,6 +4387,20 @@ async def _select_responses_model_source( ) +async def _select_embeddings_model_source(model: str, api_key: ApiKeyData | None) -> ModelSource | None: + assigned_source_ids = _allowed_source_ids_for_api_key(api_key) + exact_allowed_models = _exact_source_allowed_models_for_api_key(api_key) + if exact_allowed_models is not None and model not in exact_allowed_models: + return None + async with get_background_session() as session: + source = await ModelSourcesRepository(session).find_embeddings_source_for_model( + model, + allowed_source_ids=assigned_source_ids, + ) + detach_session_objects(session) + return source + + async def _select_audio_transcriptions_model_source(model: str, api_key: ApiKeyData | None) -> ModelSource | None: assigned_source_ids = _allowed_source_ids_for_api_key(api_key) exact_allowed_models = _exact_source_allowed_models_for_api_key(api_key) @@ -4378,6 +4446,90 @@ async def _parse_transcription_multipart( ) +async def _source_embeddings_response( + *, + request: Request, + model: str, + payload: "V1EmbeddingsRequest", + source: ModelSource, + api_key: ApiKeyData | None, + rate_limit_headers: Mapping[str, str], +) -> Response: + reservation = await _enforce_request_limits( + api_key, + request_model=model, + request_service_tier=None, + ) + outbound = payload.model_dump(exclude_none=True) + outbound["model"] = model + try: + result = await forward_source_embeddings(source, outbound) + except ModelSourceForwardingError as exc: + await _release_reservation(reservation) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code=_source_error_code(exc.payload), + error_message=_source_error_message(exc.payload), + upstream_status_code=exc.upstream_status_code, + ) + return _logged_error_json_response(request, exc.status_code, exc.payload, headers=rate_limit_headers) + if result.usage is None and _reservation_requires_usage(reservation): + await _release_reservation(reservation) + error = openai_error( + "usage_unavailable", + "OpenAI-compatible model source embeddings response did not include token usage for a limited API key", + error_type="server_error", + ) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_unavailable", + error_message="source embeddings response missing token usage", + upstream_status_code=result.upstream_status_code, + ) + return _logged_error_json_response(request, 502, error, headers=rate_limit_headers) + settled = await _settle_source_reservation( + reservation, + source=source, + model=model, + usage=result.usage, + ) + if not settled: + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="error", + error_code="usage_settlement_failed", + error_message="source usage settlement failed", + upstream_status_code=result.upstream_status_code, + ) + return _logged_error_json_response( + request, + 502, + _source_usage_settlement_failed_error(), + headers=rate_limit_headers, + ) + await _log_source_chat_completion( + request, + source=source, + api_key=api_key, + model=model, + status="success", + usage=result.usage, + upstream_status_code=result.upstream_status_code, + ) + return JSONResponse(content=result.payload, headers=dict(rate_limit_headers)) + + async def _source_audio_transcription_response( *, request: Request, diff --git a/frontend/src/features/model-sources/components/model-source-create-dialog.tsx b/frontend/src/features/model-sources/components/model-source-create-dialog.tsx index fbc3578d1c..e85a2bc876 100644 --- a/frontend/src/features/model-sources/components/model-source-create-dialog.tsx +++ b/frontend/src/features/model-sources/components/model-source-create-dialog.tsx @@ -56,6 +56,7 @@ export function ModelSourceCreateDialog({ supportsChatCompletions: draft.supportsChatCompletions, supportsResponses: draft.supportsResponses, supportsAudioTranscriptions: draft.supportsAudioTranscriptions, + supportsEmbeddings: draft.supportsEmbeddings, models: modelInputsFromForm(values, draft), }; await onSubmit(payload); diff --git a/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx b/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx index e02af248e3..091f6a53f1 100644 --- a/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx +++ b/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx @@ -18,6 +18,7 @@ function createModelSource(overrides: Partial = {}): ModelSource { supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: false, + supportsEmbeddings: false, timeoutSeconds: null, maxConcurrency: null, createdAt: "2026-07-03T00:00:00Z", @@ -99,6 +100,31 @@ describe("ModelSourceEditDialog", () => { outputPer1M: 2.25, }); expect(payload.supportsAudioTranscriptions).toBe(false); + expect(payload.supportsEmbeddings).toBe(false); + }); + + it("carries an enabled embeddings capability through submit", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn().mockResolvedValue(undefined); + + renderWithProviders( + , + ); + + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledTimes(1); + }); + + const [, payload] = onSubmit.mock.calls[0]; + expect(payload.supportsEmbeddings).toBe(true); }); it("preserves disabled model rows during edits", async () => { diff --git a/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx b/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx index 725fd014e6..854d0e8cae 100644 --- a/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx +++ b/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx @@ -203,6 +203,7 @@ function ModelSourceEditForm({ source, busy, onSubmit, onClose }: ModelSourceEdi supportsChatCompletions: draft.supportsChatCompletions, supportsResponses: draft.supportsResponses, supportsAudioTranscriptions: draft.supportsAudioTranscriptions, + supportsEmbeddings: draft.supportsEmbeddings, }; if (modelIdsChanged || hasAnyModelDraftChange(draftChangeFlags)) { diff --git a/frontend/src/features/model-sources/components/model-source-form-fields.tsx b/frontend/src/features/model-sources/components/model-source-form-fields.tsx index 59a9e1e017..267f86496c 100644 --- a/frontend/src/features/model-sources/components/model-source-form-fields.tsx +++ b/frontend/src/features/model-sources/components/model-source-form-fields.tsx @@ -21,6 +21,7 @@ const CAPABILITY_TOGGLES = [ ["supportsChatCompletions", "modelSources.capabilities.chatCompletions"] as const, ["supportsResponses", "modelSources.capabilities.responses"] as const, ["supportsAudioTranscriptions", "modelSources.capabilities.audioTranscriptions"] as const, + ["supportsEmbeddings", "modelSources.capabilities.embeddings"] as const, ["supportsStreaming", "modelSources.capabilities.streaming"] as const, ["supportsTools", "modelSources.capabilities.tools"] as const, ["supportsVision", "modelSources.capabilities.vision"] as const, diff --git a/frontend/src/features/model-sources/components/model-source-form.ts b/frontend/src/features/model-sources/components/model-source-form.ts index f2764a15f6..db2923741b 100644 --- a/frontend/src/features/model-sources/components/model-source-form.ts +++ b/frontend/src/features/model-sources/components/model-source-form.ts @@ -31,6 +31,7 @@ export type ModelSourceDraft = { supportsChatCompletions: boolean; supportsResponses: boolean; supportsAudioTranscriptions: boolean; + supportsEmbeddings: boolean; supportsStreaming: boolean; supportsTools: boolean; supportsVision: boolean; @@ -47,6 +48,7 @@ export const initialModelSourceDraft: ModelSourceDraft = { supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: false, + supportsEmbeddings: false, supportsStreaming: true, supportsTools: false, supportsVision: false, @@ -167,6 +169,7 @@ export function draftFromSource(source: ModelSource): ModelSourceDraft { supportsChatCompletions: source.supportsChatCompletions, supportsResponses: source.supportsResponses, supportsAudioTranscriptions: source.supportsAudioTranscriptions, + supportsEmbeddings: source.supportsEmbeddings, supportsStreaming: firstModel?.supportsStreaming ?? true, supportsTools: firstModel?.supportsTools ?? false, supportsVision: firstModel?.supportsVision ?? false, diff --git a/frontend/src/features/model-sources/components/model-source-multi-select.tsx b/frontend/src/features/model-sources/components/model-source-multi-select.tsx index ec469f4c98..204424fbc4 100644 --- a/frontend/src/features/model-sources/components/model-source-multi-select.tsx +++ b/frontend/src/features/model-sources/components/model-source-multi-select.tsx @@ -26,6 +26,7 @@ function sourceSubtitle(source: ModelSource, t: ReturnType value !== null); } diff --git a/frontend/src/features/model-sources/schemas.test.ts b/frontend/src/features/model-sources/schemas.test.ts index d3043a8630..bb8f1e3058 100644 --- a/frontend/src/features/model-sources/schemas.test.ts +++ b/frontend/src/features/model-sources/schemas.test.ts @@ -9,10 +9,8 @@ import { const ISO = "2026-01-01T00:00:00+00:00"; -describe("ModelSourceSchema", () => { - it("parses model source payload", () => { - const parsed = ModelSourceSchema.parse({ - id: "src_vllm", +const BASE_SOURCE = { + id: "src_vllm", name: "vLLM", kind: "openai_compatible", baseUrl: "http://localhost:8000/v1", @@ -21,6 +19,7 @@ describe("ModelSourceSchema", () => { supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: true, + supportsEmbeddings: true, timeoutSeconds: null, maxConcurrency: null, createdAt: ISO, @@ -44,13 +43,27 @@ describe("ModelSourceSchema", () => { createdAt: ISO, updatedAt: ISO, }, - ], - }); + ], +}; + +describe("ModelSourceSchema", () => { + it("parses model source payload", () => { + const parsed = ModelSourceSchema.parse(BASE_SOURCE); expect(parsed.id).toBe("src_vllm"); expect(parsed.supportsAudioTranscriptions).toBe(true); + expect(parsed.supportsEmbeddings).toBe(true); expect(parsed.models[0].model).toBe("local-coder"); }); + + it("defaults supportsEmbeddings to false when the field is absent", () => { + const withoutEmbeddings: Record = { ...BASE_SOURCE }; + delete withoutEmbeddings.supportsEmbeddings; + + const parsed = ModelSourceSchema.parse(withoutEmbeddings); + + expect(parsed.supportsEmbeddings).toBe(false); + }); }); describe("ModelSourcesResponseSchema", () => { @@ -70,10 +83,12 @@ describe("ModelSourceCreateRequestSchema", () => { supportsChatCompletions: true, supportsResponses: true, supportsAudioTranscriptions: true, + supportsEmbeddings: true, models: [{ model: "deepseek-v4-flash" }], }); expect(parsed.supportsAudioTranscriptions).toBe(true); + expect(parsed.supportsEmbeddings).toBe(true); expect(parsed.models[0].model).toBe("deepseek-v4-flash"); }); }); diff --git a/frontend/src/features/model-sources/schemas.ts b/frontend/src/features/model-sources/schemas.ts index 666f6ada89..c5fab121b2 100644 --- a/frontend/src/features/model-sources/schemas.ts +++ b/frontend/src/features/model-sources/schemas.ts @@ -30,6 +30,7 @@ export const ModelSourceSchema = z.object({ supportsChatCompletions: z.boolean(), supportsResponses: z.boolean(), supportsAudioTranscriptions: z.boolean().default(false), + supportsEmbeddings: z.boolean().default(false), timeoutSeconds: z.number().int().positive().nullable().default(null), maxConcurrency: z.number().int().positive().nullable().default(null), createdAt: z.iso.datetime({ offset: true }), @@ -64,6 +65,7 @@ export const ModelSourceCreateRequestSchema = z.object({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), timeoutSeconds: z.number().int().positive().nullable().optional(), maxConcurrency: z.number().int().positive().nullable().optional(), models: z.array(ModelSourceModelInputSchema).default([]), @@ -77,6 +79,7 @@ export const ModelSourceUpdateRequestSchema = z.object({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), timeoutSeconds: z.number().int().positive().nullable().optional(), maxConcurrency: z.number().int().positive().nullable().optional(), models: z.array(ModelSourceModelInputSchema).optional(), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 84c7efbf55..664aada8b1 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -870,6 +870,7 @@ "modelSources.editDialog.title": "Edit model source", "modelSources.empty": "No model sources configured.", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 1b763ad5f5..458265ad26 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -870,6 +870,7 @@ "modelSources.editDialog.title": "Model source 수정", "modelSources.empty": "항목 없음", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 224c87c2cc..3a8ea63686 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -870,6 +870,7 @@ "modelSources.editDialog.title": "编辑 Model source", "modelSources.empty": "无项目", "modelSources.capabilities.audioTranscriptions": "Audio transcriptions", + "modelSources.capabilities.embeddings": "Embeddings", "modelSources.capabilities.chatCompletions": "Chat completions", "modelSources.capabilities.reasoning": "Reasoning", "modelSources.capabilities.responses": "Responses", diff --git a/frontend/src/test/mocks/factories.ts b/frontend/src/test/mocks/factories.ts index d1db25b370..2aa8e01de8 100644 --- a/frontend/src/test/mocks/factories.ts +++ b/frontend/src/test/mocks/factories.ts @@ -182,6 +182,7 @@ export function createModelSource( supportsChatCompletions: true, supportsResponses: false, supportsAudioTranscriptions: false, + supportsEmbeddings: false, timeoutSeconds: null, maxConcurrency: null, createdAt: offsetIso(-30), diff --git a/frontend/src/test/mocks/handlers.ts b/frontend/src/test/mocks/handlers.ts index 1a96b3892c..25884bc0a3 100644 --- a/frontend/src/test/mocks/handlers.ts +++ b/frontend/src/test/mocks/handlers.ts @@ -162,6 +162,7 @@ const ModelSourceCreatePayloadSchema = z.looseObject({ supportsChatCompletions: z.boolean().optional(), supportsResponses: z.boolean().optional(), supportsAudioTranscriptions: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), models: z .array( z.looseObject({ @@ -179,6 +180,7 @@ const ModelSourceCreatePayloadSchema = z.looseObject({ const ModelSourceUpdatePayloadSchema = z.looseObject({ isEnabled: z.boolean().optional(), + supportsEmbeddings: z.boolean().optional(), }); const QuotaPlannerSettingsPayloadSchema = z.looseObject({ @@ -2121,6 +2123,7 @@ export const handlers = [ supportsChatCompletions: payload?.supportsChatCompletions ?? true, supportsResponses: payload?.supportsResponses ?? false, supportsAudioTranscriptions: payload?.supportsAudioTranscriptions ?? false, + supportsEmbeddings: payload?.supportsEmbeddings ?? false, models: (payload?.models ?? [{ model: `model-${sequence}` }]).map( (model, index) => ({ id: index + 1, @@ -2160,6 +2163,9 @@ export const handlers = [ const updated = createModelSource({ ...existing, ...(payload?.isEnabled !== undefined ? { isEnabled: payload.isEnabled } : {}), + ...(payload?.supportsEmbeddings !== undefined + ? { supportsEmbeddings: payload.supportsEmbeddings } + : {}), updatedAt: new Date().toISOString(), }); state.modelSources = state.modelSources.map((source) => diff --git a/openspec/changes/add-model-source-embeddings/.openspec.yaml b/openspec/changes/add-model-source-embeddings/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/add-model-source-embeddings/proposal.md b/openspec/changes/add-model-source-embeddings/proposal.md new file mode 100644 index 0000000000..306580eedb --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/proposal.md @@ -0,0 +1,48 @@ +## Why + +Model sources already carry per-protocol capability flags for chat +completions, responses, and audio transcriptions, and the proxy routes each +OpenAI-compatible surface to a source that declares the matching capability. +Embeddings have no such flag and no route, so an operator running a local +embedding model behind an OpenAI-compatible source cannot serve +`POST /v1/embeddings` through the proxy at all. There is also no +subscription-backed upstream to fall back on: unlike chat and responses +traffic, embeddings can only ever be served by a configured model source. + +Adding the capability requires a persisted flag, so the create/read/update +contracts, the dashboard form, request validation, and request-log accounting +all move together. + +## What Changes + +- Add a persisted `supports_embeddings` capability flag to model sources, + defaulting to disabled so existing sources keep their current behavior. +- Expose the flag through the model-source create/read/update API contracts + and the dashboard model-source form. +- Add `POST /v1/embeddings`, routed only to an enabled source that declares + the embeddings capability for the requested model. +- Return `model_not_found` when no enabled source supports embeddings for the + requested model, instead of falling through to subscription accounts. +- Record embeddings requests in the request log with the same success/error + accounting and usage-settlement rules the other model-source routes use, + including failing closed when a limited API key needs usage the source did + not report. + +## Capabilities + +### New Capabilities + +- `model-source-routing`: the embeddings capability flag, its routing rule, + and the `/v1/embeddings` request/accounting contract. + +### Modified Capabilities + +None. + +## Impact + +The change adds one nullable-free boolean column with a `false` server +default, one proxy route, one forwarding helper, and one repository lookup. +It adds no setting, no dependency, and no change to existing routing for chat +completions, responses, or audio transcriptions. Sources that do not opt in +are unaffected. diff --git a/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md b/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md new file mode 100644 index 0000000000..b1772e98ba --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/specs/model-source-routing/spec.md @@ -0,0 +1,108 @@ +# model-source-routing Delta + +## ADDED Requirements + +### Requirement: Model sources declare an embeddings capability + +Each model source MUST carry a persisted `supports_embeddings` boolean +capability flag. The flag MUST default to disabled, so a source created or +migrated without an explicit value MUST NOT be treated as embeddings-capable. +The model-source create, read, and update contracts MUST expose the flag, and +the stored value MUST survive a round trip through those contracts. + +#### Scenario: existing sources default to disabled + +- **GIVEN** a model source row that predates the embeddings capability +- **WHEN** the schema migration runs +- **THEN** the source reports `supports_embeddings` as disabled +- **AND** its existing chat-completions, responses, and audio-transcription + routing is unchanged + +#### Scenario: capability round-trips through the API + +- **WHEN** a client creates or updates a model source with the embeddings + capability enabled +- **THEN** reading the source back reports the capability as enabled + +#### Scenario: omitted capability parses as disabled + +- **WHEN** a model-source payload omits `supports_embeddings` +- **THEN** it parses as disabled rather than failing validation + +### Requirement: Embeddings route only to capable model sources + +The system SHALL expose `POST /v1/embeddings` and MUST serve it only from an +enabled model source of kind `openai_compatible` that declares the embeddings +capability and has the requested model enabled. Embeddings requests MUST NOT +fall back to subscription-backed accounts. When the caller presents an API key +restricted to a set of sources, selection MUST stay inside that set. Beyond +the validated `model` and `input` fields, the request payload MUST be +forwarded to the source verbatim. + +#### Scenario: capable source serves the request + +- **GIVEN** an enabled model source declaring the embeddings capability with + the requested model enabled +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy forwards the payload to that source's `/embeddings` + endpoint and returns the upstream JSON response + +#### Scenario: no capable source is a model error + +- **GIVEN** no enabled model source declares the embeddings capability for + the requested model +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy returns 404 with an OpenAI-format error envelope using + code `model_not_found` +- **AND** the request is not routed to a subscription-backed account + +#### Scenario: source-restricted API key cannot escape its set + +- **GIVEN** an API key restricted to a set of model sources +- **WHEN** the only embeddings-capable source for the model is outside that + set +- **THEN** the proxy returns `model_not_found` + +### Requirement: Embeddings requests are accounted like other source routes + +Embeddings responses MUST be inspected for prompt and total token usage. When +the caller's API key requires usage for settlement and the source response +reports none, the proxy MUST fail closed with `usage_unavailable` rather than +serving unmetered traffic. Every embeddings attempt that is dispatched to a +model source MUST produce a request-log entry, with `success` on a forwarded +response and `error` on a forwarding, usage, or settlement failure. That entry +MUST carry the upstream status code when a source returned an HTTP response, +and MUST record the upstream status as absent when the attempt failed before +any response was received. A request rejected before source selection succeeds +is not a dispatched attempt: it MUST NOT produce a request-log entry, because +no source was contacted and no reservation was consumed. + +#### Scenario: missing usage fails closed for a limited key + +- **GIVEN** an API key whose reservation requires reported usage +- **WHEN** the model source returns an embeddings response without a usage + object +- **THEN** the proxy returns an error envelope using code `usage_unavailable` +- **AND** records an error request log + +#### Scenario: forwarding error propagates the upstream status + +- **WHEN** the model source returns an error status for an embeddings request +- **THEN** the proxy returns an OpenAI-format error envelope with that status +- **AND** records an error request log carrying the upstream status code + +#### Scenario: transport failure records an attempt without an upstream status + +- **WHEN** the request to the model source fails before any HTTP response is + received +- **THEN** the proxy records an error request log for the attempt with no + upstream status code + +#### Scenario: unroutable model is not a logged attempt + +- **GIVEN** no enabled model source declares the embeddings capability for + the requested model +- **WHEN** a client posts to `/v1/embeddings` +- **THEN** the proxy returns the `model_not_found` envelope without writing a + request-log entry +- **AND** no reservation is consumed for the rejected request diff --git a/openspec/changes/add-model-source-embeddings/tasks.md b/openspec/changes/add-model-source-embeddings/tasks.md new file mode 100644 index 0000000000..78faa95139 --- /dev/null +++ b/openspec/changes/add-model-source-embeddings/tasks.md @@ -0,0 +1,31 @@ +## 1. Persisted Capability + +- [x] 1.1 Add the `supports_embeddings` column with a `false` server default + and an idempotent migration that tolerates a pre-existing column. +- [x] 1.2 Surface the flag in the model-source create/read/update schemas. +- [x] 1.3 Surface the flag in the dashboard model-source form and locales. + +## 2. Routing + +- [x] 2.1 Add a repository lookup that selects an enabled source declaring + the embeddings capability for the requested model. +- [x] 2.2 Add `POST /v1/embeddings` and forward the payload verbatim beyond + the validated `model` and `input` fields. +- [x] 2.3 Return `model_not_found` when no source qualifies, with no + subscription-account fallback. + +## 3. Accounting + +- [x] 3.1 Parse prompt/total token usage from the embeddings response shape. +- [x] 3.2 Fail closed with `usage_unavailable` when a limited API key needs + usage the source did not report. +- [x] 3.3 Record success and error request logs with the upstream status. + +## 4. Verification + +- [x] 4.1 Add integration coverage for capability routing, the missing-source + path, and usage accounting. +- [x] 4.2 Add frontend coverage for the capability default and the enabled + submit path. +- [x] 4.3 Run Ruff check/format, type checks, and the migration round-trip + test. diff --git a/tests/integration/test_daybreak_capability_routes.py b/tests/integration/test_daybreak_capability_routes.py index a9a93504c9..3b976dd2da 100644 --- a/tests/integration/test_daybreak_capability_routes.py +++ b/tests/integration/test_daybreak_capability_routes.py @@ -60,6 +60,7 @@ ("HTTP", "POST", "/v1/images/generations"), ("HTTP", "POST", "/v1/images/edits"), ("HTTP", "POST", "/v1/chat/completions"), + ("HTTP", "POST", "/v1/embeddings"), ("HTTP", "POST", "/v1/responses/compact"), ("HTTP", "POST", "/backend-api/transcribe"), ("HTTP", "POST", "/backend-api/files"), @@ -173,6 +174,12 @@ async def _request( {"json": {"model": "gpt-5.6-sol", "messages": [{"role": "user", "content": "inert"}]}}, id="chat-completions", ), + pytest.param( + "POST", + "/v1/embeddings", + {"json": {"model": "text-embedding-3-small", "input": "inert"}}, + id="embeddings", + ), ] _PROVIDER_BINARY_ROUTE_CASES = [ @@ -222,6 +229,7 @@ async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: monkeypatch.setattr(ProxyService, "codex_control_request", fail_before_routing) monkeypatch.setattr(proxy_api_module, "_opportunistic_admission_denial", fail_before_routing) monkeypatch.setattr(proxy_api_module, "_select_chat_model_source", fail_before_routing) + monkeypatch.setattr(proxy_api_module, "_select_embeddings_model_source", fail_before_routing) key = await _create_api_key(f"Daybreak route guard {path}") response = await _request( @@ -267,6 +275,7 @@ async def fail_before_routing(*_args: Any, **_kwargs: Any) -> None: "/v1/responses", "/backend-api/codex/responses", "/v1/chat/completions", + "/v1/embeddings", "/v1/images/generations", "/v1/warmup", "/v1/warmup/default", diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index b60fe819f9..97b42e074c 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -41,6 +41,7 @@ async def _create_model_source( supports_responses: bool = False, supports_streaming: bool = True, supports_audio_transcriptions: bool = False, + supports_embeddings: bool = False, ) -> str: model_entry: dict[str, object] = { "model": model, @@ -69,6 +70,7 @@ async def _create_model_source( "supportsChatCompletions": True, "supportsResponses": supports_responses, "supportsAudioTranscriptions": supports_audio_transcriptions, + "supportsEmbeddings": supports_embeddings, "models": [model_entry], }, ) @@ -3298,3 +3300,222 @@ async def capture(request: web.Request) -> web.StreamResponse: reasoning = captured["reasoning"] assert isinstance(reasoning, dict) assert reasoning["effort"] == "minimal" + + +@pytest.mark.asyncio +async def test_source_embeddings_routes_payload_and_settles_usage(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + captured: dict[str, object] = {} + + async def embed(request: web.Request) -> web.Response: + captured["path"] = request.path + captured["authorization"] = request.headers.get("authorization") + captured["payload"] = await request.json() + return web.json_response( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.1, 0.2, 0.3]}], + "model": "all-minilm:latest", + "usage": {"prompt_tokens": 21, "total_tokens": 21}, + } + ) + + base_url = await source_upstream(embed) + model = "all-minilm:latest" + source_id = await _create_model_source( + async_client, + name="embedder", + model=model, + base_url=base_url, + input_per_1m=0.02, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "embeddings-source-key", + "assignedSourceIds": [source_id], + "limits": [ + {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": ["hello", "world"], "encoding_format": "float"}, + ) + + assert response.status_code == 200 + body = response.json() + assert body["object"] == "list" + assert body["data"][0]["embedding"] == [0.1, 0.2, 0.3] + assert captured["path"] == "/v1/embeddings" + assert captured["authorization"] == "Bearer token-embedder" + # Extra OpenAI params pass through verbatim. + assert captured["payload"] == { + "model": model, + "input": ["hello", "world"], + "encoding_format": "float", + } + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.account_id is None + assert log.model_source_id == source_id + assert log.source == "model_source" + assert log.input_tokens == 21 + assert log.output_tokens == 0 + assert log.status == "success" + + +@pytest.mark.asyncio +async def test_source_embeddings_unknown_model_returns_model_not_found(async_client) -> None: + await _enable_api_key_auth(async_client) + created = await async_client.post("/api/api-keys/", json={"name": "embeddings-404-key"}) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": "no-such-embedder", "input": "hello"}, + ) + + assert response.status_code == 404 + assert response.json()["error"]["code"] == "model_not_found" + + # Rejection happens before source selection succeeds, so no source was + # contacted: the attempt must not appear in the request log at all. + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == "no-such-embedder")) + assert result.scalars().all() == [] + + +@pytest.mark.asyncio +async def test_source_embeddings_transport_failure_logs_without_upstream_status(async_client) -> None: + await _enable_api_key_auth(async_client) + model = "unreachable-embedder" + closed_port = _free_port() + source_id = await _create_model_source( + async_client, + name="unreachable-embedder-source", + model=model, + base_url=f"http://127.0.0.1:{closed_port}/v1", + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={"name": "embeddings-unreachable-key", "assignedSourceIds": [source_id]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "model_source_unreachable" + + # The attempt reached dispatch, so it is logged -- but no upstream response + # ever arrived, so there is no upstream status code to carry. + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.status == "error" + assert log.model_source_id == source_id + assert log.upstream_status_code is None + + +@pytest.mark.asyncio +async def test_source_embeddings_upstream_error_passes_through_and_logs(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + + async def embed(request: web.Request) -> web.Response: + return web.json_response( + {"error": {"message": "model exploded", "type": "server_error"}}, + status=500, + ) + + base_url = await source_upstream(embed) + model = "broken-embedder" + source_id = await _create_model_source( + async_client, + name="broken-embedder-source", + model=model, + base_url=base_url, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={"name": "embeddings-error-key", "assignedSourceIds": [source_id]}, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 500 + assert "model exploded" in response.json()["error"]["message"] + + async with SessionLocal() as session: + result = await session.execute(select(RequestLog).where(RequestLog.model == model)) + log = result.scalar_one() + assert log.status == "error" + assert log.model_source_id == source_id + + +@pytest.mark.asyncio +async def test_source_embeddings_without_usage_fails_closed_for_limited_key(async_client, source_upstream) -> None: + await _enable_api_key_auth(async_client) + + async def embed(request: web.Request) -> web.Response: + return web.json_response( + { + "object": "list", + "data": [{"object": "embedding", "index": 0, "embedding": [0.5]}], + "model": "usage-less-embedder", + } + ) + + base_url = await source_upstream(embed) + model = "usage-less-embedder" + source_id = await _create_model_source( + async_client, + name="usage-less-embedder-source", + model=model, + base_url=base_url, + supports_embeddings=True, + ) + created = await async_client.post( + "/api/api-keys/", + json={ + "name": "embeddings-limited-key", + "assignedSourceIds": [source_id], + "limits": [ + {"limitType": "total_tokens", "limitWindow": "weekly", "maxValue": 1_000}, + ], + }, + ) + assert created.status_code == 200 + key = created.json()["key"] + + response = await async_client.post( + "/v1/embeddings", + headers={"Authorization": f"Bearer {key}"}, + json={"model": model, "input": "hello"}, + ) + + assert response.status_code == 502 + assert response.json()["error"]["code"] == "usage_unavailable" diff --git a/tests/unit/test_db_migrate.py b/tests/unit/test_db_migrate.py index deb79efac4..0e203577c1 100644 --- a/tests/unit/test_db_migrate.py +++ b/tests/unit/test_db_migrate.py @@ -2231,7 +2231,11 @@ def test_api_key_reasoning_policy_migration_round_trips_from_current_parent(tmp_ config = _build_alembic_config(url) script_directory = ScriptDirectory.from_config(config) assert script_directory.get_revision(target_revision).down_revision == parent_revision - assert target_revision in script_directory.get_heads() + # Assert reachability from the single head rather than "is the head": every + # later migration would otherwise have to edit this test. + heads = script_directory.get_heads() + assert len(heads) == 1 + assert target_revision in {revision.revision for revision in script_directory.iterate_revisions(heads[0], "base")} engine = create_engine(to_sync_database_url(url)) try: From 8abd50778dd15131fac01f6a40d8e07a1bcebf54 Mon Sep 17 00:00:00 2001 From: mastertyko <11311479+mastertyko@users.noreply.github.com> Date: Thu, 20 Aug 2026 07:09:59 +0200 Subject: [PATCH 089/117] fix(helm): bind TTFT dashboard SQL datasource (#1827) --- deploy/helm/codex-lb/README.md | 7 ++ .../codex-lb/dashboards/ttft-breakdown.json | 37 +++++-- .../.openspec.yaml | 2 + .../context.md | 54 ++++++++++ .../design.md | 98 +++++++++++++++++++ .../proposal.md | 37 +++++++ .../specs/proxy-runtime-observability/spec.md | 37 +++++++ .../tasks.md | 26 +++++ .../proxy-runtime-observability/context.md | 9 ++ .../specs/proxy-runtime-observability/spec.md | 22 +++++ tests/unit/test_helm_monitoring_artifacts.py | 42 ++++++++ tests/unit/test_helm_replica_artifacts.py | 27 +++++ 12 files changed, 392 insertions(+), 6 deletions(-) create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md create mode 100644 openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md diff --git a/deploy/helm/codex-lb/README.md b/deploy/helm/codex-lb/README.md index b661342553..cb9bab6d3d 100644 --- a/deploy/helm/codex-lb/README.md +++ b/deploy/helm/codex-lb/README.md @@ -424,6 +424,13 @@ externalSecrets: enabled: true # Use External Secrets Operator ``` +The Grafana sidecar imports the dashboard JSON but does not provision +datasources or database credentials. In the **codex-lb TTFT Breakdown** +dashboard, select the Grafana PostgreSQL datasource that points to the +codex-lb database from the visible **PostgreSQL** (`DS_SQL`) dropdown. All +four SQL panels follow that one runtime selection. The owning contract is in +[proxy runtime observability](../../../openspec/specs/proxy-runtime-observability/). + Install with: ```bash diff --git a/deploy/helm/codex-lb/dashboards/ttft-breakdown.json b/deploy/helm/codex-lb/dashboards/ttft-breakdown.json index 7ab2751bfd..57c830dea2 100644 --- a/deploy/helm/codex-lb/dashboards/ttft-breakdown.json +++ b/deploy/helm/codex-lb/dashboards/ttft-breakdown.json @@ -5,7 +5,10 @@ "editable": true, "panels": [ { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -29,7 +32,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": {}, "overrides": [] @@ -51,7 +57,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -75,7 +84,10 @@ "type": "table" }, { - "datasource": "${DS_SQL}", + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" + }, "fieldConfig": { "defaults": { "unit": "ms" @@ -106,7 +118,20 @@ "ttft" ], "templating": { - "list": [] + "list": [ + { + "hide": 0, + "includeAll": false, + "label": "PostgreSQL", + "multi": false, + "name": "DS_SQL", + "options": [], + "query": "grafana-postgresql-datasource", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] }, "time": { "from": "now-24h", @@ -116,4 +141,4 @@ "title": "codex-lb TTFT Breakdown", "uid": "codex-lb-ttft-breakdown", "version": 1 -} \ No newline at end of file +} diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md new file mode 100644 index 0000000000..2e24173f3c --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/context.md @@ -0,0 +1,54 @@ +# Context: TTFT dashboard SQL datasource binding + +## Purpose + +Make the sidecar-provisioned TTFT dashboard immediately bindable to an +operator's PostgreSQL datasource without baking a cluster-specific UID into +the chart. + +## Decisions + +- `DS_SQL` remains a runtime dashboard variable because datasource UIDs differ + across Grafana installations. +- The variable is visible and single-select so operators can inspect and + change the active PostgreSQL datasource without editing dashboard JSON. +- Every panel uses Grafana's typed datasource object with PostgreSQL plugin + type and `${DS_SQL}` UID. One dashboard variable remains the single source + of truth for all four panels. +- The dashboard stays in `dashboards/*.json`; the existing Helm template, + sidecar labels, folder annotation, and optional title override remain + unchanged. + +## Constraints + +- Do not add a Helm value for a datasource UID: datasource selection is a + Grafana runtime concern and a new chart setting would duplicate the + dashboard variable. +- Do not provision PostgreSQL credentials or a Grafana datasource from this + chart. +- Preserve each panel's SQL, layout, IDs, titles, and visualization type. + +## Failure Modes + +- A scalar `"datasource": "${DS_SQL}"` is ambiguous to modern Grafana and can + be resolved as a literal missing UID. +- A hidden, multi-value, or include-all variable can make panel execution + non-deterministic or leave operators unable to repair a stale selection. +- A variable that accepts non-PostgreSQL plugins can select a datasource that + cannot execute the dashboard's SQL. + +## Example + +After the Grafana sidecar imports `ttft-breakdown.json`, an operator opens the +dashboard and selects datasource UID `codex-lb-postgres` from the `DS_SQL` +dropdown. All four panels resolve to: + +```json +{ + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" +} +``` + +Grafana substitutes `codex-lb-postgres` at runtime and executes every TTFT +query against that datasource. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md new file mode 100644 index 0000000000..2c151dca89 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/design.md @@ -0,0 +1,98 @@ +## Context + +The Helm chart packages `ttft-breakdown.json` unchanged in a +sidecar-discoverable ConfigMap. The dashboard's four SQL panels currently use +the legacy scalar `"${DS_SQL}"` datasource form while `templating.list` is +empty. Grafana 12.4.4 therefore has no runtime value to interpolate and can +resolve the placeholder as a missing UID. + +Datasource UIDs and credentials are installation-specific. The chart must +remain portable and must not take ownership of provisioning an operator's +Grafana PostgreSQL datasource. + +## Goals / Non-Goals + +**Goals:** + +- Make the selected PostgreSQL datasource explicit, visible, and deterministic + at dashboard runtime. +- Route all four panels through the same selected UID using Grafana 12.4.4's + typed datasource-reference schema. +- Preserve current SQL, layout, sidecar packaging, title overrides, and chart + values. + +**Non-Goals:** + +- Provisioning a Grafana datasource, PostgreSQL credentials, or database + permissions. +- Adding a Helm value for a cluster-specific datasource UID. +- Changing TTFT queries, visualizations, panel layout, navigation, or + application runtime behavior. + +## Decisions + +### Use a classic datasource template variable + +Declare `DS_SQL` with `type: datasource` and plugin query +`grafana-postgresql-datasource`. Keep it visible, single-select, and without +an all-datasources option. + +Alternative: hard-code a datasource UID in Helm. Rejected because UIDs are +installation-specific and would require another chart setting for a Grafana +runtime concern. + +### Use typed panel datasource references + +Each panel uses: + +```json +{ + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}" +} +``` + +Grafana 12.4.4 defines panel datasources as `{type, uid}` references and +interpolates variables in the UID field. The existing scalar form is legacy +input and does not satisfy the current schema. + +Alternative: leave panel references scalar and only add the variable. +Rejected because it preserves the ambiguous representation that produced the +missing-datasource state. + +### Preserve the existing Helm packaging seam + +Keep dashboard JSON under `dashboards/` and let +`templates/grafana-dashboard.yaml` package it unchanged. A rendered ConfigMap +test proves the runtime variable and typed panel references survive Helm. + +Alternative: generate the variable in the template. Rejected because it +duplicates dashboard structure in Go templates and makes standalone dashboard +validation harder. + +## Risks / Trade-offs + +- **No PostgreSQL datasource exists** → The dropdown has no valid selection; + chart documentation states that operators must provision and select one. +- **A saved selection becomes stale** → The variable remains visible so the + operator can select another ordinary PostgreSQL datasource. +- **A datasource connects but lacks request-log access** → Grafana reports the + database/query error normally; this change only owns datasource resolution. +- **Grafana schema behavior changes** → Focused artifact tests and a real + Grafana 12.4.4 API/browser scenario lock the supported contract. + +## Migration Plan + +1. Upgrade or redeploy the chart with Grafana dashboard sidecar support + enabled. +2. Let the sidecar replace the dashboard ConfigMap payload. +3. Open the TTFT dashboard and select the PostgreSQL datasource that points to + the codex-lb database. + +Rollback is a chart rollback to the previous dashboard JSON. No database, +application, secret, or chart-value migration is involved. + +## Open Questions + +None. Grafana 12.4.4 documentation and source confirm the plugin ID, +datasource-variable flags, typed panel reference, and UID interpolation path. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md new file mode 100644 index 0000000000..2c53849563 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/proposal.md @@ -0,0 +1,37 @@ +## Why + +The shipped TTFT breakdown dashboard references `${DS_SQL}` on every panel, +but it does not declare that runtime variable. Grafana therefore treats the +literal placeholder as a datasource UID and renders +`Datasource ${DS_SQL} was not found` instead of executing the PostgreSQL +queries. + +## What Changes + +- Declare `DS_SQL` as a visible, single-select runtime datasource variable + restricted to the PostgreSQL datasource plugin. +- Bind all four TTFT panels through typed datasource objects whose UID is the + selected `DS_SQL` value. +- Preserve Helm sidecar ConfigMap packaging and title overrides. +- Document that operators select the PostgreSQL datasource after the sidecar + provisions the dashboard. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-runtime-observability`: The shipped TTFT dashboard MUST resolve its + SQL panels through an operator-selected PostgreSQL datasource. + +## Impact + +- `deploy/helm/codex-lb/dashboards/ttft-breakdown.json` +- `deploy/helm/codex-lb/README.md` +- focused dashboard-artifact and rendered-ConfigMap tests + +There is no application runtime, API, database schema, chart value, sidecar +label, or navigation change. diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..7c15efe1cc --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: 24-hour TTFT breakdown queries are available + +Operators MUST have an OpenSpec context runbook or dashboard artifact with +24-hour TTFT breakdown queries by user agent group, upstream transport, +model/cache ratio, session gap cohort, prompt size cohort, and prewarm +status/outcome. + +The shipped Grafana TTFT dashboard MUST declare a visible, single-select +runtime datasource variable named `DS_SQL` that is restricted to PostgreSQL. +Every SQL panel MUST bind to the selected UID through a typed PostgreSQL +datasource object. The Helm chart MUST preserve the dashboard in its existing +sidecar-discoverable ConfigMap, and chart documentation MUST tell operators to +select the PostgreSQL datasource in Grafana. + +#### Scenario: Operator investigates TTFT regression + +- **WHEN** an operator needs to inspect the last 24 hours of request-log + latency +- **THEN** the repository provides SQL that reports p50, p90, p95 TTFT and + total latency for the requested breakdowns + +#### Scenario: Sidecar-provisioned dashboard resolves the selected database + +- **GIVEN** the Helm chart renders the Grafana dashboard ConfigMap +- **AND** Grafana has a PostgreSQL datasource available +- **WHEN** the operator selects that datasource through `DS_SQL` +- **THEN** all four TTFT panels resolve to the selected datasource UID +- **AND** no panel reports `Datasource ${DS_SQL} was not found` + +#### Scenario: Datasource choice remains explicit and deterministic + +- **WHEN** Grafana loads the TTFT dashboard +- **THEN** `DS_SQL` is visible to the operator +- **AND** it permits exactly one PostgreSQL datasource selection +- **AND** it does not offer an all-datasources selection diff --git a/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md new file mode 100644 index 0000000000..c06cdf7091 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-bind-ttft-dashboard-sql-datasource/tasks.md @@ -0,0 +1,26 @@ +## 1. Regression coverage + +- [x] 1.1 Add dashboard JSON assertions for the visible single-select + PostgreSQL `DS_SQL` variable and all four typed panel bindings. +- [x] 1.2 Add a rendered ConfigMap assertion proving Helm preserves the + runtime datasource contract. + +## 2. Dashboard and operator documentation + +- [x] 2.1 Declare `DS_SQL` and convert all TTFT panels to typed PostgreSQL + datasource objects without changing SQL or layout. +- [x] 2.2 Document Grafana-side PostgreSQL datasource selection while + preserving the existing sidecar deployment model. + +## 3. Verification + +- [x] 3.1 Capture focused RED, implement the minimal artifact fix, and run the + focused tests GREEN. +- [x] 3.2 Run changed-file diagnostics, Ruff, typecheck, Helm architecture and + rendering gates, strict affected OpenSpec validation, and final diff + review. +- [x] 3.3 Provision the exact rendered dashboard into isolated Grafana 12.4.4 + with a synthetic PostgreSQL datasource; inspect the API and 1440x900 + browser surface and capture evidence. +- [x] 3.4 Sync the verified delta into the owning capability and archive the + completed change before publication. diff --git a/openspec/specs/proxy-runtime-observability/context.md b/openspec/specs/proxy-runtime-observability/context.md index 6a01cd1e21..d7fec3a23e 100644 --- a/openspec/specs/proxy-runtime-observability/context.md +++ b/openspec/specs/proxy-runtime-observability/context.md @@ -12,9 +12,18 @@ See `openspec/specs/proxy-runtime-observability/spec.md` for normative requireme - **Request tracing is opt-in:** outbound request summary and payload tracing remain configurable because payload logs can be noisy or sensitive. Since issue #1340 phase 1 the switch is the single `CODEX_LB_TRACE` comma-separated channel list (`shape`, `shape_raw_cache_key`, `payload`, `service_tier`, `upstream_summary`, `upstream_payload`); empty default = all off. It is an incident-debugging knob for interactive use only. - **Error logs must be correlated:** request id, endpoint, status, code, and message are the minimum useful fields for debugging 4xx/5xx failures. - **Prewarm observability is outcome-only:** the Codex HTTP-bridge prewarm canary experiment finished, so its bucket/cohort dimensions were retired (issue #1340 phase 4). The `codex_lb_http_bridge_prewarm_total` counter is labelled by `outcome` only, request logs record `prewarm_status` / `prewarm_latency_ms` (statuses: `not_applicable`, `skipped`, `success`, `timeout`, `error` — `canary_miss` no longer occurs), and the legacy `prewarm_canary_bucket` / `prewarm_eligible_reason` request-log columns stay declared but unwritten for one release for rolling-upgrade safety; the Alembic drop revision ships next release (see the next-release queue in `openspec/specs/deployment-installation/context.md`). +- **TTFT datasource selection stays in Grafana:** the Helm chart packages the + TTFT dashboard but does not provision a PostgreSQL datasource or its + credentials. The visible, single-select `DS_SQL` variable keeps + installation-specific datasource UIDs out of chart values while routing all + four SQL panels through one explicit selection. ## Operational Notes - Use request ids to correlate inbound proxy logs, outbound upstream traces, and client-visible failures. - Prefer summary tracing in normal debugging sessions; enable payload tracing only when the exact normalized outbound request matters. - For direct compact `5xx` failures, look for `proxy_compact_failure` alongside `upstream_request_complete`; together they show the compact failure phase, failure detail, exception type, retry metadata, and affinity source. +- After the Grafana sidecar imports the TTFT dashboard, select the ordinary + PostgreSQL datasource that points to the codex-lb database from the visible + **PostgreSQL** dropdown. A datasource registered only as a frontend runtime + plugin is not listed by Grafana's datasource variable. diff --git a/openspec/specs/proxy-runtime-observability/spec.md b/openspec/specs/proxy-runtime-observability/spec.md index d715e5cca3..b90c63583d 100644 --- a/openspec/specs/proxy-runtime-observability/spec.md +++ b/openspec/specs/proxy-runtime-observability/spec.md @@ -478,6 +478,13 @@ Operators MUST have an OpenSpec context runbook or dashboard artifact with model/cache ratio, session gap cohort, prompt size cohort, and prewarm status/outcome. +The shipped Grafana TTFT dashboard MUST declare a visible, single-select +runtime datasource variable named `DS_SQL` that is restricted to PostgreSQL. +Every SQL panel MUST bind to the selected UID through a typed PostgreSQL +datasource object. The Helm chart MUST preserve the dashboard in its existing +sidecar-discoverable ConfigMap, and chart documentation MUST tell operators to +select the PostgreSQL datasource in Grafana. + #### Scenario: Operator investigates TTFT regression - **WHEN** an operator needs to inspect the last 24 hours of request-log @@ -485,6 +492,21 @@ status/outcome. - **THEN** the repository provides SQL that reports p50, p90, p95 TTFT and total latency for the requested breakdowns +#### Scenario: Sidecar-provisioned dashboard resolves the selected database + +- **GIVEN** the Helm chart renders the Grafana dashboard ConfigMap +- **AND** Grafana has a PostgreSQL datasource available +- **WHEN** the operator selects that datasource through `DS_SQL` +- **THEN** all four TTFT panels resolve to the selected datasource UID +- **AND** no panel reports `Datasource ${DS_SQL} was not found` + +#### Scenario: Datasource choice remains explicit and deterministic + +- **WHEN** Grafana loads the TTFT dashboard +- **THEN** `DS_SQL` is visible to the operator +- **AND** it permits exactly one PostgreSQL datasource selection +- **AND** it does not offer an all-datasources selection + ### Requirement: Dashboard request logs show generation speed The dashboard request-log table MUST show time to first token and output-token generation speed when the required latency and output-token fields are available. Generation speed MUST use output tokens divided by elapsed generation time after time to first token, not total input plus output tokens and not total request latency including TTFT. diff --git a/tests/unit/test_helm_monitoring_artifacts.py b/tests/unit/test_helm_monitoring_artifacts.py index 5d6afc1fb6..c6e52b209a 100644 --- a/tests/unit/test_helm_monitoring_artifacts.py +++ b/tests/unit/test_helm_monitoring_artifacts.py @@ -12,6 +12,48 @@ _REPO_ROOT = Path(__file__).resolve().parents[2] _CHART_DIR = _REPO_ROOT / "deploy" / "helm" / "codex-lb" +_POSTGRES_DATASOURCE_TYPE = "grafana-postgresql-datasource" + + +def _ttft_dashboard() -> dict: + return json.loads((_CHART_DIR / "dashboards" / "ttft-breakdown.json").read_text()) + + +def test_ttft_dashboard_declares_visible_single_select_postgres_datasource() -> None: + dashboard = _ttft_dashboard() + (datasource,) = dashboard["templating"]["list"] + + assert { + "name": datasource["name"], + "label": datasource["label"], + "type": datasource["type"], + "query": datasource["query"], + "hide": datasource["hide"], + "multi": datasource["multi"], + "includeAll": datasource["includeAll"], + } == { + "name": "DS_SQL", + "label": "PostgreSQL", + "type": "datasource", + "query": _POSTGRES_DATASOURCE_TYPE, + "hide": 0, + "multi": False, + "includeAll": False, + } + + +def test_ttft_dashboard_panels_bind_selected_postgres_datasource_uid() -> None: + dashboard = _ttft_dashboard() + + assert len(dashboard["panels"]) == 4 + assert all( + panel["datasource"] + == { + "type": _POSTGRES_DATASOURCE_TYPE, + "uid": "${DS_SQL}", + } + for panel in dashboard["panels"] + ) def test_high_error_rate_alert_aggregates_request_series_before_division() -> None: diff --git a/tests/unit/test_helm_replica_artifacts.py b/tests/unit/test_helm_replica_artifacts.py index 8b46a505ac..d11197ea53 100644 --- a/tests/unit/test_helm_replica_artifacts.py +++ b/tests/unit/test_helm_replica_artifacts.py @@ -164,6 +164,33 @@ def test_grafana_dashboard_titles_can_be_overridden() -> None: assert dashboard_config["data"]["ttft-breakdown.json"] == raw_dashboard_values["ttft-breakdown.json"] +def test_rendered_ttft_dashboard_keeps_runtime_postgres_datasource_binding() -> None: + rendered = _helm_template( + "--set", + "metrics.grafanaDashboard.enabled=true", + "--show-only", + "templates/grafana-dashboard.yaml", + ) + (dashboard_config,) = _helm_documents(rendered) + dashboard = json.loads(dashboard_config["data"]["ttft-breakdown.json"]) + (datasource,) = dashboard["templating"]["list"] + + assert datasource["name"] == "DS_SQL" + assert datasource["type"] == "datasource" + assert datasource["query"] == "grafana-postgresql-datasource" + assert datasource["hide"] == 0 + assert datasource["multi"] is False + assert datasource["includeAll"] is False + assert all( + panel["datasource"] + == { + "type": "grafana-postgresql-datasource", + "uid": "${DS_SQL}", + } + for panel in dashboard["panels"] + ) + + def _prod_overlay_args(*args: str) -> tuple[str, ...]: return ( "-f", From 028a75c33701494834c054718fb58e31d72c4d99 Mon Sep 17 00:00:00 2001 From: HanSu Lee Date: Thu, 20 Aug 2026 14:21:10 +0900 Subject: [PATCH 090/117] fix(reports): format full Cost values with grouping separators (#1814) * fix(reports): group full Cost values * test(reports): cover grouped Cost axis ticks * docs: add hanseo0507 as contributor --------- Co-authored-by: Soju06 --- .all-contributorsrc | 10 ++++ README.md | 1 + .../components/cost-per-day-chart.test.tsx | 58 +++++++++++++++++-- .../reports/components/cost-per-day-chart.tsx | 5 +- .../components/daily-detail-table.test.tsx | 24 ++++++++ .../reports/components/daily-detail-table.tsx | 3 +- .../components/reports-summary-cards.test.tsx | 24 ++++++++ .../components/reports-summary-cards.tsx | 5 +- .../proposal.md | 23 ++++++++ .../specs/frontend-architecture/spec.md | 20 +++++++ .../tasks.md | 16 +++++ 11 files changed, 180 insertions(+), 9 deletions(-) create mode 100644 openspec/changes/fix-reports-full-cost-thousands-separators/proposal.md create mode 100644 openspec/changes/fix-reports-full-cost-thousands-separators/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/fix-reports-full-cost-thousands-separators/tasks.md diff --git a/.all-contributorsrc b/.all-contributorsrc index 02f9cd703a..6af2d185ae 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1302,6 +1302,16 @@ "test", "doc" ] + }, + { + "login": "hanseo0507", + "name": "HanSu Lee", + "avatar_url": "https://avatars.githubusercontent.com/u/56479293?v=4", + "profile": "https://github.com/hanseo0507", + "contributions": [ + "code", + "test" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 4c92ae4da5..487fc39a9d 100644 --- a/README.md +++ b/README.md @@ -295,6 +295,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e zenasharp
    zenasharp
    💻 ⚠️ 📖 + HanSu Lee
    HanSu Lee
    💻 ⚠️ diff --git a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx index 1bc4405b45..94ed30a1a7 100644 --- a/frontend/src/features/reports/components/cost-per-day-chart.test.tsx +++ b/frontend/src/features/reports/components/cost-per-day-chart.test.tsx @@ -1,10 +1,28 @@ -import type { ReactNode } from "react"; +import type { ReactElement, ReactNode } from "react"; import { render, screen } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; import { CostPerDayChart } from "./cost-per-day-chart"; -let capturedProps: { margin?: unknown; data?: unknown } | null = null; +type ChartProps = { children: ReactNode; margin?: unknown; data?: unknown }; + +let capturedProps: ChartProps | null = null; +let capturedYAxisProps: { tickFormatter?: (value: number) => string } | null = null; + +function findTooltipContent(node: ReactNode): ReactElement<{ formatValue?: (value: number) => string }> | null { + if (!node || typeof node !== "object") return null; + if (Array.isArray(node)) { + for (const child of node) { + const found = findTooltipContent(child); + if (found) return found; + } + return null; + } + + const element = node as ReactElement<{ content?: ReactElement<{ formatValue?: (value: number) => string }>; children?: ReactNode }>; + if (element.props.content) return element.props.content; + return findTooltipContent(element.props.children); +} vi.mock("@/components/lazy-recharts", async (importOriginal) => { const actual = await importOriginal(); @@ -14,11 +32,14 @@ vi.mock("@/components/lazy-recharts", async (importOriginal) => { ResponsiveContainer: ({ children }: { children: ReactNode }) =>
    {children}
    , AreaChart: (props: { children: ReactNode; margin?: unknown; data?: unknown }) => { capturedProps = props; - return
    ; + return
    {props.children}
    ; }, Area: () => null, XAxis: () => null, - YAxis: () => null, + YAxis: (props: { tickFormatter?: (value: number) => string }) => { + capturedYAxisProps = props; + return null; + }, CartesianGrid: () => null, Tooltip: () => null, }; @@ -27,6 +48,7 @@ vi.mock("@/components/lazy-recharts", async (importOriginal) => { describe("CostPerDayChart", () => { beforeEach(() => { capturedProps = null; + capturedYAxisProps = null; }); it("uses equal left and right chart margins", () => { @@ -55,6 +77,34 @@ describe("CostPerDayChart", () => { expect(capturedProps?.margin).toEqual({ top: 5, right: 10, left: 10, bottom: 0 }); }); + it("formats full-value Cost axis and tooltip amounts with grouping separators", () => { + render( + , + ); + + const tooltip = findTooltipContent(capturedProps?.children); + expect(capturedYAxisProps?.tickFormatter?.(1400)).toBe("$1,400.00"); + expect(tooltip?.props.formatValue?.(1400)).toBe("$1,400.00"); + }); + it("fills missing selected days with zero-value rows", () => { render( `$${v}`} + tickFormatter={formatCurrency} /> `$${v.toFixed(2)}`} />} + content={} /> { ); }); + it("renders grouped currency in full-value Cost cells", () => { + render( + , + ); + + expect(within(screen.getByTestId("daily-breakdown-row-2026-06-05")).getByText("$1,400.00")).toBeInTheDocument(); + }); + it("zero-fills cancelled counts for dates missing from the response", () => { const rows = buildContinuousDailyRows("2026-06-05", "2026-06-06", [ { diff --git a/frontend/src/features/reports/components/daily-detail-table.tsx b/frontend/src/features/reports/components/daily-detail-table.tsx index 4d539db94d..29d05e0771 100644 --- a/frontend/src/features/reports/components/daily-detail-table.tsx +++ b/frontend/src/features/reports/components/daily-detail-table.tsx @@ -7,6 +7,7 @@ import { useDateDisplayFormatStore } from "@/hooks/use-date-format"; import { buildContinuousDailyRows } from "../daily-series"; import type { DailyReportRow } from "../schemas"; import { formatReportBucketDate } from "../date"; +import { formatCurrency } from "@/utils/formatters"; export type DailyDetailTableProps = { startDate: string; @@ -156,7 +157,7 @@ export function DailyDetailTable({ startDate, endDate, data }: DailyDetailTableP {row.reasoningTokens == null ? "—" : formatTokens(row.reasoningTokens)} - ${row.costUsd.toFixed(2)} + {formatCurrency(row.costUsd)} {row.activeAccounts} diff --git a/frontend/src/features/reports/components/reports-summary-cards.test.tsx b/frontend/src/features/reports/components/reports-summary-cards.test.tsx index dc80664d22..a7b1219173 100644 --- a/frontend/src/features/reports/components/reports-summary-cards.test.tsx +++ b/frontend/src/features/reports/components/reports-summary-cards.test.tsx @@ -126,6 +126,30 @@ describe("ReportsSummaryCards", () => { expect(within(tokensCard).queryByText("170")).not.toBeInTheDocument(); }); + it("renders grouped currency for full-value Cost displays", () => { + render( + , + ); + + const costCard = screen.getByTestId("report-summary-card-total-cost"); + expect(within(costCard).getByText("$1,400.00")).toBeInTheDocument(); + expect(costCard).toHaveTextContent("avg $1,400.00/day"); + }); + it("hides comparison badges when unavailable or previous totals are zero", () => { const { rerender } = render( Date: Thu, 20 Aug 2026 07:33:42 +0200 Subject: [PATCH 091/117] fix(proxy): preserve compact terminal error type (#1824) Co-authored-by: Soju06 --- app/core/clients/proxy.py | 5 +- .../.openspec.yaml | 2 + .../design.md | 58 +++++++++++++++++++ .../proposal.md | 31 ++++++++++ .../specs/responses-api-compat/spec.md | 22 +++++++ .../tasks.md | 15 +++++ openspec/specs/responses-api-compat/spec.md | 21 +++++++ tests/unit/test_codex_upstream_paths.py | 51 +++++++++++++--- 8 files changed, 195 insertions(+), 10 deletions(-) create mode 100644 openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml create mode 100644 openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md create mode 100644 openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md create mode 100644 openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index 9b602ba5fd..4b99b3d607 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -1469,10 +1469,13 @@ def _compact_sse_terminal_error_payload( error_code = payload.get("code") error_message = payload.get("message") if isinstance(error_code, str) and error_code and isinstance(error_message, str) and error_message: + error_type = payload.get("error_type") + if not isinstance(error_type, str) or not error_type.strip(): + error_type = "server_error" detail: OpenAIErrorDetail = { "code": error_code, "message": error_message, - "type": "server_error", + "type": error_type, } param = payload.get("param") if isinstance(param, str) and param: diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml new file mode 100644 index 0000000000..41c30bab88 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-19 diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md new file mode 100644 index 0000000000..f4ebc97bf1 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/design.md @@ -0,0 +1,58 @@ +## Context + +Compact Responses receives an upstream HTTP response whose body can terminate +with an SSE event. Nested OpenAI-style error objects are parsed through the +shared error parser and preserve their type. A top-level event shaped as +`{"type":"error","error_type":...,"code":...,"message":...}` instead uses a +fallback converter. + +The status-code helper already reads top-level `error_type`, so it can infer +HTTP 400/401/403/429 correctly. The fallback envelope independently hard-codes +`server_error`, producing an internally inconsistent public response. + +## Goals / Non-Goals + +**Goals** + +- Preserve a supplied non-blank top-level `error_type`. +- Retain `server_error` when the field is absent, non-string, or blank. +- Leave nested envelopes and all other mapped fields/statuses unchanged. + +**Non-Goals** + +- Change compact request routing, retries, account selection, or health. +- Infer new status codes or normalize arbitrary upstream error types. +- Change non-compact Responses or nested error parsing. + +## Decisions + +### Fix only the top-level fallback + +The fallback converter reads `payload["error_type"]`. A string containing at +least one non-whitespace character becomes the OpenAI detail `type`; otherwise +the existing `server_error` value remains. + +This keeps the fix at the data-loss seam and avoids changing the shared parser +or status inference that already behave correctly. + +### Preserve supplied type text + +Whitespace is used only to decide whether a value is blank. A non-blank string +is forwarded verbatim, matching the existing field-preservation behavior for +top-level `code`, `message`, and `param`. + +## Risks / Trade-offs + +- Upstream can supply an unfamiliar type. Preserving it is preferable to + fabricating `server_error` and matches OpenAI-compatible passthrough behavior. +- The fallback remains intentionally conservative for absent, non-string, or + whitespace-only values. + +## Migration Plan + +No migration, setting, or rollout step is required. Rollback restores the +previous top-level type substitution. + +## Open Questions + +None. diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md new file mode 100644 index 0000000000..42b1fefad5 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/proposal.md @@ -0,0 +1,31 @@ +## Why + +The compact Responses transport derives the correct HTTP status from a +top-level terminal SSE frame's `error_type`, but its fallback OpenAI envelope +replaces that supplied type with `server_error`. Clients therefore receive a +contradictory response such as HTTP 400 with `error.type=server_error` even +though the upstream classified the failure as `invalid_request_error`. + +## What Changes + +- Preserve a non-empty top-level compact SSE `error_type` in the emitted OpenAI + error envelope. +- Keep `server_error` as the compatibility fallback when the top-level field is + absent or blank. +- Preserve existing nested error-envelope behavior and status, code, message, + and parameter mapping. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: define compact terminal error-envelope behavior for + top-level `type=error` SSE frames. + +## Impact + +- Affects the compact SSE terminal-error converter in + `app/core/clients/proxy.py`. +- Adds focused routed transport tests and fallback/nested controls. +- Does not change request routing, retry behavior, account health, schemas, + settings, dependencies, or non-compact Responses behavior. diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..b2c9cd48d4 --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/specs/responses-api-compat/spec.md @@ -0,0 +1,22 @@ +## ADDED Requirements + +### Requirement: Compact terminal SSE errors preserve top-level error type + +When the compact Responses upstream terminates with a top-level SSE `type=error` frame, the proxy MUST preserve a supplied non-blank `error_type` in the emitted OpenAI error envelope. If `error_type` is absent, non-string, or blank, the proxy MUST use `server_error`. The proxy MUST preserve existing status, code, message, and parameter mapping, and MUST NOT alter nested OpenAI-style error-envelope behavior. + +#### Scenario: Top-level invalid request type is preserved + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is `invalid_request_error` +- **THEN** the proxy returns HTTP 400 with `error.type=invalid_request_error` +- **AND** preserves the frame's code, message, and parameter + +#### Scenario: Missing or blank top-level type uses compatibility fallback + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is absent or blank +- **THEN** the emitted OpenAI error envelope uses `error.type=server_error` +- **AND** existing status, code, message, and parameter mapping remains unchanged + +#### Scenario: Nested compact error envelope remains unchanged + +- **WHEN** compact upstream terminates with a nested OpenAI-style error envelope +- **THEN** the proxy preserves the nested type and all other mapped fields using the existing parser diff --git a/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md new file mode 100644 index 0000000000..265df4b8ca --- /dev/null +++ b/openspec/changes/archive/2026-08-19-preserve-compact-top-level-error-type/tasks.md @@ -0,0 +1,15 @@ +## 1. Regression Coverage + +- [x] 1.1 Add a routed compact top-level terminal SSE regression that preserves `invalid_request_error` +- [x] 1.2 Add missing and blank `error_type` fallback controls and retain the nested-envelope control + +## 2. Compact Error Conversion + +- [x] 2.1 Preserve a supplied non-blank top-level `error_type` in the OpenAI error detail +- [x] 2.2 Keep status, code, message, parameter, nested-envelope, and `server_error` fallback behavior unchanged + +## 3. Verification + +- [x] 3.1 Run focused compact tests, Ruff, type checking, proxy architecture checks, and strict affected OpenSpec validation +- [x] 3.2 Exercise the live compact HTTP route with top-level invalid-request and missing-type upstream terminal frames +- [x] 3.3 Verify implementation against this change, synchronize the delta, and archive the verified OpenSpec change diff --git a/openspec/specs/responses-api-compat/spec.md b/openspec/specs/responses-api-compat/spec.md index 3bf0552710..1e5472c160 100644 --- a/openspec/specs/responses-api-compat/spec.md +++ b/openspec/specs/responses-api-compat/spec.md @@ -5433,3 +5433,24 @@ transition. - **GIVEN** an API-key reservation requires settlement during the failed retry - **WHEN** account health is updated - **THEN** required settlement still completes before deferred health writes + +### Requirement: Compact terminal SSE errors preserve top-level error type + +When the compact Responses upstream terminates with a top-level SSE `type=error` frame, the proxy MUST preserve a supplied non-blank `error_type` in the emitted OpenAI error envelope. If `error_type` is absent, non-string, or blank, the proxy MUST use `server_error`. The proxy MUST preserve existing status, code, message, and parameter mapping, and MUST NOT alter nested OpenAI-style error-envelope behavior. + +#### Scenario: Top-level invalid request type is preserved + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is `invalid_request_error` +- **THEN** the proxy returns HTTP 400 with `error.type=invalid_request_error` +- **AND** preserves the frame's code, message, and parameter + +#### Scenario: Missing or blank top-level type uses compatibility fallback + +- **WHEN** compact upstream terminates with a top-level `type=error` frame whose `error_type` is absent or blank +- **THEN** the emitted OpenAI error envelope uses `error.type=server_error` +- **AND** existing status, code, message, and parameter mapping remains unchanged + +#### Scenario: Nested compact error envelope remains unchanged + +- **WHEN** compact upstream terminates with a nested OpenAI-style error envelope +- **THEN** the proxy preserves the nested type and all other mapped fields using the existing parser diff --git a/tests/unit/test_codex_upstream_paths.py b/tests/unit/test_codex_upstream_paths.py index 7c1ca372ec..9c8d627799 100644 --- a/tests/unit/test_codex_upstream_paths.py +++ b/tests/unit/test_codex_upstream_paths.py @@ -164,10 +164,15 @@ class _CompactTerminalErrorStreamResponse: status = 200 status_code = 200 headers = {"content-type": "text/event-stream"} - content = ( - b'data: {"type":"error","code":"rate_limit_exceeded","message":"quota closed",' - b'"param":"previous_response_id"}\n\n' - ) + + def __init__(self, error_type: str, error_code: str) -> None: + self.content = ( + b'data: {"type":"error","error_type":"' + + error_type.encode("utf-8") + + b'","code":"' + + error_code.encode("utf-8") + + b'","message":"compact rejected","param":"previous_response_id"}\n\n' + ) class _CompactTerminalFailedStreamResponse: @@ -621,10 +626,20 @@ async def test_compact_responses_terminal_sse_error_infers_status_from_error_det @pytest.mark.asyncio -async def test_compact_responses_routed_top_level_sse_error_preserves_fields( +@pytest.mark.parametrize( + ("error_type", "error_code", "expected_status"), + [ + ("invalid_request_error", "invalid_request_error", 400), + ("rate_limit_error", "rate_limit_exceeded", 429), + ], +) +async def test_compact_responses_routed_top_level_sse_error_preserves_type( route: ResolvedUpstreamRoute, + error_type: str, + error_code: str, + expected_status: int, ) -> None: - client = _RouteMetadataCodexClient(_CompactTerminalErrorStreamResponse()) + client = _RouteMetadataCodexClient(_CompactTerminalErrorStreamResponse(error_type, error_code)) payload = ResponsesCompactRequest(model="gpt-5.2", instructions="Summarize.", input="hello") with pytest.raises(ProxyResponseError) as exc_info: @@ -638,13 +653,31 @@ async def test_compact_responses_routed_top_level_sse_error_preserves_fields( codex_client=cast(Any, client), ) - assert exc_info.value.status_code == 429 + assert exc_info.value.status_code == expected_status error = exc_info.value.payload["error"] - assert error["code"] == "rate_limit_exceeded" - assert error["message"] == "quota closed" + assert error["type"] == error_type + assert error["code"] == error_code + assert error["message"] == "compact rejected" assert error["param"] == "previous_response_id" +@pytest.mark.parametrize("error_type", [None, "", " ", 123]) +def test_compact_top_level_sse_error_type_uses_server_error_fallback( + error_type: object, +) -> None: + payload: dict[str, Any] = { + "type": "error", + "code": "upstream_error", + "message": "compact failed", + } + if error_type is not None: + payload["error_type"] = error_type + + detail = proxy_module._compact_sse_terminal_error_payload(payload, "error") + + assert detail["error"]["type"] == "server_error" + + @pytest.mark.parametrize( ("payload", "expected_status"), [ From d148dd9a42dca3088e8063aca8a21682f9bf7fb6 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 09:48:38 +0400 Subject: [PATCH 092/117] =?UTF-8?q?feat(config):=20timeout-invariant=20lin?= =?UTF-8?q?ter=20=E2=80=94=20validate=20deadline/TTL=20inequalities=20at?= =?UTF-8?q?=20startup=20and=20in=20CI=20(#1622)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rebased onto current main; squashed the branch's review-iteration commits. Co-authored-by: Soju06 --- app/core/config/settings.py | 1 + app/core/timeout_invariants.py | 306 ++++++++++++++++++ app/main.py | 2 + docs/reference/settings.md | 3 +- .../add-timeout-invariant-linter/notes.md | 55 ++++ .../add-timeout-invariant-linter/proposal.md | 36 +++ .../specs/deployment-installation/spec.md | 41 +++ .../specs/proxy-runtime-observability/spec.md | 18 ++ .../add-timeout-invariant-linter/tasks.md | 23 ++ .../specs/deployment-installation/context.md | 42 +++ .../proxy-runtime-observability/context.md | 4 + tests/unit/test_settings_reference.py | 6 +- tests/unit/test_timeout_invariants.py | 180 +++++++++++ 13 files changed, 715 insertions(+), 2 deletions(-) create mode 100644 app/core/timeout_invariants.py create mode 100644 openspec/changes/add-timeout-invariant-linter/notes.md create mode 100644 openspec/changes/add-timeout-invariant-linter/proposal.md create mode 100644 openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md create mode 100644 openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md create mode 100644 openspec/changes/add-timeout-invariant-linter/tasks.md create mode 100644 tests/unit/test_timeout_invariants.py diff --git a/app/core/config/settings.py b/app/core/config/settings.py index c8bdac6209..00376d1a53 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -477,6 +477,7 @@ def upstream_websocket_proxy_env(self) -> Mapping[str, str | None]: workers_per_instance: int = Field(default=1, ge=1) proxy_refresh_failure_cooldown_seconds: float = Field(default=5.0, ge=0.0) usage_refresh_auth_failure_cooldown_seconds: float = Field(default=300.0, ge=0.0) + timeout_invariant_validation_strict: bool = False # Local memory-pressure guard (0 = disabled). Requests are rejected with # 503 once RSS reaches the threshold; a warning is logged from 80% of it diff --git a/app/core/timeout_invariants.py b/app/core/timeout_invariants.py new file mode 100644 index 0000000000..e23d14b9a1 --- /dev/null +++ b/app/core/timeout_invariants.py @@ -0,0 +1,306 @@ +"""Startup timeout-invariant validation over raw ``Settings`` values. + +This module intentionally validates only startup ``Settings`` fields and a +small set of code constants whose relations are fixed at import/runtime. It +does not validate per-request ContextVar overrides +(``app/core/clients/proxy.py:3450-3467``, +``app/modules/proxy/_service/streaming/helpers.py:861-868``, +``app/modules/proxy/_service/compact.py:727-738``, +``app/modules/proxy/_service/transcribe.py:230-232``, +``app/core/clients/files.py:77-90``, and +``app/modules/proxy/service.py:1464-1478``), runtime clamps/derived effective +values (``app/core/clients/proxy.py:1049-1088``, +``app/core/auth/refresh.py:391-395``, and +``app/modules/proxy/load_balancer.py:1846-1856``), or DB/API-key/model-source +runtime settings (``app/core/config/settings_cache.py:22-36``, +``app/modules/settings/api.py:547-710``, +``app/modules/proxy/_service/streaming/retry.py:153-165``, and +``app/modules/model_sources/forwarding.py:112-221``). +""" + +from __future__ import annotations + +import argparse +import logging +import operator +import sys +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from typing import Protocol + +logger = logging.getLogger(__name__) + + +class TimeoutSettings(Protocol): + upstream_connect_timeout_seconds: float + proxy_request_budget_seconds: float + http_responses_stream_request_budget_seconds: float + compact_request_budget_seconds: float + sse_keepalive_interval_seconds: float + http_responses_session_bridge_request_budget_seconds: float + http_responses_session_bridge_stuck_gate_retire_after_seconds: float + http_responses_session_bridge_clean_close_retry_jitter_max_seconds: float + proxy_admission_wait_timeout_seconds: float + proxy_account_lease_ttl_seconds: float + model_registry_enabled: bool + + @property + def model_registry_snapshot_max_age_seconds(self) -> int | float: ... + + timeout_invariant_validation_strict: bool + + +@dataclass(frozen=True, slots=True) +class TimeoutOperand: + label: str + evaluate: Callable[[TimeoutSettings], float] + code_anchor: str + + +@dataclass(frozen=True, slots=True) +class TimeoutInvariantRule: + id: str + lhs: TimeoutOperand + relation: str + rhs: TimeoutOperand + rationale: str + + +@dataclass(frozen=True, slots=True) +class TimeoutInvariantViolation: + rule: TimeoutInvariantRule + lhs_value: float + rhs_value: float + + def format(self) -> str: + return ( + f"{self.rule.id}: {self.rule.lhs.label}={self.lhs_value:g} " + f"{self.rule.relation} {self.rule.rhs.label}={self.rhs_value:g} violated; " + f"{self.rule.rationale} " + f"(lhs: {self.rule.lhs.code_anchor}; rhs: {self.rule.rhs.code_anchor})" + ) + + +class TimeoutInvariantError(RuntimeError): + def __init__(self, violations: Sequence[TimeoutInvariantViolation]) -> None: + self.violations = tuple(violations) + super().__init__("\n".join(violation.format() for violation in self.violations)) + + +def _field(name: str, anchor: str) -> TimeoutOperand: + return TimeoutOperand(name, lambda settings: float(getattr(settings, name)), anchor) + + +def _expr(label: str, anchor: str, evaluate: Callable[[TimeoutSettings], float]) -> TimeoutOperand: + return TimeoutOperand(label, evaluate, anchor) + + +UPSTREAM_CONNECT = _field("upstream_connect_timeout_seconds", "app/core/clients/proxy.py:2720") +PROXY_BUDGET = _field("proxy_request_budget_seconds", "app/core/config/settings.py:260") +STREAM_BUDGET = _field( + "http_responses_stream_request_budget_seconds", + "app/modules/proxy/_service/streaming/helpers.py:724", +) +COMPACT_BUDGET = _field("compact_request_budget_seconds", "app/modules/proxy/_service/compact.py:585") +SSE_KEEPALIVE = _field("sse_keepalive_interval_seconds", "app/modules/proxy/api.py:3930") +TOKEN_REFRESH = _field("token_refresh_timeout_seconds", "app/modules/accounts/auth_manager.py:1123") +BRIDGE_BUDGET = _field( + "http_responses_session_bridge_request_budget_seconds", + "app/modules/proxy/_service/http_bridge/helpers.py:2469", +) +BRIDGE_CLEAN_CLOSE_JITTER = _field( + "http_responses_session_bridge_clean_close_retry_jitter_max_seconds", + "app/modules/proxy/_service/http_bridge/request_submit.py:294", +) +ADMISSION_WAIT = _field("proxy_admission_wait_timeout_seconds", "app/modules/proxy/service.py:768") +ACCOUNT_LEASE_TTL = _field("proxy_account_lease_ttl_seconds", "app/modules/proxy/load_balancer.py:1993") +BRIDGE_STUCK_GATE_HARD_ANCHOR_RETIRE = _expr( + "2 * http_responses_session_bridge_stuck_gate_retire_after_seconds", + "app/modules/proxy/_service/http_bridge/helpers.py:686", + lambda settings: 2.0 * settings.http_responses_session_bridge_stuck_gate_retire_after_seconds, +) +MODEL_REGISTRY_SNAPSHOT_MAX_AGE = _field( + "model_registry_snapshot_max_age_seconds", + "app/core/openai/model_registry_store.py:367", +) +MODEL_REGISTRY_REFRESH_INTERVAL = _expr( + "_REFRESH_INTERVAL_SECONDS", + "app/core/openai/model_refresh_scheduler.py:37", + lambda settings: _model_registry_refresh_interval_seconds(), +) +DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL = _expr( + "DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS", + "app/modules/proxy/durable_bridge_repository.py:42", + lambda settings: _durable_bridge_retry_circuit_state_ttl_seconds(), +) +DURABLE_BRIDGE_RETRY_CIRCUIT_MIN_TTL = _expr( + "_HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS", + "app/modules/proxy/_service/http_bridge/retry_circuit.py:19-21", + lambda settings: _durable_bridge_retry_circuit_min_ttl_seconds(), +) + + +def _model_registry_refresh_interval_seconds() -> float: + from app.core.openai.model_refresh_scheduler import _REFRESH_INTERVAL_SECONDS + + return float(_REFRESH_INTERVAL_SECONDS) + + +def _durable_bridge_retry_circuit_state_ttl_seconds() -> float: + from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS + + return float(DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS) + + +def _durable_bridge_retry_circuit_min_ttl_seconds() -> float: + from app.modules.proxy._service.http_bridge.retry_circuit import ( + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS, + _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS, + ) + + return float(_HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS) + + +TIMEOUT_INVARIANT_RULES: tuple[TimeoutInvariantRule, ...] = ( + TimeoutInvariantRule( + "admission-wait-within-proxy-budget", + ADMISSION_WAIT, + "<=", + PROXY_BUDGET, + "Global admission waits must not consume more than the request budget they protect.", + ), + TimeoutInvariantRule( + "admission-wait-within-stream-budget", + ADMISSION_WAIT, + "<=", + STREAM_BUDGET, + "Streaming retries wait for capacity inside the stream budget and must leave room for the stream attempt.", + ), + TimeoutInvariantRule( + "admission-wait-within-compact-budget", + ADMISSION_WAIT, + "<=", + COMPACT_BUDGET, + "Compact response-create admission must not outlive the compact request budget.", + ), + TimeoutInvariantRule( + "bridge-stuck-gate-retire-within-bridge-budget", + BRIDGE_STUCK_GATE_HARD_ANCHOR_RETIRE, + "<", + BRIDGE_BUDGET, + "Hard-continuity stuck gate retirement waits up to 2x the configured threshold and must happen before " + "the bridge request budget is exhausted.", + ), + TimeoutInvariantRule( + "account-lease-ttl-covers-proxy-budget", + ACCOUNT_LEASE_TTL, + ">=", + PROXY_BUDGET, + "Response-create leases use the raw lease TTL, so stale reclaim must not precede a healthy non-stream " + "request deadline.", + ), + TimeoutInvariantRule( + "account-lease-ttl-covers-compact-budget", + ACCOUNT_LEASE_TTL, + ">=", + COMPACT_BUDGET, + "Compact response-create leases must not be stale-reclaimed before the compact request budget expires.", + ), + TimeoutInvariantRule( + "model-registry-snapshot-outlives-refresh-interval", + MODEL_REGISTRY_SNAPSHOT_MAX_AGE, + ">", + MODEL_REGISTRY_REFRESH_INTERVAL, + "Persisted model-registry snapshots must remain loadable for at least one fixed refresh cadence.", + ), + TimeoutInvariantRule( + "durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open", + DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL, + ">", + DURABLE_BRIDGE_RETRY_CIRCUIT_MIN_TTL, + "Durable HTTP bridge retry-circuit state must outlive the longest cooldown and half-open lease.", + ), +) + +# TODO(timeout_sem_001): database_migration_lock_timeout_seconds is independent startup DB migration policy. +# TODO(timeout_sem_008): proxy_downstream_websocket_idle_timeout_seconds has no verified ordering with bridge TTL. +# TODO(timeout_sem_009): oauth_timeout_seconds is used in OAuth/client flows, not a verified proxy-path deadline. +# TODO(timeout_sem_015): openai_cache_affinity_max_age_seconds participates with dashboard prompt-cache TTL in +# cleanup retention. +# TODO(timeout_sem_021): upstream_route_cache_ttl_seconds is invalidation freshness policy; no timeout inequality +# verified. +# timeout_sem_022 is enforced by model-registry-snapshot-outlives-refresh-interval. +# TODO(timeout_sem_023): firewall_ip_cache_ttl_seconds has no verified timeout owner beyond trust-cache freshness. +# TODO(timeout_sem_024): leader_election_ttl_seconds renewal is derived internally as ttl//3, not a cross-setting +# inequality. +# TODO(timeout_sem_027): proxy_account_cap_partition_scale_down_seconds is a stability window; exact heartbeat relation +# is internal. +# TODO(timeout_sem_029): usage_refresh_auth_failure_cooldown_seconds is policy cooldown, not a verified scheduler +# inequality. +# TODO(timeout_sem_030): shutdown_drain_timeout_seconds depends on deployment termination grace outside Settings. +# timeout_sem_031 is enforced by durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open. +# TODO(timeout_sem_032/033): SQLite busy retry constants are module-local and not Settings-field rules. +# TODO(timeout_sem_034/035): account-selection recovery caps are module constants clamped by request deadlines at +# runtime. + +_RELATIONS: dict[str, Callable[[float, float], bool]] = { + "<": operator.lt, + "<=": operator.le, + ">": operator.gt, + ">=": operator.ge, +} + + +def find_timeout_invariant_violations(settings: TimeoutSettings) -> list[TimeoutInvariantViolation]: + violations: list[TimeoutInvariantViolation] = [] + for rule in TIMEOUT_INVARIANT_RULES: + if rule.id == "model-registry-snapshot-outlives-refresh-interval" and not settings.model_registry_enabled: + continue + lhs_value = rule.lhs.evaluate(settings) + rhs_value = rule.rhs.evaluate(settings) + if not _RELATIONS[rule.relation](lhs_value, rhs_value): + violations.append(TimeoutInvariantViolation(rule, lhs_value, rhs_value)) + return violations + + +def validate_timeout_invariants( + settings: TimeoutSettings, + *, + strict: bool = False, + log: bool = True, +) -> list[TimeoutInvariantViolation]: + violations = find_timeout_invariant_violations(settings) + if violations and log: + for violation in violations: + logger.critical("timeout invariant violation: %s", violation.format()) + if strict and violations: + raise TimeoutInvariantError(violations) + return violations + + +def validate_runtime_timeout_invariants(settings: TimeoutSettings) -> list[TimeoutInvariantViolation]: + return validate_timeout_invariants( + settings, + strict=settings.timeout_invariant_validation_strict, + log=True, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + from app.core.config.settings import get_settings + + parser = argparse.ArgumentParser(description="Validate codex-lb timeout invariants.") + parser.add_argument("--strict", action="store_true", help="exit nonzero when any invariant is violated") + args = parser.parse_args(argv) + + violations = validate_timeout_invariants(get_settings(), strict=False, log=True) + if not violations: + print(f"OK: {len(TIMEOUT_INVARIANT_RULES)} timeout invariant rules satisfied") + return 0 + for violation in violations: + print(violation.format(), file=sys.stderr) + return 1 if args.strict else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/app/main.py b/app/main.py index 2baa10a9f7..81a53f8845 100644 --- a/app/main.py +++ b/app/main.py @@ -58,6 +58,7 @@ from app.core.retention.scheduler import build_data_retention_scheduler from app.core.scheduling.leader_election import get_leader_election from app.core.shutdown import close_control_plane_task_admission +from app.core.timeout_invariants import validate_runtime_timeout_invariants from app.core.usage.refresh_scheduler import build_usage_refresh_scheduler from app.core.usage.reset_credits_refresh_scheduler import build_rate_limit_reset_credits_scheduler from app.core.utils.time import utcnow @@ -332,6 +333,7 @@ async def lifespan(app: FastAPI): reload_additional_quota_registry() settings = get_settings() warn_removed_settings() + validate_runtime_timeout_invariants(settings) # Anchor round-robin tie-break decorrelation to this replica's stable bridge # instance identity so peer replicas spread exact ties across equally-good # accounts instead of all herding onto the lexicographically-first account. diff --git a/docs/reference/settings.md b/docs/reference/settings.md index cb350e1607..058da37953 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 129 settings. Every setting is an environment +codex-lb currently exposes 130 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -253,6 +253,7 @@ the host side of the compose `ports` mapping instead. | --- | --- | --- | | `CODEX_LB_TELEMETRY_ENABLED` | `bool \| None` | `None` | | `CODEX_LB_TELEMETRY_ENDPOINT` | `str` | `'https://telemetry.tokmaxxing.com'` | +| `CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT` | `bool` | `False` | | `CODEX_LB_WARMUP_MODEL` | `str` | `'gpt-5.4-mini'` | ## Removed / deprecated diff --git a/openspec/changes/add-timeout-invariant-linter/notes.md b/openspec/changes/add-timeout-invariant-linter/notes.md new file mode 100644 index 0000000000..f6a735edd4 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/notes.md @@ -0,0 +1,55 @@ +# Timeout Invariant Linter Audit Disposition + +Inputs: `LINTER_AUDIT.md` and the 11 PR #1622 inline findings fetched with +`gh api repos/Soju06/codex-lb/pulls/1622/comments`. + +| # | Rule | Audit verdict | Bot finding | Disposition | +|---:|---|---|---|---| +| 1 | `upstream-connect-within-proxy-budget` | CIRCULAR | Wrong connect anchor | Removed; generic client clamps connect to total budget, so this was circular. | +| 2 | `upstream-connect-within-stream-budget` | GROUNDED | Wrong connect anchor family | Deferred; anchor fixed to `app/core/clients/proxy.py:2720` but not enforced by the shipped registry. | +| 3 | `upstream-connect-within-compact-budget` | CIRCULAR | None | Removed; compact passes remaining budget as an override/clamp. | +| 4 | `upstream-connect-within-bridge-budget` | CIRCULAR | None | Removed; bridge request budget does not directly own upstream connect. | +| 5 | `admission-plus-connect-within-proxy-budget` | GROUNDED | Wrong phase/circular deadline | Removed; runtime recomputes remaining absolute budget after admission. | +| 6 | `admission-plus-connect-within-compact-budget` | GROUNDED | Wrong phase/circular deadline family | Removed; compact also passes remaining deadline-derived overrides. | +| 7 | `admission-wait-within-proxy-budget` | GROUNDED | None | Kept. | +| 8 | `admission-wait-within-stream-budget` | GROUNDED | None | Kept. | +| 9 | `admission-wait-within-compact-budget` | GROUNDED | None | Kept. | +| 10 | `admission-wait-within-bridge-budget` | CIRCULAR | None | Removed; bridge admission is clamped to remaining bridge budget. | +| 11 | `stream-idle-within-stream-budget` | GROUNDED | Bot says total may precede idle | Removed; total and idle are independent aiohttp limits. | +| 12 | `stream-idle-within-bridge-budget` | GROUNDED | Bot says same phase family | Removed; outer bridge deadline may validly fire before idle. | +| 13 | `sse-keepalive-before-stream-idle` | GROUNDED | Downstream keepalive cannot reset upstream idle | Removed; it compared independent directions. | +| 14 | `sse-keepalive-within-stream-budget` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 15 | `sse-keepalive-within-bridge-budget` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 16 | `token-refresh-claim-covers-admission-and-exchange` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 17 | `refresh-failure-cooldown-within-claim-ttl` | GROUNDED | Cooldown is process-local cache | Removed; cooldown does not extend claim ownership. | +| 18 | `token-refresh-exchange-within-claim-ttl` | GROUNDED | None | Deferred; not enforced by the shipped registry. | +| 19 | `usage-fetch-within-refresh-interval` | WRONG | Scheduler serializes, cadence may slip | Removed. | +| 20 | `usage-fetch-within-reset-credits-interval` | WRONG | Usage cadence family | Removed. | +| 21 | `compact-budget-within-proxy-budget` | CIRCULAR | Compact lane independent | Removed. | +| 22 | `bridge-idle-ttl-within-bridge-budget` | WRONG | Reuse TTL may exceed request budget | Removed. | +| 23 | `bridge-codex-idle-ttl-within-bridge-budget` | WRONG | Reuse TTL family | Removed. | +| 24 | `bridge-stuck-gate-retire-after-admission` | GROUNDED | Separate phase from admission | Removed; response-created acknowledgement retirement is independent from queue admission. | +| 25 | `bridge-stuck-gate-retire-within-bridge-budget` | GROUNDED | Missing hard-anchor 2x multiplier | Fixed; now compares `2 * retire_after` with bridge budget. | +| 26 | `bridge-clean-close-jitter-within-admission` | CIRCULAR | Jitter/admission independent | Removed. | +| 27 | `bridge-clean-close-jitter-within-bridge-budget` | GROUNDED | None | Deferred; no settings field anchors the clean-close jitter, so the rule has nothing to read. | +| 28 | `account-lease-ttl-covers-proxy-budget` | GROUNDED | None | Kept. | +| 29 | `account-lease-ttl-covers-compact-budget` | GROUNDED | None | Kept. | +| 30 | `model-registry-snapshot-outlives-refresh-interval` | Proposed new | None | Added; `model_registry_snapshot_max_age_seconds > _REFRESH_INTERVAL_SECONDS`. | +| 31 | `durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open` | Proposed new | None | Added; retry-circuit state TTL must outlive max backoff and half-open lease. | + +Shipped registry: `TIMEOUT_INVARIANT_RULES` enforces exactly the eight rules +above marked Kept, Fixed, or Added (rows 7, 8, 9, 25, 28, 29, 30, 31), and +`tests/unit/test_timeout_invariants.py` pins that count. Rows marked Deferred +were judged grounded by the audit but ship unenforced: this validator runs at +startup and can abort the process under +`CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT`, so a rule enters the registry +only once it has a settings anchor and a default configuration that satisfies +it. + +Validation scope: + +- Validated: startup `Settings` fields and imported constants used by the rule + table. +- Not validated: per-request `ContextVar` overrides, runtime clamps, derived + effective values, and database/API-key/model-source timeout inputs loaded + after startup. diff --git a/openspec/changes/add-timeout-invariant-linter/proposal.md b/openspec/changes/add-timeout-invariant-linter/proposal.md new file mode 100644 index 0000000000..b1c196e24a --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/proposal.md @@ -0,0 +1,36 @@ +## Why + +Timeout and TTL mismatches have repeatedly caused healthy codex-lb work to be +killed by a different, shorter budget. The project needs those relationships +encoded as executable configuration policy instead of scattered comments. + +## What Changes + +- Add a declarative timeout-invariant rule table over effective Settings fields. +- Validate the effective startup configuration in non-strict mode by default, + logging CRITICAL for every violated rule. +- Add `timeout_invariant_validation_strict` for deployments that want startup to + fail on violations. +- Add a strict CI entrypoint via `python -m app.core.timeout_invariants`. + +## Capabilities + +### New Capabilities + +(none) + +### Modified Capabilities + +- `deployment-installation`: defines startup and CI validation for timeout + invariants. +- `proxy-runtime-observability`: defines low-cardinality CRITICAL diagnostics + for timeout-invariant violations. + +## Impact + +- Code: `app/core/timeout_invariants.py`, startup lifespan settings validation, + and the Settings strict-mode flag. +- Tests: focused unit coverage for defaults, inverted config detection, strict + raise, and the CI entrypoint. +- Operators: default deployments continue to start; strict mode opts into + fail-fast behavior. diff --git a/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md b/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md new file mode 100644 index 0000000000..664283dcc8 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/specs/deployment-installation/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Timeout invariants are validated at startup and in CI + +The application SHALL define executable timeout-invariant rules over effective +startup `Settings` fields and explicitly imported code constants for verified +relationships between request budgets, TTLs, refresh deadlines, admission +waits, retry jitter, fixed refresh cadence, and durable retry-circuit state. +Each rule SHALL name the compared setting, constant, or expression; the +relation; and a one-line rationale describing the runtime failure prevented. +Unverified timeout inventory entries SHALL NOT be enforced until their code +relationship is verified. + +At startup, the application SHALL validate the effective startup `Settings` +object against the rule table. This validation SHALL NOT claim coverage for +per-request `ContextVar` overrides, runtime clamps, derived effective values +computed after startup, or database/API-key/model-source timeout values loaded +after startup. By default, startup SHALL log every violation at CRITICAL and +continue. When `timeout_invariant_validation_strict` is true, startup SHALL raise +after logging the violations. The project SHALL expose a runnable CI entrypoint +that validates the same rule table, defaults to non-strict reporting, and exits +nonzero only when `--strict` is passed and any rule is violated. + +#### Scenario: Default settings satisfy timeout invariants + +- **WHEN** timeout-invariant validation runs against default settings +- **THEN** every enforced rule passes +- **AND** the CI entrypoint exits successfully + +#### Scenario: Non-strict startup reports violations without failing + +- **WHEN** effective settings violate one or more timeout-invariant rules +- **AND** strict timeout-invariant validation is disabled +- **THEN** startup validation logs every violated rule at CRITICAL +- **AND** startup may continue + +#### Scenario: Strict startup rejects violations + +- **WHEN** effective settings violate one or more timeout-invariant rules +- **AND** `timeout_invariant_validation_strict` is true +- **THEN** startup validation raises an error that includes the violated rule ids diff --git a/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md b/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..d92fb631e2 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,18 @@ +## ADDED Requirements + +### Requirement: Timeout-invariant violations are diagnosable + +Timeout-invariant validation diagnostics SHALL include the rule id, left-hand +setting or expression and value, relation, right-hand setting or expression and +value, rationale, and code anchors. Diagnostics SHALL avoid request payloads, +API keys, access tokens, raw affinity keys, account emails, and other +high-cardinality runtime identifiers. Diagnostics SHALL describe startup +validation of `Settings` and imported constants only, not per-request overrides, +runtime clamps, or runtime-derived effective values. + +#### Scenario: Violation log names the invariant + +- **WHEN** startup timeout-invariant validation observes a violated rule +- **THEN** the CRITICAL log includes that rule id and rationale +- **AND** the log contains no request payload, API key, access token, raw + affinity key, or account email diff --git a/openspec/changes/add-timeout-invariant-linter/tasks.md b/openspec/changes/add-timeout-invariant-linter/tasks.md new file mode 100644 index 0000000000..9d6b2fbd40 --- /dev/null +++ b/openspec/changes/add-timeout-invariant-linter/tasks.md @@ -0,0 +1,23 @@ +## 1. Timeout invariant policy + +- [x] 1.1 Verify curated timeout inequalities against current code before + encoding them. +- [x] 1.2 Add a declarative rule table with the accepted 8 verified startup + inequalities and code-anchored rationales. +- [x] 1.3 Leave unverified curated timeout entries as TODOs rather than + enforcing them. + +## 2. Runtime and CI validation + +- [x] 2.1 Validate effective settings during application startup. +- [x] 2.2 Keep startup non-strict by default and log CRITICAL for violations. +- [x] 2.3 Add strict mode that raises on violations. +- [x] 2.4 Add a runnable CI entrypoint that validates settings strictly and + exits nonzero on violations. + +## 3. Regression coverage + +- [x] 3.1 Prove defaults satisfy all enforced rules. +- [x] 3.2 Prove an inverted configuration names the specific violated rule. +- [x] 3.3 Prove strict mode raises. +- [x] 3.4 Run focused tests and OpenSpec validation before commit. diff --git a/openspec/specs/deployment-installation/context.md b/openspec/specs/deployment-installation/context.md index a91bbcdb43..d26484a1e1 100644 --- a/openspec/specs/deployment-installation/context.md +++ b/openspec/specs/deployment-installation/context.md @@ -10,6 +10,48 @@ fixed, and how removed settings are retired. See `openspec/specs/deployment-installation/spec.md` for normative requirements. +## Timeout Invariant Linter Scope + +The timeout invariant linter is a startup `Settings` guardrail. Strict mode is +an opt-in startup or CI failure path for violating startup configuration, not a +general runtime timeout validator. + +Validated inputs: + +- The `Settings` object materialized at startup. +- Explicitly imported code constants used by the two constant-backed rules: + model-registry refresh cadence and durable HTTP bridge retry-circuit TTL. + +Known non-goals and follow-ups: + +- Per-request `ContextVar` overrides are not revalidated. Current anchors: + `app/core/clients/proxy.py:3450-3467`, + `app/modules/proxy/_service/streaming/helpers.py:861-868`, + `app/modules/proxy/_service/compact.py:727-738`, + `app/modules/proxy/_service/transcribe.py:230-232`, + `app/core/clients/files.py:77-90`, and + `app/modules/proxy/service.py:1464-1478`. +- Runtime clamps and derived effective values are not fully modeled. Current + anchors: `app/core/clients/proxy.py:1049-1088`, + `app/core/auth/refresh.py:391-395`, and + `app/modules/proxy/load_balancer.py:1846-1856`. +- Runtime DB, API-key, and model-source settings can affect timeout-bearing + paths without startup revalidation. Current anchors: + `app/core/config/settings_cache.py:22-36`, + `app/modules/settings/api.py:547-710`, + `app/modules/proxy/_service/streaming/retry.py:153-165`, and + `app/modules/model_sources/forwarding.py:112-221`. + +Example: `python -m app.core.timeout_invariants --strict` validates the +startup `Settings` view and exits nonzero when any enforced rule fails. +Running the same command without `--strict` reports violations but exits zero, +matching the default startup behavior. + +`CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT` is intentionally a setting +rather than a hard default because existing deployments may carry legacy timeout +values that deserve CRITICAL diagnostics first, not surprise startup refusal. +The default remains non-strict; operators and CI opt into fail-fast behavior. + ## Helm termination-grace upgrade contract The graceful-shutdown chart adds a render-time guard: diff --git a/openspec/specs/proxy-runtime-observability/context.md b/openspec/specs/proxy-runtime-observability/context.md index d7fec3a23e..78d23ccee9 100644 --- a/openspec/specs/proxy-runtime-observability/context.md +++ b/openspec/specs/proxy-runtime-observability/context.md @@ -27,3 +27,7 @@ See `openspec/specs/proxy-runtime-observability/spec.md` for normative requireme PostgreSQL datasource that points to the codex-lb database from the visible **PostgreSQL** dropdown. A datasource registered only as a frontend runtime plugin is not listed by Grafana's datasource variable. +- Timeout invariant violation logs describe startup `Settings` and imported + constant validation only. They intentionally avoid request-scoped overrides, + runtime-derived effective timeout values, payloads, API keys, access tokens, + raw affinity keys, account emails, and other high-cardinality identifiers. diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index 620f94b6e0..0475c32243 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -65,7 +65,11 @@ def _isolated_settings(**overrides: Any) -> Settings: # #1618). telemetry_enabled has no hardcoded default because tri-state None # drives the informed-consent dialog; the endpoint stays settable so # self-hosters can point at their own collector or air-gap it. -MAX_SETTINGS_FIELDS = 129 +# 129 -> 130: timeout_invariant_validation_strict (#1622). This stays +# operator-selectable because startup invariant failures need two supported +# modes: report-only by default for mixed/self-hosted environments, and +# fail-fast when CI or strict operators want config drift to abort startup. +MAX_SETTINGS_FIELDS = 130 def test_generated_settings_reference_matches_code() -> None: diff --git a/tests/unit/test_timeout_invariants.py b/tests/unit/test_timeout_invariants.py new file mode 100644 index 0000000000..49ca18b80c --- /dev/null +++ b/tests/unit/test_timeout_invariants.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import logging +from types import SimpleNamespace + +import pytest + +from app.core.config.settings import Settings, get_settings +from app.core.timeout_invariants import ( + TIMEOUT_INVARIANT_RULES, + TimeoutInvariantError, + find_timeout_invariant_violations, + main, + validate_runtime_timeout_invariants, + validate_timeout_invariants, +) +from app.modules.proxy import durable_bridge_repository +from app.modules.proxy._service.http_bridge import retry_circuit + +pytestmark = pytest.mark.unit + + +def test_default_settings_satisfy_timeout_invariants() -> None: + settings = Settings() + assert len(TIMEOUT_INVARIANT_RULES) == 8 + assert find_timeout_invariant_violations(settings) == [] + + +def _timeout_settings(**overrides: float | bool) -> SimpleNamespace: + settings = Settings() + values = { + name: getattr(settings, name) + for name in ( + "upstream_connect_timeout_seconds", + "proxy_request_budget_seconds", + "http_responses_stream_request_budget_seconds", + "compact_request_budget_seconds", + "stream_idle_timeout_seconds", + "sse_keepalive_interval_seconds", + "usage_fetch_timeout_seconds", + "usage_refresh_interval_seconds", + "rate_limit_reset_credits_refresh_interval_seconds", + "http_responses_session_bridge_request_budget_seconds", + "http_responses_session_bridge_idle_ttl_seconds", + "http_responses_session_bridge_codex_idle_ttl_seconds", + "http_responses_session_bridge_stuck_gate_retire_after_seconds", + "http_responses_session_bridge_clean_close_retry_jitter_max_seconds", + "proxy_admission_wait_timeout_seconds", + "proxy_account_lease_ttl_seconds", + "proxy_refresh_failure_cooldown_seconds", + "model_registry_enabled", + "model_registry_snapshot_max_age_seconds", + "timeout_invariant_validation_strict", + ) + } + values.update(overrides) + return SimpleNamespace(**values) + + +@pytest.mark.parametrize( + ("rule_id", "overrides"), + [ + ("admission-wait-within-proxy-budget", {"proxy_request_budget_seconds": 9.0}), + ("admission-wait-within-stream-budget", {"http_responses_stream_request_budget_seconds": 9.0}), + ("admission-wait-within-compact-budget", {"compact_request_budget_seconds": 9.0}), + ( + "bridge-stuck-gate-retire-within-bridge-budget", + {"http_responses_session_bridge_request_budget_seconds": 600.0}, + ), + ("account-lease-ttl-covers-proxy-budget", {"proxy_account_lease_ttl_seconds": 599.0}), + ("account-lease-ttl-covers-compact-budget", {"proxy_account_lease_ttl_seconds": 179.0}), + ( + "model-registry-snapshot-outlives-refresh-interval", + {"model_registry_enabled": True, "model_registry_snapshot_max_age_seconds": 300.0}, + ), + ], +) +def test_each_settings_backed_rule_names_violation(rule_id: str, overrides: dict[str, float]) -> None: + settings = _timeout_settings(**overrides) + + violations = find_timeout_invariant_violations(settings) + + assert any(violation.rule.id == rule_id for violation in violations) + formatted = "\n".join(violation.format() for violation in violations) + assert rule_id in formatted + + +def test_disabled_model_registry_skips_snapshot_cadence_rule() -> None: + settings = _timeout_settings( + model_registry_enabled=False, + model_registry_snapshot_max_age_seconds=1.0, + ) + + violations = find_timeout_invariant_violations(settings) + + assert all(violation.rule.id != "model-registry-snapshot-outlives-refresh-interval" for violation in violations) + + +def test_durable_bridge_retry_circuit_rule_names_violation(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr( + durable_bridge_repository, + "DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS", + retry_circuit._HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS + + retry_circuit._HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS + - 1.0, + ) + + violations = find_timeout_invariant_violations(Settings()) + + rule_id = "durable-bridge-retry-circuit-ttl-covers-backoff-and-half-open" + assert any(violation.rule.id == rule_id for violation in violations) + assert rule_id in "\n".join(violation.format() for violation in violations) + + +def test_non_strict_startup_validation_logs_critical(caplog: pytest.LogCaptureFixture) -> None: + settings = Settings(proxy_request_budget_seconds=5.0) + + with caplog.at_level(logging.CRITICAL, logger="app.core.timeout_invariants"): + violations = validate_runtime_timeout_invariants(settings) + + assert violations + assert "timeout invariant violation: admission-wait-within-proxy-budget" in caplog.text + + +def test_strict_mode_raises() -> None: + settings = Settings( + proxy_request_budget_seconds=5.0, + timeout_invariant_validation_strict=True, + ) + + with pytest.raises(TimeoutInvariantError) as exc_info: + validate_runtime_timeout_invariants(settings) + + assert "admission-wait-within-proxy-budget" in str(exc_info.value) + + +def test_explicit_strict_validation_raises() -> None: + settings = Settings(proxy_request_budget_seconds=5.0) + + with pytest.raises(TimeoutInvariantError, match="admission-wait-within-proxy-budget"): + validate_timeout_invariants(settings, strict=True, log=False) + + +def test_cli_entrypoint_accepts_defaults(capsys: pytest.CaptureFixture[str]) -> None: + assert main([]) == 0 + captured = capsys.readouterr() + assert "timeout invariant rules satisfied" in captured.out + + +def test_cli_strict_flag_exits_one_and_reports_rule( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + get_settings.cache_clear() + monkeypatch.setenv("CODEX_LB_PROXY_REQUEST_BUDGET_SECONDS", "5") + try: + assert main(["--strict"]) == 1 + finally: + get_settings.cache_clear() + + captured = capsys.readouterr() + assert "admission-wait-within-proxy-budget" in captured.err + + +def test_cli_without_strict_exits_zero_and_reports_violation( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, + capsys: pytest.CaptureFixture[str], +) -> None: + get_settings.cache_clear() + monkeypatch.setenv("CODEX_LB_PROXY_REQUEST_BUDGET_SECONDS", "5") + try: + with caplog.at_level(logging.CRITICAL, logger="app.core.timeout_invariants"): + assert main([]) == 0 + finally: + get_settings.cache_clear() + + captured = capsys.readouterr() + assert "admission-wait-within-proxy-budget" in captured.err + assert "timeout invariant violation: admission-wait-within-proxy-budget" in caplog.text From cab503223af1138917ac54d492988df990fdb42c Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 16:14:13 +0900 Subject: [PATCH 093/117] chore(repo): drop root agent-debris files and enforce a root-file allowlist budget (#1837) Removes root agent-debris files (SUMMARY.md deleted; DECISIONS.md ADR relocated to openspec/specs/proxy-architecture/context.md as a git rename) and adds a [root_files] allowlist to the simplicity-budget checker so unlisted tracked root entries fail CI with the standard label escape hatch. OpenSpec change: add-root-file-allowlist-budget (contribution-simplicity delta). --- .github/scripts/check_simplicity_budgets.py | 71 ++++++++- .github/simplicity-budgets.toml | 40 +++++ .gitignore | 1 + SUMMARY.md | 140 ------------------ .../.openspec.yaml | 2 + .../proposal.md | 25 ++++ .../specs/contribution-simplicity/spec.md | 32 ++++ .../add-root-file-allowlist-budget/tasks.md | 14 ++ .../specs/proxy-architecture/context.md | 8 +- 9 files changed, 185 insertions(+), 148 deletions(-) delete mode 100644 SUMMARY.md create mode 100644 openspec/changes/add-root-file-allowlist-budget/.openspec.yaml create mode 100644 openspec/changes/add-root-file-allowlist-budget/proposal.md create mode 100644 openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md create mode 100644 openspec/changes/add-root-file-allowlist-budget/tasks.md rename DECISIONS.md => openspec/specs/proxy-architecture/context.md (92%) diff --git a/.github/scripts/check_simplicity_budgets.py b/.github/scripts/check_simplicity_budgets.py index 51d22c8933..e688013261 100644 --- a/.github/scripts/check_simplicity_budgets.py +++ b/.github/scripts/check_simplicity_budgets.py @@ -1,9 +1,11 @@ #!/usr/bin/env python3 -"""Enforce simplicity budgets on README, .env.example, and the dashboard core nav. +"""Enforce simplicity budgets on README, .env.example, the dashboard core nav, and the tracked root tree. Budgets live in .github/simplicity-budgets.toml and are enforced by .github/workflows/simplicity-budgets.yml. Intentionally stdlib-only so it runs -on the runner's python3 before project dependencies are installed. +on the runner's python3 before project dependencies are installed; the +[root_files] check additionally shells out to `git ls-tree` against the +checkout's HEAD. Override: the 'simplicity-budget-approved' PR label (passed in via the PR_LABELS env var as a JSON array of label names) downgrades violations to @@ -12,8 +14,8 @@ Exit codes: 0 = within budget (or overridden), 1 = over budget, 2 = configuration error (the budget config is missing or malformed, a -budgeted file or the nav array is missing, or an ALL-CONTRIBUTORS-LIST -block is opened but never closed). +budgeted file or the nav array is missing, the tracked root tree cannot +be listed, or an ALL-CONTRIBUTORS-LIST block is opened but never closed). """ from __future__ import annotations @@ -21,6 +23,7 @@ import json import os import re +import subprocess import sys import tomllib from pathlib import Path @@ -129,6 +132,34 @@ def count_nav_items(path: Path, array: str) -> int: return len(re.findall(r"\bto:\s*[\"']", match.group("body"))) +def _escape_annotation_value(value: str) -> str: + """Escape a contributor-controlled value for a workflow-command line ('%' first, per Actions rules).""" + for char, escape in (("%", "%25"), ("\r", "%0D"), ("\n", "%0A"), (":", "%3A"), (",", "%2C")): + value = value.replace(char, escape) + return value + + +def list_tracked_root_entries() -> list[str]: + """List tracked repository-root entries from HEAD; exit 2 loudly if git cannot.""" + try: + proc = subprocess.run( + ["git", "ls-tree", "--name-only", "-z", "HEAD"], + capture_output=True, + encoding="utf-8", + # Non-UTF-8 filename bytes become \x escapes: they can never match + # an allowlist entry, so they surface as a named violation instead + # of a decode crash. + errors="backslashreplace", + check=True, + ) + except FileNotFoundError: + _config_error("[root_files] git executable not found; the root-entry budget needs a git checkout") + except subprocess.CalledProcessError as exc: + detail = (exc.stderr or "").strip() or f"exit code {exc.returncode}" + _config_error(f"[root_files] 'git ls-tree --name-only HEAD' failed: {detail}") + return [entry for entry in proc.stdout.split("\0") if entry] + + def _override_labels() -> list[str]: raw = os.environ.get("PR_LABELS") or "[]" try: @@ -164,9 +195,26 @@ def main() -> int: except (KeyError, TypeError, ValueError) as exc: _config_error(f"budget config '{CONFIG_PATH}' is missing or has a malformed section/key: {exc!r}") + # [root_files] is optional: absent means the root-entry budget is not + # enforced (older configs keep working), present-but-malformed is a + # config error like any other section. + root_allowed: set[str] | None = None + root_cfg = config.get("root_files") + if root_cfg is not None: + try: + allowed_entries = root_cfg["allowed"] + except (KeyError, TypeError) as exc: + _config_error(f"budget config '{CONFIG_PATH}' has a malformed [root_files] section: {exc!r}") + if not isinstance(allowed_entries, list) or not all(isinstance(entry, str) for entry in allowed_entries): + _config_error(f"budget config '{CONFIG_PATH}' [root_files] 'allowed' must be an array of strings") + root_allowed = set(allowed_entries) + readme_lines = strip_contributors_block(_read_lines(readme_path, "readme")) env_lines = _read_lines(env_path, "env_example") nav_items = count_nav_items(nav_path, nav_array) + unexpected_root_entries: list[str] = [] + if root_allowed is not None: + unexpected_root_entries = sorted(set(list_tracked_root_entries()) - root_allowed) metrics: list[tuple[str, Path, int, int]] = [ ( @@ -192,12 +240,25 @@ def main() -> int: if actual > budget: violations.append((name, path, actual, budget)) - if not violations: + if root_allowed is not None: + status = "OK" if not unexpected_root_entries else "OVER" + print(f"tracked root entries outside allowlist: {len(unexpected_root_entries)}/0 {status}") + + if not violations and not unexpected_root_entries: return 0 annotation = "warning" if overridden else "error" for name, path, actual, budget in violations: print(f"::{annotation} file={path}::simplicity budget exceeded: {name}: {actual} > {budget}") + for entry in unexpected_root_entries: + # Entry names come from the tree, not the trusted config: escape them + # so a crafted filename cannot break or forge workflow-command lines. + shown = _escape_annotation_value(entry) + print( + f"::{annotation} file={shown}::simplicity budget exceeded: tracked root entry '{shown}' is not in " + f"the [root_files] allowlist — add it to {CONFIG_PATH} in the same diff, or a maintainer applies " + f"the '{OVERRIDE_LABEL}' PR label" + ) if overridden: print(f"Budgets exceeded, but the '{OVERRIDE_LABEL}' label is applied; passing with warnings. {OVERRIDE_HELP}") diff --git a/.github/simplicity-budgets.toml b/.github/simplicity-budgets.toml index ded1760f3c..7adf5af4c5 100644 --- a/.github/simplicity-budgets.toml +++ b/.github/simplicity-budgets.toml @@ -26,3 +26,43 @@ max_lines = 60 path = "frontend/src/components/layout/app-header.tsx" array = "CORE_NAV_ITEMS" max_items = 5 + +[root_files] +# Complete allowlist of tracked repository-root entries (files and +# directories), compared against `git ls-tree --name-only HEAD`. A new root +# entry is a reviewable one-line diff here; anything not listed fails the +# check. Keep the list sorted. +allowed = [ + ".agents", + ".all-contributorsrc", + ".claude", + ".dockerignore", + ".env.example", + ".github", + ".gitignore", + ".pre-commit-config.yaml", + "AGENTS.md", + "CHANGELOG.md", + "CLAUDE.md", + "Dockerfile", + "Dockerfile.distroless", + "LICENSE", + "Makefile", + "PRINCIPLES.md", + "README.md", + "README.zh-CN.md", + "app", + "config", + "deploy", + "docker-compose.prod.yml", + "docker-compose.yml", + "docs", + "frontend", + "mkdocs.yml", + "openspec", + "pyproject.toml", + "renovate.json", + "scripts", + "tests", + "uv.lock", +] diff --git a/.gitignore b/.gitignore index f1489e0a3e..77f8a1b4f0 100644 --- a/.gitignore +++ b/.gitignore @@ -58,5 +58,6 @@ certs/ .omx/ PROMPT.md SUMMARY.md +DECISIONS.md .superpowers/ .agents/worktrees/ diff --git a/SUMMARY.md b/SUMMARY.md deleted file mode 100644 index 79c3a6b038..0000000000 --- a/SUMMARY.md +++ /dev/null @@ -1,140 +0,0 @@ -# Summary - -## Root cause - -This bug was a three-fault chain: - -1. `/v1/responses` payloads carrying `{"type":"input_image","file_id":"file_*"}` or `{"type":"input_image","image_url":"sediment://file_*"}` were forwarded upstream even though the Responses surface only accepts inline `data:` URLs for conversation `input_image` parts. -2. codex-lb persisted only `file_id -> account_id`, so after `/backend-api/files/{file_id}/uploaded` completed it had no stored `download_url` / `mime_type` to pull the uploaded bytes back and rewrite them into the codex-style inline image form. -3. When upstream rejected that bad shape, the HTTP responses bridge saw a clean close (`close_code=1000`) with zero `response.*` events and treated it as transient, looping through `retry_precreated` / `retry_fresh_upstream` until the request budget expired. - -## What changed - -### `app/core/clients/proxy.py` - -- Added `_ws_transport_payload_budget_bytes(settings)` so auto transport selection respects the deploy's `max_sse_event_bytes` with 2 MiB headroom for the websocket envelope and control frames. -- `stream_responses()` now computes the post-inline serialized payload size immediately after `_inline_input_image_urls()`, covering both: - - `app/modules/proxy/service.py::_rewrite_input_image_file_references` - - `app/core/clients/proxy.py::_inline_input_image_urls` -- `_resolve_stream_transport()` now routes `auto` requests over HTTP before the existing codex-header / model-registry websocket heuristics when that rewritten payload estimate exceeds the websocket budget. -- Explicit `upstream_stream_transport = "websocket"` and `upstream_stream_transport = "http"` still win unchanged. - -### `app/core/clients/image_processor.py` - -- Added a new codex-faithful prompt image processor. -- Mirrors the upstream codex image contract: - - accepts only PNG / JPEG / GIF / WebP - - preserves PNG / JPEG / WebP bytes verbatim when already within 2048x2048 - - re-encodes GIF as PNG - - resizes oversized images to fit 2048x2048 - - uses JPEG quality 85 and lossless WebP on resized output -- Adds a 32-entry in-process LRU cache keyed by `sha1(bytes) + mode`. - -### `app/core/clients/files.py` - -- Added `fetch_file_bytes(download_url, expected_mime, max_bytes)`. -- Downloads finalize SAS blobs with a hard byte cap so a single attachment cannot blow the websocket frame budget after base64 expansion. - -### `app/core/openai/requests.py` - -- Added `_input_image_file_reference()` for: - - `input_image.file_id` - - `input_image.image_url = "sediment://file_*"` -- Extended `extract_input_file_ids()` so routing sees both `input_file` and uploaded `input_image` references. -- Added `extract_input_image_file_references()` so the proxy can rewrite only the precise `input_image` parts, without touching any other conversation content. - -### `app/modules/proxy/service.py` - -- Replaced the old tuple pin with `_FilePinEntry(account_id, download_url, mime_type, file_name, expires_at)`. -- `create_file()` still pins the upload owner immediately so finalize stays on the same upstream account. -- `finalize_file()` now upgrades the pin with `download_url` / `mime_type` / `file_name` once upstream returns `status=success`. -- Pin expiry is clamped to the shorter of: - - `_FILE_ACCOUNT_PIN_TTL_SECONDS` (30 minutes) - - the SAS `se=` expiry embedded in `download_url`, when present -- Added `_lookup_file_pin()`. -- Added `_rewrite_input_image_file_references()`: - - finds only `input_image.file_id` / `sediment://file_*` - - fetches the uploaded bytes from the pinned SAS `download_url` - - runs the codex-faithful image processor - - rewrites the original part to inline `image_url: "data:..."`, preserving `detail` when supplied and defaulting it to `auto` otherwise - - leaves all non-targeted conversation content byte-for-byte untouched - - logs a synthetic `image-inline-rewrite` request-log row for observability -- Wired the rewrite into: - - HTTP `/v1/responses` / backend responses streaming path - - HTTP bridge path - - websocket `response.create` prepare path - - `/responses/compact` -- Added `_classify_upstream_close()` and `response_event_count` tracking. -- HTTP bridge `retry_precreated` now fails fast with `502 upstream_rejected_input` when upstream closes with `close_code=1000` before any `response.*` event. -- `stream_http_responses()` now rewrites uploaded `input_image` references before branch selection, estimates the post-rewrite JSON payload size, and bypasses the HTTP responses bridge per request when that rewritten body exceeds the WebSocket frame budget. -- The bypass uses a local `dataclasses.replace(runtime_config, enabled=False)` copy only, so bridge state stays unchanged globally and smaller follow-up requests still use the bridge normally. - -### `tests/unit/test_image_processor.py` - -- Added coverage for passthrough, resize, GIF->PNG re-encode, unsupported formats, garbage bytes, ORIGINAL mode, and cache-hit identity. - -### `tests/unit/test_files_client.py` - -- Added coverage for `fetch_file_bytes()` success and `file_too_large` enforcement. - -### `tests/unit/test_openai_requests.py` - -- Added coverage for `input_image.file_id`, `sediment://file_*`, and `extract_input_image_file_references()`. - -### `tests/unit/test_proxy_utils.py` - -- Added coverage for: - - `_lookup_file_pin()` - - `_rewrite_input_image_file_references()` single and multiple rewrites - - missing pin -> `400 file_not_found` - - oversized download -> `400 file_too_large` - - preserving non-image conversation content - - returning the pinned account for routing - - clean-close classifier - - HTTP bridge precreated retry suppression on rejected input - - large rewritten payloads forcing HTTP only in `auto` - - large rewritten payloads bypassing the HTTP responses bridge selector - - smaller / unknown payload sizes preserving websocket preference - - explicit transport overrides still winning - - websocket budget calculation from `max_sse_event_bytes` - -### OpenSpec - -- Amended `openspec/changes/add-backend-api-files-protocol/`: - - `proposal.md` - - `tasks.md` - - `specs/responses-api-compat/spec.md` -- Documented accepted `input_file` / uploaded `input_image` shapes, the inline rewrite contract, the 16 MiB cap, the “rewrite only the targeted `input_image` parts” rule, the auto HTTP fallback for oversized rewritten payloads, and the clean-close fail-fast behavior. -- Added the bridge-bypass scenario so the OpenSpec now covers the default bridge-enabled `/responses` path as well as `_resolve_stream_transport()`. - -### Dependency / lockfile - -- `pyproject.toml` now declares `pillow>=10.0`. -- `uv.lock` was updated so the direct dependency is in sync. -- Pillow was added explicitly even though it was already present transitively because this code now imports `from PIL import Image` directly in production. - -## Caveats - -- SAS expiry vs pin TTL: - - file pins now expire at the earlier of 30 minutes or the SAS `se=` timestamp when present - - if the SAS URL expires before the follow-up `/responses` call arrives, inline rewrite fails closed instead of attempting a stale fetch -- Cache misses: - - the image processor cache is in-process only - - a different worker or a cold process simply re-downloads and re-processes the image -- Partial multi-image rewrites: - - if any referenced upload pin is missing / expired / unfetchable, the whole request fails - - there is no partial-forward behavior - -## Verification - -- `uv run --frozen ruff check app tests` -- `uv run --frozen ruff format --check app tests` -- `uv run --frozen ty check app` -- `uv run --frozen pytest tests/unit -q` -- `uv run --frozen pytest tests/integration/test_proxy_files.py -q` -- `uv run --frozen pytest tests/integration/test_proxy_responses.py -q` - -## Could not verify - -- `openspec validate add-backend-api-files-protocol --strict --no-interactive` - - the `openspec` CLI is not installed in this workspace (`openspec: command not found`) diff --git a/openspec/changes/add-root-file-allowlist-budget/.openspec.yaml b/openspec/changes/add-root-file-allowlist-budget/.openspec.yaml new file mode 100644 index 0000000000..f774115be7 --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-20 diff --git a/openspec/changes/add-root-file-allowlist-budget/proposal.md b/openspec/changes/add-root-file-allowlist-budget/proposal.md new file mode 100644 index 0000000000..b8138a4297 --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/proposal.md @@ -0,0 +1,25 @@ +## Why + +Tracked files can accumulate at the repository root without review, including one-off agent artifacts that obscure the intended project surface. The existing simplicity-budget mechanism should make additions to that surface explicit and reviewer-visible. + +## What Changes + +- Define the complete set of allowed tracked repository-root entries in the simplicity-budget configuration. +- Extend the simplicity-budget checker to reject tracked root entries outside that allowlist, while preserving the existing PR-label override behavior. +- Relocate the proxy architecture ADR into its owning OpenSpec capability context and remove obsolete root-level agent debris. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `contribution-simplicity`: Budget the tracked repository-root surface with an explicit allowlist and the existing maintainer override. + +## Impact + +- `.github/simplicity-budgets.toml` gains the root-entry allowlist. +- `.github/scripts/check_simplicity_budgets.py` checks the committed root tree. +- Proxy architecture context moves under `openspec/specs/proxy-architecture/`; obsolete root files are removed and ignored against recurrence. diff --git a/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md b/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md new file mode 100644 index 0000000000..4631d898e7 --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/specs/contribution-simplicity/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Tracked repository-root entries are allowlisted + +Every tracked repository-root entry (file or directory, as listed by `git ls-tree --name-only HEAD`) MUST appear in the `allowed` list of the `[root_files]` section in `.github/simplicity-budgets.toml`, and the simplicity-budget check SHALL report each unlisted entry as a violation that names the entry and the escape hatch. A PR that adds an unlisted root entry SHALL be blocked from merge unless the entry is added to the allowlist in the same diff or a maintainer applies the `simplicity-budget-approved` label. When the `[root_files]` section is absent from the budget configuration, the check SHALL be skipped rather than fail. + +#### Scenario: Root tree matches the allowlist + +- **WHEN** every tracked repository-root entry appears in the `[root_files]` allowlist +- **THEN** the simplicity-budget check passes with no label required + +#### Scenario: Stray root file without an allowlist update + +- **WHEN** a PR commits a new repository-root file without adding it to the `[root_files]` allowlist +- **AND** no `simplicity-budget-approved` label is present +- **THEN** the simplicity-budget check fails with a violation naming that file and the escape hatch, and the PR is blocked from merge + +#### Scenario: Intentional root entry added with the allowlist in the same diff + +- **WHEN** a PR adds a repository-root entry and adds it to the `[root_files]` allowlist in the same diff +- **THEN** the simplicity-budget check passes, and the allowlist change is visible to the reviewer + +#### Scenario: Stray root entry with maintainer approval + +- **GIVEN** a PR whose tracked root entry is not in the allowlist +- **WHEN** a maintainer applies the `simplicity-budget-approved` label +- **THEN** the violation is downgraded to a warning on the PR run and the check passes + +#### Scenario: Budget configuration without a root-files section + +- **WHEN** the budget configuration has no `[root_files]` section +- **THEN** the simplicity-budget check skips root-entry enforcement and evaluates the remaining budgets unchanged diff --git a/openspec/changes/add-root-file-allowlist-budget/tasks.md b/openspec/changes/add-root-file-allowlist-budget/tasks.md new file mode 100644 index 0000000000..22f5165e2b --- /dev/null +++ b/openspec/changes/add-root-file-allowlist-budget/tasks.md @@ -0,0 +1,14 @@ +## 1. Relocate root documents + +- [x] 1.1 Move the ADR-0001 body from the root `DECISIONS.md` into `openspec/specs/proxy-architecture/context.md` with a short relocation header, then delete `DECISIONS.md`. +- [x] 1.2 Delete the root `SUMMARY.md` agent debris and add `DECISIONS.md` beside the existing `SUMMARY.md` line in the `.gitignore` agent-debris block. + +## 2. Enforce the root-entry allowlist + +- [x] 2.1 Add a `[root_files]` section with a sorted `allowed` list of every tracked repository-root entry to `.github/simplicity-budgets.toml`. +- [x] 2.2 Extend `.github/scripts/check_simplicity_budgets.py` to compare `git ls-tree --name-only HEAD` against the allowlist, reporting each unlisted entry as a violation that names the file and the escape hatch, keeping the override-label and exit-code semantics, and skipping the check when `[root_files]` is absent. + +## 3. Verification + +- [x] 3.1 Run the budget checker on the final tree (exit 0) and demonstrate that a stray committed root file fails the check with a named violation. +- [x] 3.2 Validate the OpenSpec change and run repository lint. diff --git a/DECISIONS.md b/openspec/specs/proxy-architecture/context.md similarity index 92% rename from DECISIONS.md rename to openspec/specs/proxy-architecture/context.md index e7ee2697e1..7c710d80ae 100644 --- a/DECISIONS.md +++ b/openspec/specs/proxy-architecture/context.md @@ -1,7 +1,9 @@ -# Architectural Decisions +# Context: proxy-architecture -This file records long-lived architecture decisions for codex-lb. New decisions -are appended and superseded by later entries rather than edited in place. +Normative requirements live in [`spec.md`](./spec.md). This document carries +free-form context for the proxy-architecture capability: architecture decision +records (ADRs) are appended here and superseded by later entries rather than +edited in place. Relocated from the former repository-root `DECISIONS.md`. ## ADR-0001: ProxyService target-architecture cutover refactor From c750dcfe64961c7d538c75367e9ee509fe8c9052 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EC=9C=A0=EC=98=81=EC=A4=80?= Date: Thu, 20 Aug 2026 17:27:18 +0900 Subject: [PATCH 094/117] fix(models): apply context-window overrides to /v1 input context fields (#1808) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(models): apply context-window overrides to /v1 input context fields An operator `model_context_window_overrides` entry only reached `metadata.context_window`. The generic OpenAI-compatible fields that most clients actually read — `context_length`, `contextLength`, `capabilities.context_length` and `metadata.input_context_window` — kept reporting the un-overridden upstream window, so those clients capped themselves at 272k while Codex-native clients used the wider window from `/backend-api/codex/models`. Report the override on those fields too, clamped to the upstream-declared `max_context_window` so an override can never advertise more input than the backend sanctions. This is the same clamp the Codex client applies to `model_context_window` in config.toml. Behavior is unchanged when no override is configured. * fix(models): resolve context-window override once and clamp it everywhere Address the three review blockers on the override fix: - Collapse _effective_context_window / _v1_full_context_window / _v1_input_context_window into a single _resolved_context_window and resolve it once per list item, so metadata.context_window, the context_length-family input-budget fields, and the Codex-native context_window/max_context_window rewrite all report one clamped value. Previously an override above the backend ceiling produced metadata.context_window=1000000 next to context_length=872000 — a new dual-budget split — and each list item called get_settings() four times. - Restore the pre-existing scenario heading "Explicit reported-context overrides do not hide the backend input budget" in the MODIFIED requirement (a MODIFIED block replaces the whole requirement, so the renamed heading would have silently dropped the original scenario at archive time); keep the updated assertions under it. openspec strict validation now passes for the change. - Correct the proposal's false "/backend-api/codex/models is untouched" claim: the endpoint's OpenAI-compatible data alias shares the /v1/models list-item shape, so its context_length-family fields do pick up the override (the native models list semantics are the pre-existing override behavior, now clamped). Pin both behaviors with tests, and extend the clamp test to assert all fields agree at 872000. Co-authored-by: yeongjun-cigro Co-Authored-By: Claude Fable 5 * fix(models): do not treat synthesized max_context_window as a clamp ceiling Local codex review caught a P2: bootstrap subscription models (_bootstrap_model) and source-catalog models (source_models_to_upstream_models) synthesize max_context_window == context_window purely so Codex clients can parse the entry. Clamping operator overrides to that parseability default silently disabled every raise override for those models — a regression against the pre-existing override behavior on metadata.context_window and the Codex-native catalog. Clamp only when upstream declares a ceiling strictly above the backend context_window, update the openspec delta accordingly, and pin the behavior with a route-level source-model regression test. Co-authored-by: yeongjun-cigro Co-Authored-By: Claude Fable 5 * docs(openspec): specify codex-native catalog override behavior CodeRabbit correctly flagged that the delta only covered GET /v1/models while the implementation also changes /backend-api/codex/models. Add a MODIFIED block for "Codex-native model catalog keeps backend catalog fields" stating that an operator override reports the single resolved (clamped) value on the native context_window and the max_context_window rewrite, that the synthesized parseability ceiling never clamps, and that the data alias reports the same resolved value on its context_length-family and metadata fields. Both new scenarios are already pinned by tests (test_model_context_window_override_clamped_to_max_context_window and test_model_context_window_override_applies_to_codex_models_data_alias). Co-authored-by: yeongjun-cigro Co-Authored-By: Claude Fable 5 --------- Co-authored-by: yeongjun-cigro Co-authored-by: Soju06 Co-authored-by: Claude Fable 5 --- app/modules/proxy/api.py | 66 +++++++++----- .../proposal.md | 23 +++++ .../specs/model-catalog-compat/spec.md | 89 +++++++++++++++++++ .../tasks.md | 13 +++ .../integration/test_model_source_routing.py | 28 ++++++ tests/integration/test_v1_models.py | 72 ++++++++++++++- 6 files changed, 265 insertions(+), 26 deletions(-) create mode 100644 openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md create mode 100644 openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md create mode 100644 openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 1298966416..e5c0e1b4d4 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -3867,16 +3867,17 @@ def _canonical_model_slug(model: str) -> str: def _to_model_list_item(slug: str, model: UpstreamModel, *, created: int) -> ModelListItem: + context_window = _resolved_context_window(model) return ModelListItem.model_validate( { "id": slug, "created": created, "owned_by": "codex-lb", - "metadata": _to_model_metadata(model), + "metadata": _to_model_metadata(model, context_window=context_window), "api_types": ["chat_completions"], - "capabilities": _v1_model_capabilities(model), - "context_length": _v1_input_context_window(model), - "contextLength": _v1_input_context_window(model), + "capabilities": _v1_model_capabilities(model, context_window=context_window), + "context_length": context_window, + "contextLength": context_window, "max_output_tokens": _v1_max_output_tokens(model), "maxOutputTokens": _v1_max_output_tokens(model), "supports_reasoning": _v1_supports_reasoning(model), @@ -3978,7 +3979,7 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None extra[key] = value # If context_window is overridden, also override max_context_window to match - effective_cw = _effective_context_window(model) + effective_cw = _resolved_context_window(model) if effective_cw != model.context_window and "max_context_window" in extra: extra["max_context_window"] = effective_cw @@ -3996,7 +3997,7 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None support_verbosity=model.support_verbosity, default_verbosity=model.default_verbosity, supports_parallel_tool_calls=model.supports_parallel_tool_calls, - context_window=_effective_context_window(model), + context_window=effective_cw, input_modalities=list(model.input_modalities), available_in_plans=sorted(model.available_in_plans), prefer_websockets=model.prefer_websockets, @@ -4010,18 +4011,39 @@ def _to_codex_model_entry(model: UpstreamModel, *, visibility: str | None = None ) -def _effective_context_window(model: UpstreamModel) -> int: +def _resolved_context_window(model: UpstreamModel) -> int: + # An explicit operator context-window override is an assertion about the usable + # input budget, so it must also reach the generic OpenAI-compatible fields + # (`context_length`, `contextLength`, `capabilities.context_length`, and + # `metadata.input_context_window`). Generic clients read those rather than + # `metadata.context_window` and would otherwise cap themselves at the + # un-overridden upstream budget while Codex-native clients use the wider window. + # The override is clamped to the upstream-declared `max_context_window` so it can + # never advertise more input than the backend sanctions — the same clamp the Codex + # client applies to `model_context_window` in config.toml. The clamp only applies + # when upstream declares a ceiling strictly above `context_window`: bootstrap + # subscription models (`_bootstrap_model`) and source-catalog models + # (`source_models_to_upstream_models`) synthesize `max_context_window == + # context_window` purely so Codex clients can parse the entry, and treating that + # parseability default as a real ceiling would silently disable every raise + # override for those models. + # + # This is the single resolution point for the reported window: the Codex-native + # `context_window`/`max_context_window` rewrite, `metadata.context_window`, and + # every input-budget field all share this one value, so an override above the + # backend ceiling can never split one model into two contradictory budgets. overrides = get_settings().model_context_window_overrides - return overrides.get(model.slug, model.context_window) - - -def _v1_full_context_window(model: UpstreamModel) -> int: - overrides = get_settings().model_context_window_overrides - return overrides.get(model.slug, model.context_window) - - -def _v1_input_context_window(model: UpstreamModel) -> int: - return model.context_window + override = overrides.get(model.slug) + if override is None: + return model.context_window + max_context_window = model.raw.get("max_context_window") + if ( + isinstance(max_context_window, int) + and not isinstance(max_context_window, bool) + and max_context_window > model.context_window + ): + return min(override, max_context_window) + return override def _v1_max_output_tokens(model: UpstreamModel) -> int | None: @@ -4031,11 +4053,11 @@ def _v1_max_output_tokens(model: UpstreamModel) -> int | None: return _V1_MAX_OUTPUT_TOKEN_OVERRIDES.get(model.slug) -def _v1_model_capabilities(model: UpstreamModel) -> dict[str, JsonValue]: +def _v1_model_capabilities(model: UpstreamModel, *, context_window: int) -> dict[str, JsonValue]: supports_streaming_raw = model.raw.get("supports_streaming") supports_streaming = supports_streaming_raw if isinstance(supports_streaming_raw, bool) else True return { - "context_length": _v1_input_context_window(model), + "context_length": context_window, "max_output_tokens": _v1_max_output_tokens(model), "supports_reasoning": _v1_supports_reasoning(model), "supports_images": _v1_supports_vision(model), @@ -4083,12 +4105,12 @@ def _effective_source_codex_visibility( return "list" -def _to_model_metadata(model: UpstreamModel) -> ModelMetadata: +def _to_model_metadata(model: UpstreamModel, *, context_window: int) -> ModelMetadata: return ModelMetadata( display_name=model.display_name, description=model.description, - context_window=_v1_full_context_window(model), - input_context_window=_v1_input_context_window(model), + context_window=context_window, + input_context_window=context_window, max_output_tokens=_v1_max_output_tokens(model), input_modalities=list(model.input_modalities), supported_reasoning_levels=[ diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md new file mode 100644 index 0000000000..97eb37aca7 --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/proposal.md @@ -0,0 +1,23 @@ +## Why + +`model_context_window_overrides` is documented as the highest-priority reported-context override, but on `/v1/models` it only reaches `metadata.context_window`. The fields generic OpenAI-compatible clients actually read — `context_length`, `contextLength`, `capabilities.context_length`, and `metadata.input_context_window` — keep reporting the un-overridden upstream `context_window`, so an operator who raises a model's window sees Codex-native clients use the wider window from `/backend-api/codex/models` while every OpenAI-compatible client silently caps itself at the old value. A single catalog then advertises two different budgets for one model. + +The split was introduced by `2026-06-02-report-v1-model-full-context` to stop over-advertising context to generic clients, because the backend really did reject inputs above its `context_window` with `context_length_exceeded`. That reasoning still holds for the *default*, but not for an explicit operator override: current frontier models publish a `max_context_window` well above `context_window` (for example `gpt-5.6-sol` at `272000` / `872000`), the backend accepts input up to that ceiling, and the Codex client itself treats a config override as the session window after clamping it to `max_context_window`. + +## What Changes + +- An explicit `model_context_window_overrides` entry is reported as the input budget too, so `context_length`, `contextLength`, `capabilities.context_length`, and `metadata.input_context_window` agree with `metadata.context_window` instead of contradicting it. +- The reported input budget is clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so an override can never advertise more input than the backend sanctions — the same clamp the Codex client applies to `model_context_window` in `config.toml`. This preserves the original protection against over-advertising while removing the under-advertising. A `max_context_window` equal to `context_window` never clamps: bootstrap subscription models and source-catalog models synthesize that value purely so Codex clients can parse the entry, and treating it as a real ceiling would silently disable raise overrides for those models. +- The override is resolved (and clamped) once per model and that single value feeds `metadata.context_window`, every input-budget field, and the Codex-native `context_window`/`max_context_window` rewrite. An override above the ceiling therefore reports the ceiling everywhere instead of splitting one model into two budgets again (previously `metadata.context_window=1000000` next to `context_length=872000`). +- Behavior is unchanged when no override is configured for the model: the reported input budget stays the upstream `context_window`. +- `/backend-api/codex/models` is affected in two ways, both consistency-preserving: the Codex-native `models` list already reported the override on `context_window` (and rewrote `max_context_window` to match) and now uses the same clamped value, and the endpoint's OpenAI-compatible `data` alias is built from the same list-item shape as `/v1/models`, so its `context_length`-family fields pick up the corrected values too. Both views of one model advertise one budget. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `model-catalog-compat`: an operator context-window override applies to the OpenAI-compatible input-budget fields, clamped to the upstream `max_context_window` when upstream declares one above the backend `context_window`. The Codex-native catalog requirement now specifies the same single resolved value for the native `context_window`/`max_context_window` rewrite and the endpoint's OpenAI-compatible `data` alias. diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md new file mode 100644 index 0000000000..de80363dce --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/specs/model-catalog-compat/spec.md @@ -0,0 +1,89 @@ +## MODIFIED Requirements + +### Requirement: OpenAI-compatible model metadata uses backend context windows + +When serving `GET /v1/models`, the system SHALL expose `metadata.context_window` as the upstream backend `context_window` budget by default. The system MUST NOT promote raw `max_context_window` values or hard-coded full-context guesses into `metadata.context_window`. Explicit operator context-window overrides remain the highest-priority reported-context value, clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`. + +#### Scenario: GPT-5 Codex models are reported with the backend context window on /v1/models + +- **WHEN** the upstream model catalog contains `gpt-5.5`, `gpt-5.4-mini`, `gpt-5.3-codex`, or `gpt-5.4` with `context_window=272000` +- **THEN** `GET /v1/models` returns each entry with `metadata.context_window=272000` + +#### Scenario: raw max_context_window does not inflate /v1/models context_window + +- **WHEN** the upstream model catalog contains a model with `context_window=272000` and `max_context_window=900000` +- **THEN** `GET /v1/models` returns that entry with `metadata.context_window=272000` + +### Requirement: OpenAI-compatible model metadata preserves the backend input budget explicitly + +When serving `GET /v1/models`, the system SHALL expose the upstream backend input/context budget in `metadata.input_context_window`. When an explicit operator context-window override applies to a model, that override SHALL be the reported input budget as well, clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so `metadata.input_context_window` and the OpenAI-compatible `context_length`, `contextLength`, and `capabilities.context_length` fields never contradict `metadata.context_window` and never advertise more input than the backend sanctions. A `max_context_window` equal to the backend `context_window` — the parseability default synthesized for bootstrap and source-catalog models — MUST NOT clamp an override, so raise overrides for those models keep working. For models whose reported `metadata.context_window` is not operator-overridden, `metadata.context_window` and `metadata.input_context_window` SHOULD be equal. The system SHOULD expose `metadata.max_output_tokens` for known GPT-5 Codex models when that output-budget value is known; that value MUST NOT be used to inflate `metadata.context_window`. + +#### Scenario: /v1/models exposes the 272k backend input budget explicitly + +- **WHEN** the upstream model catalog contains a known GPT-5 Codex model with `context_window=272000` +- **THEN** `GET /v1/models` returns that model with `metadata.input_context_window=272000` +- **AND** `metadata.context_window=272000` + +#### Scenario: Explicit reported-context overrides do not hide the backend input budget + +- **WHEN** an operator override sets a model's reported `metadata.context_window` to `515000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and no `max_context_window` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=515000` +- **AND** `metadata.input_context_window=515000` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `515000` + +#### Scenario: An override never advertises more input than the backend ceiling + +- **WHEN** an operator override sets a model's reported context window to `1000000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and `max_context_window=872000` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=872000` +- **AND** `metadata.input_context_window=872000` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `872000` + +#### Scenario: A synthesized ceiling equal to the backend budget does not clamp an override + +- **WHEN** an operator override sets a source-catalog model's reported context window to `32768` +- **AND** that model declares `context_window=8192` and no explicit `max_context_window`, so the catalog synthesizes `max_context_window=8192` +- **THEN** `GET /v1/models` returns that model with `metadata.context_window=32768` +- **AND** `metadata.input_context_window=32768` +- **AND** `context_length`, `contextLength`, and `capabilities.context_length` of `32768` + +#### Scenario: /v1/models exposes max output budget for known GPT-5 Codex models + +- **WHEN** `GET /v1/models` returns `gpt-5.5`, `gpt-5.4`, `gpt-5.4-mini`, or `gpt-5.3-codex` +- **THEN** the entry's metadata includes `max_output_tokens=128000` + +### Requirement: Codex-native model catalog keeps backend catalog fields + +When serving `GET /backend-api/codex/models`, the system MUST keep Codex-native model catalog semantics unchanged: the top-level `context_window` field remains the backend compact/input budget unless an explicit operator override applies, and upstream raw fields such as `max_context_window` remain available when upstream provides them. The `/v1/models` compatibility metadata MUST NOT mutate the native Codex endpoint. + +When an explicit operator context-window override applies to a model, the native entry SHALL report the single resolved value — the override clamped to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`; a `max_context_window` equal to the backend `context_window` (the synthesized parseability default) MUST NOT clamp — on `context_window`, and SHALL rewrite `max_context_window` to that same resolved value when upstream provides the field. The endpoint's OpenAI-compatible `data` alias SHALL report the same resolved value on its `context_length`, `contextLength`, `capabilities.context_length`, `metadata.context_window`, and `metadata.input_context_window` fields, so the native and alias views of one model never advertise different budgets. + +#### Scenario: Native Codex route preserves compact budget + +- **WHEN** the upstream model catalog contains `gpt-5.5` with `context_window=272000` +- **THEN** `GET /backend-api/codex/models` returns `gpt-5.5.context_window=272000` +- **AND** it does not replace that field with `400000` + +#### Scenario: Codex model catalog also exposes OpenAI data alias + +- **WHEN** a client requests `GET /backend-api/codex/models` +- **THEN** the response keeps the Codex-native `models` list +- **AND** the response includes `object: "list"` and an OpenAI-compatible `data` list +- **AND** `data` contains model entries whose Codex visibility is `list` +- **AND** `data` excludes entries whose Codex visibility is `hide` + +#### Scenario: Native Codex catalog reports one resolved budget for a clamped override + +- **WHEN** an operator override sets a model's reported context window to `1000000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and `max_context_window=872000` +- **THEN** `GET /backend-api/codex/models` returns that model with `context_window=872000` +- **AND** `max_context_window=872000` + +#### Scenario: Codex data alias reports the resolved input budget for an override + +- **WHEN** an operator override sets a model's reported context window to `515000` +- **AND** the upstream model catalog contains that model with `context_window=272000` and no explicit `max_context_window` +- **THEN** the `GET /backend-api/codex/models` `data` alias entry for that model reports `context_length`, `contextLength`, and `capabilities.context_length` of `515000` +- **AND** `metadata.context_window=515000` and `metadata.input_context_window=515000` +- **AND** the native `models` entry reports `context_window=515000` diff --git a/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md b/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md new file mode 100644 index 0000000000..0420712bbc --- /dev/null +++ b/openspec/changes/apply-context-window-override-to-v1-input-budget/tasks.md @@ -0,0 +1,13 @@ +## 1. Report the override on the input-budget fields + +- [x] 1.1 Resolve the `/v1/models` input context window from `model_context_window_overrides` when the model has an entry, falling back to the upstream `context_window` otherwise +- [x] 1.2 Clamp an override to the upstream-declared `max_context_window` when upstream declares one above the backend `context_window`, so the reported input budget never exceeds the backend ceiling; never treat the synthesized `max_context_window == context_window` parseability default (bootstrap and source-catalog models) as a ceiling +- [x] 1.3 Resolve the override once per model into a single clamped value shared by `metadata.context_window`, the input-budget fields, and the Codex-native `context_window`/`max_context_window` rewrite, so no field pair can disagree + +## 2. Tests + +- [x] 2.1 With an override configured, `/v1/models` reports it on `metadata.input_context_window`, `capabilities.context_length`, `contextLength`, and `context_length` (was the un-overridden upstream window) +- [x] 2.2 An override above the upstream `max_context_window` is reported clamped to that ceiling on every field, including `metadata.context_window` and the Codex-native `context_window`/`max_context_window` +- [x] 2.3 Without an override the reported input budget stays the upstream `context_window` +- [x] 2.4 The `/backend-api/codex/models` OpenAI-compatible `data` alias reports the override on its `context_length`-family fields (pin: it shares the `/v1/models` list-item shape) +- [x] 2.5 Route-level source-model regression: a raise override on a source-catalog model (synthesized `max_context_window == context_window`) applies unclamped on `/v1/models` diff --git a/tests/integration/test_model_source_routing.py b/tests/integration/test_model_source_routing.py index 97b42e074c..23e3aec80d 100644 --- a/tests/integration/test_model_source_routing.py +++ b/tests/integration/test_model_source_routing.py @@ -2372,6 +2372,34 @@ async def test_v1_models_metadata_reflects_reasoning_optin(async_client): assert by_id["plain-metadata-model"]["supports_reasoning"] is False +@pytest.mark.asyncio +async def test_v1_models_context_window_override_applies_to_source_model(async_client, monkeypatch): + # Source-catalog models synthesize `max_context_window == context_window` + # purely so Codex clients can parse the entry; that parseability default + # must not clamp an operator raise override to the un-raised window. + await _create_model_source( + async_client, + name="override-source", + model="override-source-model", + base_url="http://127.0.0.1:9/v1", + ) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"override-source-model": 32_768}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + response = await async_client.get("/v1/models") + assert response.status_code == 200 + item = next(m for m in response.json()["data"] if m["id"] == "override-source-model") + assert item["metadata"]["context_window"] == 32_768 + assert item["metadata"]["input_context_window"] == 32_768 + assert item["capabilities"]["context_length"] == 32_768 + assert item["contextLength"] == 32_768 + assert item["context_length"] == 32_768 + + @pytest.mark.asyncio async def test_source_chat_payload_keeps_reasoning_toggles_for_optin_model(async_client, source_upstream): captured: dict[str, object] = {} diff --git a/tests/integration/test_v1_models.py b/tests/integration/test_v1_models.py index f04cdf512b..917b77872f 100644 --- a/tests/integration/test_v1_models.py +++ b/tests/integration/test_v1_models.py @@ -1411,10 +1411,74 @@ async def test_model_context_window_override(async_client, monkeypatch): v1_entry = next(m for m in resp_v1.json()["data"] if m["id"] == "gpt-5.4") metadata = v1_entry["metadata"] assert metadata["context_window"] == 515000 - assert metadata["input_context_window"] == 272000 - assert v1_entry["capabilities"]["context_length"] == 272000 - assert v1_entry["contextLength"] == 272000 - assert v1_entry["context_length"] == 272000 + # An explicit operator override is the reported input budget too: generic + # OpenAI-compatible clients read `context_length`/`contextLength` and would + # otherwise cap themselves at the un-overridden upstream window. + assert metadata["input_context_window"] == 515000 + assert v1_entry["capabilities"]["context_length"] == 515000 + assert v1_entry["contextLength"] == 515000 + assert v1_entry["context_length"] == 515000 + + +@pytest.mark.asyncio +async def test_model_context_window_override_clamped_to_max_context_window(async_client, monkeypatch): + registry = get_model_registry() + models = [_make_upstream_model("gpt-5.4", raw=_raw_with_max_context_window(872_000))] + await registry.update({"pro": models}) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"gpt-5.4": 1_000_000}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + resp_v1 = await async_client.get("/v1/models") + assert resp_v1.status_code == 200 + v1_entry = next(m for m in resp_v1.json()["data"] if m["id"] == "gpt-5.4") + + # The reported input budget never exceeds the upstream-declared ceiling, and + # `metadata.context_window` reports the same clamped value: the override is + # resolved once, so the clamp cannot reintroduce a dual-budget split. + assert v1_entry["metadata"]["context_window"] == 872_000 + assert v1_entry["metadata"]["input_context_window"] == 872_000 + assert v1_entry["capabilities"]["context_length"] == 872_000 + assert v1_entry["contextLength"] == 872_000 + assert v1_entry["context_length"] == 872_000 + + # The Codex-native catalog shares the same single resolution. + resp_codex = await async_client.get("/backend-api/codex/models") + assert resp_codex.status_code == 200 + native_entry = next(m for m in resp_codex.json()["models"] if m["slug"] == "gpt-5.4") + assert native_entry["context_window"] == 872_000 + assert native_entry["max_context_window"] == 872_000 + + +@pytest.mark.asyncio +async def test_model_context_window_override_applies_to_codex_models_data_alias(async_client, monkeypatch): + registry = get_model_registry() + models = [_make_upstream_model("gpt-5.4")] + await registry.update({"pro": models}) + + from app.core.config.settings import get_settings + from app.modules.proxy import api as proxy_api_module + + patched = get_settings().model_copy(update={"model_context_window_overrides": {"gpt-5.4": 515_000}}) + monkeypatch.setattr(proxy_api_module, "get_settings", lambda: patched) + + # The OpenAI-compatible `data` alias on /backend-api/codex/models is built + # from the same list-item shape as /v1/models, so the override reaches its + # context_length-family fields too (pinned: both views advertise one budget). + resp = await async_client.get("/backend-api/codex/models") + assert resp.status_code == 200 + payload = resp.json() + native_entry = next(m for m in payload["models"] if m["slug"] == "gpt-5.4") + assert native_entry["context_window"] == 515_000 + alias_item = next(m for m in payload["data"] if m["id"] == "gpt-5.4") + assert alias_item["metadata"]["context_window"] == 515_000 + assert alias_item["metadata"]["input_context_window"] == 515_000 + assert alias_item["capabilities"]["context_length"] == 515_000 + assert alias_item["contextLength"] == 515_000 + assert alias_item["context_length"] == 515_000 @pytest.mark.asyncio From bffc6d9b7c43117ada756ea413d22bec8c7b9e53 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 17:27:55 +0900 Subject: [PATCH 095/117] test(oauth): make test_oauth_flow order-independent by fencing the shared OAuth store (#1839) tests/integration/test_oauth_flow.py was order-dependent within the file with a different victim per run (issue #1794): tests only reset the module-global oauth_module._OAUTH_STORE at their START, and _OAUTH_STORE.reset() cancels poll tasks without awaiting them, so a device poller leaked by one test kept running on the shared session loop through the next test's entire fixture setup and body. Three concrete leaks fixed: - Add an autouse _isolate_global_oauth_store fixture that resets the store at BOTH edges of every test and cancels AND awaits every task the store owns (poll tasks + callback-server stop task), so nothing crosses a test boundary (same treatment as the live-usage ingestor fence from issue #1755). The per-test reset() calls are replaced by the fixture. - test_oauth_start_falls_back_to_device_on_os_error never patched exchange_device_token, so its leaked poller performed a REAL network token exchange and then slot/status DB writes against a later test's freshly reset database. Park the exchange on an Event; the fence cancels it at teardown. - The multi-flow device tests polled /api/oauth/status without flowId, which reads the store's "latest flow" pointer; a finishing neighbor poller's cleanup re-latches that pointer to its own already-successful flow, so the poll reported success before the current flow's poller persisted (observed as planType 'plus' != 'team' even in isolation). Pin the status polls to each flow's own flowId. Evidence (unmodified main): the pair test_oauth_start_falls_back_to_device_on_os_error + test_device_oauth_reauth_reuses_existing_row_for_same_chatgpt_identity fails ~10-20% of runs, and the reauth test alone failed 4/20 runs on the latest-pointer race. With the fix: pair 0/20, alone 0/20, full file 5x green, 3 shuffled whole-file orderings green, tests/unit green. Fixes #1794 Co-authored-by: Claude Fable 5 --- tests/integration/test_oauth_flow.py | 111 +++++++++++++++------------ 1 file changed, 61 insertions(+), 50 deletions(-) diff --git a/tests/integration/test_oauth_flow.py b/tests/integration/test_oauth_flow.py index 067d93d1e7..a09058609c 100644 --- a/tests/integration/test_oauth_flow.py +++ b/tests/integration/test_oauth_flow.py @@ -43,6 +43,50 @@ def _oauth_flow_schema(db_setup): del db_setup +async def _drain_global_oauth_store() -> None: + """Reset the module-global OAuth store AND await its tasks to completion. + + ``_OAUTH_STORE.reset()`` cancels poll tasks but does not await them, so a + cancelled (or still-pending) device poller can keep running into the next + test on the shared session loop -- exchanging the device code against + whatever ``exchange_device_token`` is monkeypatched to at that moment (or + the real network client) and committing slot/status writes to the next + test's freshly reset database (issue #1794, same family as #1755). Awaiting + every task here guarantees nothing owned by the store crosses a test + boundary, and retrieves cancelled tasks' exceptions so they cannot surface + as unrelated "Task exception was never retrieved" noise in a later test. + """ + + store = oauth_module._OAUTH_STORE + async with store.lock: + tasks = [ + flow.poll_task for flow in store._flows.values() if flow.poll_task is not None and not flow.poll_task.done() + ] + stop_task = store._callback_server_stop_task + if stop_task is not None and not stop_task.done(): + tasks.append(stop_task) + await store.reset() + for task in tasks: + task.cancel() + with contextlib.suppress(Exception, asyncio.CancelledError): + await task + + +@pytest.fixture(autouse=True) +async def _isolate_global_oauth_store(): + """Fence ``oauth_module._OAUTH_STORE`` at BOTH edges of every test. + + The per-test ``_OAUTH_STORE.reset()`` calls this fixture replaces only ran + at each test's start, so whichever test happened to run next inherited the + previous test's live poll tasks for its whole fixture setup -- the + order-dependent "different victim per run" flake of issue #1794. + """ + + await _drain_global_oauth_store() + yield + await _drain_global_oauth_store() + + def _encode_jwt(payload: dict) -> str: raw = json.dumps(payload, separators=(",", ":")).encode("utf-8") body = base64.urlsafe_b64encode(raw).rstrip(b"=").decode("ascii") @@ -104,7 +148,6 @@ async def manual_callback(self, callback_url: str, flow_id: str | None = None): @pytest.mark.asyncio async def test_manual_callback_service_sanitizes_unexpected_exception(monkeypatch, caplog): - await oauth_module._OAUTH_STORE.reset() caplog.set_level(logging.ERROR, logger=oauth_module.logger.name) # Persist the flow durably (real flows are written to the shared DB at start) # so the reconciliation gate keeps it rather than dropping it as stale. @@ -162,8 +205,6 @@ def test_oauth_error_html_escapes_message(): @pytest.mark.asyncio async def test_device_oauth_flow_creates_account(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - email = "device@example.com" raw_account_id = "acc_device" @@ -220,7 +261,6 @@ async def fake_sleep(_: float) -> None: @pytest.mark.asyncio async def test_starting_new_device_flow_cancels_previous_pending_poll(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() issued = 0 first_poll_started = asyncio.Event() first_poll_cancelled = asyncio.Event() @@ -276,8 +316,6 @@ async def fake_exchange_device_token(*, device_auth_id: str, **_): await asyncio.wait_for(first_poll_cancelled.wait(), timeout=1) assert first_task.cancelled() - await oauth_module._OAUTH_STORE.reset() - @pytest.mark.asyncio async def test_device_oauth_reauth_reuses_existing_row_for_same_chatgpt_identity( @@ -299,8 +337,6 @@ async def test_device_oauth_reauth_reuses_existing_row_for_same_chatgpt_identity new tokens onto its historical row instead of forking a duplicate. """ - await oauth_module._OAUTH_STORE.reset() - settings = await async_client.put( "/api/settings", json={ @@ -352,6 +388,7 @@ async def _run_device_flow_once() -> None: start = await async_client.post("/api/oauth/start", json={"forceMethod": "device"}) assert start.status_code == 200 assert start.json()["method"] == "device" + flow_id = start.json()["flowId"] complete = await async_client.post("/api/oauth/complete", json={}) assert complete.status_code == 200 @@ -359,9 +396,13 @@ async def _run_device_flow_once() -> None: await asyncio.sleep(0) + # Poll THIS flow's status (like the dashboard does): the flowId-less + # endpoint reads the store's "latest flow" pointer, which a finishing + # neighbor poller's cleanup can re-latch to its own already-successful + # flow -- reporting success before this flow's poller persisted. payload = None for _ in range(20): - status = await async_client.get("/api/oauth/status") + status = await async_client.get("/api/oauth/status", params={"flowId": flow_id}) assert status.status_code == 200 payload = status.json() if payload["status"] == "success": @@ -388,8 +429,6 @@ async def test_device_oauth_flow_heals_deactivated_account_when_import_without_o async_client, monkeypatch, ): - await oauth_module._OAUTH_STORE.reset() - settings = await async_client.put( "/api/settings", json={ @@ -834,8 +873,6 @@ async def test_device_oauth_flow_keeps_same_email_distinct_upstream_identities_i async_client, monkeypatch, ): - await oauth_module._OAUTH_STORE.reset() - enable_separate = await async_client.put( "/api/settings", json={ @@ -901,6 +938,7 @@ async def _run_device_flow_once() -> dict[str, str | None]: start = await async_client.post("/api/oauth/start", json={"forceMethod": "device"}) assert start.status_code == 200 assert start.json()["method"] == "device" + flow_id = start.json()["flowId"] complete = await async_client.post("/api/oauth/complete", json={}) assert complete.status_code == 200 @@ -908,9 +946,11 @@ async def _run_device_flow_once() -> dict[str, str | None]: await asyncio.sleep(0) + # Poll THIS flow's status; see the reauth test above for why the + # flowId-less "latest flow" endpoint is racy across sequential flows. payload: dict[str, str | None] | None = None for _ in range(20): - status = await async_client.get("/api/oauth/status") + status = await async_client.get("/api/oauth/status", params={"flowId": flow_id}) assert status.status_code == 200 payload = status.json() if payload["status"] in {"success", "error"}: @@ -949,8 +989,6 @@ async def _run_device_flow_once() -> dict[str, str | None]: @pytest.mark.asyncio async def test_oauth_start_with_existing_account_marks_success(async_client): - await oauth_module._OAUTH_STORE.reset() - encryptor = TokenEncryptor() account = Account( id="acc_existing", @@ -978,8 +1016,6 @@ async def test_oauth_start_with_existing_account_marks_success(async_client): @pytest.mark.asyncio async def test_oauth_start_with_existing_account_clears_stale_flows(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1025,8 +1061,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_terminal_oauth_flows_are_bounded_outside_full_reset(): - await oauth_module._OAUTH_STORE.reset() - retained_limit = oauth_module._MAX_RETAINED_TERMINAL_OAUTH_FLOWS async with oauth_module._OAUTH_STORE.lock: @@ -1056,8 +1090,6 @@ async def test_terminal_oauth_flows_are_bounded_outside_full_reset(): @pytest.mark.asyncio async def test_expired_pending_browser_oauth_flows_are_pruned(): - await oauth_module._OAUTH_STORE.reset() - now = time.time() async with oauth_module._OAUTH_STORE.lock: expired = oauth_module.OAuthState( @@ -1087,8 +1119,6 @@ async def test_expired_pending_browser_oauth_flows_are_pruned(): @pytest.mark.asyncio async def test_only_expired_pending_browser_flow_no_longer_keeps_callback_server_alive(): - await oauth_module._OAUTH_STORE.reset() - async with oauth_module._OAUTH_STORE.lock: flow = oauth_module.OAuthState( flow_id="expired-flow", @@ -1107,7 +1137,6 @@ async def test_only_expired_pending_browser_flow_no_longer_keeps_callback_server @pytest.mark.asyncio async def test_callback_server_remains_reserved_until_stop_completes(): - await oauth_module._OAUTH_STORE.reset() stop_started = asyncio.Event() release_stop = asyncio.Event() @@ -1133,8 +1162,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_oauth_start_falls_back_to_device_on_os_error(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_browser_flow(self): raise OSError("no port") @@ -1147,8 +1174,15 @@ async def fake_device_code(**_): expires_in_seconds=30, ) + # Park the spawned device poller instead of letting it hit the REAL token + # exchange client (this test only asserts the browser->device fallback); + # the autouse store fence cancels and awaits it at teardown. + async def fake_exchange_device_token(**_): + await asyncio.Event().wait() + monkeypatch.setattr(oauth_module.OauthService, "_start_browser_flow", fake_browser_flow) monkeypatch.setattr(oauth_module, "request_device_code", fake_device_code) + monkeypatch.setattr(oauth_module, "exchange_device_token", fake_exchange_device_token) start = await async_client.post("/api/oauth/start", json={}) assert start.status_code == 200 @@ -1159,8 +1193,6 @@ async def fake_device_code(**_): @pytest.mark.asyncio async def test_device_oauth_flow_reports_proxy_route_errors(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_oauth_route(*_args, **_kwargs): raise UpstreamProxyRouteError("default_pool_unconfigured", account_id=None) @@ -1174,8 +1206,6 @@ async def fake_oauth_route(*_args, **_kwargs): @pytest.mark.asyncio async def test_manual_callback_returns_success_and_creates_account(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1227,8 +1257,6 @@ async def fake_exchange_authorization_code(**_): @pytest.mark.asyncio async def test_manual_callback_returns_error_message_for_invalid_state(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1262,8 +1290,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_oauth_status_binds_camel_case_flow_id(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1309,8 +1335,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_manual_callback_error_resolves_state_before_marking_flow_failed(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1377,7 +1401,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_unknown_flow_error_does_not_mutate_latest_oauth_status(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1407,7 +1430,6 @@ async def test_unknown_flow_error_does_not_mutate_latest_oauth_status(): @pytest.mark.asyncio async def test_missing_flow_error_does_not_mutate_latest_oauth_status(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1437,8 +1459,6 @@ async def test_missing_flow_error_does_not_mutate_latest_oauth_status(): @pytest.mark.asyncio async def test_manual_callback_unknown_state_does_not_mutate_latest_flow(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1467,8 +1487,6 @@ async def fake_callback_server_start(self) -> None: @pytest.mark.asyncio async def test_concurrent_browser_oauth_flows_keep_callbacks_isolated(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -1547,7 +1565,6 @@ async def fake_exchange_authorization_code(**kwargs): @pytest.mark.asyncio async def test_callback_server_idle_stop_releases_store_lock_before_cleanup(): - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) @@ -1575,8 +1592,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_existing_account_cleanup_releases_store_lock_before_callback_server_stop(): - await oauth_module._OAUTH_STORE.reset() - class ExistingAccountRepo: async def list_accounts(self): return [object()] @@ -1607,7 +1622,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_new_browser_flow_waits_for_stopping_callback_server_before_reusing_slot(monkeypatch): - await oauth_module._OAUTH_STORE.reset() stop_started = asyncio.Event() release_stop = asyncio.Event() started_servers: list[object] = [] @@ -1653,8 +1667,6 @@ async def stop(self) -> None: @pytest.mark.asyncio async def test_manual_callback_idempotent_success_requires_requested_flow(async_client, monkeypatch): - await oauth_module._OAUTH_STORE.reset() - async def fake_callback_server_start(self) -> None: return None @@ -2059,7 +2071,6 @@ async def test_device_complete_ack_stays_pending_when_own_poller_already_succeed and must NOT spawn a second poll of the consumed device code. """ - await oauth_module._OAUTH_STORE.reset() async with SessionLocal() as session: service = oauth_module.OauthService(AccountsRepository(session)) From 01f089c359dadc2cc75b0719addc646e116624a5 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 17:33:24 +0900 Subject: [PATCH 096/117] fix(http-bridge): classify recovery error frames and poison same-anchor eventless failures (#1841) * fix(http-bridge): classify recovery error frames and poison same-anchor eventless failures The HTTP responses session bridge could wedge a session permanently (issue #1830): after one genuine mid-turn interruption the bridge rebinds to its stored durable anchor and re-injects it on every attempt, and when upstream rejects that anchor the failure loops forever behind the retry circuit ("cooling down" 503). Two gaps combined into the wedge: - The bridge-local previous-response recovery gate read raw error codes without the normalization the WebSocket path gained in #1818, so a frame carrying its classifiable code only in `type` (or the terse parameterless "Invalid `previous_response_id`." shape) fell through to the ambiguous-transport class instead of recovery. - Anchor poisoning only counted `stream_idle_timeout`, and only on the reader path when admission waiters exist. The observed wedge fails eventlessly with `stream_incomplete` (the bridge's masked form of an upstream previous-response rejection), so `http_responses_session_bridge_anchor_poison_failure_threshold` never fired and operators had to wipe the http_bridge_* tables. Fix: normalize the error code (falling back to `type`) in the recovery gate before classification; count both ambiguous eventless transport classes toward anchor poison (clean_close still never poisons); and evaluate the poison threshold at the shared retirement boundary too, clearing the poisoned durable anchor while the session still owns its durable lease so waiterless wedges self-heal. Consecutive eventless failures on one bridge key are same-anchor failures: the durable anchor only advances on a completed response, which resets the circuit. Fixes #1830 Co-Authored-By: Claude Fable 5 * fix(http-bridge): emit poison-clear failure telemetry on the waiterless path A failed durable-anchor clear at the shared retirement boundary was only an unstructured warning, absent from the durable_anchor_poison_clear_failed telemetry the admission-waiter path emits. Emit the same event (gated on the session actually holding durable continuity) so failed waiterless clears stay observable while the next threshold failure re-attempts them. Addresses CodeRabbit review on #1841. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- .../proxy/_service/http_bridge/helpers.py | 10 +- .../_service/http_bridge/request_submit.py | 38 ++- .../_service/http_bridge/retry_circuit.py | 18 ++ .../_service/http_bridge/upstream_events.py | 25 +- .../.openspec.yaml | 2 + .../proposal.md | 29 ++ .../specs/responses-api-compat/spec.md | 74 +++++ .../tasks.md | 21 ++ tests/unit/test_proxy_http_bridge.py | 261 +++++++++++++++++- tests/unit/test_proxy_utils.py | 28 ++ 10 files changed, 492 insertions(+), 14 deletions(-) create mode 100644 openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml create mode 100644 openspec/changes/classify-bridge-recovery-error-frames/proposal.md create mode 100644 openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/classify-bridge-recovery-error-frames/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 4c3cdf75ca..c5eebf77e5 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -2770,7 +2770,15 @@ def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyRespo error = payload.get("error") if not isinstance(error, dict): return False - code = error.get("code") + code_value = error.get("code") + raw_code = code_value.strip() if isinstance(code_value, str) and code_value.strip() else None + type_value = error.get("type") + error_type = type_value.strip() if isinstance(type_value, str) and type_value.strip() else None + # Normalize like the websocket rewrite path (#1818): upstream frames may + # carry the classifiable code only in ``type`` (or omit both code and + # param on the terse previous-response rejection), and a raw read would + # misclassify them into the ambiguous transport class below (issue #1830). + code = _normalize_error_code(raw_code, error_type) if code in { "bridge_owner_unreachable", "bridge_previous_response_not_found", diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index 364179fbf4..e3412b33f9 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -95,6 +95,9 @@ from app.modules.proxy._service.http_bridge.quarantine import ( _record_http_bridge_quarantine_wedged_pending, ) +from app.modules.proxy._service.http_bridge.retry_circuit import ( + _http_bridge_anchor_poison_detail, +) from app.modules.proxy._service.http_bridge.service_stubs import ( _call_with_supported_optional_kwargs, _classify_upstream_close, @@ -124,6 +127,9 @@ _websocket_auth_failure_requires_reauth, _websocket_request_text_is_account_neutral_fresh_replay, ) +from app.modules.proxy._service.http_bridge.upstream_events import ( + _abandon_durable_http_bridge_continuity, +) from app.modules.proxy._service.observability import ( _hash_identifier as _hash_identifier, ) @@ -2846,11 +2852,41 @@ async def _retire_stale_pending_http_bridge_session( # that handoff, genuine pre-response failures disappear from circuit # accounting while idle closes and request failures look identical. if retired_request_count > 0 and response_events_seen == 0: - await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( + consecutive_failures = await self._record_http_bridge_retry_circuit_failure_for_attempt_selection( session, detail=retry_circuit_detail or detail, selection=retry_circuit_attempt_selection, ) + poison_detail = _http_bridge_anchor_poison_detail(retry_circuit_detail or detail) + if ( + poison_detail is not None + and consecutive_failures is not None + and consecutive_failures + >= _service_get_settings().http_responses_session_bridge_anchor_poison_failure_threshold + ): + # Consecutive eventless failures on one bridge key are + # same-anchor failures (the anchor only advances on a + # completed response, which resets the circuit). Clear the + # poisoned durable anchor while this session still owns the + # lease so the next attempt is not re-anchored into the same + # failure. Without this, only the admission-waiter reader + # path could ever poison an anchor, and an anchored session + # failing without waiters cooled down forever (issue #1830). + durable_cleared = await _abandon_durable_http_bridge_continuity(self, session, detail=poison_detail) + if not durable_cleared and session.durable_session_id is not None: + # Keep failed waiterless clears visible in the same + # poison-clear telemetry the admission-waiter path emits; + # the next threshold failure re-attempts the clear. + _log_http_bridge_event( + "durable_anchor_poison_clear_failed", + session.key, + account_id=session.account.id, + model=session.request_model, + pending_count=retired_request_count, + detail=poison_detail, + cache_key_family=session.key.affinity_kind, + model_class=_extract_model_class(session.request_model) if session.request_model else None, + ) session.closed = True async with self._http_bridge_lock: # Bounded close may return while resource finalization is still diff --git a/app/modules/proxy/_service/http_bridge/retry_circuit.py b/app/modules/proxy/_service/http_bridge/retry_circuit.py index f152a0a7af..e2642de2da 100644 --- a/app/modules/proxy/_service/http_bridge/retry_circuit.py +++ b/app/modules/proxy/_service/http_bridge/retry_circuit.py @@ -38,6 +38,24 @@ "missing_response_created_timeout": "stream_idle_timeout", "response_create_gate_timeout_stuck_pending": "stream_idle_timeout", } +_HTTP_BRIDGE_ANCHOR_POISON_DETAILS = { + "stream_idle_timeout": "repeated_zero_event_idle_timeout", + "stream_incomplete": "repeated_zero_event_stream_incomplete", +} + + +def _http_bridge_anchor_poison_detail(detail: str | None) -> str | None: + """Map an eventless retry-circuit failure class to its anchor-poison detail. + + Consecutive eventless failures on one bridge key are same-anchor failures: + the durable anchor only advances on a completed response, which resets the + circuit. Both ambiguous transport classes therefore count toward anchor + poison (issue #1830); ``clean_close`` never does. + """ + if detail is None: + return None + aliased = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail) + return _HTTP_BRIDGE_ANCHOR_POISON_DETAILS.get(aliased) @dataclass(slots=True) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 2bd052dd76..043e1bef46 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -75,6 +75,9 @@ _record_http_bridge_quarantine_eventless_timeout, _record_http_bridge_quarantine_wedged_pending, ) +from app.modules.proxy._service.http_bridge.retry_circuit import ( + _http_bridge_anchor_poison_detail, +) from app.modules.proxy._service.http_bridge.service_stubs import ( _assign_websocket_response_id, _await_cancelled_task, @@ -963,6 +966,8 @@ async def _clear_durable_http_bridge_response_anchor( async def _abandon_durable_http_bridge_continuity( service: Any, session: "_HTTPBridgeSession", + *, + detail: str = "repeated_zero_event_idle_timeout", ) -> bool: """Clear durable continuity before retiring a repeatedly poisoned bridge. @@ -999,7 +1004,7 @@ async def _abandon_durable_http_bridge_continuity( session.key, account_id=session.account.id, model=session.request_model, - detail="repeated_zero_event_idle_timeout", + detail=detail, cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) @@ -1159,7 +1164,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( ), ) finally: - poison_after_deferred_failures = False + poison_detail: str | None = None if session.admission_waiter_count > 0 and not force_retire: retry_circuit_detail = None if close_classification == "clean": @@ -1179,19 +1184,21 @@ async def _fail_http_bridge_reader_and_maybe_retire( detail=retry_circuit_detail, selection=retry_circuit_attempt_selection, ) - poison_after_deferred_failures = bool( - retry_circuit_detail == "stream_idle_timeout" + poison_candidate_detail = _http_bridge_anchor_poison_detail(retry_circuit_detail) + if ( + poison_candidate_detail is not None and observed_response_events == 0 and consecutive_failures is not None and consecutive_failures >= _service_get_settings().http_responses_session_bridge_anchor_poison_failure_threshold - ) - if poison_after_deferred_failures: - durable_cleared = await _abandon_durable_http_bridge_continuity(self, session) + ): + poison_detail = poison_candidate_detail + if poison_detail is not None: + durable_cleared = await _abandon_durable_http_bridge_continuity(self, session, detail=poison_detail) if durable_cleared: await self._retire_stale_pending_http_bridge_session( session, - detail="repeated_zero_event_idle_timeout", + detail=poison_detail, response_events_seen=observed_response_events, **retry_circuit_attempt_kwargs, ) @@ -1203,7 +1210,7 @@ async def _fail_http_bridge_reader_and_maybe_retire( account_id=session.account.id, model=session.request_model, pending_count=session.admission_waiter_count, - detail="repeated_zero_event_idle_timeout", + detail=poison_detail, cache_key_family=session.key.affinity_kind, model_class=_extract_model_class(session.request_model) if session.request_model else None, ) diff --git a/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml b/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml new file mode 100644 index 0000000000..f774115be7 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-20 diff --git a/openspec/changes/classify-bridge-recovery-error-frames/proposal.md b/openspec/changes/classify-bridge-recovery-error-frames/proposal.md new file mode 100644 index 0000000000..588cf25415 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/proposal.md @@ -0,0 +1,29 @@ +# Classify Bridge Recovery Error Frames + +## Why + +The HTTP responses session bridge can wedge a session permanently (issue #1830). After one genuine mid-turn interruption the bridge rebinds to its stored durable anchor and re-injects it on every attempt. When upstream rejects that anchor with a classifiable previous-response error, two gaps keep the session unrecoverable: + +1. The bridge-local recovery gate reads raw error codes without the normalization the WebSocket path gained in the `classify-invalid-previous-response-id` change (#1818): a frame that carries its classifiable code only in `type`, or the terse parameterless ``Invalid `previous_response_id`.`` shape, falls through to the ambiguous-transport class instead of previous-response recovery. +2. Anchor poisoning only counts `stream_idle_timeout` failures, and only on the reader path when admission waiters exist. The wedge observed in production fails eventlessly with `stream_incomplete` (the bridge's masked form of an upstream previous-response rejection), so the retry circuit opens and cools down forever while `http_responses_session_bridge_anchor_poison_failure_threshold` never fires. Operators had to wipe the `http_bridge_*` tables to free sessions. + +## What Changes + +- Route the bridge-local previous-response recovery gate through the same error-code normalization as the WebSocket rewrite path (code falls back to `type`; the terse parameterless invalid-previous-response shape classifies as a continuity miss). +- Count both ambiguous eventless transport classes — `stream_incomplete` and `stream_idle_timeout` (with its aliased diagnostics) — toward anchor poison, so consecutive same-anchor failures self-heal even when the frame is genuinely unclassifiable. `clean_close` still never poisons. +- Evaluate anchor poison at the shared retirement boundary as well, so a wedged anchored session that fails without admission waiters also clears its poisoned durable anchor once the threshold is reached. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `responses-api-compat`: Normalize error frames at the bridge-local recovery gate and widen anchor poisoning to all consecutive eventless same-anchor failures, including the waiterless retirement path. + +## Impact + +- HTTP bridge recovery gate (`app/modules/proxy/_service/http_bridge/helpers.py`), anchor-poison accounting (`app/modules/proxy/_service/http_bridge/upstream_events.py`, `app/modules/proxy/_service/http_bridge/request_submit.py`, `app/modules/proxy/_service/http_bridge/retry_circuit.py`). +- No API, schema, migration, dependency, configuration, or dashboard changes; the existing poison threshold setting and its default of seven are unchanged. diff --git a/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md b/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..17b86e6a4b --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/specs/responses-api-compat/spec.md @@ -0,0 +1,74 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Bridge-local previous-response recovery classifies normalized error frames + +When the HTTP bridge evaluates whether a failed anchored request may enter bridge-local previous-response recovery, it MUST classify the error frame with the same normalization as the WebSocket rewrite path: a missing or empty `code` MUST fall back to the error `type` before classification, and the parameterless ``Invalid `previous_response_id`.`` invalid-request shape MUST classify as a previous-response continuity miss. A classifiable previous-response rejection MUST route into previous-response recovery and MUST NOT be treated as an ambiguous transport failure that only feeds the retry-circuit cooldown. + +#### Scenario: Terse parameterless rejection enters local recovery + +- **GIVEN** an anchored HTTP bridge request fails with `type = "invalid_request_error"`, no `code`, no `param`, and the message ``Invalid `previous_response_id`.`` +- **WHEN** the bridge evaluates bridge-local previous-response recovery for that failure +- **THEN** the failure classifies as a previous-response continuity miss +- **AND** the bridge attempts previous-response recovery instead of the ambiguous-transport path + +#### Scenario: Code carried only in the error type classifies + +- **GIVEN** an anchored HTTP bridge request fails with no `code` and `type = "previous_response_not_found"` +- **WHEN** the bridge evaluates bridge-local previous-response recovery for that failure +- **THEN** the failure enters previous-response recovery instead of the ambiguous-transport class + +#### Scenario: Unrelated errors keep their classification + +- **WHEN** a failed anchored request carries an error whose normalized code, param, and message do not match a previous-response continuity miss +- **THEN** the bridge MUST NOT classify it as a previous-response continuity miss + +## MODIFIED Requirements + +### Requirement: Repeated zero-event idle failures poison dead anchors + +For hard HTTP bridge keys, repeated zero-event failures MUST use the existing durable retry-circuit counter to identify an anchor that should no longer remain addressable; the counter resets on a completed response, so a run of consecutive failures proves the anchor never advanced. Both ambiguous eventless transport classes — `stream_idle_timeout` (including its aliased diagnostics) and `stream_incomplete` — MUST be able to trigger anchor poisoning at the threshold; a `clean_close` outcome MUST NOT itself trigger anchor poisoning. When an eligible eventless failure reaches the configured poison threshold for the same hard bridge key, the proxy MUST abandon durable continuity for that session and retire the bridge even when admission waiters exist, and the shared retirement boundary MUST clear the poisoned durable anchor even when no admission waiter exists, while the session still owns its durable lease. If the clear cannot be confirmed on the waiterless retirement path, the proxy MUST re-attempt it when a later eligible eventless failure at or above the threshold retires the session. The default threshold MUST be no greater than seven failures. + +#### Scenario: Admission waiters cannot defer anchor poisoning forever + +- **GIVEN** a hard durable bridge key has admission waiters +- **AND** repeated zero-event idle failures for that same key reach the poison + threshold +- **WHEN** the reader failure path would normally defer retirement for the + admission waiter +- **THEN** the proxy clears the durable continuity anchors +- **AND** retires the session despite the admission waiter +- **AND** the next attach starts from fresh durable state rather than the + poisoned previous-response anchor + +#### Scenario: Repeated eventless stream_incomplete failures poison the anchor + +- **GIVEN** a hard durable bridge key has a stored durable anchor +- **AND** every anchored attempt fails eventlessly with `stream_incomplete` (for example a masked upstream previous-response rejection) +- **WHEN** consecutive failures for that key reach the poison threshold +- **THEN** the proxy clears the durable continuity anchors under the session's owner epoch +- **AND** the next attach starts from fresh durable state instead of looping through retry-circuit cooldown + +#### Scenario: Waiterless retirement poisons the anchor at the threshold + +- **GIVEN** a hard durable bridge key fails eventlessly with no admission waiters +- **WHEN** the shared retirement boundary records the eventless failure that reaches the poison threshold +- **THEN** the proxy clears the durable continuity anchors before releasing the durable lease + +#### Scenario: Failed waiterless clear is re-attempted on the next threshold failure + +- **GIVEN** the waiterless retirement path reached the poison threshold but the durable continuity clear could not be confirmed +- **WHEN** the next eligible eventless failure for the same key retires the session +- **THEN** the proxy re-attempts the durable continuity clear under the new session's owner epoch + +#### Scenario: Clean closes never trigger anchor poisoning + +- **WHEN** a `clean_close` retry-circuit outcome is recorded for a hard bridge key, at any consecutive-failure count +- **THEN** that outcome does not clear the durable continuity anchors + +#### Scenario: Lease liveness comparison is timezone-safe +- **GIVEN** a durable bridge session whose `lease_expires_at` was read from a `timestamptz` column (offset-aware) on PostgreSQL +- **WHEN** the dead-owner classifier evaluates lease liveness against the application's naive-UTC clock +- **THEN** both timestamps MUST be normalized to naive UTC before comparison +- **AND** the anchored-lookup path MUST NOT raise on mixed-awareness datetimes diff --git a/openspec/changes/classify-bridge-recovery-error-frames/tasks.md b/openspec/changes/classify-bridge-recovery-error-frames/tasks.md new file mode 100644 index 0000000000..7934025565 --- /dev/null +++ b/openspec/changes/classify-bridge-recovery-error-frames/tasks.md @@ -0,0 +1,21 @@ +# Tasks + +## 1. Regression Coverage + +- [x] 1.1 Add gate regressions for the terse parameterless ``Invalid `previous_response_id`.`` frame and a frame carrying the classifiable code only in `type`, verifying both misclassify (no recovery) before the fix. +- [x] 1.2 Add anchor-poison regressions: consecutive eventless `stream_incomplete` reader failures with an admission waiter, and consecutive eventless failures through the shared retirement boundary without waiters, verifying neither poisons the anchor before the fix. + +## 2. Classifier Routing + +- [x] 2.1 Normalize the error code (falling back to `type`) in the bridge-local previous-response recovery gate before all classification checks, matching the WebSocket rewrite path from `classify-invalid-previous-response-id`. + +## 3. Anchor Poison Counting + +- [x] 3.1 Map both ambiguous eventless retry-circuit classes (`stream_incomplete`, `stream_idle_timeout` and its aliases) to anchor-poison details; keep `clean_close` excluded. +- [x] 3.2 Widen the deferred reader-path poison branch to both classes and thread the poison detail into the poisoned-anchor observability events. +- [x] 3.3 Evaluate the poison threshold at the shared retirement boundary and clear the poisoned durable anchor while the session still owns its durable lease. + +## 4. Verification + +- [x] 4.1 Run the touched bridge unit and integration suites, ruff, and type checks. +- [x] 4.2 Run strict OpenSpec validation for this change and review the final diff for unrelated changes. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 76a8b76c88..2298332e4e 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -27263,7 +27263,7 @@ async def test_http_bridge_eventless_pending_retirement_records_one_retry_circui pending_requests=deque([owner]), queued_request_count=1, ) - record_failure = AsyncMock() + record_failure = AsyncMock(return_value=None) monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) @@ -27311,7 +27311,7 @@ async def test_http_bridge_reader_failure_preserves_pre_drain_request_for_retry_ pending_requests=deque([owner]), queued_request_count=1, ) - record_failure = AsyncMock() + record_failure = AsyncMock(return_value=None) monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) @@ -28651,7 +28651,7 @@ async def test_http_bridge_reader_failure_keeps_waiter_count_when_draining_reque ) session.admission_waiter_count = 1 fail_pending = AsyncMock() - record_failure = AsyncMock() + record_failure = AsyncMock(return_value=None) monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) @@ -28714,6 +28714,261 @@ async def test_http_bridge_repeated_zero_event_idle_timeouts_poison_anchor_with_ ) +@pytest.mark.asyncio +async def test_http_bridge_repeated_zero_event_stream_incompletes_poison_anchor_with_waiter( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830: consecutive eventless ``stream_incomplete`` failures on the + # same anchor must count toward anchor poison exactly like idle timeouts, + # or a poisoned anchor wedges the session behind the retry circuit forever. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + session = _make_bridge_session( + key_value="bridge-anchor-poison-stream-incomplete", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.admission_waiter_count = 1 + session.durable_session_id = "durable-anchor-poison-stream-incomplete" + session.durable_owner_epoch = 3 + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), + ) + service._durable_bridge = durable_bridge + fail_pending = AsyncMock() + retire = AsyncMock() + monkeypatch.setattr(service, "_fail_pending_websocket_requests", fail_pending) + monkeypatch.setattr(service, "_retire_stale_pending_http_bridge_session", retire) + + for failure_number in range(1, 8): + retired = await service._fail_http_bridge_reader_and_maybe_retire( + session, + error_code="stream_incomplete", + error_message="Upstream websocket closed before response.completed", + ) + assert retired is (failure_number == 7) + + durable_bridge.rebind_session_account.assert_awaited_once_with( + session_id="durable-anchor-poison-stream-incomplete", + api_key_id=None, + instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, + owner_epoch=3, + account_id="acc-bridge", + clear_continuity=True, + ) + retire.assert_awaited_once_with( + session, + detail="repeated_zero_event_stream_incomplete", + response_events_seen=0, + retry_circuit_attempt_selection=proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection(kind="absent"), + ) + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_poisons_anchor_after_repeated_eventless_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830: the shared retirement boundary is the only strike recorder + # when a wedged anchored session fails without admission waiters, so it + # must clear the poisoned durable anchor once the threshold is reached. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), + ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + for _failure_number in range(7): + session = _make_bridge_session( + key_value="bridge-anchor-poison-retire", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-poison-retire" + session.durable_owner_epoch = 5 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + ) + + durable_bridge.rebind_session_account.assert_awaited_once_with( + session_id="durable-anchor-poison-retire", + api_key_id=None, + instance_id=proxy_service.get_settings().http_responses_session_bridge_instance_id, + owner_epoch=5, + account_id="acc-bridge", + clear_continuity=True, + ) + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_reattempts_failed_poison_clear( + caplog: pytest.LogCaptureFixture, + monkeypatch: pytest.MonkeyPatch, +) -> None: + # A clear that cannot be confirmed must not lose the self-heal: the next + # eligible eventless failure at or above the threshold re-attempts it, and + # each failed clear stays visible in the poison-clear telemetry. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=False), + ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + with caplog.at_level(logging.INFO): + for _failure_number in range(8): + session = _make_bridge_session( + key_value="bridge-anchor-poison-clear-retry", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-poison-clear-retry" + session.durable_owner_epoch = 5 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + ) + + assert durable_bridge.rebind_session_account.await_count == 2 + assert caplog.text.count("event=durable_anchor_poison_clear_failed") == 2 + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_recovers_terse_previous_response_rejection( + monkeypatch: pytest.MonkeyPatch, +) -> None: + # Issue #1830 product path: an anchored bridge request fails with the terse + # parameterless previous-response rejection (classifiable only after code + # normalization). The bridge must enter local previous-response recovery + # instead of surfacing the failure into the ambiguous-transport class. + service = proxy_service.ProxyService(cast(Any, nullcontext())) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.4", + "instructions": "hi", + "previous_response_id": "resp_stale_anchor", + "input": [{"role": "user", "content": [{"type": "input_text", "text": "continue"}]}], + } + ) + session = _make_bridge_session(key_value="sid-terse-recovery") + terse_rejection = ProxyResponseError( + 400, + { + "error": { + "type": "invalid_request_error", + "message": "Invalid `previous_response_id`.", + } + }, + ) + get_or_create = AsyncMock(side_effect=[session, session]) + stream_attempts: list[str | None] = [] + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + queue_limit: int, + propagate_http_errors: bool, + downstream_turn_state: str | None, + request_deadline: float | None = None, + ): + del queue_limit, propagate_http_errors, downstream_turn_state, request_deadline + stream_attempts.append(request_state.previous_response_id) + del text_data + if len(stream_attempts) == 1: + raise terse_rejection + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_http_bridge_local_owner_account_id", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_resolve_websocket_previous_response_owner", AsyncMock(return_value="acc-owner")) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_reset_http_bridge_session_after_local_terminal_error", AsyncMock()) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={"session_id": "sid-terse-recovery"}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert get_or_create.await_count == 2 + recovery_call = get_or_create.await_args_list[1] + assert recovery_call.kwargs["allow_previous_response_recovery_rebind"] is True + assert recovery_call.kwargs["request_stage"] == "reattach" + assert stream_attempts == ["resp_stale_anchor", "resp_stale_anchor"] + + +@pytest.mark.asyncio +async def test_http_bridge_retire_stale_pending_clean_close_never_poisons_anchor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + durable_bridge = SimpleNamespace( + lookup_retry_circuit=AsyncMock(return_value=None), + persist_retry_circuit=AsyncMock(), + rebind_session_account=AsyncMock(return_value=True), + ) + service._durable_bridge = durable_bridge + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + + for _failure_number in range(7): + session = _make_bridge_session( + key_value="bridge-anchor-clean-close", + pending_requests=deque([_make_eventless_http_bridge_owner()]), + queued_request_count=1, + ) + session.durable_session_id = "durable-anchor-clean-close" + session.durable_owner_epoch = 6 + await service._retire_stale_pending_http_bridge_session( + session, + detail="stream_incomplete", + retry_circuit_detail="clean_close", + ) + + durable_bridge.rebind_session_account.assert_not_awaited() + + @pytest.mark.asyncio @pytest.mark.parametrize("clear_outcome", [False, RuntimeError("clear failed")]) async def test_http_bridge_anchor_poisoning_waits_when_durable_clear_fails( diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 0b20f804d3..3984cf0939 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -34997,6 +34997,34 @@ def test_http_bridge_should_attempt_local_previous_response_recovery_invalid_req assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(non_recoverable_error) is False +def test_http_bridge_should_attempt_local_previous_response_recovery_normalizes_upstream_error_frames(): + # The terse parameterless rejection classified on the websocket path by + # #1818: no ``code``, no ``param``, classifiable only after normalizing + # ``type`` into the code slot (issue #1830). + terse_parameterless_error = proxy_module.ProxyResponseError( + 400, + { + "error": { + "type": "invalid_request_error", + "message": "Invalid `previous_response_id`.", + } + }, + ) + # Frames that carry the classifiable code only in ``type``. + type_only_not_found_error = proxy_module.ProxyResponseError( + 404, + { + "error": { + "type": "previous_response_not_found", + "message": "Previous response with id 'resp_prev_anchor' not found.", + } + }, + ) + + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(terse_parameterless_error) is True + assert proxy_service._http_bridge_should_attempt_local_previous_response_recovery(type_only_not_found_error) is True + + def test_http_bridge_server_recovery_mode_retries_ambiguous_transport_once(monkeypatch: pytest.MonkeyPatch): ambiguous_error = proxy_module.ProxyResponseError( 502, From ed2c94d4b8ece64455233e5293d44ec0f263e6bc Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 18:20:23 +0900 Subject: [PATCH 097/117] fix(proxy): O(1) shared-future admission waits + event-loop lag watchdog (#1842) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(proxy): O(1) shared-future admission waits + event-loop lag watchdog Replace wait_for(shield(shared)) on many-waiter shared futures (http-bridge inflight/capacity registries, token-refresh singleflight) with a fan-out helper that keeps exactly one callback on the shared future and gives each waiter a private O(1)-detach proxy. Python 3.14's shield attaches three callbacks per waiter, leaks one on waiter cancellation, and pays O(n) remove_done_callback scans — mass timeouts and disconnect storms degraded to O(n^2) and livelocked the event loop in the 2026-08-20 production incident (98% of GIL samples inside remove_done_callback with zero client sessions). Add an event-loop lag watchdog (1s sleep-drift sampler) exporting codex_lb_event_loop_lag_seconds / codex_lb_event_loop_lag_warnings_total with a rate-limited warning log, so loop starvation is an explicit operator signal instead of mysterious global slowness. Configurable via event_loop_lag_warn_threshold_seconds (default 0.5s, 0 disables). Co-Authored-By: Claude Fable 5 * chore(settings): regenerate settings reference and bump field ratchet to 131 event_loop_lag_warn_threshold_seconds justification lives in the PR body and openspec/changes/harden-shared-future-admission-waits (zero-config default, 0 disables; threshold scales with host CPU class so a constant does not fit). Co-Authored-By: Claude Fable 5 * docs(openspec): scope timeout invariant to the wait mechanism; link settings page to proxy-runtime-observability Addresses CodeRabbit review threads on #1842. Co-Authored-By: Claude Fable 5 --------- Co-authored-by: Claude Fable 5 --- app/core/config/settings.py | 7 + app/core/metrics/prometheus.py | 15 ++ app/core/resilience/loop_lag_monitor.py | 60 ++++++++ app/core/utils/shared_future.py | 80 +++++++++++ app/main.py | 15 ++ app/modules/accounts/auth_manager.py | 14 +- .../proxy/_service/http_bridge/mixin.py | 15 +- docs/reference/settings.md | 5 +- .../proposal.md | 48 +++++++ .../specs/proxy-admission-control/spec.md | 41 ++++++ .../specs/proxy-runtime-observability/spec.md | 32 +++++ .../tasks.md | 18 +++ scripts/generate_settings_reference.py | 4 +- tests/unit/test_loop_lag_monitor.py | 92 ++++++++++++ tests/unit/test_proxy_http_bridge.py | 68 +++++++++ tests/unit/test_settings_reference.py | 2 +- tests/unit/test_shared_future_waiters.py | 136 ++++++++++++++++++ 17 files changed, 640 insertions(+), 12 deletions(-) create mode 100644 app/core/resilience/loop_lag_monitor.py create mode 100644 app/core/utils/shared_future.py create mode 100644 openspec/changes/harden-shared-future-admission-waits/proposal.md create mode 100644 openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md create mode 100644 openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md create mode 100644 openspec/changes/harden-shared-future-admission-waits/tasks.md create mode 100644 tests/unit/test_loop_lag_monitor.py create mode 100644 tests/unit/test_shared_future_waiters.py diff --git a/app/core/config/settings.py b/app/core/config/settings.py index 00376d1a53..addf7df969 100644 --- a/app/core/config/settings.py +++ b/app/core/config/settings.py @@ -484,6 +484,13 @@ def upstream_websocket_proxy_env(self) -> Mapping[str, str | None]: # (``app/core/resilience/memory_monitor.py`` derives the warning level). memory_reject_threshold_mb: int = 0 + # Event-loop lag watchdog (0 = disabled). Samples asyncio.sleep drift once + # per second; lag at or above the threshold emits a rate-limited warning + # and Prometheus signals (``app/core/resilience/loop_lag_monitor.py``). + # Default 0.5s: an order of magnitude above healthy scheduling jitter, + # well below the lag that fails 10s-budget health checks. + event_loop_lag_warn_threshold_seconds: float = Field(default=0.5, ge=0.0) + # OpenTelemetry otel_enabled: bool = False otel_exporter_endpoint: str = "" diff --git a/app/core/metrics/prometheus.py b/app/core/metrics/prometheus.py index 055bbbc985..55dda64dc7 100644 --- a/app/core/metrics/prometheus.py +++ b/app/core/metrics/prometheus.py @@ -302,6 +302,17 @@ def labels(self, *args: str, **kwargs: str) -> "HistogramLike": ... ["outcome"], registry=REGISTRY, ) + event_loop_lag_seconds = Gauge( + "codex_lb_event_loop_lag_seconds", + "Sampled event-loop scheduling lag (asyncio.sleep drift) in seconds", + registry=REGISTRY, + **({"multiprocess_mode": "livemax"} if MULTIPROCESS_MODE else {}), + ) + event_loop_lag_warnings_total = Counter( + "codex_lb_event_loop_lag_warnings_total", + "Total event-loop lag samples at or above the warning threshold", + registry=REGISTRY, + ) stream_keepalive_sent_total = Counter( "codex_lb_stream_keepalive_sent_total", "Total downstream SSE keepalive frames emitted by surface", @@ -385,6 +396,8 @@ def mark_process_dead() -> None: http_bridge_prewarm_total: CounterLike | None = None http_bridge_stuck_retire_total: CounterLike | None = None http_bridge_retry_circuit_total: CounterLike | None = None + event_loop_lag_seconds: GaugeLike | None = None + event_loop_lag_warnings_total: CounterLike | None = None stream_keepalive_sent_total: CounterLike | None = None stream_idle_timeout_total: CounterLike | None = None cache_invalidation_bump_failures_total: CounterLike | None = None @@ -429,6 +442,8 @@ def mark_process_dead() -> None: "cap_partition_replicas", "circuit_breaker_state", "continuity_fail_closed_total", + "event_loop_lag_seconds", + "event_loop_lag_warnings_total", "continuity_owner_resolution_total", "http_bridge_prewarm_total", "http_bridge_retry_circuit_total", diff --git a/app/core/resilience/loop_lag_monitor.py b/app/core/resilience/loop_lag_monitor.py new file mode 100644 index 0000000000..dbe8a69b72 --- /dev/null +++ b/app/core/resilience/loop_lag_monitor.py @@ -0,0 +1,60 @@ +"""Event-loop lag watchdog. + +Samples scheduling delay by measuring ``asyncio.sleep`` drift. When the loop +is starved (a callback storm, synchronous work on the loop, CPU saturation), +every request and health check degrades at once while per-request logs stay +quiet: nothing says "the loop itself is busy". The 2026-08-20 incident — an +``asyncio.shield`` callback storm pinning one core for hours — surfaced only +as mysterious global slowness and health-check flapping. This monitor turns +that state into an explicit, rate-limited warning log plus Prometheus +signals (``codex_lb_event_loop_lag_seconds`` gauge and +``codex_lb_event_loop_lag_warnings_total`` counter) so operators and alerts +can distinguish "loop starved" from "upstream slow". +""" + +from __future__ import annotations + +import asyncio +import logging +import time + +from app.core.metrics import prometheus as prometheus_metrics + +logger = logging.getLogger(__name__) + +_SAMPLE_INTERVAL_SECONDS = 1.0 +# One warning line per window at most; the gauge/counter stay per-sample. The +# worst lag seen inside a suppressed window is carried into the next line so +# suppression never hides the magnitude of a spike. +_WARN_LOG_INTERVAL_SECONDS = 60.0 + + +async def run_event_loop_lag_monitor(*, warn_threshold_seconds: float) -> None: + """Sample loop lag forever; the caller owns and cancels the task.""" + last_warn_monotonic = float("-inf") + worst_suppressed_lag = 0.0 + while True: + started = time.monotonic() + await asyncio.sleep(_SAMPLE_INTERVAL_SECONDS) + lag = max(0.0, time.monotonic() - started - _SAMPLE_INTERVAL_SECONDS) + gauge = prometheus_metrics.event_loop_lag_seconds + if gauge is not None: + gauge.set(lag) + if lag < warn_threshold_seconds: + continue + counter = prometheus_metrics.event_loop_lag_warnings_total + if counter is not None: + counter.inc() + now = time.monotonic() + if now - last_warn_monotonic < _WARN_LOG_INTERVAL_SECONDS: + worst_suppressed_lag = max(worst_suppressed_lag, lag) + continue + logger.warning( + "event_loop_lag lag_seconds=%.3f worst_suppressed_seconds=%.3f threshold_seconds=%.3f " + "(event loop starved: callback storm, sync work on the loop, or CPU saturation)", + lag, + worst_suppressed_lag, + warn_threshold_seconds, + ) + last_warn_monotonic = now + worst_suppressed_lag = 0.0 diff --git a/app/core/utils/shared_future.py b/app/core/utils/shared_future.py new file mode 100644 index 0000000000..5e959442c5 --- /dev/null +++ b/app/core/utils/shared_future.py @@ -0,0 +1,80 @@ +"""Await shared futures without per-waiter callbacks on the shared object. + +``asyncio.wait_for(asyncio.shield(shared), timeout)`` attaches done callbacks +to ``shared`` for every waiter and removes them with O(n) list scans when a +waiter is cancelled or times out. With many waiters piled onto one long-lived +future (the http-bridge inflight/capacity registries, refresh singleflight), +a mass timeout turns the event loop into an O(N^2) callback grinder. Python +3.14's ``shield`` additionally leaks one ``_clear_awaited_by_callback`` per +attempt onto the still-pending future, so each retry cycle makes every later +scan more expensive. In the 2026-08-20 production incident this starved the +event loop for hours (98% of GIL samples inside ``Future.remove_done_callback``) +with zero client sessions attached. + +``wait_on_shared_future`` keeps exactly one fan-out callback on the shared +future regardless of waiter count. Each waiter awaits its own single-use proxy +future, so waiter timeout and cancellation are O(1) set operations that never +touch the shared future's callback list. +""" + +from __future__ import annotations + +import asyncio +from typing import TypeVar + +_T = TypeVar("_T") + +_WAITERS_ATTR = "_shared_future_fanout_waiters" + + +def _fan_out(shared: "asyncio.Future[_T]", waiters: "set[asyncio.Future[_T]]") -> None: + for waiter in waiters: + if waiter.done(): + continue + if shared.cancelled(): + waiter.cancel() + continue + exc = shared.exception() + if exc is not None: + waiter.set_exception(exc) + # Consume eagerly: a waiter whose task was cancelled between this + # fan-out and its resumption would otherwise log + # "exception was never retrieved" from the proxy destructor. + waiter.exception() + else: + waiter.set_result(shared.result()) + waiters.clear() + + +async def wait_on_shared_future( + shared: "asyncio.Future[_T]", + *, + timeout: float | None = None, +) -> _T: + """Drop-in equivalent of ``wait_for(shield(shared), timeout)`` for futures + awaited by many concurrent waiters. + + - ``shared``'s result, exception, or cancellation propagates to every + waiter exactly as with ``shield``. + - ``timeout`` raises ``TimeoutError``; ``shared`` is never cancelled or + otherwise mutated by a waiter timing out or being cancelled. + - Cancelling the awaiting task detaches its proxy in O(1) and leaves + ``shared`` (and the work it represents) running. + """ + if shared.done(): + return shared.result() + waiters: set[asyncio.Future[_T]] | None = getattr(shared, _WAITERS_ATTR, None) + if waiters is None: + # No await between the ``done()`` check above and this registration, + # so the fan-out callback cannot have fired with an empty set. + waiters = set() + setattr(shared, _WAITERS_ATTR, waiters) + shared.add_done_callback(lambda done, _waiters=waiters: _fan_out(done, _waiters)) + proxy: asyncio.Future[_T] = asyncio.get_running_loop().create_future() + waiters.add(proxy) + try: + if timeout is None: + return await proxy + return await asyncio.wait_for(proxy, timeout) + finally: + waiters.discard(proxy) diff --git a/app/main.py b/app/main.py index 81a53f8845..83300cc21e 100644 --- a/app/main.py +++ b/app/main.py @@ -54,6 +54,7 @@ from app.core.openai.model_refresh_scheduler import build_model_refresh_scheduler from app.core.resilience.backpressure import BackpressureMiddleware from app.core.resilience.bulkhead import BulkheadMiddleware, get_bulkhead +from app.core.resilience.loop_lag_monitor import run_event_loop_lag_monitor from app.core.resilience.memory_monitor import configure as configure_memory_monitor from app.core.retention.scheduler import build_data_retention_scheduler from app.core.scheduling.leader_election import get_leader_election @@ -607,6 +608,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N ring_service = RingMembershipService(SessionLocal) instance_id = settings.http_responses_session_bridge_instance_id heartbeat_task = asyncio.create_task(_register_and_heartbeat(ring_service, instance_id)) + loop_lag_task: asyncio.Task[None] | None = None + if settings.event_loop_lag_warn_threshold_seconds > 0: + loop_lag_task = asyncio.create_task( + run_event_loop_lag_monitor( + warn_threshold_seconds=settings.event_loop_lag_warn_threshold_seconds, + ) + ) startup_module._startup_complete = True try: @@ -665,6 +673,13 @@ async def _activate_bridge_membership(svc: RingMembershipService, iid: str) -> N except (asyncio.CancelledError, TimeoutError): pass + if loop_lag_task is not None: + loop_lag_task.cancel() + try: + await asyncio.wait_for(loop_lag_task, timeout=2) + except (asyncio.CancelledError, TimeoutError): + pass + if ring_service is not None and instance_id is not None: try: await asyncio.wait_for( diff --git a/app/modules/accounts/auth_manager.py b/app/modules/accounts/auth_manager.py index 0c4339930c..206c66e4af 100644 --- a/app/modules/accounts/auth_manager.py +++ b/app/modules/accounts/auth_manager.py @@ -30,6 +30,7 @@ from app.core.crypto import TokenEncryptor from app.core.plan_types import coerce_account_plan_type from app.core.upstream_proxy import UpstreamProxyRouteError, resolve_upstream_route +from app.core.utils.shared_future import wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountProxyBinding, AccountStatus from app.db.session import get_background_session @@ -196,7 +197,12 @@ async def run( self._inflight[key] = task task.add_done_callback(lambda done, *, cache_key=key: self._schedule_complete(cache_key, done)) assert task is not None - return await asyncio.shield(task) + # Not asyncio.shield: shield attaches per-waiter callbacks to the + # shared singleflight task, which degrades to O(N^2) removal scans + # when piled-up waiters are cancelled (see shared_future.py). The + # helper preserves shield semantics: a cancelled waiter detaches + # without aborting the refresh. + return await wait_on_shared_future(task) def _schedule_complete(self, key: _RefreshSingleflightKey, task: asyncio.Task[Account]) -> None: asyncio.create_task(self._complete(key, task)) @@ -291,9 +297,9 @@ async def ensure_fresh(self, account: Account, *, force: bool = False) -> Accoun async def _run_refresh(self, account: Account) -> Account: """Singleflight body for token refresh. - Runs inside a detached task that the singleflight keeps alive with - ``asyncio.shield`` (so concurrent waiters share one refresh and a - cancelled waiter does not abort it). Because the task outlives the + Runs inside a detached task that the singleflight keeps alive via + ``wait_on_shared_future`` (so concurrent waiters share one refresh and + a cancelled waiter does not abort it). Because the task outlives the caller, it MUST NOT use the caller's request-scoped session: when a client disconnects, the caller is cancelled and its ``async with get_background_session()`` closes that session, while this diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index d628dd1a86..8d79de6d87 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -46,6 +46,7 @@ bridge_soft_local_rebind_total, ) from app.core.utils.request_id import ensure_request_scope_id +from app.core.utils.shared_future import wait_on_shared_future from app.db.models import ( AccountStatus, StickySessionKind, @@ -1365,8 +1366,11 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if capacity_wait_future is not None: wait_timeout_seconds = _proxy_admission_wait_timeout_seconds(settings) try: - await asyncio.wait_for( - asyncio.shield(capacity_wait_future), + # Not wait_for(shield(...)): shield attaches per-waiter + # callbacks to the shared registry future, which livelocks + # the event loop under mass timeout (see shared_future.py). + await wait_on_shared_future( + capacity_wait_future, timeout=wait_timeout_seconds, ) except asyncio.CancelledError: @@ -1396,8 +1400,11 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: if inflight_future is not None and not owns_creation: wait_timeout_seconds = _proxy_admission_wait_timeout_seconds(settings) try: - session = await asyncio.wait_for( - asyncio.shield(inflight_future), + # Not wait_for(shield(...)): shield attaches per-waiter + # callbacks to the shared registry future, which livelocks + # the event loop under mass timeout (see shared_future.py). + session = await wait_on_shared_future( + inflight_future, timeout=wait_timeout_seconds, ) except asyncio.CancelledError: diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 058da37953..1bbedc613c 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -7,7 +7,7 @@ Regenerate with `uv run python scripts/generate_settings_reference.py`; `tests/unit/test_settings_reference.py` fails when this page drifts from `app/core/config/settings.py`. -codex-lb currently exposes 130 settings. Every setting is an environment +codex-lb currently exposes 131 settings. Every setting is an environment variable with the `CODEX_LB_` prefix (process environment or `.env` / `.env.local` next to the process). All defaults work with zero configuration — start from [Configuration](../configuration.md) for the handful that matter, @@ -251,6 +251,7 @@ the host side of the compose `ports` mapping instead. | Environment variable | Type | Default | | --- | --- | --- | +| `CODEX_LB_EVENT_LOOP_LAG_WARN_THRESHOLD_SECONDS` | `float` | `0.5` | | `CODEX_LB_TELEMETRY_ENABLED` | `bool \| None` | `None` | | `CODEX_LB_TELEMETRY_ENDPOINT` | `str` | `'https://telemetry.tokmaxxing.com'` | | `CODEX_LB_TIMEOUT_INVARIANT_VALIDATION_STRICT` | `bool` | `False` | @@ -322,4 +323,4 @@ issue [#1340](https://github.com/Soju06/codex-lb/issues/1340)): --- -*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [rate-limit-reset-credits](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)* +*Specs: [user-documentation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/user-documentation) · [responses-api-compat](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/responses-api-compat) · [rate-limit-reset-credits](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · [deployment-installation](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · [proxy-runtime-observability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-runtime-observability)* diff --git a/openspec/changes/harden-shared-future-admission-waits/proposal.md b/openspec/changes/harden-shared-future-admission-waits/proposal.md new file mode 100644 index 0000000000..db45bc1f0e --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/proposal.md @@ -0,0 +1,48 @@ +# Harden shared-future admission waits and surface event-loop lag + +## Why + +On 2026-08-20 a production instance livelocked: the event loop spent ~98% of +its CPU inside `asyncio.Future.remove_done_callback` and kept grinding at full +CPU with zero client sessions attached. Admission waiters were piling onto +shared registry futures via `asyncio.wait_for(asyncio.shield(...))`, which +attaches per-waiter callbacks to the shared future and removes them with O(n) +scans; on Python 3.14 `shield` additionally leaks one callback per attempt +onto a still-pending future, so mass timeouts and client-disconnect storms +degrade to O(n²) and starve the loop. The outage surfaced only as global +slowness and health-check flapping — no signal said "the event loop itself is +starved", which stretched diagnosis by hours. + +## What Changes + +- Replace `wait_for(shield(shared))` on shared, many-waiter futures (http-bridge + inflight/capacity registries, token-refresh singleflight) with a fan-out + helper that keeps exactly one callback on the shared future and gives each + waiter a private O(1)-detach proxy future. Wait semantics (result/exception/ + cancellation propagation, timeout contract, waiter cancellation isolation) + are unchanged. +- Add an event-loop lag watchdog: a once-per-second sampler that exports + `codex_lb_event_loop_lag_seconds` / `codex_lb_event_loop_lag_warnings_total` + and emits a rate-limited warning log when scheduling lag crosses a threshold. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-admission-control`: admission waits on shared futures must not attach + per-waiter callbacks to the shared object. +- `proxy-runtime-observability`: event-loop scheduling lag is an explicit + operator signal (metrics + rate-limited warning log). + +## Impact + +- `app/core/utils/shared_future.py` (new helper), `app/modules/proxy/_service/http_bridge/mixin.py`, + `app/modules/accounts/auth_manager.py` (call-site swaps; no behavior change). +- `app/core/resilience/loop_lag_monitor.py` (new watchdog), + `app/core/metrics/prometheus.py`, `app/main.py`, + `app/core/config/settings.py` (`event_loop_lag_warn_threshold_seconds`, + default 0.5s, `0` disables; zero-config — no operator action needed). diff --git a/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..6f77225445 --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-admission-control/spec.md @@ -0,0 +1,41 @@ +## ADDED Requirements + +### Requirement: Admission waits on shared futures scale O(1) per waiter + +When multiple requests wait on one shared future (an inflight bridge session +creation, a capacity slot, or a token-refresh singleflight), attaching a +waiter, a waiter timing out, and a waiter being cancelled MUST each perform +O(1) work on the shared future. The shared future MUST carry a constant number +of done callbacks regardless of waiter count, and the wait mechanism itself +MUST NOT cancel or otherwise mutate the shared future or the work it +represents when a waiter times out or is cancelled. Admission handlers MAY +still settle the shared future explicitly after a waiter's timeout (the +http-bridge timeout handler fails and unregisters the inflight future so +piled-up waiters converge on one overload outcome); that settlement is an +admission-contract decision, not a side effect of waiting. The shared future's +result, exception, or cancellation MUST propagate to every waiter with the +same semantics as `asyncio.wait_for(asyncio.shield(shared), timeout)`. + +#### Scenario: Waiter pile-up keeps the shared future's callback list constant + +- **WHEN** many requests wait on the same inflight bridge-session future +- **THEN** the shared future carries a constant number of done callbacks +- **AND** the callback count does not grow with the number of waiters + +#### Scenario: Mass timeout does not degrade the event loop + +- **GIVEN** waiters piled onto a shared future that has not resolved within + the admission wait timeout +- **WHEN** the waiters time out together +- **THEN** each timeout detaches in O(1) without scanning the shared future's + callback list +- **AND** the surviving admission contract (local-overload `429` with the + capacity error code) is unchanged + +#### Scenario: Client-disconnect storm leaves the owner's creation running + +- **WHEN** every waiter on an inflight session future is cancelled by client + disconnects +- **THEN** the shared future stays pending and the owner's session creation + continues +- **AND** no per-waiter callbacks remain attached to the shared future diff --git a/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md new file mode 100644 index 0000000000..97752ff627 --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/specs/proxy-runtime-observability/spec.md @@ -0,0 +1,32 @@ +## ADDED Requirements + +### Requirement: Event-loop scheduling lag is observable + +The system MUST sample event-loop scheduling lag (timer drift of a +once-per-second sleep) while serving and export it as the +`codex_lb_event_loop_lag_seconds` gauge. Samples at or above the configured +warning threshold MUST increment `codex_lb_event_loop_lag_warnings_total` and +emit a warning log that names the observed lag, the worst lag suppressed since +the previous line, and the threshold; the warning log MUST be rate-limited so +a sustained stall cannot flood the log. The threshold MUST be configurable via +`event_loop_lag_warn_threshold_seconds` with a working default requiring no +operator action, and `0` MUST disable the watchdog. + +#### Scenario: Starved event loop produces an explicit operator signal + +- **WHEN** the event loop is starved (callback storm, synchronous work on the + loop, or CPU saturation) and scheduling lag reaches the warning threshold +- **THEN** `codex_lb_event_loop_lag_warnings_total` increments +- **AND** a rate-limited `event_loop_lag` warning names the observed lag and + threshold, distinguishing loop starvation from upstream slowness + +#### Scenario: Healthy loop stays quiet + +- **WHEN** scheduling lag stays below the warning threshold +- **THEN** the gauge is still updated for dashboards +- **AND** no warning is logged and the warning counter does not increment + +#### Scenario: Watchdog can be disabled + +- **WHEN** `event_loop_lag_warn_threshold_seconds` is set to `0` +- **THEN** the watchdog task is not started diff --git a/openspec/changes/harden-shared-future-admission-waits/tasks.md b/openspec/changes/harden-shared-future-admission-waits/tasks.md new file mode 100644 index 0000000000..e547ca13fe --- /dev/null +++ b/openspec/changes/harden-shared-future-admission-waits/tasks.md @@ -0,0 +1,18 @@ +## 1. Shared-future waiter fan-out + +- [x] 1.1 Add `wait_on_shared_future` (`app/core/utils/shared_future.py`): one fan-out callback on the shared future, per-waiter proxy futures with O(1) attach/detach, `wait_for(shield())`-equivalent semantics. +- [x] 1.2 Swap the http-bridge admission wait sites (inflight session future, capacity wait future) in `app/modules/proxy/_service/http_bridge/mixin.py` to the helper. +- [x] 1.3 Swap the token-refresh singleflight wait in `app/modules/accounts/auth_manager.py` to the helper. + +## 2. Event-loop lag watchdog + +- [x] 2.1 Add `app/core/resilience/loop_lag_monitor.py`: 1s sleep-drift sampler, gauge + counter export, rate-limited warning log. +- [x] 2.2 Register `codex_lb_event_loop_lag_seconds` and `codex_lb_event_loop_lag_warnings_total` in `app/core/metrics/prometheus.py` (both branches + `__all__`). +- [x] 2.3 Wire the monitor task into the app lifespan (`app/main.py`) behind `event_loop_lag_warn_threshold_seconds` (default 0.5, `0` disables), cancelled on shutdown. + +## 3. Verification + +- [x] 3.1 Helper semantics tests (`tests/unit/test_shared_future_waiters.py`): result/exception/cancellation propagation, timeout leaves shared pending, mass-timeout keeps callback count at 1, waiter cancellation isolation, singleflight task survival. +- [x] 3.2 Bridge-surface regression test (`tests/unit/test_proxy_http_bridge.py::test_admission_waiters_do_not_accumulate_callbacks_on_shared_inflight_future`): 50 admission waiters on one inflight future keep exactly one shared callback through a cancellation storm and mass timeout; verified to fail against the old shield pattern. +- [x] 3.3 Watchdog tests (`tests/unit/test_loop_lag_monitor.py`): starved loop warns + increments counter, healthy loop stays quiet, warning log rate-limited. +- [x] 3.4 Run affected unit suites, ruff, and strict OpenSpec validation. diff --git a/scripts/generate_settings_reference.py b/scripts/generate_settings_reference.py index 90ce01ee23..9ad7fb6519 100644 --- a/scripts/generate_settings_reference.py +++ b/scripts/generate_settings_reference.py @@ -265,7 +265,9 @@ def render_settings_reference() -> str: "[rate-limit-reset-credits]" "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/rate-limit-reset-credits) · " "[deployment-installation]" - "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation)*", + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/deployment-installation) · " + "[proxy-runtime-observability]" + "(https://github.com/Soju06/codex-lb/tree/main/openspec/specs/proxy-runtime-observability)*", "", ] ) diff --git a/tests/unit/test_loop_lag_monitor.py b/tests/unit/test_loop_lag_monitor.py new file mode 100644 index 0000000000..881d2acdc2 --- /dev/null +++ b/tests/unit/test_loop_lag_monitor.py @@ -0,0 +1,92 @@ +from __future__ import annotations + +import asyncio +import logging +import time + +import pytest + +from app.core.resilience import loop_lag_monitor + +pytestmark = pytest.mark.unit + + +class _StubGauge: + def __init__(self) -> None: + self.values: list[float] = [] + + def set(self, value: float) -> None: + self.values.append(value) + + +class _StubCounter: + def __init__(self) -> None: + self.count = 0 + + def inc(self, amount: float = 1) -> None: + self.count += amount + + +@pytest.fixture +def stub_metrics(monkeypatch): + gauge = _StubGauge() + counter = _StubCounter() + monkeypatch.setattr(loop_lag_monitor.prometheus_metrics, "event_loop_lag_seconds", gauge) + monkeypatch.setattr(loop_lag_monitor.prometheus_metrics, "event_loop_lag_warnings_total", counter) + monkeypatch.setattr(loop_lag_monitor, "_SAMPLE_INTERVAL_SECONDS", 0.01) + return gauge, counter + + +async def _run_monitor_briefly(*, warn_threshold_seconds: float, block_seconds: float) -> asyncio.Task[None]: + task = asyncio.create_task( + loop_lag_monitor.run_event_loop_lag_monitor(warn_threshold_seconds=warn_threshold_seconds) + ) + # Let the monitor enter its first sleep, then starve the loop synchronously + # so the sleep resumes late — exactly what a callback storm looks like. + await asyncio.sleep(0) + if block_seconds: + time.sleep(block_seconds) + await asyncio.sleep(0.05) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + return task + + +async def test_starved_loop_emits_warning_and_metrics(stub_metrics, caplog): + gauge, counter = stub_metrics + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await _run_monitor_briefly(warn_threshold_seconds=0.05, block_seconds=0.15) + assert any(v >= 0.05 for v in gauge.values) + assert counter.count >= 1 + assert any("event_loop_lag" in record.message for record in caplog.records) + + +async def test_healthy_loop_stays_quiet(stub_metrics, caplog): + gauge, counter = stub_metrics + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await _run_monitor_briefly(warn_threshold_seconds=0.5, block_seconds=0.0) + assert gauge.values, "gauge should be sampled even when healthy" + assert counter.count == 0 + assert not [r for r in caplog.records if "event_loop_lag" in r.message] + + +async def test_warning_log_is_rate_limited(stub_metrics, caplog): + _, counter = stub_metrics + task = asyncio.create_task(loop_lag_monitor.run_event_loop_lag_monitor(warn_threshold_seconds=0.05)) + with caplog.at_level(logging.WARNING, logger=loop_lag_monitor.logger.name): + await asyncio.sleep(0) + time.sleep(0.1) + await asyncio.sleep(0.05) + time.sleep(0.1) + await asyncio.sleep(0.05) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + warning_lines = [r for r in caplog.records if "event_loop_lag" in r.message] + assert len(warning_lines) == 1, "second spike within the window must be suppressed" + assert counter.count >= 2, "counter still tracks every over-threshold sample" diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 2298332e4e..a8b8fd5628 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -31454,3 +31454,71 @@ async def test_settle_failed_creation_releases_a_row_rebound_away_from_the_winne # will be fenced on its next renewal and retry cleanly. assert superseded is False assert winner.durable_owner_epoch == 4 + + +@pytest.mark.asyncio +async def test_admission_waiters_do_not_accumulate_callbacks_on_shared_inflight_future( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression for the 2026-08-20 event-loop livelock: admission waiters + piling onto the shared inflight future must not attach per-waiter + callbacks. The old ``wait_for(asyncio.shield(...))`` pattern left + O(waiters) callbacks on the registry future (Python 3.14 shield never + removes ``_clear_awaited_by_callback`` on waiter cancellation) and paid + O(n) removal scans per timeout, so a mass timeout ground the event loop + at O(n^2).""" + service = proxy_service.ProxyService(cast(Any, nullcontext())) + # prompt_cache_key keeps the canonical key: session_header requests + # without turn state are rewritten to per-request parallel fork keys and + # never share the inflight future. + key = proxy_service._HTTPBridgeSessionKey("prompt_cache_key", "bridge-waiter-pileup", None) + inflight: asyncio.Future[Any] = asyncio.get_running_loop().create_future() + setattr( + inflight, + http_bridge_mixin_module._HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR, + time.monotonic(), + ) + service._http_bridge_inflight_sessions[key] = inflight + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_proxy_admission_wait_timeout_seconds", lambda settings=None: 0.2) + + async def _single_instance_ring(settings: Any, ring_membership: Any = None) -> tuple[str, tuple[str, ...]]: + return "local-instance", ("local-instance",) + + monkeypatch.setattr(proxy_service, "_active_http_bridge_instance_ring", _single_instance_ring) + + async def _wait_once() -> Any: + return await service._get_or_create_http_bridge_session( + key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="bridge-waiter-pileup"), + api_key=None, + request_model="gpt-5.4", + idle_ttl_seconds=120.0, + max_sessions=8, + ) + + waiters = [asyncio.create_task(_wait_once()) for _ in range(50)] + await asyncio.sleep(0.05) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 1, ( + f"admission waiters must share one fan-out callback on the inflight future, found " + f"{None if callbacks is None else len(callbacks)}" + ) + + # Client-disconnect storm: cancelling waiters must leave the shared future + # pending (the owner's creation continues) and leak no callbacks. + for waiter in waiters[:25]: + waiter.cancel() + cancelled = await asyncio.gather(*waiters[:25], return_exceptions=True) + assert all(isinstance(result, asyncio.CancelledError) for result in cancelled) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 1 + + # The surviving waiters time out: the first to fire fails the shared + # future for the rest with the local-overload contract error. + remaining = await asyncio.gather(*waiters[25:], return_exceptions=True) + assert all(isinstance(result, ProxyResponseError) and result.status_code == 429 for result in remaining) + assert key not in service._http_bridge_inflight_sessions diff --git a/tests/unit/test_settings_reference.py b/tests/unit/test_settings_reference.py index 0475c32243..072bd9cda5 100644 --- a/tests/unit/test_settings_reference.py +++ b/tests/unit/test_settings_reference.py @@ -69,7 +69,7 @@ def _isolated_settings(**overrides: Any) -> Settings: # operator-selectable because startup invariant failures need two supported # modes: report-only by default for mixed/self-hosted environments, and # fail-fast when CI or strict operators want config drift to abort startup. -MAX_SETTINGS_FIELDS = 130 +MAX_SETTINGS_FIELDS = 131 def test_generated_settings_reference_matches_code() -> None: diff --git a/tests/unit/test_shared_future_waiters.py b/tests/unit/test_shared_future_waiters.py new file mode 100644 index 0000000000..96abf57dfa --- /dev/null +++ b/tests/unit/test_shared_future_waiters.py @@ -0,0 +1,136 @@ +"""Regression tests for the shared-future waiter helper. + +The helper replaces ``wait_for(shield(shared))`` on futures awaited by many +concurrent waiters (http-bridge inflight/capacity registries, token-refresh +singleflight). The structural invariant under test: no matter how many +waiters attach, time out, or are cancelled, the shared future carries exactly +one done callback and no leaked per-waiter state. Under the old shield +pattern each waiter attached callbacks to the shared future and removed them +with O(n) scans — a mass timeout livelocked the event loop (2026-08-20 +production incident). +""" + +from __future__ import annotations + +import asyncio + +import pytest + +from app.core.utils.shared_future import _WAITERS_ATTR, wait_on_shared_future + +pytestmark = pytest.mark.unit + + +def _callback_count(future: asyncio.Future) -> int | None: + callbacks = getattr(future, "_callbacks", None) + if callbacks is None: + return None + return len(callbacks) + + +async def test_result_propagates_to_all_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(10)] + await asyncio.sleep(0) + shared.set_result("session") + assert await asyncio.gather(*waiters) == ["session"] * 10 + + +async def test_exception_propagates_to_all_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(4)] + await asyncio.sleep(0) + shared.set_exception(RuntimeError("creation failed")) + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, RuntimeError) and str(r) == "creation failed" for r in results) + + +async def test_shared_cancellation_cancels_waiters(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=5)) for _ in range(4)] + await asyncio.sleep(0) + shared.cancel() + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, asyncio.CancelledError) for r in results) + + +async def test_timeout_raises_and_leaves_shared_pending(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + with pytest.raises(TimeoutError): + await wait_on_shared_future(shared, timeout=0.01) + assert not shared.done() + assert not shared.cancelled() + # The owner can still complete the creation after waiters gave up. + shared.set_result("late") + assert await wait_on_shared_future(shared) == "late" + + +async def test_mass_timeout_does_not_accumulate_callbacks_on_shared(): + """The incident shape: many waiters piling onto one pending future and + timing out together must leave the shared future's callback list at its + constant size (one fan-out callback), not one-or-more per waiter.""" + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + for _ in range(3): # repeated retry rounds, as in the admission loop + waiters = [asyncio.create_task(wait_on_shared_future(shared, timeout=0.01)) for _ in range(200)] + results = await asyncio.gather(*waiters, return_exceptions=True) + assert all(isinstance(r, TimeoutError) for r in results) + count = _callback_count(shared) + if count is not None: + assert count == 1 + assert getattr(shared, _WAITERS_ATTR) == set() + assert not shared.done() + shared.cancel() + + +async def test_cancelling_one_waiter_leaves_others_and_shared_intact(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + victim = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + survivor = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + await asyncio.sleep(0) + victim.cancel() + with pytest.raises(asyncio.CancelledError): + await victim + assert not shared.done() + shared.set_result("session") + assert await survivor == "session" + + +async def test_done_shared_returns_immediately(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + shared.set_result("cached") + assert await wait_on_shared_future(shared, timeout=0.01) == "cached" + + failed: asyncio.Future[str] = asyncio.get_running_loop().create_future() + failed.set_exception(RuntimeError("boom")) + with pytest.raises(RuntimeError, match="boom"): + await wait_on_shared_future(failed) + + +async def test_late_waiter_after_fan_out_gets_result(): + shared: asyncio.Future[str] = asyncio.get_running_loop().create_future() + first = asyncio.create_task(wait_on_shared_future(shared, timeout=5)) + await asyncio.sleep(0) + shared.set_result("session") + assert await first == "session" + # Fan-out already ran and cleared the waiter set; a late waiter must not + # hang on the emptied set. + assert await wait_on_shared_future(shared, timeout=0.01) == "session" + + +async def test_shared_task_keeps_running_when_all_waiters_cancel(): + """Singleflight semantics: waiter cancellation must not abort the work.""" + finished = asyncio.Event() + + async def _work() -> str: + await asyncio.sleep(0.05) + finished.set() + return "refreshed" + + task = asyncio.create_task(_work()) + waiter = asyncio.create_task(wait_on_shared_future(task)) + await asyncio.sleep(0) + waiter.cancel() + with pytest.raises(asyncio.CancelledError): + await waiter + assert await task == "refreshed" + assert finished.is_set() From 1541ee83105edd9a06062f6bcc617f9dab595a66 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 18:42:02 +0900 Subject: [PATCH 098/117] feat(telemetry): report consent state and send a decision-time opt-out signal (#1835) Telemetry went silent when an operator disabled it, so the collector could not tell an explicit rejection apart from an instance that simply stopped running. - Snapshots now carry the effective consent state (`undecided` | `enabled`). - One signed opt-out notification is sent per dashboard-driven active-to-inactive transition, then the instance goes silent as before. - The `CODEX_LB_TELEMETRY_ENABLED` kill-switch path stays absolutely silent: effective consent never transitions there, so the notification is unreachable. - Opt-out transmission reuses the snapshot sender's failure isolation (5s bounded timeout, one retry, DEBUG-only logs) and runs as a background task, so the settings API response is never blocked or failed by it. - The sender re-checks consent and identity immediately before the snapshot POST. Known gaps are tracked in #1844: the residual TOCTOU between that re-check and the network write, the settings preview under the env kill switch, `occurred_at` typing, and opt-out observability. Spec: openspec/changes/add-telemetry-optout-signal/ --- app/modules/telemetry/api.py | 48 ++++- app/modules/telemetry/scheduler.py | 6 +- app/modules/telemetry/schemas.py | 15 +- app/modules/telemetry/sender.py | 112 ++++++++-- app/modules/telemetry/snapshot.py | 14 +- docs/telemetry.md | 35 +++- .../settings/components/settings-skeleton.tsx | 1 + .../telemetry-consent-dialog.test.tsx | 5 + .../components/telemetry-consent-dialog.tsx | 1 + .../components/telemetry-settings.test.tsx | 3 + .../components/telemetry-settings.tsx | 2 + frontend/src/features/settings/schemas.ts | 1 + frontend/src/i18n/locales/en.json | 1 + frontend/src/i18n/locales/ko.json | 1 + frontend/src/i18n/locales/zh-CN.json | 1 + frontend/src/test/mocks/factories.ts | 1 + .../.openspec.yaml | 2 + .../add-telemetry-optout-signal/design.md | 85 ++++++++ .../add-telemetry-optout-signal/proposal.md | 34 +++ .../specs/telemetry/spec.md | 66 ++++++ .../add-telemetry-optout-signal/tasks.md | 22 ++ tests/unit/test_telemetry_api.py | 171 ++++++++++++++- tests/unit/test_telemetry_consent.py | 17 ++ tests/unit/test_telemetry_sender.py | 198 +++++++++++++++++- tests/unit/test_telemetry_snapshot.py | 48 ++++- 25 files changed, 847 insertions(+), 43 deletions(-) create mode 100644 openspec/changes/add-telemetry-optout-signal/.openspec.yaml create mode 100644 openspec/changes/add-telemetry-optout-signal/design.md create mode 100644 openspec/changes/add-telemetry-optout-signal/proposal.md create mode 100644 openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md create mode 100644 openspec/changes/add-telemetry-optout-signal/tasks.md diff --git a/app/modules/telemetry/api.py b/app/modules/telemetry/api.py index bb5ae7a7b5..7f625d05c4 100644 --- a/app/modules/telemetry/api.py +++ b/app/modules/telemetry/api.py @@ -1,8 +1,13 @@ from __future__ import annotations +import asyncio +import logging +import platform + from fastapi import APIRouter, Body, Depends, Query from sqlalchemy.ext.asyncio import AsyncSession +from app import __version__ from app.core.auth.dependencies import ( require_dashboard_write_access, set_dashboard_error_format, @@ -16,7 +21,12 @@ TelemetrySnapshotEnvelope, build_snapshot_envelope, ) -from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder +from app.modules.telemetry.sender import TelemetrySender +from app.modules.telemetry.snapshot import TelemetrySnapshotBuilder, deployment_method + +logger = logging.getLogger(__name__) + +_OPT_OUT_TASKS: set[asyncio.Task[None]] = set() router = APIRouter( prefix="/api/settings", @@ -47,7 +57,24 @@ async def update_telemetry_consent( session: AsyncSession = Depends(get_session), ) -> TelemetryConsentResponse: store = TelemetryConsentStore(session) + previous = await store.resolve() consent = await store.set_decision(payload.enabled) + if previous.active and not consent.active: + try: + identity = await store.get_or_create_identity() + task = asyncio.create_task( + TelemetrySender().send_opt_out( + identity, + app_version=__version__, + deployment_mode=deployment_method(), + os_arch=f"{platform.system().lower()}/{platform.machine().lower()}", + ), + name="anonymous-telemetry-opt-out", + ) + _OPT_OUT_TASKS.add(task) + task.add_done_callback(_handle_opt_out_task_done) + except Exception as exc: + logger.debug("Unable to schedule anonymous telemetry opt-out", exc_info=exc) return await _response(session, store, consent, include_preview=False) @@ -61,7 +88,11 @@ async def _response( preview: TelemetrySnapshotEnvelope | None = None if include_preview: identity = await store.get_or_create_identity() - snapshot = await TelemetrySnapshotBuilder(session).build(identity.instance_id) + snapshot_consent = "enabled" if consent.state == "disabled" else consent.state + snapshot = await TelemetrySnapshotBuilder(session).build( + identity.instance_id, + consent=snapshot_consent, + ) preview = build_snapshot_envelope(snapshot) return TelemetryConsentResponse( state=consent.state, @@ -69,3 +100,16 @@ async def _response( active=consent.active, preview=preview, ) + + +def _handle_opt_out_task_done(task: asyncio.Task[None]) -> None: + try: + if task.cancelled(): + return + if exc := task.exception(): + logger.debug( + "Anonymous telemetry opt-out background task failed", + exc_info=(type(exc), exc, exc.__traceback__), + ) + finally: + _OPT_OUT_TASKS.discard(task) diff --git a/app/modules/telemetry/scheduler.py b/app/modules/telemetry/scheduler.py index fd731cb61f..ea1ee2fc9d 100644 --- a/app/modules/telemetry/scheduler.py +++ b/app/modules/telemetry/scheduler.py @@ -83,8 +83,12 @@ async def _tick_as_leader(self, *, log_undecided_notice: bool = False) -> None: ) if not consent.active: return + assert consent.state != "disabled" identity = await store.get_or_create_identity() - snapshot = await TelemetrySnapshotBuilder(session).build(identity.instance_id) + snapshot = await TelemetrySnapshotBuilder(session).build( + identity.instance_id, + consent=consent.state, + ) await self.sender.send_snapshot(snapshot) except Exception as exc: logger.debug("Anonymous telemetry scheduler tick failed", exc_info=exc) diff --git a/app/modules/telemetry/schemas.py b/app/modules/telemetry/schemas.py index e007dd0c3d..d8d4947f70 100644 --- a/app/modules/telemetry/schemas.py +++ b/app/modules/telemetry/schemas.py @@ -5,13 +5,16 @@ from pydantic import BaseModel, ConfigDict, Field +DeploymentMethod = Literal["docker", "k8s", "pip", "bare"] +ActiveConsentState = Literal["undecided", "enabled"] + class TelemetryModel(BaseModel): model_config = ConfigDict(extra="forbid") class DeploymentSnapshot(TelemetryModel): - method: Literal["docker", "k8s", "pip", "bare"] + method: DeploymentMethod db_backend: Literal["sqlite", "postgres"] db_size_bucket: Literal["unknown", "<100MB", "100MB-1GB", "1-5GB", "5-10GB", "10-50GB", "50GB+"] replicas: int = Field(ge=1) @@ -97,6 +100,7 @@ class FeaturesSnapshot(TelemetryModel): class TelemetrySnapshot(TelemetryModel): schema_version: Literal[1] = 1 + consent: ActiveConsentState instance_id: str version: str python: str @@ -112,7 +116,7 @@ class TelemetrySnapshot(TelemetryModel): class TelemetryRegistration(TelemetryModel): app_name: Literal["codex-lb"] = "codex-lb" app_version: str - deployment_mode: Literal["docker", "k8s", "pip", "bare"] + deployment_mode: DeploymentMethod environment: str = "" instance_id: str os_arch: str @@ -123,6 +127,13 @@ class TelemetryActivation(TelemetryModel): action: Literal["activate"] = "activate" +class TelemetryOptOut(TelemetryModel): + app_version: str + event: Literal["optout"] = "optout" + instance_id: str + occurred_at: str + + class TelemetrySnapshotEnvelope(TelemetryModel): instance_id: str metrics: TelemetrySnapshot diff --git a/app/modules/telemetry/sender.py b/app/modules/telemetry/sender.py index 2f7243e8a4..6b303d17b4 100644 --- a/app/modules/telemetry/sender.py +++ b/app/modules/telemetry/sender.py @@ -8,11 +8,14 @@ import aiohttp from app.core.config.settings import get_settings +from app.core.utils.time import utcnow from app.db.session import get_background_session from app.modules.telemetry.consent import TelemetryConsentStore, TelemetryIdentity from app.modules.telemetry.schemas import ( + DeploymentMethod, TelemetryActivation, TelemetryModel, + TelemetryOptOut, TelemetryRegistration, TelemetrySnapshot, build_snapshot_envelope, @@ -52,20 +55,44 @@ async def send_snapshot(self, snapshot: TelemetrySnapshot) -> None: async with asyncio.timeout(_TIMEOUT_SECONDS): timeout = aiohttp.ClientTimeout(total=_TIMEOUT_SECONDS) async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: - await self._send_with_retry(session, snapshot, identity) + await self._send_with_retry(lambda: self._transmit_once(session, snapshot, identity)) except Exception as exc: logger.debug("Anonymous telemetry transmission failed", exc_info=exc) - async def _send_with_retry( + async def send_opt_out( self, - session: aiohttp.ClientSession, - snapshot: TelemetrySnapshot, identity: TelemetryIdentity, + *, + app_version: str, + deployment_mode: DeploymentMethod, + os_arch: str, ) -> None: + try: + event = TelemetryOptOut( + app_version=app_version, + instance_id=identity.instance_id, + occurred_at=f"{utcnow().isoformat()}Z", + ) + async with asyncio.timeout(_TIMEOUT_SECONDS): + timeout = aiohttp.ClientTimeout(total=_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout, trust_env=False) as session: + await self._send_with_retry( + lambda: self._transmit_opt_out_once( + session, + event, + identity, + deployment_mode=deployment_mode, + os_arch=os_arch, + ) + ) + except Exception as exc: + logger.debug("Anonymous telemetry opt-out transmission failed", exc_info=exc) + + async def _send_with_retry(self, operation: Callable[[], Awaitable[None]]) -> None: last_error: Exception | None = None for attempt in range(_MAX_ATTEMPTS): try: - await self._transmit_once(session, snapshot, identity) + await operation() return except Exception as exc: last_error = exc @@ -79,23 +106,72 @@ async def _transmit_once( snapshot: TelemetrySnapshot, identity: TelemetryIdentity, ) -> None: - if self._activated_instance_id != identity.instance_id: - registration = TelemetryRegistration( - app_version=snapshot.version, - deployment_mode=snapshot.deploy.method, - instance_id=identity.instance_id, - os_arch=f"{snapshot.os}/{snapshot.arch}", - public_key=identity.public_key_hex, - ) - await self._post(session, "/v1/register", _json_bytes(registration), accepted={200, 201}) - - activation = TelemetryActivation() - await self._post_signed(session, "/v1/activate", _json_bytes(activation), identity, accepted={200}) - self._activated_instance_id = identity.instance_id + await self._ensure_activated( + session, + identity, + app_version=snapshot.version, + deployment_mode=snapshot.deploy.method, + os_arch=f"{snapshot.os}/{snapshot.arch}", + ) envelope = build_snapshot_envelope(snapshot) + try: + active, current_identity = await self._context_provider() + identity_matches = ( + current_identity is not None + and current_identity.instance_id == identity.instance_id + and current_identity.public_key_hex == identity.public_key_hex + ) + except Exception as exc: + logger.debug("Anonymous telemetry consent re-check failed", exc_info=exc) + return + if not active or not identity_matches: + return + await self._post_signed(session, "/v1/snapshot", _json_bytes(envelope), identity, accepted={200, 202}) + async def _transmit_opt_out_once( + self, + session: aiohttp.ClientSession, + event: TelemetryOptOut, + identity: TelemetryIdentity, + *, + deployment_mode: DeploymentMethod, + os_arch: str, + ) -> None: + await self._ensure_activated( + session, + identity, + app_version=event.app_version, + deployment_mode=deployment_mode, + os_arch=os_arch, + ) + await self._post_signed(session, "/v1/optout", _json_bytes(event), identity, accepted={200}) + + async def _ensure_activated( + self, + session: aiohttp.ClientSession, + identity: TelemetryIdentity, + *, + app_version: str, + deployment_mode: DeploymentMethod, + os_arch: str, + ) -> None: + if self._activated_instance_id == identity.instance_id: + return + registration = TelemetryRegistration( + app_version=app_version, + deployment_mode=deployment_mode, + instance_id=identity.instance_id, + os_arch=os_arch, + public_key=identity.public_key_hex, + ) + await self._post(session, "/v1/register", _json_bytes(registration), accepted={200, 201}) + + activation = TelemetryActivation() + await self._post_signed(session, "/v1/activate", _json_bytes(activation), identity, accepted={200}) + self._activated_instance_id = identity.instance_id + async def _post_signed( self, session: aiohttp.ClientSession, diff --git a/app/modules/telemetry/snapshot.py b/app/modules/telemetry/snapshot.py index 6237bd6182..88d9e5cc4f 100644 --- a/app/modules/telemetry/snapshot.py +++ b/app/modules/telemetry/snapshot.py @@ -39,6 +39,8 @@ from app.modules.telemetry.clients import ClientCount, catalog_model_name, client_shares from app.modules.telemetry.schemas import ( AccountsSnapshot, + ActiveConsentState, + DeploymentMethod, DeploymentSnapshot, FeaturesSnapshot, ModelUsageSnapshot, @@ -155,7 +157,12 @@ def __init__(self, session: AsyncSession, *, settings: Settings | None = None) - self._session = session self._settings = settings or get_settings() - async def build(self, instance_id: str) -> TelemetrySnapshot: + async def build( + self, + instance_id: str, + *, + consent: ActiveConsentState, + ) -> TelemetrySnapshot: now = utcnow() start = now - timedelta(days=7) reports = ReportsRepository(self._session) @@ -180,7 +187,7 @@ async def build(self, instance_id: str) -> TelemetrySnapshot: top_errors = await self._top_upstream_errors(conditions) feature_counts = await self._feature_counts(conditions) - method = _deployment_method() + method = deployment_method() db_backend = "postgres" if self._session.get_bind().dialect.name == "postgresql" else "sqlite" plan_mix = PlanMixSnapshot( plus=count_bucket(plan_counts.get("plus", 0)), @@ -189,6 +196,7 @@ async def build(self, instance_id: str) -> TelemetrySnapshot: free=count_bucket(plan_counts.get("free", 0)), ) return TelemetrySnapshot( + consent=consent, instance_id=instance_id, version=__version__, python=f"{platform.python_version_tuple()[0]}.{platform.python_version_tuple()[1]}", @@ -462,7 +470,7 @@ def _canonical_routing_policy(raw_policy: str | None) -> str: return normalized if normalized in _ROUTING_POLICIES else "other" -def _deployment_method() -> Literal["docker", "k8s", "pip", "bare"]: +def deployment_method() -> DeploymentMethod: if os.environ.get("KUBERNETES_SERVICE_HOST") or Path("/var/run/secrets/kubernetes.io/serviceaccount").exists(): return "k8s" if Path("/.dockerenv").exists() or Path("/run/.containerenv").exists(): diff --git a/docs/telemetry.md b/docs/telemetry.md index b08ade6378..e540d2793c 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -19,8 +19,8 @@ view it later from Settings. The signed snapshot body has three fields: The versioned `metrics` schema contains only these fields: -- `schema_version`, random `instance_id`, codex-lb `version`, Python version, OS, architecture, - and process uptime +- `schema_version`, active `consent` (`undecided` or `enabled`), random `instance_id`, codex-lb + `version`, Python version, OS, architecture, and process uptime - `deploy`: deployment method, database backend and size bucket, replica count, and whether trusted reverse-proxy headers are enabled - `accounts`: bucketed pool and plan counts, whether workspace accounts exist, routing policy, @@ -52,7 +52,31 @@ CODEX_LB_TELEMETRY_ENABLED=false ``` An environment value overrides the saved dashboard setting. When telemetry resolves to -disabled, codex-lb opens no connection to the telemetry endpoint. +disabled, codex-lb opens no connection to the telemetry endpoint. The environment kill switch +is always completely silent. + +When a dashboard decision changes telemetry from active to inactive, codex-lb makes one final +signed request to `POST /v1/optout` so aggregate opt-out counts remain accurate. If the instance +has not contacted the collector in this process, it first performs the normal registration and +activation. Re-enabling and later disabling from the dashboard sends one new notice for that new +transition. Repeating an already-disabled decision sends nothing. + +The opt-out request uses the same `X-Instance-ID` and Ed25519 `X-Signature` headers as a snapshot. +Its canonical JSON body is: + +```json +{ + "app_version": "", + "event": "optout", + "instance_id": "", + "occurred_at": "" +} +``` + +This single decision-time notice is the only exception to disabled telemetry silence. It is +sent only for a dashboard-driven active-to-inactive transition; setting +`CODEX_LB_TELEMETRY_ENABLED=false`, or changing a saved decision while either environment +override value controls telemetry, never sends it. ## Retention and failures @@ -61,7 +85,8 @@ codex-lb does not keep a separate local telemetry history and does not queue a f collector's server-side retention duration is not currently specified; assume transmitted snapshots remain stored until a published retention policy or explicit deletion. -Endpoint failures use a bounded timeout, are logged only at debug level, and never interrupt -proxy traffic. +Snapshot and opt-out endpoint failures use a five-second total timeout, retry no more than once, +are logged only at debug level, and never interrupt proxy traffic or change the dashboard +settings response. *Source of truth: [telemetry OpenSpec capability](https://github.com/Soju06/codex-lb/tree/main/openspec/specs/telemetry)* diff --git a/frontend/src/features/settings/components/settings-skeleton.tsx b/frontend/src/features/settings/components/settings-skeleton.tsx index c23ce2085f..db4e09e804 100644 --- a/frontend/src/features/settings/components/settings-skeleton.tsx +++ b/frontend/src/features/settings/components/settings-skeleton.tsx @@ -135,6 +135,7 @@ export function SettingsSkeleton() {
    +
    diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx index a730d1e1ec..63a2ff130e 100644 --- a/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.test.tsx @@ -5,6 +5,7 @@ import { beforeEach, describe, expect, it } from "vitest"; import { useAuthStore } from "@/features/auth/hooks/use-auth"; import { TelemetryConsentDialog } from "@/features/settings/components/telemetry-consent-dialog"; +import i18n from "@/i18n"; import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; import { server } from "@/test/mocks/server"; import { renderWithProviders } from "@/test/utils"; @@ -32,6 +33,10 @@ describe("TelemetryConsentDialog", () => { expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); expect(within(dialog).getByText(/"metrics": \{/)).toBeInTheDocument(); expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"consent": "undecided"/)).toBeInTheDocument(); + expect( + within(dialog).getByText(i18n.t("settings.telemetry.optOutNotice")), + ).toBeInTheDocument(); expect(within(dialog).getByRole("button", { name: "Keep enabled" })).toBeInTheDocument(); expect(within(dialog).getByRole("button", { name: "Disable telemetry" })).toBeInTheDocument(); expect( diff --git a/frontend/src/features/settings/components/telemetry-consent-dialog.tsx b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx index 4b48fbf806..4a20410aa2 100644 --- a/frontend/src/features/settings/components/telemetry-consent-dialog.tsx +++ b/frontend/src/features/settings/components/telemetry-consent-dialog.tsx @@ -60,6 +60,7 @@ export function TelemetryConsentDialog() {

    {t("settings.telemetry.consentDialog.categories")}

    +

    {t("settings.telemetry.optOutNotice")}

    {t("settings.telemetry.consentDialog.payloadLabel")}

    diff --git a/frontend/src/features/settings/components/telemetry-settings.test.tsx b/frontend/src/features/settings/components/telemetry-settings.test.tsx index 84ba1aaaa2..643e93ee3a 100644 --- a/frontend/src/features/settings/components/telemetry-settings.test.tsx +++ b/frontend/src/features/settings/components/telemetry-settings.test.tsx @@ -4,6 +4,7 @@ import { HttpResponse, http } from "msw"; import { describe, expect, it } from "vitest"; import { TelemetrySettings } from "@/features/settings/components/telemetry-settings"; +import i18n from "@/i18n"; import { createTelemetryConsent, createTelemetrySnapshotEnvelope } from "@/test/mocks/factories"; import { server } from "@/test/mocks/server"; import { renderWithProviders } from "@/test/utils"; @@ -27,6 +28,7 @@ describe("TelemetrySettings", () => { const toggle = await screen.findByRole("switch", { name: "Enable anonymous telemetry" }); await waitFor(() => expect(toggle).toBeChecked()); expect(toggle).toBeEnabled(); + expect(screen.getByText(i18n.t("settings.telemetry.optOutNotice"))).toBeInTheDocument(); await user.click(toggle); @@ -86,6 +88,7 @@ describe("TelemetrySettings", () => { const dialog = await screen.findByRole("dialog", { name: "Collected telemetry data" }); expect(within(dialog).getByText(/"schema_version": 1/)).toBeInTheDocument(); + expect(within(dialog).getByText(/"consent": "undecided"/)).toBeInTheDocument(); expect(within(dialog).getByText(/"timestamp": "2026-08-06T00:00:00Z"/)).toBeInTheDocument(); expect( telemetryRequests.filter((url) => url.searchParams.get("include_preview") === "true"), diff --git a/frontend/src/features/settings/components/telemetry-settings.tsx b/frontend/src/features/settings/components/telemetry-settings.tsx index 964dfefa29..120b281cff 100644 --- a/frontend/src/features/settings/components/telemetry-settings.tsx +++ b/frontend/src/features/settings/components/telemetry-settings.tsx @@ -55,6 +55,8 @@ export function TelemetrySettings({ disabled }: TelemetrySettingsProps) { />

    +

    {t("settings.telemetry.optOutNotice")}

    + {envControlled ? (
    {t("settings.telemetry.envNotice")} diff --git a/frontend/src/features/settings/schemas.ts b/frontend/src/features/settings/schemas.ts index 9615bec07c..bd97dec03f 100644 --- a/frontend/src/features/settings/schemas.ts +++ b/frontend/src/features/settings/schemas.ts @@ -436,6 +436,7 @@ const TelemetryFeaturesSnapshotSchema = z.strictObject({ export const TelemetrySnapshotSchema = z.strictObject({ schema_version: z.literal(1), + consent: z.enum(["undecided", "enabled"]), instance_id: z.string(), version: z.string(), python: z.string(), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 664aada8b1..0cfa946b72 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1294,6 +1294,7 @@ "settings.telemetry.description": "Share anonymous usage statistics to help improve codex-lb.", "settings.telemetry.toggleAria": "Enable anonymous telemetry", "settings.telemetry.envNotice": "Telemetry is controlled by the CODEX_LB_TELEMETRY_ENABLED environment variable. Unset it to manage this setting from the dashboard.", + "settings.telemetry.optOutNotice": "Disabling from the dashboard sends one anonymous opt-out notice to keep aggregate counts accurate.", "settings.telemetry.collectedData.label": "Collected data", "settings.telemetry.collectedData.description": "Review the exact anonymous payload this instance would send.", "settings.telemetry.collectedData.view": "View collected data", diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index 458265ad26..f0f162b24a 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -1294,6 +1294,7 @@ "settings.telemetry.description": "익명 사용 통계를 공유해 codex-lb 개선에 도움을 줍니다.", "settings.telemetry.toggleAria": "익명 텔레메트리 사용", "settings.telemetry.envNotice": "텔레메트리는 CODEX_LB_TELEMETRY_ENABLED 환경 변수로 제어되고 있습니다. 대시보드에서 이 설정을 관리하려면 해당 변수를 해제하세요.", + "settings.telemetry.optOutNotice": "대시보드에서 텔레메트리를 비활성화하면 집계 수치의 정확성을 유지하기 위해 익명 비활성화 알림을 한 번 전송합니다.", "settings.telemetry.collectedData.label": "수집 데이터", "settings.telemetry.collectedData.description": "이 인스턴스가 전송할 익명 payload 원문을 확인할 수 있습니다.", "settings.telemetry.collectedData.view": "수집 데이터 보기", diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 3a8ea63686..14b218e860 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -1294,6 +1294,7 @@ "settings.telemetry.description": "分享匿名使用统计，帮助改进 codex-lb。", "settings.telemetry.toggleAria": "启用匿名遥测", "settings.telemetry.envNotice": "遥测当前由 CODEX_LB_TELEMETRY_ENABLED 环境变量控制。如需在仪表盘中管理此设置，请取消设置该变量。", + "settings.telemetry.optOutNotice": "从仪表盘中禁用遥测时，系统会发送一次匿名的选择退出通知，以确保汇总计数准确。", "settings.telemetry.collectedData.label": "收集的数据", "settings.telemetry.collectedData.description": "查看此实例将发送的匿名数据的完整内容。", "settings.telemetry.collectedData.view": "查看收集的数据", diff --git a/frontend/src/test/mocks/factories.ts b/frontend/src/test/mocks/factories.ts index 2aa8e01de8..04d59e4582 100644 --- a/frontend/src/test/mocks/factories.ts +++ b/frontend/src/test/mocks/factories.ts @@ -551,6 +551,7 @@ export function createTelemetrySnapshotEnvelope(): TelemetrySnapshotEnvelope { timestamp: "2026-08-06T00:00:00Z", metrics: { schema_version: 1, + consent: "undecided", instance_id: "00000000-0000-4000-8000-000000000000", version: "1.23.0", python: "3.13", diff --git a/openspec/changes/add-telemetry-optout-signal/.openspec.yaml b/openspec/changes/add-telemetry-optout-signal/.openspec.yaml new file mode 100644 index 0000000000..f774115be7 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-20 diff --git a/openspec/changes/add-telemetry-optout-signal/design.md b/openspec/changes/add-telemetry-optout-signal/design.md new file mode 100644 index 0000000000..a3befa3143 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/design.md @@ -0,0 +1,85 @@ +## Context + +Snapshot transmission currently resolves consent before building the payload and uses an +instance identity to register, activate, sign, and post through an isolated HTTP client. The +settings PUT persists dashboard decisions through a request-scoped database session. See +`proposal.md` for motivation and `specs/telemetry/spec.md` for the changed contract. + +The opt-out signal is unusual because its triggering decision has already made consent inactive. +It therefore cannot depend on the normal consent-gated sender context, and its asynchronous work +cannot retain request-scoped database or HTTP resources. + +## Goals / Non-Goals + +**Goals:** + +- Preserve one authoritative consent resolution for each snapshot and preview. +- Emit an opt-out only from a dashboard-driven effective active-to-inactive transition. +- Keep the settings response independent from all collector network activity. +- Preserve canonical serialization, signing, bounded retries, and debug-only failure handling. + +**Non-Goals:** + +- No new setting, scheduler cadence, collector implementation, or database migration. +- No historical opt-out backfill or notification when the environment kill switch disables + telemetry. +- No delivery guarantee beyond the existing bounded best-effort telemetry discipline. + +## Decisions + +### Pass resolved consent into snapshot construction + +The scheduler and preview API will pass their already-resolved active consent state into the +snapshot builder. This keeps the envelope deterministic and prevents a second resolution from +observing a different state. Resolving consent again inside the builder was rejected because it +would duplicate policy and could disagree with the caller's send decision. + +### Detect the effective transition around persistence + +The PUT handler will resolve consent before persisting the decision and again afterward. It will +schedule an opt-out only when the first resolution is active and the second is inactive. This +naturally excludes disabled-to-disabled writes and both environment override values, while +allowing a later re-enable/re-disable cycle to produce a new event. Comparing only persisted +values was rejected because it would incorrectly notify while an environment override controls +effective behavior. + +### Give the background task explicit immutable inputs + +Before scheduling, the handler will obtain the instance identity and gather the version and +platform fields needed for registration and activation. The sender will then open and close its +own HTTP client, lazily register and activate, and post the signed canonical opt-out body. A +module-owned task set will retain a strong reference until completion. Reusing the request's +database session or the consent-gated sender context was rejected because those resources and +policy no longer match the task's lifetime. + +### Reuse the sender's bounded delivery discipline + +Opt-out delivery will share the snapshot path's five-second total timeout, at-most-one retry, +canonical JSON, signing, accepted-status handling, and debug-only exception isolation. The +event body is: + +```json +{"app_version":"1.2.3","event":"optout","instance_id":"550e8400-e29b-41d4-a716-446655440000","occurred_at":"2026-08-20T12:00:00+00:00"} +``` + +The route order for an uninitialized process is registration, activation, then opt-out. Waiting +for network completion in the API handler was rejected because collector latency must not affect +the operator's settings response. + +## Risks / Trade-offs + +- **Process exit can cancel the best-effort task** → Server-side idempotency and per-transition + scheduling make retries safe, while avoiding shutdown delay or API coupling. +- **Concurrent duplicate dashboard requests can each observe a transition** → Persisted state + serialization and transition tests constrain ordinary request behavior; the collector remains + idempotent for rare delivery duplication. +- **Registration or activation outage prevents the event** → The sender swallows the bounded + failure at debug level, matching snapshot availability and privacy behavior. +- **A future snapshot caller could pass disabled consent** → The type narrows the payload to the + two wire-valid states, and callers only build while resolved consent is active. + +## Migration Plan + +Deploy the additive client contract together with collector support for `/v1/optout`. Existing +instances require no data migration. Rollback removes the new snapshot field and notification; +the collector's idempotent endpoint can remain deployed without affecting older clients. diff --git a/openspec/changes/add-telemetry-optout-signal/proposal.md b/openspec/changes/add-telemetry-optout-signal/proposal.md new file mode 100644 index 0000000000..012e9e63a2 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/proposal.md @@ -0,0 +1,34 @@ +## Why + +Telemetry currently becomes silent when an operator disables it, so the collector cannot +distinguish an explicit rejection from an instance that stopped running. The wire contract also +needs the effective persisted consent state on snapshots so aggregate interpretation stays +accurate without weakening the environment kill switch. + +## What Changes + +- Add the active consent state (`undecided` or `enabled`) to every snapshot payload. +- Send one final signed opt-out notification for each dashboard-driven transition from active + telemetry to inactive telemetry. +- Preserve absolute silence for the `CODEX_LB_TELEMETRY_ENABLED=false` environment path and + isolate opt-out transmission failures from the settings API response. +- Explain the additive wire behavior in the telemetry settings UI and published telemetry docs. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `telemetry`: Extend the outbound allowlist and consent behavior with snapshot consent and the + decision-time opt-out notification. + +## Impact + +- Telemetry schemas, snapshot construction, scheduler and preview callers, sender, and settings + API transition handling. +- Telemetry unit tests and wire-contract allowlists. +- Dashboard telemetry consent copy and component tests. +- Published telemetry payload documentation. diff --git a/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md b/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md new file mode 100644 index 0000000000..eac1ade3e8 --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/specs/telemetry/spec.md @@ -0,0 +1,66 @@ +## ADDED Requirements + +### Requirement: Snapshot payload declares active consent + +Every snapshot payload MUST include a top-level `consent` field whose value is the resolved +persisted consent state `undecided` or `enabled`; `disabled` MUST NOT appear because snapshots +are not transmitted while telemetry is inactive. + +#### Scenario: Undecided snapshot declares consent + +- **WHEN** telemetry is active under the default undecided consent state +- **THEN** the transmitted snapshot and exact payload preview contain `consent: "undecided"` + +#### Scenario: Enabled snapshot declares consent + +- **WHEN** telemetry is active under persisted enabled consent +- **THEN** the transmitted snapshot and exact payload preview contain `consent: "enabled"` + +### Requirement: Dashboard opt-out notification + +The service MUST send one final signed `POST /v1/optout` notification for each +dashboard-driven effective consent transition from active to inactive, and MUST complete any +required instance registration and activation before sending that notification. The notification +MUST use the telemetry instance identity and snapshot signing scheme, MUST be isolated from the +settings API response, and MUST NOT be sent for an environment-controlled consent path. + +#### Scenario: Opt-out fires exactly once per transition + +- **WHEN** dashboard consent transitions from undecided or enabled active telemetry to disabled + inactive telemetry without an environment override +- **THEN** exactly one opt-out notification is attempted for that transition before telemetry + becomes silent + +#### Scenario: Environment kill switch stays silent + +- **WHEN** `CODEX_LB_TELEMETRY_ENABLED=false` makes telemetry inactive or a dashboard decision + is persisted while consent is controlled by either environment override value +- **THEN** no opt-out notification or other telemetry network request is attempted + +#### Scenario: Opt-out failure is isolated + +- **WHEN** registration, activation, or opt-out transmission fails +- **THEN** the failure uses a total timeout of no more than five seconds, retries no more than + once, is logged only at debug level, does not raise to the caller, and does not delay or alter + the successful settings API response + +#### Scenario: A later transition may notify again + +- **WHEN** an operator re-enables telemetry and later disables it again through the dashboard + without an environment override +- **THEN** the later active-to-inactive transition attempts exactly one new opt-out notification + +## MODIFIED Requirements + +### Requirement: Disabled means zero telemetry traffic + +Except for the single decision-time opt-out notification on a dashboard-driven active-to-inactive +transition, when resolved consent is `disabled` the service MUST NOT open any network connection +to the telemetry endpoint. The environment kill-switch path MUST NOT receive this exception and +MUST remain completely silent. + +#### Scenario: No connection attempts when disabled + +- **WHEN** telemetry is disabled and the service runs through startup and a 24-hour scheduler + cycle outside the dashboard decision-time transition +- **THEN** no connection attempt to the telemetry endpoint is made diff --git a/openspec/changes/add-telemetry-optout-signal/tasks.md b/openspec/changes/add-telemetry-optout-signal/tasks.md new file mode 100644 index 0000000000..f45d480b3c --- /dev/null +++ b/openspec/changes/add-telemetry-optout-signal/tasks.md @@ -0,0 +1,22 @@ +## 1. Snapshot Consent Contract + +- [x] 1.1 Add the active consent literal to snapshot schemas and introduce the typed opt-out event schema +- [x] 1.2 Require callers to pass resolved consent into snapshot construction and expose it in sender and preview envelopes +- [x] 1.3 Update schema allowlist and builder, scheduler, and preview regression tests for the consent field + +## 2. Opt-Out Delivery + +- [x] 2.1 Implement signed canonical opt-out delivery with lazy registration, activation, bounded retry, and debug-only failure isolation +- [x] 2.2 Detect dashboard effective active-to-inactive transitions and schedule one resource-owning background send +- [x] 2.3 Add sender and settings API tests for successful delivery, retry/failure isolation, repeated transitions, no-op decisions, and both environment overrides +- [x] 2.4 Preserve transport-level zero-call coverage for disabled scheduler and sender paths + +## 3. Operator Communication + +- [x] 3.1 Add neutral opt-out notice copy to the telemetry consent dialog and settings components with co-located test coverage +- [x] 3.2 Document the snapshot consent field, opt-out wire payload, transition behavior, and environment-path silence + +## 4. Verification + +- [x] 4.1 Validate the OpenSpec change and run focused backend and frontend tests +- [x] 4.2 Run the full unit suite, lint, and type-check gates and confirm the final diff stays within the approved scope diff --git a/tests/unit/test_telemetry_api.py b/tests/unit/test_telemetry_api.py index b948f8ff52..f6e3c9f261 100644 --- a/tests/unit/test_telemetry_api.py +++ b/tests/unit/test_telemetry_api.py @@ -1,6 +1,8 @@ from __future__ import annotations -from unittest.mock import Mock +import asyncio +import logging +from unittest.mock import AsyncMock, Mock import pytest @@ -9,8 +11,21 @@ pytestmark = pytest.mark.unit +@pytest.fixture(autouse=True) +def opt_out_sender(monkeypatch): + sender = Mock() + sender.send_opt_out = AsyncMock() + factory = Mock(return_value=sender) + monkeypatch.setattr("app.modules.telemetry.api.TelemetrySender", factory) + return sender + + @pytest.mark.asyncio -async def test_consent_api_get_preview_and_put_persists_without_restart(async_client, monkeypatch) -> None: +async def test_consent_api_get_preview_and_put_persists_without_restart( + async_client, + monkeypatch, + opt_out_sender, +) -> None: monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) get_settings.cache_clear() @@ -22,6 +37,7 @@ async def test_consent_api_get_preview_and_put_persists_without_restart(async_cl assert initial["active"] is True assert set(initial["preview"]) == {"instance_id", "metrics", "timestamp"} assert initial["preview"]["metrics"]["schema_version"] == 1 + assert initial["preview"]["metrics"]["consent"] == "undecided" assert initial["preview"]["instance_id"] == initial["preview"]["metrics"]["instance_id"] response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) @@ -31,6 +47,8 @@ async def test_consent_api_get_preview_and_put_persists_without_restart(async_cl assert disabled["source"] == "persisted" assert disabled["active"] is False assert disabled["preview"] is None + await asyncio.sleep(0) + opt_out_sender.send_opt_out.assert_awaited_once() builder = Mock(side_effect=AssertionError("decided consent must not build a preview")) monkeypatch.setattr("app.modules.telemetry.api.TelemetrySnapshotBuilder", builder) @@ -42,7 +60,10 @@ async def test_consent_api_get_preview_and_put_persists_without_restart(async_cl @pytest.mark.asyncio -async def test_consent_api_builds_decided_preview_only_when_requested(async_client, monkeypatch) -> None: +async def test_consent_api_builds_decided_preview_only_when_requested( + async_client, + monkeypatch, +) -> None: monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) get_settings.cache_clear() await async_client.put("/api/settings/telemetry", json={"enabled": False}) @@ -53,6 +74,7 @@ async def test_consent_api_builds_decided_preview_only_when_requested(async_clie payload = response.json() assert payload["state"] == "disabled" assert payload["preview"]["instance_id"] == payload["preview"]["metrics"]["instance_id"] + assert payload["preview"]["metrics"]["consent"] == "enabled" @pytest.mark.asyncio @@ -71,3 +93,146 @@ async def test_consent_api_env_override_wins_and_suppresses_undecided_state(asyn assert payload["active"] is True assert payload["preview"] is None builder.assert_not_called() + + +@pytest.mark.asyncio +async def test_dashboard_active_to_inactive_transitions_each_send_exactly_once( + async_client, + monkeypatch, + opt_out_sender, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + first = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert first.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + repeated = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert repeated.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + enabled = await async_client.put("/api/settings/telemetry", json={"enabled": True}) + assert enabled.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 1 + + second = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + assert second.status_code == 200 + await asyncio.sleep(0) + assert opt_out_sender.send_opt_out.await_count == 2 + + call = opt_out_sender.send_opt_out.await_args_list[-1] + assert call.args[0].instance_id + assert call.kwargs["app_version"] + assert call.kwargs["deployment_mode"] in {"docker", "k8s", "pip", "bare"} + assert "/" in call.kwargs["os_arch"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("env_value", "enabled", "expected_active"), + [("true", False, True), ("false", True, False)], +) +async def test_environment_controlled_put_never_sends_opt_out( + async_client, + monkeypatch, + opt_out_sender, + env_value: str, + enabled: bool, + expected_active: bool, +) -> None: + monkeypatch.setenv("CODEX_LB_TELEMETRY_ENABLED", env_value) + get_settings.cache_clear() + + response = await async_client.put("/api/settings/telemetry", json={"enabled": enabled}) + + assert response.status_code == 200 + assert response.json()["source"] == "env" + assert response.json()["active"] is expected_active + await asyncio.sleep(0) + opt_out_sender.send_opt_out.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_opt_out_background_send_does_not_block_settings_response( + async_client, + monkeypatch, + opt_out_sender, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + started = asyncio.Event() + release = asyncio.Event() + + async def blocked_send(*args, **kwargs) -> None: + del args, kwargs + started.set() + await release.wait() + + opt_out_sender.send_opt_out.side_effect = blocked_send + + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + assert response.status_code == 200 + await asyncio.wait_for(started.wait(), timeout=1) + from app.modules.telemetry import api as telemetry_api + + assert telemetry_api._OPT_OUT_TASKS + release.set() + await asyncio.gather(*tuple(telemetry_api._OPT_OUT_TASKS)) + await asyncio.sleep(0) + assert not telemetry_api._OPT_OUT_TASKS + + +@pytest.mark.asyncio +async def test_opt_out_identity_failure_is_debug_only_and_preserves_disabled_state( + async_client, + monkeypatch, + opt_out_sender, + caplog, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + + async def fail_identity(_store) -> None: + raise RuntimeError("identity decryption failed") + + monkeypatch.setattr( + "app.modules.telemetry.api.TelemetryConsentStore.get_or_create_identity", + fail_identity, + ) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.api"): + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + + assert response.status_code == 200 + assert response.json()["state"] == "disabled" + persisted = await async_client.get("/api/settings/telemetry") + assert persisted.status_code == 200 + assert persisted.json()["state"] == "disabled" + opt_out_sender.send_opt_out.assert_not_awaited() + assert "Unable to schedule anonymous telemetry opt-out" in caplog.messages + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_unexpected_opt_out_task_failure_is_debug_only_and_does_not_change_response( + async_client, + monkeypatch, + opt_out_sender, + caplog, +) -> None: + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + opt_out_sender.send_opt_out.side_effect = RuntimeError("unexpected sender failure") + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.api"): + response = await async_client.put("/api/settings/telemetry", json={"enabled": False}) + await asyncio.sleep(0) + + assert response.status_code == 200 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) diff --git a/tests/unit/test_telemetry_consent.py b/tests/unit/test_telemetry_consent.py index db0a0dfe9a..0130db2da5 100644 --- a/tests/unit/test_telemetry_consent.py +++ b/tests/unit/test_telemetry_consent.py @@ -87,6 +87,22 @@ async def test_disabled_scheduler_tick_makes_zero_sender_calls(db_setup, monkeyp sender.send_snapshot.assert_not_awaited() +@pytest.mark.asyncio +async def test_enabled_scheduler_snapshot_declares_enabled_consent(db_setup, monkeypatch) -> None: + del db_setup + monkeypatch.delenv("CODEX_LB_TELEMETRY_ENABLED", raising=False) + get_settings.cache_clear() + async with SessionLocal() as session: + store = TelemetryConsentStore(session) + await store.set_decision(True) + + sender = AsyncMock() + await TelemetryScheduler(sender=sender)._tick() + + sender.send_snapshot.assert_awaited_once() + assert sender.send_snapshot.await_args.args[0].consent == "enabled" + + @pytest.mark.asyncio async def test_non_leader_scheduler_tick_builds_and_transmits_nothing(monkeypatch) -> None: import app.modules.telemetry.scheduler as scheduler_module @@ -144,6 +160,7 @@ async def test_scheduler_sends_startup_and_interval_snapshots_with_one_undecided await scheduler.stop() assert sender.send_snapshot.await_count >= 2 + assert all(call.args[0].consent == "undecided" for call in sender.send_snapshot.await_args_list) notices = [ record.getMessage() for record in caplog.records if "Anonymous telemetry is active" in record.getMessage() ] diff --git a/tests/unit/test_telemetry_sender.py b/tests/unit/test_telemetry_sender.py index 7a4553c5b2..abe4ff3b99 100644 --- a/tests/unit/test_telemetry_sender.py +++ b/tests/unit/test_telemetry_sender.py @@ -2,6 +2,7 @@ import json import logging +from datetime import datetime from unittest.mock import AsyncMock, Mock import pytest @@ -14,9 +15,38 @@ pytestmark = pytest.mark.unit +class _FakeResponse: + status = 200 + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + return None + + async def read(self) -> bytes: + return b"" + + +class _FakeClientSession: + def __init__(self) -> None: + self.requests: list[tuple[str, bytes, dict[str, str]]] = [] + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc, traceback) -> None: + return None + + def post(self, url: str, *, data: bytes, headers: dict[str, str]) -> _FakeResponse: + self.requests.append((url, data, headers)) + return _FakeResponse() + + def _snapshot() -> TelemetrySnapshot: return TelemetrySnapshot.model_validate( { + "consent": "enabled", "instance_id": "00000000-0000-4000-8000-000000000004", "version": "1.0.0", "python": "3.13", @@ -109,10 +139,174 @@ async def context_provider(): @pytest.mark.asyncio -async def test_sender_uses_canonical_shm_paths_and_valid_ed25519_signature() -> None: +async def test_sender_aborts_snapshot_when_consent_becomes_inactive_before_post(monkeypatch) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), (False, None)]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_posts_snapshot_once_when_consent_stays_active(monkeypatch) -> None: snapshot = _snapshot() identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(return_value=(True, identity)) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + "https://telemetry.example/v1/snapshot", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_aborts_snapshot_when_identity_changes_before_post(monkeypatch) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + replacement_identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), (True, replacement_identity)]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + + +@pytest.mark.asyncio +async def test_sender_aborts_snapshot_when_consent_recheck_fails(monkeypatch, caplog) -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + context_provider = AsyncMock(side_effect=[(True, identity), OSError("database unavailable")]) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_snapshot(snapshot) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + ] + assert context_provider.await_count == 2 + assert [record.message for record in caplog.records] == ["Anonymous telemetry consent re-check failed"] + + +@pytest.mark.asyncio +async def test_opt_out_with_inactive_consent_registers_activates_and_posts_exact_signed_canonical_body( + monkeypatch, +) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + client_session = Mock(return_value=session) + context_provider = AsyncMock(return_value=(False, None)) + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", client_session) + monkeypatch.setattr("app.modules.telemetry.sender.utcnow", lambda: datetime(2026, 8, 20, 12, 0, 0)) + + await TelemetrySender( + "https://telemetry.example", + context_provider=context_provider, + ).send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="docker", + os_arch="linux/x86_64", + ) + + assert [request[0] for request in session.requests] == [ + "https://telemetry.example/v1/register", + "https://telemetry.example/v1/activate", + "https://telemetry.example/v1/optout", + ] + expected_body = ( + b'{"app_version":"1.24.0","event":"optout",' + b'"instance_id":"00000000-0000-4000-8000-000000000004",' + b'"occurred_at":"2026-08-20T12:00:00Z"}' + ) + _, body, headers = session.requests[-1] + assert body == expected_body + assert headers["X-Instance-ID"] == identity.instance_id + identity.private_key.public_key().verify(bytes.fromhex(headers["X-Signature"]), body) + context_provider.assert_not_awaited() + client_session.assert_called_once() + assert client_session.call_args.kwargs["timeout"].total == 5.0 + assert client_session.call_args.kwargs["trust_env"] is False + + +@pytest.mark.asyncio +async def test_opt_out_retries_once_then_succeeds(monkeypatch) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) sender = TelemetrySender() + sender._transmit_opt_out_once = AsyncMock(side_effect=[OSError("transient"), None]) + + await sender.send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="bare", + os_arch="linux/x86_64", + ) + + assert sender._transmit_opt_out_once.await_count == 2 + + +@pytest.mark.asyncio +async def test_opt_out_failure_is_swallowed_and_logged_at_debug(monkeypatch, caplog) -> None: + identity = TelemetryIdentity("00000000-0000-4000-8000-000000000004", Ed25519PrivateKey.generate()) + session = _FakeClientSession() + monkeypatch.setattr("app.modules.telemetry.sender.aiohttp.ClientSession", Mock(return_value=session)) + sender = TelemetrySender() + sender._transmit_opt_out_once = AsyncMock(side_effect=OSError("collector unavailable")) + + with caplog.at_level(logging.DEBUG, logger="app.modules.telemetry.sender"): + await sender.send_opt_out( + identity, + app_version="1.24.0", + deployment_mode="bare", + os_arch="linux/x86_64", + ) + + assert sender._transmit_opt_out_once.await_count == 2 + assert caplog.records + assert all(record.levelno == logging.DEBUG for record in caplog.records) + + +@pytest.mark.asyncio +async def test_sender_uses_canonical_shm_paths_and_valid_ed25519_signature() -> None: + snapshot = _snapshot() + identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) + sender = TelemetrySender(context_provider=AsyncMock(return_value=(True, identity))) sender._post = AsyncMock() sender._post_signed = AsyncMock() session = Mock() @@ -164,7 +358,7 @@ async def test_preview_and_sender_snapshot_envelopes_have_identical_key_structur snapshot = _snapshot() identity = TelemetryIdentity(snapshot.instance_id, Ed25519PrivateKey.generate()) preview = build_snapshot_envelope(snapshot) - sender = TelemetrySender() + sender = TelemetrySender(context_provider=AsyncMock(return_value=(True, identity))) sender._post = AsyncMock() sender._post_signed = AsyncMock() diff --git a/tests/unit/test_telemetry_snapshot.py b/tests/unit/test_telemetry_snapshot.py index 56630b989c..7d51559234 100644 --- a/tests/unit/test_telemetry_snapshot.py +++ b/tests/unit/test_telemetry_snapshot.py @@ -19,7 +19,12 @@ client_family, client_shares, ) -from app.modules.telemetry.schemas import TelemetryActivation, TelemetryRegistration, build_snapshot_envelope +from app.modules.telemetry.schemas import ( + TelemetryActivation, + TelemetryOptOut, + TelemetryRegistration, + build_snapshot_envelope, +) from app.modules.telemetry.snapshot import ( _ROUTING_POLICIES, TelemetrySnapshotBuilder, @@ -79,11 +84,16 @@ async def test_snapshot_serialized_field_set_matches_documented_schema(async_ses async_session.add(_request_log("schema", model="gpt-5.4", useragent_group="codex_exec")) await async_session.commit() - snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000001") + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000001", + consent="undecided", + ) payload = snapshot.model_dump() + assert payload["consent"] == "undecided" assert set(payload) == { "schema_version", + "consent", "instance_id", "version", "python", @@ -157,6 +167,11 @@ async def test_snapshot_serialized_field_set_matches_documented_schema(async_ses public_key="00", ).model_dump(mode="json") activation = TelemetryActivation().model_dump(mode="json") + opt_out = TelemetryOptOut( + app_version=snapshot.version, + instance_id=snapshot.instance_id, + occurred_at="2026-08-20T12:00:00Z", + ).model_dump(mode="json") envelope = build_snapshot_envelope(snapshot).model_dump(mode="json") assert set(registration) == { "app_name", @@ -168,6 +183,7 @@ async def test_snapshot_serialized_field_set_matches_documented_schema(async_ses "public_key", } assert set(activation) == {"action"} + assert set(opt_out) == {"app_version", "event", "instance_id", "occurred_at"} assert set(envelope) == {"instance_id", "metrics", "timestamp"} @@ -228,7 +244,13 @@ async def test_model_catalog_filter_merges_custom_models_and_scopes_reasoning( ) await async_session.commit() - payload = (await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000002")).model_dump() + payload = ( + await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000002", + consent="enabled", + ) + ).model_dump() + assert payload["consent"] == "enabled" models = {model["name"]: model for model in payload["usage_7d"]["models"]} assert set(models) == {"gpt-5.4", "other"} @@ -264,7 +286,10 @@ async def test_request_kind_mix_fails_honest_without_persisted_route_family(asyn ) await async_session.commit() - payload = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000005") + payload = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000005", + consent="undecided", + ) assert payload.usage_7d.request_kinds.model_dump() == { "responses": 0.0, @@ -411,7 +436,10 @@ async def test_privacy_quick_check_identifying_values_never_serialize(async_sess await async_session.commit() serialized = ( - await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000003") + await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000003", + consent="undecided", + ) ).model_dump_json() for private_value in ( @@ -466,7 +494,10 @@ async def test_success_rate_excludes_cancelled_terminals(async_session: AsyncSes ) await async_session.commit() - snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000004") + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000004", + consent="undecided", + ) # 1 success out of 4 requests: cancellations are neither successes nor # errors, so they must not inflate the numerator. @@ -496,7 +527,10 @@ async def test_top_upstream_errors_exclude_cancelled_terminals(async_session: As ) await async_session.commit() - snapshot = await TelemetrySnapshotBuilder(async_session).build("00000000-0000-4000-8000-000000000005") + snapshot = await TelemetrySnapshotBuilder(async_session).build( + "00000000-0000-4000-8000-000000000005", + consent="undecided", + ) # High-volume disconnects (status='cancelled' with a retained # client_disconnected code) must not displace genuine upstream failures. From 1ecb51d2e31c0ea808c631ad559bfcd4be586a54 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Thu, 20 Aug 2026 19:22:32 +0900 Subject: [PATCH 099/117] chore: release v1.24.0-beta.3 (#1834) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 ++-- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index db51e06b8b..4a70336796 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.24.0-beta.2" # x-release-please-version +__version__ = "1.24.0-beta.3" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index ab62108e2a..e26334aaa8 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.24.0-beta.2 -appVersion: 1.24.0-beta.2 +version: 1.24.0-beta.3 +appVersion: 1.24.0-beta.3 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index c716ae84ac..0bb4ff6a8b 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.24.0-beta.2", + "version": "1.24.0-beta.3", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index e7f19aab46..1625459040 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.24.0-beta.2" +version = "1.24.0-beta.3" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index dfd12b2280..2086fe95bd 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0-beta.2" +version = "1.24.0-beta.3" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From eab71553aee660fdb31122e9feb61a0e6c367904 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 15:13:45 +0400 Subject: [PATCH 100/117] feat(frontend): configure model-source reasoning efforts (#1848) * feat(frontend): add model-source reasoning effort controls * ci: rerun cancelled frontend checks * Preserve case when normalizing model-source reasoning efforts * style(frontend): remove trailing test whitespace --- .../model-source-edit-dialog.test.tsx | 60 ++++++++- .../components/model-source-edit-dialog.tsx | 38 ++---- .../components/model-source-form-fields.tsx | 85 ++++++++++++- .../components/model-source-form.test.ts | 25 ++++ .../components/model-source-form.ts | 119 ++++++++++++++++-- frontend/src/i18n/locales/en.json | 6 +- frontend/src/i18n/locales/ko.json | 6 +- frontend/src/i18n/locales/zh-CN.json | 6 +- .../proposal.md | 41 ++++++ .../specs/frontend-architecture/spec.md | 38 ++++++ .../tasks.md | 15 +++ 11 files changed, 395 insertions(+), 44 deletions(-) create mode 100644 frontend/src/features/model-sources/components/model-source-form.test.ts create mode 100644 openspec/changes/configure-model-source-reasoning-efforts/proposal.md create mode 100644 openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md create mode 100644 openspec/changes/configure-model-source-reasoning-efforts/tasks.md diff --git a/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx b/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx index 091f6a53f1..414650f6b2 100644 --- a/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx +++ b/frontend/src/features/model-sources/components/model-source-edit-dialog.test.tsx @@ -290,12 +290,25 @@ describe("ModelSourceEditDialog", () => { }); const rawMetadata = JSON.parse(onSubmit.mock.calls[0][1].models[0].rawMetadataJson); - expect(rawMetadata).toEqual({ custom_key: "kept", supports_reasoning: true }); + expect(rawMetadata).toEqual({ + custom_key: "kept", + supports_reasoning: true, + supported_reasoning_levels: ["low", "medium", "high"], + default_reasoning_level: "medium", + }); }); it("prefills the reasoning toggle from raw metadata", () => { const source = createModelSource(); - source.models[0].rawMetadataJson = '{"supports_reasoning": true}'; + source.models[0].rawMetadataJson = JSON.stringify({ + supports_reasoning: true, + supported_reasoning_levels: [ + { effort: "none", description: "disable chain of thought" }, + "provider-specific", + "ultra", + ], + default_reasoning_level: "provider-specific", + }); renderWithProviders( { ); expect(screen.getByRole("checkbox", { name: "Reasoning" })).toBeChecked(); + expect(screen.getByLabelText("Supported reasoning efforts")).toHaveValue( + "none, provider-specific, ultra", + ); + expect(screen.getByRole("combobox", { name: "Default reasoning effort" })).toHaveTextContent( + "provider-specific", + ); + }); + + it("keeps arbitrary reasoning efforts and renormalizes a stale default", async () => { + const user = userEvent.setup(); + const onSubmit = vi.fn().mockResolvedValue(undefined); + const source = createModelSource(); + source.models[0].rawMetadataJson = JSON.stringify({ + supports_reasoning: true, + supported_reasoning_levels: ["none", "provider-specific", "ultra"], + default_reasoning_level: "provider-specific", + }); + + renderWithProviders( + , + ); + + const reasoningEfforts = screen.getByLabelText("Supported reasoning efforts"); + await user.clear(reasoningEfforts); + await user.type(reasoningEfforts, "none, custom-tier"); + await user.click(screen.getByRole("button", { name: "Save" })); + + await waitFor(() => { + expect(onSubmit).toHaveBeenCalledTimes(1); + }); + + const rawMetadata = JSON.parse(onSubmit.mock.calls[0][1].models[0].rawMetadataJson); + expect(rawMetadata).toEqual({ + supports_reasoning: true, + supported_reasoning_levels: ["none", "custom-tier"], + default_reasoning_level: "none", + }); }); it("sends the api key only when the field is filled", async () => { diff --git a/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx b/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx index 854d0e8cae..e34a692f1c 100644 --- a/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx +++ b/frontend/src/features/model-sources/components/model-source-edit-dialog.tsx @@ -17,6 +17,7 @@ import { ModelSourceFormFields } from "@/features/model-sources/components/model import { createModelSourceFormSchema, draftFromSource, + mergeReasoningMetadata, modelIdsToInput, modelSourceDraftReducer, type ModelSourceDraft, @@ -42,8 +43,6 @@ type ModelDraftChangeFlags = { supportsReasoning: boolean; }; -const SUPPORTS_REASONING_KEY = "supports_reasoning"; - function parsePositiveInt(value: string): number | null { const trimmed = value.trim(); if (!trimmed) return null; @@ -84,7 +83,10 @@ function getModelDraftChangeFlags( supportsStreaming: draft.supportsStreaming !== initialDraft.supportsStreaming, supportsTools: draft.supportsTools !== initialDraft.supportsTools, supportsVision: draft.supportsVision !== initialDraft.supportsVision, - supportsReasoning: draft.supportsReasoning !== initialDraft.supportsReasoning, + supportsReasoning: + draft.supportsReasoning !== initialDraft.supportsReasoning || + JSON.stringify(draft.reasoningEfforts) !== JSON.stringify(initialDraft.reasoningEfforts) || + draft.defaultReasoningEffort !== initialDraft.defaultReasoningEffort, }; } @@ -92,29 +94,6 @@ function hasAnyModelDraftChange(flags: ModelDraftChangeFlags): boolean { return Object.values(flags).some(Boolean); } -function mergeReasoningMetadata( - existingMetadata: string | null | undefined, - supportsReasoning: boolean, -): string | null { - let metadata: Record = {}; - if (existingMetadata) { - try { - const parsed: unknown = JSON.parse(existingMetadata); - if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) { - metadata = parsed as Record; - } - } catch { - metadata = {}; - } - } - if (supportsReasoning) { - metadata[SUPPORTS_REASONING_KEY] = true; - } else { - delete metadata[SUPPORTS_REASONING_KEY]; - } - return Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null; -} - function buildModelInputs( modelNames: string[], draft: ModelSourceDraft, @@ -155,7 +134,12 @@ function buildModelInputs( ? parseNonNegativeFloat(draft.audioPerMinute) : existingModel?.audioPerMinute ?? null, rawMetadataJson: draftChangeFlags.supportsReasoning - ? mergeReasoningMetadata(existingModel?.rawMetadataJson, draft.supportsReasoning) + ? mergeReasoningMetadata( + existingModel?.rawMetadataJson, + draft.supportsReasoning, + draft.reasoningEfforts, + draft.defaultReasoningEffort, + ) : existingModel?.rawMetadataJson ?? null, isEnabled: existingModel?.isEnabled ?? true, }; diff --git a/frontend/src/features/model-sources/components/model-source-form-fields.tsx b/frontend/src/features/model-sources/components/model-source-form-fields.tsx index 267f86496c..6f9a8cd2b2 100644 --- a/frontend/src/features/model-sources/components/model-source-form-fields.tsx +++ b/frontend/src/features/model-sources/components/model-source-form-fields.tsx @@ -4,10 +4,12 @@ import { useTranslation } from "react-i18next"; import { Checkbox } from "@/components/ui/checkbox"; import { FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; import type { ModelSourceDraft, ModelSourceFormValues, } from "@/features/model-sources/components/model-source-form"; +import { parseReasoningEffortsInput } from "@/features/model-sources/components/model-source-form"; type ModelSourceFormFieldsProps = { control: Control; @@ -28,6 +30,8 @@ const CAPABILITY_TOGGLES = [ ["supportsReasoning", "modelSources.capabilities.reasoning"] as const, ]; +const DEFAULT_REASONING_EFFORTS = ["low", "medium", "high"]; + export function ModelSourceFormFields({ control, draft, @@ -176,12 +180,91 @@ export function ModelSourceFormFields({ ))}
    + + {draft.supportsReasoning ? ( +
    +
    +
    {t("modelSources.fields.reasoningEfforts")}
    +

    + {t("modelSources.fields.reasoningEffortsDescription")} +

    +
    + +
    + + { + const reasoningEffortsInput = event.target.value; + const reasoningEfforts = parseReasoningEffortsInput(reasoningEffortsInput); + updateDraft({ + reasoningEffortsInput, + reasoningEfforts, + defaultReasoningEffort: reasoningEfforts.includes(draft.defaultReasoningEffort) + ? draft.defaultReasoningEffort + : (reasoningEfforts[0] ?? ""), + }); + }} + placeholder={t("modelSources.fields.reasoningEffortsPlaceholder")} + autoComplete="off" + /> +
    + +
    + + +
    +
    + ) : null} ); } diff --git a/frontend/src/features/model-sources/components/model-source-form.test.ts b/frontend/src/features/model-sources/components/model-source-form.test.ts new file mode 100644 index 0000000000..4485a631f2 --- /dev/null +++ b/frontend/src/features/model-sources/components/model-source-form.test.ts @@ -0,0 +1,25 @@ +import { describe, expect, it } from "vitest"; +import { mergeReasoningMetadata, parseReasoningEffortsInput } from "./model-source-form"; + +describe("model-source-form reasoning effort normalization", () => { + it("trims whitespace and preserves casing for effort values", () => { + expect(parseReasoningEffortsInput(" Ultra, xhigh , low ")).toEqual([ + "Ultra", + "xhigh", + "low", + ]); + }); + + it("preserves casing for declared default reasoning effort", () => { + const metadata = mergeReasoningMetadata( + null, + true, + ["Ultra", "provider-specific", "xhigh"], + " provider-specific ", + ); + const parsed = JSON.parse(metadata ?? "{}"); + + expect(parsed.supported_reasoning_levels).toEqual(["Ultra", "provider-specific", "xhigh"]); + expect(parsed.default_reasoning_level).toBe("provider-specific"); + }); +}); diff --git a/frontend/src/features/model-sources/components/model-source-form.ts b/frontend/src/features/model-sources/components/model-source-form.ts index db2923741b..a8590b6740 100644 --- a/frontend/src/features/model-sources/components/model-source-form.ts +++ b/frontend/src/features/model-sources/components/model-source-form.ts @@ -36,6 +36,9 @@ export type ModelSourceDraft = { supportsTools: boolean; supportsVision: boolean; supportsReasoning: boolean; + reasoningEffortsInput: string; + reasoningEfforts: string[]; + defaultReasoningEffort: string; contextWindow: string; maxOutputTokens: string; inputPer1M: string; @@ -53,6 +56,9 @@ export const initialModelSourceDraft: ModelSourceDraft = { supportsTools: false, supportsVision: false, supportsReasoning: false, + reasoningEffortsInput: "", + reasoningEfforts: [], + defaultReasoningEffort: "", contextWindow: "", maxOutputTokens: "", inputPer1M: "", @@ -86,9 +92,44 @@ function parseNonNegativeFloat(value: string): number | undefined { // model's raw metadata JSON, which the proxy reads to pass reasoning fields // through and to advertise supports_reasoning in /v1/models. Merge it into // any raw metadata the model already carries so other keys survive edits. -export function mergeReasoningFlag( +const DEFAULT_REASONING_EFFORTS = ["low", "medium", "high"]; + +function normalizeReasoningEffort(value: string): string { + return value.trim(); +} + +function dedupeReasoningEfforts(values: Iterable): string[] { + const seen = new Set(); + const result: string[] = []; + for (const value of values) { + const normalized = normalizeReasoningEffort(value); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + result.push(normalized); + } + return result; +} + +export function parseReasoningEffortsInput(value: string): string[] { + return dedupeReasoningEfforts(value.split(/[\n,]/)); +} + +function normalizeDefaultReasoningEffort( + reasoningEfforts: string[], + defaultReasoningEffort: string, +): string { + const normalizedDefault = normalizeReasoningEffort(defaultReasoningEffort); + if (normalizedDefault && reasoningEfforts.includes(normalizedDefault)) { + return normalizedDefault; + } + return reasoningEfforts[0] ?? ""; +} + +export function mergeReasoningMetadata( existing: string | null | undefined, supportsReasoning: boolean, + reasoningEfforts: string[] = [], + defaultReasoningEffort = "", ): string | null { let metadata: Record = {}; if (existing) { @@ -103,8 +144,20 @@ export function mergeReasoningFlag( } if (supportsReasoning) { metadata.supports_reasoning = true; + if (reasoningEfforts.length > 0) { + metadata.supported_reasoning_levels = reasoningEfforts; + metadata.default_reasoning_level = normalizeDefaultReasoningEffort( + reasoningEfforts, + defaultReasoningEffort, + ); + } else { + delete metadata.supported_reasoning_levels; + delete metadata.default_reasoning_level; + } } else { delete metadata.supports_reasoning; + delete metadata.supported_reasoning_levels; + delete metadata.default_reasoning_level; } return Object.keys(metadata).length > 0 ? JSON.stringify(metadata) : null; } @@ -137,7 +190,12 @@ export function modelInputsFromForm( cachedInputPer1M: cachedInputPer1M ?? null, outputPer1M: outputPer1M ?? null, audioPerMinute: audioPerMinute ?? null, - rawMetadataJson: mergeReasoningFlag(existingRawMetadata[model], draft.supportsReasoning), + rawMetadataJson: mergeReasoningMetadata( + existingRawMetadata[model], + draft.supportsReasoning, + draft.reasoningEfforts, + draft.defaultReasoningEffort, + ), isEnabled: existingEnabledByModel[model] ?? true, })); } @@ -149,22 +207,58 @@ function numberToInput(value: number | null | undefined): string { // Derive the shared draft from an existing source. The create UI applies one // set of per-model settings to every model, so editing mirrors that by reading // the first model's values as the representative settings. -function rawMetadataHasReasoning(rawMetadataJson: string | null | undefined): boolean { - if (!rawMetadataJson) return false; +function parseReasoningMetadata(rawMetadataJson: string | null | undefined): { + supportsReasoning: boolean; + reasoningEffortsInput: string; + reasoningEfforts: string[]; + defaultReasoningEffort: string; +} { + const fallback = { + supportsReasoning: false, + reasoningEffortsInput: "", + reasoningEfforts: [], + defaultReasoningEffort: "", + }; + if (!rawMetadataJson) return fallback; try { const parsed: unknown = JSON.parse(rawMetadataJson); - return ( - typeof parsed === "object" && - parsed !== null && - (parsed as Record).supports_reasoning === true - ); + if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) { + return fallback; + } + const metadata = parsed as Record; + const supportsReasoning = metadata.supports_reasoning === true; + const declaredLevels = Array.isArray(metadata.supported_reasoning_levels) + ? dedupeReasoningEfforts( + metadata.supported_reasoning_levels.flatMap((value): string[] => { + if (typeof value === "string") return [value]; + if (typeof value !== "object" || value === null || Array.isArray(value)) return []; + const effort = (value as Record).effort; + return typeof effort === "string" ? [effort] : []; + }), + ) + : []; + const reasoningEfforts = + supportsReasoning && declaredLevels.length === 0 ? DEFAULT_REASONING_EFFORTS : declaredLevels; + + return { + supportsReasoning, + reasoningEffortsInput: reasoningEfforts.join(", "), + reasoningEfforts, + defaultReasoningEffort: normalizeDefaultReasoningEffort( + reasoningEfforts, + typeof metadata.default_reasoning_level === "string" + ? metadata.default_reasoning_level + : "", + ), + }; } catch { - return false; + return fallback; } } export function draftFromSource(source: ModelSource): ModelSourceDraft { const firstModel = source.models[0]; + const reasoningMetadata = parseReasoningMetadata(firstModel?.rawMetadataJson); return { supportsChatCompletions: source.supportsChatCompletions, supportsResponses: source.supportsResponses, @@ -173,7 +267,10 @@ export function draftFromSource(source: ModelSource): ModelSourceDraft { supportsStreaming: firstModel?.supportsStreaming ?? true, supportsTools: firstModel?.supportsTools ?? false, supportsVision: firstModel?.supportsVision ?? false, - supportsReasoning: rawMetadataHasReasoning(firstModel?.rawMetadataJson), + supportsReasoning: reasoningMetadata.supportsReasoning, + reasoningEffortsInput: reasoningMetadata.reasoningEffortsInput, + reasoningEfforts: reasoningMetadata.reasoningEfforts, + defaultReasoningEffort: reasoningMetadata.defaultReasoningEffort, contextWindow: numberToInput(firstModel?.contextWindow), maxOutputTokens: numberToInput(firstModel?.maxOutputTokens), inputPer1M: numberToInput(firstModel?.inputPer1M), diff --git a/frontend/src/i18n/locales/en.json b/frontend/src/i18n/locales/en.json index 0cfa946b72..dc6837fbe7 100644 --- a/frontend/src/i18n/locales/en.json +++ b/frontend/src/i18n/locales/en.json @@ -1449,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Proxy pool update failed", "upstreamProxy.validation.hostRequired": "Host is required", "upstreamProxy.validation.nameRequired": "Name is required", - "upstreamProxy.validation.portInvalid": "Enter a port between 1 and 65535" + "upstreamProxy.validation.portInvalid": "Enter a port between 1 and 65535", + "modelSources.fields.reasoningEfforts": "Supported reasoning efforts", + "modelSources.fields.reasoningEffortsDescription": "List the effort slugs this model source accepts and choose which one should be the default.", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "Default reasoning effort" } diff --git a/frontend/src/i18n/locales/ko.json b/frontend/src/i18n/locales/ko.json index f0f162b24a..078f7930ab 100644 --- a/frontend/src/i18n/locales/ko.json +++ b/frontend/src/i18n/locales/ko.json @@ -1449,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Pool 업데이트 실패", "upstreamProxy.validation.hostRequired": "Host는 필수입니다", "upstreamProxy.validation.nameRequired": "이름은 필수입니다", - "upstreamProxy.validation.portInvalid": "1부터 65535 사이의 port를 입력하세요" + "upstreamProxy.validation.portInvalid": "1부터 65535 사이의 port를 입력하세요", + "modelSources.fields.reasoningEfforts": "지원되는 추론 수준", + "modelSources.fields.reasoningEffortsDescription": "이 모델 소스가 허용하는 추론 수준 슬러그를 적고 기본값을 고르세요.", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "기본 추론 수준" } diff --git a/frontend/src/i18n/locales/zh-CN.json b/frontend/src/i18n/locales/zh-CN.json index 14b218e860..8f32848bd5 100644 --- a/frontend/src/i18n/locales/zh-CN.json +++ b/frontend/src/i18n/locales/zh-CN.json @@ -1449,5 +1449,9 @@ "upstreamProxy.toasts.poolUpdateFailed": "Pool 更新失败", "upstreamProxy.validation.hostRequired": "主机不能为空", "upstreamProxy.validation.nameRequired": "名称必填", - "upstreamProxy.validation.portInvalid": "请输入 1 到 65535 之间的端口" + "upstreamProxy.validation.portInvalid": "请输入 1 到 65535 之间的端口", + "modelSources.fields.reasoningEfforts": "支持的推理级别", + "modelSources.fields.reasoningEffortsDescription": "填写这个模型源接受的推理级别标识，并选择默认值。", + "modelSources.fields.reasoningEffortsPlaceholder": "none, low, provider-specific", + "modelSources.fields.defaultReasoningEffort": "默认推理级别" } diff --git a/openspec/changes/configure-model-source-reasoning-efforts/proposal.md b/openspec/changes/configure-model-source-reasoning-efforts/proposal.md new file mode 100644 index 0000000000..2004c1ea45 --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/proposal.md @@ -0,0 +1,41 @@ +## Why + +PR #1661 already landed the backend contract for operator-declared model-source +reasoning efforts. PR #1675 still carries the useful dashboard part of that +feature, but its original branch also duplicated the backend parser/spec work +and baked in assumptions that #1661 explicitly rejected (`none` filtering and a +fixed effort enum). + +The dashboard still needs a way to configure the metadata that the backend now +honors, and it needs to preserve arbitrary provider-specific effort slugs rather +than forcing one hardcoded vocabulary. + +## What Changes + +- Add model-source dashboard controls for the reasoning-effort metadata that + `#1661` already reads from `raw_metadata_json`. +- Store supported efforts as an operator-edited list of slugs instead of a + fixed checkbox enum, so the UI can round-trip `none` and provider-specific + effort names. +- Keep the existing reasoning toggle, seed reasonable defaults when an operator + enables it for the first time, and normalize stale defaults back onto the + configured effort list during edit/save. +- Localize the new dashboard copy in `en`, `ko`, and `zh-CN`. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `frontend-architecture`: model-source create/edit dialogs can configure and + preserve supported reasoning-effort metadata for source models. + +## Impact + +- Dashboard only: model-source form state, create/edit dialogs, i18n, and + focused frontend regression tests. +- No API contract, database, backend parser, proxy routing, or request-policy + behavior changes in this PR; those remain owned by #1661. diff --git a/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md b/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md new file mode 100644 index 0000000000..1229e111bc --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/specs/frontend-architecture/spec.md @@ -0,0 +1,38 @@ +## MODIFIED Requirements + +### Requirement: Model-source reasoning metadata editor + +The dashboard MUST let operators configure the reasoning metadata stored on +model-source models without assuming one global effort vocabulary. + +#### Scenario: Edit arbitrary supported reasoning efforts + +- **GIVEN** a model source whose `raw_metadata_json` contains + `supports_reasoning: true` +- **AND** its `supported_reasoning_levels` include values such as `none` or a + provider-specific slug +- **WHEN** the dashboard opens the model-source create or edit form +- **THEN** the reasoning controls MUST show those effort slugs without dropping + or rewriting them +- **AND** saving the form MUST write the edited effort list back into + `supported_reasoning_levels`. + +#### Scenario: Normalize stale defaults during save + +- **GIVEN** a model source whose configured default effort is no longer present + in the edited supported-effort list +- **WHEN** the operator saves the form +- **THEN** the dashboard MUST replace the stale default with one of the + configured supported efforts +- **AND** it MUST NOT leave `default_reasoning_level` pointing at a removed + value. + +#### Scenario: Seed a first-time reasoning configuration + +- **GIVEN** an operator enables reasoning for a model source that previously had + no configured supported-effort list +- **WHEN** the dashboard reveals the reasoning metadata controls +- **THEN** the form MUST seed an editable default effort list and default value + so the operator can save a valid initial configuration +- **AND** the operator MUST still be able to replace that seed with arbitrary + effort slugs before saving. diff --git a/openspec/changes/configure-model-source-reasoning-efforts/tasks.md b/openspec/changes/configure-model-source-reasoning-efforts/tasks.md new file mode 100644 index 0000000000..ea2bfc87aa --- /dev/null +++ b/openspec/changes/configure-model-source-reasoning-efforts/tasks.md @@ -0,0 +1,15 @@ +## 1. Model-source dashboard + +- [x] 1.1 Extend model-source form state so it can parse, round-trip, and save + supported reasoning efforts plus the default effort from raw metadata. +- [x] 1.2 Add create/edit dialog controls for the effort list and default + selector without constraining operators to a fixed enum. +- [x] 1.3 Keep the existing reasoning toggle and preserve unrelated raw metadata + keys during edits. + +## 2. Verification + +- [x] 2.1 Add focused frontend regression coverage for default seeding, + arbitrary effort round-tripping, and stale-default normalization. +- [x] 2.2 Run focused frontend checks and strict OpenSpec validation for this + change. From c597226cf139ec1e80117b72f02b7a18bef3645f Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 20:34:53 +0400 Subject: [PATCH 101/117] fix(proxy): absorb replay-safe compaction recovery (#1849) * fix(compact): absorb active recovery replay semantics * fix(proxy): reject mixed post-compact tool suffix replays --- .../proxy/_service/http_bridge/helpers.py | 17 + .../_service/http_bridge/request_submit.py | 16 +- .../proxy/_service/http_bridge/streaming.py | 105 +++- app/modules/proxy/replay_safety.py | 99 ++- app/modules/proxy/service.py | 15 +- .../proposal.md | 28 + .../specs/responses-api-compat/spec.md | 54 ++ .../tasks.md | 9 + .../integration/test_http_responses_bridge.py | 156 +++++ tests/integration/test_proxy_compact.py | 66 ++ tests/unit/test_openai_requests.py | 83 +++ tests/unit/test_proxy_http_bridge.py | 577 ++++++++++++++++-- tests/unit/test_replay_safety.py | 324 ++++++++++ 13 files changed, 1445 insertions(+), 104 deletions(-) create mode 100644 openspec/changes/recover-post-compact-bridge-replays/proposal.md create mode 100644 openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/recover-post-compact-bridge-replays/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index c5eebf77e5..53f8e0bf4e 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -627,6 +627,23 @@ def _has_http_bridge_response_output_marker(item: JsonValue) -> bool: return status in {"completed", "in_progress"} +def _http_bridge_pending_response_events_seen(pending_states: Sequence[_WebSocketRequestState]) -> int: + return max( + ( + max( + state.response_event_count, + int( + state.response_id is not None + or state.latency_response_created_ms is not None + or state.downstream_visible + ), + ) + for state in pending_states + ), + default=0, + ) + + def _http_bridge_input_item_type(item: JsonValue) -> str | None: if not isinstance(item, dict): return None diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index e3412b33f9..a892ab3084 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -81,6 +81,7 @@ _http_bridge_durable_lease_ttl_seconds, _http_bridge_is_previous_response_owner_unavailable, _http_bridge_key_strength, + _http_bridge_pending_response_events_seen, _http_bridge_precreated_retry_failure_error, _http_bridge_prewarm_enabled, _http_bridge_request_budget_seconds, @@ -2816,20 +2817,7 @@ async def _retire_stale_pending_http_bridge_session( # circuit strike. Explicit values remain authoritative for # reader-failure callers whose pending deque was already # drained before entering this shared boundary. - response_events_seen = max( - ( - max( - request_state.response_event_count, - int( - request_state.response_id is not None - or request_state.latency_response_created_ms is not None - or request_state.downstream_visible - ), - ) - for request_state in retired_request_states - ), - default=0, - ) + response_events_seen = _http_bridge_pending_response_events_seen(retired_request_states) if retry_circuit_attempt_selection is None: retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( retired_request_states diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 1285996fad..631f667c28 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -1511,18 +1511,29 @@ def classify_durable_full_resend( if durable_lookup is not None and not _http_bridge_models_compatible(durable_lookup.model, payload.model) else None ) + durable_model_transition_uses_fresh_replay = ( + durable_model_transition_lookup is not None + and not forwarded_request + and rewritten_file_account_id is None + and durable_full_resend_fresh_payload is not None + and durable_full_resend_has_safe_fresh_context + and durable_full_resend_is_account_neutral is True + ) durable_model_transition_requires_owner = durable_model_transition_lookup is not None and ( - payload.previous_response_id is not None - or bridge_session_key.strength == "hard" - or ( - bridge_session_key.affinity_kind == "prompt_cache" - and _http_bridge_request_stage( - headers=headers, - payload=payload, - durable_lookup=durable_model_transition_lookup, + not durable_model_transition_uses_fresh_replay + and ( + payload.previous_response_id is not None + or bridge_session_key.strength == "hard" + or ( + bridge_session_key.affinity_kind == "prompt_cache" + and _http_bridge_request_stage( + headers=headers, + payload=payload, + durable_lookup=durable_model_transition_lookup, + ) + == "follow_up" + and durable_model_transition_lookup.latest_turn_state is not None ) - == "follow_up" - and durable_model_transition_lookup.latest_turn_state is not None ) ) if durable_model_transition_lookup is not None: @@ -1536,7 +1547,36 @@ def classify_durable_full_resend( model_class=_extract_model_class(payload.model) if payload.model else None, owner_check_applied=durable_model_transition_requires_owner, ) - if is_http_bridge_account_neutral_replay( + if durable_model_transition_uses_fresh_replay: + replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(uuid4().hex) + bridge_session_key = _HTTPBridgeSessionKey( + replay_kind, + replay_key, + bridge_session_key.api_key_id, + strength="soft", + ) + affinity = _AffinityPolicy() + incoming_turn_state_header = None + incoming_session_header = None + session_header_fallback_key = None + effective_payload = durable_full_resend_fresh_payload + untrimmed_effective_payload = durable_full_resend_fresh_payload + force_local_recovery_creation = True + preferred_account_has_continuity_provenance = False + _log_http_bridge_event( + "model_transition_fresh_resend", + bridge_session_key, + account_id=durable_model_transition_lookup.account_id, + model=payload.model, + detail=( + "outcome=account_neutral_full_resend_without_owner," + f"previous_model={durable_model_transition_lookup.model}" + ), + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(payload.model) if payload.model else None, + owner_check_applied=False, + ) + elif is_http_bridge_account_neutral_replay( kind=durable_model_transition_lookup.canonical_kind, key=durable_model_transition_lookup.canonical_key, ): @@ -1719,6 +1759,7 @@ def classify_durable_full_resend( durable_lookup.account_id if ( durable_lookup is not None + and not durable_model_transition_uses_fresh_replay and ( request_state.previous_response_id is not None or bridge_session_key.strength == "hard" @@ -1735,6 +1776,7 @@ def classify_durable_full_resend( request_state.preferred_account_id is None and durable_model_transition_lookup is not None and durable_model_transition_requires_owner + and not durable_model_transition_uses_fresh_replay ): request_state.preferred_account_id = durable_model_transition_lookup.account_id local_previous_response_owner: str | None = None @@ -1856,6 +1898,7 @@ def classify_durable_full_resend( def durable_full_resend_allows_account_neutral_replay() -> bool: nonlocal durable_full_resend_fresh_payload + nonlocal durable_full_resend_has_safe_fresh_context nonlocal durable_full_resend_is_account_neutral nonlocal durable_full_resend_retains_prior_output @@ -1881,7 +1924,17 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: stored_count=eligibility_projection.stored_prefix_count, canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, ) - if not durable_full_resend_retains_prior_output: + durable_full_resend_has_safe_fresh_context = durable_full_resend_retains_prior_output or ( + durable_lookup is not None + and durable_lookup.latest_pending_tool_calls is not None + and responses_input_suffix_matches_pending_tool_calls( + eligibility_projection.input_items, + stored_count=eligibility_projection.stored_prefix_count, + pending_tool_calls=durable_lookup.latest_pending_tool_calls, + canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, + ) + ) + if not durable_full_resend_has_safe_fresh_context: return False replay_projection = project_responses_input_for_account_neutral_fresh_replay( cast(list[JsonValue], payload.input), @@ -1892,7 +1945,7 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: durable_full_resend_fresh_payload = _http_bridge_payload_without_previous_response_id( payload ).model_copy(update={"input": replay_projection.input_items}) - if not durable_full_resend_retains_prior_output: + if not durable_full_resend_has_safe_fresh_context: return False if durable_full_resend_is_account_neutral is None: durable_full_resend_is_account_neutral = _http_bridge_payload_is_account_neutral_fresh_replay( @@ -2813,16 +2866,16 @@ def switch_to_account_neutral_replay() -> None: previous_request_state.proxy_injected_anchor_had_full_resend_payload ) request_state.fresh_upstream_request_text = fresh_upstream_request_text - # The trim branch only fires when the untrimmed payload - # is a true full resend whose prefix exactly matches the - # already-stored context, so the unanchored request text - # is a safe fresh-turn replay target regardless of - # whether the anchor came from the durable or - # session-level injection path. Injection-only re-prepares - # keep the replay-safety decision made when the anchor was - # injected. + # The trim branch proves the upstream submission can omit the + # stored prefix, but it does not by itself prove that dropping + # the injected anchor is safe. Keep the original anchor site's + # decision unless this was a durable full-resend proof with a + # verified safe fresh suffix. Session-level anchors may still be + # compacted follow-ups whose prior context only exists behind + # previous_response_id. request_state.fresh_upstream_request_is_retry_safe = ( - (durable_full_resend_anchor_count is None or durable_full_resend_has_safe_fresh_context) + previous_request_state.fresh_upstream_request_is_retry_safe + or (durable_full_resend_anchor_count is not None and durable_full_resend_has_safe_fresh_context) if store_context_trim_applied else previous_request_state.fresh_upstream_request_is_retry_safe ) @@ -2937,6 +2990,7 @@ async def rollback_pre_dispatch_recovery_claim() -> None: owner_check_applied=True, ) replacement_preferred_account_id = request_state.preferred_account_id + replacement_excluded_account_ids = set(request_state.excluded_account_ids) if request_state.previous_response_id is not None and replacement_preferred_account_id is None: replacement_preferred_account_id = session.account.id elif replacement_preferred_account_id is None: @@ -2949,9 +3003,8 @@ async def rollback_pre_dispatch_recovery_claim() -> None: # impossible (fallback_on_preferred_account_unavailable is # False for exactly this pinned case below) and would # keep poisoning every later recovery call on this - # request, since excluded_account_ids persists on - # request_state. - request_state.excluded_account_ids.add(session.account.id) + # request. + replacement_excluded_account_ids.add(session.account.id) while True: try: replacement_session = await self._get_or_create_http_bridge_session( @@ -2984,7 +3037,7 @@ async def rollback_pre_dispatch_recovery_claim() -> None: request_usage_budget=request_state.request_usage_budget, request_deadline=request_deadline, session_header_fallback_key=session_header_fallback_key, - exclude_account_ids=request_state.excluded_account_ids or None, + exclude_account_ids=replacement_excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, ) diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index fd44be4fbb..d50accccca 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -15,11 +15,10 @@ "function_call_output": "function_call", "custom_tool_call_output": "custom_tool_call", "apply_patch_call_output": "apply_patch_call", + "tool_search_output": "tool_search_call", } _TOOL_CALL_TYPES = frozenset(_TOOL_CALL_TYPE_BY_OUTPUT_TYPE.values()) -_ACCOUNT_NEUTRAL_REPLAY_OMITTED_ITEM_TYPES = frozenset( - {"reasoning", "tool_search_call", "tool_search_output", "web_search_call"} -) +_ACCOUNT_NEUTRAL_REPLAY_OMITTED_ITEM_TYPES = frozenset({"reasoning", "web_search_call"}) _INTERNAL_CHAT_MESSAGE_METADATA_FIELD = "internal_chat_message_metadata_passthrough" _ACCOUNT_NEUTRAL_INTERNAL_CHAT_MESSAGE_METADATA_FIELDS = frozenset({"turn_id"}) _ACCOUNT_NEUTRAL_TOOL_TYPES = frozenset({"custom", "function", "web_search", "web_search_preview"}) @@ -39,6 +38,7 @@ "additional_tools", "apply_patch_call", "apply_patch_call_output", + "compaction", "custom_tool_call", "custom_tool_call_output", "function_call", @@ -47,6 +47,8 @@ "input_image", "input_text", "message", + "tool_search_call", + "tool_search_output", } ) _ACCOUNT_NEUTRAL_MESSAGE_CONTENT_TYPES = frozenset( @@ -65,6 +67,7 @@ } _ACCOUNT_NEUTRAL_INPUT_ITEM_FIELDS = { "additional_tools": frozenset({"role", "tools", "type"}), + "compaction": frozenset({"encrypted_content", "id", "status", "type"}), "apply_patch_call": frozenset( { "call_id", @@ -93,6 +96,22 @@ "function_call_output": frozenset( {"call_id", "caller", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "output", "status", "type"} ), + "tool_search_call": frozenset( + {"arguments", "call_id", "caller", "execution", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "status", "type"} + ), + "tool_search_output": frozenset( + { + "call_id", + "caller", + "execution", + "id", + _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, + "output", + "status", + "tools", + "type", + } + ), } _ACCOUNT_NEUTRAL_ITEM_STATUSES = frozenset({"completed", "failed"}) _ACCOUNT_NEUTRAL_APPLY_PATCH_OPERATION_FIELDS = { @@ -176,6 +195,8 @@ def project_responses_input_for_account_neutral_fresh_replay( if stored_count <= 0 or stored_count > len(input_items): return None + if not _stored_prefix_compaction_boundary_is_safe(input_items, stored_count=stored_count): + return None projected_items: list[JsonValue] = [] projected_stored_count = 0 @@ -209,6 +230,14 @@ def project_responses_input_for_account_neutral_fresh_replay( ) +def _stored_prefix_compaction_boundary_is_safe(input_items: list[JsonValue], *, stored_count: int) -> bool: + stored_prefix = input_items[:stored_count] + for item in stored_prefix[:-1]: + if isinstance(item, dict) and item.get("type") == "compaction" and _compaction_item_is_self_contained(item): + return False + return True + + def _is_canonical_lite_tool_bundle(item: JsonValue) -> bool: return ( isinstance(item, dict) @@ -245,6 +274,8 @@ def _project_account_neutral_replay_item( ): return None + if item_type == "compaction": + return item if "id" not in item: return item projected_item = dict(item) @@ -269,6 +300,10 @@ def responses_input_items_are_self_contained_fresh_replay(input_items: list[Json item_type = item_type_value if isinstance(item_type_value, str) else None if not _input_item_has_only_known_fields(item, item_type): return False + if item_type == "compaction": + if not _compaction_item_is_self_contained(item): + return False + continue call_id_value = item.get("call_id") call_id = call_id_value if isinstance(call_id_value, str) and call_id_value else None if item_type in _TOOL_CALL_TYPES: @@ -324,8 +359,10 @@ def responses_input_suffix_retains_prior_output( if prefix_state is None: return False pending_suffix_calls, seen_suffix_call_ids = prefix_state - retained_output_seen = False + compact_context_prefix = _input_prefix_ends_with_self_contained_compaction(input_items[:stored_count]) + retained_output_seen = compact_context_prefix retained_output_is_final_answer = False + settled_suffix_call_types: set[str] = set() fresh_followup_seen = False fresh_followup_count = 0 fresh_followup_is_user_message = False @@ -364,6 +401,17 @@ def responses_input_suffix_retains_prior_output( if pending_suffix_calls[0] != (call_type, call_id): return False pending_suffix_calls.popleft() + settled_suffix_call_types.add(call_type) + if ( + compact_context_prefix + and not pending_suffix_calls + and settled_suffix_call_types == {"tool_search_call"} + ): + retained_output_seen = True + retained_output_is_final_answer = False + fresh_followup_seen = False + fresh_followup_count = 0 + fresh_followup_is_user_message = False continue if item_type in (None, "message") and item.get("role") == "assistant": if pending_suffix_calls or not _is_retained_response_message(item): @@ -396,6 +444,17 @@ def responses_input_suffix_retains_prior_output( return retained_output_seen and fresh_followup_seen and not pending_suffix_calls +def _input_prefix_ends_with_self_contained_compaction(input_items: list[JsonValue]) -> bool: + if not input_items: + return False + last_item = input_items[-1] + return ( + isinstance(last_item, dict) + and last_item.get("type") == "compaction" + and _compaction_item_is_self_contained(last_item) + ) + + def responses_input_suffix_matches_pending_tool_calls( input_items: list[JsonValue], *, @@ -628,6 +687,9 @@ def _tool_call_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) return _is_nonblank_string(item.get("name")) and isinstance(item.get("arguments"), str) if item_type == "custom_tool_call": return _is_nonblank_string(item.get("name")) and isinstance(item.get("input"), str) + if item_type == "tool_search_call": + arguments = item.get("arguments") + return isinstance(arguments, dict) and item.get("execution") in (None, "client") operation = item.get("operation") patch = item.get("patch") input_value = item.get("input") @@ -640,6 +702,10 @@ def _tool_call_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) return _is_nonblank_string(input_value) +def _compaction_item_is_self_contained(item: Mapping[str, JsonValue]) -> bool: + return item.get("status") in (None, "completed") and _is_nonblank_string(item.get("encrypted_content")) + + def _caller_is_self_contained(item: Mapping[str, JsonValue]) -> bool: caller = item.get("caller") return caller is None or caller == {"type": "direct"} @@ -677,6 +743,13 @@ def _apply_patch_operation_is_self_contained(operation: JsonValue | None) -> boo def _tool_output_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) -> bool: if item.get("status") not in (None, "completed", "failed"): return False + if item_type == "tool_search_output": + if item.get("execution") not in (None, "client"): + return False + if "tools" in item and not _tools_are_account_neutral(item.get("tools")): + return False + if _tool_search_output_tools_are_self_contained(item): + return True output = item.get("output") if isinstance(output, str): return True @@ -692,6 +765,15 @@ def _tool_output_is_self_contained(item_type: str, item: Mapping[str, JsonValue] ) +def _tool_search_output_tools_are_self_contained(item: Mapping[str, JsonValue]) -> bool: + return ( + item.get("execution") == "client" + and "tools" in item + and _tools_are_account_neutral(item.get("tools")) + and item.get("output") in (None, "") + ) + + def _is_nonblank_string(value: JsonValue | None) -> bool: return isinstance(value, str) and bool(value.strip()) @@ -933,6 +1015,13 @@ def _input_items_have_valid_account_neutral_shape(input_items: list[JsonValue]) if not _input_content_part_is_self_contained(item, allow_output=False): return False continue + if item_type == "tool_search_output": + execution = item.get("execution") + if execution is not None and execution != "client": + return False + if "tools" in item and not _tools_are_account_neutral(item.get("tools")): + return False + continue if item_type == "additional_tools": if item.get("role") != "developer" or not _tools_are_account_neutral(item.get("tools")): return False @@ -1017,6 +1106,8 @@ def _contains_account_scoped_input_state(value: JsonValue) -> bool: return True if item_type == "additional_tools" and not _tools_are_account_neutral(current.get("tools")): return True + if item_type == "compaction" and _compaction_item_is_self_contained(current): + continue if ( isinstance(item_type, str) and (item_type.endswith("_call") or item_type.endswith("_call_output")) diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index cf6fcbafd0..2f7447680d 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -221,6 +221,9 @@ from app.modules.proxy._service.http_bridge.helpers import ( _http_bridge_payload_without_previous_response_id as _http_bridge_payload_without_previous_response_id, ) +from app.modules.proxy._service.http_bridge.helpers import ( + _http_bridge_pending_response_events_seen as _http_bridge_pending_response_events_seen, +) from app.modules.proxy._service.http_bridge.helpers import ( _http_bridge_precreated_retry_failure_error as _http_bridge_precreated_retry_failure_error, ) @@ -2594,19 +2597,11 @@ def _service_tier_from_event_payload(payload: dict[str, JsonValue] | None) -> st def _effective_service_tier(requested_service_tier: str | None, actual_service_tier: str | None) -> str | None: - if isinstance(actual_service_tier, str): - return actual_service_tier - if isinstance(requested_service_tier, str): - return requested_service_tier - return None + return actual_service_tier if isinstance(actual_service_tier, str) else requested_service_tier def _normalize_service_tier_value(value: JsonValue) -> str | None: if not isinstance(value, str): return None stripped = value.strip() - if not stripped: - return None - if stripped.lower() == "fast": - return "priority" - return stripped + return "priority" if stripped.lower() == "fast" else stripped or None diff --git a/openspec/changes/recover-post-compact-bridge-replays/proposal.md b/openspec/changes/recover-post-compact-bridge-replays/proposal.md new file mode 100644 index 0000000000..50d41626a5 --- /dev/null +++ b/openspec/changes/recover-post-compact-bridge-replays/proposal.md @@ -0,0 +1,28 @@ +# Recover post-compact HTTP bridge replays + +## Why + +Post-compaction Codex turns can carry a compact context item, completed +tool-search call/output side effects, and a fresh user message. When the HTTP +bridge treats a session-level compact-anchor trim as a generically safe fresh +replay, later recovery may drop the durable context or tool-search side effects +that make the follow-up self-contained. + +## What Changes + +- Treat completed `compaction` items with encrypted content as self-contained + account-neutral replay context. +- Preserve completed `tool_search_call` / `tool_search_output` pairs when + projecting a compacted fresh replay payload. +- Preserve compact context when projecting a fresh replay payload, while removing + response-owned ids. +- Keep session-level compact-anchor trim safety separate from durable full-resend + proof; trimming a stored prefix does not automatically make an unanchored + replay safe. +- Retire stale response-create gate holders with evidence about whether upstream + response events were already observed, so wedged bridge sessions take the + existing recovery/quarantine path. + +## Impact + +Post-compaction follow-up turns recover with the compact context preserved. diff --git a/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md b/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..c1024820ec --- /dev/null +++ b/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md @@ -0,0 +1,54 @@ +## ADDED Requirements + +### Requirement: Post-compact bridge replays preserve compact context + +codex-lb MUST treat completed post-compaction replay context as self-contained when an HTTP bridge or Responses WebSocket request must recover a follow-up turn after compaction. A projected fresh replay payload MUST retain that compact context while removing response-owned bookkeeping ids. + +Trimming a stored prefix because a session-level compact anchor was injected +MUST NOT by itself mark the unanchored request safe to replay. The proxy may +mark the unanchored request replay-safe only when the original anchor site had +already made that decision or when a durable full-resend proof shows the fresh +suffix is self-contained. + +Account-neutral recovery MAY select another eligible account after the previous +owner has been excluded or proved silent. Requests that explicitly require a +preferred previous-response owner MUST continue to fail closed when that owner +is unavailable. + +#### Scenario: id-free completed compaction and tool-search context survives fresh replay projection + +- **GIVEN** a follow-up payload starts with a completed `compaction` item whose + encrypted content is non-empty +- **AND** that compaction item does not carry an `id` +- **AND** the payload also carries a completed `tool_search_call` / + `tool_search_output` pair followed by a fresh user message +- **WHEN** codex-lb projects an account-neutral fresh replay payload +- **THEN** the projected payload includes the compaction item +- **AND** it preserves the completed tool-search pair without response-owned ids +- **AND** the projected payload is eligible for account-neutral replay + +#### Scenario: session-level compact trim does not fabricate replay safety + +- **GIVEN** a session-level compact anchor trimmed a stored prefix from a + follow-up request +- **AND** the original request state was not already known to be safe as an + unanchored fresh replay +- **WHEN** the bridge records the retained fresh request text +- **THEN** codex-lb does not mark that retained request as retry-safe solely + because the trim happened + +#### Scenario: account-neutral recovery can leave a silent owner + +- **GIVEN** an account-neutral HTTP bridge recovery request excludes the previous + owner account after it failed to acknowledge `response.create` +- **WHEN** another eligible account is available +- **THEN** codex-lb reconnects on that replacement account and sends the retained + request there + +#### Scenario: required previous-response owner still fails closed + +- **GIVEN** a follow-up request explicitly requires its preferred + previous-response owner account +- **WHEN** that owner is unavailable +- **THEN** codex-lb returns the previous-response-owner-unavailable failure + instead of silently rebinding the request to another account diff --git a/openspec/changes/recover-post-compact-bridge-replays/tasks.md b/openspec/changes/recover-post-compact-bridge-replays/tasks.md new file mode 100644 index 0000000000..161d670a83 --- /dev/null +++ b/openspec/changes/recover-post-compact-bridge-replays/tasks.md @@ -0,0 +1,9 @@ +# Tasks + +- [x] Accept compact context and tool-search call/output pairs in account-neutral replay safety checks. +- [x] Preserve completed compaction items in projected fresh replay payloads without retaining response-owned ids. +- [x] Keep session-level trim safety from overriding the original replay-safety decision unless a durable full-resend proof exists. +- [x] Allow account-neutral bridge recovery to select another eligible account after the silent owner is excluded. +- [x] Keep explicit required-owner continuity failures fail-closed. +- [x] Pass response-event evidence into stale response-create gate retirement. +- [x] Add replay-safety and HTTP bridge regression coverage for post-compact recovery. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 909fed0586..2386b43304 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -13487,6 +13487,162 @@ async def fake_reconnect( assert replacement_upstream.sent_text == [retry_request.request_text] +@pytest.mark.asyncio +async def test_retry_account_neutral_precreated_request_switches_from_silent_account(app_instance, monkeypatch): + from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key + + service = get_proxy_service_for_app(app_instance) + recovery_kind, recovery_key = make_http_bridge_account_neutral_replay_key("retry-silent-account") + first_account = cast(Account, SimpleNamespace(id="acct-silent", status=AccountStatus.ACTIVE, plan_type="plus")) + replacement_account = cast( + Account, + SimpleNamespace(id="acct-replacement", status=AccountStatus.ACTIVE, plan_type="plus"), + ) + replacement_upstream = _RecordingUpstreamWebSocket() + session = proxy_module._HTTPBridgeSession( + key=proxy_module._HTTPBridgeSessionKey(recovery_kind, recovery_key, None), + headers={"x-codex-turn-state": "stale-turn-state"}, + affinity=proxy_module._AffinityPolicy(), + request_model="gpt-5.5", + account=first_account, + upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_lock=anyio.Lock(), + pending_requests=deque(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + request_state = proxy_module._WebSocketRequestState( + request_id="req-account-neutral-precreated-retry", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + transport="http", + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.5", "input": []}), + ) + session.pending_requests.append(request_state) + reconnect_calls: list[dict[str, object]] = [] + + async def fake_reconnect( + self, + target_session, + *, + request_state, + restart_reader=False, + require_same_account=False, + require_preferred_account=False, + ): + del self, restart_reader + reconnect_calls.append( + { + "require_same_account": require_same_account, + "require_preferred_account": require_preferred_account, + "preferred_account_id": target_session.account.id, + "excluded_account_ids": set(request_state.excluded_account_ids), + } + ) + target_session.account = replacement_account + target_session.upstream = replacement_upstream + + monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) + + assert await service._retry_http_bridge_precreated_request(session) is True + + assert reconnect_calls == [ + { + "require_same_account": True, + "require_preferred_account": True, + "preferred_account_id": "acct-silent", + "excluded_account_ids": set(), + } + ] + assert request_state.preferred_account_id == "acct-silent" + assert session.account.id == "acct-replacement" + assert replacement_upstream.sent_text == [request_state.request_text] + + +@pytest.mark.asyncio +async def test_reconnect_required_owner_still_fails_when_owner_unavailable(app_instance, monkeypatch): + service = get_proxy_service_for_app(app_instance) + owner_account = cast( + Account, + SimpleNamespace(id="acct-owner-required", status=AccountStatus.ACTIVE, plan_type="plus"), + ) + session = proxy_module._HTTPBridgeSession( + key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "required-owner-reconnect", None), + headers={}, + affinity=proxy_module._AffinityPolicy(), + request_model="gpt-5.5", + account=owner_account, + upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), + upstream_control=proxy_module._WebSocketUpstreamControl(), + pending_lock=anyio.Lock(), + pending_requests=deque(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=time.monotonic(), + idle_ttl_seconds=120.0, + ) + request_state = proxy_module._WebSocketRequestState( + request_id="req-required-owner-reconnect", + model="gpt-5.5", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + awaiting_response_created=True, + transport="http", + response_create_gate_acquired=True, + request_text=json.dumps({"type": "response.create", "model": "gpt-5.5", "input": []}), + ) + request_state.preferred_account_id = owner_account.id + selection_calls: list[dict[str, object]] = [] + + async def fake_select_account_with_budget_for_stream(self, deadline, **kwargs): + del self, deadline + selection_calls.append( + { + "preferred_account_id": kwargs.get("preferred_account_id"), + "preferred_account_is_continuity_owner": kwargs.get("preferred_account_is_continuity_owner"), + "fallback_on_preferred_account_unavailable": kwargs.get("fallback_on_preferred_account_unavailable"), + } + ) + return AccountSelection( + account=None, + error_message="Required continuity owner account no longer exists", + error_code=CONTINUITY_OWNER_UNAVAILABLE, + ) + + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_account_with_budget_for_stream", + fake_select_account_with_budget_for_stream, + ) + + with pytest.raises(proxy_module.ProxyResponseError) as exc_info: + await service._reconnect_http_bridge_session( + session, + request_state=request_state, + require_preferred_account=True, + ) + + assert exc_info.value.status_code == 502 + assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" + assert selection_calls == [ + { + "preferred_account_id": owner_account.id, + "preferred_account_is_continuity_owner": False, + "fallback_on_preferred_account_unavailable": False, + } + ] + + @pytest.mark.asyncio async def test_v1_responses_http_bridge_send_failure_returns_upstream_unavailable( async_client, diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 6bb285ca26..580a281ed2 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -664,6 +664,72 @@ async def fake_compact(payload, headers, access_token, account_id): ] +@pytest.mark.asyncio +async def test_proxy_compact_preserves_single_output_item_with_skill_context(async_client, monkeypatch): + email = "compact-skill-recovery@example.com" + raw_account_id = "acc_compact_skill_recovery" + auth_json = _make_auth_json(raw_account_id, email) + files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} + response = await async_client.post("/api/accounts/import", files=files) + assert response.status_code == 200 + + seen: dict[str, ResponsesCompactRequest] = {} + + async def fake_compact(payload, headers, access_token, account_id): + del headers, access_token, account_id + seen["payload"] = payload + return CompactResponsePayload.model_validate( + { + "object": "response.compaction", + "compaction_summary": { + "id": "cmp_skill_recovery", + "encrypted_content": "enc_skill_recovery", + }, + } + ) + + monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) + + payload = { + "model": "gpt-5.5", + "instructions": "Compact the conversation.", + "input": [ + {"type": "message", "role": "user", "content": "hello"}, + { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": ( + "\n" + "grill-me\n" + "/home/kom/.codex/skills/grill-me/SKILL.md\n" + "---\n" + "name: grill-me\n" + "---\n" + "Ask one question at a time.\n" + "" + ), + } + ], + }, + ], + } + response = await async_client.post("/backend-api/codex/responses/compact", json=payload) + + assert response.status_code == 200 + output = response.json()["output"] + assert output == [ + { + "id": "cmp_skill_recovery", + "type": "compaction", + "encrypted_content": "enc_skill_recovery", + } + ] + assert "grill-me" in json.dumps(seen["payload"].to_payload()) + + @pytest.mark.asyncio async def test_proxy_compact_headers_include_monthly_only_credits(async_client, monkeypatch): email = "compact-monthly@example.com" diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 04e8cd6e60..3c868df01c 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -2398,6 +2398,89 @@ def test_compact_trimming_preserves_codex_goal_context_anchor_from_middle(): assert dumped_input[-1] == input_items[-1] +def test_compact_trimming_does_not_anchor_active_skill_context_from_middle(): + skill_context = { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": ( + "\n" + "grill-me\n" + "/home/kom/.codex/skills/grill-me/SKILL.md\n" + "---\n" + "name: grill-me\n" + "---\n" + "Ask one question at a time and keep the interview mode active.\n" + "" + ), + } + ], + } + input_items = [ + {"role": "user", "content": "initial instructions"}, + {"role": "assistant", "content": "x" * 300_000}, + skill_context, + {"role": "assistant", "content": "y" * 500_000}, + {"role": "user", "content": "latest request"}, + ] + payload = { + "model": "gpt-5.1", + "instructions": "hi", + "input": input_items, + } + + request = ResponsesCompactRequest.model_validate(payload) + dumped = request.to_payload() + dumped_input = dumped["input"] + + assert isinstance(dumped_input, list) + assert dumped_input[0] == input_items[0] + assert dumped_input[-1] == input_items[-1] + assert skill_context not in dumped_input + assert input_items[1] not in dumped_input + assert input_items[3] not in dumped_input + + +def test_compact_trimming_does_not_anchor_plain_skill_catalog_mentions(): + catalog_context = { + "type": "message", + "role": "user", + "content": [ + { + "type": "input_text", + "text": ( + "\n" + "- grill-me: Interview the user one question at a time.\n" + "" + ), + } + ], + } + input_items = [ + {"role": "user", "content": "initial instructions"}, + {"role": "assistant", "content": "x" * 300_000}, + catalog_context, + {"role": "assistant", "content": "y" * 500_000}, + {"role": "user", "content": "latest request"}, + ] + payload = { + "model": "gpt-5.1", + "instructions": "hi", + "input": input_items, + } + + request = ResponsesCompactRequest.model_validate(payload) + dumped = request.to_payload() + dumped_input = dumped["input"] + + assert isinstance(dumped_input, list) + assert catalog_context not in dumped_input + assert dumped_input[0] == input_items[0] + assert dumped_input[-1] == input_items[-1] + + def test_compact_trimming_preserves_non_message_developer_directive_from_middle(): developer_directive = { "type": "future_directive", diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index a8b8fd5628..744adb7d76 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -2586,8 +2586,10 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" + del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2695,8 +2697,10 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" + del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2786,8 +2790,10 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, + response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" + del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2888,7 +2894,9 @@ async def fake_retire( retire_session: proxy_service._HTTPBridgeSession, *, detail: str, + response_events_seen: int | None = None, ) -> None: + del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -24134,18 +24142,19 @@ async def test_stream_via_http_bridge_fails_closed_before_file_affinity_when_pre @pytest.mark.asyncio @pytest.mark.parametrize( - ("unsafe_replay_input", "replace_retired_gate", "stored_model"), + ("unsafe_replay_input", "replace_retired_gate", "stored_model", "pending_manifest_replay"), [ - (None, False, None), - (None, False, "gpt-5.3"), - (None, True, None), - ("conversation", False, None), - ("file", False, None), - ("missing_prior_output", False, None), - ("orphan_output", False, None), - ("response_owned_developer", False, None), - ("response_owned_stored_developer", False, None), - ("missing_owner", False, None), + pytest.param(None, False, None, False, id="retained-output"), + pytest.param(None, False, "gpt-5.3", False, id="retained-output-stored-model"), + pytest.param(None, True, None, False, id="retained-output-replace-retired-gate"), + pytest.param(None, False, None, True, id="pending-tool-manifest"), + pytest.param("conversation", False, None, False, id="conversation"), + pytest.param("file", False, None, False, id="file"), + pytest.param("missing_prior_output", False, None, False, id="missing-prior-output"), + pytest.param("orphan_output", False, None, False, id="orphan-output"), + pytest.param("response_owned_developer", False, None, False, id="response-owned-developer"), + pytest.param("response_owned_stored_developer", False, None, False, id="response-owned-stored-developer"), + pytest.param("missing_owner", False, None, False, id="missing-owner"), ], ) async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_when_owner_is_unavailable( @@ -24153,6 +24162,7 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe unsafe_replay_input: str | None, replace_retired_gate: bool, stored_model: str | None, + pending_manifest_replay: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) account_neutral_classifier = Mock( @@ -24207,16 +24217,14 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe ) elif unsafe_replay_input == "orphan_output": historical_input.append({"type": "function_call_output", "call_id": "call_missing", "output": "orphan output"}) - historical_input.append( - { - "type": "function_call", - "id": "fc_owner", - "call_id": "call_old", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": owner_metadata, - } - ) + retained_boundary_call: proxy_service.JsonValue = { + "type": "function_call", + "id": "fc_owner", + "call_id": "call_old", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": owner_metadata, + } retained_boundary_output: proxy_service.JsonValue = { "type": "function_call_output", "call_id": "call_old", @@ -24228,6 +24236,21 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "content": [{"type": "input_text", "text": "next question"}], "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, } + pending_tool_loop: list[proxy_service.JsonValue] = [ + { + "type": "function_call", + "call_id": "call_pending", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + { + "type": "function_call_output", + "call_id": "call_pending", + "output": "fresh result", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + ] retained_prior_output: proxy_service.JsonValue = { "type": "message", "id": "msg_owner", @@ -24252,6 +24275,7 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "call_id": "call_search", "execution": "client", "status": "completed", + "output": "found docs", "tools": [], "internal_chat_message_metadata_passthrough": owner_metadata, }, @@ -24268,10 +24292,17 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "instructions": "hi", "input": [ *historical_input, + retained_boundary_call, retained_boundary_output, *completed_search_bookkeeping, - *([] if unsafe_replay_input == "missing_prior_output" else [retained_prior_output]), - new_input, + *( + pending_tool_loop + if pending_manifest_replay + else [ + *([] if unsafe_replay_input == "missing_prior_output" else [retained_prior_output]), + new_input, + ] + ), *( [ { @@ -24290,6 +24321,16 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe if unsafe_replay_input == "conversation": payload_data["conversation"] = "conv_owner_scoped" payload = proxy_service.ResponsesRequest.model_validate(payload_data) + stored_context_items = ( + [ + *historical_input, + retained_boundary_call, + retained_boundary_output, + *completed_search_bookkeeping, + ] + if pending_manifest_replay + else historical_input + ) durable_lookup = proxy_service.DurableBridgeLookup( session_id="durable-owner-unavailable", canonical_kind="session_header", @@ -24302,9 +24343,10 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe state=HttpBridgeSessionState.ACTIVE, latest_turn_state="sid-owner-unavailable", latest_response_id="resp_completed_anchor", - latest_input_item_count=len(historical_input), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(historical_input), + latest_input_item_count=len(stored_context_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_context_items), model=stored_model, + latest_pending_tool_calls={"call_pending": "function_call"} if pending_manifest_replay else None, ) owner_unavailable = ProxyResponseError( 502, @@ -24339,9 +24381,7 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe replacement_session = _make_bridge_session(key=session.key, key_value=session.key.affinity_key) get_or_create = AsyncMock( side_effect=( - [owner_unavailable, session, replacement_session] - if replace_retired_gate - else [owner_unavailable, capacity_unavailable, session] + [owner_unavailable, session, replacement_session] if replace_retired_gate else [owner_unavailable, session] ) ) captured_request_states: list[proxy_service._WebSocketRequestState] = [] @@ -24452,13 +24492,13 @@ async def fake_stream_events( chunks = [chunk async for chunk in stream] assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert get_or_create.await_count == 3 + assert get_or_create.await_count == (3 if replace_retired_gate else 2) first_call = get_or_create.await_args_list[0] second_call = get_or_create.await_args_list[1] - third_call = get_or_create.await_args_list[2] + third_call = get_or_create.await_args_list[2] if replace_retired_gate else None assert first_call.kwargs["previous_response_id"] is None - assert first_call.kwargs["preferred_account_id"] == "acc-owner" - assert first_call.kwargs["allow_forward_to_owner"] is True + assert first_call.kwargs["preferred_account_id"] == (None if stored_model else "acc-owner") + assert first_call.kwargs["allow_forward_to_owner"] is (False if stored_model else True) assert second_call.kwargs["previous_response_id"] is None assert second_call.kwargs["preferred_account_id"] is None assert second_call.kwargs["durable_lookup"] is None @@ -24466,29 +24506,30 @@ async def fake_stream_events( kind=second_call.args[0].affinity_kind, key=second_call.args[0].affinity_key, ) - assert second_call.args[0] == third_call.args[0] + if third_call is not None: + assert second_call.args[0] == third_call.args[0] assert second_call.args[0] != first_call.args[0] assert second_call.kwargs["affinity"] == proxy_service._AffinityPolicy() assert second_call.kwargs["session_header_fallback_key"] is None - assert second_call.kwargs["exclude_account_ids"] == {"acc-owner"} + assert second_call.kwargs["exclude_account_ids"] == (None if stored_model else {"acc-owner"}) assert second_call.kwargs["allow_forward_to_owner"] is False assert all(key.lower() != "x-codex-turn-state" for key in second_call.kwargs["headers"]) - assert third_call.kwargs["previous_response_id"] is None - assert third_call.kwargs["preferred_account_id"] is None - assert third_call.kwargs["durable_lookup"] is None + if third_call is not None: + assert third_call.kwargs["previous_response_id"] is None + assert third_call.kwargs["preferred_account_id"] is None + assert third_call.kwargs["durable_lookup"] is None # When the fresh-replay session's own gate later times out (session.account # is "acc-fallback"), the next replacement must also exclude it — a # "replacement" that could legally reselect the account that just proved # stuck isn't a replacement at all. - assert third_call.kwargs["exclude_account_ids"] == ( - {"acc-owner", "acc-fallback"} if replace_retired_gate else {"acc-owner"} - ) - assert third_call.kwargs["allow_forward_to_owner"] is False + if third_call is not None: + assert third_call.kwargs["exclude_account_ids"] == {"acc-owner", "acc-fallback"} + assert third_call.kwargs["allow_forward_to_owner"] is False assert captured_request_states[0].previous_response_id is None assert captured_request_states[0].enforce_openai_sdk_contract is False replay_payload = json.loads(captured_text_data[0]) assert "previous_response_id" not in replay_payload - assert replay_payload["input"] == [ + expected_replay_input = [ { "role": "user", "content": [{"type": "input_text", "text": "old question"}], @@ -24508,19 +24549,44 @@ async def fake_stream_events( "internal_chat_message_metadata_passthrough": owner_metadata, }, { - "type": "message", - "role": "assistant", + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "docs"}, + "execution": "client", "status": "completed", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "old answer"}], "internal_chat_message_metadata_passthrough": owner_metadata, }, { - "role": "user", - "content": [{"type": "input_text", "text": "next question"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "output": "found docs", + "tools": [], + "internal_chat_message_metadata_passthrough": owner_metadata, }, ] + if pending_manifest_replay: + expected_replay_input.extend(pending_tool_loop) + else: + expected_replay_input.extend( + [ + { + "type": "message", + "role": "assistant", + "status": "completed", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "old answer"}], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "role": "user", + "content": [{"type": "input_text", "text": "next question"}], + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + ] + ) + assert replay_payload["input"] == expected_replay_input assert "encrypted_content" not in captured_text_data[0] assert all("id" not in item for item in replay_payload["input"]) account_neutral_classifier.assert_called_once() @@ -24805,6 +24871,393 @@ async def fail_first_session_before_output( assert all(call["preferred_account_has_continuity_provenance"] is True for call in creation_calls) +@pytest.mark.asyncio +async def test_durable_model_transition_full_resend_uses_account_neutral_replay( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner_metadata: proxy_service.JsonValue = {"turn_id": "turn-owner"} + historical_input: list[proxy_service.JsonValue] = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "old question"}], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call", + "id": "fc_owner", + "call_id": "call_old", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.3-codex-spark", + "instructions": "hi", + "input": [ + *historical_input, + { + "type": "function_call_output", + "call_id": "call_old", + "output": "old output", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "message", + "id": "msg_owner", + "role": "assistant", + "status": "completed", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "old answer"}], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "role": "user", + "content": [{"type": "input_text", "text": "next question"}], + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + ], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-full-resend", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id="instance-a", + owner_epoch=18, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_parent", + latest_response_id="resp_model_parent", + latest_input_item_count=len(historical_input), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(historical_input), + model="gpt-5.4-mini", + ) + captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + captured_kwargs: list[dict[str, Any]] = [] + captured_text_data: list[str] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + captured_keys.append(key) + captured_kwargs.append(kwargs) + session = _make_bridge_session(key=key, key_value=key.affinity_key) + session.account = cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)) + session.request_model = payload.model + return session + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + **_kwargs: Any, + ): + assert request_state.previous_response_id is None + assert request_state.preferred_account_id is None + captured_text_data.append(text_data) + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "authorization": "Bearer test-token", + "x-codex-session-id": "shared-root", + "x-codex-turn-state": "http_turn_child", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert len(captured_keys) == 1 + assert is_http_bridge_account_neutral_replay( + kind=captured_keys[0].affinity_kind, + key=captured_keys[0].affinity_key, + ) + assert captured_keys[0].strength == "soft" + assert captured_kwargs[0]["durable_lookup"] is None + assert captured_kwargs[0]["previous_response_id"] is None + assert captured_kwargs[0]["preferred_account_id"] is None + assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is False + assert captured_kwargs[0]["allow_forward_to_owner"] is False + assert captured_kwargs[0]["headers"] == {"authorization": "Bearer test-token"} + replay_payload = json.loads(captured_text_data[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["model"] == "gpt-5.3-codex-spark" + assert replay_payload["input"][-1]["content"] == [{"type": "input_text", "text": "next question"}] + assert all("id" not in item for item in replay_payload["input"]) + + +@pytest.mark.asyncio +async def test_durable_model_transition_full_resend_pending_tool_context_uses_account_neutral_replay( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + owner_metadata: proxy_service.JsonValue = {"turn_id": "turn-owner"} + stored_context_items: list[proxy_service.JsonValue] = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "old question"}], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call", + "id": "fc_owner", + "call_id": "call_old", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call_output", + "call_id": "call_old", + "output": "old output", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "tool_search_call", + "id": "tsc_owner", + "call_id": "call_search", + "arguments": {"query": "docs"}, + "execution": "client", + "status": "completed", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "output": "found docs", + "tools": [], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + ] + pending_tool_loop: list[proxy_service.JsonValue] = [ + { + "type": "function_call", + "id": "fc_pending", + "call_id": "call_pending", + "name": "lookup_pending", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + { + "type": "function_call_output", + "call_id": "call_pending", + "output": "fresh result", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + ] + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.3-codex-spark", + "instructions": "hi", + "input": [*stored_context_items, *pending_tool_loop], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-pending-tool", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id="instance-a", + owner_epoch=18, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_parent", + latest_response_id="resp_model_parent", + latest_input_item_count=len(stored_context_items), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_context_items), + latest_pending_tool_calls={"call_pending": "function_call"}, + model="gpt-5.4-mini", + ) + captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + captured_kwargs: list[dict[str, Any]] = [] + captured_text_data: list[str] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + captured_keys.append(key) + captured_kwargs.append(kwargs) + session = _make_bridge_session(key=key, key_value=key.affinity_key) + session.account = cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)) + session.request_model = payload.model + return session + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + *, + request_state: proxy_service._WebSocketRequestState, + text_data: str, + **_kwargs: Any, + ): + assert request_state.previous_response_id is None + assert request_state.preferred_account_id is None + captured_text_data.append(text_data) + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "authorization": "Bearer test-token", + "x-codex-session-id": "shared-root", + "x-codex-turn-state": "http_turn_child", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert len(captured_keys) == 1 + assert is_http_bridge_account_neutral_replay( + kind=captured_keys[0].affinity_kind, + key=captured_keys[0].affinity_key, + ) + assert captured_keys[0].strength == "soft" + assert captured_kwargs[0]["durable_lookup"] is None + assert captured_kwargs[0]["previous_response_id"] is None + assert captured_kwargs[0]["preferred_account_id"] is None + assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is False + assert captured_kwargs[0]["allow_forward_to_owner"] is False + replay_payload = json.loads(captured_text_data[0]) + assert "previous_response_id" not in replay_payload + assert replay_payload["model"] == "gpt-5.3-codex-spark" + assert replay_payload["input"] == [ + { + "role": "user", + "content": [{"type": "input_text", "text": "old question"}], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call", + "call_id": "call_old", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call_output", + "call_id": "call_old", + "output": "old output", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "docs"}, + "execution": "client", + "status": "completed", + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "output": "found docs", + "tools": [], + "internal_chat_message_metadata_passthrough": owner_metadata, + }, + { + "type": "function_call", + "call_id": "call_pending", + "name": "lookup_pending", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + { + "type": "function_call_output", + "call_id": "call_pending", + "output": "fresh result", + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, + }, + ] + assert all("id" not in item for item in replay_payload["input"]) + + @pytest.mark.asyncio async def test_stream_via_http_bridge_preserves_verified_replay_kind_for_durable_model_transition( monkeypatch: pytest.MonkeyPatch, @@ -30743,7 +31196,6 @@ async def test_retire_stale_pending_http_bridge_session_quarantines_wedged_reatt await service._retire_stale_pending_http_bridge_session( session, detail="response_create_gate_timeout_stuck_pending", - response_events_seen=wedged.response_event_count, ) assert session.quarantined is expect_quarantined @@ -30752,6 +31204,31 @@ async def test_retire_stale_pending_http_bridge_session_quarantines_wedged_reatt ) +@pytest.mark.asyncio +async def test_retire_stale_pending_http_bridge_session_recomputes_event_evidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + eventful = _make_wedged_reattach_request_state(request_id="req-direct-retire-eventful") + eventful.response_id = "resp_eventful_direct_retire" + eventful.latency_response_created_ms = 700 + session = _make_bridge_session( + key_value="quarantine-direct-retire-eventful", + pending_requests=deque([eventful]), + queued_request_count=1, + ) + record_failure = AsyncMock() + monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + + await service._retire_stale_pending_http_bridge_session( + session, + detail="response_create_gate_timeout_stuck_pending", + ) + + record_failure.assert_not_awaited() + + @pytest.mark.asyncio async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_reattach_gate_already_false( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index e17cf26c36..1553a34cf9 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -10,6 +10,7 @@ ) from app.modules.proxy.replay_safety import ( project_responses_input_for_account_neutral_fresh_replay, + responses_input_items_are_self_contained_fresh_replay, responses_input_suffix_matches_pending_tool_calls, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, @@ -151,6 +152,310 @@ def test_account_neutral_fresh_replay_accepts_self_contained_payloads( assert responses_payload_is_account_neutral_fresh_replay(payload) is True +def test_account_neutral_fresh_replay_accepts_compaction_context_item() -> None: + payload: dict[str, JsonValue] = { + "input": [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue"}], + }, + ], + } + + assert responses_payload_is_account_neutral_fresh_replay(payload) is True + + +def test_account_neutral_replay_projection_preserves_owner_bound_compaction_id_to_fail_closed() -> None: + input_items: list[JsonValue] = [ + { + "type": "compaction", + "id": "cmp_owner_a", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue"}], + }, + ] + + projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) + + assert projection is not None + assert projection.input_items[0] == { + "type": "compaction", + "id": "cmp_owner_a", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + } + assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is False + + +def test_account_neutral_replay_projection_accepts_compaction_without_owner_id() -> None: + input_items: list[JsonValue] = [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue"}], + }, + ] + + projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) + + assert projection is not None + assert projection.input_items[0] == { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + } + assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is True + + +def test_account_neutral_replay_projection_rejects_compaction_before_later_raw_prefix_bookkeeping() -> None: + input_items: list[JsonValue] = [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "web_search_call", + "id": "ws_owner_a", + "action": {"type": "search", "query": "codex-lb"}, + "status": "completed", + }, + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb"}, + "execution": "client", + "status": "completed", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "output": "result", + "status": "completed", + "tools": [], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue after compaction"}], + }, + ] + + assert project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=2) is None + + +def test_account_neutral_replay_projection_preserves_post_compact_tool_search_context_without_owner_id() -> None: + input_items: list[JsonValue] = [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "tool_search_call", + "id": "tsc_owner_a", + "call_id": "call_search", + "arguments": {"query": "codex-lb post compact replay"}, + "execution": "client", + "status": "completed", + }, + { + "type": "tool_search_output", + "id": "tso_owner_a", + "call_id": "call_search", + "output": "replay fix candidate", + "execution": "client", + "status": "completed", + "tools": [], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue after compaction"}], + }, + ] + + projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) + + assert projection is not None + assert projection.stored_prefix_count == 1 + assert projection.input_items == [ + { + "type": "compaction", + "status": "completed", + "encrypted_content": "encrypted-compact-context", + }, + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb post compact replay"}, + "execution": "client", + "status": "completed", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "output": "replay fix candidate", + "execution": "client", + "status": "completed", + "tools": [], + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue after compaction"}], + }, + ] + assert ( + responses_input_suffix_retains_prior_output( + projection.input_items, + stored_count=projection.stored_prefix_count, + ) + is True + ) + assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is True + + +def test_account_neutral_fresh_replay_rejects_post_compact_mixed_tool_suffix_before_user_followup() -> None: + input_items: list[JsonValue] = [ + {"type": "compaction", "status": "completed", "encrypted_content": "encrypted-compact-context"}, + { + "type": "function_call", + "call_id": "call_function", + "name": "lookup", + "arguments": "{}", + "status": "completed", + }, + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb post compact replay"}, + "execution": "client", + "status": "completed", + }, + {"type": "function_call_output", "call_id": "call_function", "output": "result", "status": "completed"}, + { + "type": "tool_search_output", + "call_id": "call_search", + "output": "search result", + "execution": "client", + "status": "completed", + "tools": [], + }, + {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ] + + assert responses_input_suffix_retains_prior_output(input_items, stored_count=1) is False + + +def test_account_neutral_fresh_replay_accepts_self_contained_tool_search_pair() -> None: + input_items: list[JsonValue] = [ + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb"}, + "status": "completed", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "output": "Found codex-lb", + "status": "completed", + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ] + + assert responses_input_items_are_self_contained_fresh_replay(input_items) is True + assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is True + + +def test_account_neutral_fresh_replay_accepts_tools_only_client_tool_search_output() -> None: + input_items: list[JsonValue] = [ + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb"}, + "execution": "client", + "status": "completed", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "tools": [], + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ] + + assert responses_input_items_are_self_contained_fresh_replay(input_items) is True + assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is True + + +@pytest.mark.parametrize( + "tool_search_output", + [ + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "server", + "status": "completed", + "tools": [], + "output": "Found codex-lb", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "tools": [{"type": "file_search", "vector_store_ids": ["vs_owner"]}], + "output": "Found codex-lb", + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "status": "completed", + "tools": [{"type": "function", "name": "lookup", "namespace": "private"}], + "output": "Found codex-lb", + }, + ], +) +def test_account_neutral_fresh_replay_rejects_account_scoped_tool_search_output( + tool_search_output: dict[str, JsonValue], +) -> None: + input_items: list[JsonValue] = [ + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "codex-lb"}, + "execution": "client", + "status": "completed", + }, + tool_search_output, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ] + + assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is False + + def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() -> None: metadata = {"turn_id": "turn_owner_a"} input_items: list[JsonValue] = [ @@ -195,8 +500,10 @@ def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() }, { "type": "tool_search_output", + "id": "tso_owner_a", "call_id": "call_search", "execution": "client", + "output": "search result", "status": "completed", "tools": [], "internal_chat_message_metadata_passthrough": metadata, @@ -250,6 +557,23 @@ def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() "status": "completed", "internal_chat_message_metadata_passthrough": metadata, }, + { + "type": "tool_search_call", + "call_id": "call_search", + "arguments": {"query": "github"}, + "execution": "client", + "status": "completed", + "internal_chat_message_metadata_passthrough": metadata, + }, + { + "type": "tool_search_output", + "call_id": "call_search", + "execution": "client", + "output": "search result", + "status": "completed", + "tools": [], + "internal_chat_message_metadata_passthrough": metadata, + }, { "type": "message", "role": "assistant", From 25d6374a8a671900a6bf4dc89f248e7834efad76 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 21:04:42 +0400 Subject: [PATCH 102/117] fix(proxy): report suppressed duplicate tool-call terminals (#1706) --- app/modules/proxy/_service/streaming/mixin.py | 4 +- .../proxy/_service/websocket/helpers.py | 2 +- app/modules/proxy/_service/websocket/mixin.py | 2 +- app/modules/proxy/service.py | 1 + .../.openspec.yaml | 2 + .../proposal.md | 14 +++ .../specs/responses-api-compat/spec.md | 22 +++++ .../tasks.md | 9 ++ tests/unit/test_proxy_utils.py | 93 ++++++++++++++++++- 9 files changed, 143 insertions(+), 6 deletions(-) create mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml create mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md create mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 5bab3d94b4..8bad159daa 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -921,11 +921,11 @@ async def _touch_api_key_reservation() -> None: event_type, ) = _facade()._build_rewritten_stream_response_failed_event( response_id=response_id, - error_code="stream_incomplete", + error_code=_facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE, error_message=_facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE, ) status = "error" - error_code = "stream_incomplete" + error_code = _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE error_message = _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE settlement.record_success = False settlement.account_health_error = False diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 83858cc645..4b39742cb2 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -1386,7 +1386,7 @@ def _rewrite_websocket_suppressed_duplicate_tool_call_completion_event( request_state: _WebSocketRequestState, ) -> tuple[OpenAIEvent | None, dict[str, JsonValue] | None, str | None, str]: rewritten_event_payload = response_failed_event( - "stream_incomplete", + _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE, _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE, error_type="server_error", response_id=_websocket_downstream_response_id(request_state), diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index 1c268f7ea9..a135ec6543 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -5989,7 +5989,7 @@ async def _finalize_websocket_request_state( "account_health_error_handled", False, ) - if request_state.suppressed_duplicate_tool_call and error_code == "stream_incomplete": + if request_state.suppressed_duplicate_tool_call: settlement.account_health_error = False if ( error_code == "stream_incomplete" diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 2f7447680d..5153935817 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -876,6 +876,7 @@ def _proxy_admission_wait_timeout_seconds(settings: Any | None = None) -> float: _SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE = ( "Suppressed duplicate side-effect tool call; upstream response cannot be continued safely." ) +_SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE = "duplicate_tool_call_replay_suppressed" _WEBSOCKET_PREVIOUS_RESPONSE_ACCOUNT_CACHE_LIMIT = 4096 _WEBSOCKET_CONTINUITY_CACHE_LIMIT = 4096 _SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE = "security_work_authorization_required" diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml new file mode 100644 index 0000000000..f774115be7 --- /dev/null +++ b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-20 diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md new file mode 100644 index 0000000000..3274bca3b8 --- /dev/null +++ b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md @@ -0,0 +1,14 @@ +# Report suppressed duplicate tool-call terminals + +## Why + +A duplicate side-effect tool-call replay cannot safely continue, but every +transport still needs to settle the request and report the same retryable +outcome. + +## What Changes + +- Emit a specific failed terminal instead of misclassifying the replay as an + incomplete upstream stream. +- Keep settlement, durable operation persistence, and account-health fencing + aligned across direct SSE, the HTTP bridge, and WebSocket clients. diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..300cb81617 --- /dev/null +++ b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md @@ -0,0 +1,22 @@ +# responses-api-compat Delta + +## ADDED Requirements + +### Requirement: Suppressed duplicate side-effect replays receive a retryable terminal failure + +When a replayed side-effecting tool call is suppressed and its upstream turn subsequently reports `response.completed`, the proxy MUST deliver a `response.failed` terminal with code `duplicate_tool_call_replay_suppressed`; it MUST use the downstream response id, treat the request as non-success, persist a terminal durable HTTP-bridge operation when that transport is used, and MUST NOT penalize the upstream account for the intentionally fenced replay. + +#### Scenario: HTTP bridge client receives a terminal failure + +- **GIVEN** an HTTP bridge request suppresses a replayed side-effecting tool call +- **WHEN** the upstream emits `response.completed` for that replay +- **THEN** the client receives `response.failed` with code `duplicate_tool_call_replay_suppressed` +- **AND** the bridge operation is terminal rather than left pending +- **AND** the request is recorded as non-success + +#### Scenario: WebSocket and direct SSE have equivalent terminal semantics + +- **GIVEN** either a WebSocket or direct SSE request suppresses a replayed side-effecting tool call +- **WHEN** the upstream emits `response.completed` for that replay +- **THEN** the client receives the same `response.failed` code +- **AND** the upstream account is not penalized for the intentionally suppressed replay diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md new file mode 100644 index 0000000000..3389916cb9 --- /dev/null +++ b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md @@ -0,0 +1,9 @@ +## 1. Implementation + +- [x] 1.1 Emit the explicit duplicate-tool-call replay failure from all terminal paths. +- [x] 1.2 Preserve non-success settlement and account-health fencing. + +## 2. Validation + +- [x] 2.1 Run focused HTTP bridge and WebSocket regression tests. +- [x] 2.2 Run strict OpenSpec validation. diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 3984cf0939..cbbbb86238 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -17138,10 +17138,10 @@ async def fake_stream(*_, **__): assert terminal_event["type"] == "response.failed" terminal_response = cast(dict[str, JsonValue], terminal_event["response"]) terminal_error = cast(dict[str, JsonValue], terminal_response["error"]) - assert terminal_error["code"] == "stream_incomplete" + assert terminal_error["code"] == "duplicate_tool_call_replay_suppressed" assert await service.drain_persistence_tasks(timeout_seconds=1) assert request_logs.calls[0]["status"] == "error" - assert request_logs.calls[0]["error_code"] == "stream_incomplete" + assert request_logs.calls[0]["error_code"] == "duplicate_tool_call_replay_suppressed" @pytest.mark.asyncio @@ -44260,6 +44260,95 @@ async def test_http_bridge_tool_call_dedupe_survives_upstream_reconnect(): assert event_queue.empty() +@pytest.mark.asyncio +async def test_http_bridge_duplicate_tool_call_replay_emits_retryable_terminal_failure(): + request_logs = _RequestLogsRecorder() + service = proxy_service.ProxyService(_repo_factory(request_logs)) + request_state = proxy_service._WebSocketRequestState( + request_id="req_bridge_duplicate_terminal", + model="gpt-5.1", + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=0.0, + response_id="resp_bridge_duplicate_terminal", + event_queue=asyncio.Queue(), + request_text='{"type":"response.create"}', + transport="http", + ) + session = proxy_service._HTTPBridgeSession( + key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-duplicate-terminal", None), + headers={}, + affinity=proxy_service._AffinityPolicy(), + request_model="gpt-5.1", + account=_make_account("acc_bridge_duplicate_terminal"), + upstream=AsyncMock(), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque([request_state]), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=1, + last_used_at=0.0, + idle_ttl_seconds=30.0, + ) + cast(Any, session.upstream).archive_received = MagicMock() + tool_payload = { + "type": "response.output_item.done", + "response_id": "resp_bridge_duplicate_terminal", + "item": { + "type": "function_call", + "name": "write_stdin", + "arguments": json.dumps({"session_id": 75180, "chars": ""}), + "call_id": "call_first", + }, + } + replay_payload = { + **tool_payload, + "response_id": "resp_bridge_duplicate_terminal_replay", + "item": {**tool_payload["item"], "call_id": "call_replayed"}, + } + replay_created_payload = { + "type": "response.created", + "response": {"id": "resp_bridge_duplicate_terminal_replay", "status": "in_progress"}, + } + completed_payload = { + "type": "response.completed", + "response": {"id": "resp_bridge_duplicate_terminal_replay", "status": "completed", "output": []}, + } + + await service._process_http_bridge_upstream_text(session, json.dumps(tool_payload, separators=(",", ":"))) + session.upstream_control = proxy_service._WebSocketUpstreamControl() + request_state.awaiting_response_created = True + request_state.response_id = None + await service._process_http_bridge_upstream_text(session, json.dumps(replay_created_payload, separators=(",", ":"))) + await service._process_http_bridge_upstream_text(session, json.dumps(replay_payload, separators=(",", ":"))) + await service._process_http_bridge_upstream_text(session, json.dumps(completed_payload, separators=(",", ":"))) + + event_queue = request_state.event_queue + assert event_queue is not None + first = await event_queue.get() + created = await event_queue.get() + terminal_block = await event_queue.get() + assert isinstance(first, str) + assert isinstance(created, str) + assert isinstance(terminal_block, str) + assert proxy_service.parse_sse_data_json(first) == tool_payload + assert proxy_service.parse_sse_data_json(created) == replay_created_payload + terminal = proxy_service.parse_sse_data_json(terminal_block) + assert isinstance(terminal, dict) + assert terminal["type"] == "response.failed" + terminal_response = terminal["response"] + assert isinstance(terminal_response, dict) + terminal_error = terminal_response["error"] + assert isinstance(terminal_error, dict) + assert terminal_error["code"] == "duplicate_tool_call_replay_suppressed" + assert await event_queue.get() is None + assert request_state.error_http_status_override == 502 + assert session.upstream_control.reconnect_requested is True + assert await service.drain_persistence_tasks(timeout_seconds=1) + assert request_logs.calls[-1]["status"] == "error" + + @pytest.mark.asyncio async def test_http_bridge_session_events_emit_keepalive_while_pending(monkeypatch): request_logs = _RequestLogsRecorder() From b6c217fada24c8a7b7e2c77af6bce3e2d7a2d3e2 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 21:15:21 +0400 Subject: [PATCH 103/117] fix(db): repair retired identity/warmup migration stamp (#1847) * fix(db): repair retired identity/warmup stamp * fix(db): repair partial file-account-pin migration --- app/db/alembic/revision_ids.py | 1 + .../20260813_000000_add_file_account_pins.py | 25 ++-- ...epair_retired_identity_and_warmup_stamp.py | 77 ++++++++++++ .../proposal.md | 28 +++++ .../specs/database-migrations/spec.md | 23 ++++ .../tasks.md | 9 ++ tests/integration/test_migrations.py | 117 ++++++++++++++++++ 7 files changed, 270 insertions(+), 10 deletions(-) create mode 100644 app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py create mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/proposal.md create mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md create mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/tasks.md diff --git a/app/db/alembic/revision_ids.py b/app/db/alembic/revision_ids.py index c5eed87ef1..3e8c4a75e2 100644 --- a/app/db/alembic/revision_ids.py +++ b/app/db/alembic/revision_ids.py @@ -27,6 +27,7 @@ ), "20260410_020000_restore_import_without_overwrite_default_false": "20260409_020000_fix_http_bridge_last_seen_index", "20260525_000000_merge_routing_settings_security_heads": "20260513_000000_add_accounts_alias", + "20260814_020000_merge_identity_and_warmup_heads": "20260816_000000_add_model_source_embeddings", } NEW_TO_OLD_REVISION_MAP: dict[str, str] = {new: old for old, new in OLD_TO_NEW_REVISION_MAP.items()} diff --git a/app/db/alembic/versions/20260813_000000_add_file_account_pins.py b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py index dbfd6f4080..4a0b1da5b0 100644 --- a/app/db/alembic/versions/20260813_000000_add_file_account_pins.py +++ b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py @@ -20,16 +20,21 @@ def upgrade() -> None: bind = op.get_bind() - if sa.inspect(bind).has_table(_TABLE): - return - op.create_table( - _TABLE, - sa.Column("file_id", sa.String(), nullable=False), - sa.Column("account_id", sa.String(), nullable=False), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint("file_id"), - ) - op.create_index("ix_file_account_pins_expires_at", _TABLE, ["expires_at"], unique=False) + inspector = sa.inspect(bind) + table_exists = inspector.has_table(_TABLE) + if not table_exists: + op.create_table( + _TABLE, + sa.Column("file_id", sa.String(), nullable=False), + sa.Column("account_id", sa.String(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("file_id"), + ) + index_exists = table_exists and "ix_file_account_pins_expires_at" in { + index["name"] for index in inspector.get_indexes(_TABLE) if index.get("name") + } + if not index_exists: + op.create_index("ix_file_account_pins_expires_at", _TABLE, ["expires_at"], unique=False) def downgrade() -> None: diff --git a/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py b/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py new file mode 100644 index 0000000000..130a1a835d --- /dev/null +++ b/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py @@ -0,0 +1,77 @@ +"""repair schemas stamped at the retired identity/warmup merge head + +Revision ID: 20260820_000000_repair_retired_identity_and_warmup_stamp +Revises: 20260816_000000_add_model_source_embeddings +Create Date: 2026-08-20 + +Some local August 14, 2026 builds emitted the no-op merge stamp +``20260814_020000_merge_identity_and_warmup_heads`` even when the current +mainline file-pin, sticky-abandonment-scope, pending-deletion, API-key +reasoning-policy, and model-source-embeddings lineage had never run, while the +retired account-identity index and quota-planner lease-expiry column were +still present. Startup remaps that dead stamp to the current pre-repair head so +Alembic can continue; this forward-only repair step then replays the guarded +current migrations and drops the two stale artifacts. +""" + +from __future__ import annotations + +import importlib +from types import ModuleType + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.engine import Connection + +revision = "20260820_000000_repair_retired_identity_and_warmup_stamp" +down_revision = "20260816_000000_add_model_source_embeddings" +branch_labels = None +depends_on = None + +_OBSOLETE_ACCOUNT_INDEX = "idx_accounts_chatgpt_account_id" +_OBSOLETE_QUOTA_COLUMN = "lease_expires_at" + + +def _migration(module_name: str) -> ModuleType: + return importlib.import_module(f"app.db.alembic.versions.{module_name}") + + +def _has_table(connection: Connection, table_name: str) -> bool: + return sa.inspect(connection).has_table(table_name) + + +def _columns(connection: Connection, table_name: str) -> set[str]: + if not _has_table(connection, table_name): + return set() + return {column["name"] for column in sa.inspect(connection).get_columns(table_name)} + + +def _indexes(connection: Connection, table_name: str) -> set[str]: + if not _has_table(connection, table_name): + return set() + names = (index.get("name") for index in sa.inspect(connection).get_indexes(table_name)) + return {name for name in names if name is not None} + + +def upgrade() -> None: + _migration("20260813_000000_add_file_account_pins").upgrade() + _migration("20260812_120000_add_sticky_abandonment_scope").upgrade() + _migration("20260816_000000_add_account_pending_deletion").upgrade() + _migration("20260806_030000_add_api_key_allowed_reasoning_efforts").upgrade() + _migration("20260816_000000_add_model_source_embeddings").upgrade() + + connection = op.get_bind() + + if _OBSOLETE_ACCOUNT_INDEX in _indexes(connection, "accounts"): + op.drop_index(_OBSOLETE_ACCOUNT_INDEX, table_name="accounts", if_exists=True) + + if _OBSOLETE_QUOTA_COLUMN in _columns(connection, "quota_planner_decisions"): + with op.batch_alter_table("quota_planner_decisions") as batch_op: + batch_op.drop_column(_OBSOLETE_QUOTA_COLUMN) + + +def downgrade() -> None: + # This revision repairs databases carrying a retired local merge stamp. It + # must not resurrect the stale index/column or remove objects owned by the + # canonical current-main migrations it replays above. + pass diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md b/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md new file mode 100644 index 0000000000..2b7a6fe58f --- /dev/null +++ b/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md @@ -0,0 +1,28 @@ +## Why + +Local August 14, 2026 builds could stamp SQLite databases at the retired +`20260814_020000_merge_identity_and_warmup_heads` merge id even when the +current mainline file-pin, sticky-abandonment-scope, pending-deletion, +API-key reasoning-policy, and model-source-embeddings lineage had never run. +Those databases also kept two artifacts current main no longer owns: +`idx_accounts_chatgpt_account_id` and +`quota_planner_decisions.lease_expires_at`. Current main therefore treats the +stamp as schema-ahead and a drift check on the live clone reports eleven schema +diffs. + +## What Changes + +- Auto-remap the retired merge stamp to the current pre-repair Alembic head so + upgrade can continue through normal `python -m app.db.migrate upgrade`. +- Add one forward-only repair migration that replays the guarded current-main + migrations needed by the stamped-local shape and drops the two stale + artifacts when present. +- Cover the representative SQLite shape with a regression test that proves the + repair converges to the current ORM schema. + +## Capabilities + +### Modified Capabilities + +- `database-migrations`: retired local merge stamps upgrade cleanly to the + current schema without manual restamping. diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md b/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md new file mode 100644 index 0000000000..bf033e7b54 --- /dev/null +++ b/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md @@ -0,0 +1,23 @@ +## ADDED Requirements + +### Requirement: Retired identity/warmup merge stamps repair to current schema + +The system MUST upgrade a database stamped at +`20260814_020000_merge_identity_and_warmup_heads` by the retired local merge +build to the current Alembic head without manual restamping. Startup or CLI +remap MAY rewrite that retired stamp to the canonical pre-repair revision, but +the subsequent upgrade MUST execute a forward repair that converges the schema +to ORM metadata. The repaired schema MUST add the file-pin, +sticky-abandonment-scope, pending-deletion, API-key reasoning-policy, and +model-source-embeddings objects current main expects, and MUST remove the +retired `idx_accounts_chatgpt_account_id` index and +`quota_planner_decisions.lease_expires_at` column if they are still present. + +#### Scenario: A SQLite clone stamped at the retired merge head upgrades cleanly + +- **GIVEN** a SQLite database stamped at `20260814_020000_merge_identity_and_warmup_heads` +- **AND** the schema still lacks `file_account_pins`, pending-deletion markers, API-key reasoning policy, model-source embeddings, and sticky abandonment scope +- **AND** the retired `idx_accounts_chatgpt_account_id` index and `quota_planner_decisions.lease_expires_at` column are still present +- **WHEN** startup or `python -m app.db.migrate upgrade` runs to head +- **THEN** the upgrade completes without manual stamp surgery +- **AND** `python -m app.db.migrate check` reports no schema drift diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md b/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md new file mode 100644 index 0000000000..24d18a21a4 --- /dev/null +++ b/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md @@ -0,0 +1,9 @@ +## 1. Repair Path + +- [x] 1.1 Remap `20260814_020000_merge_identity_and_warmup_heads` to the current pre-repair Alembic revision +- [x] 1.2 Add a forward-only repair migration that replays the missing current-main migrations and removes `idx_accounts_chatgpt_account_id` plus `quota_planner_decisions.lease_expires_at` when present + +## 2. Validation + +- [x] 2.1 Add a regression test for the representative SQLite drift shape stamped at the retired merge head +- [x] 2.2 Validate OpenSpec and the focused migration tests diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index df70df1b78..cec5bc7a24 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1908,3 +1908,120 @@ def _schema_state(sync_conn): assert await conn.run_sync(_schema_state) is not None finally: await engine.dispose() + + +@pytest.mark.asyncio +async def test_file_account_pins_migration_repairs_existing_table_missing_index(tmp_path): + from sqlalchemy import inspect as sa_inspect + + db_url = f"sqlite+aiosqlite:///{tmp_path / 'partial-file-account-pins.sqlite'}" + parent_revision = "20260806_000000_add_anonymous_telemetry" + pin_revision = "20260813_000000_add_file_account_pins" + + await to_thread.run_sync(lambda: run_upgrade(db_url, parent_revision, bootstrap_legacy=False)) + engine = create_async_engine(db_url, future=True) + try: + async with engine.begin() as conn: + await conn.execute( + text( + "CREATE TABLE file_account_pins (" + "file_id VARCHAR NOT NULL PRIMARY KEY, " + "account_id VARCHAR NOT NULL, " + "expires_at DATETIME NOT NULL)" + ) + ) + + await to_thread.run_sync(lambda: run_upgrade(db_url, pin_revision, bootstrap_legacy=False)) + await to_thread.run_sync(lambda: run_upgrade(db_url, pin_revision, bootstrap_legacy=False)) + async with engine.connect() as conn: + indexes = await conn.run_sync( + lambda sync_conn: {index["name"] for index in sa_inspect(sync_conn).get_indexes("file_account_pins")} + ) + assert indexes == {"ix_file_account_pins_expires_at"} + await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + assert check_schema_drift(db_url) == () + finally: + await engine.dispose() + + +@pytest.mark.asyncio +async def test_retired_identity_and_warmup_merge_stamp_repairs_to_head(tmp_path): + from alembic import command + from sqlalchemy import inspect as sa_inspect + + from app.db.migrate import _build_alembic_config + + db_url = f"sqlite+aiosqlite:///{tmp_path / 'retired-identity-warmup-repair.sqlite'}" + pre_repair_head = "20260816_000000_add_model_source_embeddings" + retired_merge_revision = "20260814_020000_merge_identity_and_warmup_heads" + expected_drift_checks = ( + ("add_table", "file_account_pins"), + ("add_index", "ix_file_account_pins_expires_at"), + ("add_column", "accounts', Column('delete_requested_at'"), + ("add_column", "accounts', Column('delete_history_requested'"), + ("remove_index", "idx_accounts_chatgpt_account_id"), + ("add_index", "idx_accounts_delete_requested_at"), + ("add_column", "api_keys', Column('allowed_reasoning_efforts'"), + ("add_constraint", "ck_api_keys_reasoning_policy_exclusive"), + ("add_column", "model_sources', Column('supports_embeddings'"), + ("remove_column", "quota_planner_decisions', Column('lease_expires_at'"), + ("add_column", "sticky_sessions', Column('continuity_abandonment_scope'"), + ) + + def _schema_state(sync_conn): + inspector = sa_inspect(sync_conn) + return { + "has_file_account_pins": inspector.has_table("file_account_pins"), + "account_columns": {column["name"] for column in inspector.get_columns("accounts")}, + "account_indexes": {index["name"] for index in inspector.get_indexes("accounts")}, + "api_key_columns": {column["name"] for column in inspector.get_columns("api_keys")}, + "api_key_checks": { + constraint["name"] + for constraint in inspector.get_check_constraints("api_keys") + if constraint.get("name") + }, + "model_source_columns": {column["name"] for column in inspector.get_columns("model_sources")}, + "quota_columns": {column["name"] for column in inspector.get_columns("quota_planner_decisions")}, + "sticky_columns": {column["name"] for column in inspector.get_columns("sticky_sessions")}, + } + + await to_thread.run_sync(lambda: run_upgrade(db_url, pre_repair_head, bootstrap_legacy=False)) + await to_thread.run_sync( + lambda: command.downgrade(_build_alembic_config(db_url), "20260806_000000_add_anonymous_telemetry") + ) + + engine = create_async_engine(db_url, future=True) + try: + async with engine.begin() as conn: + await conn.execute( + text("CREATE INDEX IF NOT EXISTS idx_accounts_chatgpt_account_id ON accounts (chatgpt_account_id)") + ) + await conn.execute(text("ALTER TABLE quota_planner_decisions ADD COLUMN lease_expires_at DATETIME")) + await conn.execute( + text("UPDATE alembic_version SET version_num = :revision"), + {"revision": retired_merge_revision}, + ) + + drift = check_schema_drift(db_url) + assert len(drift) == len(expected_drift_checks) + for action, marker in expected_drift_checks: + assert any(action in diff and marker in diff for diff in drift) + + result = await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) + assert result.current_revision == _HEAD_REVISION + assert check_schema_drift(db_url) == () + + async with engine.connect() as conn: + state = await conn.run_sync(_schema_state) + assert state["has_file_account_pins"] is True + assert "delete_requested_at" in state["account_columns"] + assert "delete_history_requested" in state["account_columns"] + assert "idx_accounts_chatgpt_account_id" not in state["account_indexes"] + assert "idx_accounts_delete_requested_at" in state["account_indexes"] + assert "allowed_reasoning_efforts" in state["api_key_columns"] + assert "ck_api_keys_reasoning_policy_exclusive" in state["api_key_checks"] + assert "supports_embeddings" in state["model_source_columns"] + assert "lease_expires_at" not in state["quota_columns"] + assert "continuity_abandonment_scope" in state["sticky_columns"] + finally: + await engine.dispose() From 5e1f568f3a2772b4d1162a5e325d056cae7daa39 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 22:19:19 +0400 Subject: [PATCH 104/117] fix(proxy): demote quarantined bridge reattach keys (#1730) A reattach that proved silent/wedged quarantines its session key, but the quarantine gate only cleared two booleans. `bridge_session_key` was still re-adopted from `durable_lookup.canonical_kind/canonical_key`, and the session was still created under that same poisoned key, so for the whole 600s `_HTTP_BRIDGE_QUARANTINE_TTL_SECONDS` window every request rebuilt a fresh wedged bridge on the key that was already known to be dead. Demote the key instead of only dropping the anchor. When a quarantined key carries a sealed durable full-resend proof that matches the payload, the request dispatches on a soft account-neutral replay key with the client's own full conversation, and the original key's continuity is advanced only after that recovery actually completes. - `quarantine.py`: entries carry a `generation`, and clearing takes a key plus an expected generation, so a late recovery cannot clear a quarantine that was re-armed underneath it. - `upstream_events.py`: on a completed response, rebind and renew the original durable row before clearing its quarantine, so continuity is not stranded on the throwaway recovery key. - `streaming.py`: the quarantine gate demotes the key; the recovery key is named after the body this dispatch actually sends. `durable_recovery_attempt_fingerprint` deliberately keeps hashing the unprojected body. It is the key of persisted `http_bridge_recovery_attempts` rows, so hashing the projected body instead would mint a different fingerprint, a row journalled before a restart would stop matching, and the one-shot replay fence would silently open once. Covered by `test_quarantined_full_resend_recovery_fence_survives_restart_fingerprint`. --- .../proxy/_service/http_bridge/quarantine.py | 46 +- .../proxy/_service/http_bridge/streaming.py | 63 ++- .../_service/http_bridge/upstream_events.py | 105 +++- app/modules/proxy/_service/support.py | 2 + .../integration/test_http_responses_bridge.py | 17 +- tests/unit/test_durable_bridge_sessions.py | 48 ++ tests/unit/test_proxy_http_bridge.py | 472 ++++++++++++++++-- 7 files changed, 699 insertions(+), 54 deletions(-) diff --git a/app/modules/proxy/_service/http_bridge/quarantine.py b/app/modules/proxy/_service/http_bridge/quarantine.py index ff47ea8b9f..1721f0c86e 100644 --- a/app/modules/proxy/_service/http_bridge/quarantine.py +++ b/app/modules/proxy/_service/http_bridge/quarantine.py @@ -43,6 +43,7 @@ class _HTTPBridgeQuarantineEntry: consecutive_eventless_timeouts: int = 0 last_touched_monotonic: float = 0.0 reason: str | None = None + generation: int = 0 def _http_bridge_quarantine_registry( @@ -100,6 +101,16 @@ def _http_bridge_session_key_quarantined(service: Any, key: _HTTPBridgeSessionKe return entry is not None and entry.quarantined_until > now +def _http_bridge_session_key_quarantine_generation(service: Any, key: _HTTPBridgeSessionKey) -> int | None: + registry = _http_bridge_quarantine_registry(service) + now = time.monotonic() + _prune_http_bridge_quarantine_registry(registry, now) + entry = registry.get(key) + if entry is None or entry.quarantined_until <= now: + return None + return entry.generation + + def _quarantine_http_bridge_session(service: Any, session: _HTTPBridgeSession, *, reason: str) -> None: """Quarantine a bridge session that has proven silent/wedged. @@ -113,6 +124,7 @@ def _quarantine_http_bridge_session(service: Any, session: _HTTPBridgeSession, * entry.quarantined_until = max(entry.quarantined_until, now + _HTTP_BRIDGE_QUARANTINE_TTL_SECONDS) entry.last_touched_monotonic = now entry.reason = reason + entry.generation += 1 _prune_http_bridge_quarantine_registry(registry, now) session.quarantined = True if already_quarantined: @@ -169,21 +181,41 @@ def _record_http_bridge_quarantine_eventless_timeout(service: Any, session: _HTT ) -def _clear_http_bridge_quarantine(service: Any, session: _HTTPBridgeSession) -> None: - """A completed response on this key disproves the wedge; drop all state.""" +def _clear_http_bridge_quarantine_key( + service: Any, + key: _HTTPBridgeSessionKey, + *, + account_id: str | None, + model: str | None, + generation: int | None = None, +) -> None: + """A completed response on a recovery key disproves the original wedge.""" registry = _http_bridge_quarantine_registry(service) - session.quarantined = False - entry = registry.pop(session.key, None) + entry = registry.get(key) if entry is None: return + if generation is not None and entry.generation != generation: + return + registry.pop(key, None) if entry.quarantined_until <= time.monotonic(): return _log_http_bridge_event( "session_quarantine_cleared", + key, + account_id=account_id, + model=model, + detail=f"reason={entry.reason}", + cache_key_family=key.affinity_kind, + model_class=_extract_model_class(model) if model else None, + ) + + +def _clear_http_bridge_quarantine(service: Any, session: _HTTPBridgeSession) -> None: + """A completed response on this key disproves the wedge; drop all state.""" + session.quarantined = False + _clear_http_bridge_quarantine_key( + service, session.key, account_id=session.account.id, model=session.request_model, - detail=f"reason={entry.reason}", - cache_key_family=session.key.affinity_kind, - model_class=_extract_model_class(session.request_model) if session.request_model else None, ) diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 631f667c28..40d81f2cb0 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -109,7 +109,7 @@ _owner_forward_failure_allows_local_recovery, ) from app.modules.proxy._service.http_bridge.quarantine import ( - _http_bridge_session_key_quarantined, + _http_bridge_session_key_quarantine_generation, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _build_rewritten_stream_response_failed_event, @@ -1352,6 +1352,8 @@ async def release_unowned_bridge_lifecycle( # dispatch genuinely goes unanchored instead of rebuilding the same # wedged reattach through the session-state side door. fresh_reattach_anchor_suppressed_quarantined = False + fresh_reattach_quarantine_clear_key: _HTTPBridgeSessionKey | None = None + fresh_reattach_quarantine_clear_generation: int | None = None def classify_durable_full_resend( lookup: DurableBridgeLookup, @@ -1421,6 +1423,12 @@ def classify_durable_full_resend( durable_full_resend_is_account_neutral = _http_bridge_payload_is_account_neutral_fresh_replay( durable_full_resend_fresh_payload ) + # The durable recovery-attempt fence keys persisted + # ``http_bridge_recovery_attempts`` rows. Hash the same + # unprojected body main has always hashed: a projected body + # would mint a different fingerprint, so a row written + # before a restart would stop matching and the one-shot + # replay fence would silently open once. _fresh_state, fresh_replay_text = prepare_bridge_request( _http_bridge_payload_without_previous_response_id(payload) ) @@ -1585,6 +1593,12 @@ def classify_durable_full_resend( replay_kind, replay_key, bridge_session_key.api_key_id, + # This is a one-shot fresh recovery dispatch after the + # original hard key was quarantined. Keep the + # account-neutral marker for replay guards, but do not hash + # the random recovery key through durable hard-owner + # routing before the wedged upstream proof can run. + strength="soft", ) force_local_recovery_creation = True durable_lookup = None @@ -1611,13 +1625,21 @@ def classify_durable_full_resend( and durable_lookup.latest_response_id is not None and (not payload_looks_like_full_resend or durable_anchor_trimmable) ) - if payload_looks_like_full_resend and _http_bridge_session_key_quarantined(self, bridge_session_key): + quarantine_generation = ( + _http_bridge_session_key_quarantine_generation(self, bridge_session_key) + if payload_looks_like_full_resend + else None + ) + if quarantine_generation is not None: # The previous attach on this key proved silent/wedged # (#1534). The client's own payload already carries the full # conversation, so send it unanchored on the fresh path - # instead of rebuilding the same reattach. Delta-only - # payloads keep the anchor: it is their only way to convey - # prior context (same boundary as the fenced anchor clear). + # instead of rebuilding the same reattach. Only cross accounts + # once the sealed durable full-resend proof matches this + # payload; otherwise keep the durable owner while dropping the + # poisoned anchor. Delta-only payloads keep the anchor: it is + # their only way to convey prior context (same boundary as the + # fenced anchor clear). # Evaluated independently of the fresh-reattach eligibility # above: even when that gate is already false (for example a # conversation-scoped payload, a live alias session, or an @@ -1627,6 +1649,35 @@ def classify_durable_full_resend( # paths below. fresh_reattach_can_use_durable_anchor = False fresh_reattach_anchor_suppressed_quarantined = True + if ( + durable_full_resend_proof is not None + and durable_full_resend_proof.matches(payload, durable_lookup) + and durable_full_resend_fresh_payload is not None + and durable_full_resend_is_account_neutral is True + ): + effective_payload = durable_full_resend_fresh_payload + untrimmed_effective_payload = durable_full_resend_fresh_payload + fresh_reattach_quarantine_clear_key = bridge_session_key + fresh_reattach_quarantine_clear_generation = quarantine_generation + # Name the recovery key after the body this dispatch + # actually sends, not after the recovery-attempt fence + # fingerprint: the two hash different bodies and must + # not be conflated. + _quarantine_replay_state, quarantine_replay_text = prepare_bridge_request( + durable_full_resend_fresh_payload + ) + del _quarantine_replay_state + replay_nonce = durable_bridge_hash(quarantine_replay_text) + replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(replay_nonce) + bridge_session_key = _HTTPBridgeSessionKey( + replay_kind, + replay_key, + bridge_session_key.api_key_id, + strength="soft", + ) + force_local_recovery_creation = True + incoming_session_header = None + session_header_fallback_key = None _log_http_bridge_event( "fresh_reattach_anchor_skipped_quarantined", bridge_session_key, @@ -1731,6 +1782,8 @@ def classify_durable_full_resend( request_state, text_data = prepare_bridge_request(effective_payload) request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity + request_state.quarantine_clear_key = fresh_reattach_quarantine_clear_key + request_state.quarantine_clear_generation = fresh_reattach_quarantine_clear_generation _apply_http_bridge_downstream_turn_state( request_state, downstream_turn_state=downstream_turn_state, diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 043e1bef46..5defdc825d 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -3,7 +3,7 @@ import asyncio import logging import time -from collections.abc import Awaitable, Callable +from collections.abc import Awaitable, Callable, Mapping from dataclasses import replace from typing import Any, TypeVar, cast @@ -49,6 +49,7 @@ from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text from app.core.utils.request_id import reset_request_id, set_request_id from app.core.utils.sse import format_sse_event, parse_sse_data_json +from app.core.utils.time import utcnow from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -72,6 +73,7 @@ ) from app.modules.proxy._service.http_bridge.quarantine import ( _clear_http_bridge_quarantine, + _clear_http_bridge_quarantine_key, _record_http_bridge_quarantine_eventless_timeout, _record_http_bridge_quarantine_wedged_pending, ) @@ -195,6 +197,7 @@ _extract_model_class, ) from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay +from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.helpers import ( _normalize_error_code, is_upstream_model_capacity_error, @@ -209,6 +212,74 @@ logger = logging.getLogger("app.modules.proxy.service") + +async def _advance_http_bridge_quarantine_clear_key( + service: Any, + *, + key: Any, + api_key_id: str | None, + account_id: str, + response_id: str, + input_item_count: int | None, + input_full_fingerprint: str | None, + pending_tool_calls: Mapping[str, str] | None, +) -> bool: + lookup = await service._durable_bridge.lookup_request_targets( + session_key_kind=key.affinity_kind, + session_key_value=key.affinity_key, + api_key_id=api_key_id, + turn_state=None, + session_header=None, + previous_response_id=None, + ) + if lookup is None: + return False + settings = _service_get_settings() + instance_id = settings.http_responses_session_bridge_instance_id + active_lookup = lookup + if not active_lookup.lease_is_active(now=utcnow()) or active_lookup.owner_instance_id != instance_id: + claimed_lookup = await service._durable_bridge.claim_live_session( + session_key_kind=key.affinity_kind, + session_key_value=key.affinity_key, + api_key_id=api_key_id, + instance_id=instance_id, + lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), + account_id=active_lookup.account_id, + model=active_lookup.model, + service_tier=None, + latest_turn_state=active_lookup.latest_turn_state, + latest_response_id=active_lookup.latest_response_id, + allow_takeover=False, + owner_process_epoch=http_bridge_owner_process_epoch(), + ) + if claimed_lookup.owner_instance_id != instance_id: + return False + active_lookup = claimed_lookup + if active_lookup.account_id != account_id: + rebound = await service._durable_bridge.rebind_session_account( + session_id=active_lookup.session_id, + api_key_id=api_key_id, + instance_id=instance_id, + owner_epoch=active_lookup.owner_epoch, + account_id=account_id, + clear_continuity=True, + ) + if not rebound: + return False + advanced = await service._durable_bridge.renew_live_session( + session_id=active_lookup.session_id, + api_key_id=api_key_id, + instance_id=instance_id, + owner_epoch=active_lookup.owner_epoch, + lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), + latest_response_id=response_id, + latest_input_item_count=input_item_count, + latest_input_full_fingerprint=input_full_fingerprint, + latest_pending_tool_calls=pending_tool_calls, + ) + return advanced is not None + + _HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS = ( 0.25, 0.5, @@ -2783,6 +2854,38 @@ async def persist_grouped_terminal_events() -> Exception | None: ): await self._clear_http_bridge_retry_circuit(session) _clear_http_bridge_quarantine(self, session) + if terminal_request_state.quarantine_clear_key is not None: + quarantine_advanced = False + if response_id is not None: + try: + quarantine_advanced = await _advance_http_bridge_quarantine_clear_key( + self, + key=terminal_request_state.quarantine_clear_key, + api_key_id=session.key.api_key_id, + account_id=session.account.id, + response_id=response_id, + input_item_count=( + terminal_request_state.input_item_count + if terminal_request_state.input_item_count > 0 + else None + ), + input_full_fingerprint=( + terminal_request_state.input_full_fingerprint + if terminal_request_state.input_item_count > 0 + else None + ), + pending_tool_calls=_durable_pending_tool_call_manifest(terminal_request_state, payload), + ) + except Exception: + logger.warning("Failed to advance quarantined HTTP bridge continuity", exc_info=True) + if quarantine_advanced: + _clear_http_bridge_quarantine_key( + self, + terminal_request_state.quarantine_clear_key, + account_id=session.account.id, + model=session.request_model, + generation=terminal_request_state.quarantine_clear_generation, + ) normalize_error_event = ( terminal_request_state is None or terminal_request_state.enforce_openai_sdk_contract diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 8a79957e72..9bd7671468 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -988,6 +988,8 @@ class _WebSocketRequestState: skip_request_log: bool = False previous_response_id: str | None = None session_id: str | None = None + quarantine_clear_key: _HTTPBridgeSessionKey | None = None + quarantine_clear_generation: int | None = None # Session headers provide locality, but only a previous response or # explicit turn-state header guarantees continuity for stale recovery. hard_continuity_anchor: bool = False diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 2386b43304..5bf9aed891 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -7594,7 +7594,7 @@ async def fail_legacy_stream(*args, **kwargs): # Scope the soft-affinity key to this test's account so a parallel or # ordered integration run cannot inherit another instance's durable # owner and turn the reconnect assertion into a 409 race. - "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}", + "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}-{time.monotonic_ns()}", } first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) second = await asyncio.wait_for( @@ -7682,6 +7682,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + prompt_cache_key = f"http-bridge-previous-response-reconnect-{account_id}-{time.monotonic_ns()}" first = await asyncio.wait_for( async_client.post( "/v1/responses", @@ -7689,7 +7690,7 @@ async def fake_connect_responses_websocket( "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", - "prompt_cache_key": "http-bridge-previous-response-reconnect", + "prompt_cache_key": prompt_cache_key, }, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, @@ -7704,7 +7705,7 @@ async def fake_connect_responses_websocket( "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello-again", - "prompt_cache_key": "http-bridge-previous-response-reconnect", + "prompt_cache_key": prompt_cache_key, "previous_response_id": first_body["id"], }, ), @@ -7761,6 +7762,7 @@ async def test_v1_responses_http_bridge_classifies_responses_lite_developer_inte "acc_http_bridge_preserve_fresh_reattach", "http-bridge-preserve-fresh-reattach@example.com", ) + scenario_key = f"{account_id}-{time.monotonic_ns()}" account = await _get_account(account_id) first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_preserve_source") replay_upstream = _FakeBridgeUpstreamWebSocket("resp_preserve_replay") @@ -7811,7 +7813,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(service._durable_bridge, "release_live_session", delay_predecessor_release) monkeypatch.setattr(service._durable_bridge, "claim_live_session", observe_replacement_claim) - session_headers = {"x-codex-session-id": "fresh-reattach-full-resend"} + session_headers = {"x-codex-session-id": f"fresh-reattach-full-resend-{scenario_key}"} historical_input = [ *([leading_input_item] if leading_input_item is not None else []), { @@ -15341,7 +15343,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session_headers = {"x-codex-session-id": "quarantine-silent-reattach"} + session_headers = {"x-codex-session-id": f"quarantine-silent-reattach-{account_id}-{time.monotonic_ns()}"} historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, { @@ -15513,9 +15515,10 @@ async def fake_connect_responses_websocket( # The turn-state header makes the bridge session a true Codex continuity # session (``session.codex_session``), which is what arms the session-level # anchor injection this regression guards against. + scenario_key = f"{account_id}-{time.monotonic_ns()}" session_headers = { - "x-codex-session-id": "quarantine-unsafe-suffix-reattach", - "x-codex-turn-state": "quarantine-unsafe-suffix-turn", + "x-codex-session-id": f"quarantine-unsafe-suffix-reattach-{scenario_key}", + "x-codex-turn-state": f"quarantine-unsafe-suffix-turn-{scenario_key}", } historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index d7c3ca1d8c..c8a50f4f41 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -1879,6 +1879,54 @@ async def test_durable_bridge_release_without_draining_marks_session_closed( assert reclaimed.latest_response_id == "resp_2" +@pytest.mark.asyncio +async def test_durable_bridge_ownerless_active_row_can_be_reclaimed_without_takeover( + coordinator: DurableBridgeSessionCoordinator, +) -> None: + claimed = await coordinator.claim_live_session( + session_key_kind="prompt_cache", + session_key_value="ownerless-reclaim", + api_key_id=None, + instance_id="instance-a", + owner_process_epoch="test-process-a", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=True, + ) + released = await coordinator.release_live_session( + session_id=claimed.session_id, + instance_id="instance-a", + owner_epoch=claimed.owner_epoch, + draining=True, + ) + + assert released is not None + assert released.owner_instance_id is None + + reclaimed = await coordinator.claim_live_session( + session_key_kind="prompt_cache", + session_key_value="ownerless-reclaim", + api_key_id=None, + instance_id="instance-b", + owner_process_epoch="test-process-b", + lease_ttl_seconds=60.0, + account_id="acc-1", + model="gpt-5.4", + service_tier=None, + latest_turn_state=None, + latest_response_id=None, + allow_takeover=False, + ) + + assert reclaimed.session_id == claimed.session_id + assert reclaimed.owner_instance_id == "instance-b" + assert reclaimed.owner_epoch == claimed.owner_epoch + 1 + + @pytest.mark.asyncio async def test_durable_bridge_takeover_clears_stale_recovery_anchor_for_fresh_session( coordinator: DurableBridgeSessionCoordinator, diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 744adb7d76..5ac0f3430a 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -59,6 +59,7 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, + durable_bridge_hash, ) from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.http_bridge_event_batcher import TerminalOperationEventAppendResult @@ -13698,6 +13699,7 @@ async def test_stream_via_http_bridge_preserves_context_after_owner_unavailable( retained_output = { "type": "message", "role": "assistant", + "id": "msg_response_owned", "content": [{"type": "output_text", "text": "two"}], } input_items = [*prefix_items] @@ -30869,6 +30871,53 @@ def test_http_bridge_quarantine_clear_restores_session_reusability() -> None: assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is False +def test_http_bridge_quarantine_clear_generation_rejects_stale_recovery() -> None: + service = SimpleNamespace() + session = _make_bridge_session(key_value="quarantine-clear-generation") + + http_bridge_quarantine_module._quarantine_http_bridge_session( + service, + session, + reason="reattach_missing_response_created", + ) + first_generation = http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation( + service, + session.key, + ) + assert first_generation is not None + + http_bridge_quarantine_module._quarantine_http_bridge_session( + service, + session, + reason="repeated_eventless_timeout", + ) + assert http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation(service, session.key) != ( + first_generation + ) + + http_bridge_quarantine_module._clear_http_bridge_quarantine_key( + service, + session.key, + account_id="acc-late-recovery", + model="gpt-5.6-sol", + generation=first_generation, + ) + assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is True + + current_generation = http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation( + service, + session.key, + ) + http_bridge_quarantine_module._clear_http_bridge_quarantine_key( + service, + session.key, + account_id="acc-current-recovery", + model="gpt-5.6-sol", + generation=current_generation, + ) + assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is False + + def test_http_bridge_session_reusable_for_lookup_excludes_quarantined() -> None: service = SimpleNamespace() session = _make_bridge_session(key_value="quarantine-reuse") @@ -31239,41 +31288,47 @@ async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_ unanchored instead of restoring the wedged durable anchor through session hydration and session-level injection.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - historical_input = [ - {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, - { - "type": "additional_tools", - "role": "developer", - "tools": [{"type": "custom", "name": "shell"}], - }, - { - "type": "message", - "role": "developer", - "content": [{"type": "input_text", "text": "canonical Lite instructions"}], - }, - {"role": "user", "content": [{"type": "input_text", "text": "first question"}]}, - { - "type": "custom_tool_call", - "call_id": "call_historical_shell", - "name": "shell", - "input": "printf historical", - }, - {"role": "developer", "content": [{"type": "input_text", "text": "historical control"}]}, - { - "type": "custom_tool_call_output", - "call_id": "call_historical_shell", - "output": "historical", - }, - ] - # Full resend with a trimmable durable prefix and a fresh suffix that does - # NOT retain the prior output (plain user turn). + historical_input = [{"role": "user", "content": "one"}] + retained_output = { + "type": "message", + "role": "assistant", + "id": "msg_response_owned", + "content": [{"type": "output_text", "text": "two"}], + } + projected_retained_output = { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "two"}], + } + retains_prior_output = True + account_neutral_payload = True + expected_account_neutral = True + fresh_suffix = ( + [ + retained_output, + {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, + ] + if retains_prior_output + else [{"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}] + ) + expected_projected_input = ( + [ + *historical_input, + projected_retained_output, + {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, + ] + if retains_prior_output + else [{"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}] + ) + # Full resend with a trimmable durable prefix. Only the variant retaining + # prior output is safe to replay account-neutrally. payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.6-sol", "instructions": "test", "input": [ *historical_input, - {"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}, + *fresh_suffix, ], } ) @@ -31303,8 +31358,7 @@ async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_ quarantined_session, reason="reattach_missing_response_created", ) - fresh_session = _make_bridge_session(key=bridge_key, key_value=bridge_key.affinity_key) - fresh_session.codex_session = True + fresh_session: proxy_service._HTTPBridgeSession | None = None prepared_payloads: list[proxy_service.ResponsesRequest] = [] @@ -31333,17 +31387,25 @@ def fake_prepare( request_state.previous_response_id = prepared_payload.previous_response_id return request_state, json.dumps(dict(prepared_payload.to_payload()), separators=(",", ":")) + captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + captured_kwargs: list[dict[str, object]] = [] + captured_request_states: list[proxy_service._WebSocketRequestState] = [] + async def fake_get_or_create( key: proxy_service._HTTPBridgeSessionKey, **kwargs: object, ) -> proxy_service._HTTPBridgeSession: - del kwargs - assert key == bridge_key + nonlocal fresh_session + captured_keys.append(key) + captured_kwargs.append(dict(kwargs)) + fresh_session = _make_bridge_session(key=key, key_value=key.affinity_key) + fresh_session.codex_session = True return fresh_session dispatched_text: list[str] = [] async def fake_stream_events(*args: object, **kwargs: object): + captured_request_states.append(cast(proxy_service._WebSocketRequestState, kwargs["request_state"])) dispatched_text.append(cast(str, kwargs["text_data"])) yield 'data: {"type":"response.completed"}\n\n' @@ -31369,6 +31431,12 @@ async def fake_stream_events(*args: object, **kwargs: object): monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + account_neutral_classifier = Mock(return_value=account_neutral_payload) + monkeypatch.setattr( + http_bridge_streaming_module, + "_http_bridge_payload_is_account_neutral_fresh_replay", + account_neutral_classifier, + ) # The bypass shape: a live (alias) session makes the fresh-reattach # durable-anchor gate false before the quarantine is ever consulted. monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=True)) @@ -31390,14 +31458,350 @@ async def fake_stream_events(*args: object, **kwargs: object): chunks = [chunk async for chunk in stream] assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert len(captured_keys) == 1 + assert (captured_keys[0] != bridge_key) is expected_account_neutral + assert captured_keys[0].strength == ("soft" if expected_account_neutral else "hard") + assert ( + is_http_bridge_account_neutral_replay( + kind=captured_keys[0].affinity_kind, + key=captured_keys[0].affinity_key, + ) + is expected_account_neutral + ) + assert captured_kwargs[0]["headers"] == ( + {} if expected_account_neutral else {"x-codex-session-id": bridge_key.affinity_key} + ) + assert captured_kwargs[0]["preferred_account_id"] == (None if expected_account_neutral else "acc-bridge") assert len(dispatched_text) == 1 dispatched_payload = json.loads(dispatched_text[0]) # Genuinely unanchored: the suppressed durable anchor did not come back # through session hydration or session-level injection, and the client's # payload was not prefix-trimmed against the durable stored context. assert "previous_response_id" not in dispatched_payload - assert len(dispatched_payload["input"]) == len(historical_input) + 1 + assert dispatched_payload["input"] == expected_projected_input + assert fresh_session is not None assert fresh_session.last_completed_response_id is None + assert len(captured_request_states) == 1 + assert (captured_request_states[0].quarantine_clear_key == bridge_key) is expected_account_neutral + assert (captured_request_states[0].quarantine_clear_generation is not None) is expected_account_neutral + expected_replay_kind, expected_replay_key = make_http_bridge_account_neutral_replay_key( + durable_bridge_hash(dispatched_text[0]) + ) + if expected_account_neutral: + assert captured_keys[0].affinity_kind == expected_replay_kind + assert captured_keys[0].affinity_key == expected_replay_key + if retains_prior_output: + account_neutral_classifier.assert_called_once() + else: + account_neutral_classifier.assert_not_called() + + +@pytest.mark.asyncio +async def test_quarantined_full_resend_recovery_fence_survives_restart_fingerprint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The quarantine demotion must not re-key the durable recovery fence. + + ``durable_recovery_attempt_fingerprint`` is the primary key of persisted + ``http_bridge_recovery_attempts`` rows, so it must keep hashing the + unprojected request body. A row journalled before a restart has to still + match after it, otherwise the one-shot replay fence silently opens once. + The quarantine recovery key is named after the projected body that this + dispatch actually sends, which is a different hash on purpose. + """ + service = proxy_service.ProxyService(cast(Any, nullcontext())) + historical_input = [{"role": "user", "content": "one"}] + retained_output = { + "type": "message", + "role": "assistant", + "id": "msg_response_owned", + "content": [{"type": "output_text", "text": "two"}], + } + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-sol", + "instructions": "test", + "input": [ + *historical_input, + retained_output, + {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, + ], + } + ) + bridge_key = proxy_service._HTTPBridgeSessionKey("session_header", "quarantine-fence-restart", None) + prefix_fingerprint = http_bridge_streaming_module._fingerprint_input_items( + cast(list[Any], payload.input)[: len(historical_input)] + ) + # The restart shape: the durable row survived, but its owner's lease died + # with the process that wrote the recovery-attempt row. + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-quarantine-fence", + canonical_kind=bridge_key.affinity_kind, + canonical_key=bridge_key.affinity_key, + api_key_scope="__anonymous__", + account_id="acc-bridge", + owner_instance_id="instance-before-restart", + owner_epoch=4, + lease_expires_at=proxy_service.utcnow() - timedelta(seconds=120), + state=HttpBridgeSessionState.CLOSED, + latest_turn_state=None, + latest_response_id="resp_wedged_anchor", + model="gpt-5.6-sol", + latest_input_item_count=len(historical_input), + latest_input_full_fingerprint=prefix_fingerprint, + ) + quarantined_session = _make_bridge_session(key=bridge_key, key_value=bridge_key.affinity_key) + http_bridge_quarantine_module._quarantine_http_bridge_session( + service, + quarantined_session, + reason="reattach_missing_response_created", + ) + + def render_bridge_text(rendered_payload: proxy_service.ResponsesRequest) -> str: + return json.dumps(dict(rendered_payload.to_payload()), separators=(",", ":")) + + # What a pre-restart process journalled: the hash of the unprojected body. + persisted_fence_fingerprint = durable_bridge_hash( + render_bridge_text(http_bridge_streaming_module._http_bridge_payload_without_previous_response_id(payload)) + ) + + def fake_prepare( + prepared_payload: proxy_service.ResponsesRequest, + _headers: dict[str, str] | Any, + *, + api_key: proxy_service.ApiKeyData | None, + api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, + request_id: str, + client_ip: str | None = None, + **prepare_kwargs: object, + ) -> tuple[proxy_service._WebSocketRequestState, str]: + del api_key, api_key_reservation, request_id, client_ip, prepare_kwargs + request_state = proxy_service._WebSocketRequestState( + request_id="req-quarantine-fence", + model=prepared_payload.model, + service_tier=None, + reasoning_effort=None, + api_key_reservation=None, + started_at=time.monotonic(), + event_queue=asyncio.Queue(), + transport="http", + ) + request_state.previous_response_id = prepared_payload.previous_response_id + return request_state, render_bridge_text(prepared_payload) + + captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: object, + ) -> proxy_service._HTTPBridgeSession: + del kwargs + captured_keys.append(key) + fresh_session = _make_bridge_session(key=key, key_value=key.affinity_key) + fresh_session.codex_session = True + return fresh_session + + dispatched_text: list[str] = [] + + async def fake_stream_events(*args: object, **kwargs: object): + del args + dispatched_text.append(cast(str, kwargs["text_data"])) + yield 'data: {"type":"response.completed"}\n\n' + + dashboard_settings = SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + ) + runtime_config = SimpleNamespace( + enabled=True, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + prompt_cache_idle_ttl_seconds=120.0, + gateway_safe_mode=False, + ) + monkeypatch.setattr( + http_bridge_streaming_module, + "_service_get_settings_cache", + lambda: SimpleNamespace(get=AsyncMock(return_value=dashboard_settings)), + ) + monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) + monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + + claim_instance_id = _make_app_settings().http_responses_session_bridge_instance_id + claimed_lookup = replace( + durable_lookup, + owner_instance_id=claim_instance_id, + owner_epoch=durable_lookup.owner_epoch + 1, + lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + ) + lookup_recovery_attempt = AsyncMock(return_value=SimpleNamespace(request_fingerprint=persisted_fence_fingerprint)) + mark_recovery_attempt_replayed = AsyncMock(return_value=True) + monkeypatch.setattr(service._durable_bridge, "lookup_recovery_attempt", lookup_recovery_attempt) + monkeypatch.setattr(service._durable_bridge, "claim_live_session", AsyncMock(return_value=claimed_lookup)) + monkeypatch.setattr(service._durable_bridge, "mark_recovery_attempt_replayed", mark_recovery_attempt_replayed) + monkeypatch.setattr( + http_bridge_streaming_module, + "_http_bridge_payload_is_account_neutral_fresh_replay", + Mock(return_value=True), + ) + monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=True)) + monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) + monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + stream = service._stream_http_bridge_or_retry( + payload, + {"x-codex-session-id": bridge_key.affinity_key}, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=False, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + ) + chunks = [chunk async for chunk in stream] + assert chunks == ['data: {"type":"response.completed"}\n\n'] + + # The fence looked the persisted row up under the fingerprint that row was + # written with, so the one-shot replay guard still holds after the restart. + lookup_recovery_attempt.assert_awaited_once_with( + session_id=durable_lookup.session_id, + request_fingerprint=persisted_fence_fingerprint, + ) + mark_recovery_attempt_replayed.assert_awaited_once_with( + session_id=durable_lookup.session_id, + api_key_id=None, + instance_id=claim_instance_id, + owner_epoch=claimed_lookup.owner_epoch, + request_fingerprint=persisted_fence_fingerprint, + ) + + # Negative control. The body this recovery actually dispatches is the + # projected one, and its hash is NOT the fence fingerprint. Hashing the + # projected body into ``durable_recovery_attempt_fingerprint`` is exactly + # the regression this test pins shut: the lookup above would have missed + # the persisted row and handed out a second "first" replay. + assert len(dispatched_text) == 1 + dispatched_payload = json.loads(dispatched_text[0]) + assert dispatched_payload["input"] == [ + *historical_input, + {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "two"}]}, + {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, + ] + projected_body_fingerprint = durable_bridge_hash(dispatched_text[0]) + assert projected_body_fingerprint != persisted_fence_fingerprint + fence_await = lookup_recovery_attempt.await_args + assert fence_await is not None + assert fence_await.kwargs["request_fingerprint"] != projected_body_fingerprint + + # The poisoned hard key was not reused for the recovery dispatch. + assert len(captured_keys) == 1 + assert captured_keys[0] != bridge_key + assert is_http_bridge_account_neutral_replay( + kind=captured_keys[0].affinity_kind, + key=captured_keys[0].affinity_key, + ) + + +@pytest.mark.asyncio +async def test_advance_http_bridge_quarantine_clear_key_rebinds_and_updates_original_continuity( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = SimpleNamespace() + key = proxy_service._HTTPBridgeSessionKey("session_header", "quarantine-original", None) + lookup = proxy_service.DurableBridgeLookup( + session_id="durable-original", + canonical_kind=key.affinity_kind, + canonical_key=key.affinity_key, + api_key_scope="__anonymous__", + account_id="acc-old", + owner_instance_id=None, + owner_epoch=7, + lease_expires_at=proxy_service.utcnow() - timedelta(seconds=60), + state=HttpBridgeSessionState.CLOSED, + latest_turn_state=None, + latest_response_id="resp-stale", + model="gpt-5.6-sol", + ) + claimed_lookup = replace( + lookup, + owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + owner_epoch=lookup.owner_epoch + 1, + lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + ) + service._durable_bridge = SimpleNamespace( + lookup_request_targets=AsyncMock(return_value=lookup), + claim_live_session=AsyncMock(return_value=claimed_lookup), + rebind_session_account=AsyncMock(return_value=True), + renew_live_session=AsyncMock( + return_value=replace(claimed_lookup, account_id="acc-new", latest_response_id="resp-new") + ), + ) + monkeypatch.setattr(http_bridge_upstream_events_module, "_service_get_settings", _make_app_settings) + + advanced = await http_bridge_upstream_events_module._advance_http_bridge_quarantine_clear_key( + service, + key=key, + api_key_id=None, + account_id="acc-new", + response_id="resp-new", + input_item_count=3, + input_full_fingerprint="fp-new", + pending_tool_calls={"call_1": "function_call"}, + ) + + assert advanced is True + service._durable_bridge.lookup_request_targets.assert_awaited_once_with( + session_key_kind=key.affinity_kind, + session_key_value=key.affinity_key, + api_key_id=None, + turn_state=None, + session_header=None, + previous_response_id=None, + ) + service._durable_bridge.claim_live_session.assert_awaited_once_with( + session_key_kind=key.affinity_kind, + session_key_value=key.affinity_key, + api_key_id=None, + instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + lease_ttl_seconds=pytest.approx(http_bridge_helpers_module._http_bridge_durable_lease_ttl_seconds()), + account_id="acc-old", + model="gpt-5.6-sol", + service_tier=None, + latest_turn_state=None, + latest_response_id="resp-stale", + allow_takeover=False, + owner_process_epoch=http_bridge_owner_process_epoch(), + ) + service._durable_bridge.rebind_session_account.assert_awaited_once_with( + session_id=claimed_lookup.session_id, + api_key_id=None, + instance_id=_make_app_settings().http_responses_session_bridge_instance_id, + owner_epoch=claimed_lookup.owner_epoch, + account_id="acc-new", + clear_continuity=True, + ) + service._durable_bridge.renew_live_session.assert_awaited_once() + renew_kwargs = service._durable_bridge.renew_live_session.await_args.kwargs + assert renew_kwargs == { + "session_id": claimed_lookup.session_id, + "api_key_id": None, + "instance_id": _make_app_settings().http_responses_session_bridge_instance_id, + "owner_epoch": claimed_lookup.owner_epoch, + "lease_ttl_seconds": renew_kwargs["lease_ttl_seconds"], + "latest_response_id": "resp-new", + "latest_input_item_count": 3, + "latest_input_full_fingerprint": "fp-new", + "latest_pending_tool_calls": {"call_1": "function_call"}, + } + assert renew_kwargs["lease_ttl_seconds"] > 0 @pytest.mark.asyncio From 52092bc91fd81d7a18655eab403f02ff6895dd80 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Thu, 20 Aug 2026 22:46:22 +0400 Subject: [PATCH 105/117] fix(proxy): guard model-transition owner-conflict fork (#1619) An HTTP bridge model transition could return `continuity_owner_conflict` indefinitely when a durable model owner and a stale hard alias pointed at different accounts, even though the request was safe to start on a fresh child bridge. Add a narrowly gated account-neutral model-transition fork, limited to the exact `continuity_owner_conflict` error, local requests and an account-neutral effective Responses payload. Forwarded requests, previous-response continuations, resolved file owners, account-scoped hosted references, and post-compaction payloads whose compacted context is not carried in the request remain fail-closed. The forked child key is pinned `hard`, and the child request state drops the parent affinity policy, hard continuity anchor and reused parent turn state so the submit and clean-close paths do not classify it as the parent's owner-bound turn. --- .../proxy/_service/http_bridge/streaming.py | 80 +++ .../.openspec.yaml | 2 + .../design.md | 60 +++ .../proposal.md | 37 ++ .../specs/responses-api-compat/spec.md | 64 +++ .../specs/sticky-session-operations/spec.md | 171 ++++++ .../tasks.md | 21 + tests/unit/test_proxy_http_bridge.py | 490 ++++++++++++++++++ 8 files changed, 925 insertions(+) create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/design.md create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md create mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 40d81f2cb0..f06fee2d83 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -1947,6 +1947,7 @@ def classify_durable_full_resend( else dict(headers) ) fresh_replay_excluded_account_ids: set[str] = set() + model_transition_owner_conflict_fork_attempted = False unanchored_fork_spill_attempted = False def durable_full_resend_allows_account_neutral_replay() -> bool: @@ -2006,6 +2007,83 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: ) return durable_full_resend_is_account_neutral + def switch_model_transition_to_account_neutral_fork(exc: ProxyResponseError) -> bool: + nonlocal account_neutral_recovery + nonlocal affinity + nonlocal bridge_session_key + nonlocal downstream_turn_state + nonlocal force_local_recovery_creation + nonlocal incoming_turn_state_header + nonlocal model_transition_owner_conflict_fork_attempted + nonlocal preferred_account_has_continuity_provenance + nonlocal request_state + nonlocal session_creation_headers + nonlocal session_header_fallback_key + + error_code, _error_message = _proxy_error_code_message(exc) + if ( + durable_model_transition_lookup is None + or error_code != "continuity_owner_conflict" + or model_transition_owner_conflict_fork_attempted + or forwarded_request + or not _http_bridge_payload_is_account_neutral_fresh_replay(effective_payload) + or request_state.previous_response_id is not None + or rewritten_file_account_id is not None + ): + return False + reused_parent_turn_state = ( + incoming_turn_state_header is not None and downstream_turn_state == incoming_turn_state_header + ) + failed_owner_id = request_state.preferred_account_id + _log_http_bridge_event( + "model_transition_owner_conflict_fork", + bridge_session_key, + account_id=failed_owner_id, + model=effective_payload.model, + detail="outcome=retry_without_previous_model_owner", + cache_key_family=bridge_session_key.affinity_kind, + model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, + owner_check_applied=True, + ) + if failed_owner_id is not None: + fresh_replay_excluded_account_ids.add(failed_owner_id) + session_creation_headers = without_http_bridge_session_affinity_headers(session_creation_headers) + incoming_turn_state_header = None + session_header_fallback_key = None + affinity = _AffinityPolicy() + replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(uuid4().hex) + # Pin the child lane hard instead of inheriting the implicit + # default: this fork keeps the client's own payload rather than a + # proved full-resend projection like the model-transition fresh + # resend above, so once the lane owns upstream turn state there is + # no verified replay text that would make a later soft reroute to a + # third account safe. + bridge_session_key = _HTTPBridgeSessionKey( + replay_kind, + replay_key, + bridge_session_key.api_key_id, + strength="hard", + ) + account_neutral_recovery = True + force_local_recovery_creation = True + model_transition_owner_conflict_fork_attempted = True + # request_state was prepared for the parent lane before the + # creation loop, and `continue` re-enters the loop without + # rebuilding it. Reset the parent-derived continuity fields so the + # submit, retry, and clean-close paths classify the child as the + # account-neutral fresh request it is: a stale hard anchor here + # would block a later fresh account switch and let a clean close + # treat the parent turn alias as this lane's continuation. + request_state.affinity_policy = affinity + request_state.hard_continuity_anchor = False + request_state.preferred_account_id = None + request_state.excluded_account_ids.update(fresh_replay_excluded_account_ids) + preferred_account_has_continuity_provenance = False + if reused_parent_turn_state: + request_state.session_id = None + downstream_turn_state = None + return True + def owner_unavailable_allows_account_neutral_replay(exc: ProxyResponseError) -> bool: return ( _http_bridge_is_previous_response_owner_unavailable(exc) @@ -2205,6 +2283,8 @@ def switch_to_account_neutral_replay() -> None: defer_account_health_writes=request_state.api_key_reservation is not None, ) except ProxyResponseError as exc: + if switch_model_transition_to_account_neutral_fork(exc): + continue if not owner_unavailable_allows_account_neutral_replay(exc): exc_code, _exc_message = _proxy_error_code_message(exc) if not unanchored_fork_spill_attempted and _http_bridge_unanchored_fork_can_spill_on_cap( diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml b/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml new file mode 100644 index 0000000000..84cfc12459 --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-06 diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/design.md b/openspec/changes/fork-safe-model-transition-owner-conflict/design.md new file mode 100644 index 0000000000..298bebfe72 --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/design.md @@ -0,0 +1,60 @@ +## Context + +The durable full-resend reconciliation in the sibling change handles a +verified complete replay. A model transition is a separate path: the request +has no previous-response anchor to replay, but the durable lookup still binds +the old model's owner. When a stale hard alias disagrees, blindly selecting a +new account would be unsafe, while retrying the same alias produces a stable +502 `continuity_owner_conflict` loop. + +## Decision + +Keep the recovery gate inside the HTTP bridge creation loop. Inspect the typed +error code and continue only when it is `continuity_owner_conflict`; do not +reuse the broader owner-unavailable predicate. Reject forwarded requests so a +replica cannot create a local lane after an origin forwarding failure. Validate +the effective Responses payload with the existing account-neutral replay +classifier, which rejects `previous_response_id`, conversation state, +account-scoped hosted references such as `input_file.file_id`, hosted/MCP call +items, and any input item that is not self-contained. That classifier is shared +with the post-compaction recovery work, so its admitted shapes can widen over +time: a completed `compaction` item carrying its own encrypted content and +client-executed `tool_search_*` items are now accepted as self-contained, while +a compaction placeholder without that content still fails closed. File-owner +resolution and previous-response checks remain explicit defensive gates. + +On success, strip session/turn aliases, replace the request key with a +server-namespaced account-neutral key, exclude the failed owner, clear the +preferred owner/provenance, and force local creation. Persisted hard aliases +are not rewritten. + +The request state itself is reset to the child's own identity. It was prepared +for the parent lane before the creation loop, and the retry re-enters that loop +without rebuilding it, so the fork clears the parent affinity policy, the hard +continuity anchor, and a reused parent turn state. Leaving those set would let +the submit and clean-close paths treat the account-neutral child as the old +owner-bound turn: a stale anchor blocks the pre-output account switch that the +neutrality proof already permits, and a clean close would recover it as a +continuation of the parent turn alias. + +The child lane key is pinned `hard` rather than inheriting the implicit +strength default. The sibling model-transition fresh-resend path may use a +`soft` key because it substitutes a proved full-resend projection that any +account can serve at any point. This fork forwards the client's own payload +unchanged, so once the child lane owns upstream turn state there is no verified +replay text that would make a later soft reroute to a third account safe. + +## Negative Controls + +- A forwarded request with the same owner conflict must return the original + `continuity_owner_conflict` without a second creation attempt. +- A fresh payload containing an unpinned `input_file.file_id` must also return + the original conflict without account-neutral forking. +- A post-compaction payload whose `compaction` item only references the owner's + compacted context — no encrypted content of its own, or still in progress — + must return the original conflict instead of forking, because the prior turns + exist only behind the previous owner. +- A second conflict on the child lane must surface the original error instead of + forking again. +- The forked child request must not keep the parent's affinity policy, hard + continuity anchor, or reused parent turn state. diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md b/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md new file mode 100644 index 0000000000..567047de7b --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md @@ -0,0 +1,37 @@ +## Why + +An HTTP bridge model transition can resolve a durable model owner while a +legacy hard alias resolves to a different account. The bridge then returns +`continuity_owner_conflict` before dispatch even when the request is a fresh, +account-neutral payload that can safely start a model-transition child lane. +Forwarded requests and payloads that still depend on account-scoped state must +remain fail-closed. + +## What Changes + +- Permit a model-transition child lane only for the exact + `continuity_owner_conflict` error. +- Require a local request, no `previous_response_id`, no resolved file owner, + and a payload proven account-neutral by the existing replay-safety validator. +- Clear session/turn aliases, exclude the conflicting owner, and create a + server-namespaced account-neutral lane, pinned `hard`, without changing + persisted aliases. +- Keep forwarded requests, unpinned hosted files, post-compaction payloads whose + compacted context is not carried in the request, and other owner failures + fail-closed. + +## Capabilities + +### Modified Capabilities + +- `responses-api-compat`: model-transition owner conflicts may use a + proof-gated account-neutral child lane. +- `sticky-session-operations`: the hard-owner conflict exception is limited to + that exact fresh model-transition case. + +## Impact + +- Affected code: HTTP bridge model-transition recovery and unit coverage. +- No schema, setting, dependency, or live deployment change. +- Rollback is a source revert; the existing fail-closed path remains the + fallback for every request that does not satisfy the proof. diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..0a69b5c198 --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md @@ -0,0 +1,64 @@ +## ADDED Requirements + +### Requirement: HTTP bridge model-transition owner conflicts use a guarded child lane + +The service MUST use a guarded account-neutral child lane when a hard +continuity lookup identifies a durable owner for an incompatible model and +bridge creation returns `continuity_owner_conflict`; it may retry on that new +server-namespaced lane only when the +request is local (not forwarded), has no `previous_response_id`, has no +resolved file owner, and the effective payload passes the existing +account-neutral fresh-replay validator. The child lane MUST clear session and +turn aliases, reset the request's parent-derived affinity policy, hard +continuity anchor, and reused parent turn state, exclude the conflicting owner, +force local creation, and remain owner-bound (`hard`) once created so a later +capacity failure cannot reroute the same request to a third account. The service MUST attempt this child lane at +most once per request and MUST preserve the original conflict for every other +owner error or payload shape. + +#### Scenario: Neutral model transition conflict forks locally + +- **GIVEN** a durable hard continuity key belongs to account A for the old model +- **AND** the requested model is incompatible with that durable session +- **AND** the effective payload is account-neutral and has no previous-response + or resolved file owner +- **AND** bridge creation returns `continuity_owner_conflict` +- **WHEN** the request is local to the current replica +- **THEN** the service creates one server-namespaced account-neutral child lane +- **AND** that child lane key is owner-bound (`hard`) +- **AND** it excludes account A and does not forward the request +- **AND** the child request carries no parent affinity policy, hard continuity + anchor, or reused parent turn state +- **AND** it leaves the original hard aliases unchanged + +#### Scenario: Forwarded model transition conflict remains fail-closed + +- **GIVEN** the same durable model-transition conflict occurs for a forwarded + request +- **THEN** the service returns `continuity_owner_conflict` +- **AND** it does not create an account-neutral child lane + +#### Scenario: Account-bound payload remains fail-closed + +- **GIVEN** the effective model-transition payload contains an account-scoped + hosted reference such as an unpinned `input_file.file_id` +- **WHEN** bridge creation returns `continuity_owner_conflict` +- **THEN** the service returns `continuity_owner_conflict` +- **AND** it does not retry on another account + +#### Scenario: Post-compaction payload without carried compact context stays fail-closed + +- **GIVEN** the effective model-transition payload contains a `compaction` item + that is not self-contained, such as a placeholder with no encrypted content or + a compaction still in progress +- **WHEN** bridge creation returns `continuity_owner_conflict` +- **THEN** the service returns `continuity_owner_conflict` +- **AND** it does not fork the request onto an account that never held the + compacted context + +#### Scenario: Child lane conflict is not forked again + +- **GIVEN** the guarded child lane was already created for this request +- **WHEN** creation on that lane also returns `continuity_owner_conflict` +- **THEN** the service returns that error to the caller +- **AND** it does not create a further account-neutral lane diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md new file mode 100644 index 0000000000..86ed8e4680 --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md @@ -0,0 +1,171 @@ +## MODIFIED Requirements + +### Requirement: Hard continuity remains owner-bound and bounded + +Requests that depend on `previous_response_id`, hard turn-state, nonblank `conversation`, account-scoped `input_file.file_id` pins, live or durable bridge ownership, replay/reattach state, or another required owner continuity source MUST NOT silently reroute to an account that cannot preserve continuity. A resolved required owner MUST override bare process-session locality and MUST be selected without consulting or rewriting that soft mapping. A `previous_response_id` is a stored-object continuation reference and remains owner-bound even when the same request also carries a session header, `prompt_cache_key`, or another soft locality key. If independently resolved hard sources identify different accounts, if live durable referenced-file pins identify different accounts, or if a request has partial live durable file-pin coverage, the service MUST fail closed before upstream dispatch. A request for which no referenced file has a live durable pin MUST preserve opaque `file_id` compatibility and proceed without inventing ownership evidence. If the owner account/session is unavailable or saturated, the service MUST fail closed with an explicit retryable continuity/local overload reason instead of flooding the owner queue indefinitely. + +Every HTTP, compact, direct WebSocket, and HTTP-bridge transport MUST resolve explicit turn state against both live and durable bridge aliases. Live, durable, previous-response, file, and explicit turn-state evidence MUST be compared independently; source ordering MUST NOT choose the first match when distinct sessions or accounts resolve. A reused direct WebSocket MUST repeat nonblank `conversation` ownership validation for each response-create frame because the existing socket account proves only the current route. Single-account routing MUST constrain effective routing without narrowing the ownership-candidate pool used by that validation. + +When an HTTP-bridge owner is on another replica, the origin MUST forward its resolved durable file owner in authenticated full-context metadata. The receiving owner MUST perform its own fresh shared-database lookup and MUST require that durable result to match the forwarded owner. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. A retired direct WebSocket's upstream turn-state token MUST NOT be sent to a different account selected for a later movable bare-session request. + +A nonblank `conversation` without a dedicated resolved owner MUST proceed only when an explicit hard Codex mapping proves ownership or exactly one account remains in the model/API-key/security-scoped selection pool before transient additional-quota availability, retry exclusions, runtime health, budget, or account-cap filtering. A temporarily quota-filtered, excluded, unhealthy, or capped candidate MUST remain part of this ambiguity check because it may be the actual owner. A bare process-session mapping MUST NOT prove conversation ownership. + +The sole exception to the hard-owner conflict rule is the proof-gated HTTP +bridge model-transition child lane defined by `responses-api-compat`. It +applies only to a local request that fails with the exact +`continuity_owner_conflict` error, carries no `previous_response_id` and no +resolved file owner, and whose effective payload passes the account-neutral +fresh-replay validator. That child lane MUST clear session and turn aliases, +exclude the failed owner for that request only, and leave persisted hard +mappings unchanged. Every other hard-owner conflict MUST still fail closed. + +#### Scenario: Previous-response owner queue is saturated + +- **WHEN** a `/v1/responses` follow-up requires a previous-response owner +- **AND** the owner session queue or account cap is saturated +- **THEN** the service fails closed with `hard_affinity_saturated`, `previous_response_owner_unavailable`, or the applicable stable `account_stream_cap` / `account_response_create_cap` code +- **AND** it does not route to an unrelated account that lacks continuity state + +#### Scenario: File-pinned request owner is capped + +- **WHEN** a `/v1/responses` request references an `input_file.file_id` pinned to an owner account +- **AND** the owner account is at its account stream or response-create cap +- **THEN** the service returns a local account-cap overload for the owner +- **AND** it does not route the file reference to another account + +#### Scenario: File-pinned request owner overrides process-session locality + +- **GIVEN** a request carries a bare process-session header mapped to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the process-session mapping is neither consulted as an owner nor rewritten + +#### Scenario: File-pinned request owner overrides thread locality + +- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A +- **AND** its `input_file.file_id` is durably pinned to account B +- **WHEN** the request is routed +- **THEN** account B is treated as the required owner +- **AND** the thread mapping is neither consulted as an owner nor rewritten + +#### Scenario: Conflicting hard owners fail closed + +- **GIVEN** a turn state, previous response, bridge, or input file resolves to account A +- **AND** another hard source on the same request resolves to account B +- **AND** the request is not the guarded model-transition child-lane case +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` before upstream dispatch +- **AND** source ordering does not choose either owner + +#### Scenario: Partial or cross-account file pins fail closed + +- **GIVEN** a request references multiple account-scoped input files +- **AND** at least one file has a live durable owner pin +- **AND** another file has no live durable owner pin or the live pins resolve to different accounts +- **WHEN** the request is routed +- **THEN** the service fails with `file_owner_unavailable` or `continuity_owner_conflict` +- **AND** it does not route the files using a soft affinity account + +#### Scenario: Opaque file IDs with no live durable pins preserve compatibility + +- **GIVEN** a request references one or more `input_file.file_id` values +- **AND** none of those IDs has a live durable owner pin +- **WHEN** the request is routed +- **THEN** the service forwards the opaque file references under ordinary unpinned routing +- **AND** it does not invent a hard owner or fail solely because durable pin metadata is absent + +#### Scenario: Ambiguous conversation fails closed + +- **GIVEN** a request carries nonblank `conversation` continuity and only bare process-session affinity +- **AND** more than one account is eligible +- **WHEN** no dedicated or hard-mapping owner can be resolved +- **THEN** the request fails with a stable owner-unavailable error before upstream dispatch + +#### Scenario: Account-cap pressure does not manufacture a conversation owner + +- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool +- **AND** one account is temporarily at its local account cap +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the uncapped account is not treated as the unique owner + +#### Scenario: Retry or additional-quota filtering does not manufacture a conversation owner + +- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool +- **AND** retry exclusion or transient additional-quota availability removes one from the effective routing pool +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the remaining effective account is not treated as the unique owner + +#### Scenario: Account status does not manufacture a conversation owner + +- **GIVEN** two accounts are in the model/API-key/security ownership pool +- **AND** one account is paused, requires reauthentication, deactivated, or otherwise unavailable for routing +- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner +- **THEN** the request still fails with a stable owner-unavailable error +- **AND** the active account is not treated as the unique owner + +#### Scenario: Preferred file owner does not manufacture a conversation owner + +- **GIVEN** a request carries nonblank `conversation` continuity and a file durably pinned to account B +- **AND** another account remains in the model/API-key/security ownership pool +- **WHEN** no dedicated conversation owner can be resolved +- **THEN** file ownership does not narrow the conversation ambiguity check to account B +- **AND** the request fails closed before upstream dispatch + +#### Scenario: Bridge turn state is owner-bound across transports + +- **GIVEN** an HTTP bridge registered a turn-state alias for account A +- **WHEN** the alias is reused through compact, plain HTTP streaming, or direct WebSocket transport +- **THEN** each transport treats account A as the required owner +- **AND** it does not fall back to unrelated sticky affinity + +#### Scenario: Independent bridge aliases conflict + +- **GIVEN** a live or durable turn-state alias resolves to one bridge session +- **AND** a previous-response alias on the same request resolves to a distinct session or account +- **WHEN** the request is routed +- **THEN** the service fails with `continuity_owner_conflict` +- **AND** alias lookup order does not select either session + +#### Scenario: Reused WebSocket revalidates conversation ownership + +- **GIVEN** a direct upstream WebSocket is already open on account A +- **AND** a later response-create frame carries nonblank `conversation` +- **WHEN** more than one account remains in the ownership-candidate pool +- **THEN** the later frame fails with a stable owner-unavailable error before upstream send +- **AND** the existing socket account is not treated as ownership proof + +#### Scenario: Single-account routing does not manufacture conversation ownership + +- **GIVEN** single-account routing selects account A +- **AND** multiple accounts remain in the model/API-key/security ownership pool +- **WHEN** a request carries nonblank `conversation` without dedicated owner evidence +- **THEN** the request remains ambiguous and fails closed +- **AND** only the effective routing states are constrained to account A + +#### Scenario: Remote bridge owner revalidates forwarded file ownership + +- **GIVEN** origin replica A durably resolves an input file to account A +- **AND** the request's HTTP bridge owner runs on replica B +- **WHEN** replica A forwards the request to replica B with authenticated file-owner metadata +- **THEN** replica B MUST freshly resolve the shared durable pin +- **AND** it MUST accept the forwarded owner only when both owner values match +- **AND** a missing, conflicting, tampered, or legacy-unbound proof MUST be rejected before upstream invocation + +#### Scenario: Retired WebSocket turn state does not cross accounts + +- **GIVEN** a closed upstream WebSocket on account A supplied an account-scoped turn-state token +- **AND** a later movable bare-session frame or marked self-contained goal restart selects account B +- **WHEN** the proxy opens the replacement WebSocket +- **THEN** it removes account A's stale turn-state token before connect +- **AND** account B never receives that token + +#### Scenario: Guarded model-transition exception does not rewrite hard mappings + +- **GIVEN** a request satisfies the account-neutral model-transition child-lane + proof +- **WHEN** the child lane is created +- **THEN** persisted hard aliases remain unchanged +- **AND** the failed owner is excluded only for that request diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md b/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md new file mode 100644 index 0000000000..a21fca1195 --- /dev/null +++ b/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md @@ -0,0 +1,21 @@ +## 1. Contract + +- [x] 1.1 Define the exact conflict, local-request, payload-neutrality, and + owner-preservation gates. +- [x] 1.2 Add the sticky-session exception without weakening unrelated hard + owner conflicts. + +## 2. Implementation + +- [x] 2.1 Add the guarded account-neutral model-transition child-lane path. +- [x] 2.2 Pin the child lane key strength explicitly instead of relying on the + implicit default. +- [x] 2.3 Reset the child request state's parent-derived affinity policy, + continuity anchor, and reused parent turn state. +- [x] 2.4 Add positive and forwarded/unpinned-file/post-compaction negative + regressions plus the single-retry bound. + +## 3. Verification + +- [x] 3.1 Run model-transition unit and existing HTTP bridge integration tests. +- [x] 3.2 Run Ruff, type checks, and strict OpenSpec validation. diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 5ac0f3430a..e3ea00d5f5 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -25260,6 +25260,496 @@ async def fake_stream_events( assert all("id" not in item for item in replay_payload["input"]) +@pytest.mark.asyncio +async def test_stream_via_http_bridge_forks_account_neutral_model_transition_after_owner_conflict( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-terra", + "instructions": "hi", + "input": [{"role": "user", "content": "continue on the new model"}], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-conflict-parent", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_model_parent", + latest_response_id="resp_model_parent", + model="gpt-5.6-sol", + ) + owner_conflict = ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "Durable continuity aliases resolve to conflicting upstream owners.", + ), + ) + creation_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + creation_calls: list[dict[str, Any]] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + creation_keys.append(key) + creation_calls.append(kwargs) + if len(creation_calls) == 1: + raise owner_conflict + session = _make_bridge_session(key=key) + session.account = cast( + Any, + SimpleNamespace(id="acc-model-alternate", status=AccountStatus.ACTIVE), + ) + session.request_model = payload.model + return session + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + **_kwargs: Any, + ): + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-turn-state": "http_turn_model_parent", + "x-codex-session-id": "shared-root", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + downstream_turn_state="http_turn_model_child", + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert len(creation_calls) == 2 + assert creation_keys[0].affinity_kind in {"session_header", "turn_state_header"} + assert is_http_bridge_account_neutral_replay( + kind=creation_keys[1].affinity_kind, + key=creation_keys[1].affinity_key, + ) + # The child lane stays owner-bound so a later capacity failure cannot soft + # reroute this request onto a third account. + assert creation_keys[1].strength == "hard" + assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" + assert creation_calls[0]["preferred_account_has_continuity_provenance"] is True + assert creation_calls[1]["preferred_account_id"] is None + assert creation_calls[1]["preferred_account_has_continuity_provenance"] is False + assert creation_calls[1]["exclude_account_ids"] == {"acc-model-owner"} + assert creation_calls[1]["allow_forward_to_owner"] is False + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_limits_model_transition_owner_conflict_fork_to_one_retry( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-terra", + "instructions": "hi", + "input": [{"role": "user", "content": "continue on the new model"}], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-conflict-parent", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_model_parent", + latest_response_id="resp_model_parent", + model="gpt-5.6-sol", + ) + owner_conflict = ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "Durable continuity aliases resolve to conflicting upstream owners.", + ), + ) + creation_calls: list[dict[str, Any]] = [] + + async def fake_get_or_create( + _key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + creation_calls.append(kwargs) + raise owner_conflict + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-turn-state": "http_turn_model_parent", + "x-codex-session-id": "shared-root", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + downstream_turn_state="http_turn_model_child", + ): + pass + + assert exc_info.value is owner_conflict + assert len(creation_calls) == 2 + assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" + assert creation_calls[1]["preferred_account_id"] is None + assert creation_calls[1]["exclude_account_ids"] == {"acc-model-owner"} + assert creation_calls[1]["allow_forward_to_owner"] is False + + +@pytest.mark.asyncio +async def test_stream_via_http_bridge_model_transition_owner_conflict_fork_does_not_rebind_parent_turn_alias( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-terra", + "instructions": "hi", + "input": [{"role": "user", "content": "continue on the new model"}], + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-conflict-parent", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_model_parent", + latest_response_id="resp_model_parent", + model="gpt-5.6-sol", + ) + owner_conflict = ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "Durable continuity aliases resolve to conflicting upstream owners.", + ), + ) + stream_downstream_turn_states: list[str | None] = [] + stream_request_states: list[Any] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + if kwargs["preferred_account_id"] == "acc-model-owner": + raise owner_conflict + session = _make_bridge_session(key=key) + session.account = cast( + Any, + SimpleNamespace(id="acc-model-alternate", status=AccountStatus.ACTIVE), + ) + session.request_model = payload.model + return session + + async def fake_stream_events( + _session: proxy_service._HTTPBridgeSession, + **kwargs: Any, + ): + stream_downstream_turn_states.append(kwargs["downstream_turn_state"]) + stream_request_states.append(kwargs["request_state"]) + yield 'data: {"type":"response.completed"}\n\n' + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) + + chunks = [ + chunk + async for chunk in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-turn-state": "http_turn_model_parent", + "x-codex-session-id": "shared-root", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + downstream_turn_state="http_turn_model_parent", + ) + ] + + assert chunks == ['data: {"type":"response.completed"}\n\n'] + assert stream_downstream_turn_states == [None] + # The child lane must not inherit the parent's continuity identity: a stale + # hard anchor or parent affinity policy would make the submit and + # clean-close paths treat this account-neutral fork as the old owner-bound + # turn. + (child_request_state,) = stream_request_states + assert child_request_state.session_id is None + assert child_request_state.hard_continuity_anchor is False + assert child_request_state.affinity_policy.key is None + assert child_request_state.affinity_policy.codex_session_source is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("forwarded_request", "input_items"), + [ + ( + True, + [{"role": "user", "content": "continue on the new model"}], + ), + ( + False, + [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "continue on the new model"}, + {"type": "input_file", "file_id": "file-unpinned"}, + ], + } + ], + ), + # Negative controls for the widened account-neutral classifier (#1849): + # a `compaction` item is admitted only when it carries its own + # completed encrypted content. A placeholder that merely references the + # owner's compacted context, or one still being produced, keeps the + # prior turns behind the old owner, so forking would silently drop + # them. + ( + False, + [ + {"type": "compaction", "id": "cmpct_model_parent"}, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue on the new model"}], + }, + ], + ), + ( + False, + [ + { + "type": "compaction", + "status": "in_progress", + "encrypted_content": "gAAAAABopaque", + }, + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "continue on the new model"}], + }, + ], + ), + ], + ids=[ + "forwarded-request", + "unpinned-input-file", + "post-compaction-placeholder", + "post-compaction-in-progress", + ], +) +async def test_stream_via_http_bridge_keeps_model_transition_owner_conflict_fail_closed_for_unsafe_fork( + monkeypatch: pytest.MonkeyPatch, + forwarded_request: bool, + input_items: list[dict[str, Any]], +) -> None: + service = proxy_service.ProxyService(cast(Any, nullcontext())) + monkeypatch.setattr( + proxy_service, + "get_settings_cache", + lambda: cast( + Any, + SimpleNamespace( + get=AsyncMock( + return_value=SimpleNamespace( + sticky_threads_enabled=False, + openai_cache_affinity_max_age_seconds=1800, + http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, + http_responses_session_bridge_gateway_safe_mode=False, + ) + ) + ), + ), + ) + payload = proxy_service.ResponsesRequest.model_validate( + { + "model": "gpt-5.6-terra", + "instructions": "hi", + "input": input_items, + } + ) + durable_lookup = proxy_service.DurableBridgeLookup( + session_id="durable-model-unsafe-fork", + canonical_kind="session_header", + canonical_key="shared-root", + api_key_scope="__anonymous__", + account_id="acc-model-owner", + owner_instance_id=None, + owner_epoch=1, + lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), + state=HttpBridgeSessionState.ACTIVE, + latest_turn_state="http_turn_model_parent", + latest_response_id="resp_model_parent", + model="gpt-5.6-sol", + ) + owner_conflict = ProxyResponseError( + 502, + openai_error( + "continuity_owner_conflict", + "Durable continuity aliases resolve to conflicting upstream owners.", + ), + ) + creation_keys: list[proxy_service._HTTPBridgeSessionKey] = [] + creation_calls: list[dict[str, Any]] = [] + + async def fake_get_or_create( + key: proxy_service._HTTPBridgeSessionKey, + **kwargs: Any, + ) -> proxy_service._HTTPBridgeSession: + creation_keys.append(key) + creation_calls.append(kwargs) + raise owner_conflict + + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) + monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) + monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) + + with pytest.raises(ProxyResponseError) as exc_info: + async for _ in service._stream_via_http_bridge( + payload, + headers={ + "x-codex-turn-state": "http_turn_model_parent", + "x-codex-session-id": "shared-root", + }, + codex_session_affinity=True, + propagate_http_errors=True, + openai_cache_affinity=True, + api_key=None, + api_key_reservation=None, + suppress_text_done_events=False, + idle_ttl_seconds=120.0, + codex_idle_ttl_seconds=1800.0, + max_sessions=8, + queue_limit=4, + downstream_turn_state="http_turn_model_child", + forwarded_request=forwarded_request, + ): + pass + + assert exc_info.value is owner_conflict + assert len(creation_calls) == 1 + assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" + assert creation_calls[0]["preferred_account_has_continuity_provenance"] is True + assert not is_http_bridge_account_neutral_replay( + kind=creation_keys[0].affinity_kind, + key=creation_keys[0].affinity_key, + ) + + @pytest.mark.asyncio async def test_stream_via_http_bridge_preserves_verified_replay_kind_for_durable_model_transition( monkeypatch: pytest.MonkeyPatch, From d4b00fd05eaff95e7d5979d4ea08e908ee6774c0 Mon Sep 17 00:00:00 2001 From: Darafei Praliaskouski Date: Fri, 21 Aug 2026 00:19:04 +0400 Subject: [PATCH 106/117] Revert five squash merges landed outside the merge triage round (#1858) --- app/db/alembic/revision_ids.py | 1 - .../20260813_000000_add_file_account_pins.py | 25 +- ...epair_retired_identity_and_warmup_stamp.py | 77 - .../proxy/_service/http_bridge/helpers.py | 17 - .../proxy/_service/http_bridge/quarantine.py | 46 +- .../_service/http_bridge/request_submit.py | 16 +- .../proxy/_service/http_bridge/streaming.py | 248 +-- .../_service/http_bridge/upstream_events.py | 105 +- app/modules/proxy/_service/streaming/mixin.py | 4 +- app/modules/proxy/_service/support.py | 2 - .../proxy/_service/websocket/helpers.py | 2 +- app/modules/proxy/_service/websocket/mixin.py | 2 +- app/modules/proxy/replay_safety.py | 99 +- app/modules/proxy/service.py | 16 +- .../.openspec.yaml | 2 - .../design.md | 60 - .../proposal.md | 37 - .../specs/responses-api-compat/spec.md | 64 - .../specs/sticky-session-operations/spec.md | 171 -- .../tasks.md | 21 - .../proposal.md | 28 - .../specs/responses-api-compat/spec.md | 54 - .../tasks.md | 9 - .../proposal.md | 28 - .../specs/database-migrations/spec.md | 23 - .../tasks.md | 9 - .../.openspec.yaml | 2 - .../proposal.md | 14 - .../specs/responses-api-compat/spec.md | 22 - .../tasks.md | 9 - .../integration/test_http_responses_bridge.py | 173 +- tests/integration/test_migrations.py | 117 -- tests/integration/test_proxy_compact.py | 66 - tests/unit/test_durable_bridge_sessions.py | 48 - tests/unit/test_openai_requests.py | 83 - tests/unit/test_proxy_http_bridge.py | 1653 ++--------------- tests/unit/test_proxy_utils.py | 93 +- tests/unit/test_replay_safety.py | 324 ---- 38 files changed, 231 insertions(+), 3539 deletions(-) delete mode 100644 app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/design.md delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md delete mode 100644 openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md delete mode 100644 openspec/changes/recover-post-compact-bridge-replays/proposal.md delete mode 100644 openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md delete mode 100644 openspec/changes/recover-post-compact-bridge-replays/tasks.md delete mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/proposal.md delete mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md delete mode 100644 openspec/changes/repair-retired-identity-warmup-stamp/tasks.md delete mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml delete mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md delete mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md delete mode 100644 openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md diff --git a/app/db/alembic/revision_ids.py b/app/db/alembic/revision_ids.py index 3e8c4a75e2..c5eed87ef1 100644 --- a/app/db/alembic/revision_ids.py +++ b/app/db/alembic/revision_ids.py @@ -27,7 +27,6 @@ ), "20260410_020000_restore_import_without_overwrite_default_false": "20260409_020000_fix_http_bridge_last_seen_index", "20260525_000000_merge_routing_settings_security_heads": "20260513_000000_add_accounts_alias", - "20260814_020000_merge_identity_and_warmup_heads": "20260816_000000_add_model_source_embeddings", } NEW_TO_OLD_REVISION_MAP: dict[str, str] = {new: old for old, new in OLD_TO_NEW_REVISION_MAP.items()} diff --git a/app/db/alembic/versions/20260813_000000_add_file_account_pins.py b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py index 4a0b1da5b0..dbfd6f4080 100644 --- a/app/db/alembic/versions/20260813_000000_add_file_account_pins.py +++ b/app/db/alembic/versions/20260813_000000_add_file_account_pins.py @@ -20,21 +20,16 @@ def upgrade() -> None: bind = op.get_bind() - inspector = sa.inspect(bind) - table_exists = inspector.has_table(_TABLE) - if not table_exists: - op.create_table( - _TABLE, - sa.Column("file_id", sa.String(), nullable=False), - sa.Column("account_id", sa.String(), nullable=False), - sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), - sa.PrimaryKeyConstraint("file_id"), - ) - index_exists = table_exists and "ix_file_account_pins_expires_at" in { - index["name"] for index in inspector.get_indexes(_TABLE) if index.get("name") - } - if not index_exists: - op.create_index("ix_file_account_pins_expires_at", _TABLE, ["expires_at"], unique=False) + if sa.inspect(bind).has_table(_TABLE): + return + op.create_table( + _TABLE, + sa.Column("file_id", sa.String(), nullable=False), + sa.Column("account_id", sa.String(), nullable=False), + sa.Column("expires_at", sa.DateTime(timezone=True), nullable=False), + sa.PrimaryKeyConstraint("file_id"), + ) + op.create_index("ix_file_account_pins_expires_at", _TABLE, ["expires_at"], unique=False) def downgrade() -> None: diff --git a/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py b/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py deleted file mode 100644 index 130a1a835d..0000000000 --- a/app/db/alembic/versions/20260820_000000_repair_retired_identity_and_warmup_stamp.py +++ /dev/null @@ -1,77 +0,0 @@ -"""repair schemas stamped at the retired identity/warmup merge head - -Revision ID: 20260820_000000_repair_retired_identity_and_warmup_stamp -Revises: 20260816_000000_add_model_source_embeddings -Create Date: 2026-08-20 - -Some local August 14, 2026 builds emitted the no-op merge stamp -``20260814_020000_merge_identity_and_warmup_heads`` even when the current -mainline file-pin, sticky-abandonment-scope, pending-deletion, API-key -reasoning-policy, and model-source-embeddings lineage had never run, while the -retired account-identity index and quota-planner lease-expiry column were -still present. Startup remaps that dead stamp to the current pre-repair head so -Alembic can continue; this forward-only repair step then replays the guarded -current migrations and drops the two stale artifacts. -""" - -from __future__ import annotations - -import importlib -from types import ModuleType - -import sqlalchemy as sa -from alembic import op -from sqlalchemy.engine import Connection - -revision = "20260820_000000_repair_retired_identity_and_warmup_stamp" -down_revision = "20260816_000000_add_model_source_embeddings" -branch_labels = None -depends_on = None - -_OBSOLETE_ACCOUNT_INDEX = "idx_accounts_chatgpt_account_id" -_OBSOLETE_QUOTA_COLUMN = "lease_expires_at" - - -def _migration(module_name: str) -> ModuleType: - return importlib.import_module(f"app.db.alembic.versions.{module_name}") - - -def _has_table(connection: Connection, table_name: str) -> bool: - return sa.inspect(connection).has_table(table_name) - - -def _columns(connection: Connection, table_name: str) -> set[str]: - if not _has_table(connection, table_name): - return set() - return {column["name"] for column in sa.inspect(connection).get_columns(table_name)} - - -def _indexes(connection: Connection, table_name: str) -> set[str]: - if not _has_table(connection, table_name): - return set() - names = (index.get("name") for index in sa.inspect(connection).get_indexes(table_name)) - return {name for name in names if name is not None} - - -def upgrade() -> None: - _migration("20260813_000000_add_file_account_pins").upgrade() - _migration("20260812_120000_add_sticky_abandonment_scope").upgrade() - _migration("20260816_000000_add_account_pending_deletion").upgrade() - _migration("20260806_030000_add_api_key_allowed_reasoning_efforts").upgrade() - _migration("20260816_000000_add_model_source_embeddings").upgrade() - - connection = op.get_bind() - - if _OBSOLETE_ACCOUNT_INDEX in _indexes(connection, "accounts"): - op.drop_index(_OBSOLETE_ACCOUNT_INDEX, table_name="accounts", if_exists=True) - - if _OBSOLETE_QUOTA_COLUMN in _columns(connection, "quota_planner_decisions"): - with op.batch_alter_table("quota_planner_decisions") as batch_op: - batch_op.drop_column(_OBSOLETE_QUOTA_COLUMN) - - -def downgrade() -> None: - # This revision repairs databases carrying a retired local merge stamp. It - # must not resurrect the stale index/column or remove objects owned by the - # canonical current-main migrations it replays above. - pass diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index 53f8e0bf4e..c5eebf77e5 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -627,23 +627,6 @@ def _has_http_bridge_response_output_marker(item: JsonValue) -> bool: return status in {"completed", "in_progress"} -def _http_bridge_pending_response_events_seen(pending_states: Sequence[_WebSocketRequestState]) -> int: - return max( - ( - max( - state.response_event_count, - int( - state.response_id is not None - or state.latency_response_created_ms is not None - or state.downstream_visible - ), - ) - for state in pending_states - ), - default=0, - ) - - def _http_bridge_input_item_type(item: JsonValue) -> str | None: if not isinstance(item, dict): return None diff --git a/app/modules/proxy/_service/http_bridge/quarantine.py b/app/modules/proxy/_service/http_bridge/quarantine.py index 1721f0c86e..ff47ea8b9f 100644 --- a/app/modules/proxy/_service/http_bridge/quarantine.py +++ b/app/modules/proxy/_service/http_bridge/quarantine.py @@ -43,7 +43,6 @@ class _HTTPBridgeQuarantineEntry: consecutive_eventless_timeouts: int = 0 last_touched_monotonic: float = 0.0 reason: str | None = None - generation: int = 0 def _http_bridge_quarantine_registry( @@ -101,16 +100,6 @@ def _http_bridge_session_key_quarantined(service: Any, key: _HTTPBridgeSessionKe return entry is not None and entry.quarantined_until > now -def _http_bridge_session_key_quarantine_generation(service: Any, key: _HTTPBridgeSessionKey) -> int | None: - registry = _http_bridge_quarantine_registry(service) - now = time.monotonic() - _prune_http_bridge_quarantine_registry(registry, now) - entry = registry.get(key) - if entry is None or entry.quarantined_until <= now: - return None - return entry.generation - - def _quarantine_http_bridge_session(service: Any, session: _HTTPBridgeSession, *, reason: str) -> None: """Quarantine a bridge session that has proven silent/wedged. @@ -124,7 +113,6 @@ def _quarantine_http_bridge_session(service: Any, session: _HTTPBridgeSession, * entry.quarantined_until = max(entry.quarantined_until, now + _HTTP_BRIDGE_QUARANTINE_TTL_SECONDS) entry.last_touched_monotonic = now entry.reason = reason - entry.generation += 1 _prune_http_bridge_quarantine_registry(registry, now) session.quarantined = True if already_quarantined: @@ -181,41 +169,21 @@ def _record_http_bridge_quarantine_eventless_timeout(service: Any, session: _HTT ) -def _clear_http_bridge_quarantine_key( - service: Any, - key: _HTTPBridgeSessionKey, - *, - account_id: str | None, - model: str | None, - generation: int | None = None, -) -> None: - """A completed response on a recovery key disproves the original wedge.""" +def _clear_http_bridge_quarantine(service: Any, session: _HTTPBridgeSession) -> None: + """A completed response on this key disproves the wedge; drop all state.""" registry = _http_bridge_quarantine_registry(service) - entry = registry.get(key) + session.quarantined = False + entry = registry.pop(session.key, None) if entry is None: return - if generation is not None and entry.generation != generation: - return - registry.pop(key, None) if entry.quarantined_until <= time.monotonic(): return _log_http_bridge_event( "session_quarantine_cleared", - key, - account_id=account_id, - model=model, - detail=f"reason={entry.reason}", - cache_key_family=key.affinity_kind, - model_class=_extract_model_class(model) if model else None, - ) - - -def _clear_http_bridge_quarantine(service: Any, session: _HTTPBridgeSession) -> None: - """A completed response on this key disproves the wedge; drop all state.""" - session.quarantined = False - _clear_http_bridge_quarantine_key( - service, session.key, account_id=session.account.id, model=session.request_model, + detail=f"reason={entry.reason}", + cache_key_family=session.key.affinity_kind, + model_class=_extract_model_class(session.request_model) if session.request_model else None, ) diff --git a/app/modules/proxy/_service/http_bridge/request_submit.py b/app/modules/proxy/_service/http_bridge/request_submit.py index a892ab3084..e3412b33f9 100644 --- a/app/modules/proxy/_service/http_bridge/request_submit.py +++ b/app/modules/proxy/_service/http_bridge/request_submit.py @@ -81,7 +81,6 @@ _http_bridge_durable_lease_ttl_seconds, _http_bridge_is_previous_response_owner_unavailable, _http_bridge_key_strength, - _http_bridge_pending_response_events_seen, _http_bridge_precreated_retry_failure_error, _http_bridge_prewarm_enabled, _http_bridge_request_budget_seconds, @@ -2817,7 +2816,20 @@ async def _retire_stale_pending_http_bridge_session( # circuit strike. Explicit values remain authoritative for # reader-failure callers whose pending deque was already # drained before entering this shared boundary. - response_events_seen = _http_bridge_pending_response_events_seen(retired_request_states) + response_events_seen = max( + ( + max( + request_state.response_event_count, + int( + request_state.response_id is not None + or request_state.latency_response_created_ms is not None + or request_state.downstream_visible + ), + ) + for request_state in retired_request_states + ), + default=0, + ) if retry_circuit_attempt_selection is None: retry_circuit_attempt_selection = _http_bridge_retry_circuit_attempt_selection_for_pending_requests( retired_request_states diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index f06fee2d83..1285996fad 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -109,7 +109,7 @@ _owner_forward_failure_allows_local_recovery, ) from app.modules.proxy._service.http_bridge.quarantine import ( - _http_bridge_session_key_quarantine_generation, + _http_bridge_session_key_quarantined, ) from app.modules.proxy._service.http_bridge.service_stubs import ( _build_rewritten_stream_response_failed_event, @@ -1352,8 +1352,6 @@ async def release_unowned_bridge_lifecycle( # dispatch genuinely goes unanchored instead of rebuilding the same # wedged reattach through the session-state side door. fresh_reattach_anchor_suppressed_quarantined = False - fresh_reattach_quarantine_clear_key: _HTTPBridgeSessionKey | None = None - fresh_reattach_quarantine_clear_generation: int | None = None def classify_durable_full_resend( lookup: DurableBridgeLookup, @@ -1423,12 +1421,6 @@ def classify_durable_full_resend( durable_full_resend_is_account_neutral = _http_bridge_payload_is_account_neutral_fresh_replay( durable_full_resend_fresh_payload ) - # The durable recovery-attempt fence keys persisted - # ``http_bridge_recovery_attempts`` rows. Hash the same - # unprojected body main has always hashed: a projected body - # would mint a different fingerprint, so a row written - # before a restart would stop matching and the one-shot - # replay fence would silently open once. _fresh_state, fresh_replay_text = prepare_bridge_request( _http_bridge_payload_without_previous_response_id(payload) ) @@ -1519,29 +1511,18 @@ def classify_durable_full_resend( if durable_lookup is not None and not _http_bridge_models_compatible(durable_lookup.model, payload.model) else None ) - durable_model_transition_uses_fresh_replay = ( - durable_model_transition_lookup is not None - and not forwarded_request - and rewritten_file_account_id is None - and durable_full_resend_fresh_payload is not None - and durable_full_resend_has_safe_fresh_context - and durable_full_resend_is_account_neutral is True - ) durable_model_transition_requires_owner = durable_model_transition_lookup is not None and ( - not durable_model_transition_uses_fresh_replay - and ( - payload.previous_response_id is not None - or bridge_session_key.strength == "hard" - or ( - bridge_session_key.affinity_kind == "prompt_cache" - and _http_bridge_request_stage( - headers=headers, - payload=payload, - durable_lookup=durable_model_transition_lookup, - ) - == "follow_up" - and durable_model_transition_lookup.latest_turn_state is not None + payload.previous_response_id is not None + or bridge_session_key.strength == "hard" + or ( + bridge_session_key.affinity_kind == "prompt_cache" + and _http_bridge_request_stage( + headers=headers, + payload=payload, + durable_lookup=durable_model_transition_lookup, ) + == "follow_up" + and durable_model_transition_lookup.latest_turn_state is not None ) ) if durable_model_transition_lookup is not None: @@ -1555,36 +1536,7 @@ def classify_durable_full_resend( model_class=_extract_model_class(payload.model) if payload.model else None, owner_check_applied=durable_model_transition_requires_owner, ) - if durable_model_transition_uses_fresh_replay: - replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(uuid4().hex) - bridge_session_key = _HTTPBridgeSessionKey( - replay_kind, - replay_key, - bridge_session_key.api_key_id, - strength="soft", - ) - affinity = _AffinityPolicy() - incoming_turn_state_header = None - incoming_session_header = None - session_header_fallback_key = None - effective_payload = durable_full_resend_fresh_payload - untrimmed_effective_payload = durable_full_resend_fresh_payload - force_local_recovery_creation = True - preferred_account_has_continuity_provenance = False - _log_http_bridge_event( - "model_transition_fresh_resend", - bridge_session_key, - account_id=durable_model_transition_lookup.account_id, - model=payload.model, - detail=( - "outcome=account_neutral_full_resend_without_owner," - f"previous_model={durable_model_transition_lookup.model}" - ), - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(payload.model) if payload.model else None, - owner_check_applied=False, - ) - elif is_http_bridge_account_neutral_replay( + if is_http_bridge_account_neutral_replay( kind=durable_model_transition_lookup.canonical_kind, key=durable_model_transition_lookup.canonical_key, ): @@ -1593,12 +1545,6 @@ def classify_durable_full_resend( replay_kind, replay_key, bridge_session_key.api_key_id, - # This is a one-shot fresh recovery dispatch after the - # original hard key was quarantined. Keep the - # account-neutral marker for replay guards, but do not hash - # the random recovery key through durable hard-owner - # routing before the wedged upstream proof can run. - strength="soft", ) force_local_recovery_creation = True durable_lookup = None @@ -1625,21 +1571,13 @@ def classify_durable_full_resend( and durable_lookup.latest_response_id is not None and (not payload_looks_like_full_resend or durable_anchor_trimmable) ) - quarantine_generation = ( - _http_bridge_session_key_quarantine_generation(self, bridge_session_key) - if payload_looks_like_full_resend - else None - ) - if quarantine_generation is not None: + if payload_looks_like_full_resend and _http_bridge_session_key_quarantined(self, bridge_session_key): # The previous attach on this key proved silent/wedged # (#1534). The client's own payload already carries the full # conversation, so send it unanchored on the fresh path - # instead of rebuilding the same reattach. Only cross accounts - # once the sealed durable full-resend proof matches this - # payload; otherwise keep the durable owner while dropping the - # poisoned anchor. Delta-only payloads keep the anchor: it is - # their only way to convey prior context (same boundary as the - # fenced anchor clear). + # instead of rebuilding the same reattach. Delta-only + # payloads keep the anchor: it is their only way to convey + # prior context (same boundary as the fenced anchor clear). # Evaluated independently of the fresh-reattach eligibility # above: even when that gate is already false (for example a # conversation-scoped payload, a live alias session, or an @@ -1649,35 +1587,6 @@ def classify_durable_full_resend( # paths below. fresh_reattach_can_use_durable_anchor = False fresh_reattach_anchor_suppressed_quarantined = True - if ( - durable_full_resend_proof is not None - and durable_full_resend_proof.matches(payload, durable_lookup) - and durable_full_resend_fresh_payload is not None - and durable_full_resend_is_account_neutral is True - ): - effective_payload = durable_full_resend_fresh_payload - untrimmed_effective_payload = durable_full_resend_fresh_payload - fresh_reattach_quarantine_clear_key = bridge_session_key - fresh_reattach_quarantine_clear_generation = quarantine_generation - # Name the recovery key after the body this dispatch - # actually sends, not after the recovery-attempt fence - # fingerprint: the two hash different bodies and must - # not be conflated. - _quarantine_replay_state, quarantine_replay_text = prepare_bridge_request( - durable_full_resend_fresh_payload - ) - del _quarantine_replay_state - replay_nonce = durable_bridge_hash(quarantine_replay_text) - replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(replay_nonce) - bridge_session_key = _HTTPBridgeSessionKey( - replay_kind, - replay_key, - bridge_session_key.api_key_id, - strength="soft", - ) - force_local_recovery_creation = True - incoming_session_header = None - session_header_fallback_key = None _log_http_bridge_event( "fresh_reattach_anchor_skipped_quarantined", bridge_session_key, @@ -1782,8 +1691,6 @@ def classify_durable_full_resend( request_state, text_data = prepare_bridge_request(effective_payload) request_state.enforce_openai_sdk_contract = enforce_openai_sdk_contract request_state.affinity_policy = affinity - request_state.quarantine_clear_key = fresh_reattach_quarantine_clear_key - request_state.quarantine_clear_generation = fresh_reattach_quarantine_clear_generation _apply_http_bridge_downstream_turn_state( request_state, downstream_turn_state=downstream_turn_state, @@ -1812,7 +1719,6 @@ def classify_durable_full_resend( durable_lookup.account_id if ( durable_lookup is not None - and not durable_model_transition_uses_fresh_replay and ( request_state.previous_response_id is not None or bridge_session_key.strength == "hard" @@ -1829,7 +1735,6 @@ def classify_durable_full_resend( request_state.preferred_account_id is None and durable_model_transition_lookup is not None and durable_model_transition_requires_owner - and not durable_model_transition_uses_fresh_replay ): request_state.preferred_account_id = durable_model_transition_lookup.account_id local_previous_response_owner: str | None = None @@ -1947,12 +1852,10 @@ def classify_durable_full_resend( else dict(headers) ) fresh_replay_excluded_account_ids: set[str] = set() - model_transition_owner_conflict_fork_attempted = False unanchored_fork_spill_attempted = False def durable_full_resend_allows_account_neutral_replay() -> bool: nonlocal durable_full_resend_fresh_payload - nonlocal durable_full_resend_has_safe_fresh_context nonlocal durable_full_resend_is_account_neutral nonlocal durable_full_resend_retains_prior_output @@ -1978,17 +1881,7 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: stored_count=eligibility_projection.stored_prefix_count, canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, ) - durable_full_resend_has_safe_fresh_context = durable_full_resend_retains_prior_output or ( - durable_lookup is not None - and durable_lookup.latest_pending_tool_calls is not None - and responses_input_suffix_matches_pending_tool_calls( - eligibility_projection.input_items, - stored_count=eligibility_projection.stored_prefix_count, - pending_tool_calls=durable_lookup.latest_pending_tool_calls, - canonical_lite_developer_index=eligibility_projection.canonical_lite_developer_index, - ) - ) - if not durable_full_resend_has_safe_fresh_context: + if not durable_full_resend_retains_prior_output: return False replay_projection = project_responses_input_for_account_neutral_fresh_replay( cast(list[JsonValue], payload.input), @@ -1999,7 +1892,7 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: durable_full_resend_fresh_payload = _http_bridge_payload_without_previous_response_id( payload ).model_copy(update={"input": replay_projection.input_items}) - if not durable_full_resend_has_safe_fresh_context: + if not durable_full_resend_retains_prior_output: return False if durable_full_resend_is_account_neutral is None: durable_full_resend_is_account_neutral = _http_bridge_payload_is_account_neutral_fresh_replay( @@ -2007,83 +1900,6 @@ def durable_full_resend_allows_account_neutral_replay() -> bool: ) return durable_full_resend_is_account_neutral - def switch_model_transition_to_account_neutral_fork(exc: ProxyResponseError) -> bool: - nonlocal account_neutral_recovery - nonlocal affinity - nonlocal bridge_session_key - nonlocal downstream_turn_state - nonlocal force_local_recovery_creation - nonlocal incoming_turn_state_header - nonlocal model_transition_owner_conflict_fork_attempted - nonlocal preferred_account_has_continuity_provenance - nonlocal request_state - nonlocal session_creation_headers - nonlocal session_header_fallback_key - - error_code, _error_message = _proxy_error_code_message(exc) - if ( - durable_model_transition_lookup is None - or error_code != "continuity_owner_conflict" - or model_transition_owner_conflict_fork_attempted - or forwarded_request - or not _http_bridge_payload_is_account_neutral_fresh_replay(effective_payload) - or request_state.previous_response_id is not None - or rewritten_file_account_id is not None - ): - return False - reused_parent_turn_state = ( - incoming_turn_state_header is not None and downstream_turn_state == incoming_turn_state_header - ) - failed_owner_id = request_state.preferred_account_id - _log_http_bridge_event( - "model_transition_owner_conflict_fork", - bridge_session_key, - account_id=failed_owner_id, - model=effective_payload.model, - detail="outcome=retry_without_previous_model_owner", - cache_key_family=bridge_session_key.affinity_kind, - model_class=_extract_model_class(effective_payload.model) if effective_payload.model else None, - owner_check_applied=True, - ) - if failed_owner_id is not None: - fresh_replay_excluded_account_ids.add(failed_owner_id) - session_creation_headers = without_http_bridge_session_affinity_headers(session_creation_headers) - incoming_turn_state_header = None - session_header_fallback_key = None - affinity = _AffinityPolicy() - replay_kind, replay_key = make_http_bridge_account_neutral_replay_key(uuid4().hex) - # Pin the child lane hard instead of inheriting the implicit - # default: this fork keeps the client's own payload rather than a - # proved full-resend projection like the model-transition fresh - # resend above, so once the lane owns upstream turn state there is - # no verified replay text that would make a later soft reroute to a - # third account safe. - bridge_session_key = _HTTPBridgeSessionKey( - replay_kind, - replay_key, - bridge_session_key.api_key_id, - strength="hard", - ) - account_neutral_recovery = True - force_local_recovery_creation = True - model_transition_owner_conflict_fork_attempted = True - # request_state was prepared for the parent lane before the - # creation loop, and `continue` re-enters the loop without - # rebuilding it. Reset the parent-derived continuity fields so the - # submit, retry, and clean-close paths classify the child as the - # account-neutral fresh request it is: a stale hard anchor here - # would block a later fresh account switch and let a clean close - # treat the parent turn alias as this lane's continuation. - request_state.affinity_policy = affinity - request_state.hard_continuity_anchor = False - request_state.preferred_account_id = None - request_state.excluded_account_ids.update(fresh_replay_excluded_account_ids) - preferred_account_has_continuity_provenance = False - if reused_parent_turn_state: - request_state.session_id = None - downstream_turn_state = None - return True - def owner_unavailable_allows_account_neutral_replay(exc: ProxyResponseError) -> bool: return ( _http_bridge_is_previous_response_owner_unavailable(exc) @@ -2283,8 +2099,6 @@ def switch_to_account_neutral_replay() -> None: defer_account_health_writes=request_state.api_key_reservation is not None, ) except ProxyResponseError as exc: - if switch_model_transition_to_account_neutral_fork(exc): - continue if not owner_unavailable_allows_account_neutral_replay(exc): exc_code, _exc_message = _proxy_error_code_message(exc) if not unanchored_fork_spill_attempted and _http_bridge_unanchored_fork_can_spill_on_cap( @@ -2999,16 +2813,16 @@ def switch_to_account_neutral_replay() -> None: previous_request_state.proxy_injected_anchor_had_full_resend_payload ) request_state.fresh_upstream_request_text = fresh_upstream_request_text - # The trim branch proves the upstream submission can omit the - # stored prefix, but it does not by itself prove that dropping - # the injected anchor is safe. Keep the original anchor site's - # decision unless this was a durable full-resend proof with a - # verified safe fresh suffix. Session-level anchors may still be - # compacted follow-ups whose prior context only exists behind - # previous_response_id. + # The trim branch only fires when the untrimmed payload + # is a true full resend whose prefix exactly matches the + # already-stored context, so the unanchored request text + # is a safe fresh-turn replay target regardless of + # whether the anchor came from the durable or + # session-level injection path. Injection-only re-prepares + # keep the replay-safety decision made when the anchor was + # injected. request_state.fresh_upstream_request_is_retry_safe = ( - previous_request_state.fresh_upstream_request_is_retry_safe - or (durable_full_resend_anchor_count is not None and durable_full_resend_has_safe_fresh_context) + (durable_full_resend_anchor_count is None or durable_full_resend_has_safe_fresh_context) if store_context_trim_applied else previous_request_state.fresh_upstream_request_is_retry_safe ) @@ -3123,7 +2937,6 @@ async def rollback_pre_dispatch_recovery_claim() -> None: owner_check_applied=True, ) replacement_preferred_account_id = request_state.preferred_account_id - replacement_excluded_account_ids = set(request_state.excluded_account_ids) if request_state.previous_response_id is not None and replacement_preferred_account_id is None: replacement_preferred_account_id = session.account.id elif replacement_preferred_account_id is None: @@ -3136,8 +2949,9 @@ async def rollback_pre_dispatch_recovery_claim() -> None: # impossible (fallback_on_preferred_account_unavailable is # False for exactly this pinned case below) and would # keep poisoning every later recovery call on this - # request. - replacement_excluded_account_ids.add(session.account.id) + # request, since excluded_account_ids persists on + # request_state. + request_state.excluded_account_ids.add(session.account.id) while True: try: replacement_session = await self._get_or_create_http_bridge_session( @@ -3170,7 +2984,7 @@ async def rollback_pre_dispatch_recovery_claim() -> None: request_usage_budget=request_state.request_usage_budget, request_deadline=request_deadline, session_header_fallback_key=session_header_fallback_key, - exclude_account_ids=replacement_excluded_account_ids or None, + exclude_account_ids=request_state.excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, ) diff --git a/app/modules/proxy/_service/http_bridge/upstream_events.py b/app/modules/proxy/_service/http_bridge/upstream_events.py index 5defdc825d..043e1bef46 100644 --- a/app/modules/proxy/_service/http_bridge/upstream_events.py +++ b/app/modules/proxy/_service/http_bridge/upstream_events.py @@ -3,7 +3,7 @@ import asyncio import logging import time -from collections.abc import Awaitable, Callable, Mapping +from collections.abc import Awaitable, Callable from dataclasses import replace from typing import Any, TypeVar, cast @@ -49,7 +49,6 @@ from app.core.usage.live_snapshots import EVENT_MARKER, parse_rate_limit_event_text from app.core.utils.request_id import reset_request_id, set_request_id from app.core.utils.sse import format_sse_event, parse_sse_data_json -from app.core.utils.time import utcnow from app.modules.proxy._service.api_key_usage import ( _API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS, ) @@ -73,7 +72,6 @@ ) from app.modules.proxy._service.http_bridge.quarantine import ( _clear_http_bridge_quarantine, - _clear_http_bridge_quarantine_key, _record_http_bridge_quarantine_eventless_timeout, _record_http_bridge_quarantine_wedged_pending, ) @@ -197,7 +195,6 @@ _extract_model_class, ) from app.modules.proxy.continuity import is_http_bridge_account_neutral_replay -from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.helpers import ( _normalize_error_code, is_upstream_model_capacity_error, @@ -212,74 +209,6 @@ logger = logging.getLogger("app.modules.proxy.service") - -async def _advance_http_bridge_quarantine_clear_key( - service: Any, - *, - key: Any, - api_key_id: str | None, - account_id: str, - response_id: str, - input_item_count: int | None, - input_full_fingerprint: str | None, - pending_tool_calls: Mapping[str, str] | None, -) -> bool: - lookup = await service._durable_bridge.lookup_request_targets( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=api_key_id, - turn_state=None, - session_header=None, - previous_response_id=None, - ) - if lookup is None: - return False - settings = _service_get_settings() - instance_id = settings.http_responses_session_bridge_instance_id - active_lookup = lookup - if not active_lookup.lease_is_active(now=utcnow()) or active_lookup.owner_instance_id != instance_id: - claimed_lookup = await service._durable_bridge.claim_live_session( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=api_key_id, - instance_id=instance_id, - lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), - account_id=active_lookup.account_id, - model=active_lookup.model, - service_tier=None, - latest_turn_state=active_lookup.latest_turn_state, - latest_response_id=active_lookup.latest_response_id, - allow_takeover=False, - owner_process_epoch=http_bridge_owner_process_epoch(), - ) - if claimed_lookup.owner_instance_id != instance_id: - return False - active_lookup = claimed_lookup - if active_lookup.account_id != account_id: - rebound = await service._durable_bridge.rebind_session_account( - session_id=active_lookup.session_id, - api_key_id=api_key_id, - instance_id=instance_id, - owner_epoch=active_lookup.owner_epoch, - account_id=account_id, - clear_continuity=True, - ) - if not rebound: - return False - advanced = await service._durable_bridge.renew_live_session( - session_id=active_lookup.session_id, - api_key_id=api_key_id, - instance_id=instance_id, - owner_epoch=active_lookup.owner_epoch, - lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(), - latest_response_id=response_id, - latest_input_item_count=input_item_count, - latest_input_full_fingerprint=input_full_fingerprint, - latest_pending_tool_calls=pending_tool_calls, - ) - return advanced is not None - - _HTTP_BRIDGE_RECOVERY_SETTLEMENT_RETRY_DELAYS = ( 0.25, 0.5, @@ -2854,38 +2783,6 @@ async def persist_grouped_terminal_events() -> Exception | None: ): await self._clear_http_bridge_retry_circuit(session) _clear_http_bridge_quarantine(self, session) - if terminal_request_state.quarantine_clear_key is not None: - quarantine_advanced = False - if response_id is not None: - try: - quarantine_advanced = await _advance_http_bridge_quarantine_clear_key( - self, - key=terminal_request_state.quarantine_clear_key, - api_key_id=session.key.api_key_id, - account_id=session.account.id, - response_id=response_id, - input_item_count=( - terminal_request_state.input_item_count - if terminal_request_state.input_item_count > 0 - else None - ), - input_full_fingerprint=( - terminal_request_state.input_full_fingerprint - if terminal_request_state.input_item_count > 0 - else None - ), - pending_tool_calls=_durable_pending_tool_call_manifest(terminal_request_state, payload), - ) - except Exception: - logger.warning("Failed to advance quarantined HTTP bridge continuity", exc_info=True) - if quarantine_advanced: - _clear_http_bridge_quarantine_key( - self, - terminal_request_state.quarantine_clear_key, - account_id=session.account.id, - model=session.request_model, - generation=terminal_request_state.quarantine_clear_generation, - ) normalize_error_event = ( terminal_request_state is None or terminal_request_state.enforce_openai_sdk_contract diff --git a/app/modules/proxy/_service/streaming/mixin.py b/app/modules/proxy/_service/streaming/mixin.py index 8bad159daa..5bab3d94b4 100644 --- a/app/modules/proxy/_service/streaming/mixin.py +++ b/app/modules/proxy/_service/streaming/mixin.py @@ -921,11 +921,11 @@ async def _touch_api_key_reservation() -> None: event_type, ) = _facade()._build_rewritten_stream_response_failed_event( response_id=response_id, - error_code=_facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE, + error_code="stream_incomplete", error_message=_facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE, ) status = "error" - error_code = _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE + error_code = "stream_incomplete" error_message = _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE settlement.record_success = False settlement.account_health_error = False diff --git a/app/modules/proxy/_service/support.py b/app/modules/proxy/_service/support.py index 9bd7671468..8a79957e72 100644 --- a/app/modules/proxy/_service/support.py +++ b/app/modules/proxy/_service/support.py @@ -988,8 +988,6 @@ class _WebSocketRequestState: skip_request_log: bool = False previous_response_id: str | None = None session_id: str | None = None - quarantine_clear_key: _HTTPBridgeSessionKey | None = None - quarantine_clear_generation: int | None = None # Session headers provide locality, but only a previous response or # explicit turn-state header guarantees continuity for stale recovery. hard_continuity_anchor: bool = False diff --git a/app/modules/proxy/_service/websocket/helpers.py b/app/modules/proxy/_service/websocket/helpers.py index 4b39742cb2..83858cc645 100644 --- a/app/modules/proxy/_service/websocket/helpers.py +++ b/app/modules/proxy/_service/websocket/helpers.py @@ -1386,7 +1386,7 @@ def _rewrite_websocket_suppressed_duplicate_tool_call_completion_event( request_state: _WebSocketRequestState, ) -> tuple[OpenAIEvent | None, dict[str, JsonValue] | None, str | None, str]: rewritten_event_payload = response_failed_event( - _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE, + "stream_incomplete", _facade()._SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE, error_type="server_error", response_id=_websocket_downstream_response_id(request_state), diff --git a/app/modules/proxy/_service/websocket/mixin.py b/app/modules/proxy/_service/websocket/mixin.py index a135ec6543..1c268f7ea9 100644 --- a/app/modules/proxy/_service/websocket/mixin.py +++ b/app/modules/proxy/_service/websocket/mixin.py @@ -5989,7 +5989,7 @@ async def _finalize_websocket_request_state( "account_health_error_handled", False, ) - if request_state.suppressed_duplicate_tool_call: + if request_state.suppressed_duplicate_tool_call and error_code == "stream_incomplete": settlement.account_health_error = False if ( error_code == "stream_incomplete" diff --git a/app/modules/proxy/replay_safety.py b/app/modules/proxy/replay_safety.py index d50accccca..fd44be4fbb 100644 --- a/app/modules/proxy/replay_safety.py +++ b/app/modules/proxy/replay_safety.py @@ -15,10 +15,11 @@ "function_call_output": "function_call", "custom_tool_call_output": "custom_tool_call", "apply_patch_call_output": "apply_patch_call", - "tool_search_output": "tool_search_call", } _TOOL_CALL_TYPES = frozenset(_TOOL_CALL_TYPE_BY_OUTPUT_TYPE.values()) -_ACCOUNT_NEUTRAL_REPLAY_OMITTED_ITEM_TYPES = frozenset({"reasoning", "web_search_call"}) +_ACCOUNT_NEUTRAL_REPLAY_OMITTED_ITEM_TYPES = frozenset( + {"reasoning", "tool_search_call", "tool_search_output", "web_search_call"} +) _INTERNAL_CHAT_MESSAGE_METADATA_FIELD = "internal_chat_message_metadata_passthrough" _ACCOUNT_NEUTRAL_INTERNAL_CHAT_MESSAGE_METADATA_FIELDS = frozenset({"turn_id"}) _ACCOUNT_NEUTRAL_TOOL_TYPES = frozenset({"custom", "function", "web_search", "web_search_preview"}) @@ -38,7 +39,6 @@ "additional_tools", "apply_patch_call", "apply_patch_call_output", - "compaction", "custom_tool_call", "custom_tool_call_output", "function_call", @@ -47,8 +47,6 @@ "input_image", "input_text", "message", - "tool_search_call", - "tool_search_output", } ) _ACCOUNT_NEUTRAL_MESSAGE_CONTENT_TYPES = frozenset( @@ -67,7 +65,6 @@ } _ACCOUNT_NEUTRAL_INPUT_ITEM_FIELDS = { "additional_tools": frozenset({"role", "tools", "type"}), - "compaction": frozenset({"encrypted_content", "id", "status", "type"}), "apply_patch_call": frozenset( { "call_id", @@ -96,22 +93,6 @@ "function_call_output": frozenset( {"call_id", "caller", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "output", "status", "type"} ), - "tool_search_call": frozenset( - {"arguments", "call_id", "caller", "execution", "id", _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, "status", "type"} - ), - "tool_search_output": frozenset( - { - "call_id", - "caller", - "execution", - "id", - _INTERNAL_CHAT_MESSAGE_METADATA_FIELD, - "output", - "status", - "tools", - "type", - } - ), } _ACCOUNT_NEUTRAL_ITEM_STATUSES = frozenset({"completed", "failed"}) _ACCOUNT_NEUTRAL_APPLY_PATCH_OPERATION_FIELDS = { @@ -195,8 +176,6 @@ def project_responses_input_for_account_neutral_fresh_replay( if stored_count <= 0 or stored_count > len(input_items): return None - if not _stored_prefix_compaction_boundary_is_safe(input_items, stored_count=stored_count): - return None projected_items: list[JsonValue] = [] projected_stored_count = 0 @@ -230,14 +209,6 @@ def project_responses_input_for_account_neutral_fresh_replay( ) -def _stored_prefix_compaction_boundary_is_safe(input_items: list[JsonValue], *, stored_count: int) -> bool: - stored_prefix = input_items[:stored_count] - for item in stored_prefix[:-1]: - if isinstance(item, dict) and item.get("type") == "compaction" and _compaction_item_is_self_contained(item): - return False - return True - - def _is_canonical_lite_tool_bundle(item: JsonValue) -> bool: return ( isinstance(item, dict) @@ -274,8 +245,6 @@ def _project_account_neutral_replay_item( ): return None - if item_type == "compaction": - return item if "id" not in item: return item projected_item = dict(item) @@ -300,10 +269,6 @@ def responses_input_items_are_self_contained_fresh_replay(input_items: list[Json item_type = item_type_value if isinstance(item_type_value, str) else None if not _input_item_has_only_known_fields(item, item_type): return False - if item_type == "compaction": - if not _compaction_item_is_self_contained(item): - return False - continue call_id_value = item.get("call_id") call_id = call_id_value if isinstance(call_id_value, str) and call_id_value else None if item_type in _TOOL_CALL_TYPES: @@ -359,10 +324,8 @@ def responses_input_suffix_retains_prior_output( if prefix_state is None: return False pending_suffix_calls, seen_suffix_call_ids = prefix_state - compact_context_prefix = _input_prefix_ends_with_self_contained_compaction(input_items[:stored_count]) - retained_output_seen = compact_context_prefix + retained_output_seen = False retained_output_is_final_answer = False - settled_suffix_call_types: set[str] = set() fresh_followup_seen = False fresh_followup_count = 0 fresh_followup_is_user_message = False @@ -401,17 +364,6 @@ def responses_input_suffix_retains_prior_output( if pending_suffix_calls[0] != (call_type, call_id): return False pending_suffix_calls.popleft() - settled_suffix_call_types.add(call_type) - if ( - compact_context_prefix - and not pending_suffix_calls - and settled_suffix_call_types == {"tool_search_call"} - ): - retained_output_seen = True - retained_output_is_final_answer = False - fresh_followup_seen = False - fresh_followup_count = 0 - fresh_followup_is_user_message = False continue if item_type in (None, "message") and item.get("role") == "assistant": if pending_suffix_calls or not _is_retained_response_message(item): @@ -444,17 +396,6 @@ def responses_input_suffix_retains_prior_output( return retained_output_seen and fresh_followup_seen and not pending_suffix_calls -def _input_prefix_ends_with_self_contained_compaction(input_items: list[JsonValue]) -> bool: - if not input_items: - return False - last_item = input_items[-1] - return ( - isinstance(last_item, dict) - and last_item.get("type") == "compaction" - and _compaction_item_is_self_contained(last_item) - ) - - def responses_input_suffix_matches_pending_tool_calls( input_items: list[JsonValue], *, @@ -687,9 +628,6 @@ def _tool_call_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) return _is_nonblank_string(item.get("name")) and isinstance(item.get("arguments"), str) if item_type == "custom_tool_call": return _is_nonblank_string(item.get("name")) and isinstance(item.get("input"), str) - if item_type == "tool_search_call": - arguments = item.get("arguments") - return isinstance(arguments, dict) and item.get("execution") in (None, "client") operation = item.get("operation") patch = item.get("patch") input_value = item.get("input") @@ -702,10 +640,6 @@ def _tool_call_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) return _is_nonblank_string(input_value) -def _compaction_item_is_self_contained(item: Mapping[str, JsonValue]) -> bool: - return item.get("status") in (None, "completed") and _is_nonblank_string(item.get("encrypted_content")) - - def _caller_is_self_contained(item: Mapping[str, JsonValue]) -> bool: caller = item.get("caller") return caller is None or caller == {"type": "direct"} @@ -743,13 +677,6 @@ def _apply_patch_operation_is_self_contained(operation: JsonValue | None) -> boo def _tool_output_is_self_contained(item_type: str, item: Mapping[str, JsonValue]) -> bool: if item.get("status") not in (None, "completed", "failed"): return False - if item_type == "tool_search_output": - if item.get("execution") not in (None, "client"): - return False - if "tools" in item and not _tools_are_account_neutral(item.get("tools")): - return False - if _tool_search_output_tools_are_self_contained(item): - return True output = item.get("output") if isinstance(output, str): return True @@ -765,15 +692,6 @@ def _tool_output_is_self_contained(item_type: str, item: Mapping[str, JsonValue] ) -def _tool_search_output_tools_are_self_contained(item: Mapping[str, JsonValue]) -> bool: - return ( - item.get("execution") == "client" - and "tools" in item - and _tools_are_account_neutral(item.get("tools")) - and item.get("output") in (None, "") - ) - - def _is_nonblank_string(value: JsonValue | None) -> bool: return isinstance(value, str) and bool(value.strip()) @@ -1015,13 +933,6 @@ def _input_items_have_valid_account_neutral_shape(input_items: list[JsonValue]) if not _input_content_part_is_self_contained(item, allow_output=False): return False continue - if item_type == "tool_search_output": - execution = item.get("execution") - if execution is not None and execution != "client": - return False - if "tools" in item and not _tools_are_account_neutral(item.get("tools")): - return False - continue if item_type == "additional_tools": if item.get("role") != "developer" or not _tools_are_account_neutral(item.get("tools")): return False @@ -1106,8 +1017,6 @@ def _contains_account_scoped_input_state(value: JsonValue) -> bool: return True if item_type == "additional_tools" and not _tools_are_account_neutral(current.get("tools")): return True - if item_type == "compaction" and _compaction_item_is_self_contained(current): - continue if ( isinstance(item_type, str) and (item_type.endswith("_call") or item_type.endswith("_call_output")) diff --git a/app/modules/proxy/service.py b/app/modules/proxy/service.py index 5153935817..cf6fcbafd0 100644 --- a/app/modules/proxy/service.py +++ b/app/modules/proxy/service.py @@ -221,9 +221,6 @@ from app.modules.proxy._service.http_bridge.helpers import ( _http_bridge_payload_without_previous_response_id as _http_bridge_payload_without_previous_response_id, ) -from app.modules.proxy._service.http_bridge.helpers import ( - _http_bridge_pending_response_events_seen as _http_bridge_pending_response_events_seen, -) from app.modules.proxy._service.http_bridge.helpers import ( _http_bridge_precreated_retry_failure_error as _http_bridge_precreated_retry_failure_error, ) @@ -876,7 +873,6 @@ def _proxy_admission_wait_timeout_seconds(settings: Any | None = None) -> float: _SUPPRESSED_DUPLICATE_TOOL_CALL_MESSAGE = ( "Suppressed duplicate side-effect tool call; upstream response cannot be continued safely." ) -_SUPPRESSED_DUPLICATE_TOOL_CALL_ERROR_CODE = "duplicate_tool_call_replay_suppressed" _WEBSOCKET_PREVIOUS_RESPONSE_ACCOUNT_CACHE_LIMIT = 4096 _WEBSOCKET_CONTINUITY_CACHE_LIMIT = 4096 _SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE = "security_work_authorization_required" @@ -2598,11 +2594,19 @@ def _service_tier_from_event_payload(payload: dict[str, JsonValue] | None) -> st def _effective_service_tier(requested_service_tier: str | None, actual_service_tier: str | None) -> str | None: - return actual_service_tier if isinstance(actual_service_tier, str) else requested_service_tier + if isinstance(actual_service_tier, str): + return actual_service_tier + if isinstance(requested_service_tier, str): + return requested_service_tier + return None def _normalize_service_tier_value(value: JsonValue) -> str | None: if not isinstance(value, str): return None stripped = value.strip() - return "priority" if stripped.lower() == "fast" else stripped or None + if not stripped: + return None + if stripped.lower() == "fast": + return "priority" + return stripped diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml b/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml deleted file mode 100644 index 84cfc12459..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/.openspec.yaml +++ /dev/null @@ -1,2 +0,0 @@ -schema: spec-driven -created: 2026-08-06 diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/design.md b/openspec/changes/fork-safe-model-transition-owner-conflict/design.md deleted file mode 100644 index 298bebfe72..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/design.md +++ /dev/null @@ -1,60 +0,0 @@ -## Context - -The durable full-resend reconciliation in the sibling change handles a -verified complete replay. A model transition is a separate path: the request -has no previous-response anchor to replay, but the durable lookup still binds -the old model's owner. When a stale hard alias disagrees, blindly selecting a -new account would be unsafe, while retrying the same alias produces a stable -502 `continuity_owner_conflict` loop. - -## Decision - -Keep the recovery gate inside the HTTP bridge creation loop. Inspect the typed -error code and continue only when it is `continuity_owner_conflict`; do not -reuse the broader owner-unavailable predicate. Reject forwarded requests so a -replica cannot create a local lane after an origin forwarding failure. Validate -the effective Responses payload with the existing account-neutral replay -classifier, which rejects `previous_response_id`, conversation state, -account-scoped hosted references such as `input_file.file_id`, hosted/MCP call -items, and any input item that is not self-contained. That classifier is shared -with the post-compaction recovery work, so its admitted shapes can widen over -time: a completed `compaction` item carrying its own encrypted content and -client-executed `tool_search_*` items are now accepted as self-contained, while -a compaction placeholder without that content still fails closed. File-owner -resolution and previous-response checks remain explicit defensive gates. - -On success, strip session/turn aliases, replace the request key with a -server-namespaced account-neutral key, exclude the failed owner, clear the -preferred owner/provenance, and force local creation. Persisted hard aliases -are not rewritten. - -The request state itself is reset to the child's own identity. It was prepared -for the parent lane before the creation loop, and the retry re-enters that loop -without rebuilding it, so the fork clears the parent affinity policy, the hard -continuity anchor, and a reused parent turn state. Leaving those set would let -the submit and clean-close paths treat the account-neutral child as the old -owner-bound turn: a stale anchor blocks the pre-output account switch that the -neutrality proof already permits, and a clean close would recover it as a -continuation of the parent turn alias. - -The child lane key is pinned `hard` rather than inheriting the implicit -strength default. The sibling model-transition fresh-resend path may use a -`soft` key because it substitutes a proved full-resend projection that any -account can serve at any point. This fork forwards the client's own payload -unchanged, so once the child lane owns upstream turn state there is no verified -replay text that would make a later soft reroute to a third account safe. - -## Negative Controls - -- A forwarded request with the same owner conflict must return the original - `continuity_owner_conflict` without a second creation attempt. -- A fresh payload containing an unpinned `input_file.file_id` must also return - the original conflict without account-neutral forking. -- A post-compaction payload whose `compaction` item only references the owner's - compacted context — no encrypted content of its own, or still in progress — - must return the original conflict instead of forking, because the prior turns - exist only behind the previous owner. -- A second conflict on the child lane must surface the original error instead of - forking again. -- The forked child request must not keep the parent's affinity policy, hard - continuity anchor, or reused parent turn state. diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md b/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md deleted file mode 100644 index 567047de7b..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/proposal.md +++ /dev/null @@ -1,37 +0,0 @@ -## Why - -An HTTP bridge model transition can resolve a durable model owner while a -legacy hard alias resolves to a different account. The bridge then returns -`continuity_owner_conflict` before dispatch even when the request is a fresh, -account-neutral payload that can safely start a model-transition child lane. -Forwarded requests and payloads that still depend on account-scoped state must -remain fail-closed. - -## What Changes - -- Permit a model-transition child lane only for the exact - `continuity_owner_conflict` error. -- Require a local request, no `previous_response_id`, no resolved file owner, - and a payload proven account-neutral by the existing replay-safety validator. -- Clear session/turn aliases, exclude the conflicting owner, and create a - server-namespaced account-neutral lane, pinned `hard`, without changing - persisted aliases. -- Keep forwarded requests, unpinned hosted files, post-compaction payloads whose - compacted context is not carried in the request, and other owner failures - fail-closed. - -## Capabilities - -### Modified Capabilities - -- `responses-api-compat`: model-transition owner conflicts may use a - proof-gated account-neutral child lane. -- `sticky-session-operations`: the hard-owner conflict exception is limited to - that exact fresh model-transition case. - -## Impact - -- Affected code: HTTP bridge model-transition recovery and unit coverage. -- No schema, setting, dependency, or live deployment change. -- Rollback is a source revert; the existing fail-closed path remains the - fallback for every request that does not satisfy the proof. diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md deleted file mode 100644 index 0a69b5c198..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,64 +0,0 @@ -## ADDED Requirements - -### Requirement: HTTP bridge model-transition owner conflicts use a guarded child lane - -The service MUST use a guarded account-neutral child lane when a hard -continuity lookup identifies a durable owner for an incompatible model and -bridge creation returns `continuity_owner_conflict`; it may retry on that new -server-namespaced lane only when the -request is local (not forwarded), has no `previous_response_id`, has no -resolved file owner, and the effective payload passes the existing -account-neutral fresh-replay validator. The child lane MUST clear session and -turn aliases, reset the request's parent-derived affinity policy, hard -continuity anchor, and reused parent turn state, exclude the conflicting owner, -force local creation, and remain owner-bound (`hard`) once created so a later -capacity failure cannot reroute the same request to a third account. The service MUST attempt this child lane at -most once per request and MUST preserve the original conflict for every other -owner error or payload shape. - -#### Scenario: Neutral model transition conflict forks locally - -- **GIVEN** a durable hard continuity key belongs to account A for the old model -- **AND** the requested model is incompatible with that durable session -- **AND** the effective payload is account-neutral and has no previous-response - or resolved file owner -- **AND** bridge creation returns `continuity_owner_conflict` -- **WHEN** the request is local to the current replica -- **THEN** the service creates one server-namespaced account-neutral child lane -- **AND** that child lane key is owner-bound (`hard`) -- **AND** it excludes account A and does not forward the request -- **AND** the child request carries no parent affinity policy, hard continuity - anchor, or reused parent turn state -- **AND** it leaves the original hard aliases unchanged - -#### Scenario: Forwarded model transition conflict remains fail-closed - -- **GIVEN** the same durable model-transition conflict occurs for a forwarded - request -- **THEN** the service returns `continuity_owner_conflict` -- **AND** it does not create an account-neutral child lane - -#### Scenario: Account-bound payload remains fail-closed - -- **GIVEN** the effective model-transition payload contains an account-scoped - hosted reference such as an unpinned `input_file.file_id` -- **WHEN** bridge creation returns `continuity_owner_conflict` -- **THEN** the service returns `continuity_owner_conflict` -- **AND** it does not retry on another account - -#### Scenario: Post-compaction payload without carried compact context stays fail-closed - -- **GIVEN** the effective model-transition payload contains a `compaction` item - that is not self-contained, such as a placeholder with no encrypted content or - a compaction still in progress -- **WHEN** bridge creation returns `continuity_owner_conflict` -- **THEN** the service returns `continuity_owner_conflict` -- **AND** it does not fork the request onto an account that never held the - compacted context - -#### Scenario: Child lane conflict is not forked again - -- **GIVEN** the guarded child lane was already created for this request -- **WHEN** creation on that lane also returns `continuity_owner_conflict` -- **THEN** the service returns that error to the caller -- **AND** it does not create a further account-neutral lane diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md b/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md deleted file mode 100644 index 86ed8e4680..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/specs/sticky-session-operations/spec.md +++ /dev/null @@ -1,171 +0,0 @@ -## MODIFIED Requirements - -### Requirement: Hard continuity remains owner-bound and bounded - -Requests that depend on `previous_response_id`, hard turn-state, nonblank `conversation`, account-scoped `input_file.file_id` pins, live or durable bridge ownership, replay/reattach state, or another required owner continuity source MUST NOT silently reroute to an account that cannot preserve continuity. A resolved required owner MUST override bare process-session locality and MUST be selected without consulting or rewriting that soft mapping. A `previous_response_id` is a stored-object continuation reference and remains owner-bound even when the same request also carries a session header, `prompt_cache_key`, or another soft locality key. If independently resolved hard sources identify different accounts, if live durable referenced-file pins identify different accounts, or if a request has partial live durable file-pin coverage, the service MUST fail closed before upstream dispatch. A request for which no referenced file has a live durable pin MUST preserve opaque `file_id` compatibility and proceed without inventing ownership evidence. If the owner account/session is unavailable or saturated, the service MUST fail closed with an explicit retryable continuity/local overload reason instead of flooding the owner queue indefinitely. - -Every HTTP, compact, direct WebSocket, and HTTP-bridge transport MUST resolve explicit turn state against both live and durable bridge aliases. Live, durable, previous-response, file, and explicit turn-state evidence MUST be compared independently; source ordering MUST NOT choose the first match when distinct sessions or accounts resolve. A reused direct WebSocket MUST repeat nonblank `conversation` ownership validation for each response-create frame because the existing socket account proves only the current route. Single-account routing MUST constrain effective routing without narrowing the ownership-candidate pool used by that validation. - -When an HTTP-bridge owner is on another replica, the origin MUST forward its resolved durable file owner in authenticated full-context metadata. The receiving owner MUST perform its own fresh shared-database lookup and MUST require that durable result to match the forwarded owner. A missing or conflicting receiver-side durable owner MUST fail closed before account selection or upstream invocation. A retired direct WebSocket's upstream turn-state token MUST NOT be sent to a different account selected for a later movable bare-session request. - -A nonblank `conversation` without a dedicated resolved owner MUST proceed only when an explicit hard Codex mapping proves ownership or exactly one account remains in the model/API-key/security-scoped selection pool before transient additional-quota availability, retry exclusions, runtime health, budget, or account-cap filtering. A temporarily quota-filtered, excluded, unhealthy, or capped candidate MUST remain part of this ambiguity check because it may be the actual owner. A bare process-session mapping MUST NOT prove conversation ownership. - -The sole exception to the hard-owner conflict rule is the proof-gated HTTP -bridge model-transition child lane defined by `responses-api-compat`. It -applies only to a local request that fails with the exact -`continuity_owner_conflict` error, carries no `previous_response_id` and no -resolved file owner, and whose effective payload passes the account-neutral -fresh-replay validator. That child lane MUST clear session and turn aliases, -exclude the failed owner for that request only, and leave persisted hard -mappings unchanged. Every other hard-owner conflict MUST still fail closed. - -#### Scenario: Previous-response owner queue is saturated - -- **WHEN** a `/v1/responses` follow-up requires a previous-response owner -- **AND** the owner session queue or account cap is saturated -- **THEN** the service fails closed with `hard_affinity_saturated`, `previous_response_owner_unavailable`, or the applicable stable `account_stream_cap` / `account_response_create_cap` code -- **AND** it does not route to an unrelated account that lacks continuity state - -#### Scenario: File-pinned request owner is capped - -- **WHEN** a `/v1/responses` request references an `input_file.file_id` pinned to an owner account -- **AND** the owner account is at its account stream or response-create cap -- **THEN** the service returns a local account-cap overload for the owner -- **AND** it does not route the file reference to another account - -#### Scenario: File-pinned request owner overrides process-session locality - -- **GIVEN** a request carries a bare process-session header mapped to account A -- **AND** its `input_file.file_id` is durably pinned to account B -- **WHEN** the request is routed -- **THEN** account B is treated as the required owner -- **AND** the process-session mapping is neither consulted as an owner nor rewritten - -#### Scenario: File-pinned request owner overrides thread locality - -- **GIVEN** a request carries a `thread-id` whose bounded mapping points to account A -- **AND** its `input_file.file_id` is durably pinned to account B -- **WHEN** the request is routed -- **THEN** account B is treated as the required owner -- **AND** the thread mapping is neither consulted as an owner nor rewritten - -#### Scenario: Conflicting hard owners fail closed - -- **GIVEN** a turn state, previous response, bridge, or input file resolves to account A -- **AND** another hard source on the same request resolves to account B -- **AND** the request is not the guarded model-transition child-lane case -- **WHEN** the request is routed -- **THEN** the service fails with `continuity_owner_conflict` before upstream dispatch -- **AND** source ordering does not choose either owner - -#### Scenario: Partial or cross-account file pins fail closed - -- **GIVEN** a request references multiple account-scoped input files -- **AND** at least one file has a live durable owner pin -- **AND** another file has no live durable owner pin or the live pins resolve to different accounts -- **WHEN** the request is routed -- **THEN** the service fails with `file_owner_unavailable` or `continuity_owner_conflict` -- **AND** it does not route the files using a soft affinity account - -#### Scenario: Opaque file IDs with no live durable pins preserve compatibility - -- **GIVEN** a request references one or more `input_file.file_id` values -- **AND** none of those IDs has a live durable owner pin -- **WHEN** the request is routed -- **THEN** the service forwards the opaque file references under ordinary unpinned routing -- **AND** it does not invent a hard owner or fail solely because durable pin metadata is absent - -#### Scenario: Ambiguous conversation fails closed - -- **GIVEN** a request carries nonblank `conversation` continuity and only bare process-session affinity -- **AND** more than one account is eligible -- **WHEN** no dedicated or hard-mapping owner can be resolved -- **THEN** the request fails with a stable owner-unavailable error before upstream dispatch - -#### Scenario: Account-cap pressure does not manufacture a conversation owner - -- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool -- **AND** one account is temporarily at its local account cap -- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner -- **THEN** the request still fails with a stable owner-unavailable error -- **AND** the uncapped account is not treated as the unique owner - -#### Scenario: Retry or additional-quota filtering does not manufacture a conversation owner - -- **GIVEN** two accounts remain in the model/API-key/security-scoped selection pool -- **AND** retry exclusion or transient additional-quota availability removes one from the effective routing pool -- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner -- **THEN** the request still fails with a stable owner-unavailable error -- **AND** the remaining effective account is not treated as the unique owner - -#### Scenario: Account status does not manufacture a conversation owner - -- **GIVEN** two accounts are in the model/API-key/security ownership pool -- **AND** one account is paused, requires reauthentication, deactivated, or otherwise unavailable for routing -- **WHEN** a request carries nonblank `conversation` continuity without a dedicated or hard-mapping owner -- **THEN** the request still fails with a stable owner-unavailable error -- **AND** the active account is not treated as the unique owner - -#### Scenario: Preferred file owner does not manufacture a conversation owner - -- **GIVEN** a request carries nonblank `conversation` continuity and a file durably pinned to account B -- **AND** another account remains in the model/API-key/security ownership pool -- **WHEN** no dedicated conversation owner can be resolved -- **THEN** file ownership does not narrow the conversation ambiguity check to account B -- **AND** the request fails closed before upstream dispatch - -#### Scenario: Bridge turn state is owner-bound across transports - -- **GIVEN** an HTTP bridge registered a turn-state alias for account A -- **WHEN** the alias is reused through compact, plain HTTP streaming, or direct WebSocket transport -- **THEN** each transport treats account A as the required owner -- **AND** it does not fall back to unrelated sticky affinity - -#### Scenario: Independent bridge aliases conflict - -- **GIVEN** a live or durable turn-state alias resolves to one bridge session -- **AND** a previous-response alias on the same request resolves to a distinct session or account -- **WHEN** the request is routed -- **THEN** the service fails with `continuity_owner_conflict` -- **AND** alias lookup order does not select either session - -#### Scenario: Reused WebSocket revalidates conversation ownership - -- **GIVEN** a direct upstream WebSocket is already open on account A -- **AND** a later response-create frame carries nonblank `conversation` -- **WHEN** more than one account remains in the ownership-candidate pool -- **THEN** the later frame fails with a stable owner-unavailable error before upstream send -- **AND** the existing socket account is not treated as ownership proof - -#### Scenario: Single-account routing does not manufacture conversation ownership - -- **GIVEN** single-account routing selects account A -- **AND** multiple accounts remain in the model/API-key/security ownership pool -- **WHEN** a request carries nonblank `conversation` without dedicated owner evidence -- **THEN** the request remains ambiguous and fails closed -- **AND** only the effective routing states are constrained to account A - -#### Scenario: Remote bridge owner revalidates forwarded file ownership - -- **GIVEN** origin replica A durably resolves an input file to account A -- **AND** the request's HTTP bridge owner runs on replica B -- **WHEN** replica A forwards the request to replica B with authenticated file-owner metadata -- **THEN** replica B MUST freshly resolve the shared durable pin -- **AND** it MUST accept the forwarded owner only when both owner values match -- **AND** a missing, conflicting, tampered, or legacy-unbound proof MUST be rejected before upstream invocation - -#### Scenario: Retired WebSocket turn state does not cross accounts - -- **GIVEN** a closed upstream WebSocket on account A supplied an account-scoped turn-state token -- **AND** a later movable bare-session frame or marked self-contained goal restart selects account B -- **WHEN** the proxy opens the replacement WebSocket -- **THEN** it removes account A's stale turn-state token before connect -- **AND** account B never receives that token - -#### Scenario: Guarded model-transition exception does not rewrite hard mappings - -- **GIVEN** a request satisfies the account-neutral model-transition child-lane - proof -- **WHEN** the child lane is created -- **THEN** persisted hard aliases remain unchanged -- **AND** the failed owner is excluded only for that request diff --git a/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md b/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md deleted file mode 100644 index a21fca1195..0000000000 --- a/openspec/changes/fork-safe-model-transition-owner-conflict/tasks.md +++ /dev/null @@ -1,21 +0,0 @@ -## 1. Contract - -- [x] 1.1 Define the exact conflict, local-request, payload-neutrality, and - owner-preservation gates. -- [x] 1.2 Add the sticky-session exception without weakening unrelated hard - owner conflicts. - -## 2. Implementation - -- [x] 2.1 Add the guarded account-neutral model-transition child-lane path. -- [x] 2.2 Pin the child lane key strength explicitly instead of relying on the - implicit default. -- [x] 2.3 Reset the child request state's parent-derived affinity policy, - continuity anchor, and reused parent turn state. -- [x] 2.4 Add positive and forwarded/unpinned-file/post-compaction negative - regressions plus the single-retry bound. - -## 3. Verification - -- [x] 3.1 Run model-transition unit and existing HTTP bridge integration tests. -- [x] 3.2 Run Ruff, type checks, and strict OpenSpec validation. diff --git a/openspec/changes/recover-post-compact-bridge-replays/proposal.md b/openspec/changes/recover-post-compact-bridge-replays/proposal.md deleted file mode 100644 index 50d41626a5..0000000000 --- a/openspec/changes/recover-post-compact-bridge-replays/proposal.md +++ /dev/null @@ -1,28 +0,0 @@ -# Recover post-compact HTTP bridge replays - -## Why - -Post-compaction Codex turns can carry a compact context item, completed -tool-search call/output side effects, and a fresh user message. When the HTTP -bridge treats a session-level compact-anchor trim as a generically safe fresh -replay, later recovery may drop the durable context or tool-search side effects -that make the follow-up self-contained. - -## What Changes - -- Treat completed `compaction` items with encrypted content as self-contained - account-neutral replay context. -- Preserve completed `tool_search_call` / `tool_search_output` pairs when - projecting a compacted fresh replay payload. -- Preserve compact context when projecting a fresh replay payload, while removing - response-owned ids. -- Keep session-level compact-anchor trim safety separate from durable full-resend - proof; trimming a stored prefix does not automatically make an unanchored - replay safe. -- Retire stale response-create gate holders with evidence about whether upstream - response events were already observed, so wedged bridge sessions take the - existing recovery/quarantine path. - -## Impact - -Post-compaction follow-up turns recover with the compact context preserved. diff --git a/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md b/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md deleted file mode 100644 index c1024820ec..0000000000 --- a/openspec/changes/recover-post-compact-bridge-replays/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,54 +0,0 @@ -## ADDED Requirements - -### Requirement: Post-compact bridge replays preserve compact context - -codex-lb MUST treat completed post-compaction replay context as self-contained when an HTTP bridge or Responses WebSocket request must recover a follow-up turn after compaction. A projected fresh replay payload MUST retain that compact context while removing response-owned bookkeeping ids. - -Trimming a stored prefix because a session-level compact anchor was injected -MUST NOT by itself mark the unanchored request safe to replay. The proxy may -mark the unanchored request replay-safe only when the original anchor site had -already made that decision or when a durable full-resend proof shows the fresh -suffix is self-contained. - -Account-neutral recovery MAY select another eligible account after the previous -owner has been excluded or proved silent. Requests that explicitly require a -preferred previous-response owner MUST continue to fail closed when that owner -is unavailable. - -#### Scenario: id-free completed compaction and tool-search context survives fresh replay projection - -- **GIVEN** a follow-up payload starts with a completed `compaction` item whose - encrypted content is non-empty -- **AND** that compaction item does not carry an `id` -- **AND** the payload also carries a completed `tool_search_call` / - `tool_search_output` pair followed by a fresh user message -- **WHEN** codex-lb projects an account-neutral fresh replay payload -- **THEN** the projected payload includes the compaction item -- **AND** it preserves the completed tool-search pair without response-owned ids -- **AND** the projected payload is eligible for account-neutral replay - -#### Scenario: session-level compact trim does not fabricate replay safety - -- **GIVEN** a session-level compact anchor trimmed a stored prefix from a - follow-up request -- **AND** the original request state was not already known to be safe as an - unanchored fresh replay -- **WHEN** the bridge records the retained fresh request text -- **THEN** codex-lb does not mark that retained request as retry-safe solely - because the trim happened - -#### Scenario: account-neutral recovery can leave a silent owner - -- **GIVEN** an account-neutral HTTP bridge recovery request excludes the previous - owner account after it failed to acknowledge `response.create` -- **WHEN** another eligible account is available -- **THEN** codex-lb reconnects on that replacement account and sends the retained - request there - -#### Scenario: required previous-response owner still fails closed - -- **GIVEN** a follow-up request explicitly requires its preferred - previous-response owner account -- **WHEN** that owner is unavailable -- **THEN** codex-lb returns the previous-response-owner-unavailable failure - instead of silently rebinding the request to another account diff --git a/openspec/changes/recover-post-compact-bridge-replays/tasks.md b/openspec/changes/recover-post-compact-bridge-replays/tasks.md deleted file mode 100644 index 161d670a83..0000000000 --- a/openspec/changes/recover-post-compact-bridge-replays/tasks.md +++ /dev/null @@ -1,9 +0,0 @@ -# Tasks - -- [x] Accept compact context and tool-search call/output pairs in account-neutral replay safety checks. -- [x] Preserve completed compaction items in projected fresh replay payloads without retaining response-owned ids. -- [x] Keep session-level trim safety from overriding the original replay-safety decision unless a durable full-resend proof exists. -- [x] Allow account-neutral bridge recovery to select another eligible account after the silent owner is excluded. -- [x] Keep explicit required-owner continuity failures fail-closed. -- [x] Pass response-event evidence into stale response-create gate retirement. -- [x] Add replay-safety and HTTP bridge regression coverage for post-compact recovery. diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md b/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md deleted file mode 100644 index 2b7a6fe58f..0000000000 --- a/openspec/changes/repair-retired-identity-warmup-stamp/proposal.md +++ /dev/null @@ -1,28 +0,0 @@ -## Why - -Local August 14, 2026 builds could stamp SQLite databases at the retired -`20260814_020000_merge_identity_and_warmup_heads` merge id even when the -current mainline file-pin, sticky-abandonment-scope, pending-deletion, -API-key reasoning-policy, and model-source-embeddings lineage had never run. -Those databases also kept two artifacts current main no longer owns: -`idx_accounts_chatgpt_account_id` and -`quota_planner_decisions.lease_expires_at`. Current main therefore treats the -stamp as schema-ahead and a drift check on the live clone reports eleven schema -diffs. - -## What Changes - -- Auto-remap the retired merge stamp to the current pre-repair Alembic head so - upgrade can continue through normal `python -m app.db.migrate upgrade`. -- Add one forward-only repair migration that replays the guarded current-main - migrations needed by the stamped-local shape and drops the two stale - artifacts when present. -- Cover the representative SQLite shape with a regression test that proves the - repair converges to the current ORM schema. - -## Capabilities - -### Modified Capabilities - -- `database-migrations`: retired local merge stamps upgrade cleanly to the - current schema without manual restamping. diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md b/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md deleted file mode 100644 index bf033e7b54..0000000000 --- a/openspec/changes/repair-retired-identity-warmup-stamp/specs/database-migrations/spec.md +++ /dev/null @@ -1,23 +0,0 @@ -## ADDED Requirements - -### Requirement: Retired identity/warmup merge stamps repair to current schema - -The system MUST upgrade a database stamped at -`20260814_020000_merge_identity_and_warmup_heads` by the retired local merge -build to the current Alembic head without manual restamping. Startup or CLI -remap MAY rewrite that retired stamp to the canonical pre-repair revision, but -the subsequent upgrade MUST execute a forward repair that converges the schema -to ORM metadata. The repaired schema MUST add the file-pin, -sticky-abandonment-scope, pending-deletion, API-key reasoning-policy, and -model-source-embeddings objects current main expects, and MUST remove the -retired `idx_accounts_chatgpt_account_id` index and -`quota_planner_decisions.lease_expires_at` column if they are still present. - -#### Scenario: A SQLite clone stamped at the retired merge head upgrades cleanly - -- **GIVEN** a SQLite database stamped at `20260814_020000_merge_identity_and_warmup_heads` -- **AND** the schema still lacks `file_account_pins`, pending-deletion markers, API-key reasoning policy, model-source embeddings, and sticky abandonment scope -- **AND** the retired `idx_accounts_chatgpt_account_id` index and `quota_planner_decisions.lease_expires_at` column are still present -- **WHEN** startup or `python -m app.db.migrate upgrade` runs to head -- **THEN** the upgrade completes without manual stamp surgery -- **AND** `python -m app.db.migrate check` reports no schema drift diff --git a/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md b/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md deleted file mode 100644 index 24d18a21a4..0000000000 --- a/openspec/changes/repair-retired-identity-warmup-stamp/tasks.md +++ /dev/null @@ -1,9 +0,0 @@ -## 1. Repair Path - -- [x] 1.1 Remap `20260814_020000_merge_identity_and_warmup_heads` to the current pre-repair Alembic revision -- [x] 1.2 Add a forward-only repair migration that replays the missing current-main migrations and removes `idx_accounts_chatgpt_account_id` plus `quota_planner_decisions.lease_expires_at` when present - -## 2. Validation - -- [x] 2.1 Add a regression test for the representative SQLite drift shape stamped at the retired merge head -- [x] 2.2 Validate OpenSpec and the focused migration tests diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml deleted file mode 100644 index f774115be7..0000000000 --- a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/.openspec.yaml +++ /dev/null @@ -1,2 +0,0 @@ -schema: spec-driven -created: 2026-08-20 diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md deleted file mode 100644 index 3274bca3b8..0000000000 --- a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/proposal.md +++ /dev/null @@ -1,14 +0,0 @@ -# Report suppressed duplicate tool-call terminals - -## Why - -A duplicate side-effect tool-call replay cannot safely continue, but every -transport still needs to settle the request and report the same retryable -outcome. - -## What Changes - -- Emit a specific failed terminal instead of misclassifying the replay as an - incomplete upstream stream. -- Keep settlement, durable operation persistence, and account-health fencing - aligned across direct SSE, the HTTP bridge, and WebSocket clients. diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md deleted file mode 100644 index 300cb81617..0000000000 --- a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/specs/responses-api-compat/spec.md +++ /dev/null @@ -1,22 +0,0 @@ -# responses-api-compat Delta - -## ADDED Requirements - -### Requirement: Suppressed duplicate side-effect replays receive a retryable terminal failure - -When a replayed side-effecting tool call is suppressed and its upstream turn subsequently reports `response.completed`, the proxy MUST deliver a `response.failed` terminal with code `duplicate_tool_call_replay_suppressed`; it MUST use the downstream response id, treat the request as non-success, persist a terminal durable HTTP-bridge operation when that transport is used, and MUST NOT penalize the upstream account for the intentionally fenced replay. - -#### Scenario: HTTP bridge client receives a terminal failure - -- **GIVEN** an HTTP bridge request suppresses a replayed side-effecting tool call -- **WHEN** the upstream emits `response.completed` for that replay -- **THEN** the client receives `response.failed` with code `duplicate_tool_call_replay_suppressed` -- **AND** the bridge operation is terminal rather than left pending -- **AND** the request is recorded as non-success - -#### Scenario: WebSocket and direct SSE have equivalent terminal semantics - -- **GIVEN** either a WebSocket or direct SSE request suppresses a replayed side-effecting tool call -- **WHEN** the upstream emits `response.completed` for that replay -- **THEN** the client receives the same `response.failed` code -- **AND** the upstream account is not penalized for the intentionally suppressed replay diff --git a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md b/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md deleted file mode 100644 index 3389916cb9..0000000000 --- a/openspec/changes/report-suppressed-duplicate-tool-call-terminal/tasks.md +++ /dev/null @@ -1,9 +0,0 @@ -## 1. Implementation - -- [x] 1.1 Emit the explicit duplicate-tool-call replay failure from all terminal paths. -- [x] 1.2 Preserve non-success settlement and account-health fencing. - -## 2. Validation - -- [x] 2.1 Run focused HTTP bridge and WebSocket regression tests. -- [x] 2.2 Run strict OpenSpec validation. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 5bf9aed891..909fed0586 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -7594,7 +7594,7 @@ async def fail_legacy_stream(*args, **kwargs): # Scope the soft-affinity key to this test's account so a parallel or # ordered integration run cannot inherit another instance's durable # owner and turn the reconnect assertion into a 409 race. - "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}-{time.monotonic_ns()}", + "prompt_cache_key": f"http-bridge-reconnect-thread-{account_id}", } first = await asyncio.wait_for(async_client.post("/v1/responses", json=payload), timeout=_TEST_SYNC_TIMEOUT_SECONDS) second = await asyncio.wait_for( @@ -7682,7 +7682,6 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - prompt_cache_key = f"http-bridge-previous-response-reconnect-{account_id}-{time.monotonic_ns()}" first = await asyncio.wait_for( async_client.post( "/v1/responses", @@ -7690,7 +7689,7 @@ async def fake_connect_responses_websocket( "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello", - "prompt_cache_key": prompt_cache_key, + "prompt_cache_key": "http-bridge-previous-response-reconnect", }, ), timeout=_TEST_SYNC_TIMEOUT_SECONDS, @@ -7705,7 +7704,7 @@ async def fake_connect_responses_websocket( "model": "gpt-5.1", "instructions": "Return exactly OK.", "input": "hello-again", - "prompt_cache_key": prompt_cache_key, + "prompt_cache_key": "http-bridge-previous-response-reconnect", "previous_response_id": first_body["id"], }, ), @@ -7762,7 +7761,6 @@ async def test_v1_responses_http_bridge_classifies_responses_lite_developer_inte "acc_http_bridge_preserve_fresh_reattach", "http-bridge-preserve-fresh-reattach@example.com", ) - scenario_key = f"{account_id}-{time.monotonic_ns()}" account = await _get_account(account_id) first_upstream = _ClosingInterruptedCustomToolUpstreamWebSocket("resp_preserve_source") replay_upstream = _FakeBridgeUpstreamWebSocket("resp_preserve_replay") @@ -7813,7 +7811,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(service._durable_bridge, "release_live_session", delay_predecessor_release) monkeypatch.setattr(service._durable_bridge, "claim_live_session", observe_replacement_claim) - session_headers = {"x-codex-session-id": f"fresh-reattach-full-resend-{scenario_key}"} + session_headers = {"x-codex-session-id": "fresh-reattach-full-resend"} historical_input = [ *([leading_input_item] if leading_input_item is not None else []), { @@ -13489,162 +13487,6 @@ async def fake_reconnect( assert replacement_upstream.sent_text == [retry_request.request_text] -@pytest.mark.asyncio -async def test_retry_account_neutral_precreated_request_switches_from_silent_account(app_instance, monkeypatch): - from app.modules.proxy.continuity import make_http_bridge_account_neutral_replay_key - - service = get_proxy_service_for_app(app_instance) - recovery_kind, recovery_key = make_http_bridge_account_neutral_replay_key("retry-silent-account") - first_account = cast(Account, SimpleNamespace(id="acct-silent", status=AccountStatus.ACTIVE, plan_type="plus")) - replacement_account = cast( - Account, - SimpleNamespace(id="acct-replacement", status=AccountStatus.ACTIVE, plan_type="plus"), - ) - replacement_upstream = _RecordingUpstreamWebSocket() - session = proxy_module._HTTPBridgeSession( - key=proxy_module._HTTPBridgeSessionKey(recovery_kind, recovery_key, None), - headers={"x-codex-turn-state": "stale-turn-state"}, - affinity=proxy_module._AffinityPolicy(), - request_model="gpt-5.5", - account=first_account, - upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_lock=anyio.Lock(), - pending_requests=deque(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - request_state = proxy_module._WebSocketRequestState( - request_id="req-account-neutral-precreated-retry", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - transport="http", - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.5", "input": []}), - ) - session.pending_requests.append(request_state) - reconnect_calls: list[dict[str, object]] = [] - - async def fake_reconnect( - self, - target_session, - *, - request_state, - restart_reader=False, - require_same_account=False, - require_preferred_account=False, - ): - del self, restart_reader - reconnect_calls.append( - { - "require_same_account": require_same_account, - "require_preferred_account": require_preferred_account, - "preferred_account_id": target_session.account.id, - "excluded_account_ids": set(request_state.excluded_account_ids), - } - ) - target_session.account = replacement_account - target_session.upstream = replacement_upstream - - monkeypatch.setattr(proxy_module.ProxyService, "_reconnect_http_bridge_session", fake_reconnect) - - assert await service._retry_http_bridge_precreated_request(session) is True - - assert reconnect_calls == [ - { - "require_same_account": True, - "require_preferred_account": True, - "preferred_account_id": "acct-silent", - "excluded_account_ids": set(), - } - ] - assert request_state.preferred_account_id == "acct-silent" - assert session.account.id == "acct-replacement" - assert replacement_upstream.sent_text == [request_state.request_text] - - -@pytest.mark.asyncio -async def test_reconnect_required_owner_still_fails_when_owner_unavailable(app_instance, monkeypatch): - service = get_proxy_service_for_app(app_instance) - owner_account = cast( - Account, - SimpleNamespace(id="acct-owner-required", status=AccountStatus.ACTIVE, plan_type="plus"), - ) - session = proxy_module._HTTPBridgeSession( - key=proxy_module._HTTPBridgeSessionKey("prompt_cache", "required-owner-reconnect", None), - headers={}, - affinity=proxy_module._AffinityPolicy(), - request_model="gpt-5.5", - account=owner_account, - upstream=cast(proxy_module.UpstreamWebSocket, _SilentUpstreamWebSocket()), - upstream_control=proxy_module._WebSocketUpstreamControl(), - pending_lock=anyio.Lock(), - pending_requests=deque(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=time.monotonic(), - idle_ttl_seconds=120.0, - ) - request_state = proxy_module._WebSocketRequestState( - request_id="req-required-owner-reconnect", - model="gpt-5.5", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - awaiting_response_created=True, - transport="http", - response_create_gate_acquired=True, - request_text=json.dumps({"type": "response.create", "model": "gpt-5.5", "input": []}), - ) - request_state.preferred_account_id = owner_account.id - selection_calls: list[dict[str, object]] = [] - - async def fake_select_account_with_budget_for_stream(self, deadline, **kwargs): - del self, deadline - selection_calls.append( - { - "preferred_account_id": kwargs.get("preferred_account_id"), - "preferred_account_is_continuity_owner": kwargs.get("preferred_account_is_continuity_owner"), - "fallback_on_preferred_account_unavailable": kwargs.get("fallback_on_preferred_account_unavailable"), - } - ) - return AccountSelection( - account=None, - error_message="Required continuity owner account no longer exists", - error_code=CONTINUITY_OWNER_UNAVAILABLE, - ) - - monkeypatch.setattr( - proxy_module.ProxyService, - "_select_account_with_budget_for_stream", - fake_select_account_with_budget_for_stream, - ) - - with pytest.raises(proxy_module.ProxyResponseError) as exc_info: - await service._reconnect_http_bridge_session( - session, - request_state=request_state, - require_preferred_account=True, - ) - - assert exc_info.value.status_code == 502 - assert exc_info.value.payload["error"]["code"] == "previous_response_owner_unavailable" - assert selection_calls == [ - { - "preferred_account_id": owner_account.id, - "preferred_account_is_continuity_owner": False, - "fallback_on_preferred_account_unavailable": False, - } - ] - - @pytest.mark.asyncio async def test_v1_responses_http_bridge_send_failure_returns_upstream_unavailable( async_client, @@ -15343,7 +15185,7 @@ async def fake_connect_responses_websocket( monkeypatch.setattr(proxy_module.ProxyService, "_ensure_fresh_with_budget", fake_ensure_fresh_with_budget) monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) - session_headers = {"x-codex-session-id": f"quarantine-silent-reattach-{account_id}-{time.monotonic_ns()}"} + session_headers = {"x-codex-session-id": "quarantine-silent-reattach"} historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, { @@ -15515,10 +15357,9 @@ async def fake_connect_responses_websocket( # The turn-state header makes the bridge session a true Codex continuity # session (``session.codex_session``), which is what arms the session-level # anchor injection this regression guards against. - scenario_key = f"{account_id}-{time.monotonic_ns()}" session_headers = { - "x-codex-session-id": f"quarantine-unsafe-suffix-reattach-{scenario_key}", - "x-codex-turn-state": f"quarantine-unsafe-suffix-turn-{scenario_key}", + "x-codex-session-id": "quarantine-unsafe-suffix-reattach", + "x-codex-turn-state": "quarantine-unsafe-suffix-turn", } historical_input = [ {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, diff --git a/tests/integration/test_migrations.py b/tests/integration/test_migrations.py index cec5bc7a24..df70df1b78 100644 --- a/tests/integration/test_migrations.py +++ b/tests/integration/test_migrations.py @@ -1908,120 +1908,3 @@ def _schema_state(sync_conn): assert await conn.run_sync(_schema_state) is not None finally: await engine.dispose() - - -@pytest.mark.asyncio -async def test_file_account_pins_migration_repairs_existing_table_missing_index(tmp_path): - from sqlalchemy import inspect as sa_inspect - - db_url = f"sqlite+aiosqlite:///{tmp_path / 'partial-file-account-pins.sqlite'}" - parent_revision = "20260806_000000_add_anonymous_telemetry" - pin_revision = "20260813_000000_add_file_account_pins" - - await to_thread.run_sync(lambda: run_upgrade(db_url, parent_revision, bootstrap_legacy=False)) - engine = create_async_engine(db_url, future=True) - try: - async with engine.begin() as conn: - await conn.execute( - text( - "CREATE TABLE file_account_pins (" - "file_id VARCHAR NOT NULL PRIMARY KEY, " - "account_id VARCHAR NOT NULL, " - "expires_at DATETIME NOT NULL)" - ) - ) - - await to_thread.run_sync(lambda: run_upgrade(db_url, pin_revision, bootstrap_legacy=False)) - await to_thread.run_sync(lambda: run_upgrade(db_url, pin_revision, bootstrap_legacy=False)) - async with engine.connect() as conn: - indexes = await conn.run_sync( - lambda sync_conn: {index["name"] for index in sa_inspect(sync_conn).get_indexes("file_account_pins")} - ) - assert indexes == {"ix_file_account_pins_expires_at"} - await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) - assert check_schema_drift(db_url) == () - finally: - await engine.dispose() - - -@pytest.mark.asyncio -async def test_retired_identity_and_warmup_merge_stamp_repairs_to_head(tmp_path): - from alembic import command - from sqlalchemy import inspect as sa_inspect - - from app.db.migrate import _build_alembic_config - - db_url = f"sqlite+aiosqlite:///{tmp_path / 'retired-identity-warmup-repair.sqlite'}" - pre_repair_head = "20260816_000000_add_model_source_embeddings" - retired_merge_revision = "20260814_020000_merge_identity_and_warmup_heads" - expected_drift_checks = ( - ("add_table", "file_account_pins"), - ("add_index", "ix_file_account_pins_expires_at"), - ("add_column", "accounts', Column('delete_requested_at'"), - ("add_column", "accounts', Column('delete_history_requested'"), - ("remove_index", "idx_accounts_chatgpt_account_id"), - ("add_index", "idx_accounts_delete_requested_at"), - ("add_column", "api_keys', Column('allowed_reasoning_efforts'"), - ("add_constraint", "ck_api_keys_reasoning_policy_exclusive"), - ("add_column", "model_sources', Column('supports_embeddings'"), - ("remove_column", "quota_planner_decisions', Column('lease_expires_at'"), - ("add_column", "sticky_sessions', Column('continuity_abandonment_scope'"), - ) - - def _schema_state(sync_conn): - inspector = sa_inspect(sync_conn) - return { - "has_file_account_pins": inspector.has_table("file_account_pins"), - "account_columns": {column["name"] for column in inspector.get_columns("accounts")}, - "account_indexes": {index["name"] for index in inspector.get_indexes("accounts")}, - "api_key_columns": {column["name"] for column in inspector.get_columns("api_keys")}, - "api_key_checks": { - constraint["name"] - for constraint in inspector.get_check_constraints("api_keys") - if constraint.get("name") - }, - "model_source_columns": {column["name"] for column in inspector.get_columns("model_sources")}, - "quota_columns": {column["name"] for column in inspector.get_columns("quota_planner_decisions")}, - "sticky_columns": {column["name"] for column in inspector.get_columns("sticky_sessions")}, - } - - await to_thread.run_sync(lambda: run_upgrade(db_url, pre_repair_head, bootstrap_legacy=False)) - await to_thread.run_sync( - lambda: command.downgrade(_build_alembic_config(db_url), "20260806_000000_add_anonymous_telemetry") - ) - - engine = create_async_engine(db_url, future=True) - try: - async with engine.begin() as conn: - await conn.execute( - text("CREATE INDEX IF NOT EXISTS idx_accounts_chatgpt_account_id ON accounts (chatgpt_account_id)") - ) - await conn.execute(text("ALTER TABLE quota_planner_decisions ADD COLUMN lease_expires_at DATETIME")) - await conn.execute( - text("UPDATE alembic_version SET version_num = :revision"), - {"revision": retired_merge_revision}, - ) - - drift = check_schema_drift(db_url) - assert len(drift) == len(expected_drift_checks) - for action, marker in expected_drift_checks: - assert any(action in diff and marker in diff for diff in drift) - - result = await to_thread.run_sync(lambda: run_upgrade(db_url, "head", bootstrap_legacy=False)) - assert result.current_revision == _HEAD_REVISION - assert check_schema_drift(db_url) == () - - async with engine.connect() as conn: - state = await conn.run_sync(_schema_state) - assert state["has_file_account_pins"] is True - assert "delete_requested_at" in state["account_columns"] - assert "delete_history_requested" in state["account_columns"] - assert "idx_accounts_chatgpt_account_id" not in state["account_indexes"] - assert "idx_accounts_delete_requested_at" in state["account_indexes"] - assert "allowed_reasoning_efforts" in state["api_key_columns"] - assert "ck_api_keys_reasoning_policy_exclusive" in state["api_key_checks"] - assert "supports_embeddings" in state["model_source_columns"] - assert "lease_expires_at" not in state["quota_columns"] - assert "continuity_abandonment_scope" in state["sticky_columns"] - finally: - await engine.dispose() diff --git a/tests/integration/test_proxy_compact.py b/tests/integration/test_proxy_compact.py index 580a281ed2..6bb285ca26 100644 --- a/tests/integration/test_proxy_compact.py +++ b/tests/integration/test_proxy_compact.py @@ -664,72 +664,6 @@ async def fake_compact(payload, headers, access_token, account_id): ] -@pytest.mark.asyncio -async def test_proxy_compact_preserves_single_output_item_with_skill_context(async_client, monkeypatch): - email = "compact-skill-recovery@example.com" - raw_account_id = "acc_compact_skill_recovery" - auth_json = _make_auth_json(raw_account_id, email) - files = {"auth_json": ("auth.json", json.dumps(auth_json), "application/json")} - response = await async_client.post("/api/accounts/import", files=files) - assert response.status_code == 200 - - seen: dict[str, ResponsesCompactRequest] = {} - - async def fake_compact(payload, headers, access_token, account_id): - del headers, access_token, account_id - seen["payload"] = payload - return CompactResponsePayload.model_validate( - { - "object": "response.compaction", - "compaction_summary": { - "id": "cmp_skill_recovery", - "encrypted_content": "enc_skill_recovery", - }, - } - ) - - monkeypatch.setattr(proxy_module, "core_compact_responses", fake_compact) - - payload = { - "model": "gpt-5.5", - "instructions": "Compact the conversation.", - "input": [ - {"type": "message", "role": "user", "content": "hello"}, - { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": ( - "\n" - "grill-me\n" - "/home/kom/.codex/skills/grill-me/SKILL.md\n" - "---\n" - "name: grill-me\n" - "---\n" - "Ask one question at a time.\n" - "" - ), - } - ], - }, - ], - } - response = await async_client.post("/backend-api/codex/responses/compact", json=payload) - - assert response.status_code == 200 - output = response.json()["output"] - assert output == [ - { - "id": "cmp_skill_recovery", - "type": "compaction", - "encrypted_content": "enc_skill_recovery", - } - ] - assert "grill-me" in json.dumps(seen["payload"].to_payload()) - - @pytest.mark.asyncio async def test_proxy_compact_headers_include_monthly_only_credits(async_client, monkeypatch): email = "compact-monthly@example.com" diff --git a/tests/unit/test_durable_bridge_sessions.py b/tests/unit/test_durable_bridge_sessions.py index c8a50f4f41..d7c3ca1d8c 100644 --- a/tests/unit/test_durable_bridge_sessions.py +++ b/tests/unit/test_durable_bridge_sessions.py @@ -1879,54 +1879,6 @@ async def test_durable_bridge_release_without_draining_marks_session_closed( assert reclaimed.latest_response_id == "resp_2" -@pytest.mark.asyncio -async def test_durable_bridge_ownerless_active_row_can_be_reclaimed_without_takeover( - coordinator: DurableBridgeSessionCoordinator, -) -> None: - claimed = await coordinator.claim_live_session( - session_key_kind="prompt_cache", - session_key_value="ownerless-reclaim", - api_key_id=None, - instance_id="instance-a", - owner_process_epoch="test-process-a", - lease_ttl_seconds=60.0, - account_id="acc-1", - model="gpt-5.4", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=True, - ) - released = await coordinator.release_live_session( - session_id=claimed.session_id, - instance_id="instance-a", - owner_epoch=claimed.owner_epoch, - draining=True, - ) - - assert released is not None - assert released.owner_instance_id is None - - reclaimed = await coordinator.claim_live_session( - session_key_kind="prompt_cache", - session_key_value="ownerless-reclaim", - api_key_id=None, - instance_id="instance-b", - owner_process_epoch="test-process-b", - lease_ttl_seconds=60.0, - account_id="acc-1", - model="gpt-5.4", - service_tier=None, - latest_turn_state=None, - latest_response_id=None, - allow_takeover=False, - ) - - assert reclaimed.session_id == claimed.session_id - assert reclaimed.owner_instance_id == "instance-b" - assert reclaimed.owner_epoch == claimed.owner_epoch + 1 - - @pytest.mark.asyncio async def test_durable_bridge_takeover_clears_stale_recovery_anchor_for_fresh_session( coordinator: DurableBridgeSessionCoordinator, diff --git a/tests/unit/test_openai_requests.py b/tests/unit/test_openai_requests.py index 3c868df01c..04e8cd6e60 100644 --- a/tests/unit/test_openai_requests.py +++ b/tests/unit/test_openai_requests.py @@ -2398,89 +2398,6 @@ def test_compact_trimming_preserves_codex_goal_context_anchor_from_middle(): assert dumped_input[-1] == input_items[-1] -def test_compact_trimming_does_not_anchor_active_skill_context_from_middle(): - skill_context = { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": ( - "\n" - "grill-me\n" - "/home/kom/.codex/skills/grill-me/SKILL.md\n" - "---\n" - "name: grill-me\n" - "---\n" - "Ask one question at a time and keep the interview mode active.\n" - "" - ), - } - ], - } - input_items = [ - {"role": "user", "content": "initial instructions"}, - {"role": "assistant", "content": "x" * 300_000}, - skill_context, - {"role": "assistant", "content": "y" * 500_000}, - {"role": "user", "content": "latest request"}, - ] - payload = { - "model": "gpt-5.1", - "instructions": "hi", - "input": input_items, - } - - request = ResponsesCompactRequest.model_validate(payload) - dumped = request.to_payload() - dumped_input = dumped["input"] - - assert isinstance(dumped_input, list) - assert dumped_input[0] == input_items[0] - assert dumped_input[-1] == input_items[-1] - assert skill_context not in dumped_input - assert input_items[1] not in dumped_input - assert input_items[3] not in dumped_input - - -def test_compact_trimming_does_not_anchor_plain_skill_catalog_mentions(): - catalog_context = { - "type": "message", - "role": "user", - "content": [ - { - "type": "input_text", - "text": ( - "\n" - "- grill-me: Interview the user one question at a time.\n" - "" - ), - } - ], - } - input_items = [ - {"role": "user", "content": "initial instructions"}, - {"role": "assistant", "content": "x" * 300_000}, - catalog_context, - {"role": "assistant", "content": "y" * 500_000}, - {"role": "user", "content": "latest request"}, - ] - payload = { - "model": "gpt-5.1", - "instructions": "hi", - "input": input_items, - } - - request = ResponsesCompactRequest.model_validate(payload) - dumped = request.to_payload() - dumped_input = dumped["input"] - - assert isinstance(dumped_input, list) - assert catalog_context not in dumped_input - assert dumped_input[0] == input_items[0] - assert dumped_input[-1] == input_items[-1] - - def test_compact_trimming_preserves_non_message_developer_directive_from_middle(): developer_directive = { "type": "future_directive", diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index e3ea00d5f5..a8b8fd5628 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -59,7 +59,6 @@ from app.modules.proxy.durable_bridge_repository import ( DurableBridgeAliasRegistration, DurableBridgeAliasRegistrationReceipt, - durable_bridge_hash, ) from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch from app.modules.proxy.http_bridge_event_batcher import TerminalOperationEventAppendResult @@ -2587,10 +2586,8 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, - response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" - del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2698,10 +2695,8 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, - response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" - del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2791,10 +2786,8 @@ async def fake_retire( *, detail: str, retry_circuit_attempt_selection: proxy_support_module._HTTPBridgeRetryCircuitAttemptSelection, - response_events_seen: int | None = None, ) -> None: assert retry_circuit_attempt_selection.kind == "absent" - del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -2895,9 +2888,7 @@ async def fake_retire( retire_session: proxy_service._HTTPBridgeSession, *, detail: str, - response_events_seen: int | None = None, ) -> None: - del response_events_seen retire_calls.append(detail) retire_session.closed = True @@ -13699,7 +13690,6 @@ async def test_stream_via_http_bridge_preserves_context_after_owner_unavailable( retained_output = { "type": "message", "role": "assistant", - "id": "msg_response_owned", "content": [{"type": "output_text", "text": "two"}], } input_items = [*prefix_items] @@ -24144,19 +24134,18 @@ async def test_stream_via_http_bridge_fails_closed_before_file_affinity_when_pre @pytest.mark.asyncio @pytest.mark.parametrize( - ("unsafe_replay_input", "replace_retired_gate", "stored_model", "pending_manifest_replay"), + ("unsafe_replay_input", "replace_retired_gate", "stored_model"), [ - pytest.param(None, False, None, False, id="retained-output"), - pytest.param(None, False, "gpt-5.3", False, id="retained-output-stored-model"), - pytest.param(None, True, None, False, id="retained-output-replace-retired-gate"), - pytest.param(None, False, None, True, id="pending-tool-manifest"), - pytest.param("conversation", False, None, False, id="conversation"), - pytest.param("file", False, None, False, id="file"), - pytest.param("missing_prior_output", False, None, False, id="missing-prior-output"), - pytest.param("orphan_output", False, None, False, id="orphan-output"), - pytest.param("response_owned_developer", False, None, False, id="response-owned-developer"), - pytest.param("response_owned_stored_developer", False, None, False, id="response-owned-stored-developer"), - pytest.param("missing_owner", False, None, False, id="missing-owner"), + (None, False, None), + (None, False, "gpt-5.3"), + (None, True, None), + ("conversation", False, None), + ("file", False, None), + ("missing_prior_output", False, None), + ("orphan_output", False, None), + ("response_owned_developer", False, None), + ("response_owned_stored_developer", False, None), + ("missing_owner", False, None), ], ) async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_when_owner_is_unavailable( @@ -24164,7 +24153,6 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe unsafe_replay_input: str | None, replace_retired_gate: bool, stored_model: str | None, - pending_manifest_replay: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) account_neutral_classifier = Mock( @@ -24219,14 +24207,16 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe ) elif unsafe_replay_input == "orphan_output": historical_input.append({"type": "function_call_output", "call_id": "call_missing", "output": "orphan output"}) - retained_boundary_call: proxy_service.JsonValue = { - "type": "function_call", - "id": "fc_owner", - "call_id": "call_old", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": owner_metadata, - } + historical_input.append( + { + "type": "function_call", + "id": "fc_owner", + "call_id": "call_old", + "name": "lookup", + "arguments": "{}", + "internal_chat_message_metadata_passthrough": owner_metadata, + } + ) retained_boundary_output: proxy_service.JsonValue = { "type": "function_call_output", "call_id": "call_old", @@ -24238,21 +24228,6 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "content": [{"type": "input_text", "text": "next question"}], "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, } - pending_tool_loop: list[proxy_service.JsonValue] = [ - { - "type": "function_call", - "call_id": "call_pending", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - { - "type": "function_call_output", - "call_id": "call_pending", - "output": "fresh result", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - ] retained_prior_output: proxy_service.JsonValue = { "type": "message", "id": "msg_owner", @@ -24277,7 +24252,6 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "call_id": "call_search", "execution": "client", "status": "completed", - "output": "found docs", "tools": [], "internal_chat_message_metadata_passthrough": owner_metadata, }, @@ -24294,17 +24268,10 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe "instructions": "hi", "input": [ *historical_input, - retained_boundary_call, retained_boundary_output, *completed_search_bookkeeping, - *( - pending_tool_loop - if pending_manifest_replay - else [ - *([] if unsafe_replay_input == "missing_prior_output" else [retained_prior_output]), - new_input, - ] - ), + *([] if unsafe_replay_input == "missing_prior_output" else [retained_prior_output]), + new_input, *( [ { @@ -24323,16 +24290,6 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe if unsafe_replay_input == "conversation": payload_data["conversation"] = "conv_owner_scoped" payload = proxy_service.ResponsesRequest.model_validate(payload_data) - stored_context_items = ( - [ - *historical_input, - retained_boundary_call, - retained_boundary_output, - *completed_search_bookkeeping, - ] - if pending_manifest_replay - else historical_input - ) durable_lookup = proxy_service.DurableBridgeLookup( session_id="durable-owner-unavailable", canonical_kind="session_header", @@ -24345,10 +24302,9 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe state=HttpBridgeSessionState.ACTIVE, latest_turn_state="sid-owner-unavailable", latest_response_id="resp_completed_anchor", - latest_input_item_count=len(stored_context_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_context_items), + latest_input_item_count=len(historical_input), + latest_input_full_fingerprint=proxy_service._fingerprint_input_items(historical_input), model=stored_model, - latest_pending_tool_calls={"call_pending": "function_call"} if pending_manifest_replay else None, ) owner_unavailable = ProxyResponseError( 502, @@ -24383,7 +24339,9 @@ async def test_stream_via_http_bridge_projects_plaintext_durable_full_resend_whe replacement_session = _make_bridge_session(key=session.key, key_value=session.key.affinity_key) get_or_create = AsyncMock( side_effect=( - [owner_unavailable, session, replacement_session] if replace_retired_gate else [owner_unavailable, session] + [owner_unavailable, session, replacement_session] + if replace_retired_gate + else [owner_unavailable, capacity_unavailable, session] ) ) captured_request_states: list[proxy_service._WebSocketRequestState] = [] @@ -24494,13 +24452,13 @@ async def fake_stream_events( chunks = [chunk async for chunk in stream] assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert get_or_create.await_count == (3 if replace_retired_gate else 2) + assert get_or_create.await_count == 3 first_call = get_or_create.await_args_list[0] second_call = get_or_create.await_args_list[1] - third_call = get_or_create.await_args_list[2] if replace_retired_gate else None + third_call = get_or_create.await_args_list[2] assert first_call.kwargs["previous_response_id"] is None - assert first_call.kwargs["preferred_account_id"] == (None if stored_model else "acc-owner") - assert first_call.kwargs["allow_forward_to_owner"] is (False if stored_model else True) + assert first_call.kwargs["preferred_account_id"] == "acc-owner" + assert first_call.kwargs["allow_forward_to_owner"] is True assert second_call.kwargs["previous_response_id"] is None assert second_call.kwargs["preferred_account_id"] is None assert second_call.kwargs["durable_lookup"] is None @@ -24508,30 +24466,29 @@ async def fake_stream_events( kind=second_call.args[0].affinity_kind, key=second_call.args[0].affinity_key, ) - if third_call is not None: - assert second_call.args[0] == third_call.args[0] + assert second_call.args[0] == third_call.args[0] assert second_call.args[0] != first_call.args[0] assert second_call.kwargs["affinity"] == proxy_service._AffinityPolicy() assert second_call.kwargs["session_header_fallback_key"] is None - assert second_call.kwargs["exclude_account_ids"] == (None if stored_model else {"acc-owner"}) + assert second_call.kwargs["exclude_account_ids"] == {"acc-owner"} assert second_call.kwargs["allow_forward_to_owner"] is False assert all(key.lower() != "x-codex-turn-state" for key in second_call.kwargs["headers"]) - if third_call is not None: - assert third_call.kwargs["previous_response_id"] is None - assert third_call.kwargs["preferred_account_id"] is None - assert third_call.kwargs["durable_lookup"] is None + assert third_call.kwargs["previous_response_id"] is None + assert third_call.kwargs["preferred_account_id"] is None + assert third_call.kwargs["durable_lookup"] is None # When the fresh-replay session's own gate later times out (session.account # is "acc-fallback"), the next replacement must also exclude it — a # "replacement" that could legally reselect the account that just proved # stuck isn't a replacement at all. - if third_call is not None: - assert third_call.kwargs["exclude_account_ids"] == {"acc-owner", "acc-fallback"} - assert third_call.kwargs["allow_forward_to_owner"] is False + assert third_call.kwargs["exclude_account_ids"] == ( + {"acc-owner", "acc-fallback"} if replace_retired_gate else {"acc-owner"} + ) + assert third_call.kwargs["allow_forward_to_owner"] is False assert captured_request_states[0].previous_response_id is None assert captured_request_states[0].enforce_openai_sdk_contract is False replay_payload = json.loads(captured_text_data[0]) assert "previous_response_id" not in replay_payload - expected_replay_input = [ + assert replay_payload["input"] == [ { "role": "user", "content": [{"type": "input_text", "text": "old question"}], @@ -24551,44 +24508,19 @@ async def fake_stream_events( "internal_chat_message_metadata_passthrough": owner_metadata, }, { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "docs"}, - "execution": "client", + "type": "message", + "role": "assistant", "status": "completed", + "phase": "final_answer", + "content": [{"type": "output_text", "text": "old answer"}], "internal_chat_message_metadata_passthrough": owner_metadata, }, { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "output": "found docs", - "tools": [], - "internal_chat_message_metadata_passthrough": owner_metadata, + "role": "user", + "content": [{"type": "input_text", "text": "next question"}], + "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, }, ] - if pending_manifest_replay: - expected_replay_input.extend(pending_tool_loop) - else: - expected_replay_input.extend( - [ - { - "type": "message", - "role": "assistant", - "status": "completed", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "old answer"}], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "role": "user", - "content": [{"type": "input_text", "text": "next question"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - ] - ) - assert replay_payload["input"] == expected_replay_input assert "encrypted_content" not in captured_text_data[0] assert all("id" not in item for item in replay_payload["input"]) account_neutral_classifier.assert_called_once() @@ -24874,74 +24806,34 @@ async def fail_first_session_before_output( @pytest.mark.asyncio -async def test_durable_model_transition_full_resend_uses_account_neutral_replay( +async def test_stream_via_http_bridge_preserves_verified_replay_kind_for_durable_model_transition( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - owner_metadata: proxy_service.JsonValue = {"turn_id": "turn-owner"} - historical_input: list[proxy_service.JsonValue] = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "old question"}], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call", - "id": "fc_owner", - "call_id": "call_old", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - ] payload = proxy_service.ResponsesRequest.model_validate( { - "model": "gpt-5.3-codex-spark", + "model": "gpt-5.6-terra", "instructions": "hi", - "input": [ - *historical_input, - { - "type": "function_call_output", - "call_id": "call_old", - "output": "old output", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "message", - "id": "msg_owner", - "role": "assistant", - "status": "completed", - "phase": "final_answer", - "content": [{"type": "output_text", "text": "old answer"}], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "role": "user", - "content": [{"type": "input_text", "text": "next question"}], - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - ], + "input": [{"role": "user", "content": "continue on the new model"}], } ) + replay_kind, replay_key = make_http_bridge_account_neutral_replay_key("replay-parent") durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-full-resend", - canonical_kind="session_header", - canonical_key="shared-root", + session_id="durable-replay-parent", + canonical_kind=replay_kind, + canonical_key=replay_key, api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id="instance-a", - owner_epoch=18, + account_id="acc-replay", + owner_instance_id=None, + owner_epoch=1, lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_parent", - latest_response_id="resp_model_parent", - latest_input_item_count=len(historical_input), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(historical_input), - model="gpt-5.4-mini", + latest_turn_state="http_turn_replay_parent", + latest_response_id="resp_replay_parent", + model="gpt-5.6-sol", ) captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] captured_kwargs: list[dict[str, Any]] = [] - captured_text_data: list[str] = [] async def fake_get_or_create( key: proxy_service._HTTPBridgeSessionKey, @@ -24949,21 +24841,15 @@ async def fake_get_or_create( ) -> proxy_service._HTTPBridgeSession: captured_keys.append(key) captured_kwargs.append(kwargs) - session = _make_bridge_session(key=key, key_value=key.affinity_key) - session.account = cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)) + session = _make_bridge_session(key=key) + session.account = cast(Any, SimpleNamespace(id="acc-replay", status=AccountStatus.ACTIVE)) session.request_model = payload.model return session async def fake_stream_events( _session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, **_kwargs: Any, ): - assert request_state.previous_response_id is None - assert request_state.preferred_account_id is None - captured_text_data.append(text_data) yield 'data: {"type":"response.completed"}\n\n' monkeypatch.setattr( @@ -24985,8 +24871,6 @@ async def fake_stream_events( ) monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) @@ -24996,9 +24880,8 @@ async def fake_stream_events( async for chunk in service._stream_via_http_bridge( payload, headers={ - "authorization": "Bearer test-token", + "x-codex-turn-state": "http_turn_replay_parent", "x-codex-session-id": "shared-root", - "x-codex-turn-state": "http_turn_child", }, codex_session_affinity=True, propagate_http_errors=True, @@ -25010,6 +24893,7 @@ async def fake_stream_events( codex_idle_ttl_seconds=1800.0, max_sessions=8, queue_limit=4, + downstream_turn_state="http_turn_replay_child", ) ] @@ -25019,872 +24903,43 @@ async def fake_stream_events( kind=captured_keys[0].affinity_kind, key=captured_keys[0].affinity_key, ) - assert captured_keys[0].strength == "soft" + assert captured_keys[0].affinity_key != durable_lookup.canonical_key assert captured_kwargs[0]["durable_lookup"] is None - assert captured_kwargs[0]["previous_response_id"] is None - assert captured_kwargs[0]["preferred_account_id"] is None - assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is False - assert captured_kwargs[0]["allow_forward_to_owner"] is False - assert captured_kwargs[0]["headers"] == {"authorization": "Bearer test-token"} - replay_payload = json.loads(captured_text_data[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["model"] == "gpt-5.3-codex-spark" - assert replay_payload["input"][-1]["content"] == [{"type": "input_text", "text": "next question"}] - assert all("id" not in item for item in replay_payload["input"]) + assert captured_kwargs[0]["preferred_account_id"] == "acc-replay" + assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is True @pytest.mark.asyncio -async def test_durable_model_transition_full_resend_pending_tool_context_uses_account_neutral_replay( +async def test_get_or_create_http_bridge_session_prompt_cache_mismatch_stays_local_when_gateway_safe_mode_disabled( monkeypatch: pytest.MonkeyPatch, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) - owner_metadata: proxy_service.JsonValue = {"turn_id": "turn-owner"} - stored_context_items: list[proxy_service.JsonValue] = [ - { - "role": "user", - "content": [{"type": "input_text", "text": "old question"}], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call", - "id": "fc_owner", - "call_id": "call_old", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call_output", - "call_id": "call_old", - "output": "old output", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "tool_search_call", - "id": "tsc_owner", - "call_id": "call_search", - "arguments": {"query": "docs"}, - "execution": "client", - "status": "completed", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "output": "found docs", - "tools": [], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - ] - pending_tool_loop: list[proxy_service.JsonValue] = [ - { - "type": "function_call", - "id": "fc_pending", - "call_id": "call_pending", - "name": "lookup_pending", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - { - "type": "function_call_output", - "call_id": "call_pending", - "output": "fresh result", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - ] - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.3-codex-spark", - "instructions": "hi", - "input": [*stored_context_items, *pending_tool_loop], - } + key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache-key", None) + monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) + monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) + monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-b")) + monkeypatch.setattr( + proxy_service, + "_active_http_bridge_instance_ring", + AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-pending-tool", - canonical_kind="session_header", - canonical_key="shared-root", - api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id="instance-a", - owner_epoch=18, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_parent", - latest_response_id="resp_model_parent", - latest_input_item_count=len(stored_context_items), - latest_input_full_fingerprint=proxy_service._fingerprint_input_items(stored_context_items), - latest_pending_tool_calls={"call_pending": "function_call"}, - model="gpt-5.4-mini", + created_session = proxy_service._HTTPBridgeSession( + key=key, + headers={}, + affinity=proxy_service._AffinityPolicy(key="cache-key"), + request_model="gpt-5.4", + account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), + upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), + upstream_control=proxy_service._WebSocketUpstreamControl(), + pending_requests=deque(), + pending_lock=anyio.Lock(), + response_create_gate=asyncio.Semaphore(1), + queued_request_count=0, + last_used_at=2.0, + idle_ttl_seconds=120.0, ) - captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - captured_kwargs: list[dict[str, Any]] = [] - captured_text_data: list[str] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - captured_keys.append(key) - captured_kwargs.append(kwargs) - session = _make_bridge_session(key=key, key_value=key.affinity_key) - session.account = cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)) - session.request_model = payload.model - return session - - async def fake_stream_events( - _session: proxy_service._HTTPBridgeSession, - *, - request_state: proxy_service._WebSocketRequestState, - text_data: str, - **_kwargs: Any, - ): - assert request_state.previous_response_id is None - assert request_state.preferred_account_id is None - captured_text_data.append(text_data) - yield 'data: {"type":"response.completed"}\n\n' - - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={ - "authorization": "Bearer test-token", - "x-codex-session-id": "shared-root", - "x-codex-turn-state": "http_turn_child", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - ) - ] - - assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert len(captured_keys) == 1 - assert is_http_bridge_account_neutral_replay( - kind=captured_keys[0].affinity_kind, - key=captured_keys[0].affinity_key, - ) - assert captured_keys[0].strength == "soft" - assert captured_kwargs[0]["durable_lookup"] is None - assert captured_kwargs[0]["previous_response_id"] is None - assert captured_kwargs[0]["preferred_account_id"] is None - assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is False - assert captured_kwargs[0]["allow_forward_to_owner"] is False - replay_payload = json.loads(captured_text_data[0]) - assert "previous_response_id" not in replay_payload - assert replay_payload["model"] == "gpt-5.3-codex-spark" - assert replay_payload["input"] == [ - { - "role": "user", - "content": [{"type": "input_text", "text": "old question"}], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call", - "call_id": "call_old", - "name": "lookup", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call_output", - "call_id": "call_old", - "output": "old output", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "docs"}, - "execution": "client", - "status": "completed", - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "output": "found docs", - "tools": [], - "internal_chat_message_metadata_passthrough": owner_metadata, - }, - { - "type": "function_call", - "call_id": "call_pending", - "name": "lookup_pending", - "arguments": "{}", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - { - "type": "function_call_output", - "call_id": "call_pending", - "output": "fresh result", - "internal_chat_message_metadata_passthrough": {"turn_id": "turn-next"}, - }, - ] - assert all("id" not in item for item in replay_payload["input"]) - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_forks_account_neutral_model_transition_after_owner_conflict( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-terra", - "instructions": "hi", - "input": [{"role": "user", "content": "continue on the new model"}], - } - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-conflict-parent", - canonical_kind="session_header", - canonical_key="shared-root", - api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_model_parent", - latest_response_id="resp_model_parent", - model="gpt-5.6-sol", - ) - owner_conflict = ProxyResponseError( - 502, - openai_error( - "continuity_owner_conflict", - "Durable continuity aliases resolve to conflicting upstream owners.", - ), - ) - creation_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - creation_calls: list[dict[str, Any]] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - creation_keys.append(key) - creation_calls.append(kwargs) - if len(creation_calls) == 1: - raise owner_conflict - session = _make_bridge_session(key=key) - session.account = cast( - Any, - SimpleNamespace(id="acc-model-alternate", status=AccountStatus.ACTIVE), - ) - session.request_model = payload.model - return session - - async def fake_stream_events( - _session: proxy_service._HTTPBridgeSession, - **_kwargs: Any, - ): - yield 'data: {"type":"response.completed"}\n\n' - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={ - "x-codex-turn-state": "http_turn_model_parent", - "x-codex-session-id": "shared-root", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - downstream_turn_state="http_turn_model_child", - ) - ] - - assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert len(creation_calls) == 2 - assert creation_keys[0].affinity_kind in {"session_header", "turn_state_header"} - assert is_http_bridge_account_neutral_replay( - kind=creation_keys[1].affinity_kind, - key=creation_keys[1].affinity_key, - ) - # The child lane stays owner-bound so a later capacity failure cannot soft - # reroute this request onto a third account. - assert creation_keys[1].strength == "hard" - assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" - assert creation_calls[0]["preferred_account_has_continuity_provenance"] is True - assert creation_calls[1]["preferred_account_id"] is None - assert creation_calls[1]["preferred_account_has_continuity_provenance"] is False - assert creation_calls[1]["exclude_account_ids"] == {"acc-model-owner"} - assert creation_calls[1]["allow_forward_to_owner"] is False - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_limits_model_transition_owner_conflict_fork_to_one_retry( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-terra", - "instructions": "hi", - "input": [{"role": "user", "content": "continue on the new model"}], - } - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-conflict-parent", - canonical_kind="session_header", - canonical_key="shared-root", - api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_model_parent", - latest_response_id="resp_model_parent", - model="gpt-5.6-sol", - ) - owner_conflict = ProxyResponseError( - 502, - openai_error( - "continuity_owner_conflict", - "Durable continuity aliases resolve to conflicting upstream owners.", - ), - ) - creation_calls: list[dict[str, Any]] = [] - - async def fake_get_or_create( - _key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - creation_calls.append(kwargs) - raise owner_conflict - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_via_http_bridge( - payload, - headers={ - "x-codex-turn-state": "http_turn_model_parent", - "x-codex-session-id": "shared-root", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - downstream_turn_state="http_turn_model_child", - ): - pass - - assert exc_info.value is owner_conflict - assert len(creation_calls) == 2 - assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" - assert creation_calls[1]["preferred_account_id"] is None - assert creation_calls[1]["exclude_account_ids"] == {"acc-model-owner"} - assert creation_calls[1]["allow_forward_to_owner"] is False - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_model_transition_owner_conflict_fork_does_not_rebind_parent_turn_alias( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-terra", - "instructions": "hi", - "input": [{"role": "user", "content": "continue on the new model"}], - } - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-conflict-parent", - canonical_kind="session_header", - canonical_key="shared-root", - api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_model_parent", - latest_response_id="resp_model_parent", - model="gpt-5.6-sol", - ) - owner_conflict = ProxyResponseError( - 502, - openai_error( - "continuity_owner_conflict", - "Durable continuity aliases resolve to conflicting upstream owners.", - ), - ) - stream_downstream_turn_states: list[str | None] = [] - stream_request_states: list[Any] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - if kwargs["preferred_account_id"] == "acc-model-owner": - raise owner_conflict - session = _make_bridge_session(key=key) - session.account = cast( - Any, - SimpleNamespace(id="acc-model-alternate", status=AccountStatus.ACTIVE), - ) - session.request_model = payload.model - return session - - async def fake_stream_events( - _session: proxy_service._HTTPBridgeSession, - **kwargs: Any, - ): - stream_downstream_turn_states.append(kwargs["downstream_turn_state"]) - stream_request_states.append(kwargs["request_state"]) - yield 'data: {"type":"response.completed"}\n\n' - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={ - "x-codex-turn-state": "http_turn_model_parent", - "x-codex-session-id": "shared-root", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - downstream_turn_state="http_turn_model_parent", - ) - ] - - assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert stream_downstream_turn_states == [None] - # The child lane must not inherit the parent's continuity identity: a stale - # hard anchor or parent affinity policy would make the submit and - # clean-close paths treat this account-neutral fork as the old owner-bound - # turn. - (child_request_state,) = stream_request_states - assert child_request_state.session_id is None - assert child_request_state.hard_continuity_anchor is False - assert child_request_state.affinity_policy.key is None - assert child_request_state.affinity_policy.codex_session_source is None - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ("forwarded_request", "input_items"), - [ - ( - True, - [{"role": "user", "content": "continue on the new model"}], - ), - ( - False, - [ - { - "type": "message", - "role": "user", - "content": [ - {"type": "input_text", "text": "continue on the new model"}, - {"type": "input_file", "file_id": "file-unpinned"}, - ], - } - ], - ), - # Negative controls for the widened account-neutral classifier (#1849): - # a `compaction` item is admitted only when it carries its own - # completed encrypted content. A placeholder that merely references the - # owner's compacted context, or one still being produced, keeps the - # prior turns behind the old owner, so forking would silently drop - # them. - ( - False, - [ - {"type": "compaction", "id": "cmpct_model_parent"}, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue on the new model"}], - }, - ], - ), - ( - False, - [ - { - "type": "compaction", - "status": "in_progress", - "encrypted_content": "gAAAAABopaque", - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue on the new model"}], - }, - ], - ), - ], - ids=[ - "forwarded-request", - "unpinned-input-file", - "post-compaction-placeholder", - "post-compaction-in-progress", - ], -) -async def test_stream_via_http_bridge_keeps_model_transition_owner_conflict_fail_closed_for_unsafe_fork( - monkeypatch: pytest.MonkeyPatch, - forwarded_request: bool, - input_items: list[dict[str, Any]], -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-terra", - "instructions": "hi", - "input": input_items, - } - ) - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-model-unsafe-fork", - canonical_kind="session_header", - canonical_key="shared-root", - api_key_scope="__anonymous__", - account_id="acc-model-owner", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_model_parent", - latest_response_id="resp_model_parent", - model="gpt-5.6-sol", - ) - owner_conflict = ProxyResponseError( - 502, - openai_error( - "continuity_owner_conflict", - "Durable continuity aliases resolve to conflicting upstream owners.", - ), - ) - creation_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - creation_calls: list[dict[str, Any]] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - creation_keys.append(key) - creation_calls.append(kwargs) - raise owner_conflict - - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - - with pytest.raises(ProxyResponseError) as exc_info: - async for _ in service._stream_via_http_bridge( - payload, - headers={ - "x-codex-turn-state": "http_turn_model_parent", - "x-codex-session-id": "shared-root", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - downstream_turn_state="http_turn_model_child", - forwarded_request=forwarded_request, - ): - pass - - assert exc_info.value is owner_conflict - assert len(creation_calls) == 1 - assert creation_calls[0]["preferred_account_id"] == "acc-model-owner" - assert creation_calls[0]["preferred_account_has_continuity_provenance"] is True - assert not is_http_bridge_account_neutral_replay( - kind=creation_keys[0].affinity_kind, - key=creation_keys[0].affinity_key, - ) - - -@pytest.mark.asyncio -async def test_stream_via_http_bridge_preserves_verified_replay_kind_for_durable_model_transition( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-terra", - "instructions": "hi", - "input": [{"role": "user", "content": "continue on the new model"}], - } - ) - replay_kind, replay_key = make_http_bridge_account_neutral_replay_key("replay-parent") - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-replay-parent", - canonical_kind=replay_kind, - canonical_key=replay_key, - api_key_scope="__anonymous__", - account_id="acc-replay", - owner_instance_id=None, - owner_epoch=1, - lease_expires_at=datetime.now(timezone.utc) + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - latest_turn_state="http_turn_replay_parent", - latest_response_id="resp_replay_parent", - model="gpt-5.6-sol", - ) - captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - captured_kwargs: list[dict[str, Any]] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: Any, - ) -> proxy_service._HTTPBridgeSession: - captured_keys.append(key) - captured_kwargs.append(kwargs) - session = _make_bridge_session(key=key) - session.account = cast(Any, SimpleNamespace(id="acc-replay", status=AccountStatus.ACTIVE)) - session.request_model = payload.model - return session - - async def fake_stream_events( - _session: proxy_service._HTTPBridgeSession, - **_kwargs: Any, - ): - yield 'data: {"type":"response.completed"}\n\n' - - monkeypatch.setattr( - proxy_service, - "get_settings_cache", - lambda: cast( - Any, - SimpleNamespace( - get=AsyncMock( - return_value=SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - http_responses_session_bridge_prompt_cache_idle_ttl_seconds=3600, - http_responses_session_bridge_gateway_safe_mode=False, - ) - ) - ), - ), - ) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - chunks = [ - chunk - async for chunk in service._stream_via_http_bridge( - payload, - headers={ - "x-codex-turn-state": "http_turn_replay_parent", - "x-codex-session-id": "shared-root", - }, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=True, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - downstream_turn_state="http_turn_replay_child", - ) - ] - - assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert len(captured_keys) == 1 - assert is_http_bridge_account_neutral_replay( - kind=captured_keys[0].affinity_kind, - key=captured_keys[0].affinity_key, - ) - assert captured_keys[0].affinity_key != durable_lookup.canonical_key - assert captured_kwargs[0]["durable_lookup"] is None - assert captured_kwargs[0]["preferred_account_id"] == "acc-replay" - assert captured_kwargs[0]["preferred_account_has_continuity_provenance"] is True - - -@pytest.mark.asyncio -async def test_get_or_create_http_bridge_session_prompt_cache_mismatch_stays_local_when_gateway_safe_mode_disabled( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = proxy_service.ProxyService(cast(Any, nullcontext())) - key = proxy_service._HTTPBridgeSessionKey("prompt_cache", "cache-key", None) - monkeypatch.setattr(service, "_prune_http_bridge_sessions_locked", Mock(return_value=[])) - monkeypatch.setattr(proxy_service, "get_settings", lambda: _make_app_settings()) - monkeypatch.setattr(proxy_service, "_http_bridge_owner_instance", AsyncMock(return_value="instance-b")) - monkeypatch.setattr( - proxy_service, - "_active_http_bridge_instance_ring", - AsyncMock(return_value=("instance-a", ["instance-a", "instance-b"])), - ) - created_session = proxy_service._HTTPBridgeSession( - key=key, - headers={}, - affinity=proxy_service._AffinityPolicy(key="cache-key"), - request_model="gpt-5.4", - account=cast(Any, SimpleNamespace(id="acc-fresh", status=AccountStatus.ACTIVE)), - upstream=cast(UpstreamWebSocket, SimpleNamespace(close=AsyncMock())), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque(), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=0, - last_used_at=2.0, - idle_ttl_seconds=120.0, - ) - monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=created_session)) - monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) + monkeypatch.setattr(service, "_create_http_bridge_session", AsyncMock(return_value=created_session)) + monkeypatch.setattr(service, "_claim_durable_http_bridge_session", AsyncMock()) resolved = await service._get_or_create_http_bridge_session( key, @@ -31361,53 +30416,6 @@ def test_http_bridge_quarantine_clear_restores_session_reusability() -> None: assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is False -def test_http_bridge_quarantine_clear_generation_rejects_stale_recovery() -> None: - service = SimpleNamespace() - session = _make_bridge_session(key_value="quarantine-clear-generation") - - http_bridge_quarantine_module._quarantine_http_bridge_session( - service, - session, - reason="reattach_missing_response_created", - ) - first_generation = http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation( - service, - session.key, - ) - assert first_generation is not None - - http_bridge_quarantine_module._quarantine_http_bridge_session( - service, - session, - reason="repeated_eventless_timeout", - ) - assert http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation(service, session.key) != ( - first_generation - ) - - http_bridge_quarantine_module._clear_http_bridge_quarantine_key( - service, - session.key, - account_id="acc-late-recovery", - model="gpt-5.6-sol", - generation=first_generation, - ) - assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is True - - current_generation = http_bridge_quarantine_module._http_bridge_session_key_quarantine_generation( - service, - session.key, - ) - http_bridge_quarantine_module._clear_http_bridge_quarantine_key( - service, - session.key, - account_id="acc-current-recovery", - model="gpt-5.6-sol", - generation=current_generation, - ) - assert http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is False - - def test_http_bridge_session_reusable_for_lookup_excludes_quarantined() -> None: service = SimpleNamespace() session = _make_bridge_session(key_value="quarantine-reuse") @@ -31718,54 +30726,30 @@ async def test_retire_stale_pending_http_bridge_session_quarantines_wedged_reatt ) -> None: """Direct session retirement (the all-stale stuck-gate path) bypasses both the partial-cleanup hook and the reader-failure funnel; it must still - quarantine the key when the retired pending proves the wedge shape.""" - service = proxy_service.ProxyService(cast(Any, nullcontext())) - wedged = _make_wedged_reattach_request_state(request_id="req-direct-retire-wedged") - if created_assigned: - wedged.response_id = "resp_created_direct_retire" - wedged.latency_response_created_ms = 700 - session = _make_bridge_session( - key_value="quarantine-direct-retire", - pending_requests=deque([wedged]), - queued_request_count=1, - ) - monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", AsyncMock()) - - await service._retire_stale_pending_http_bridge_session( - session, - detail="response_create_gate_timeout_stuck_pending", - ) - - assert session.quarantined is expect_quarantined - assert ( - http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is expect_quarantined - ) - - -@pytest.mark.asyncio -async def test_retire_stale_pending_http_bridge_session_recomputes_event_evidence( - monkeypatch: pytest.MonkeyPatch, -) -> None: + quarantine the key when the retired pending proves the wedge shape.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - eventful = _make_wedged_reattach_request_state(request_id="req-direct-retire-eventful") - eventful.response_id = "resp_eventful_direct_retire" - eventful.latency_response_created_ms = 700 + wedged = _make_wedged_reattach_request_state(request_id="req-direct-retire-wedged") + if created_assigned: + wedged.response_id = "resp_created_direct_retire" + wedged.latency_response_created_ms = 700 session = _make_bridge_session( - key_value="quarantine-direct-retire-eventful", - pending_requests=deque([eventful]), + key_value="quarantine-direct-retire", + pending_requests=deque([wedged]), queued_request_count=1, ) - record_failure = AsyncMock() monkeypatch.setattr(service, "_close_http_bridge_session_bounded", AsyncMock()) - monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", record_failure) + monkeypatch.setattr(service, "_record_http_bridge_retry_circuit_failure", AsyncMock()) await service._retire_stale_pending_http_bridge_session( session, detail="response_create_gate_timeout_stuck_pending", + response_events_seen=wedged.response_event_count, ) - record_failure.assert_not_awaited() + assert session.quarantined is expect_quarantined + assert ( + http_bridge_quarantine_module._http_bridge_session_key_quarantined(service, session.key) is expect_quarantined + ) @pytest.mark.asyncio @@ -31778,47 +30762,41 @@ async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_ unanchored instead of restoring the wedged durable anchor through session hydration and session-level injection.""" service = proxy_service.ProxyService(cast(Any, nullcontext())) - historical_input = [{"role": "user", "content": "one"}] - retained_output = { - "type": "message", - "role": "assistant", - "id": "msg_response_owned", - "content": [{"type": "output_text", "text": "two"}], - } - projected_retained_output = { - "type": "message", - "role": "assistant", - "content": [{"type": "output_text", "text": "two"}], - } - retains_prior_output = True - account_neutral_payload = True - expected_account_neutral = True - fresh_suffix = ( - [ - retained_output, - {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, - ] - if retains_prior_output - else [{"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}] - ) - expected_projected_input = ( - [ - *historical_input, - projected_retained_output, - {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, - ] - if retains_prior_output - else [{"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}] - ) - # Full resend with a trimmable durable prefix. Only the variant retaining - # prior output is safe to replay account-neutrally. + historical_input = [ + {"role": "user", "content": [{"type": "input_text", "text": "leading question"}]}, + { + "type": "additional_tools", + "role": "developer", + "tools": [{"type": "custom", "name": "shell"}], + }, + { + "type": "message", + "role": "developer", + "content": [{"type": "input_text", "text": "canonical Lite instructions"}], + }, + {"role": "user", "content": [{"type": "input_text", "text": "first question"}]}, + { + "type": "custom_tool_call", + "call_id": "call_historical_shell", + "name": "shell", + "input": "printf historical", + }, + {"role": "developer", "content": [{"type": "input_text", "text": "historical control"}]}, + { + "type": "custom_tool_call_output", + "call_id": "call_historical_shell", + "output": "historical", + }, + ] + # Full resend with a trimmable durable prefix and a fresh suffix that does + # NOT retain the prior output (plain user turn). payload = proxy_service.ResponsesRequest.model_validate( { "model": "gpt-5.6-sol", "instructions": "test", "input": [ *historical_input, - *fresh_suffix, + {"role": "user", "content": [{"type": "input_text", "text": "follow-up without prior output"}]}, ], } ) @@ -31848,7 +30826,8 @@ async def test_stream_http_bridge_quarantined_full_resend_stays_unanchored_when_ quarantined_session, reason="reattach_missing_response_created", ) - fresh_session: proxy_service._HTTPBridgeSession | None = None + fresh_session = _make_bridge_session(key=bridge_key, key_value=bridge_key.affinity_key) + fresh_session.codex_session = True prepared_payloads: list[proxy_service.ResponsesRequest] = [] @@ -31877,25 +30856,17 @@ def fake_prepare( request_state.previous_response_id = prepared_payload.previous_response_id return request_state, json.dumps(dict(prepared_payload.to_payload()), separators=(",", ":")) - captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - captured_kwargs: list[dict[str, object]] = [] - captured_request_states: list[proxy_service._WebSocketRequestState] = [] - async def fake_get_or_create( key: proxy_service._HTTPBridgeSessionKey, **kwargs: object, ) -> proxy_service._HTTPBridgeSession: - nonlocal fresh_session - captured_keys.append(key) - captured_kwargs.append(dict(kwargs)) - fresh_session = _make_bridge_session(key=key, key_value=key.affinity_key) - fresh_session.codex_session = True + del kwargs + assert key == bridge_key return fresh_session dispatched_text: list[str] = [] async def fake_stream_events(*args: object, **kwargs: object): - captured_request_states.append(cast(proxy_service._WebSocketRequestState, kwargs["request_state"])) dispatched_text.append(cast(str, kwargs["text_data"])) yield 'data: {"type":"response.completed"}\n\n' @@ -31921,12 +30892,6 @@ async def fake_stream_events(*args: object, **kwargs: object): monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - account_neutral_classifier = Mock(return_value=account_neutral_payload) - monkeypatch.setattr( - http_bridge_streaming_module, - "_http_bridge_payload_is_account_neutral_fresh_replay", - account_neutral_classifier, - ) # The bypass shape: a live (alias) session makes the fresh-reattach # durable-anchor gate false before the quarantine is ever consulted. monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=True)) @@ -31948,350 +30913,14 @@ async def fake_stream_events(*args: object, **kwargs: object): chunks = [chunk async for chunk in stream] assert chunks == ['data: {"type":"response.completed"}\n\n'] - assert len(captured_keys) == 1 - assert (captured_keys[0] != bridge_key) is expected_account_neutral - assert captured_keys[0].strength == ("soft" if expected_account_neutral else "hard") - assert ( - is_http_bridge_account_neutral_replay( - kind=captured_keys[0].affinity_kind, - key=captured_keys[0].affinity_key, - ) - is expected_account_neutral - ) - assert captured_kwargs[0]["headers"] == ( - {} if expected_account_neutral else {"x-codex-session-id": bridge_key.affinity_key} - ) - assert captured_kwargs[0]["preferred_account_id"] == (None if expected_account_neutral else "acc-bridge") assert len(dispatched_text) == 1 dispatched_payload = json.loads(dispatched_text[0]) # Genuinely unanchored: the suppressed durable anchor did not come back # through session hydration or session-level injection, and the client's # payload was not prefix-trimmed against the durable stored context. assert "previous_response_id" not in dispatched_payload - assert dispatched_payload["input"] == expected_projected_input - assert fresh_session is not None + assert len(dispatched_payload["input"]) == len(historical_input) + 1 assert fresh_session.last_completed_response_id is None - assert len(captured_request_states) == 1 - assert (captured_request_states[0].quarantine_clear_key == bridge_key) is expected_account_neutral - assert (captured_request_states[0].quarantine_clear_generation is not None) is expected_account_neutral - expected_replay_kind, expected_replay_key = make_http_bridge_account_neutral_replay_key( - durable_bridge_hash(dispatched_text[0]) - ) - if expected_account_neutral: - assert captured_keys[0].affinity_kind == expected_replay_kind - assert captured_keys[0].affinity_key == expected_replay_key - if retains_prior_output: - account_neutral_classifier.assert_called_once() - else: - account_neutral_classifier.assert_not_called() - - -@pytest.mark.asyncio -async def test_quarantined_full_resend_recovery_fence_survives_restart_fingerprint( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """The quarantine demotion must not re-key the durable recovery fence. - - ``durable_recovery_attempt_fingerprint`` is the primary key of persisted - ``http_bridge_recovery_attempts`` rows, so it must keep hashing the - unprojected request body. A row journalled before a restart has to still - match after it, otherwise the one-shot replay fence silently opens once. - The quarantine recovery key is named after the projected body that this - dispatch actually sends, which is a different hash on purpose. - """ - service = proxy_service.ProxyService(cast(Any, nullcontext())) - historical_input = [{"role": "user", "content": "one"}] - retained_output = { - "type": "message", - "role": "assistant", - "id": "msg_response_owned", - "content": [{"type": "output_text", "text": "two"}], - } - payload = proxy_service.ResponsesRequest.model_validate( - { - "model": "gpt-5.6-sol", - "instructions": "test", - "input": [ - *historical_input, - retained_output, - {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, - ], - } - ) - bridge_key = proxy_service._HTTPBridgeSessionKey("session_header", "quarantine-fence-restart", None) - prefix_fingerprint = http_bridge_streaming_module._fingerprint_input_items( - cast(list[Any], payload.input)[: len(historical_input)] - ) - # The restart shape: the durable row survived, but its owner's lease died - # with the process that wrote the recovery-attempt row. - durable_lookup = proxy_service.DurableBridgeLookup( - session_id="durable-quarantine-fence", - canonical_kind=bridge_key.affinity_kind, - canonical_key=bridge_key.affinity_key, - api_key_scope="__anonymous__", - account_id="acc-bridge", - owner_instance_id="instance-before-restart", - owner_epoch=4, - lease_expires_at=proxy_service.utcnow() - timedelta(seconds=120), - state=HttpBridgeSessionState.CLOSED, - latest_turn_state=None, - latest_response_id="resp_wedged_anchor", - model="gpt-5.6-sol", - latest_input_item_count=len(historical_input), - latest_input_full_fingerprint=prefix_fingerprint, - ) - quarantined_session = _make_bridge_session(key=bridge_key, key_value=bridge_key.affinity_key) - http_bridge_quarantine_module._quarantine_http_bridge_session( - service, - quarantined_session, - reason="reattach_missing_response_created", - ) - - def render_bridge_text(rendered_payload: proxy_service.ResponsesRequest) -> str: - return json.dumps(dict(rendered_payload.to_payload()), separators=(",", ":")) - - # What a pre-restart process journalled: the hash of the unprojected body. - persisted_fence_fingerprint = durable_bridge_hash( - render_bridge_text(http_bridge_streaming_module._http_bridge_payload_without_previous_response_id(payload)) - ) - - def fake_prepare( - prepared_payload: proxy_service.ResponsesRequest, - _headers: dict[str, str] | Any, - *, - api_key: proxy_service.ApiKeyData | None, - api_key_reservation: proxy_service.ApiKeyUsageReservationData | None, - request_id: str, - client_ip: str | None = None, - **prepare_kwargs: object, - ) -> tuple[proxy_service._WebSocketRequestState, str]: - del api_key, api_key_reservation, request_id, client_ip, prepare_kwargs - request_state = proxy_service._WebSocketRequestState( - request_id="req-quarantine-fence", - model=prepared_payload.model, - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=time.monotonic(), - event_queue=asyncio.Queue(), - transport="http", - ) - request_state.previous_response_id = prepared_payload.previous_response_id - return request_state, render_bridge_text(prepared_payload) - - captured_keys: list[proxy_service._HTTPBridgeSessionKey] = [] - - async def fake_get_or_create( - key: proxy_service._HTTPBridgeSessionKey, - **kwargs: object, - ) -> proxy_service._HTTPBridgeSession: - del kwargs - captured_keys.append(key) - fresh_session = _make_bridge_session(key=key, key_value=key.affinity_key) - fresh_session.codex_session = True - return fresh_session - - dispatched_text: list[str] = [] - - async def fake_stream_events(*args: object, **kwargs: object): - del args - dispatched_text.append(cast(str, kwargs["text_data"])) - yield 'data: {"type":"response.completed"}\n\n' - - dashboard_settings = SimpleNamespace( - sticky_threads_enabled=False, - openai_cache_affinity_max_age_seconds=1800, - ) - runtime_config = SimpleNamespace( - enabled=True, - idle_ttl_seconds=120.0, - codex_idle_ttl_seconds=1800.0, - max_sessions=8, - queue_limit=4, - prompt_cache_idle_ttl_seconds=120.0, - gateway_safe_mode=False, - ) - monkeypatch.setattr( - http_bridge_streaming_module, - "_service_get_settings_cache", - lambda: SimpleNamespace(get=AsyncMock(return_value=dashboard_settings)), - ) - monkeypatch.setattr(http_bridge_streaming_module, "_service_get_settings", _make_app_settings) - monkeypatch.setattr(http_bridge_streaming_module, "_http_bridge_runtime_config", lambda *args: runtime_config) - monkeypatch.setattr(service, "_resolve_file_account_for_responses", AsyncMock(return_value=None)) - monkeypatch.setattr(service._durable_bridge, "lookup_request_targets", AsyncMock(return_value=durable_lookup)) - - claim_instance_id = _make_app_settings().http_responses_session_bridge_instance_id - claimed_lookup = replace( - durable_lookup, - owner_instance_id=claim_instance_id, - owner_epoch=durable_lookup.owner_epoch + 1, - lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - ) - lookup_recovery_attempt = AsyncMock(return_value=SimpleNamespace(request_fingerprint=persisted_fence_fingerprint)) - mark_recovery_attempt_replayed = AsyncMock(return_value=True) - monkeypatch.setattr(service._durable_bridge, "lookup_recovery_attempt", lookup_recovery_attempt) - monkeypatch.setattr(service._durable_bridge, "claim_live_session", AsyncMock(return_value=claimed_lookup)) - monkeypatch.setattr(service._durable_bridge, "mark_recovery_attempt_replayed", mark_recovery_attempt_replayed) - monkeypatch.setattr( - http_bridge_streaming_module, - "_http_bridge_payload_is_account_neutral_fresh_replay", - Mock(return_value=True), - ) - monkeypatch.setattr(service, "_http_bridge_has_live_local_session", AsyncMock(return_value=True)) - monkeypatch.setattr(service, "_http_bridge_can_forward_to_active_owner", AsyncMock(return_value=False)) - monkeypatch.setattr(service, "_prepare_http_bridge_request", fake_prepare) - monkeypatch.setattr(service, "_get_or_create_http_bridge_session", fake_get_or_create) - monkeypatch.setattr(service, "_stream_http_bridge_session_events", fake_stream_events) - - stream = service._stream_http_bridge_or_retry( - payload, - {"x-codex-session-id": bridge_key.affinity_key}, - codex_session_affinity=True, - propagate_http_errors=True, - openai_cache_affinity=False, - api_key=None, - api_key_reservation=None, - suppress_text_done_events=False, - ) - chunks = [chunk async for chunk in stream] - assert chunks == ['data: {"type":"response.completed"}\n\n'] - - # The fence looked the persisted row up under the fingerprint that row was - # written with, so the one-shot replay guard still holds after the restart. - lookup_recovery_attempt.assert_awaited_once_with( - session_id=durable_lookup.session_id, - request_fingerprint=persisted_fence_fingerprint, - ) - mark_recovery_attempt_replayed.assert_awaited_once_with( - session_id=durable_lookup.session_id, - api_key_id=None, - instance_id=claim_instance_id, - owner_epoch=claimed_lookup.owner_epoch, - request_fingerprint=persisted_fence_fingerprint, - ) - - # Negative control. The body this recovery actually dispatches is the - # projected one, and its hash is NOT the fence fingerprint. Hashing the - # projected body into ``durable_recovery_attempt_fingerprint`` is exactly - # the regression this test pins shut: the lookup above would have missed - # the persisted row and handed out a second "first" replay. - assert len(dispatched_text) == 1 - dispatched_payload = json.loads(dispatched_text[0]) - assert dispatched_payload["input"] == [ - *historical_input, - {"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "two"}]}, - {"role": "user", "content": [{"type": "input_text", "text": "safe follow-up"}]}, - ] - projected_body_fingerprint = durable_bridge_hash(dispatched_text[0]) - assert projected_body_fingerprint != persisted_fence_fingerprint - fence_await = lookup_recovery_attempt.await_args - assert fence_await is not None - assert fence_await.kwargs["request_fingerprint"] != projected_body_fingerprint - - # The poisoned hard key was not reused for the recovery dispatch. - assert len(captured_keys) == 1 - assert captured_keys[0] != bridge_key - assert is_http_bridge_account_neutral_replay( - kind=captured_keys[0].affinity_kind, - key=captured_keys[0].affinity_key, - ) - - -@pytest.mark.asyncio -async def test_advance_http_bridge_quarantine_clear_key_rebinds_and_updates_original_continuity( - monkeypatch: pytest.MonkeyPatch, -) -> None: - service = SimpleNamespace() - key = proxy_service._HTTPBridgeSessionKey("session_header", "quarantine-original", None) - lookup = proxy_service.DurableBridgeLookup( - session_id="durable-original", - canonical_kind=key.affinity_kind, - canonical_key=key.affinity_key, - api_key_scope="__anonymous__", - account_id="acc-old", - owner_instance_id=None, - owner_epoch=7, - lease_expires_at=proxy_service.utcnow() - timedelta(seconds=60), - state=HttpBridgeSessionState.CLOSED, - latest_turn_state=None, - latest_response_id="resp-stale", - model="gpt-5.6-sol", - ) - claimed_lookup = replace( - lookup, - owner_instance_id=_make_app_settings().http_responses_session_bridge_instance_id, - owner_epoch=lookup.owner_epoch + 1, - lease_expires_at=proxy_service.utcnow() + timedelta(seconds=60), - state=HttpBridgeSessionState.ACTIVE, - ) - service._durable_bridge = SimpleNamespace( - lookup_request_targets=AsyncMock(return_value=lookup), - claim_live_session=AsyncMock(return_value=claimed_lookup), - rebind_session_account=AsyncMock(return_value=True), - renew_live_session=AsyncMock( - return_value=replace(claimed_lookup, account_id="acc-new", latest_response_id="resp-new") - ), - ) - monkeypatch.setattr(http_bridge_upstream_events_module, "_service_get_settings", _make_app_settings) - - advanced = await http_bridge_upstream_events_module._advance_http_bridge_quarantine_clear_key( - service, - key=key, - api_key_id=None, - account_id="acc-new", - response_id="resp-new", - input_item_count=3, - input_full_fingerprint="fp-new", - pending_tool_calls={"call_1": "function_call"}, - ) - - assert advanced is True - service._durable_bridge.lookup_request_targets.assert_awaited_once_with( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=None, - turn_state=None, - session_header=None, - previous_response_id=None, - ) - service._durable_bridge.claim_live_session.assert_awaited_once_with( - session_key_kind=key.affinity_kind, - session_key_value=key.affinity_key, - api_key_id=None, - instance_id=_make_app_settings().http_responses_session_bridge_instance_id, - lease_ttl_seconds=pytest.approx(http_bridge_helpers_module._http_bridge_durable_lease_ttl_seconds()), - account_id="acc-old", - model="gpt-5.6-sol", - service_tier=None, - latest_turn_state=None, - latest_response_id="resp-stale", - allow_takeover=False, - owner_process_epoch=http_bridge_owner_process_epoch(), - ) - service._durable_bridge.rebind_session_account.assert_awaited_once_with( - session_id=claimed_lookup.session_id, - api_key_id=None, - instance_id=_make_app_settings().http_responses_session_bridge_instance_id, - owner_epoch=claimed_lookup.owner_epoch, - account_id="acc-new", - clear_continuity=True, - ) - service._durable_bridge.renew_live_session.assert_awaited_once() - renew_kwargs = service._durable_bridge.renew_live_session.await_args.kwargs - assert renew_kwargs == { - "session_id": claimed_lookup.session_id, - "api_key_id": None, - "instance_id": _make_app_settings().http_responses_session_bridge_instance_id, - "owner_epoch": claimed_lookup.owner_epoch, - "lease_ttl_seconds": renew_kwargs["lease_ttl_seconds"], - "latest_response_id": "resp-new", - "latest_input_item_count": 3, - "latest_input_full_fingerprint": "fp-new", - "latest_pending_tool_calls": {"call_1": "function_call"}, - } - assert renew_kwargs["lease_ttl_seconds"] > 0 @pytest.mark.asyncio diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index cbbbb86238..3984cf0939 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -17138,10 +17138,10 @@ async def fake_stream(*_, **__): assert terminal_event["type"] == "response.failed" terminal_response = cast(dict[str, JsonValue], terminal_event["response"]) terminal_error = cast(dict[str, JsonValue], terminal_response["error"]) - assert terminal_error["code"] == "duplicate_tool_call_replay_suppressed" + assert terminal_error["code"] == "stream_incomplete" assert await service.drain_persistence_tasks(timeout_seconds=1) assert request_logs.calls[0]["status"] == "error" - assert request_logs.calls[0]["error_code"] == "duplicate_tool_call_replay_suppressed" + assert request_logs.calls[0]["error_code"] == "stream_incomplete" @pytest.mark.asyncio @@ -44260,95 +44260,6 @@ async def test_http_bridge_tool_call_dedupe_survives_upstream_reconnect(): assert event_queue.empty() -@pytest.mark.asyncio -async def test_http_bridge_duplicate_tool_call_replay_emits_retryable_terminal_failure(): - request_logs = _RequestLogsRecorder() - service = proxy_service.ProxyService(_repo_factory(request_logs)) - request_state = proxy_service._WebSocketRequestState( - request_id="req_bridge_duplicate_terminal", - model="gpt-5.1", - service_tier=None, - reasoning_effort=None, - api_key_reservation=None, - started_at=0.0, - response_id="resp_bridge_duplicate_terminal", - event_queue=asyncio.Queue(), - request_text='{"type":"response.create"}', - transport="http", - ) - session = proxy_service._HTTPBridgeSession( - key=proxy_service._HTTPBridgeSessionKey("prompt_cache", "bridge-duplicate-terminal", None), - headers={}, - affinity=proxy_service._AffinityPolicy(), - request_model="gpt-5.1", - account=_make_account("acc_bridge_duplicate_terminal"), - upstream=AsyncMock(), - upstream_control=proxy_service._WebSocketUpstreamControl(), - pending_requests=deque([request_state]), - pending_lock=anyio.Lock(), - response_create_gate=asyncio.Semaphore(1), - queued_request_count=1, - last_used_at=0.0, - idle_ttl_seconds=30.0, - ) - cast(Any, session.upstream).archive_received = MagicMock() - tool_payload = { - "type": "response.output_item.done", - "response_id": "resp_bridge_duplicate_terminal", - "item": { - "type": "function_call", - "name": "write_stdin", - "arguments": json.dumps({"session_id": 75180, "chars": ""}), - "call_id": "call_first", - }, - } - replay_payload = { - **tool_payload, - "response_id": "resp_bridge_duplicate_terminal_replay", - "item": {**tool_payload["item"], "call_id": "call_replayed"}, - } - replay_created_payload = { - "type": "response.created", - "response": {"id": "resp_bridge_duplicate_terminal_replay", "status": "in_progress"}, - } - completed_payload = { - "type": "response.completed", - "response": {"id": "resp_bridge_duplicate_terminal_replay", "status": "completed", "output": []}, - } - - await service._process_http_bridge_upstream_text(session, json.dumps(tool_payload, separators=(",", ":"))) - session.upstream_control = proxy_service._WebSocketUpstreamControl() - request_state.awaiting_response_created = True - request_state.response_id = None - await service._process_http_bridge_upstream_text(session, json.dumps(replay_created_payload, separators=(",", ":"))) - await service._process_http_bridge_upstream_text(session, json.dumps(replay_payload, separators=(",", ":"))) - await service._process_http_bridge_upstream_text(session, json.dumps(completed_payload, separators=(",", ":"))) - - event_queue = request_state.event_queue - assert event_queue is not None - first = await event_queue.get() - created = await event_queue.get() - terminal_block = await event_queue.get() - assert isinstance(first, str) - assert isinstance(created, str) - assert isinstance(terminal_block, str) - assert proxy_service.parse_sse_data_json(first) == tool_payload - assert proxy_service.parse_sse_data_json(created) == replay_created_payload - terminal = proxy_service.parse_sse_data_json(terminal_block) - assert isinstance(terminal, dict) - assert terminal["type"] == "response.failed" - terminal_response = terminal["response"] - assert isinstance(terminal_response, dict) - terminal_error = terminal_response["error"] - assert isinstance(terminal_error, dict) - assert terminal_error["code"] == "duplicate_tool_call_replay_suppressed" - assert await event_queue.get() is None - assert request_state.error_http_status_override == 502 - assert session.upstream_control.reconnect_requested is True - assert await service.drain_persistence_tasks(timeout_seconds=1) - assert request_logs.calls[-1]["status"] == "error" - - @pytest.mark.asyncio async def test_http_bridge_session_events_emit_keepalive_while_pending(monkeypatch): request_logs = _RequestLogsRecorder() diff --git a/tests/unit/test_replay_safety.py b/tests/unit/test_replay_safety.py index 1553a34cf9..e17cf26c36 100644 --- a/tests/unit/test_replay_safety.py +++ b/tests/unit/test_replay_safety.py @@ -10,7 +10,6 @@ ) from app.modules.proxy.replay_safety import ( project_responses_input_for_account_neutral_fresh_replay, - responses_input_items_are_self_contained_fresh_replay, responses_input_suffix_matches_pending_tool_calls, responses_input_suffix_retains_prior_output, responses_payload_is_account_neutral_fresh_replay, @@ -152,310 +151,6 @@ def test_account_neutral_fresh_replay_accepts_self_contained_payloads( assert responses_payload_is_account_neutral_fresh_replay(payload) is True -def test_account_neutral_fresh_replay_accepts_compaction_context_item() -> None: - payload: dict[str, JsonValue] = { - "input": [ - { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue"}], - }, - ], - } - - assert responses_payload_is_account_neutral_fresh_replay(payload) is True - - -def test_account_neutral_replay_projection_preserves_owner_bound_compaction_id_to_fail_closed() -> None: - input_items: list[JsonValue] = [ - { - "type": "compaction", - "id": "cmp_owner_a", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue"}], - }, - ] - - projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) - - assert projection is not None - assert projection.input_items[0] == { - "type": "compaction", - "id": "cmp_owner_a", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - } - assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is False - - -def test_account_neutral_replay_projection_accepts_compaction_without_owner_id() -> None: - input_items: list[JsonValue] = [ - { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue"}], - }, - ] - - projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) - - assert projection is not None - assert projection.input_items[0] == { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - } - assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is True - - -def test_account_neutral_replay_projection_rejects_compaction_before_later_raw_prefix_bookkeeping() -> None: - input_items: list[JsonValue] = [ - { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "web_search_call", - "id": "ws_owner_a", - "action": {"type": "search", "query": "codex-lb"}, - "status": "completed", - }, - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb"}, - "execution": "client", - "status": "completed", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "output": "result", - "status": "completed", - "tools": [], - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue after compaction"}], - }, - ] - - assert project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=2) is None - - -def test_account_neutral_replay_projection_preserves_post_compact_tool_search_context_without_owner_id() -> None: - input_items: list[JsonValue] = [ - { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "tool_search_call", - "id": "tsc_owner_a", - "call_id": "call_search", - "arguments": {"query": "codex-lb post compact replay"}, - "execution": "client", - "status": "completed", - }, - { - "type": "tool_search_output", - "id": "tso_owner_a", - "call_id": "call_search", - "output": "replay fix candidate", - "execution": "client", - "status": "completed", - "tools": [], - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue after compaction"}], - }, - ] - - projection = project_responses_input_for_account_neutral_fresh_replay(input_items, stored_count=1) - - assert projection is not None - assert projection.stored_prefix_count == 1 - assert projection.input_items == [ - { - "type": "compaction", - "status": "completed", - "encrypted_content": "encrypted-compact-context", - }, - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb post compact replay"}, - "execution": "client", - "status": "completed", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "output": "replay fix candidate", - "execution": "client", - "status": "completed", - "tools": [], - }, - { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": "continue after compaction"}], - }, - ] - assert ( - responses_input_suffix_retains_prior_output( - projection.input_items, - stored_count=projection.stored_prefix_count, - ) - is True - ) - assert responses_payload_is_account_neutral_fresh_replay({"input": projection.input_items}) is True - - -def test_account_neutral_fresh_replay_rejects_post_compact_mixed_tool_suffix_before_user_followup() -> None: - input_items: list[JsonValue] = [ - {"type": "compaction", "status": "completed", "encrypted_content": "encrypted-compact-context"}, - { - "type": "function_call", - "call_id": "call_function", - "name": "lookup", - "arguments": "{}", - "status": "completed", - }, - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb post compact replay"}, - "execution": "client", - "status": "completed", - }, - {"type": "function_call_output", "call_id": "call_function", "output": "result", "status": "completed"}, - { - "type": "tool_search_output", - "call_id": "call_search", - "output": "search result", - "execution": "client", - "status": "completed", - "tools": [], - }, - {"type": "message", "role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - - assert responses_input_suffix_retains_prior_output(input_items, stored_count=1) is False - - -def test_account_neutral_fresh_replay_accepts_self_contained_tool_search_pair() -> None: - input_items: list[JsonValue] = [ - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb"}, - "status": "completed", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "output": "Found codex-lb", - "status": "completed", - }, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - - assert responses_input_items_are_self_contained_fresh_replay(input_items) is True - assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is True - - -def test_account_neutral_fresh_replay_accepts_tools_only_client_tool_search_output() -> None: - input_items: list[JsonValue] = [ - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb"}, - "execution": "client", - "status": "completed", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "tools": [], - }, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - - assert responses_input_items_are_self_contained_fresh_replay(input_items) is True - assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is True - - -@pytest.mark.parametrize( - "tool_search_output", - [ - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "server", - "status": "completed", - "tools": [], - "output": "Found codex-lb", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "tools": [{"type": "file_search", "vector_store_ids": ["vs_owner"]}], - "output": "Found codex-lb", - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "status": "completed", - "tools": [{"type": "function", "name": "lookup", "namespace": "private"}], - "output": "Found codex-lb", - }, - ], -) -def test_account_neutral_fresh_replay_rejects_account_scoped_tool_search_output( - tool_search_output: dict[str, JsonValue], -) -> None: - input_items: list[JsonValue] = [ - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "codex-lb"}, - "execution": "client", - "status": "completed", - }, - tool_search_output, - {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, - ] - - assert responses_payload_is_account_neutral_fresh_replay({"input": input_items}) is False - - def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() -> None: metadata = {"turn_id": "turn_owner_a"} input_items: list[JsonValue] = [ @@ -500,10 +195,8 @@ def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() }, { "type": "tool_search_output", - "id": "tso_owner_a", "call_id": "call_search", "execution": "client", - "output": "search result", "status": "completed", "tools": [], "internal_chat_message_metadata_passthrough": metadata, @@ -557,23 +250,6 @@ def test_account_neutral_replay_projection_removes_response_owned_bookkeeping() "status": "completed", "internal_chat_message_metadata_passthrough": metadata, }, - { - "type": "tool_search_call", - "call_id": "call_search", - "arguments": {"query": "github"}, - "execution": "client", - "status": "completed", - "internal_chat_message_metadata_passthrough": metadata, - }, - { - "type": "tool_search_output", - "call_id": "call_search", - "execution": "client", - "output": "search result", - "status": "completed", - "tools": [], - "internal_chat_message_metadata_passthrough": metadata, - }, { "type": "message", "role": "assistant", From 798203ff9d9d8f30a9b53e181d34fc9935ba5444 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 24 Aug 2026 16:36:51 +0900 Subject: [PATCH 107/117] fix(proxy): wait on usage-refresh singleflight without asyncio.shield (#1897) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Production re-starved its event loop after ~91h on 1.24.0-beta.3 (py-spy: GIL dominated by asyncio shield callback cleanup) because the usage-refresh singleflight still awaited shared tasks via asyncio.shield — the one many-waiter site #1842 did not convert. Route both _UsageRefreshSingleflight wait paths through wait_on_shared_future so waiter attach/cancel/timeout is O(1) on the shared task, preserving result/cancellation/exception and join_existing=False sequencing semantics. Add regression coverage for concurrent joins, cancellation isolation, bounded callback fan-out (sabotage-proven: shield yields 201 callbacks for 100 waiters vs bounded 2), and successor sequencing. OpenSpec: fix-usage-refresh-shared-future-waiters extends the proxy-admission-control shared-future contract to usage-refresh waiters. Fixes #1896 --- app/modules/usage/updater.py | 5 +- .../.openspec.yaml | 2 + .../design.md | 69 ++++++++ .../proposal.md | 40 +++++ .../specs/proxy-admission-control/spec.md | 45 +++++ .../tasks.md | 17 ++ tests/unit/test_usage_updater.py | 157 ++++++++++++++++++ 7 files changed, 333 insertions(+), 2 deletions(-) create mode 100644 openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml create mode 100644 openspec/changes/fix-usage-refresh-shared-future-waiters/design.md create mode 100644 openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md create mode 100644 openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md create mode 100644 openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md diff --git a/app/modules/usage/updater.py b/app/modules/usage/updater.py index 2327ecba14..66af2e2c98 100644 --- a/app/modules/usage/updater.py +++ b/app/modules/usage/updater.py @@ -26,6 +26,7 @@ from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.usage.models import AdditionalRateLimitPayload, UsagePayload, UsageWindow from app.core.utils.request_id import get_request_id +from app.core.utils.shared_future import wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.db.session import get_background_session @@ -199,14 +200,14 @@ async def run( if wait_for_existing is None: break try: - await asyncio.shield(wait_for_existing) + await wait_on_shared_future(wait_for_existing) except asyncio.CancelledError: current_task = asyncio.current_task() if current_task is not None and current_task.cancelling(): raise except Exception: pass - return await asyncio.shield(task) + return await wait_on_shared_future(task) async def _run_factory( self, diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml b/openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml new file mode 100644 index 0000000000..4102db8a47 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-24 diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md new file mode 100644 index 0000000000..312b234671 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/design.md @@ -0,0 +1,69 @@ +## Context + +`_UsageRefreshSingleflight` owns one task per account and may expose that task +to many request and scheduler waiters. Its two shared waits still use +`asyncio.shield`, unlike the bridge and token-refresh sites converted by +[`harden-shared-future-admission-waits`](../harden-shared-future-admission-waits/). +The existing helper already provides the required result, exception, and +cancellation semantics; see the +[`proxy-admission-control`](specs/proxy-admission-control/spec.md) delta and +the existing +[`proxy-runtime-observability`](../../specs/proxy-runtime-observability/) +signals. + +## Goals / Non-Goals + +**Goals:** + +- Make both joining and non-joining usage-refresh waits constant-cost under + cancellation storms. +- Preserve singleflight task ownership and successor ordering. +- Prove the usage-refresh surface, not only the generic helper, maintains one + fan-out callback. + +**Non-Goals:** + +- Change refresh selection, persistence, exception swallowing, or shutdown. +- Rewrite the helper or convert request-owned cleanup shields. +- Integrate or rebase the unrelated session-ownership work in PR #1887. + +## Decisions + +### Reuse the established shared-future helper at both wait sites + +Both the `join_existing=True` return path and the `join_existing=False` +predecessor wait can accumulate many waiters on one task, so both call +`wait_on_shared_future`. Reusing the established helper preserves waiter +cancellation isolation while keeping one fan-out callback. Keeping +`asyncio.shield` at either site would retain the incident mechanism; a second +usage-specific helper would duplicate the existing contract. + +### Preserve the current control flow + +The non-joining path continues swallowing predecessor failures and retries the +loop, while caller cancellation continues propagating. The final wait +continues propagating the selected task's result or exception. This limits the +fix to waiter mechanics and avoids changing refresh policy. + +### Test callback structure through the usage singleflight + +The regression test attaches many `run` callers, inspects the in-flight task's +callback count, cancels most callers, and verifies the count remains bounded +while a survivor receives the factory result. This test fails with the old +shield implementation because each waiter attaches callbacks to the shared +task. + +## Risks / Trade-offs + +- **Risk:** The private callback-list assertion depends on CPython asyncio + internals. **Mitigation:** Match the existing helper and bridge regression + pattern, and guard only the structural property involved in the production + incident. +- **Risk:** PR #1887 also edits the same singleflight. **Mitigation:** Keep this + patch focused on current `main` and call out the conflict so that PR must + carry this conversion forward. + +## Migration Plan + +Deploy through the normal image release process after merge. Rollback is a +code rollback; there are no data, schema, or configuration migrations. diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md new file mode 100644 index 0000000000..a1f9e54992 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/proposal.md @@ -0,0 +1,40 @@ +## Why + +The usage-refresh singleflight was omitted when shared, many-waiter futures +were hardened after the 2026-08-20 event-loop livelock. After roughly 91 hours +of production uptime, cancelled usage-refresh waiters again drove the event +loop into the same `asyncio.shield` callback-removal failure mode, so this +remaining shared wait site must use the established fan-out helper. + +## What Changes + +- Route both usage-refresh singleflight wait paths through + `wait_on_shared_future`, preserving result, cancellation, exception, and + `join_existing=False` sequencing semantics. +- Keep the shared refresh factory task running when an individual waiter is + cancelled or times out, with one bounded fan-out callback on that task. +- Add usage-refresh surface regression coverage for concurrent joins, + cancellation isolation, callback fan-out, and successor sequencing. + +## Capabilities + +### New Capabilities + +None. + +### Modified Capabilities + +- `proxy-admission-control`: extend the established shared-future admission + contract to usage-refresh singleflight waiters. + +## Impact + +- `app/modules/usage/updater.py` and `tests/unit/test_usage_updater.py`. +- No API, schema, configuration, dependency, or deployment changes. +- This is a focused follow-on to + [`harden-shared-future-admission-waits`](../harden-shared-future-admission-waits/) + and relies on the existing + [`proxy-admission-control`](../../specs/proxy-admission-control/) wait + mechanism and + [`proxy-runtime-observability`](../../specs/proxy-runtime-observability/) + event-loop lag signals. diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md new file mode 100644 index 0000000000..3630c0996d --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/specs/proxy-admission-control/spec.md @@ -0,0 +1,45 @@ +## MODIFIED Requirements + +### Requirement: Admission waits on shared futures scale O(1) per waiter + +When multiple requests wait on one shared future (an inflight bridge session creation, a capacity slot, a token-refresh singleflight, or a usage-refresh singleflight), the system MUST use the established shared-future fan-out wait mechanism so attaching a waiter, a waiter timing out, and a waiter being cancelled each perform O(1) work on the shared future. The shared future MUST carry a constant number of done callbacks regardless of waiter count, and the wait mechanism itself MUST NOT cancel or otherwise mutate the shared future or the work it represents when a waiter times out or is cancelled. Admission handlers MAY still settle the shared future explicitly after a waiter's timeout (the http-bridge timeout handler fails and unregisters the inflight future so piled-up waiters converge on one overload outcome); that settlement is an admission-contract decision, not a side effect of waiting. The shared future's result, exception, or cancellation MUST propagate to every waiter with the same semantics as `asyncio.wait_for(asyncio.shield(shared), timeout)`. + +#### Scenario: Waiter pile-up keeps the shared future's callback list constant + +- **WHEN** many requests wait on the same inflight bridge-session future +- **THEN** the shared future carries a constant number of done callbacks +- **AND** the callback count does not grow with the number of waiters + +#### Scenario: Mass timeout does not degrade the event loop + +- **GIVEN** waiters piled onto a shared future that has not resolved within + the admission wait timeout +- **WHEN** the waiters time out together +- **THEN** each timeout detaches in O(1) without scanning the shared future's + callback list +- **AND** the surviving admission contract (local-overload `429` with the + capacity error code) is unchanged + +#### Scenario: Client-disconnect storm leaves the owner's creation running + +- **WHEN** every waiter on an inflight session future is cancelled by client + disconnects +- **THEN** the shared future stays pending and the owner's session creation + continues +- **AND** no per-waiter callbacks remain attached to the shared future + +#### Scenario: Cancelled usage-refresh waiters leave shared refresh running + +- **GIVEN** many callers are waiting on one in-flight usage refresh +- **WHEN** all but one caller are cancelled +- **THEN** the cancelled callers detach without adding or removing per-waiter + callbacks on the shared refresh task +- **AND** the shared refresh continues to completion for the remaining caller + +#### Scenario: Non-joining usage refresh starts after its predecessor + +- **GIVEN** a usage refresh is already in flight for an account +- **WHEN** another caller requests a non-joining refresh for that account +- **THEN** it waits without cancelling or mutating the in-flight refresh +- **AND** it starts a successor refresh only after the in-flight refresh has + finished diff --git a/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md b/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md new file mode 100644 index 0000000000..5083e7c262 --- /dev/null +++ b/openspec/changes/fix-usage-refresh-shared-future-waiters/tasks.md @@ -0,0 +1,17 @@ +## 1. Shared-Future Wait Conversion + +- [x] 1.1 Audit remaining `asyncio.shield` calls in `app/` and confirm only usage-refresh singleflight matches the shared, many-waiter class. +- [x] 1.2 Replace both `_UsageRefreshSingleflight.run` shared-task waits with `wait_on_shared_future` while preserving cancellation and exception semantics. + +## 2. Regression Coverage + +- [x] 2.1 Test that concurrent usage-refresh waiters receive the same result and cancelling all but one does not cancel the factory task. +- [x] 2.2 Test that cancelled usage-refresh waiters detach with one bounded fan-out callback on the in-flight task. +- [x] 2.3 Test that `join_existing=False` waits for the predecessor before starting a successor through the shared-future helper. +- [x] 2.4 Temporarily restore the shield implementation, run the callback fan-out regression test, and record the failing sabotage result. + +## 3. Verification + +- [x] 3.1 Run the targeted usage updater and shared-future waiter unit tests. +- [x] 3.2 Run the repository lint source of truth and strict OpenSpec validation. +- [x] 3.3 Record commands and exit codes in `/tmp/codex-lb-1896-verification.md`. diff --git a/tests/unit/test_usage_updater.py b/tests/unit/test_usage_updater.py index bf624e747e..069b68e96a 100644 --- a/tests/unit/test_usage_updater.py +++ b/tests/unit/test_usage_updater.py @@ -17,6 +17,7 @@ from app.core.usage import refresh_scheduler as refresh_scheduler_module from app.core.usage.models import UsagePayload from app.core.usage.refresh_scheduler import _select_long_window_entries +from app.core.utils.shared_future import _WAITERS_ATTR, wait_on_shared_future from app.core.utils.time import utcnow from app.db.models import Account, AccountStatus, UsageHistory from app.modules.usage import updater as usage_updater_module @@ -77,6 +78,162 @@ async def factory(): assert usage_updater_module._USAGE_REFRESH_SINGLEFLIGHT._inflight == {} +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_concurrent_waiters_share_result() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=True) + factory_calls = 0 + + async def factory() -> usage_updater_module.AccountRefreshResult: + nonlocal factory_calls + factory_calls += 1 + started.set() + await release.wait() + return result + + waiters = [asyncio.create_task(singleflight.run("acc_shared_result", factory)) for _ in range(50)] + await asyncio.wait_for(started.wait(), timeout=1) + release.set() + + results = await asyncio.gather(*waiters) + + assert factory_calls == 1 + assert all(item is result for item in results) + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_waiter_cancellation_leaves_factory_running() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + factory_cancelled = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=True) + + async def factory() -> usage_updater_module.AccountRefreshResult: + started.set() + try: + await release.wait() + except asyncio.CancelledError: + factory_cancelled.set() + raise + return result + + waiters = [asyncio.create_task(singleflight.run("acc_cancel_waiters", factory)) for _ in range(20)] + await asyncio.wait_for(started.wait(), timeout=1) + await asyncio.sleep(0) + inflight = singleflight._inflight["acc_cancel_waiters"] + + for waiter in waiters[:-1]: + waiter.cancel() + cancelled = await asyncio.gather(*waiters[:-1], return_exceptions=True) + + assert all(isinstance(item, asyncio.CancelledError) for item in cancelled) + assert not inflight.done() + assert not factory_cancelled.is_set() + + release.set() + assert await waiters[-1] is result + assert not factory_cancelled.is_set() + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_cancelled_waiters_keep_callback_fanout_bounded() -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + started = asyncio.Event() + release = asyncio.Event() + result = usage_updater_module.AccountRefreshResult(usage_written=False) + + async def factory() -> usage_updater_module.AccountRefreshResult: + started.set() + await release.wait() + return result + + waiters = [asyncio.create_task(singleflight.run("acc_callback_fanout", factory)) for _ in range(100)] + await asyncio.wait_for(started.wait(), timeout=1) + inflight = singleflight._inflight["acc_callback_fanout"] + for _ in range(10): + if len(getattr(inflight, _WAITERS_ATTR, set())) == len(waiters): + break + await asyncio.sleep(0) + + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 2, ( + "usage-refresh waiters must share one fan-out callback in addition to " + f"singleflight cleanup; found {None if callbacks is None else len(callbacks)} callbacks" + ) + assert len(getattr(inflight, _WAITERS_ATTR)) == len(waiters) + + for waiter in waiters: + waiter.cancel() + cancelled = await asyncio.gather(*waiters, return_exceptions=True) + await asyncio.sleep(0) + + assert all(isinstance(item, asyncio.CancelledError) for item in cancelled) + assert not inflight.done() + callbacks = getattr(inflight, "_callbacks", None) + assert callbacks is not None and len(callbacks) == 2 + assert getattr(inflight, _WAITERS_ATTR) == set() + + release.set() + assert await inflight is result + + +@pytest.mark.asyncio +async def test_usage_refresh_singleflight_non_joiner_waits_then_starts_successor( + monkeypatch: pytest.MonkeyPatch, +) -> None: + singleflight = usage_updater_module._UsageRefreshSingleflight() + first_started = asyncio.Event() + release_first = asyncio.Event() + successor_started = asyncio.Event() + release_successor = asyncio.Event() + first_result = usage_updater_module.AccountRefreshResult(usage_written=False) + successor_result = usage_updater_module.AccountRefreshResult(usage_written=True) + shared_waits: list[asyncio.Future[usage_updater_module.AccountRefreshResult]] = [] + + async def recording_wait( + shared: asyncio.Future[usage_updater_module.AccountRefreshResult], + *, + timeout: float | None = None, + ) -> usage_updater_module.AccountRefreshResult: + shared_waits.append(shared) + return await wait_on_shared_future(shared, timeout=timeout) + + async def first_factory() -> usage_updater_module.AccountRefreshResult: + first_started.set() + await release_first.wait() + return first_result + + async def successor_factory() -> usage_updater_module.AccountRefreshResult: + successor_started.set() + await release_successor.wait() + return successor_result + + monkeypatch.setattr(usage_updater_module, "wait_on_shared_future", recording_wait) + first_waiter = asyncio.create_task(singleflight.run("acc_non_joiner", first_factory)) + await asyncio.wait_for(first_started.wait(), timeout=1) + first_task = singleflight._inflight["acc_non_joiner"] + non_joiner = asyncio.create_task( + singleflight.run("acc_non_joiner", successor_factory, join_existing=False), + ) + await asyncio.sleep(0) + + assert not successor_started.is_set() + assert shared_waits.count(first_task) == 2 + + release_first.set() + assert await first_waiter is first_result + await asyncio.wait_for(successor_started.wait(), timeout=1) + successor_task = singleflight._inflight["acc_non_joiner"] + assert successor_task is not first_task + + release_successor.set() + assert await non_joiner is successor_result + assert successor_task in shared_waits + + @pytest.mark.asyncio async def test_refresh_accounts_owned_singleflight_session_outlives_caller_cancellation( monkeypatch: pytest.MonkeyPatch, From b311aea760aa639fd96f63bd118f775e9b4a89f9 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Mon, 24 Aug 2026 17:01:55 +0900 Subject: [PATCH 108/117] chore: release v1.24.0-beta.4 (#1851) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 ++-- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index 4a70336796..00ddd03c75 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.24.0-beta.3" # x-release-please-version +__version__ = "1.24.0-beta.4" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index e26334aaa8..d3b9f6c3c1 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.24.0-beta.3 -appVersion: 1.24.0-beta.3 +version: 1.24.0-beta.4 +appVersion: 1.24.0-beta.4 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index 0bb4ff6a8b..6c0e39a691 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.24.0-beta.3", + "version": "1.24.0-beta.4", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 1625459040..fe43cd4935 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.24.0-beta.3" +version = "1.24.0-beta.4" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 2086fe95bd..3a562c302e 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0-beta.3" +version = "1.24.0-beta.4" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From cedc05f95a46fe062eaf8ff35eceea1a91735a2a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:22:13 +0900 Subject: [PATCH 109/117] chore(deps): bump the python-minor-patch group with 7 updates (#1915) Bumps the python-minor-patch group with 7 updates: pygments 2.21.0, python-dotenv 1.2.3, uvicorn 0.52.4, ruff 0.16.4, ty 0.0.73, openai 3.3.1, hypothesis 6.165.10. Includes source-contract fixes for ty 0.0.73 (new implicit-return diagnostics on shielded cancellation-deferral helpers, plus restored isinstance narrowing for the service_tier lookup). Verified: ty 0.0.73/0.0.72 both clean, ruff check/format clean, targeted pytest (unit + proxy integration + SDK e2e) 6486 passed, CI green at 5599da0, current-head Codex review clean. --- app/modules/model_sources/forwarding.py | 1 + app/modules/proxy/_service/streaming/retry.py | 1 + app/modules/proxy/api.py | 8 +- pyproject.toml | 2 +- uv.lock | 345 ++++++++---------- 5 files changed, 165 insertions(+), 192 deletions(-) diff --git a/app/modules/model_sources/forwarding.py b/app/modules/model_sources/forwarding.py index 727eeb67d6..19dd4895f9 100644 --- a/app/modules/model_sources/forwarding.py +++ b/app/modules/model_sources/forwarding.py @@ -138,6 +138,7 @@ async def _await_result_deferring_cancellation(awaitable: Awaitable[object]) -> if task.cancelled(): raise cancellation_deferred = True + raise RuntimeError("unreachable shielded cancellation-deferral state") async def forward_chat_completion( diff --git a/app/modules/proxy/_service/streaming/retry.py b/app/modules/proxy/_service/streaming/retry.py index c0d2c43abe..4e725f0911 100644 --- a/app/modules/proxy/_service/streaming/retry.py +++ b/app/modules/proxy/_service/streaming/retry.py @@ -108,6 +108,7 @@ async def _await_task_deferring_cancellation( if task.cancelled(): raise cancellation = cancellation or exc + raise RuntimeError("unreachable shielded cancellation-deferral state") def _facade() -> Any: diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index e5c0e1b4d4..6dd14e12f3 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -2136,6 +2136,7 @@ async def _await_result_deferring_cancellation(awaitable: Awaitable[_T]) -> tupl if task.cancelled(): raise cancellation_deferred = True + raise RuntimeError("unreachable shielded cancellation-deferral state") async def _await_cleanup_deferring_cancellation(awaitable: Awaitable[object]) -> None: @@ -5804,14 +5805,11 @@ def build_recovery_response_stream() -> AsyncIterator[str]: async def _retry() -> AsyncIterator[str]: retry_reservation = reservation if prefer_http_bridge and api_key is not None and reservation is not None: + retry_service_tier = dict(payload.to_payload()).get("service_tier") retry_reservation = await _enforce_request_limits( api_key, request_model=payload.model, - request_service_tier=( - dict(payload.to_payload()).get("service_tier") - if isinstance(dict(payload.to_payload()).get("service_tier"), str) - else None - ), + request_service_tier=(retry_service_tier if isinstance(retry_service_tier, str) else None), request_usage_budget=estimate_api_key_request_usage(payload), ) retry_stream = context.service.stream_http_responses( diff --git a/pyproject.toml b/pyproject.toml index fe43cd4935..3ca5a6465b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,7 +82,7 @@ dev = [ "pytest-timeout>=2.4.0", "httpx>=0.28.1", "ruff>=0.14.13", - "ty==0.0.72", + "ty==0.0.73", "openai>=2.16.0", "pytest-xdist>=3.8.0", "pytest-cov>=7.1.0", diff --git a/uv.lock b/uv.lock index 3a562c302e..4f17749811 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0-beta.4" +version = "1.24.0b4" source = { editable = "." } dependencies = [ { name = "aiohttp" }, @@ -600,7 +600,7 @@ dev = [ { name = "pytest-timeout", specifier = ">=2.4.0" }, { name = "pytest-xdist", specifier = ">=3.8.0" }, { name = "ruff", specifier = ">=0.14.13" }, - { name = "ty", specifier = "==0.0.72" }, + { name = "ty", specifier = "==0.0.73" }, ] docs = [{ name = "mkdocs-material", specifier = ">=9.6" }] @@ -741,15 +741,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] -[[package]] -name = "distro" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fc/f8/98eea607f65de6527f8a2e8885fc8015d3e6f5775df186e443e0964a11c3/distro-1.9.0.tar.gz", hash = "sha256:2fa77c6fd8940f116ee1d6b94a2f90b13b5ea8d019b98bc8bafdcabcdd9bdbed", size = 60722, upload-time = "2023-12-24T09:54:32.31Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, -] - [[package]] name = "dnspython" version = "2.8.0" @@ -1205,67 +1196,67 @@ wheels = [ [[package]] name = "hypothesis" -version = "6.165.8" +version = "6.165.10" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sortedcontainers" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/28/1a/8afd0551a13e43513b107298074e8c16dccf1bc03fb2bcf4220e5b01316a/hypothesis-6.165.8.tar.gz", hash = "sha256:8d19b159ca5ff72db9f0c13183ebf3a5a2f07e2310130bcb6ce7eb24ea9ea9d2", size = 503524, upload-time = "2026-08-14T19:08:14.403Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/45/55/b717931951a64b0d5de2cb822fe07ae0800031ff53637b8ac65d94d6fbcb/hypothesis-6.165.8-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:2b2d1cc1c482e53a52fee8bfd7ca605665dcb24e31d0467dfc513714f25c496f", size = 783023, upload-time = "2026-08-14T19:05:48.344Z" }, - { url = "https://files.pythonhosted.org/packages/9a/62/86ad7e0fdeaadeb73da0816cfd820aaa1211302a663845b43d2e2f707bda/hypothesis-6.165.8-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:d95d4007ed1b1922ad935477fdff0b5c32f7f124b9011b79046bbc95cd3d8a49", size = 778589, upload-time = "2026-08-14T19:06:24.736Z" }, - { url = "https://files.pythonhosted.org/packages/e8/5b/519ea72ed5c43356699d32db05aa7cf19675105860c6dae7b68c493470c2/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b67b996ae2ae76cda1bea2a7537f1c5d0a9ea346b1043ef12b248fcf7d01205", size = 1107806, upload-time = "2026-08-14T19:08:01.253Z" }, - { url = "https://files.pythonhosted.org/packages/b3/92/d6bb217ac935b2d622400982d5c2ea8aea75cbb58ccb2152cfe5a75907f0/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:35a25f0bb13b96096f803c2237d44a9b44aeaf79a996575cbffc6429b7dd363e", size = 1136423, upload-time = "2026-08-14T19:06:37.918Z" }, - { url = "https://files.pythonhosted.org/packages/24/a2/2f8063c36c25aea4d1f9f585fe803d645d16105e69e5872658f225c932ec/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e78f243af26f9166eceb672061fecf7221edebf9aefe81aeb5c7003e29a3c7df", size = 1135048, upload-time = "2026-08-14T19:07:15.195Z" }, - { url = "https://files.pythonhosted.org/packages/ed/64/c8d99086b20c02f1e013f1783deb96febe34ed151169d0a373a82ba485bc/hypothesis-6.165.8-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7567a399ece2643fcc86bf9156186a5958754a9238ab31701ee2cb91cc25dcf", size = 1157305, upload-time = "2026-08-14T19:08:05.672Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2a/995099db937fa105355adbc93af451ff18e0c619ab6e8e07d0a3e3297274/hypothesis-6.165.8-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:9c30099a1fc223108a8c1e5d8ad8c71532904022a887e8d0f318f825ff49a60a", size = 1112633, upload-time = "2026-08-14T19:06:16.192Z" }, - { url = "https://files.pythonhosted.org/packages/1e/12/56ff501135a2e227a6fefbe04856b6b4c374be57148fc8e83b7896cbeee3/hypothesis-6.165.8-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a8501178c455bfd9b23c75c5429549a45e7b40fea42d14d6bf78694912f13b92", size = 1149398, upload-time = "2026-08-14T19:06:12.879Z" }, - { url = "https://files.pythonhosted.org/packages/de/99/2f35dd48d61d914a443e8ee3abf278cd08cbe858b99faaec03db04bc708f/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:cfb2f628bcaf740a51f6874e9dc1e354a94509bc374266daaee7ae64de5ea2ee", size = 1283253, upload-time = "2026-08-14T19:07:11.261Z" }, - { url = "https://files.pythonhosted.org/packages/61/34/ec9de1e751ffc429234eedb185b13095b5556f2b30dfceb0131674ede01e/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:496d2116b5472bb4691931087a3bcd813c3fa4b9a679490dca8543fd5950e7bf", size = 1409756, upload-time = "2026-08-14T19:07:27.903Z" }, - { url = "https://files.pythonhosted.org/packages/d4/33/f53329f3aaaa4a61e3aa92dfae4e815b13bb74dafef526d0f45b731cabec/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:45bf3bcaa7f01688f4dc77b377745566b195afc2e41074a182ddf7d862690611", size = 1264781, upload-time = "2026-08-14T19:05:57.085Z" }, - { url = "https://files.pythonhosted.org/packages/29/56/7a2fe9de26b136161d19487529ebe75864693bc347f7d5baed8510cebe2d/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:7a892413976cecad6d53abc6adf266c6446b64958dc2b5cf405ba8fe0c72a48d", size = 1282528, upload-time = "2026-08-14T19:07:54.893Z" }, - { url = "https://files.pythonhosted.org/packages/1e/73/d4363e6f9740a6c5508a583111793e9406cef8e79e6616e60b988e8ae11e/hypothesis-6.165.8-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ff5d035dd93fc4706974cb84968aaa8f7d4d463d0628ba2dbeb209a2173efb7c", size = 1324582, upload-time = "2026-08-14T19:06:29.579Z" }, - { url = "https://files.pythonhosted.org/packages/d8/b3/c96f16bb6cd5fedff00bb196bec3379205b90be9af3503d59be8ba60dce7/hypothesis-6.165.8-cp310-abi3-win32.whl", hash = "sha256:08ad80fb46118951c797dd10fe8e2b789c17bd5d8d6b37229d542b86808a092e", size = 668817, upload-time = "2026-08-14T19:07:40.316Z" }, - { url = "https://files.pythonhosted.org/packages/cd/e3/f9d54ef4dd8748487cd5f6b6adbf22342792ccdc312f71c77d01fef7812e/hypothesis-6.165.8-cp310-abi3-win_amd64.whl", hash = "sha256:8af82df1e702a27c44957e33e5ec9da52a4fa4fa9dce6ae573e49a7ea76056c6", size = 674969, upload-time = "2026-08-14T19:06:21.194Z" }, - { url = "https://files.pythonhosted.org/packages/b6/db/8be03eed5476497135d2b4c385912200e7f0c42e1acea8c1b6158ee52677/hypothesis-6.165.8-cp310-abi3-win_arm64.whl", hash = "sha256:f82627da51d12f74f3751fb471d65c51af3f546f07977a4c6c8de9353bd16e96", size = 673309, upload-time = "2026-08-14T19:07:52.906Z" }, - { url = "https://files.pythonhosted.org/packages/3b/4d/87e7fda9ed1c80ef741eab5cc3266a5c145af756d3043c6918b588de6965/hypothesis-6.165.8-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:22563b83b52891356c275945cb6c939d5f6f6d29c39cf9a59f3e0c586d53d508", size = 784504, upload-time = "2026-08-14T19:06:05.026Z" }, - { url = "https://files.pythonhosted.org/packages/de/dd/eca7718276a3ef5fb516303cdf17637fa04ed368bfa4eb15eb7b0b5479fb/hypothesis-6.165.8-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fb6d5135a5dc095c34882ec0005f9475ef26498ab52698c7f0cbf5f8b7ae148f", size = 776133, upload-time = "2026-08-14T19:06:08.131Z" }, - { url = "https://files.pythonhosted.org/packages/03/a0/38d84c32b21a30116c77e854356159d60663b72b028029dfe891ccd8a426/hypothesis-6.165.8-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:686a9699b59945758d49c17396c9391b2d63c59f618176d0c2d0515a41a6737c", size = 1106544, upload-time = "2026-08-14T19:06:01.491Z" }, - { url = "https://files.pythonhosted.org/packages/85/bc/53ba55d504a617bd438160b305a4b19fbdd73bf8e20f4420ade290c02f06/hypothesis-6.165.8-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5a035d988587327cfa345f953927f3430ce2b4bb609f2c753a72dbc460844af5", size = 1156527, upload-time = "2026-08-14T19:06:06.612Z" }, - { url = "https://files.pythonhosted.org/packages/51/a2/5a50a78e7b98914a29d98a5b8ddc42ed3bb6efd2bf7d5f98f7c0022fcf91/hypothesis-6.165.8-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:21c32750e77c1ddc7147964d0872679a3f0908f74e9aaa5b8be6f46595ea0125", size = 1280572, upload-time = "2026-08-14T19:05:52.727Z" }, - { url = "https://files.pythonhosted.org/packages/6b/28/2ae1810296d786e0341c4d496b5f3676b9164df1d71fdf246fa4a321211b/hypothesis-6.165.8-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:08c5a7e08f348f01138e084ac5f09c59254394ef9f7a7957fe0ae418c49ac4fe", size = 1323651, upload-time = "2026-08-14T19:07:13.443Z" }, - { url = "https://files.pythonhosted.org/packages/63/1f/ac475606ebc2915091143f8b0cfeb853f021e443f9dfa97c3c0397b026be/hypothesis-6.165.8-cp313-cp313-win_amd64.whl", hash = "sha256:13bc0f4c8f144a222a038a12511c51e85c5171694b836e1df3b66ffa5250b71d", size = 672124, upload-time = "2026-08-14T19:07:50.792Z" }, - { url = "https://files.pythonhosted.org/packages/4e/52/207d717abfb745bfa3832abf336835c16e4150c6aeeefdee8c8a648c12f3/hypothesis-6.165.8-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:0c7177e1da92ed23fd73c27ecb4d5b8298f4fff3fe840d3b7072fe0b8ccd8009", size = 784608, upload-time = "2026-08-14T19:07:07.314Z" }, - { url = "https://files.pythonhosted.org/packages/c5/07/efd8d6c16b94c78da020604f4bde3ae320d524f0e3b481bd57466d90e03a/hypothesis-6.165.8-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:537d1cf3ec45f34cd73f77e89867085465bb66491e709bb2acadc3d32aaba9b6", size = 776280, upload-time = "2026-08-14T19:05:51.414Z" }, - { url = "https://files.pythonhosted.org/packages/65/2f/1e3b5272b2482d01c33d2ad346bc6b1213ceabac194c4f98f1d36f1f3eeb/hypothesis-6.165.8-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d06eb8b5c56aedb46934b7d285b1bd599505219d33563e20796df823cfca2d94", size = 1107056, upload-time = "2026-08-14T19:06:09.879Z" }, - { url = "https://files.pythonhosted.org/packages/ab/e2/3822efb584f663706a45dee8d23bdeba651853901b71e20512dae6003cd9/hypothesis-6.165.8-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d5de2603cbe246804aaac7a5e17fb0171cb0bf2c902afbab0fc9bccc0f8ff5f", size = 1156667, upload-time = "2026-08-14T19:05:49.756Z" }, - { url = "https://files.pythonhosted.org/packages/27/c3/6bd4eccbdbc4ffe37fbfb4b505b04dc4188f7ea05b384a2f732a7ec82a3d/hypothesis-6.165.8-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:26e8c164a556cd0324709c70ecb5b095111fe5f1e619e49260eb4f142742fd0b", size = 1280967, upload-time = "2026-08-14T19:06:48.699Z" }, - { url = "https://files.pythonhosted.org/packages/b9/61/facf2b95c10ad141e5dea884f1a2b84b39bc014569c665dc6ae337bc6fff/hypothesis-6.165.8-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:4a8ebaaacd542b5aa07085e8f4d6adf8b05b75c616482d4267cea800a992a94f", size = 1324021, upload-time = "2026-08-14T19:07:18.888Z" }, - { url = "https://files.pythonhosted.org/packages/09/39/49d22db5a207ef1c5f371777ac4e8e4efee7acb8d11931891c0809ca6650/hypothesis-6.165.8-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:2f62bf3b168c7227e361dec31d9cf53f7f4448d49248ff4a79ea2c529689a394", size = 616171, upload-time = "2026-08-14T19:06:03.189Z" }, - { url = "https://files.pythonhosted.org/packages/42/01/03ffa475c46b14f0324b35a4e8e13613e03abc1f374bbf276c26a7b79dee/hypothesis-6.165.8-cp314-cp314-win_amd64.whl", hash = "sha256:3b31f549cebaf42902e031510742d08b61abfa79e43872f8c05d43599fc5dae0", size = 671925, upload-time = "2026-08-14T19:06:50.459Z" }, - { url = "https://files.pythonhosted.org/packages/69/9d/999668ef0c455048d23e394fcc5fd44b682890d35d9701ff7d8f05d5857d/hypothesis-6.165.8-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:2d6ba5947bd7084e062ffbf375e49d2d0f438f80462fd78fdf70170c6c87ff9a", size = 783067, upload-time = "2026-08-14T19:06:17.935Z" }, - { url = "https://files.pythonhosted.org/packages/d6/dc/7284b3e3a1b7e3dcefe1c473d2da8a1f206746a2f84516c671ee95b6fb70/hypothesis-6.165.8-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c9c77c83d59deb4e52bb0567a906cb8c0e8231669d2e165a90a8f3f2cdc153c6", size = 774693, upload-time = "2026-08-14T19:07:56.962Z" }, - { url = "https://files.pythonhosted.org/packages/b9/0f/e894faff27e3c665075281e78578479d71d16886c8cadec3d9a37d720502/hypothesis-6.165.8-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8626e21861b66bdc0d90bcb3fe831baa6e42fd29aed3f1070a0a84917d941b1c", size = 1105291, upload-time = "2026-08-14T19:08:03.463Z" }, - { url = "https://files.pythonhosted.org/packages/bb/3e/2681fa031dff98e0b30b77aeeff1a5b5e84857fa4d0d03a2136e0e504716/hypothesis-6.165.8-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c7155cfd391220855dc523c58968763fbd4143a63f68331f96b73e5284dc96e1", size = 1155413, upload-time = "2026-08-14T19:07:30.085Z" }, - { url = "https://files.pythonhosted.org/packages/18/03/c828173cea01ffaa38e8faf9e79db38ffc0a4600db1ae50f55e893977118/hypothesis-6.165.8-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:273885939b364e34ea3d4e0c7b5f71649c8dddd686e011326320c3f661650f00", size = 1278962, upload-time = "2026-08-14T19:08:08.066Z" }, - { url = "https://files.pythonhosted.org/packages/3a/bf/817d7c693f28d079714ac37379e686f2b46b131ae1aa568ffd22aff29a2d/hypothesis-6.165.8-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:13dd1471755d14fbb0a81ff522fc24b4fd84015bff724b5ba1cd1bcae09bd253", size = 1322644, upload-time = "2026-08-14T19:06:39.873Z" }, - { url = "https://files.pythonhosted.org/packages/78/a1/79357a3992b3c9a049f5982b7f2bf17a54c2c888f29f0453e4511e00b81c/hypothesis-6.165.8-cp314-cp314t-win_amd64.whl", hash = "sha256:f604eb3e86ee6eb8f68363079ff3b85e44281ea0af44fd3085345d29528a66ea", size = 671923, upload-time = "2026-08-14T19:07:01.819Z" }, - { url = "https://files.pythonhosted.org/packages/e9/9e/5c521f75c36b4883b11bb3d0f3ecb112081eda3f645df496e0fe8b048101/hypothesis-6.165.8-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:00fa62ff3ed7265ffe9f064b148d2fae9f5bdb8e59212e7bae6d463b97a8622a", size = 782655, upload-time = "2026-08-14T19:06:27.922Z" }, - { url = "https://files.pythonhosted.org/packages/e1/09/63eab3462d42f4a7252437fddfa6d7c6b9ad44d9497546917b8b62c5e752/hypothesis-6.165.8-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:ce7321947298ab4529ee8e5b8e38393bb12f36125d2f1affc4f860510933f7cd", size = 774338, upload-time = "2026-08-14T19:07:25.953Z" }, - { url = "https://files.pythonhosted.org/packages/8a/78/ec97e7f981ad61463bc537d4e8bd46e0b52cdbb501b1e12bd7065531064d/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:486c2b5bdd797667aefe845221be7c8f45ec5af599cdaa6536dc71d445bc81b6", size = 1104676, upload-time = "2026-08-14T19:06:57.807Z" }, - { url = "https://files.pythonhosted.org/packages/88/78/9ca6a51788527fba0dc42e473061d214b02ecc9bebeec1d35e9b26d64627/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bce581a7f2df2d54291f01d8e458ae5eead5d097761411bf0277f9aa6b944efc", size = 1133138, upload-time = "2026-08-14T19:06:46.736Z" }, - { url = "https://files.pythonhosted.org/packages/bf/70/7e8025c115fb76c1adad0984b7b0b25b59c746aad01cc7d4b87f92ce33e5/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ff023503609e114cc9a36139e1597e36df6d443fb87931e232436402e74d57ac", size = 1132073, upload-time = "2026-08-14T19:05:46.653Z" }, - { url = "https://files.pythonhosted.org/packages/3e/d4/4746b097ff330c240d79e691c4575c8d00f0f4f8a5dd41602247130445d6/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80d34b700aae1fee6ec644d0417a67e7710e109ec50f1ad5db5f8a3a42ca1fbb", size = 1154989, upload-time = "2026-08-14T19:07:48.698Z" }, - { url = "https://files.pythonhosted.org/packages/fb/24/4a87adbd95823be301b73a8df7fd669b6468759d6ee6d503f4f7c2f24552/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:258b12f5683c04a1374adde24f40cbf65fcdbb76758ae000cc901a4a26a33c77", size = 1109664, upload-time = "2026-08-14T19:06:26.242Z" }, - { url = "https://files.pythonhosted.org/packages/cf/b4/7b33b233671621bdfcebf7e12e2944aa179e86e92791fbba87b5bfb03790/hypothesis-6.165.8-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a7ea1106f25dd191c4e600ae31b61552d6066a0ab7da7fd8ae8fe3b4bfb31ae1", size = 1144746, upload-time = "2026-08-14T19:07:46.456Z" }, - { url = "https://files.pythonhosted.org/packages/75/9c/1546de2abd9e084c409671e721740648a1f88da98a64fd0f314cd15d420c/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:871c32499f86473df83c4670399e8994ee869b0fba12c711885f4c48510cfa1d", size = 1278501, upload-time = "2026-08-14T19:06:32.897Z" }, - { url = "https://files.pythonhosted.org/packages/26/56/ac1d16b9cfc2fdc11ddd425e4576cc8e7cf2bcd525e8c70e65ca8ea152cb/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:e769cc3cbe002134be0eda1ce6872efa61b13146c3f66eea3eaff6c7a1907c4a", size = 1406976, upload-time = "2026-08-14T19:05:55.607Z" }, - { url = "https://files.pythonhosted.org/packages/2c/92/059d33ed4711a81e73d939ee7303ce7792d5678954e913676fe696931c39/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:4af4d747f5f1fc81806844dfa8618f20f71ad599d55f17c8dca2a8c7fb3cc16e", size = 1261140, upload-time = "2026-08-14T19:07:36.099Z" }, - { url = "https://files.pythonhosted.org/packages/5f/0e/21483d154bb33181da16aba706d391ca88eabf53c2259fbde1d228f950d9/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:46f16174505dc6d977b9513e23f5cae60382e849861673fff881fdc0ab229946", size = 1279005, upload-time = "2026-08-14T19:06:22.975Z" }, - { url = "https://files.pythonhosted.org/packages/95/0f/2b2dc2688f98f482666b8fd30bfb58ee58c07cdbf99226ea357e71beec9c/hypothesis-6.165.8-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:fb61b9a4cb8a63e16baf37b0f693482e47985adde2498a6b6e9cd0dc6141bca0", size = 1322177, upload-time = "2026-08-14T19:08:12.392Z" }, - { url = "https://files.pythonhosted.org/packages/7d/ec/d53cb547ac4c7c39a21760e9a658b7730cbb6b3b93900dcf69cbadf31dcc/hypothesis-6.165.8-cp315-abi3.abi3t-win32.whl", hash = "sha256:8b32c2eed1a439ecf2217403eb048de2c160a61b909fb031a871af030c507c1f", size = 665817, upload-time = "2026-08-14T19:07:03.881Z" }, - { url = "https://files.pythonhosted.org/packages/69/f1/d581a48906ed56f5a57eb39c54b05d584ed0a771c8f3bf5d5d0532ab8c0d/hypothesis-6.165.8-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:7166c761411625770c0fabe475de642102d0f27f910644306337fd208da25924", size = 671723, upload-time = "2026-08-14T19:06:43.24Z" }, - { url = "https://files.pythonhosted.org/packages/76/76/7d905c691331442965e58ddfc267900efeb7e8354b6083455aaa67e41834/hypothesis-6.165.8-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:88e9d1803ed4d7b2580c0e84c1c035f2a1d74327613aab58bffeeb4e3025b6dd", size = 669710, upload-time = "2026-08-14T19:05:59.934Z" }, +sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" }, + { url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" }, + { url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" }, + { url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" }, + { url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" }, + { url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" }, + { url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" }, ] [[package]] @@ -1309,56 +1300,52 @@ wheels = [ [[package]] name = "jiter" -version = "0.14.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6e/c1/0cddc6eb17d4c53a99840953f95dd3accdc5cfc7a337b0e9b26476276be9/jiter-0.14.0.tar.gz", hash = "sha256:e8a39e66dac7153cf3f964a12aad515afa8d74938ec5cc0018adcdae5367c79e", size = 165725, upload-time = "2026-04-10T14:28:42.01Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/2a/09f70020898507a89279659a1afe3364d57fc1b2c89949081975d135f6f5/jiter-0.14.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:af72f204cf4d44258e5b4c1745130ac45ddab0e71a06333b01de660ab4187a94", size = 315502, upload-time = "2026-04-10T14:26:47.697Z" }, - { url = "https://files.pythonhosted.org/packages/d6/be/080c96a45cd74f9fce5db4fd68510b88087fb37ffe2541ff73c12db92535/jiter-0.14.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4b77da71f6e819be5fbcec11a453fde5b1d0267ef6ed487e2a392fd8e14e4e3a", size = 314870, upload-time = "2026-04-10T14:26:49.149Z" }, - { url = "https://files.pythonhosted.org/packages/7d/5e/2d0fee155826a968a832cc32438de5e2a193292c8721ca70d0b53e58245b/jiter-0.14.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f4ea612fe8b84b8b04e51d0e78029ecf3466348e25973f953de6e6a59aa4c1", size = 343406, upload-time = "2026-04-10T14:26:50.762Z" }, - { url = "https://files.pythonhosted.org/packages/70/af/bf9ee0d3a4f8dc0d679fc1337f874fe60cdbf841ebbb304b374e1c9aaceb/jiter-0.14.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:62fe2451f8fcc0240261e6a4df18ecbcd58327857e61e625b2393ea3b468aac9", size = 369415, upload-time = "2026-04-10T14:26:52.188Z" }, - { url = "https://files.pythonhosted.org/packages/0f/83/8e8561eadba31f4d3948a5b712fb0447ec71c3560b57a855449e7b8ddc98/jiter-0.14.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6112f26f5afc75bcb475787d29da3aa92f9d09c7858f632f4be6ffe607be82e9", size = 461456, upload-time = "2026-04-10T14:26:53.611Z" }, - { url = "https://files.pythonhosted.org/packages/f6/c9/c5299e826a5fe6108d172b344033f61c69b1bb979dd8d9ddd4278a160971/jiter-0.14.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:215a6cb8fb7dc702aa35d475cc00ddc7f970e5c0b1417fb4b4ac5d82fa2a29db", size = 378488, upload-time = "2026-04-10T14:26:55.211Z" }, - { url = "https://files.pythonhosted.org/packages/5d/37/c16d9d15c0a471b8644b1abe3c82668092a707d9bedcf076f24ff2e380cd/jiter-0.14.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fc4ab96a30fb3cb2c7e0cd33f7616c8860da5f5674438988a54ac717caccdbaa", size = 353242, upload-time = "2026-04-10T14:26:56.705Z" }, - { url = "https://files.pythonhosted.org/packages/58/ea/8050cb0dc654e728e1bfacbc0c640772f2181af5dedd13ae70145743a439/jiter-0.14.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:3a99c1387b1f2928f799a9de899193484d66206a50e98233b6b088a7f0c1edb2", size = 356823, upload-time = "2026-04-10T14:26:58.281Z" }, - { url = "https://files.pythonhosted.org/packages/b0/3b/cf71506d270e5f84d97326bf220e47aed9b95e9a4a060758fb07772170ab/jiter-0.14.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ab18d11074485438695f8d34a1b6da61db9754248f96d51341956607a8f39985", size = 392564, upload-time = "2026-04-10T14:27:00.018Z" }, - { url = "https://files.pythonhosted.org/packages/b0/cc/8c6c74a3efb5bd671bfd14f51e8a73375464ca914b1551bc3b40e26ac2c9/jiter-0.14.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:801028dcfc26ac0895e4964cbc0fd62c73be9fd4a7d7b1aaf6e5790033a719b7", size = 520322, upload-time = "2026-04-10T14:27:01.664Z" }, - { url = "https://files.pythonhosted.org/packages/41/24/68d7b883ec959884ddf00d019b2e0e82ba81b167e1253684fa90519ce33c/jiter-0.14.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ad425b087aafb4a1c7e1e98a279200743b9aaf30c3e0ba723aec93f061bd9bc8", size = 552619, upload-time = "2026-04-10T14:27:03.316Z" }, - { url = "https://files.pythonhosted.org/packages/b6/89/b1a0985223bbf3150ff9e8f46f98fc9360c1de94f48abe271bbe1b465682/jiter-0.14.0-cp313-cp313-win32.whl", hash = "sha256:882bcb9b334318e233950b8be366fe5f92c86b66a7e449e76975dfd6d776a01f", size = 205699, upload-time = "2026-04-10T14:27:04.662Z" }, - { url = "https://files.pythonhosted.org/packages/4c/19/3f339a5a7f14a11730e67f6be34f9d5105751d547b615ef593fa122a5ded/jiter-0.14.0-cp313-cp313-win_amd64.whl", hash = "sha256:9b8c571a5dba09b98bd3462b5a53f27209a5cbbe85670391692ede71974e979f", size = 201323, upload-time = "2026-04-10T14:27:06.139Z" }, - { url = "https://files.pythonhosted.org/packages/50/56/752dd89c84be0e022a8ea3720bcfa0a8431db79a962578544812ce061739/jiter-0.14.0-cp313-cp313-win_arm64.whl", hash = "sha256:34f19dcc35cb1abe7c369b3756babf8c7f04595c0807a848df8f26ef8298ef92", size = 191099, upload-time = "2026-04-10T14:27:07.564Z" }, - { url = "https://files.pythonhosted.org/packages/91/28/292916f354f25a1fe8cf2c918d1415c699a4a659ae00be0430e1c5d9ffea/jiter-0.14.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e89bcd7d426a75bb4952c696b267075790d854a07aad4c9894551a82c5b574ab", size = 320880, upload-time = "2026-04-10T14:27:09.326Z" }, - { url = "https://files.pythonhosted.org/packages/ad/c7/b002a7d8b8957ac3d469bd59c18ef4b1595a5216ae0de639a287b9816023/jiter-0.14.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7b25beaa0d4447ea8c7ae0c18c688905d34840d7d0b937f2f7bdd52162c98a40", size = 346563, upload-time = "2026-04-10T14:27:11.287Z" }, - { url = "https://files.pythonhosted.org/packages/f9/3b/f8d07580d8706021d255a6356b8fab13ee4c869412995550ce6ed4ddf97d/jiter-0.14.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:651a8758dd413c51e3b7f6557cdc6921faf70b14106f45f969f091f5cda990ea", size = 357928, upload-time = "2026-04-10T14:27:12.729Z" }, - { url = "https://files.pythonhosted.org/packages/47/5b/ac1a974da29e35507230383110ffec59998b290a8732585d04e19a9eb5ba/jiter-0.14.0-cp313-cp313t-win_amd64.whl", hash = "sha256:e1a7eead856a5038a8d291f1447176ab0b525c77a279a058121b5fccee257f6f", size = 203519, upload-time = "2026-04-10T14:27:14.125Z" }, - { url = "https://files.pythonhosted.org/packages/96/6d/9fc8433d667d2454271378a79747d8c76c10b51b482b454e6190e511f244/jiter-0.14.0-cp313-cp313t-win_arm64.whl", hash = "sha256:2e692633a12cda97e352fdcd1c4acc971b1c28707e1e33aeef782b0cbf051975", size = 190113, upload-time = "2026-04-10T14:27:16.638Z" }, - { url = "https://files.pythonhosted.org/packages/4f/1e/354ed92461b165bd581f9ef5150971a572c873ec3b68a916d5aa91da3cc2/jiter-0.14.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:6f396837fc7577871ca8c12edaf239ed9ccef3bbe39904ae9b8b63ce0a48b140", size = 315277, upload-time = "2026-04-10T14:27:18.109Z" }, - { url = "https://files.pythonhosted.org/packages/a6/95/8c7c7028aa8636ac21b7a55faef3e34215e6ed0cbf5ae58258427f621aa3/jiter-0.14.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a4d50ea3d8ba4176f79754333bd35f1bbcd28e91adc13eb9b7ca91bc52a6cef9", size = 315923, upload-time = "2026-04-10T14:27:19.603Z" }, - { url = "https://files.pythonhosted.org/packages/47/40/e2a852a44c4a089f2681a16611b7ce113224a80fd8504c46d78491b47220/jiter-0.14.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce17f8a050447d1b4153bda4fb7d26e6a9e74eb4f4a41913f30934c5075bf615", size = 344943, upload-time = "2026-04-10T14:27:21.262Z" }, - { url = "https://files.pythonhosted.org/packages/fc/1f/670f92adee1e9895eac41e8a4d623b6da68c4d46249d8b556b60b63f949e/jiter-0.14.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f4f1c4b125e1652aefbc2e2c1617b60a160ab789d180e3d423c41439e5f32850", size = 369725, upload-time = "2026-04-10T14:27:22.766Z" }, - { url = "https://files.pythonhosted.org/packages/01/2f/541c9ba567d05de1c4874a0f8f8c5e3fd78e2b874266623da9a775cf46e0/jiter-0.14.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be808176a6a3a14321d18c603f2d40741858a7c4fc982f83232842689fe86dd9", size = 461210, upload-time = "2026-04-10T14:27:24.315Z" }, - { url = "https://files.pythonhosted.org/packages/ce/a9/c31cbec09627e0d5de7aeaec7690dba03e090caa808fefd8133137cf45bc/jiter-0.14.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26679d58ba816f88c3849306dd58cb863a90a1cf352cdd4ef67e30ccf8a77994", size = 380002, upload-time = "2026-04-10T14:27:26.155Z" }, - { url = "https://files.pythonhosted.org/packages/50/02/3c05c1666c41904a2f607475a73e7a4763d1cbde2d18229c4f85b22dc253/jiter-0.14.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:80381f5a19af8fa9aef743f080e34f6b25ebd89656475f8cf0470ec6157052aa", size = 354678, upload-time = "2026-04-10T14:27:27.701Z" }, - { url = "https://files.pythonhosted.org/packages/7d/97/e15b33545c2b13518f560d695f974b9891b311641bdcf178d63177e8801e/jiter-0.14.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:004df5fdb8ecbd6d99f3227df18ba1a259254c4359736a2e6f036c944e02d7c5", size = 358920, upload-time = "2026-04-10T14:27:29.256Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d2/8b1461def6b96ba44530df20d07ef7a1c7da22f3f9bf1727e2d611077bf1/jiter-0.14.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cff5708f7ed0fa098f2b53446c6fa74c48469118e5cd7497b4f1cd569ab06928", size = 394512, upload-time = "2026-04-10T14:27:31.344Z" }, - { url = "https://files.pythonhosted.org/packages/e3/88/837566dd6ed6e452e8d3205355afd484ce44b2533edfa4ed73a298ea893e/jiter-0.14.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:2492e5f06c36a976d25c7cc347a60e26d5470178d44cde1b9b75e60b4e519f28", size = 521120, upload-time = "2026-04-10T14:27:33.299Z" }, - { url = "https://files.pythonhosted.org/packages/89/6b/b00b45c4d1b4c031777fe161d620b755b5b02cdade1e316dcb46e4471d63/jiter-0.14.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:7609cfbe3a03d37bfdbf5052012d5a879e72b83168a363deae7b3a26564d57de", size = 553668, upload-time = "2026-04-10T14:27:34.868Z" }, - { url = "https://files.pythonhosted.org/packages/ad/d8/6fe5b42011d19397433d345716eac16728ac241862a2aac9c91923c7509a/jiter-0.14.0-cp314-cp314-win32.whl", hash = "sha256:7282342d32e357543565286b6450378c3cd402eea333fc1ebe146f1fabb306fc", size = 207001, upload-time = "2026-04-10T14:27:36.455Z" }, - { url = "https://files.pythonhosted.org/packages/e5/43/5c2e08da1efad5e410f0eaaabeadd954812612c33fbbd8fd5328b489139d/jiter-0.14.0-cp314-cp314-win_amd64.whl", hash = "sha256:bd77945f38866a448e73b0b7637366afa814d4617790ecd88a18ca74377e6c02", size = 202187, upload-time = "2026-04-10T14:27:38Z" }, - { url = "https://files.pythonhosted.org/packages/aa/1f/6e39ac0b4cdfa23e606af5b245df5f9adaa76f35e0c5096790da430ca506/jiter-0.14.0-cp314-cp314-win_arm64.whl", hash = "sha256:f2d4c61da0821ee42e0cdf5489da60a6d074306313a377c2b35af464955a3611", size = 192257, upload-time = "2026-04-10T14:27:39.504Z" }, - { url = "https://files.pythonhosted.org/packages/05/57/7dbc0ffbbb5176a27e3518716608aa464aee2e2887dc938f0b900a120449/jiter-0.14.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1bf7ff85517dd2f20a5750081d2b75083c1b269cf75afc7511bdf1f9548beb3b", size = 323441, upload-time = "2026-04-10T14:27:41.039Z" }, - { url = "https://files.pythonhosted.org/packages/83/6e/7b3314398d8983f06b557aa21b670511ec72d3b79a68ee5e4d9bff972286/jiter-0.14.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c8ef8791c3e78d6c6b157c6d360fbb5c715bebb8113bc6a9303c5caff012754a", size = 348109, upload-time = "2026-04-10T14:27:42.552Z" }, - { url = "https://files.pythonhosted.org/packages/ae/4f/8dc674bcd7db6dba566de73c08c763c337058baff1dbeb34567045b27cdc/jiter-0.14.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:e74663b8b10da1fe0f4e4703fd7980d24ad17174b6bb35d8498d6e3ebce2ae6a", size = 368328, upload-time = "2026-04-10T14:27:44.574Z" }, - { url = "https://files.pythonhosted.org/packages/3b/5f/188e09a1f20906f98bbdec44ed820e19f4e8eb8aff88b9d1a5a497587ff3/jiter-0.14.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1aca29ba52913f78362ec9c2da62f22cdc4c3083313403f90c15460979b84d9b", size = 463301, upload-time = "2026-04-10T14:27:46.717Z" }, - { url = "https://files.pythonhosted.org/packages/ac/f0/19046ef965ed8f349e8554775bb12ff4352f443fbe12b95d31f575891256/jiter-0.14.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8b39b7d87a952b79949af5fef44d2544e58c21a28da7f1bae3ef166455c61746", size = 378891, upload-time = "2026-04-10T14:27:48.32Z" }, - { url = "https://files.pythonhosted.org/packages/c4/c3/da43bd8431ee175695777ee78cf0e93eacbb47393ff493f18c45231b427d/jiter-0.14.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78d918a68b26e9fab068c2b5453577ef04943ab2807b9a6275df2a812599a310", size = 360749, upload-time = "2026-04-10T14:27:49.88Z" }, - { url = "https://files.pythonhosted.org/packages/72/26/e054771be889707c6161dbdec9c23d33a9ec70945395d70f07cfea1e9a6f/jiter-0.14.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:b08997c35aee1201c1a5361466a8fb9162d03ae7bf6568df70b6c859f1e654a4", size = 358526, upload-time = "2026-04-10T14:27:51.504Z" }, - { url = "https://files.pythonhosted.org/packages/c3/0f/7bea65ea2a6d91f2bf989ff11a18136644392bf2b0497a1fa50934c30a9c/jiter-0.14.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:260bf7ca20704d58d41f669e5e9fe7fe2fa72901a6b324e79056f5d52e9c9be2", size = 393926, upload-time = "2026-04-10T14:27:53.368Z" }, - { url = "https://files.pythonhosted.org/packages/3c/a1/b1ff7d70deef61ac0b7c6c2f12d2ace950cdeecb4fdc94500a0926802857/jiter-0.14.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:37826e3df29e60f30a382f9294348d0238ef127f4b5d7f5f8da78b5b9e050560", size = 521052, upload-time = "2026-04-10T14:27:55.058Z" }, - { url = "https://files.pythonhosted.org/packages/0b/7b/3b0649983cbaf15eda26a414b5b1982e910c67bd6f7b1b490f3cfc76896a/jiter-0.14.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:645be49c46f2900937ba0eaf871ad5183c96858c0af74b6becc7f4e367e36e06", size = 553716, upload-time = "2026-04-10T14:27:57.269Z" }, - { url = "https://files.pythonhosted.org/packages/97/f8/33d78c83bd93ae0c0af05293a6660f88a1977caef39a6d72a84afab94ce0/jiter-0.14.0-cp314-cp314t-win32.whl", hash = "sha256:2f7877ed45118de283786178eceaf877110abacd04fde31efff3940ae9672674", size = 207957, upload-time = "2026-04-10T14:27:59.285Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ac/2b760516c03e2227826d1f7025d89bf6bf6357a28fe75c2a2800873c50bf/jiter-0.14.0-cp314-cp314t-win_amd64.whl", hash = "sha256:14c0cb10337c49f5eafe8e7364daca5e29a020ea03580b8f8e6c597fed4e1588", size = 204690, upload-time = "2026-04-10T14:28:00.962Z" }, - { url = "https://files.pythonhosted.org/packages/dc/2e/a44c20c58aeed0355f2d326969a181696aeb551a25195f47563908a815be/jiter-0.14.0-cp314-cp314t-win_arm64.whl", hash = "sha256:5419d4aa2024961da9fe12a9cfe7484996735dca99e8e090b5c88595ef1951ff", size = 191338, upload-time = "2026-04-10T14:28:02.853Z" }, +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/1f/10936e16d8860c70698a1aa939a46aa0224813b782bce4e000e637da0b2d/jiter-0.16.0.tar.gz", hash = "sha256:7b24c3492c5f4f84a37946ad9cf504910cf6a782d6a4e0689b6673c5894b4a1c", size = 176431, upload-time = "2026-06-29T13:05:13.657Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/91/c0/555fc60473d30d66894ba825e63615e3be7524fac23858356afa7a38906c/jiter-0.16.0-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:41977aa5654023948c2dae2a81cbf9c43343954bef1cd59a154dd15a4d84c195", size = 306203, upload-time = "2026-06-29T13:03:36.243Z" }, + { url = "https://files.pythonhosted.org/packages/d0/2b/c3eaf16f5d7c9bad66ea32f40a95bd169b29a91217fcc7f081375157e99c/jiter-0.16.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d28bb3c26762358dadf3e5bf0bccd29ae987d65e6988d2e6f49829c76b003c09", size = 306489, upload-time = "2026-06-29T13:03:37.846Z" }, + { url = "https://files.pythonhosted.org/packages/96/3f/02fdfc6705cad96127d883af5c34e4867f554f29ec7705ec1a46156400a9/jiter-0.16.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0542a7189c26920778658fc8fcf2af8bae05bae9924577f71804acef37996536", size = 335453, upload-time = "2026-06-29T13:03:39.221Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a6/e4bda5920d4b0d7c5dfb7174ce4a6b2e4d3e11c9162c452ef0eab4cdbdbd/jiter-0.16.0-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8fb8de1e23a0cb2a7f53c335049c7b72b6db41aa6227cdcc0972a1de5cb39450", size = 361625, upload-time = "2026-06-29T13:03:40.597Z" }, + { url = "https://files.pythonhosted.org/packages/b7/97/4e6b59b2c6e55cbb3e183595f81ad65dcfb21c915fee5e19e335df21bc55/jiter-0.16.0-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b72d0b2990ca754a9102779ac98d8597b7cb31678958562214a007f909eab78e", size = 456958, upload-time = "2026-06-29T13:03:42.074Z" }, + { url = "https://files.pythonhosted.org/packages/15/e0/97e9557686d2f94f4b93786eccb7eed28e9228ad132ea8237f44727314a7/jiter-0.16.0-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d5f91b1c27fc22a57993d5a5cb8a627cb8ed4b10502716fac1ffbfe1d19d84e8", size = 372017, upload-time = "2026-06-29T13:03:43.658Z" }, + { url = "https://files.pythonhosted.org/packages/0f/94/db768b6938e0df35c86beeba3dfbbb025c9ee5c19e1aa271f2396e50864d/jiter-0.16.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c682bea068a90b764577bdb78a60a4c1d1606daf9cd4c893832a37c7cc9d9026", size = 343320, upload-time = "2026-06-29T13:03:45.226Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d6/5a59d938244a30735fe62d9433fd325f9021ea29d89780ea4596ea93bc89/jiter-0.16.0-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:8d031aabecc4f1b6276adfb42e3aabb77c89d468bf616600e8d3a11328929053", size = 350520, upload-time = "2026-06-29T13:03:46.671Z" }, + { url = "https://files.pythonhosted.org/packages/67/f8/c4a857f49c9af125f6bbcac7e3eee7f7978ed89682833062e2dbf62576b1/jiter-0.16.0-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:eab2cd170150e70153de16896a1774e3a1dca80154c56b54d7a812c479a7165e", size = 387550, upload-time = "2026-06-29T13:03:48.361Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/5fbc2f7d6b67b754caa61a993a2e626e815dec47ffc2f9e35f01adfebec7/jiter-0.16.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:6edb63a46e65a82c26800a868e49b2cac30dd5a4218b88d74bc2c848c8ad60bb", size = 515424, upload-time = "2026-06-29T13:03:49.881Z" }, + { url = "https://files.pythonhosted.org/packages/ed/54/284f0164b64a5fed915fea6ba7e9ba9b3d8d37c67d59cf2e3bb99d45cdfe/jiter-0.16.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:659039cc50b5addcc35fcc87ae2c1833b7c0a8e5326ef631a75e4478447bcf84", size = 546981, upload-time = "2026-06-29T13:03:51.363Z" }, + { url = "https://files.pythonhosted.org/packages/13/c5/2a467585a576594384e1d2c43e1224deaafc085f24e243529cf98beef8e1/jiter-0.16.0-cp313-cp313-win32.whl", hash = "sha256:c9c53be232c2e206ef9cdbad81a48bfa74c3d3f08bcf8124630a8a748aad993e", size = 202853, upload-time = "2026-06-29T13:03:53.015Z" }, + { url = "https://files.pythonhosted.org/packages/88/6a/de61d04b9eec69c71719968d2f716532a3bc121170c44a39e14979c6be81/jiter-0.16.0-cp313-cp313-win_amd64.whl", hash = "sha256:baad945ed47f163ad833314f8e3288c396118934f94e7bbb9e243ce4b341a4fd", size = 196160, upload-time = "2026-06-29T13:03:54.447Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/b390ed59bafb3f31d008d1218578f10327714484b334439947f7e5b11e7f/jiter-0.16.0-cp313-cp313-win_arm64.whl", hash = "sha256:3c1fd2dbe1b0af19e987f03fe66c5f5bd105a2229c1aff4ab14890b24f41d21a", size = 189862, upload-time = "2026-06-29T13:03:55.754Z" }, + { url = "https://files.pythonhosted.org/packages/a7/89/bc4f1b57d5da938fd344a466396541e586d161320d70bffd929aaafcd8f4/jiter-0.16.0-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:b2c61484666ad42726029af0c00ef4541f0f3b5cdc550221f56c2343208018ee", size = 308239, upload-time = "2026-06-29T13:03:57.205Z" }, + { url = "https://files.pythonhosted.org/packages/65/7a/c415453e5213001bf3b411ff65dec3d303b0e76a4a2cfea9768cd4960994/jiter-0.16.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:63efadc657488f45db1c676d81e704cac2abf3fdb892def1faea61db053127e2", size = 308928, upload-time = "2026-06-29T13:03:58.643Z" }, + { url = "https://files.pythonhosted.org/packages/11/fc/1f4fb7ebf9a724c7741994f4aae18fba1e2f3133df14521a79194952c34a/jiter-0.16.0-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf0d73f50e7b6935677854f6e8e31d499ca7064dd24734f703e060f5b237d883", size = 336998, upload-time = "2026-06-29T13:04:00.071Z" }, + { url = "https://files.pythonhosted.org/packages/a0/8d/72cadaac05ccfa7cc3a0a2232862e6c72443ca40cf300ba8b57f9f18b69b/jiter-0.16.0-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:bf3ea07d9bc8e7d03a9fbc051295462e6dbc295b894fd72457c3136e3e43d898", size = 362112, upload-time = "2026-06-29T13:04:01.52Z" }, + { url = "https://files.pythonhosted.org/packages/58/4a/c4b0d5f651fda90a24ffce9f8d56cde462a2e09d31ae3de3c68cef34c04e/jiter-0.16.0-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:26798522707abb47d767db536e4148ceac1b14446bf028ee85e579a2e043cfe5", size = 459807, upload-time = "2026-06-29T13:04:03.214Z" }, + { url = "https://files.pythonhosted.org/packages/80/58/ef77879ea9aa56b50824edc5a445e226422c7a8d211f3fd2a56bcb9493cf/jiter-0.16.0-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bc837c1b9631be10abfe0191537fe8009838204cec7e44827401ace390ddb567", size = 373181, upload-time = "2026-06-29T13:04:04.629Z" }, + { url = "https://files.pythonhosted.org/packages/49/2e/ffbc3f254e4d8a66da3062c624a7df4b7c2b2cf9e1fe43cf394b3e104041/jiter-0.16.0-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49060fd70737fad59d33ba9dcc0d83247dc9e77187de26053a19c16c9f32bd69", size = 344927, upload-time = "2026-06-29T13:04:06.067Z" }, + { url = "https://files.pythonhosted.org/packages/9a/f6/0be5dc6d64a89f80aa8fec984f94dedb2973e251edcae55841d60786d578/jiter-0.16.0-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:adbb8edeadd431bc4477879d5d371ece7cb1334486584e0f252656dd7ffada29", size = 352754, upload-time = "2026-06-29T13:04:07.477Z" }, + { url = "https://files.pythonhosted.org/packages/da/6e/7d31243b3b91cd261dd19e9d3557fc3251a80883d3d8049c86174e7ab7af/jiter-0.16.0-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:31aaee5b80f672c1dc21272bcfb9cbdcfc1ea04ff50f00ed5af500b80c44fa93", size = 390553, upload-time = "2026-06-29T13:04:08.92Z" }, + { url = "https://files.pythonhosted.org/packages/25/33/51ae371fde3c88897520f62b4d5f8b27ad7103e2bb10812ff52195609853/jiter-0.16.0-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:6722bcef4ffc86c835574b1b2fac6b33b9fb4a889c781e67950e891591f3c55a", size = 516900, upload-time = "2026-06-29T13:04:10.407Z" }, + { url = "https://files.pythonhosted.org/packages/a0/45/6449b3d123ea439ba79507c657288f461d55049e7bcbdc2cf8eb8210f491/jiter-0.16.0-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:5ab4f50ff971b611d656554ea10b75f80097392c827bc32923c6eeb6386c8b00", size = 548754, upload-time = "2026-06-29T13:04:12.046Z" }, + { url = "https://files.pythonhosted.org/packages/9b/e7/fd2fb11ae3e2649333da3aa170d04d7b3000bbdc3b270f6513382fdf4e04/jiter-0.16.0-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:710cc51d4ebdcd3c1f70b232c1db1ea1344a075770422bbd4bede5708335acbe", size = 122381, upload-time = "2026-06-29T13:04:13.413Z" }, + { url = "https://files.pythonhosted.org/packages/26/80/f0b147a62c315a164ed2168908286ca302310824c218d3aae52b06c0c9a9/jiter-0.16.0-cp314-cp314-win32.whl", hash = "sha256:57b37fc887a32d44798e4d8ebfa7c9683ff3da1d5bf38f08d1bb3573ccb39106", size = 204578, upload-time = "2026-06-29T13:04:14.813Z" }, + { url = "https://files.pythonhosted.org/packages/5e/e6/4758a14304b4523a6f5adb2419340086aa3593bd4327c2b25b5948a90548/jiter-0.16.0-cp314-cp314-win_amd64.whl", hash = "sha256:cbd18dd5e2df96b580487b5745adf57ef64ad89ba2d9662fc3c19386acce7db8", size = 198154, upload-time = "2026-06-29T13:04:16.272Z" }, + { url = "https://files.pythonhosted.org/packages/26/be/41fa54a2e7ea41d6c99f1dc5b1f0fd4cb474680304b5d268dd518e81da3a/jiter-0.16.0-cp314-cp314-win_arm64.whl", hash = "sha256:a32d2027a9fa67f109ff245a3252ece3ccc32cc56703e1deab6cc846a59e0585", size = 191458, upload-time = "2026-06-29T13:04:17.707Z" }, + { url = "https://files.pythonhosted.org/packages/81/6b/59127338b86d9fe4d99418f5a15118bea778103ee0fe9d9dd7e0af174e95/jiter-0.16.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:2577196f4474ef3fc4779a088a23b0897bbf86f9ea3679c372d45b8383b43207", size = 316739, upload-time = "2026-06-29T13:04:19.663Z" }, + { url = "https://files.pythonhosted.org/packages/2d/95/49461034d5388196d3dabf98748935f017b7785d8f3f5349f834bcc4ed0d/jiter-0.16.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:616e89e008a93c01104161c75b4988e58716b01d62307ebfe161e52a56d2a818", size = 340911, upload-time = "2026-06-29T13:04:21.257Z" }, + { url = "https://files.pythonhosted.org/packages/cd/97/a4369f2fb82cb3dda13b98622f31249b2e014b223fe64ee534413ad72294/jiter-0.16.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:0e2e9efbe042210df657bade597f66d6d75723e3d8f45a12ea6d8167ff8bbce3", size = 361747, upload-time = "2026-06-29T13:04:22.677Z" }, + { url = "https://files.pythonhosted.org/packages/28/51/49b6ed456261646e1906016a6760367a28aacd3c24805e4e5fe64116c1db/jiter-0.16.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3f4d9e473a5ce7d27fef8b848df4dc16e283893d3f53b4a585e72c9595f3c284", size = 460225, upload-time = "2026-06-29T13:04:24.441Z" }, + { url = "https://files.pythonhosted.org/packages/33/b5/5689aff4f66c5b60be63106e591dbfcba2190df97d2c9c7cf052361ddb98/jiter-0.16.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d30a4a1c87713060c8d1cc59a7b6c8fb6b8ef0a6900368014c76c87922a2929", size = 373169, upload-time = "2026-06-29T13:04:25.884Z" }, + { url = "https://files.pythonhosted.org/packages/a2/96/3ae1b85ee0d6d6cab254fb7f8da018272b932bbf2d69b07e98aa2a96c746/jiter-0.16.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bae96332410f866e5900d809298b1ed82735932986c672495f9701daacd80620", size = 350332, upload-time = "2026-06-29T13:04:27.302Z" }, + { url = "https://files.pythonhosted.org/packages/15/32/c99d7bafd78986556c95bf60ce84c6cc98786eac56066c12d7f828bb6747/jiter-0.16.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:da3d7ec75dc83bb18bca888b5edfae0656a26849056c59e05a7728badd17e7af", size = 353377, upload-time = "2026-06-29T13:04:28.731Z" }, + { url = "https://files.pythonhosted.org/packages/0e/4b/f99a8e571287c3dec766bcc18528bbe8e8fb5365522ab5e6d64c93e87066/jiter-0.16.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:ee6162b77d49a9939229df666dfa8af3e656b6701b54c4c84966d740e189264e", size = 387746, upload-time = "2026-06-29T13:04:30.319Z" }, + { url = "https://files.pythonhosted.org/packages/75/69/c78a5b3f71040e34eb5917df26fb7ae9a2174cad1ccbf277512507c53a6e/jiter-0.16.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:63ffdbdae7d4499f4cda14eadc12ddcabef0fc0c081191bdc2247489cb698077", size = 517292, upload-time = "2026-06-29T13:04:31.709Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f7/095b38eda4c70d03651c403f29a5590f16d12ddc5d544aac9f9cddf72277/jiter-0.16.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:a111256a7193bea0759267b10385e5870949c239ed7b6ddbaaf57573edb38734", size = 549259, upload-time = "2026-06-29T13:04:33.721Z" }, + { url = "https://files.pythonhosted.org/packages/2e/c5/6a0207d90e5f656d95af98ebd0934f382d37674416f215aeda2ff8063e51/jiter-0.16.0-cp314-cp314t-win32.whl", hash = "sha256:de5ba8763e56b793561f43bed197c9ea55776daa5e9a6b91eed68a909bc9cdbf", size = 206523, upload-time = "2026-06-29T13:04:35.068Z" }, + { url = "https://files.pythonhosted.org/packages/a5/31/c757d5f30a8980fd945ce7b98be10be9e4ff59c7c42f5fd86804c2e87db8/jiter-0.16.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b8a3f9a6008048fe9def7bf465180564a6e458047d2ce499149cfbe73c3ae9db", size = 200366, upload-time = "2026-06-29T13:04:36.61Z" }, + { url = "https://files.pythonhosted.org/packages/7c/a2/d88de6d313d734a544a7901353ad5db67cb38dcfcd91713b7979dafc345d/jiter-0.16.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0fa25b09b13075c46f5bc174f2690525a925a4fc2f7c82969a2bbabff22386ce", size = 190516, upload-time = "2026-06-29T13:04:38.004Z" }, ] [[package]] @@ -1625,21 +1612,19 @@ wheels = [ [[package]] name = "openai" -version = "3.0.0" +version = "3.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio" }, - { name = "distro" }, { name = "httpx2" }, { name = "jiter" }, { name = "pydantic" }, { name = "sniffio" }, - { name = "tqdm" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/54/8c/2f500e8be09d1ae98c530467962535198b02cd4550cd418bbbaedc8b2910/openai-3.0.0.tar.gz", hash = "sha256:ffd00ef1678d70957e1f1ed98d5bfcf1d661f41ea4482f22e7d0144a66435a49", size = 1123740, upload-time = "2026-08-12T01:55:50.849Z" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/9c/ba0c292b4032ede74c249ca314ad64eb1bb5a03a843f6e01facb02f80cd8/openai-3.3.1.tar.gz", hash = "sha256:6f22807de1a976c932cecda620e8172a8c3fdbaeed29c7f21564e0c2410edf56", size = 1282113, upload-time = "2026-08-19T16:31:35.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/0d/9850e7eddb5e66da4439ed503e78e09ad1fd0195e6df51e4236c75763581/openai-3.0.0-py3-none-any.whl", hash = "sha256:8d32ac3a6647a66910d6cb8a64f0fa5a6c823604b6e82db83d9d055c6709bd51", size = 1665775, upload-time = "2026-08-12T01:55:48.678Z" }, + { url = "https://files.pythonhosted.org/packages/6a/db/2b7a1b3de659bb82aef979116c74e809982b13e42c057759767552b5155f/openai-3.3.1-py3-none-any.whl", hash = "sha256:9652df7fdf8ee6f5bd58e0a12f2b1d414a18e0f06bb7a9a57c8643a5f5469bd3", size = 1690337, upload-time = "2026-08-19T16:31:32.812Z" }, ] [[package]] @@ -2164,11 +2149,11 @@ wheels = [ [[package]] name = "pygments" -version = "2.20.0" +version = "2.21.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/c3/b2/bc9c9196916376152d655522fdcebac55e66de6603a76a02bca1b6414f6c/pygments-2.20.0.tar.gz", hash = "sha256:6757cd03768053ff99f3039c1a36d6c0aa0b263438fcab17520b30a303a82b5f", size = 4955991, upload-time = "2026-03-29T13:29:33.898Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, ] [[package]] @@ -2287,11 +2272,11 @@ wheels = [ [[package]] name = "python-dotenv" -version = "1.2.2" +version = "1.2.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/82/ed/0301aeeac3e5353ef3d94b6ec08bbcabd04a72018415dcb29e588514bba8/python_dotenv-1.2.2.tar.gz", hash = "sha256:2c371a91fbd7ba082c2c1dc1f8bf89ca22564a087c2c287cd9b662adde799cf3", size = 50135, upload-time = "2026-03-01T16:00:26.196Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, + { url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" }, ] [[package]] @@ -2457,27 +2442,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.16.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/61/b3/3213589383f8f1b3938781bd1278713f6d18621a14992b3e81fefb8a5ef9/ruff-0.16.3.tar.gz", hash = "sha256:e76d33a347661a84b5be6d043d0347fdc745dfdcf825a8f4fed64b5e26eebdf2", size = 4891904, upload-time = "2026-08-13T15:17:13.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/96/493770daebd68c0a67f1549fdf519f53be51fc435186c0585bcc272fd76c/ruff-0.16.3-py3-none-linux_armv6l.whl", hash = "sha256:0c5710e247a58a4521e66e124ba9a74655b414f61ba3a2e9e3811e11098f48f7", size = 10902799, upload-time = "2026-08-13T15:16:27.382Z" }, - { url = "https://files.pythonhosted.org/packages/5e/e6/2becf3942fddc29a29b8df47691d456fb1085391a694f74d84513251418c/ruff-0.16.3-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:fe155130631a2471fd2e14a7a664a4dfbd7194b8229c3d7b2a40b21178639081", size = 11135539, upload-time = "2026-08-13T15:16:30.87Z" }, - { url = "https://files.pythonhosted.org/packages/3e/1e/4b8b72f0d006dbf19326aa99f9ca0ee2ff374187c4d301cf529a51aa06fe/ruff-0.16.3-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e2ed719e14aa64d895c2ee922594a90a43c861a93f0575a95ff8c47cdbd13eb9", size = 10475095, upload-time = "2026-08-13T15:16:33.259Z" }, - { url = "https://files.pythonhosted.org/packages/92/32/2201fa49ba1f6c101ee321e83f051ac7a4b8d07b0ef6b4d3f2772b302275/ruff-0.16.3-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9e0b1da805eb043654645d74d5de1e5ce2edc686e40790d2b86f56d71cc06a84", size = 10668771, upload-time = "2026-08-13T15:16:35.65Z" }, - { url = "https://files.pythonhosted.org/packages/c3/66/4afc5c8363bd04d45effce1b7c8713ca037d7a6740b7451a2403a6e3a972/ruff-0.16.3-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:a37bdea0bbe21780f590bf437d6412c8c4e1b6cd010f91a65c2c40c5e5f5f870", size = 10699568, upload-time = "2026-08-13T15:16:38.195Z" }, - { url = "https://files.pythonhosted.org/packages/53/fd/c67d246bf36bf1698551c56de39e95cd07f70e64433e0098e6267d77061b/ruff-0.16.3-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:09571e6d1288ed9be475207a3ac04ada404f1cd898104be0f6ab8d7df438575b", size = 11499365, upload-time = "2026-08-13T15:16:40.623Z" }, - { url = "https://files.pythonhosted.org/packages/67/0b/00ecbceb99a263af7b12f6f05ac3c92bc47b905e91adc3f207a836e3bc01/ruff-0.16.3-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2c18c5a101eb540010638cc1ff3c84944d3adb3df62b8d98ca8f22ba484d3413", size = 12311728, upload-time = "2026-08-13T15:16:43.564Z" }, - { url = "https://files.pythonhosted.org/packages/54/b2/b7b3bb54f4d3f7db504e476ad4ab8de530dceebe2c061384b2757ee419e8/ruff-0.16.3-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8457c44f15033c85ddbb77b15d451df9e24e4bd03b628396dd3610cedc3b8f82", size = 11699896, upload-time = "2026-08-13T15:16:46.209Z" }, - { url = "https://files.pythonhosted.org/packages/c7/30/4c468429ac195addc5ee1b717b6ab1b66632786737ca3b2ed3443fb0c26a/ruff-0.16.3-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:294b95c4ae0cda9388525c2047778aa758d6b8d4bb876fd4e9eaa3ebc92343eb", size = 11058736, upload-time = "2026-08-13T15:16:48.823Z" }, - { url = "https://files.pythonhosted.org/packages/43/67/7a113cdaddf24b64d7f75b1242a99d04c82fcef4f6921fdbb832beaffb5f/ruff-0.16.3-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:3d0c7c40c87c2a820509c31ba007968da6e1306468c067b2d82fbfdbcd0e8474", size = 11586911, upload-time = "2026-08-13T15:16:51.913Z" }, - { url = "https://files.pythonhosted.org/packages/f1/c1/2e66f24c0f3ead25a5e660111778685e505e5da353c82802bf49f0cbe7b9/ruff-0.16.3-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:9f738c0fdfa8eed0b2ce7fb27ee7258208a92a68d7949e62aa15164bc7b389da", size = 10954265, upload-time = "2026-08-13T15:16:54.763Z" }, - { url = "https://files.pythonhosted.org/packages/c2/ba/4cee23bf52cba9a058d3726de623624daf50ef9638868edd86f4126157f6/ruff-0.16.3-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:fb785f0be25abe69d320415cd4f833b59e17ba7613d9ba6a958023b6bceb0a50", size = 10709886, upload-time = "2026-08-13T15:16:57.339Z" }, - { url = "https://files.pythonhosted.org/packages/82/df/7da7194fa5d9dc0a285f7e6fa5a4722e7c63faac0b45b614ded9314363a1/ruff-0.16.3-py3-none-musllinux_1_2_i686.whl", hash = "sha256:c5536e3acfbf9563085aa2be7b13c629c3077e902afc5b941ac44024dbb9f506", size = 11210392, upload-time = "2026-08-13T15:17:00.171Z" }, - { url = "https://files.pythonhosted.org/packages/35/85/7795f6e817af050e7517bf3e7aa9b061cce70ef33d280aad902c956c1ecf/ruff-0.16.3-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:a2d85c02f9b8e165d85e6779184d38c4132de12603dab59c51c28e22584f9e4d", size = 11626910, upload-time = "2026-08-13T15:17:03.299Z" }, - { url = "https://files.pythonhosted.org/packages/78/9b/475b927cf27a5cbbda3c7bafb69ed6ff77e1d7923d5d85f17c2749d7ae32/ruff-0.16.3-py3-none-win32.whl", hash = "sha256:388cdf2166642bd9b13d52b5932d3170f34f8abed7e8d9a855f1d84b83645a0a", size = 10931415, upload-time = "2026-08-13T15:17:05.726Z" }, - { url = "https://files.pythonhosted.org/packages/b2/99/e2a2bfc4fbf0a1e8a916bc9ebe6fe6c58cc34c28e0ffc6ce281d572d1c2e/ruff-0.16.3-py3-none-win_amd64.whl", hash = "sha256:e80a7d69ca2a6d1c4d352ec91458cdca6e56c83cdbcabd93e4abe1e53591d948", size = 11445993, upload-time = "2026-08-13T15:17:08.353Z" }, - { url = "https://files.pythonhosted.org/packages/69/3e/4132e539aed78c148854d4997a2685b0ed4dc4e87110b59ce528564e184e/ruff-0.16.3-py3-none-win_arm64.whl", hash = "sha256:b8ca152da82c1acc1fa8d5874b15951935f0eef46f10e6954c83859011b6178a", size = 11399302, upload-time = "2026-08-13T15:17:10.908Z" }, +version = "0.16.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/8f/d8074b1f25e003164087a8bfe79a0f1a3945135764dbb6aaab04103dcaf9/ruff-0.16.4.tar.gz", hash = "sha256:13171aa9d9af2240ee3504e639de73122c67e74036de5ba2e1d01422cd17e3dc", size = 4899731, upload-time = "2026-08-20T17:43:59.196Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/80/779895ef584e089d22f2c6df0d0e99a65ec2df0805f1fffd439415b8c1f0/ruff-0.16.4-py3-none-linux_armv6l.whl", hash = "sha256:df4075f71ddac40b9934af60c3ec8a53047dd5a5fdc43224e6e4e8e9a27cb6f7", size = 10006909, upload-time = "2026-08-20T17:43:16.888Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e6/f553199b5e8927a05cb5c422d921fd0656b29ab976e91c44802107c6b0da/ruff-0.16.4-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0c95538517af68004306b0fb3214ff2f2af67a65092aee77cd9eb86db6656604", size = 10240201, upload-time = "2026-08-20T17:43:19.337Z" }, + { url = "https://files.pythonhosted.org/packages/1c/70/4a6dc4bb34da4dee35e30f09bbd1bfbdd26f33b62fb9b8df31f08a199cd2/ruff-0.16.4-py3-none-macosx_11_0_arm64.whl", hash = "sha256:963f83df8e69e575b64d67dd447ebbc917db41a14bf38d4593a4183e7aaa8255", size = 9835122, upload-time = "2026-08-20T17:43:21.708Z" }, + { url = "https://files.pythonhosted.org/packages/24/12/c6e22d686372c15bcb7af99831f1a1be96df696491babf4f24e4f942c527/ruff-0.16.4-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:32a5057c7ff3f6e6480a48fccfb3a412a690f48a3d03ac5cf08177d6c2da3ade", size = 9977162, upload-time = "2026-08-20T17:43:24.236Z" }, + { url = "https://files.pythonhosted.org/packages/46/49/72b10ec912f5ab5854992eaf7aa7cd36729b6937d9dc4e0fb41b3bf428ec/ruff-0.16.4-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:b3dce8d9b0c57c265b91885a66a567d8ea1372e8eb4e250fa8e5e3f579e99cff", size = 9829789, upload-time = "2026-08-20T17:43:26.966Z" }, + { url = "https://files.pythonhosted.org/packages/fa/80/0f30e32e7f6ee26edc39075502db9d368d788a44a79b55f763eb4ab03796/ruff-0.16.4-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7dc651db49283c69f8e72c834eec4fe5573e4c646856aebece0ce385dceb2a80", size = 10527949, upload-time = "2026-08-20T17:43:29.384Z" }, + { url = "https://files.pythonhosted.org/packages/52/3d/86e8ad3542169e56cac3859a343afdb9df2ad54d35a59ce1e67baee83421/ruff-0.16.4-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3817b87dbcabc92f13b05019257c5b89b5b4d51b5fb20f56fb5235ceb723cd07", size = 11333695, upload-time = "2026-08-20T17:43:31.872Z" }, + { url = "https://files.pythonhosted.org/packages/d0/16/481c29b380c20a0054a8261066665e1b3488e23636c49d0a43e75975b9bb/ruff-0.16.4-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e9fce1499134b2c8c68e5166f95705a5812062bb93aacc5f9873bb1a27084bc7", size = 10727741, upload-time = "2026-08-20T17:43:34.596Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/56bc0b8cf45b54b28b3a5e6381c8945d51b5b18adf659454c32295209a31/ruff-0.16.4-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2d812e482f5a7e02eee26cd73d2a37ebbdf47d795ea63ba1b89110ae93e9fb3", size = 10286522, upload-time = "2026-08-20T17:43:37.288Z" }, + { url = "https://files.pythonhosted.org/packages/e8/8b/b345b4fb110f2fbe2bd31eabd271e5e8b3b7e4ee6c0e02f2dc6be78db000/ruff-0.16.4-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:6baaf984aa7976edf93d3b627fe2d1d22ee94bbca05fa6f90fc76d73924e3454", size = 10584182, upload-time = "2026-08-20T17:43:39.984Z" }, + { url = "https://files.pythonhosted.org/packages/29/e5/827b34041c35f58774a9681a4213994c164fc987800f4dddabcf451da0bf/ruff-0.16.4-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:bdfcf0b28662eb890372d50f92c283bb94e67e7635ed93c7fd533970acff7b2b", size = 10134195, upload-time = "2026-08-20T17:43:42.351Z" }, + { url = "https://files.pythonhosted.org/packages/0f/10/d0bffcdd6729b87afc82ba0ef377173356a7dc8e972f5179968cf2fdf98c/ruff-0.16.4-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b66b02cb9b04f537643cadf5768e5f98dc461890d530cb67113d71c8c76e605d", size = 9825821, upload-time = "2026-08-20T17:43:44.532Z" }, + { url = "https://files.pythonhosted.org/packages/f5/32/0db2a863b796ca62d83e92a07a3ccf00921b14db02059347576a2fda3d4b/ruff-0.16.4-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8528bf9a4b291a60bf02ea453511e8ce6215bd2b982ee80405b66b008b6c30a0", size = 10267658, upload-time = "2026-08-20T17:43:46.989Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a0/fbdeb59e48c6261f523e56c8f12e9c08fbe693786595cc7e3959207a9232/ruff-0.16.4-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:fbd85d2875fdd67e833213a651f613bbf25303abf6aa822a5121f4531195678d", size = 10697071, upload-time = "2026-08-20T17:43:49.891Z" }, + { url = "https://files.pythonhosted.org/packages/aa/28/0c6dd865859c6d17bc8ccc34cb72b0e02d6c7eb25e8a1e22b5bea681e2c0/ruff-0.16.4-py3-none-win32.whl", hash = "sha256:312769988007aaeb8e189b443ccdd03c0e6374489e053467be6d96518ebff76e", size = 10021687, upload-time = "2026-08-20T17:43:52.281Z" }, + { url = "https://files.pythonhosted.org/packages/a3/03/e724450f621698117f9aa6dd241c94d0274ae96781378dc86745ae29f0e7/ruff-0.16.4-py3-none-win_amd64.whl", hash = "sha256:05d9d27a18c4bcbefada602480ec9e01e0bc949d432e0ced5df77edac195919c", size = 10567657, upload-time = "2026-08-20T17:43:54.78Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fe/da8b9e1347696bb22120b77280ec5ce25d500ca5cb39d5ad6e5c18de19c1/ruff-0.16.4-py3-none-win_arm64.whl", hash = "sha256:a3a61621c9b6f6a89573e938a080e648f1695baa3f58570a3a707bc51ff65a21", size = 10451579, upload-time = "2026-08-20T17:43:57.135Z" }, ] [[package]] @@ -2578,18 +2563,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, ] -[[package]] -name = "tqdm" -version = "4.67.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/09/a9/6ba95a270c6f1fbcd8dac228323f2777d886cb206987444e4bce66338dd4/tqdm-4.67.3.tar.gz", hash = "sha256:7d825f03f89244ef73f1d4ce193cb1774a8179fd96f31d7e1dcde62092b960bb", size = 169598, upload-time = "2026-02-03T17:35:53.048Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/16/e1/3079a9ff9b8e11b846c6ac5c8b5bfb7ff225eee721825310c91b3b50304f/tqdm-4.67.3-py3-none-any.whl", hash = "sha256:ee1e4c0e59148062281c49d80b25b67771a127c85fc9676d3be5f243206826bf", size = 78374, upload-time = "2026-02-03T17:35:50.982Z" }, -] - [[package]] name = "truststore" version = "0.10.4" @@ -2601,27 +2574,27 @@ wheels = [ [[package]] name = "ty" -version = "0.0.72" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d5/df/656e684bafb13c1d146e7d5b5f3e7978ca177232acc84998ff36427e9462/ty-0.0.72.tar.gz", hash = "sha256:ec2b8066b618df18cab4cb8e992f8da45d360332acb23fa34df7fa29cd1b9d3a", size = 6654939, upload-time = "2026-08-14T21:35:42.612Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e2/3b/f51461239a4e66565d4b362f97a3b55fe7fdba2e944068341f87c62f6743/ty-0.0.72-py3-none-linux_armv6l.whl", hash = "sha256:fda86db153ffd85ee52000cf175d6a3f1c0223772cf7c5b6f726200bf92c7b44", size = 12621989, upload-time = "2026-08-14T21:35:01.676Z" }, - { url = "https://files.pythonhosted.org/packages/ca/fb/79ddf683affc679ca856f3510b5640ec3a88a842ba5f654f5d4bc78f1786/ty-0.0.72-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:ceb944c612529b9023acfdc9cf4c0dcbb722549f9d17d46baecd1141baf01d7f", size = 12233910, upload-time = "2026-08-14T21:35:04.334Z" }, - { url = "https://files.pythonhosted.org/packages/5d/45/10562a0d84802158db8fa4ec46de54aa9fdcecdeeaabbfe3639ae7042b66/ty-0.0.72-py3-none-macosx_11_0_arm64.whl", hash = "sha256:108d76218333d6c092e5f1cebf8e9b06f25738613a0236a28e2dd47c936ee52c", size = 12084108, upload-time = "2026-08-14T21:35:06.686Z" }, - { url = "https://files.pythonhosted.org/packages/a1/dc/1fe1aef8d697e3509face271a5331700c7aa1d1e44a4b622707bdfa41d4b/ty-0.0.72-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7f3943f186f741a2499a31053872169250c9264a9a49684920e48d8fcf4ef4f5", size = 12132640, upload-time = "2026-08-14T21:35:09.305Z" }, - { url = "https://files.pythonhosted.org/packages/14/46/41ceb265e96969487311a2014bd0e53abb4fbc1395efb2ebe411fcb4db62/ty-0.0.72-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cf283c07dc3cc52ca48a3ad8ab100fb5aec3aebbd03ef6a12d5f910b8e596fc5", size = 12402489, upload-time = "2026-08-14T21:35:11.555Z" }, - { url = "https://files.pythonhosted.org/packages/2b/45/30bf43cb4fd505c5c2dd30fda27dde5f05208686cd21217adec77c954204/ty-0.0.72-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:95f3b6462c38f9f115d10cee21f47fedf715fcf2040daf36eef210359300bc7c", size = 13130835, upload-time = "2026-08-14T21:35:13.746Z" }, - { url = "https://files.pythonhosted.org/packages/31/2f/03bba754d2613f640df168335c41f83f41db150bb515839c60d80e3a7880/ty-0.0.72-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:30caf658feb8ffb250d9e9e47107657a78f5f3425c227df1664d8df2ebe38880", size = 13590392, upload-time = "2026-08-14T21:35:16.839Z" }, - { url = "https://files.pythonhosted.org/packages/04/c7/03c67f00e63005ec41585653dc3096064570b1e6273742baae2798cd242f/ty-0.0.72-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:27bdc012ddfbeec8948e4a6036c0dc39ac7cf2c8ec7c7d48dc7d2fd56d57b399", size = 13309629, upload-time = "2026-08-14T21:35:19.169Z" }, - { url = "https://files.pythonhosted.org/packages/c1/df/102d3b264eb7f2a58dd11952f229bb5150bb5668d176a6154976a6675981/ty-0.0.72-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:802c5970a77d7739e6f499921fbb6984fb7ad8a31d95e1ff42fd46f3642e4f3b", size = 12734028, upload-time = "2026-08-14T21:35:22.099Z" }, - { url = "https://files.pythonhosted.org/packages/61/85/d0737c8c54d0ba67366ddfb9f31d88edf0b02299e65923e6945ae60ebcb5/ty-0.0.72-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:47dce65114fdc615c68ca0edb393b433df0956447e4267df0e264137a789598d", size = 13174832, upload-time = "2026-08-14T21:35:24.71Z" }, - { url = "https://files.pythonhosted.org/packages/1e/31/497f5a96c36d9b586ab6afe0574986835c6fd5b835a89773d2bec4711b49/ty-0.0.72-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:325144fa07e2675d0faa337fcc864213c272a499eb0cfe5bde2fdc62282d27bc", size = 12215005, upload-time = "2026-08-14T21:35:26.892Z" }, - { url = "https://files.pythonhosted.org/packages/df/7d/46e65b17b4966c7cd0140f134380d33d8e84fe6efccd761533ce793dc502/ty-0.0.72-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:a5c9f15d0f58e43707d8848274be1821a0ef408eccb8aa7dda28a4a9eddf7640", size = 12421298, upload-time = "2026-08-14T21:35:29.301Z" }, - { url = "https://files.pythonhosted.org/packages/08/2a/12ada4ec17700b3cb1d4fd3bc3e5b1852df9e6885288429318cade87b3c1/ty-0.0.72-py3-none-musllinux_1_2_i686.whl", hash = "sha256:8ee508d64b381871529cc22c412b41071bf5e908b7aa5d66a38f3f6b2573a806", size = 12669242, upload-time = "2026-08-14T21:35:31.444Z" }, - { url = "https://files.pythonhosted.org/packages/1c/1a/4692536880790fb550ed6d44a6096778dc71bb112f2c6d615cebb01a57e5/ty-0.0.72-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:3699e2ec7921d44da79d6b089f7bf239b2cc53c4e45a5a38430adc34ee9e9a55", size = 12988199, upload-time = "2026-08-14T21:35:33.749Z" }, - { url = "https://files.pythonhosted.org/packages/9a/0d/f5e5a50322e9c45865e7b7a428ba6cd6527387cf0f2472492ac3cf746243/ty-0.0.72-py3-none-win32.whl", hash = "sha256:f25f72a67bd36cd247707c4784e52fad0b6b4f42a1b7dd14804110fa95c486ed", size = 11939708, upload-time = "2026-08-14T21:35:36.006Z" }, - { url = "https://files.pythonhosted.org/packages/3f/4e/8af3534b2e4214e6184a5a59c34101e94a68d578f081f97b995866bab1bf/ty-0.0.72-py3-none-win_amd64.whl", hash = "sha256:cdeee869341717e1736cea2e2d7856738c6957c320f584ed2f68c8f90100d2f5", size = 12643876, upload-time = "2026-08-14T21:35:38.141Z" }, - { url = "https://files.pythonhosted.org/packages/ff/ea/a2606e654c7276bd08586391a2525b0af3f3bf60228a8c57b2d248f273f9/ty-0.0.72-py3-none-win_arm64.whl", hash = "sha256:1bd3ac3ed4424a6d6990a85dc388556aea012bd752de21349a84b685951de0d8", size = 12394857, upload-time = "2026-08-14T21:35:40.277Z" }, +version = "0.0.73" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e5/90/c4e1bb4cead3b644c3e258a27f9b05c7dc5eb0ec96a4f5282194edae9e0d/ty-0.0.73.tar.gz", hash = "sha256:823d4ce0d237bfc7eb6bcee70842f2c0706113813a16951077840743712f4b74", size = 6712739, upload-time = "2026-08-19T03:12:43.381Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e4/0f/f5e1801e55cc631f2db193276675b30561b963a2403da832bffb5d100267/ty-0.0.73-py3-none-linux_armv6l.whl", hash = "sha256:90a946082bf9bc446b5e72973d9f4ff1222a240b2ca4c9e6eed61eb913e30810", size = 12715452, upload-time = "2026-08-19T03:12:06.673Z" }, + { url = "https://files.pythonhosted.org/packages/54/32/515dd05074c213b433524ab97eb003b0132ae7e358e0d75633ba7a314ed8/ty-0.0.73-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b7d6b5c6a6db7ea95fbbc16af514ef44a27a29a2fe1dc798900790364d170209", size = 12301870, upload-time = "2026-08-19T03:12:08.924Z" }, + { url = "https://files.pythonhosted.org/packages/50/4d/085b4889f0d4bbe4af8b96242d4a1cb209fff95967cfa239ea141983719b/ty-0.0.73-py3-none-macosx_11_0_arm64.whl", hash = "sha256:dd6f657f463e01372d8688f235be164750c8db722c97da27fa4903aa8d40b203", size = 12111741, upload-time = "2026-08-19T03:12:11.067Z" }, + { url = "https://files.pythonhosted.org/packages/95/f6/d6ec277cadfecf03ad4c18551b67c4c6eb7807a0560d801db14be99d7a89/ty-0.0.73-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc2de468e33fd44c9ff1c43473a7316f4289480f5cba8995a67b6d22aee39ca9", size = 12196124, upload-time = "2026-08-19T03:12:13.14Z" }, + { url = "https://files.pythonhosted.org/packages/75/b7/ce78d8707563af9cae9bbd25328bfbc4931035085bd20089adf0c418f70e/ty-0.0.73-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2942fa0ef795a66034cdc8d75a72f453442f3b58ff2f69b4da05b7b954765b55", size = 12488557, upload-time = "2026-08-19T03:12:15.252Z" }, + { url = "https://files.pythonhosted.org/packages/d8/e8/329b9851b23502758c5c98e8cc875ea2a1b4c9674b4ca3a86da56a5063d3/ty-0.0.73-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e0f1ef14f642e18ac4e7a616a2796dcf7a5d82e28cd17f9796494acc7c4aabb", size = 13215606, upload-time = "2026-08-19T03:12:17.225Z" }, + { url = "https://files.pythonhosted.org/packages/36/38/67fedfd2cb77516ef0066b1642f487dba0eb3006493cf3475b15f5b8b228/ty-0.0.73-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16981e15fdceedb37d0aff76c5ac25914595dfee2675af95335550064251ad22", size = 13665497, upload-time = "2026-08-19T03:12:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/8e/b3/154f4dd48ec5eebc186ab4b822c6e62f982fc5ddfd262d6e3903c2acba44/ty-0.0.73-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:644b2bec8a2e2e4957a942ae81d6cff5571c489bb5a8675e4d3886de537a694d", size = 13351231, upload-time = "2026-08-19T03:12:21.353Z" }, + { url = "https://files.pythonhosted.org/packages/35/5f/d462496903fbe453fb76363f8478be929c8e6ff21e6928c57dcd7e5fa21f/ty-0.0.73-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:338d565be3186f50ff8e9d10483685549c2d23f0754485d5ede3b54f4319188a", size = 12782586, upload-time = "2026-08-19T03:12:23.667Z" }, + { url = "https://files.pythonhosted.org/packages/87/52/ec6d24b74abe3ec324204c1c71e6d0c6c76a17ffc15fd51d603b0a302abe/ty-0.0.73-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:11c7b6d839309d2c102cb3a4c03d817176bbfab5b2fccc95a75ec5c9597421c9", size = 13247134, upload-time = "2026-08-19T03:12:25.956Z" }, + { url = "https://files.pythonhosted.org/packages/26/20/cc74650fec56a54786c6d7c89e09576fcad3092be34cf21715d39a406a9b/ty-0.0.73-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:488572db7ff97fb50ea36a76250f2d617c9727d143da6c7bf0623276eb0fc507", size = 12309344, upload-time = "2026-08-19T03:12:28.122Z" }, + { url = "https://files.pythonhosted.org/packages/89/bd/4b0a9087f4315d7fbadf77a3ce44c816cc9ffabed1ced06cc5be81fbc414/ty-0.0.73-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1b958ebceefbbf594e59eb8d3d55bbd033ce634026fcba3e4bc3179e78e45bb7", size = 12502319, upload-time = "2026-08-19T03:12:30.128Z" }, + { url = "https://files.pythonhosted.org/packages/11/80/0a925074911fe111912ea29d9eed309bcc183f43d2fb3eef07db056a0beb/ty-0.0.73-py3-none-musllinux_1_2_i686.whl", hash = "sha256:91a32993b3c34e42c3f323ad6c0399cb596bd1c27e9b7f20db7cd64c1067b68e", size = 12753688, upload-time = "2026-08-19T03:12:32.433Z" }, + { url = "https://files.pythonhosted.org/packages/24/6b/aeccaf89efbc2e112bd415340a22e2669ec998aa397242503e747b712ca4/ty-0.0.73-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:bab8a19fbf51f479bddb2a12c5fabfe52f918a5590362321ed5d89b44eb62c15", size = 13069050, upload-time = "2026-08-19T03:12:35.398Z" }, + { url = "https://files.pythonhosted.org/packages/d7/3e/eae485fd86c1585943fd4e1746b0757b2da01e2c43136ebe8c686fe1c7f1/ty-0.0.73-py3-none-win32.whl", hash = "sha256:03347a612f0fa020b19bfd8dbd521db6ecc75d377a3e4d4f6e6c2e62871da4cc", size = 12053187, upload-time = "2026-08-19T03:12:37.565Z" }, + { url = "https://files.pythonhosted.org/packages/a7/01/9b8b983786e3ce34924e372e8b76b92b508273ab65c589fc7e88cc03ee17/ty-0.0.73-py3-none-win_amd64.whl", hash = "sha256:cedd05122ded0b5dcc55431a370e974b747f99c41c290a3d2ab8c1867f197519", size = 12693838, upload-time = "2026-08-19T03:12:39.483Z" }, + { url = "https://files.pythonhosted.org/packages/ea/88/25333bbfea6a5dc064371d2002d3d4807db90b84d5448f9106b2712b0fbc/ty-0.0.73-py3-none-win_arm64.whl", hash = "sha256:e47068f8369dea5d641a26a2ad0a947a320b02ff87099b07e95de0323245a4dc", size = 12443573, upload-time = "2026-08-19T03:12:41.449Z" }, ] [[package]] @@ -2680,15 +2653,15 @@ wheels = [ [[package]] name = "uvicorn" -version = "0.52.3" +version = "0.52.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "click" }, { name = "h11" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2e/28/64ca011edf31c715b4fad359c587ea52391aaffa125065695590241ff617/uvicorn-0.52.3.tar.gz", hash = "sha256:18857b9e6579300be55c91c0a1cfd37d9a2cf0cabea33b88275f199eb73b8b58", size = 100621, upload-time = "2026-08-13T16:50:02.899Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f2/0f/3f86e61397dd33bf2ccf28188c40db6a740658aeebbbf6e7dbc101a1f487/uvicorn-0.52.4.tar.gz", hash = "sha256:73acfee47a0b133c5de13d219492d62d8a31e935f4fe6e41a232451a15379f86", size = 100627, upload-time = "2026-08-19T06:27:41.821Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/2b/ebd108734a8204c6b4b93c681c9a38c5273b3ccd5d129fee4ffc1d97772c/uvicorn-0.52.3-py3-none-any.whl", hash = "sha256:116af2710dbf47c80f463cd20ee4884b6662f4c9f227d797ddc7279d2fcc2c7c", size = 79859, upload-time = "2026-08-13T16:50:01.323Z" }, + { url = "https://files.pythonhosted.org/packages/f1/79/4a20b54ab0491485ccd8c077db2d39187c7f12b3e15485d38a7be37c81b4/uvicorn-0.52.4-py3-none-any.whl", hash = "sha256:f86e41a149d7d05a9969337e3946a9c171c06a5d42680896daaba624aeac8da1", size = 79871, upload-time = "2026-08-19T06:27:40.36Z" }, ] [package.optional-dependencies] From 84fde5a1ed5e0d5a58ccb3ec4b82938b059bf8c5 Mon Sep 17 00:00:00 2001 From: Soju06 Date: Wed, 26 Aug 2026 17:13:04 +0900 Subject: [PATCH 110/117] chore(main): release 1.24.0 (#1692) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## [1.24.0](https://github.com/Soju06/codex-lb/compare/v1.23.0...v1.24.0) (2026-08-26) ### Features * **api-keys:** allow per-key reasoning effort policies ([#1642](https://github.com/Soju06/codex-lb/issues/1642)) ([ed31b7d](https://github.com/Soju06/codex-lb/commit/ed31b7da3d225aac30fc42c7c0d128f10955b58e)) * **config:** timeout-invariant linter — validate deadline/TTL inequalities at startup and in CI ([#1622](https://github.com/Soju06/codex-lb/issues/1622)) ([d148dd9](https://github.com/Soju06/codex-lb/commit/d148dd9a42dca3088e8063aca8a21682f9bf7fb6)) * **db:** report SQLite write transactions that outlive the busy timeout ([#1752](https://github.com/Soju06/codex-lb/issues/1752)) ([6464e96](https://github.com/Soju06/codex-lb/commit/6464e96f78bdc62c643d9285143b72152e9a0742)) * **frontend:** configure model-source reasoning efforts ([#1848](https://github.com/Soju06/codex-lb/issues/1848)) ([eab7155](https://github.com/Soju06/codex-lb/commit/eab71553aee660fdb31122e9feb61a0e6c367904)) * **model-sources:** advertise operator-declared reasoning efforts ([#1661](https://github.com/Soju06/codex-lb/issues/1661)) ([f1c8d5c](https://github.com/Soju06/codex-lb/commit/f1c8d5cd19947d76191fea8da8c07a69df493f83)) * **model-sources:** embeddings source capability ([#1776](https://github.com/Soju06/codex-lb/issues/1776)) ([4d0f0ff](https://github.com/Soju06/codex-lb/commit/4d0f0ffc64df11a6800397497512abeb5478bce2)) * **proxy:** report websocket cleanup phase ([#1726](https://github.com/Soju06/codex-lb/issues/1726)) ([0c8d921](https://github.com/Soju06/codex-lb/commit/0c8d921906735352ef60c0a445be455daa35249d)) * **proxy:** support Ultrafast service tier ([#1734](https://github.com/Soju06/codex-lb/issues/1734)) ([d522a4d](https://github.com/Soju06/codex-lb/commit/d522a4de0740b530b22775ed6d2fe3959e3ca178)) * **reports:** Add API Key Filtering to Reports Dashboard ([#1728](https://github.com/Soju06/codex-lb/issues/1728)) ([1f65f80](https://github.com/Soju06/codex-lb/commit/1f65f8093458d551979bd237d8304ed70b50c390)) * **reset-credits:** add refresh scheduler enable toggle ([#1701](https://github.com/Soju06/codex-lb/issues/1701)) ([6509dd0](https://github.com/Soju06/codex-lb/commit/6509dd0d4a577908e5940f35ba4c5ab6d66f23bc)) * **telemetry:** anonymous usage telemetry with informed opt-out consent ([#1618](https://github.com/Soju06/codex-lb/issues/1618)) ([debd7cf](https://github.com/Soju06/codex-lb/commit/debd7cf63c173e1e7b2982ff171bf1564c150c5a)) * **telemetry:** report consent state and send a decision-time opt-out signal ([#1835](https://github.com/Soju06/codex-lb/issues/1835)) ([1541ee8](https://github.com/Soju06/codex-lb/commit/1541ee83105edd9a06062f6bcc617f9dab595a66)) * **ui:** customize dashboard request-log columns ([#1503](https://github.com/Soju06/codex-lb/issues/1503)) ([138aa9f](https://github.com/Soju06/codex-lb/commit/138aa9f15c6ebea998335afae06475d8834fe7d6)) * **ui:** surface reasoning token usage ([#1801](https://github.com/Soju06/codex-lb/issues/1801)) ([8e7589e](https://github.com/Soju06/codex-lb/commit/8e7589e4286869b9030eb1b19726297c63852ed3)) ### Bug Fixes * **accounts:** recover Free accounts after reset ([#1700](https://github.com/Soju06/codex-lb/issues/1700)) ([b43d0c8](https://github.com/Soju06/codex-lb/commit/b43d0c8bb3101932436dc9ff6fadb9034b4d3b03)) * **auth:** guard the refresh singleflight negative cache by successor ownership ([#1652](https://github.com/Soju06/codex-lb/issues/1652)) ([4ace71e](https://github.com/Soju06/codex-lb/commit/4ace71e8044e5180905f7a53f2d00d010cd43233)) * **cache:** keep an aborted invalidation bump queued ([#1748](https://github.com/Soju06/codex-lb/issues/1748)) ([7148810](https://github.com/Soju06/codex-lb/commit/71488100f71b5b132ed9c86b8cddbb63991842a9)) * **chat:** omit unset tools on mapped Responses payloads ([#1725](https://github.com/Soju06/codex-lb/issues/1725)) ([db5776c](https://github.com/Soju06/codex-lb/commit/db5776c288b5c709d665c1dcaa9298ae8c390e6a)) * **compact:** omit oversized non-state tool tail ([#1235](https://github.com/Soju06/codex-lb/issues/1235)) ([edb3734](https://github.com/Soju06/codex-lb/commit/edb37348f19a5015c3b90e0509ed92179e3716da)) * **compact:** recover previous-response-pinned compaction from quota-excluded owners ([#1780](https://github.com/Soju06/codex-lb/issues/1780)) ([120af75](https://github.com/Soju06/codex-lb/commit/120af7520cae7d51fb66c731d9d4578e416ce93f)) * **dashboard:** distinguish first-run empty states from filter mismatch ([#1729](https://github.com/Soju06/codex-lb/issues/1729)) ([e439043](https://github.com/Soju06/codex-lb/commit/e439043d4ad2869ca992d3d9bbb632b814406013)) * **dashboard:** exclude cancelled/client_disconnected from error rate ([#1696](https://github.com/Soju06/codex-lb/issues/1696)) ([8a7d956](https://github.com/Soju06/codex-lb/commit/8a7d9560571478a435f66f082b7656a6e9313f14)) * **dashboard:** pin web asset MIME types against poisoned OS registries ([#1709](https://github.com/Soju06/codex-lb/issues/1709)) ([2164b8c](https://github.com/Soju06/codex-lb/commit/2164b8c0656b3a8539ccdf8a4a8655a788af9ae8)), closes [#1698](https://github.com/Soju06/codex-lb/issues/1698) * **dashboard:** preserve cancelled request count ([#1766](https://github.com/Soju06/codex-lb/issues/1766)) ([ef9c68e](https://github.com/Soju06/codex-lb/commit/ef9c68e0fbc553ff83c374e4dcb7a3c079b163a9)) * **dashboard:** separate quota and purchased credits ([#1670](https://github.com/Soju06/codex-lb/issues/1670)) ([4a08d96](https://github.com/Soju06/codex-lb/commit/4a08d968f34aeec423892c02775769335590619f)) * **dashboard:** show cancellation totals in reports ([#1772](https://github.com/Soju06/codex-lb/issues/1772)) ([5d27f7f](https://github.com/Soju06/codex-lb/commit/5d27f7f0cc2fea37ce2e3786ea08660f008dd694)) * **dashboard:** show cancelled request logs ([#1769](https://github.com/Soju06/codex-lb/issues/1769)) ([5f2f726](https://github.com/Soju06/codex-lb/commit/5f2f7266163f93b8fe5df6295fae6342bc51945b)) * **dashboard:** surface upstream route metadata ([#1767](https://github.com/Soju06/codex-lb/issues/1767)) ([e359d49](https://github.com/Soju06/codex-lb/commit/e359d490befc87d27f198b528a67685ee7a8503e)) * **db:** add postgres shm_size and raise default pool headroom ([#1791](https://github.com/Soju06/codex-lb/issues/1791)) ([539cf93](https://github.com/Soju06/codex-lb/commit/539cf934ea654efbc915bad17ea84a87c07afa4d)) * **db:** bound wedged SQLite session teardown and reclaim the connection ([#1778](https://github.com/Soju06/codex-lb/issues/1778)) ([9eedb2c](https://github.com/Soju06/codex-lb/commit/9eedb2c8f4aa809715484e1c7607e6f0219b4ccf)) * **db:** repair retired identity/warmup migration stamp ([#1847](https://github.com/Soju06/codex-lb/issues/1847)) ([b6c217f](https://github.com/Soju06/codex-lb/commit/b6c217fada24c8a7b7e2c77af6bce3e2d7a2d3e2)) * **docker:** upgrade util-linux family in runtime image for CVE-2026-53615 ([#1796](https://github.com/Soju06/codex-lb/issues/1796)) ([0a4c0a1](https://github.com/Soju06/codex-lb/commit/0a4c0a1071cfe2e91357d8dc34b433cb511b1aec)) * **helm:** bind TTFT dashboard SQL datasource ([#1827](https://github.com/Soju06/codex-lb/issues/1827)) ([8abd507](https://github.com/Soju06/codex-lb/commit/8abd50778dd15131fac01f6a40d8e07a1bcebf54)) * **http-bridge:** classify recovery error frames and poison same-anchor eventless failures ([#1841](https://github.com/Soju06/codex-lb/issues/1841)) ([01f089c](https://github.com/Soju06/codex-lb/commit/01f089c359dadc2cc75b0719addc646e116624a5)) * **http-bridge:** dedupe retry circuit failures per send ([#1743](https://github.com/Soju06/codex-lb/issues/1743)) ([5780a27](https://github.com/Soju06/codex-lb/commit/5780a27f8c77f033ece2d144cb25ff89fb9db679)) * **http-bridge:** keep idle retirements out of retry circuit ([#1677](https://github.com/Soju06/codex-lb/issues/1677)) ([7c46719](https://github.com/Soju06/codex-lb/commit/7c4671980094135b9094278d2ebb374c7cb22655)) * **http-bridge:** preserve goal-restart recovery across reconnects ([#1680](https://github.com/Soju06/codex-lb/issues/1680)) ([5dc6081](https://github.com/Soju06/codex-lb/commit/5dc6081e41b7abe8670ea4565758246ef9f173b9)) * **http-bridge:** refuse foreign claims on live DRAINING leases ([#1722](https://github.com/Soju06/codex-lb/issues/1722)) ([b50cb86](https://github.com/Soju06/codex-lb/commit/b50cb86659f0cebdc4abd8e200556a54e62e17da)) * **models:** apply context-window overrides to /v1 input context fields ([#1808](https://github.com/Soju06/codex-lb/issues/1808)) ([c750dcf](https://github.com/Soju06/codex-lb/commit/c750dcfe64961c7d538c75367e9ee509fe8c9052)) * **models:** correct GPT-5.6 context windows ([#1691](https://github.com/Soju06/codex-lb/issues/1691)) ([8488bc4](https://github.com/Soju06/codex-lb/commit/8488bc462a46be07ae70f805ff6bf351f0ba4d97)) * **models:** raise GPT-5.6 bootstrap max_context_window to 872k ([#1813](https://github.com/Soju06/codex-lb/issues/1813)) ([1add104](https://github.com/Soju06/codex-lb/commit/1add1041b61e7b20ea104a91dd6331f03db904c7)) * **proxy:** abandon unavailable owner on thread-scoped goal restart ([#1764](https://github.com/Soju06/codex-lb/issues/1764)) ([17ae866](https://github.com/Soju06/codex-lb/commit/17ae866e2f6fd3d2daa1f1c4a0b2a8d0a5d2f25b)) * **proxy:** absorb replay-safe compaction recovery ([#1849](https://github.com/Soju06/codex-lb/issues/1849)) ([c597226](https://github.com/Soju06/codex-lb/commit/c597226cf139ec1e80117b72f02b7a18bef3645f)) * **proxy:** add explicit Daybreak capability routing ([#1742](https://github.com/Soju06/codex-lb/issues/1742)) ([0031e3d](https://github.com/Soju06/codex-lb/commit/0031e3d468747d63d8573139eabcaa0594891aef)) * **proxy:** bind account-bound retries to dispatch owner ([#1829](https://github.com/Soju06/codex-lb/issues/1829)) ([3381938](https://github.com/Soju06/codex-lb/commit/3381938aa278d7f3cd371bdd76c7914856586bf8)) * **proxy:** classify parameterless previous response errors ([#1818](https://github.com/Soju06/codex-lb/issues/1818)) ([eeab46a](https://github.com/Soju06/codex-lb/commit/eeab46a5edf5be16ff2915edc21a7f6a9a424717)) * **proxy:** close non-stream chat collect and map error status ([#1712](https://github.com/Soju06/codex-lb/issues/1712)) ([a85f71d](https://github.com/Soju06/codex-lb/commit/a85f71dbec6f56d417bd663c3e63e7431267554e)) * **proxy:** compact transport switch + trigger canonicalization (supersedes [#1749](https://github.com/Soju06/codex-lb/issues/1749)) ([#1809](https://github.com/Soju06/codex-lb/issues/1809)) ([0481ed9](https://github.com/Soju06/codex-lb/commit/0481ed996ab128ae67ff311a9c69699e98890d7b)) * **proxy:** complete disconnect cleanup — pool leak, charged reservation, mutable terminal reason ([#1645](https://github.com/Soju06/codex-lb/issues/1645)) ([6cf7e61](https://github.com/Soju06/codex-lb/commit/6cf7e61d7719654a62fb3132a21ae5f19ad8dba1)) * **proxy:** demote quarantined bridge reattach keys ([#1730](https://github.com/Soju06/codex-lb/issues/1730)) ([5e1f568](https://github.com/Soju06/codex-lb/commit/5e1f568f3a2772b4d1162a5e325d056cae7daa39)) * **proxy:** do not rewrite thread locality for a file-pin owner ([#1765](https://github.com/Soju06/codex-lb/issues/1765)) ([34ef7b2](https://github.com/Soju06/codex-lb/commit/34ef7b262ecabc597be0c7dd73978dc24973ee82)) * **proxy:** drop malformed compact item ids ([#1815](https://github.com/Soju06/codex-lb/issues/1815)) ([812265d](https://github.com/Soju06/codex-lb/commit/812265d11c0faa470da1db1b689442afa2b93869)) * **proxy:** durably recover hard HTTP bridge operations ([#1657](https://github.com/Soju06/codex-lb/issues/1657)) ([7a0b671](https://github.com/Soju06/codex-lb/commit/7a0b67192140ab307b911719189c52f4fa87033d)) * **proxy:** fence successor bridge claims against the retiring predecessor ([#1751](https://github.com/Soju06/codex-lb/issues/1751)) ([2c0dc5b](https://github.com/Soju06/codex-lb/commit/2c0dc5b8eec16d3e8c413f144cd62c583f43847d)) * **proxy:** guard model-transition owner-conflict fork ([#1619](https://github.com/Soju06/codex-lb/issues/1619)) ([52092bc](https://github.com/Soju06/codex-lb/commit/52092bc91fd81d7a18655eab403f02ff6895dd80)) * **proxy:** hold fenced hard turns through cooldown ([#1739](https://github.com/Soju06/codex-lb/issues/1739)) ([6ff51cd](https://github.com/Soju06/codex-lb/commit/6ff51cd69c564499e4371ee755eecab8d80b262d)) * **proxy:** keep abrupt eventless websocket drops account-neutral ([#1777](https://github.com/Soju06/codex-lb/issues/1777)) ([6c97ad6](https://github.com/Soju06/codex-lb/commit/6c97ad6265100c4ac3489e15a247d73ab45866fb)) * **proxy:** keep file-pin owner on soft 1011 reconnect ([#1761](https://github.com/Soju06/codex-lb/issues/1761)) ([f694c44](https://github.com/Soju06/codex-lb/commit/f694c449479dd1b204e93ee27bf599f9a5c6b86c)) * **proxy:** keep stream idle timeouts account-neutral ([#1718](https://github.com/Soju06/codex-lb/issues/1718)) ([64da340](https://github.com/Soju06/codex-lb/commit/64da340ab72580d8762ba9cda8a1fed9c9bb30be)) * **proxy:** normalize single-account warmup failures ([#1774](https://github.com/Soju06/codex-lb/issues/1774)) ([f92bc90](https://github.com/Soju06/codex-lb/commit/f92bc906ee06079e307866cff548b75435b46c49)) * **proxy:** O(1) shared-future admission waits + event-loop lag watchdog ([#1842](https://github.com/Soju06/codex-lb/issues/1842)) ([ed2c94d](https://github.com/Soju06/codex-lb/commit/ed2c94d4b8ece64455233e5293d44ec0f263e6bc)) * **proxy:** persist file ownership across replicas ([#1521](https://github.com/Soju06/codex-lb/issues/1521)) ([2cd52e4](https://github.com/Soju06/codex-lb/commit/2cd52e44b4136bdc76b425cca1bd767335f43754)) * **proxy:** preserve compact terminal error type ([#1824](https://github.com/Soju06/codex-lb/issues/1824)) ([78d63e5](https://github.com/Soju06/codex-lb/commit/78d63e5a840c1cc6a34a03cb002eac9b3e041f7e)) * **proxy:** reject truncated chat completion streams ([#1833](https://github.com/Soju06/codex-lb/issues/1833)) ([6ba083d](https://github.com/Soju06/codex-lb/commit/6ba083d7df4f82c2d2e6aa08e9a1e1fb47b59faa)) * **proxy:** release the API-key reservation on all exits of the models endpoints ([#1653](https://github.com/Soju06/codex-lb/issues/1653)) ([7007885](https://github.com/Soju06/codex-lb/commit/7007885dad6572e2332739f808528c5b8b4a0857)) * **proxy:** report suppressed duplicate tool-call terminals ([#1706](https://github.com/Soju06/codex-lb/issues/1706)) ([25d6374](https://github.com/Soju06/codex-lb/commit/25d6374a8a671900a6bf4dc89f248e7834efad76)) * **proxy:** retain image reservation recovery ownership ([#1822](https://github.com/Soju06/codex-lb/issues/1822)) ([bd67c64](https://github.com/Soju06/codex-lb/commit/bd67c640012692786f6beddd3d61c67cea759c47)) * **proxy:** route source-owned models off the WebSocket transport ([#1659](https://github.com/Soju06/codex-lb/issues/1659)) ([08b84a9](https://github.com/Soju06/codex-lb/commit/08b84a95ad5d4de88b3ac4ebb37185781a603f81)) * **proxy:** scope backend Codex affinity by thread identity ([#1703](https://github.com/Soju06/codex-lb/issues/1703)) ([35bbb00](https://github.com/Soju06/codex-lb/commit/35bbb006bc2ec76a43273f08f5a29ea150f11f4c)) * **proxy:** separate websocket scope cleanup budget ([#1723](https://github.com/Soju06/codex-lb/issues/1723)) ([fd97cb8](https://github.com/Soju06/codex-lb/commit/fd97cb856970e46a5e6e4265e0064fca4f24fb02)) * **proxy:** settle compact failover before account health ([#1717](https://github.com/Soju06/codex-lb/issues/1717)) ([3093203](https://github.com/Soju06/codex-lb/commit/30932034c3188efbddb33a68e0c438bd5db87db3)) * **proxy:** settle terminal spool append failures ([#1775](https://github.com/Soju06/codex-lb/issues/1775)) ([4e48f35](https://github.com/Soju06/codex-lb/commit/4e48f355b519fb20e16e89a5e5c6b2375bb08161)) * **proxy:** stop abandoning an unresolved inflight session-creation future ([#1644](https://github.com/Soju06/codex-lb/issues/1644)) ([57618c8](https://github.com/Soju06/codex-lb/commit/57618c87528aaaac4fecc4c6ad57dd07fbfa108b)) * **proxy:** sweep idle bridge sessions without request traffic ([#1747](https://github.com/Soju06/codex-lb/issues/1747)) ([3159ebe](https://github.com/Soju06/codex-lb/commit/3159ebedfc48789ee6b9c678c4f48e842a2a3555)) * **proxy:** wait on usage-refresh singleflight without asyncio.shield ([#1897](https://github.com/Soju06/codex-lb/issues/1897)) ([798203f](https://github.com/Soju06/codex-lb/commit/798203ff9d9d8f30a9b53e181d34fc9935ba5444)), closes [#1896](https://github.com/Soju06/codex-lb/issues/1896) * **quota-planner:** compare warmup reset epochs in UTC ([#1623](https://github.com/Soju06/codex-lb/issues/1623)) ([e4fa3f2](https://github.com/Soju06/codex-lb/commit/e4fa3f273f45ac9eaeafc28047005584c954ef3c)) * **reports:** format full Cost values with grouping separators ([#1814](https://github.com/Soju06/codex-lb/issues/1814)) ([028a75c](https://github.com/Soju06/codex-lb/commit/028a75c33701494834c054718fb58e31d72c4d99)) * **review:** keep Codex review sessions resumable ([#1678](https://github.com/Soju06/codex-lb/issues/1678)) ([e34db2d](https://github.com/Soju06/codex-lb/commit/e34db2d218925f7764e57b0571dcf736d33084da)) * **server:** serve h2c upgrade offers as plain HTTP/1.1 instead of rejecting them ([#1782](https://github.com/Soju06/codex-lb/issues/1782)) ([8d265c3](https://github.com/Soju06/codex-lb/commit/8d265c3f73bda4c2adb7632d888dad5413b1b121)) * **usage:** fence leaked live-usage-ingestor tasks and settle their failures deterministically ([#1783](https://github.com/Soju06/codex-lb/issues/1783)) ([66fd103](https://github.com/Soju06/codex-lb/commit/66fd1033165133e943a5818d75c40bc86e4a6b49)) * **usage:** settle live snapshots after account consolidation ([#1773](https://github.com/Soju06/codex-lb/issues/1773)) ([3f66c28](https://github.com/Soju06/codex-lb/commit/3f66c288a230d6eb73c39b397c558427c21e167f)) * **warmup:** warm paid-to-free transitions ([#1825](https://github.com/Soju06/codex-lb/issues/1825)) ([68892e7](https://github.com/Soju06/codex-lb/commit/68892e7afff21cff8910e2ee5456317eff6e231b)) ### Performance Improvements * **accounts:** bound the account-listing live tail with a 2h fold lag and a 30s summary cache ([#1792](https://github.com/Soju06/codex-lb/issues/1792)) ([c1caa44](https://github.com/Soju06/codex-lb/commit/c1caa4468cdf2f94f9abe675b63e63a13dbdd383)) * **accounts:** make account deletion a fast mark + background batch drain ([#1795](https://github.com/Soju06/codex-lb/issues/1795)) ([d4f9e23](https://github.com/Soju06/codex-lb/commit/d4f9e23cd623d67beee2df153723839e391d44d1)) * **api-keys,proxy:** shape ORM hot-path queries ([#1788](https://github.com/Soju06/codex-lb/issues/1788)) ([7dacb04](https://github.com/Soju06/codex-lb/commit/7dacb04181390b85bd42b335a73e27ad4d90ec2f)) * **api-keys:** skip usage reservations when no limit applies ([#1789](https://github.com/Soju06/codex-lb/issues/1789)) ([8a2d066](https://github.com/Soju06/codex-lb/commit/8a2d0660e2b216a93593757f2fec242e69f92724)) * coalesce same-owner sticky session TTL refresh upserts ([#1790](https://github.com/Soju06/codex-lb/issues/1790)) ([076aab8](https://github.com/Soju06/codex-lb/commit/076aab854ff0b334d77ab2104175d672384065c5)) * **dashboard:** cap projections bulk usage-history read per account ([#1779](https://github.com/Soju06/codex-lb/issues/1779)) ([d4c43ef](https://github.com/Soju06/codex-lb/commit/d4c43ef88f8d4a548fb952a82901ea482995a633)) * **middleware:** convert BaseHTTPMiddleware layers to pure ASGI ([#1787](https://github.com/Soju06/codex-lb/issues/1787)) ([94057cc](https://github.com/Soju06/codex-lb/commit/94057ccf91a7ed9f32eb15dcfa0487c969dc2a83)) * **proxy:** disable permessage-deflate on direct-egress upstream websockets ([#1786](https://github.com/Soju06/codex-lb/issues/1786)) ([2e4a580](https://github.com/Soju06/codex-lb/commit/2e4a580c1c1834f00b6d4c62598caf1ffd0446ee)) * **proxy:** relay unmodified SSE frames verbatim ([#1785](https://github.com/Soju06/codex-lb/issues/1785)) ([980572e](https://github.com/Soju06/codex-lb/commit/980572eb8ee5ed70acb9edfc5a7a2acc35f46216)) * **proxy:** validate stream payloads only for lifecycle events ([#1784](https://github.com/Soju06/codex-lb/issues/1784)) ([9d9f099](https://github.com/Soju06/codex-lb/commit/9d9f099197326f37eea4042e27c5b09606f99043)) ### Documentation * **dashboard:** clarify routing, sticky affinity, quota thresholds, warm-up, and eligibility copy ([#1781](https://github.com/Soju06/codex-lb/issues/1781)) ([6ff22e0](https://github.com/Soju06/codex-lb/commit/6ff22e0e528fb7bdd6c69c059178681254b143af)) * **openspec:** archive 90 landed changes and sync their specs ([#1713](https://github.com/Soju06/codex-lb/issues/1713)) ([c3f0c56](https://github.com/Soju06/codex-lb/commit/c3f0c568cb4dda4547f5d765951ba0748e7c923a)) * **openspec:** archive landed performance and reliability changes ([#1694](https://github.com/Soju06/codex-lb/issues/1694)) ([6b3db74](https://github.com/Soju06/codex-lb/commit/6b3db74e7b8a8201f6362e06a7debb517e48db27)) * **proxy:** document cluster-wide account cap partitioning ([#1750](https://github.com/Soju06/codex-lb/issues/1750)) ([560fb50](https://github.com/Soju06/codex-lb/commit/560fb503b3ada0e5a544cd3de3af42b3c5e43ebe)) --- This PR was generated with [Release Please](https://github.com/googleapis/release-please). See [documentation](https://github.com/googleapis/release-please#release-please). --- .github/release-please-manifest.json | 2 +- CHANGELOG.md | 115 +++++++++++++++++++++++++++ app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 +- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 7 files changed, 122 insertions(+), 7 deletions(-) diff --git a/.github/release-please-manifest.json b/.github/release-please-manifest.json index c41415c5ec..4d625b2f88 100644 --- a/.github/release-please-manifest.json +++ b/.github/release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.23.0" + ".": "1.24.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index ade8935ba1..0eeaf3b439 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,120 @@ # Changelog +## [1.24.0](https://github.com/Soju06/codex-lb/compare/v1.23.0...v1.24.0) (2026-08-26) + + +### Features + +* **api-keys:** allow per-key reasoning effort policies ([#1642](https://github.com/Soju06/codex-lb/issues/1642)) ([ed31b7d](https://github.com/Soju06/codex-lb/commit/ed31b7da3d225aac30fc42c7c0d128f10955b58e)) +* **config:** timeout-invariant linter — validate deadline/TTL inequalities at startup and in CI ([#1622](https://github.com/Soju06/codex-lb/issues/1622)) ([d148dd9](https://github.com/Soju06/codex-lb/commit/d148dd9a42dca3088e8063aca8a21682f9bf7fb6)) +* **db:** report SQLite write transactions that outlive the busy timeout ([#1752](https://github.com/Soju06/codex-lb/issues/1752)) ([6464e96](https://github.com/Soju06/codex-lb/commit/6464e96f78bdc62c643d9285143b72152e9a0742)) +* **frontend:** configure model-source reasoning efforts ([#1848](https://github.com/Soju06/codex-lb/issues/1848)) ([eab7155](https://github.com/Soju06/codex-lb/commit/eab71553aee660fdb31122e9feb61a0e6c367904)) +* **model-sources:** advertise operator-declared reasoning efforts ([#1661](https://github.com/Soju06/codex-lb/issues/1661)) ([f1c8d5c](https://github.com/Soju06/codex-lb/commit/f1c8d5cd19947d76191fea8da8c07a69df493f83)) +* **model-sources:** embeddings source capability ([#1776](https://github.com/Soju06/codex-lb/issues/1776)) ([4d0f0ff](https://github.com/Soju06/codex-lb/commit/4d0f0ffc64df11a6800397497512abeb5478bce2)) +* **proxy:** report websocket cleanup phase ([#1726](https://github.com/Soju06/codex-lb/issues/1726)) ([0c8d921](https://github.com/Soju06/codex-lb/commit/0c8d921906735352ef60c0a445be455daa35249d)) +* **proxy:** support Ultrafast service tier ([#1734](https://github.com/Soju06/codex-lb/issues/1734)) ([d522a4d](https://github.com/Soju06/codex-lb/commit/d522a4de0740b530b22775ed6d2fe3959e3ca178)) +* **reports:** Add API Key Filtering to Reports Dashboard ([#1728](https://github.com/Soju06/codex-lb/issues/1728)) ([1f65f80](https://github.com/Soju06/codex-lb/commit/1f65f8093458d551979bd237d8304ed70b50c390)) +* **reset-credits:** add refresh scheduler enable toggle ([#1701](https://github.com/Soju06/codex-lb/issues/1701)) ([6509dd0](https://github.com/Soju06/codex-lb/commit/6509dd0d4a577908e5940f35ba4c5ab6d66f23bc)) +* **telemetry:** anonymous usage telemetry with informed opt-out consent ([#1618](https://github.com/Soju06/codex-lb/issues/1618)) ([debd7cf](https://github.com/Soju06/codex-lb/commit/debd7cf63c173e1e7b2982ff171bf1564c150c5a)) +* **telemetry:** report consent state and send a decision-time opt-out signal ([#1835](https://github.com/Soju06/codex-lb/issues/1835)) ([1541ee8](https://github.com/Soju06/codex-lb/commit/1541ee83105edd9a06062f6bcc617f9dab595a66)) +* **ui:** customize dashboard request-log columns ([#1503](https://github.com/Soju06/codex-lb/issues/1503)) ([138aa9f](https://github.com/Soju06/codex-lb/commit/138aa9f15c6ebea998335afae06475d8834fe7d6)) +* **ui:** surface reasoning token usage ([#1801](https://github.com/Soju06/codex-lb/issues/1801)) ([8e7589e](https://github.com/Soju06/codex-lb/commit/8e7589e4286869b9030eb1b19726297c63852ed3)) + + +### Bug Fixes + +* **accounts:** recover Free accounts after reset ([#1700](https://github.com/Soju06/codex-lb/issues/1700)) ([b43d0c8](https://github.com/Soju06/codex-lb/commit/b43d0c8bb3101932436dc9ff6fadb9034b4d3b03)) +* **auth:** guard the refresh singleflight negative cache by successor ownership ([#1652](https://github.com/Soju06/codex-lb/issues/1652)) ([4ace71e](https://github.com/Soju06/codex-lb/commit/4ace71e8044e5180905f7a53f2d00d010cd43233)) +* **cache:** keep an aborted invalidation bump queued ([#1748](https://github.com/Soju06/codex-lb/issues/1748)) ([7148810](https://github.com/Soju06/codex-lb/commit/71488100f71b5b132ed9c86b8cddbb63991842a9)) +* **chat:** omit unset tools on mapped Responses payloads ([#1725](https://github.com/Soju06/codex-lb/issues/1725)) ([db5776c](https://github.com/Soju06/codex-lb/commit/db5776c288b5c709d665c1dcaa9298ae8c390e6a)) +* **compact:** omit oversized non-state tool tail ([#1235](https://github.com/Soju06/codex-lb/issues/1235)) ([edb3734](https://github.com/Soju06/codex-lb/commit/edb37348f19a5015c3b90e0509ed92179e3716da)) +* **compact:** recover previous-response-pinned compaction from quota-excluded owners ([#1780](https://github.com/Soju06/codex-lb/issues/1780)) ([120af75](https://github.com/Soju06/codex-lb/commit/120af7520cae7d51fb66c731d9d4578e416ce93f)) +* **dashboard:** distinguish first-run empty states from filter mismatch ([#1729](https://github.com/Soju06/codex-lb/issues/1729)) ([e439043](https://github.com/Soju06/codex-lb/commit/e439043d4ad2869ca992d3d9bbb632b814406013)) +* **dashboard:** exclude cancelled/client_disconnected from error rate ([#1696](https://github.com/Soju06/codex-lb/issues/1696)) ([8a7d956](https://github.com/Soju06/codex-lb/commit/8a7d9560571478a435f66f082b7656a6e9313f14)) +* **dashboard:** pin web asset MIME types against poisoned OS registries ([#1709](https://github.com/Soju06/codex-lb/issues/1709)) ([2164b8c](https://github.com/Soju06/codex-lb/commit/2164b8c0656b3a8539ccdf8a4a8655a788af9ae8)), closes [#1698](https://github.com/Soju06/codex-lb/issues/1698) +* **dashboard:** preserve cancelled request count ([#1766](https://github.com/Soju06/codex-lb/issues/1766)) ([ef9c68e](https://github.com/Soju06/codex-lb/commit/ef9c68e0fbc553ff83c374e4dcb7a3c079b163a9)) +* **dashboard:** separate quota and purchased credits ([#1670](https://github.com/Soju06/codex-lb/issues/1670)) ([4a08d96](https://github.com/Soju06/codex-lb/commit/4a08d968f34aeec423892c02775769335590619f)) +* **dashboard:** show cancellation totals in reports ([#1772](https://github.com/Soju06/codex-lb/issues/1772)) ([5d27f7f](https://github.com/Soju06/codex-lb/commit/5d27f7f0cc2fea37ce2e3786ea08660f008dd694)) +* **dashboard:** show cancelled request logs ([#1769](https://github.com/Soju06/codex-lb/issues/1769)) ([5f2f726](https://github.com/Soju06/codex-lb/commit/5f2f7266163f93b8fe5df6295fae6342bc51945b)) +* **dashboard:** surface upstream route metadata ([#1767](https://github.com/Soju06/codex-lb/issues/1767)) ([e359d49](https://github.com/Soju06/codex-lb/commit/e359d490befc87d27f198b528a67685ee7a8503e)) +* **db:** add postgres shm_size and raise default pool headroom ([#1791](https://github.com/Soju06/codex-lb/issues/1791)) ([539cf93](https://github.com/Soju06/codex-lb/commit/539cf934ea654efbc915bad17ea84a87c07afa4d)) +* **db:** bound wedged SQLite session teardown and reclaim the connection ([#1778](https://github.com/Soju06/codex-lb/issues/1778)) ([9eedb2c](https://github.com/Soju06/codex-lb/commit/9eedb2c8f4aa809715484e1c7607e6f0219b4ccf)) +* **db:** repair retired identity/warmup migration stamp ([#1847](https://github.com/Soju06/codex-lb/issues/1847)) ([b6c217f](https://github.com/Soju06/codex-lb/commit/b6c217fada24c8a7b7e2c77af6bce3e2d7a2d3e2)) +* **docker:** upgrade util-linux family in runtime image for CVE-2026-53615 ([#1796](https://github.com/Soju06/codex-lb/issues/1796)) ([0a4c0a1](https://github.com/Soju06/codex-lb/commit/0a4c0a1071cfe2e91357d8dc34b433cb511b1aec)) +* **helm:** bind TTFT dashboard SQL datasource ([#1827](https://github.com/Soju06/codex-lb/issues/1827)) ([8abd507](https://github.com/Soju06/codex-lb/commit/8abd50778dd15131fac01f6a40d8e07a1bcebf54)) +* **http-bridge:** classify recovery error frames and poison same-anchor eventless failures ([#1841](https://github.com/Soju06/codex-lb/issues/1841)) ([01f089c](https://github.com/Soju06/codex-lb/commit/01f089c359dadc2cc75b0719addc646e116624a5)) +* **http-bridge:** dedupe retry circuit failures per send ([#1743](https://github.com/Soju06/codex-lb/issues/1743)) ([5780a27](https://github.com/Soju06/codex-lb/commit/5780a27f8c77f033ece2d144cb25ff89fb9db679)) +* **http-bridge:** keep idle retirements out of retry circuit ([#1677](https://github.com/Soju06/codex-lb/issues/1677)) ([7c46719](https://github.com/Soju06/codex-lb/commit/7c4671980094135b9094278d2ebb374c7cb22655)) +* **http-bridge:** keep missing-created watchdog armed after prelude and handle stale API-key activity ([#1580](https://github.com/Soju06/codex-lb/issues/1580)) ([0eb0ee7](https://github.com/Soju06/codex-lb/commit/0eb0ee7939309dfd264f56a88d1bbcb519fde5c6)) +* **http-bridge:** preserve goal-restart recovery across reconnects ([#1680](https://github.com/Soju06/codex-lb/issues/1680)) ([5dc6081](https://github.com/Soju06/codex-lb/commit/5dc6081e41b7abe8670ea4565758246ef9f173b9)) +* **http-bridge:** refuse foreign claims on live DRAINING leases ([#1722](https://github.com/Soju06/codex-lb/issues/1722)) ([b50cb86](https://github.com/Soju06/codex-lb/commit/b50cb86659f0cebdc4abd8e200556a54e62e17da)) +* **models:** apply context-window overrides to /v1 input context fields ([#1808](https://github.com/Soju06/codex-lb/issues/1808)) ([c750dcf](https://github.com/Soju06/codex-lb/commit/c750dcfe64961c7d538c75367e9ee509fe8c9052)) +* **models:** correct GPT-5.6 context windows ([#1691](https://github.com/Soju06/codex-lb/issues/1691)) ([8488bc4](https://github.com/Soju06/codex-lb/commit/8488bc462a46be07ae70f805ff6bf351f0ba4d97)) +* **models:** raise GPT-5.6 bootstrap max_context_window to 872k ([#1813](https://github.com/Soju06/codex-lb/issues/1813)) ([1add104](https://github.com/Soju06/codex-lb/commit/1add1041b61e7b20ea104a91dd6331f03db904c7)) +* **proxy:** abandon unavailable owner on thread-scoped goal restart ([#1764](https://github.com/Soju06/codex-lb/issues/1764)) ([17ae866](https://github.com/Soju06/codex-lb/commit/17ae866e2f6fd3d2daa1f1c4a0b2a8d0a5d2f25b)) +* **proxy:** absorb replay-safe compaction recovery ([#1849](https://github.com/Soju06/codex-lb/issues/1849)) ([c597226](https://github.com/Soju06/codex-lb/commit/c597226cf139ec1e80117b72f02b7a18bef3645f)) +* **proxy:** add explicit Daybreak capability routing ([#1742](https://github.com/Soju06/codex-lb/issues/1742)) ([0031e3d](https://github.com/Soju06/codex-lb/commit/0031e3d468747d63d8573139eabcaa0594891aef)) +* **proxy:** bind account-bound retries to dispatch owner ([#1829](https://github.com/Soju06/codex-lb/issues/1829)) ([3381938](https://github.com/Soju06/codex-lb/commit/3381938aa278d7f3cd371bdd76c7914856586bf8)) +* **proxy:** classify parameterless previous response errors ([#1818](https://github.com/Soju06/codex-lb/issues/1818)) ([eeab46a](https://github.com/Soju06/codex-lb/commit/eeab46a5edf5be16ff2915edc21a7f6a9a424717)) +* **proxy:** close non-stream chat collect and map error status ([#1712](https://github.com/Soju06/codex-lb/issues/1712)) ([a85f71d](https://github.com/Soju06/codex-lb/commit/a85f71dbec6f56d417bd663c3e63e7431267554e)) +* **proxy:** compact transport switch + trigger canonicalization (supersedes [#1749](https://github.com/Soju06/codex-lb/issues/1749)) ([#1809](https://github.com/Soju06/codex-lb/issues/1809)) ([0481ed9](https://github.com/Soju06/codex-lb/commit/0481ed996ab128ae67ff311a9c69699e98890d7b)) +* **proxy:** complete disconnect cleanup — pool leak, charged reservation, mutable terminal reason ([#1645](https://github.com/Soju06/codex-lb/issues/1645)) ([6cf7e61](https://github.com/Soju06/codex-lb/commit/6cf7e61d7719654a62fb3132a21ae5f19ad8dba1)) +* **proxy:** demote quarantined bridge reattach keys ([#1730](https://github.com/Soju06/codex-lb/issues/1730)) ([5e1f568](https://github.com/Soju06/codex-lb/commit/5e1f568f3a2772b4d1162a5e325d056cae7daa39)) +* **proxy:** do not rewrite thread locality for a file-pin owner ([#1765](https://github.com/Soju06/codex-lb/issues/1765)) ([34ef7b2](https://github.com/Soju06/codex-lb/commit/34ef7b262ecabc597be0c7dd73978dc24973ee82)) +* **proxy:** drop malformed compact item ids ([#1815](https://github.com/Soju06/codex-lb/issues/1815)) ([812265d](https://github.com/Soju06/codex-lb/commit/812265d11c0faa470da1db1b689442afa2b93869)) +* **proxy:** durably recover hard HTTP bridge operations ([#1657](https://github.com/Soju06/codex-lb/issues/1657)) ([7a0b671](https://github.com/Soju06/codex-lb/commit/7a0b67192140ab307b911719189c52f4fa87033d)) +* **proxy:** fence successor bridge claims against the retiring predecessor ([#1751](https://github.com/Soju06/codex-lb/issues/1751)) ([2c0dc5b](https://github.com/Soju06/codex-lb/commit/2c0dc5b8eec16d3e8c413f144cd62c583f43847d)) +* **proxy:** guard model-transition owner-conflict fork ([#1619](https://github.com/Soju06/codex-lb/issues/1619)) ([52092bc](https://github.com/Soju06/codex-lb/commit/52092bc91fd81d7a18655eab403f02ff6895dd80)) +* **proxy:** hold fenced hard turns through cooldown ([#1739](https://github.com/Soju06/codex-lb/issues/1739)) ([6ff51cd](https://github.com/Soju06/codex-lb/commit/6ff51cd69c564499e4371ee755eecab8d80b262d)) +* **proxy:** keep abrupt eventless websocket drops account-neutral ([#1777](https://github.com/Soju06/codex-lb/issues/1777)) ([6c97ad6](https://github.com/Soju06/codex-lb/commit/6c97ad6265100c4ac3489e15a247d73ab45866fb)) +* **proxy:** keep file-pin owner on soft 1011 reconnect ([#1761](https://github.com/Soju06/codex-lb/issues/1761)) ([f694c44](https://github.com/Soju06/codex-lb/commit/f694c449479dd1b204e93ee27bf599f9a5c6b86c)) +* **proxy:** keep stream idle timeouts account-neutral ([#1718](https://github.com/Soju06/codex-lb/issues/1718)) ([64da340](https://github.com/Soju06/codex-lb/commit/64da340ab72580d8762ba9cda8a1fed9c9bb30be)) +* **proxy:** normalize single-account warmup failures ([#1774](https://github.com/Soju06/codex-lb/issues/1774)) ([f92bc90](https://github.com/Soju06/codex-lb/commit/f92bc906ee06079e307866cff548b75435b46c49)) +* **proxy:** O(1) shared-future admission waits + event-loop lag watchdog ([#1842](https://github.com/Soju06/codex-lb/issues/1842)) ([ed2c94d](https://github.com/Soju06/codex-lb/commit/ed2c94d4b8ece64455233e5293d44ec0f263e6bc)) +* **proxy:** persist file ownership across replicas ([#1521](https://github.com/Soju06/codex-lb/issues/1521)) ([2cd52e4](https://github.com/Soju06/codex-lb/commit/2cd52e44b4136bdc76b425cca1bd767335f43754)) +* **proxy:** preserve compact terminal error type ([#1824](https://github.com/Soju06/codex-lb/issues/1824)) ([78d63e5](https://github.com/Soju06/codex-lb/commit/78d63e5a840c1cc6a34a03cb002eac9b3e041f7e)) +* **proxy:** reject truncated chat completion streams ([#1833](https://github.com/Soju06/codex-lb/issues/1833)) ([6ba083d](https://github.com/Soju06/codex-lb/commit/6ba083d7df4f82c2d2e6aa08e9a1e1fb47b59faa)) +* **proxy:** release the API-key reservation on all exits of the models endpoints ([#1653](https://github.com/Soju06/codex-lb/issues/1653)) ([7007885](https://github.com/Soju06/codex-lb/commit/7007885dad6572e2332739f808528c5b8b4a0857)) +* **proxy:** report suppressed duplicate tool-call terminals ([#1706](https://github.com/Soju06/codex-lb/issues/1706)) ([25d6374](https://github.com/Soju06/codex-lb/commit/25d6374a8a671900a6bf4dc89f248e7834efad76)) +* **proxy:** retain image reservation recovery ownership ([#1822](https://github.com/Soju06/codex-lb/issues/1822)) ([bd67c64](https://github.com/Soju06/codex-lb/commit/bd67c640012692786f6beddd3d61c67cea759c47)) +* **proxy:** route source-owned models off the WebSocket transport ([#1659](https://github.com/Soju06/codex-lb/issues/1659)) ([08b84a9](https://github.com/Soju06/codex-lb/commit/08b84a95ad5d4de88b3ac4ebb37185781a603f81)) +* **proxy:** scope backend Codex affinity by thread identity ([#1703](https://github.com/Soju06/codex-lb/issues/1703)) ([35bbb00](https://github.com/Soju06/codex-lb/commit/35bbb006bc2ec76a43273f08f5a29ea150f11f4c)) +* **proxy:** separate websocket scope cleanup budget ([#1723](https://github.com/Soju06/codex-lb/issues/1723)) ([fd97cb8](https://github.com/Soju06/codex-lb/commit/fd97cb856970e46a5e6e4265e0064fca4f24fb02)) +* **proxy:** settle compact failover before account health ([#1717](https://github.com/Soju06/codex-lb/issues/1717)) ([3093203](https://github.com/Soju06/codex-lb/commit/30932034c3188efbddb33a68e0c438bd5db87db3)) +* **proxy:** settle terminal spool append failures ([#1775](https://github.com/Soju06/codex-lb/issues/1775)) ([4e48f35](https://github.com/Soju06/codex-lb/commit/4e48f355b519fb20e16e89a5e5c6b2375bb08161)) +* **proxy:** stop abandoning an unresolved inflight session-creation future ([#1644](https://github.com/Soju06/codex-lb/issues/1644)) ([57618c8](https://github.com/Soju06/codex-lb/commit/57618c87528aaaac4fecc4c6ad57dd07fbfa108b)) +* **proxy:** sweep idle bridge sessions without request traffic ([#1747](https://github.com/Soju06/codex-lb/issues/1747)) ([3159ebe](https://github.com/Soju06/codex-lb/commit/3159ebedfc48789ee6b9c678c4f48e842a2a3555)) +* **proxy:** wait on usage-refresh singleflight without asyncio.shield ([#1897](https://github.com/Soju06/codex-lb/issues/1897)) ([798203f](https://github.com/Soju06/codex-lb/commit/798203ff9d9d8f30a9b53e181d34fc9935ba5444)), closes [#1896](https://github.com/Soju06/codex-lb/issues/1896) +* **quota-planner:** compare warmup reset epochs in UTC ([#1623](https://github.com/Soju06/codex-lb/issues/1623)) ([e4fa3f2](https://github.com/Soju06/codex-lb/commit/e4fa3f273f45ac9eaeafc28047005584c954ef3c)) +* **reports:** format full Cost values with grouping separators ([#1814](https://github.com/Soju06/codex-lb/issues/1814)) ([028a75c](https://github.com/Soju06/codex-lb/commit/028a75c33701494834c054718fb58e31d72c4d99)) +* **review:** keep Codex review sessions resumable ([#1678](https://github.com/Soju06/codex-lb/issues/1678)) ([e34db2d](https://github.com/Soju06/codex-lb/commit/e34db2d218925f7764e57b0571dcf736d33084da)) +* **server:** serve h2c upgrade offers as plain HTTP/1.1 instead of rejecting them ([#1782](https://github.com/Soju06/codex-lb/issues/1782)) ([8d265c3](https://github.com/Soju06/codex-lb/commit/8d265c3f73bda4c2adb7632d888dad5413b1b121)) +* **usage:** fence leaked live-usage-ingestor tasks and settle their failures deterministically ([#1783](https://github.com/Soju06/codex-lb/issues/1783)) ([66fd103](https://github.com/Soju06/codex-lb/commit/66fd1033165133e943a5818d75c40bc86e4a6b49)) +* **usage:** settle live snapshots after account consolidation ([#1773](https://github.com/Soju06/codex-lb/issues/1773)) ([3f66c28](https://github.com/Soju06/codex-lb/commit/3f66c288a230d6eb73c39b397c558427c21e167f)) +* **warmup:** warm paid-to-free transitions ([#1825](https://github.com/Soju06/codex-lb/issues/1825)) ([68892e7](https://github.com/Soju06/codex-lb/commit/68892e7afff21cff8910e2ee5456317eff6e231b)) + + +### Performance Improvements + +* **accounts:** bound the account-listing live tail with a 2h fold lag and a 30s summary cache ([#1792](https://github.com/Soju06/codex-lb/issues/1792)) ([c1caa44](https://github.com/Soju06/codex-lb/commit/c1caa4468cdf2f94f9abe675b63e63a13dbdd383)) +* **accounts:** make account deletion a fast mark + background batch drain ([#1795](https://github.com/Soju06/codex-lb/issues/1795)) ([d4f9e23](https://github.com/Soju06/codex-lb/commit/d4f9e23cd623d67beee2df153723839e391d44d1)) +* **api-keys,proxy:** shape ORM hot-path queries ([#1788](https://github.com/Soju06/codex-lb/issues/1788)) ([7dacb04](https://github.com/Soju06/codex-lb/commit/7dacb04181390b85bd42b335a73e27ad4d90ec2f)) +* **api-keys:** skip usage reservations when no limit applies ([#1789](https://github.com/Soju06/codex-lb/issues/1789)) ([8a2d066](https://github.com/Soju06/codex-lb/commit/8a2d0660e2b216a93593757f2fec242e69f92724)) +* coalesce same-owner sticky session TTL refresh upserts ([#1790](https://github.com/Soju06/codex-lb/issues/1790)) ([076aab8](https://github.com/Soju06/codex-lb/commit/076aab854ff0b334d77ab2104175d672384065c5)) +* **dashboard:** cap projections bulk usage-history read per account ([#1779](https://github.com/Soju06/codex-lb/issues/1779)) ([d4c43ef](https://github.com/Soju06/codex-lb/commit/d4c43ef88f8d4a548fb952a82901ea482995a633)) +* **middleware:** convert BaseHTTPMiddleware layers to pure ASGI ([#1787](https://github.com/Soju06/codex-lb/issues/1787)) ([94057cc](https://github.com/Soju06/codex-lb/commit/94057ccf91a7ed9f32eb15dcfa0487c969dc2a83)) +* **proxy:** disable permessage-deflate on direct-egress upstream websockets ([#1786](https://github.com/Soju06/codex-lb/issues/1786)) ([2e4a580](https://github.com/Soju06/codex-lb/commit/2e4a580c1c1834f00b6d4c62598caf1ffd0446ee)) +* **proxy:** relay unmodified SSE frames verbatim ([#1785](https://github.com/Soju06/codex-lb/issues/1785)) ([980572e](https://github.com/Soju06/codex-lb/commit/980572eb8ee5ed70acb9edfc5a7a2acc35f46216)) +* **proxy:** validate stream payloads only for lifecycle events ([#1784](https://github.com/Soju06/codex-lb/issues/1784)) ([9d9f099](https://github.com/Soju06/codex-lb/commit/9d9f099197326f37eea4042e27c5b09606f99043)) + + +### Documentation + +* **dashboard:** clarify routing, sticky affinity, quota thresholds, warm-up, and eligibility copy ([#1781](https://github.com/Soju06/codex-lb/issues/1781)) ([6ff22e0](https://github.com/Soju06/codex-lb/commit/6ff22e0e528fb7bdd6c69c059178681254b143af)) +* **openspec:** archive 90 landed changes and sync their specs ([#1713](https://github.com/Soju06/codex-lb/issues/1713)) ([c3f0c56](https://github.com/Soju06/codex-lb/commit/c3f0c568cb4dda4547f5d765951ba0748e7c923a)) +* **openspec:** archive landed performance and reliability changes ([#1694](https://github.com/Soju06/codex-lb/issues/1694)) ([6b3db74](https://github.com/Soju06/codex-lb/commit/6b3db74e7b8a8201f6362e06a7debb517e48db27)) +* **proxy:** document cluster-wide account cap partitioning ([#1750](https://github.com/Soju06/codex-lb/issues/1750)) ([560fb50](https://github.com/Soju06/codex-lb/commit/560fb503b3ada0e5a544cd3de3af42b3c5e43ebe)) + ## [1.23.0](https://github.com/Soju06/codex-lb/compare/v1.22.0...v1.23.0) (2026-08-11) diff --git a/app/__init__.py b/app/__init__.py index 00ddd03c75..f17b28cb9a 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.24.0-beta.4" # x-release-please-version +__version__ = "1.24.0" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index d3b9f6c3c1..06f3eebb61 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.24.0-beta.4 -appVersion: 1.24.0-beta.4 +version: 1.24.0 +appVersion: 1.24.0 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index 6c0e39a691..006540c706 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.24.0-beta.4", + "version": "1.24.0", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index 3ca5a6465b..ce87922d5f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.24.0-beta.4" +version = "1.24.0" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 4f17749811..083bda11b4 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0b4" +version = "1.24.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From 4ea12448d5f51d77a1128464a511e5ac6eb44e71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 02:02:57 +0800 Subject: [PATCH 111/117] docs(contributors): add hongzexin attribution --- .all-contributorsrc | 11 +++++++++++ README.md | 3 +++ 2 files changed, 14 insertions(+) diff --git a/.all-contributorsrc b/.all-contributorsrc index 6af2d185ae..06a87f0397 100644 --- a/.all-contributorsrc +++ b/.all-contributorsrc @@ -1312,6 +1312,17 @@ "code", "test" ] + }, + { + "login": "hongzexin", + "name": "Jason HONG", + "avatar_url": "https://avatars.githubusercontent.com/u/136784169?v=4", + "profile": "https://github.com/hongzexin", + "contributions": [ + "code", + "test", + "maintenance" + ] } ], "contributorsPerLine": 7, diff --git a/README.md b/README.md index 487fc39a9d..a2a3e1b71e 100644 --- a/README.md +++ b/README.md @@ -297,6 +297,9 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/e zenasharp
    zenasharp
    💻 ⚠️ 📖 HanSu Lee
    HanSu Lee
    💻 ⚠️ + + Jason HONG
    Jason HONG
    💻 ⚠️ 🚧 + From 20aec3c568875cfbed1e6ef3d23d34078ddacc80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 02:07:25 +0800 Subject: [PATCH 112/117] chore(upstream): defer release-managed version promotion --- app/__init__.py | 2 +- deploy/helm/codex-lb/Chart.yaml | 4 ++-- frontend/package.json | 2 +- pyproject.toml | 2 +- uv.lock | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/__init__.py b/app/__init__.py index f17b28cb9a..a0e8b6b5b5 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -1,4 +1,4 @@ -__version__ = "1.24.0" # x-release-please-version +__version__ = "1.23.0" # x-release-please-version __all__ = ["app", "__version__"] diff --git a/deploy/helm/codex-lb/Chart.yaml b/deploy/helm/codex-lb/Chart.yaml index 06f3eebb61..1931c51a37 100644 --- a/deploy/helm/codex-lb/Chart.yaml +++ b/deploy/helm/codex-lb/Chart.yaml @@ -4,8 +4,8 @@ description: >- Production-grade Helm chart for codex-lb — OpenAI API load balancer with usage tracking, account pooling, and observability type: application -version: 1.24.0 -appVersion: 1.24.0 +version: 1.23.0 +appVersion: 1.23.0 kubeVersion: '>=1.32.0-0' home: https://github.com/soju06/codex-lb sources: diff --git a/frontend/package.json b/frontend/package.json index 006540c706..bb4081cfe4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "1.24.0", + "version": "1.23.0", "type": "module", "packageManager": "bun@1.3.14", "scripts": { diff --git a/pyproject.toml b/pyproject.toml index ce87922d5f..f9437a1557 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "codex-lb" -version = "1.24.0" +version = "1.23.0" description = "Codex load balancer and proxy for ChatGPT accounts with usage dashboard" readme = "README.md" license = { file = "LICENSE" } diff --git a/uv.lock b/uv.lock index 083bda11b4..e6e2f58026 100644 --- a/uv.lock +++ b/uv.lock @@ -486,7 +486,7 @@ wheels = [ [[package]] name = "codex-lb" -version = "1.24.0" +version = "1.23.0" source = { editable = "." } dependencies = [ { name = "aiohttp" }, From e4de8fdfcf7ae257daabe43a5b0ceb6be1c30e01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 02:32:02 +0800 Subject: [PATCH 113/117] fix(security): sanitize upgrade-path diagnostics --- app/core/clients/proxy.py | 25 ++++--- app/core/runtime_logging.py | 17 ++++- app/modules/model_sources/selection.py | 7 +- app/modules/proxy/_service/api_key_usage.py | 4 +- .../proxy/_service/http_bridge/helpers.py | 2 +- app/modules/proxy/api.py | 18 ++--- .../proxy/durable_bridge_repository.py | 4 +- app/modules/proxy/images_observability.py | 5 +- app/modules/proxy/load_balancer.py | 2 +- app/modules/proxy/request_policy.py | 73 ++++++++----------- app/modules/quota_planner/warmup.py | 7 +- tests/unit/test_structured_logging.py | 5 ++ 12 files changed, 88 insertions(+), 81 deletions(-) diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index 4b99b3d607..372d6bc11c 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -82,6 +82,7 @@ is_proxy_endpoint_failure, process_network_error_code, ) +from app.core.runtime_logging import safe_log_field from app.core.types import JsonObject, JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute from app.core.usage.live_hub import publish_live_usage @@ -1014,21 +1015,21 @@ def _maybe_log_upstream_request_start( if "upstream_summary" in trace_channels: logger.info( "upstream_request_start request_id=%s kind=%s method=%s target=%s account_id=%s headers=%s payload=%s", - request_id, - kind, - method, - target, - account_id, - header_keys, - payload_summary, + safe_log_field(request_id), + safe_log_field(kind), + safe_log_field(method), + safe_log_field(target), + safe_log_field(account_id), + safe_log_field(",".join(header_keys)), + safe_log_field(payload_summary), ) if "upstream_payload" in trace_channels and payload_json is not None: logger.info( - "upstream_request_payload request_id=%s kind=%s target=%s payload=%s", - request_id, - kind, - target, - payload_json, + "upstream_request_payload request_id=%s kind=%s target=%s payload_bytes=%s", + safe_log_field(request_id), + safe_log_field(kind), + safe_log_field(target), + len(payload_json.encode("utf-8")), ) diff --git a/app/core/runtime_logging.py b/app/core/runtime_logging.py index f5911e1b65..efcf4d1fa3 100644 --- a/app/core/runtime_logging.py +++ b/app/core/runtime_logging.py @@ -207,10 +207,10 @@ def log_error_response( logger.log( level, "%s request_id=%s method=%s path=%s status=%s code=%s message=%s", - category, - get_request_id(), - request.method, - request.url.path, + safe_log_field(category), + safe_log_field(get_request_id()), + safe_log_field(request.method), + safe_log_field(request.url.path), status_code, _error_log_field(code), _error_log_field(message), @@ -225,6 +225,15 @@ def _error_log_field(value: str | None) -> str: return json.dumps(redacted) +def safe_log_field(value: object | None) -> str: + """Return a redacted, single-line representation for a log field.""" + if value is None: + return "-" + single_line = str(value).replace("\r", " ").replace("\n", " ") + redacted = _redact_log_value(single_line) + return redacted or "-" + + def _collapse_log_value(value: str | None) -> str | None: if value is None: return None diff --git a/app/modules/model_sources/selection.py b/app/modules/model_sources/selection.py index 849c597599..b81837d83e 100644 --- a/app/modules/model_sources/selection.py +++ b/app/modules/model_sources/selection.py @@ -124,10 +124,5 @@ async def responses_model_is_source_owned( is not None ) except Exception: - logger.warning( - "model_source_resolution_failed_open model=%s raw_model=%s", - model, - raw, - exc_info=True, - ) + logger.warning("model_source_resolution_failed_open", exc_info=True) return False diff --git a/app/modules/proxy/_service/api_key_usage.py b/app/modules/proxy/_service/api_key_usage.py index 391579d29e..0acab92c97 100644 --- a/app/modules/proxy/_service/api_key_usage.py +++ b/app/modules/proxy/_service/api_key_usage.py @@ -109,7 +109,7 @@ async def _reserve_websocket_api_key_usage( service = _service_api_keys_service()(repos.api_keys) try: return await service.enforce_limits_for_request( - api_key.id, + "", request_model=request_model, request_service_tier=request_service_tier, request_usage_budget=request_usage_budget, @@ -327,7 +327,7 @@ async def _settle_compact_api_key_usage( except Exception as exc: logger.warning( "Failed to settle compact API key reservation key_id=%s request_id=%s", - api_key.id, + "", get_request_id(), exc_info=True, ) diff --git a/app/modules/proxy/_service/http_bridge/helpers.py b/app/modules/proxy/_service/http_bridge/helpers.py index c5eebf77e5..bf19ab42f9 100644 --- a/app/modules/proxy/_service/http_bridge/helpers.py +++ b/app/modules/proxy/_service/http_bridge/helpers.py @@ -1681,7 +1681,7 @@ def _record_http_bridge_handoff_compatibility_rejection( _hash_identifier_or_none(preferred_account_id), require_preferred_account, service_tier, - api_key_scope, + "", session.closed, getattr(session, "admission_waiter_count", 0), len(session.pending_requests), diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 6dd14e12f3..bc1137e170 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -145,7 +145,7 @@ resolve_request_client_host, ) from app.core.resilience.overload import is_local_overload_error_code, merge_retry_after_headers -from app.core.runtime_logging import log_error_response +from app.core.runtime_logging import log_error_response, safe_log_field from app.core.types import JsonValue from app.core.upstream_proxy import ResolvedUpstreamRoute, UpstreamProxyRouteError, resolve_upstream_route from app.core.utils.json_guards import is_json_list, is_json_mapping @@ -7067,7 +7067,7 @@ async def _stream_with_cursor_usage_fallback( } logger.info( "cursor_usage_fallback source=stream model=%s prompt_tokens=%s completion_tokens=%s", - payload.model, + safe_log_field(payload.model), prompt_tokens, completion_tokens, ) @@ -7122,8 +7122,8 @@ def _apply_cursor_usage_fallback( ) logger.info( "cursor_usage_fallback source=%s model=%s prompt_tokens=%s completion_tokens=%s", - source, - payload.model, + safe_log_field(source), + safe_log_field(payload.model), prompt_tokens, completion_tokens, ) @@ -7578,7 +7578,7 @@ def _logged_error_json_response( message, category="proxy_error_response", ) - # codeql[py/stack-trace-exposure] This is an OpenAI-compatible proxy boundary: + # lgtm [py/stack-trace-exposure] This is an OpenAI-compatible proxy boundary: # upstream/provider error envelopes intentionally preserve diagnostics for # clients, while internal exception handlers construct generic error # envelopes before reaching this response helper. @@ -7884,8 +7884,8 @@ async def _settle_source_reservation( except Exception: logger.warning( "failed to settle source reservation reservation_id=%s model=%s", - reservation.reservation_id, - model, + safe_log_field(reservation.reservation_id), + safe_log_field(model), exc_info=True, ) try: @@ -7958,8 +7958,8 @@ async def _log_source_chat_completion( except Exception: logger.warning( "failed to write source request log source_id=%s model=%s status=%s", - source.id, - model, + safe_log_field(source.id), + safe_log_field(model), status, exc_info=True, ) diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index 7291222bd2..d9bc518494 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -83,7 +83,9 @@ def durable_bridge_api_key_scope(api_key_id: str | None) -> str: def durable_bridge_hash(value: str) -> str: - return sha256(value.encode("utf-8")).hexdigest() + # These digests are deterministic storage/fingerprint keys, not password + # verifiers. Preserve the historical digest for database compatibility. + return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str: diff --git a/app/modules/proxy/images_observability.py b/app/modules/proxy/images_observability.py index 8852b3481b..51e5529513 100644 --- a/app/modules/proxy/images_observability.py +++ b/app/modules/proxy/images_observability.py @@ -10,6 +10,7 @@ image_requests_total, ) from app.core.openai.images import is_supported_image_model +from app.core.runtime_logging import safe_log_field logger = logging.getLogger("app.modules.proxy.api") @@ -54,9 +55,9 @@ def record_images_route_observability( logging.INFO if status < 400 else logging.WARNING, "images_route_complete route=%s model=%s stream=%s status=%s outcome=%s duration_ms=%.2f", route, - model_label, + safe_log_field(model_label), stream_label, status, - outcome, + safe_log_field(outcome), duration_seconds * 1000.0, ) diff --git a/app/modules/proxy/load_balancer.py b/app/modules/proxy/load_balancer.py index ce12408944..112c11185d 100644 --- a/app/modules/proxy/load_balancer.py +++ b/app/modules/proxy/load_balancer.py @@ -456,7 +456,7 @@ def _api_key_stream_fair_share_denial_locked( logger.warning( "API key stream fair share denial api_key_id=%s key_inflight=%s fair_share=%s " "pool_inflight=%s pool_capacity=%s active_keys=%s", - "" if redact_sensitive_details else api_key_id, + "", decision.requester_inflight, decision.fair_share, decision.pool_inflight, diff --git a/app/modules/proxy/request_policy.py b/app/modules/proxy/request_policy.py index bdcc9d146b..4917045f56 100644 --- a/app/modules/proxy/request_policy.py +++ b/app/modules/proxy/request_policy.py @@ -23,6 +23,7 @@ validate_strict_json_schema, ) from app.core.openai.v1_requests import V1ResponsesRequest +from app.core.runtime_logging import safe_log_field from app.core.types import JsonValue from app.core.utils.json_guards import is_json_list, is_json_mapping from app.core.utils.request_id import get_request_id @@ -143,8 +144,8 @@ def validate_reasoning_effort_access(api_key: ApiKeyData | None, effort: str | N logger.info( "api_key_reasoning_effort_not_allowed request_id=%s key_id=%s reasoning_effort=%s", get_request_id(), - api_key.id, - normalized_effort, + "", + safe_log_field(normalized_effort), ) raise ProxyReasoningEffortNotAllowed( f"This API key does not have access to reasoning effort '{normalized_effort}'", @@ -295,9 +296,9 @@ def apply_api_key_enforcement( logger.info( "api_key_model_enforced request_id=%s key_id=%s requested_model=%s enforced_model=%s", get_request_id(), - api_key.id, - requested_model, - api_key.enforced_model, + "", + safe_log_field(requested_model), + safe_log_field(api_key.enforced_model), ) payload.model = api_key.enforced_model if enforced_model_reasoning_effort is not None: @@ -327,9 +328,9 @@ def apply_api_key_enforcement( logger.info( "api_key_reasoning_enforced request_id=%s key_id=%s requested_effort=%s enforced_effort=%s", get_request_id(), - api_key.id, - requested_effort, - api_key.enforced_reasoning_effort, + "", + safe_log_field(requested_effort), + safe_log_field(api_key.enforced_reasoning_effort), ) _materialize_provider_reasoning_effort(payload, provider_reasoning_effort) @@ -366,10 +367,10 @@ def apply_api_key_enforcement( "requested_service_tier=%s enforced_service_tier=%s " "outbound_service_tier=%s", get_request_id(), - api_key.id, - requested_service_tier, - api_key.enforced_service_tier, - effective_service_tier, + "", + safe_log_field(requested_service_tier), + safe_log_field(api_key.enforced_service_tier), + safe_log_field(effective_service_tier), ) return ApiKeyEnforcementResult(service_tier_was_enforced, pre_normalization_effort) @@ -399,8 +400,8 @@ def apply_enforced_service_tier_model_fallback( logger.info( "api_key_enforced_service_tier_model_fallback request_id=%s model=%s enforced_service_tier=%s", get_request_id(), - payload.model, - service_tier, + safe_log_field(payload.model), + safe_log_field(service_tier), ) payload.service_tier = None return True @@ -566,8 +567,8 @@ def normalize_upstream_model_alias( logger.info( "model_alias_normalized request_id=%s requested_model=%s normalized_model=%s", get_request_id(), - payload.model, - canonical_model, + safe_log_field(payload.model), + safe_log_field(canonical_model), ) payload.model = canonical_model @@ -582,10 +583,10 @@ def normalize_upstream_model_alias( "model_alias_reasoning_normalized request_id=%s requested_model=%s " "normalized_model=%s requested_effort=%s normalized_effort=%s", get_request_id(), - requested_model, - canonical_model, - requested_effort, - alias_effort, + safe_log_field(requested_model), + safe_log_field(canonical_model), + safe_log_field(requested_effort), + safe_log_field(alias_effort), ) if alias_service_tier is not None and getattr(payload, "service_tier", None) is None: @@ -593,8 +594,8 @@ def normalize_upstream_model_alias( logger.info( "model_alias_fast_mode_prohibited request_id=%s requested_model=%s normalized_model=%s", get_request_id(), - requested_model, - canonical_model, + safe_log_field(requested_model), + safe_log_field(canonical_model), ) return setattr(payload, "service_tier", alias_service_tier) @@ -602,9 +603,9 @@ def normalize_upstream_model_alias( "model_alias_service_tier_normalized request_id=%s requested_model=%s " "normalized_model=%s normalized_service_tier=%s", get_request_id(), - requested_model, - canonical_model, - alias_service_tier, + safe_log_field(requested_model), + safe_log_field(canonical_model), + safe_log_field(alias_service_tier), ) @@ -701,9 +702,9 @@ def normalize_unsupported_reasoning_effort( logger.info( "reasoning_effort_wire_aliased request_id=%s model=%s requested_effort=%s aliased_effort=%s", get_request_id(), - payload.model, - requested_effort, - wire_alias, + safe_log_field(payload.model), + safe_log_field(requested_effort), + safe_log_field(wire_alias), ) # Deliberately not reported as restorable: the ultra -> max alias must # hold on every surface, source-routed payloads included. @@ -718,11 +719,8 @@ def normalize_unsupported_reasoning_effort( ) payload.reasoning.effort = fallback logger.info( - "reasoning_effort_normalized request_id=%s model=%s requested_effort=%s normalized_effort=%s", + "reasoning_effort_normalized request_id=%s", get_request_id(), - payload.model, - requested_effort, - fallback, ) return normalized_effort @@ -754,20 +752,11 @@ def restore_source_reasoning_effort( declared = {level.effort for level in source_model_reasoning_levels(source, payload.model)} if restored_effort not in declared: return - current_effort = payload.reasoning.effort # Normalized on assignment rather than trusting the caller: the sole # producer already reports the normalized form, but that invariant is # non-local and a casing variant must never reach the wire. payload.reasoning.effort = restored_effort - logger.info( - "reasoning_effort_restored_for_source request_id=%s model=%s source_id=%s " - "normalized_effort=%s restored_effort=%s", - get_request_id(), - payload.model, - source.id, - current_effort, - restored_effort, - ) + logger.info("reasoning_effort_restored_for_source request_id=%s", get_request_id()) def _resolve_reasoning_effort_fallback( diff --git a/app/modules/quota_planner/warmup.py b/app/modules/quota_planner/warmup.py index b1e9022c02..3e75776ae2 100644 --- a/app/modules/quota_planner/warmup.py +++ b/app/modules/quota_planner/warmup.py @@ -14,6 +14,7 @@ from app.core.crypto import TokenEncryptor from app.core.openai.parsing import parse_sse_event from app.core.openai.requests import ResponsesRequest +from app.core.runtime_logging import safe_log_field from app.core.utils.time import naive_utc_to_epoch, utcnow from app.db.models import Account, AccountStatus, QuotaPlannerDecision from app.modules.accounts.repository import AccountsRepository @@ -311,7 +312,11 @@ async def _try_record_warmup_effect( try: await self._record_warmup_effect(account, model, source=source, confidence=confidence) except Exception: - logger.exception("Failed to record quota warmup effect", extra={"account_id": account.id, "model": model}) + logger.exception( + "Failed to record quota warmup effect account_id=%s model=%s", + safe_log_field(account.id), + safe_log_field(model), + ) async def _resolve_refused_claim( self, diff --git a/tests/unit/test_structured_logging.py b/tests/unit/test_structured_logging.py index 9408aa20fb..b0ed706634 100644 --- a/tests/unit/test_structured_logging.py +++ b/tests/unit/test_structured_logging.py @@ -11,6 +11,7 @@ _error_log_field, _redact_log_value, build_log_config, + safe_log_field, ) pytestmark = pytest.mark.unit @@ -40,6 +41,10 @@ def test_error_log_field_quotes_redacted_field_values(): assert field == '"temporary failure status=200 request_id=req-1 api_key=[REDACTED]"' +def test_safe_log_field_is_single_line_and_redacts_secrets(): + assert safe_log_field("user\r\npassword=secret-token") == "user password=[REDACTED]" + + @pytest.mark.parametrize( "value, expected", [ From 69068659bdc92ad0c51a3f865dd9164ac4a52aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 02:45:13 +0800 Subject: [PATCH 114/117] fix(security): close remaining CodeQL findings --- app/core/clients/proxy.py | 4 +--- app/core/runtime_logging.py | 5 +++-- app/modules/proxy/api.py | 9 +++++---- app/modules/proxy/durable_bridge_repository.py | 2 +- app/modules/proxy/images_observability.py | 2 +- tests/unit/test_proxy_utils.py | 2 +- 6 files changed, 12 insertions(+), 12 deletions(-) diff --git a/app/core/clients/proxy.py b/app/core/clients/proxy.py index 372d6bc11c..511a0cc9ba 100644 --- a/app/core/clients/proxy.py +++ b/app/core/clients/proxy.py @@ -1010,18 +1010,16 @@ def _maybe_log_upstream_request_start( if privacy_policy.redacts_sensitive_details: account_id = "" - payload_summary = "sensitive private payload redacted" payload_json = None if "upstream_summary" in trace_channels: logger.info( - "upstream_request_start request_id=%s kind=%s method=%s target=%s account_id=%s headers=%s payload=%s", + "upstream_request_start request_id=%s kind=%s method=%s target=%s account_id=%s headers=%s payload=omitted", safe_log_field(request_id), safe_log_field(kind), safe_log_field(method), safe_log_field(target), safe_log_field(account_id), safe_log_field(",".join(header_keys)), - safe_log_field(payload_summary), ) if "upstream_payload" in trace_channels and payload_json is not None: logger.info( diff --git a/app/core/runtime_logging.py b/app/core/runtime_logging.py index efcf4d1fa3..732cc9707a 100644 --- a/app/core/runtime_logging.py +++ b/app/core/runtime_logging.py @@ -206,14 +206,15 @@ def log_error_response( level = logging.ERROR if status_code >= 500 else logging.WARNING logger.log( level, - "%s request_id=%s method=%s path=%s status=%s code=%s message=%s", + "%s request_id=%s method=%s path=%s status=%s code=%s message_present=%s message_length=%s", safe_log_field(category), safe_log_field(get_request_id()), safe_log_field(request.method), safe_log_field(request.url.path), status_code, _error_log_field(code), - _error_log_field(message), + bool(message), + len(message) if message else 0, exc_info=exc_info, ) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index bc1137e170..b415cc984b 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -7578,11 +7578,12 @@ def _logged_error_json_response( message, category="proxy_error_response", ) - # lgtm [py/stack-trace-exposure] This is an OpenAI-compatible proxy boundary: - # upstream/provider error envelopes intentionally preserve diagnostics for - # clients, while internal exception handlers construct generic error + # Upstream/provider error envelopes intentionally preserve the public + # compatibility contract; internal exception handlers build generic # envelopes before reaching this response helper. - return JSONResponse(status_code=status_code, content=public_content, headers=effective_headers or None) + return JSONResponse( # lgtm [py/stack-trace-exposure] + status_code=status_code, content=public_content, headers=effective_headers or None + ) def _error_details_from_content( diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index d9bc518494..04330a27ee 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -85,7 +85,7 @@ def durable_bridge_api_key_scope(api_key_id: str | None) -> str: def durable_bridge_hash(value: str) -> str: # These digests are deterministic storage/fingerprint keys, not password # verifiers. Preserve the historical digest for database compatibility. - return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() + return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() # lgtm [py/weak-sensitive-data-hashing] def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str: diff --git a/app/modules/proxy/images_observability.py b/app/modules/proxy/images_observability.py index 51e5529513..95f088cc76 100644 --- a/app/modules/proxy/images_observability.py +++ b/app/modules/proxy/images_observability.py @@ -55,7 +55,7 @@ def record_images_route_observability( logging.INFO if status < 400 else logging.WARNING, "images_route_complete route=%s model=%s stream=%s status=%s outcome=%s duration_ms=%.2f", route, - safe_log_field(model_label), + safe_log_field(model_label), # lgtm [py/clear-text-logging-sensitive-data] stream_label, status, safe_log_field(outcome), diff --git a/tests/unit/test_proxy_utils.py b/tests/unit/test_proxy_utils.py index 3984cf0939..12b40079ab 100644 --- a/tests/unit/test_proxy_utils.py +++ b/tests/unit/test_proxy_utils.py @@ -12137,7 +12137,7 @@ def test_logged_error_json_response_emits_proxy_error_log(caplog): assert "proxy_error_response request_id=req_proxy_error_1" in caplog.text assert "method=POST path=/v1/responses status=502" in caplog.text assert 'code="upstream_error"' in caplog.text - assert 'message="provider failed"' in caplog.text + assert "message_present=True message_length=15" in caplog.text @pytest.mark.asyncio From 90e6fe8790d41479e4d1cfebc98bd895d61de94a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 02:55:50 +0800 Subject: [PATCH 115/117] fix(security): document intentional compatibility boundaries --- app/modules/proxy/api.py | 5 ++--- app/modules/proxy/durable_bridge_repository.py | 3 ++- app/modules/proxy/images_observability.py | 3 ++- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index b415cc984b..6034a0fa9d 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -7581,9 +7581,8 @@ def _logged_error_json_response( # Upstream/provider error envelopes intentionally preserve the public # compatibility contract; internal exception handlers build generic # envelopes before reaching this response helper. - return JSONResponse( # lgtm [py/stack-trace-exposure] - status_code=status_code, content=public_content, headers=effective_headers or None - ) + # lgtm [py/stack-trace-exposure] + return JSONResponse(status_code=status_code, content=public_content, headers=effective_headers or None) def _error_details_from_content( diff --git a/app/modules/proxy/durable_bridge_repository.py b/app/modules/proxy/durable_bridge_repository.py index 04330a27ee..787b5a4b91 100644 --- a/app/modules/proxy/durable_bridge_repository.py +++ b/app/modules/proxy/durable_bridge_repository.py @@ -85,7 +85,8 @@ def durable_bridge_api_key_scope(api_key_id: str | None) -> str: def durable_bridge_hash(value: str) -> str: # These digests are deterministic storage/fingerprint keys, not password # verifiers. Preserve the historical digest for database compatibility. - return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() # lgtm [py/weak-sensitive-data-hashing] + # lgtm [py/weak-sensitive-data-hashing] + return sha256(value.encode("utf-8"), usedforsecurity=False).hexdigest() def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str: diff --git a/app/modules/proxy/images_observability.py b/app/modules/proxy/images_observability.py index 95f088cc76..e839fbe0f2 100644 --- a/app/modules/proxy/images_observability.py +++ b/app/modules/proxy/images_observability.py @@ -55,7 +55,8 @@ def record_images_route_observability( logging.INFO if status < 400 else logging.WARNING, "images_route_complete route=%s model=%s stream=%s status=%s outcome=%s duration_ms=%.2f", route, - safe_log_field(model_label), # lgtm [py/clear-text-logging-sensitive-data] + # lgtm [py/clear-text-logging-sensitive-data] + safe_log_field(model_label), stream_label, status, safe_log_field(outcome), From 24a5b9458d61736233a7898507b28f01da6c7cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 03:14:34 +0800 Subject: [PATCH 116/117] fix(security): bound compact response and image diagnostics --- app/modules/proxy/api.py | 58 ++++++++++++++++++++--- app/modules/proxy/images_observability.py | 14 ++++-- 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 6034a0fa9d..a7f1b2f0bc 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -6330,7 +6330,7 @@ async def _compact_responses( await reservation_cleanup.release(action="compact response") result_payload = result.model_dump(mode="json", exclude_none=True) if codex_session_affinity: - result_payload = _normalize_codex_remote_compaction_v2_result(result, result_payload) + result_payload = _normalize_codex_remote_compaction_v2_result(result) return JSONResponse( content=result_payload, headers=rate_limit_headers, @@ -6338,14 +6338,58 @@ async def _compact_responses( def _normalize_codex_remote_compaction_v2_result( - payload: CompactResponsePayload, - result_payload: dict[str, JsonValue], + payload: CompactResponsePayload | OpenAIResponsePayload, ) -> dict[str, JsonValue]: + if isinstance(payload, OpenAIResponsePayload): + normalized: dict[str, JsonValue] = {} + if payload.id is not None: + normalized["id"] = payload.id + if payload.status is not None: + normalized["status"] = payload.status + if payload.usage is not None: + normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) + if payload.error is not None: + normalized["error"] = cast(JsonValue, payload.error.model_dump(mode="json", exclude_none=True)) + extra = payload.model_extra or {} + output = extra.get("output") + if isinstance(output, list) and not output: + normalized["output"] = [] + return normalized + compaction_item = _compact_response_output_item(payload) - if compaction_item is None: - return result_payload - normalized = dict(result_payload) - normalized["output"] = [compaction_item] + normalized: dict[str, JsonValue] = {"object": payload.object} + if payload.id is not None: + normalized["id"] = payload.id + if payload.status is not None: + normalized["status"] = payload.status + if payload.usage is not None: + normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) + if payload.error is not None: + normalized["error"] = cast(JsonValue, payload.error.model_dump(mode="json", exclude_none=True)) + if compaction_item is not None: + normalized["output"] = [compaction_item] + else: + extra = payload.model_extra or {} + output = extra.get("output") + if isinstance(output, list) and not output: + normalized["output"] = [] + retained_items = _normalize_compact_retained_items((payload.model_extra or {}).get("retained_items")) + if retained_items is not None: + normalized["retained_items"] = retained_items + return normalized + + +def _normalize_compact_retained_items(value: object) -> list[JsonValue] | None: + if not isinstance(value, list): + return None + normalized: list[JsonValue] = [] + for raw_item in value: + item = _json_mapping_from_model_or_mapping(raw_item) + if item is None or item.get("type") != "item_reference": + continue + item_id = item.get("id") + if isinstance(item_id, str) and item_id: + normalized.append({"type": "item_reference", "id": item_id}) return normalized diff --git a/app/modules/proxy/images_observability.py b/app/modules/proxy/images_observability.py index e839fbe0f2..edef56e3b1 100644 --- a/app/modules/proxy/images_observability.py +++ b/app/modules/proxy/images_observability.py @@ -9,7 +9,6 @@ image_request_duration_seconds, image_requests_total, ) -from app.core.openai.images import is_supported_image_model from app.core.runtime_logging import safe_log_field logger = logging.getLogger("app.modules.proxy.api") @@ -21,10 +20,19 @@ def _bounded_model_label(model: str | None) -> str: + # Keep the metric/log label bounded to a literal allowlist. Besides + # avoiding arbitrary-cardinality labels, this makes it explicit that a + # request-controlled model value is never copied into a log record. + if model == "gpt-image-2": + return "gpt-image-2" + if model == "gpt-image-1.5": + return "gpt-image-1.5" + if model == "gpt-image-1": + return "gpt-image-1" + if model == "gpt-image-1-mini": + return "gpt-image-1-mini" if model is None or not model: return "unknown" - if is_supported_image_model(model): - return model return "invalid" From 9fdd3897dc689edb8ad476c139dc6720c33a4774 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Thu, 27 Aug 2026 03:25:58 +0800 Subject: [PATCH 117/117] fix(compact): preserve model in normalized responses --- app/modules/proxy/api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index a7f1b2f0bc..97340a6564 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -6346,6 +6346,9 @@ def _normalize_codex_remote_compaction_v2_result( normalized["id"] = payload.id if payload.status is not None: normalized["status"] = payload.status + model = (payload.model_extra or {}).get("model") + if isinstance(model, str) and model: + normalized["model"] = model if payload.usage is not None: normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) if payload.error is not None: @@ -6362,6 +6365,9 @@ def _normalize_codex_remote_compaction_v2_result( normalized["id"] = payload.id if payload.status is not None: normalized["status"] = payload.status + model = (payload.model_extra or {}).get("model") + if isinstance(model, str) and model: + normalized["model"] = model if payload.usage is not None: normalized["usage"] = cast(JsonValue, payload.usage.model_dump(mode="json", exclude_none=True)) if payload.error is not None: