From 6e2750a98c3335a8979c7cfdc3a1fb6f622eaf41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Fri, 28 Aug 2026 10:58:20 +0800 Subject: [PATCH 1/2] fix(proxy-responses): restart verified v1 goals after stale anchors Source sidechat: 01a045d1-ecf5-7840-b404-4c9b6098216d\nRecovered from: a347912b8334af52133bfc4436381fb78afb8888 --- .../proxy/_service/http_bridge/mixin.py | 23 +- .../proxy/_service/http_bridge/streaming.py | 24 +- app/modules/proxy/api.py | 36 +++ .../.openspec.yaml | 2 + .../design.md | 121 ++++++++ .../proposal.md | 73 +++++ .../specs/responses-api-compat/spec.md | 98 +++++++ .../tasks.md | 52 ++++ .../integration/test_http_responses_bridge.py | 277 +++++++++++++++++- tests/unit/test_proxy_api_websocket_auth.py | 123 ++++++++ tests/unit/test_proxy_http_bridge.py | 7 +- 11 files changed, 819 insertions(+), 17 deletions(-) create mode 100644 openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/.openspec.yaml create mode 100644 openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/design.md create mode 100644 openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/proposal.md create mode 100644 openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/specs/responses-api-compat/spec.md create mode 100644 openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md diff --git a/app/modules/proxy/_service/http_bridge/mixin.py b/app/modules/proxy/_service/http_bridge/mixin.py index adeb42abd0..f74aa46b7f 100644 --- a/app/modules/proxy/_service/http_bridge/mixin.py +++ b/app/modules/proxy/_service/http_bridge/mixin.py @@ -364,6 +364,7 @@ async def _get_or_create_http_bridge_session( exclude_account_ids: Collection[str] | None = None, deferred_account_backoff_lifecycle: _DeferredAccountBackoffLifecycle | None = None, defer_account_health_writes: bool = False, + force_new_turn_state_session: bool = False, ) -> "_HTTPBridgeSession": ... @overload async def _get_or_create_http_bridge_session( @@ -397,6 +398,7 @@ async def _get_or_create_http_bridge_session( exclude_account_ids: Collection[str] | None = None, deferred_account_backoff_lifecycle: _DeferredAccountBackoffLifecycle | None = None, defer_account_health_writes: bool = False, + force_new_turn_state_session: bool = False, ) -> "_HTTPBridgeSession | _HTTPBridgeOwnerForward": ... async def _get_or_create_http_bridge_session( self, @@ -429,6 +431,7 @@ async def _get_or_create_http_bridge_session( exclude_account_ids: Collection[str] | None = None, deferred_account_backoff_lifecycle: _DeferredAccountBackoffLifecycle | None = None, defer_account_health_writes: bool = False, + force_new_turn_state_session: bool = False, ) -> "_HTTPBridgeSession | _HTTPBridgeOwnerForward": settings = _service_get_settings() request_scope_id = ensure_request_scope_id() @@ -678,14 +681,16 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: key=key.affinity_key, ): key = _HTTPBridgeSessionKey("turn_state_header", incoming_turn_state, api_key_id) - elif ( - fallback_key := _alias_fallback_key(incoming_session_key, initial_session_key, api_key_id) - ) is not None: - key = fallback_key - used_session_header_fallback = True else: - key = _HTTPBridgeSessionKey("turn_state_header", incoming_turn_state, api_key_id) - missing_turn_state_alias = True + fallback_key = ( + _alias_fallback_key(incoming_session_key, initial_session_key, api_key_id) + if not force_new_turn_state_session + else None + ) + key = fallback_key or _HTTPBridgeSessionKey( + "turn_state_header", incoming_turn_state, api_key_id + ) + missing_turn_state_alias = not (used_session_header_fallback := fallback_key is not None) pruned_sessions = self._prune_http_bridge_sessions_locked() if pruned_sessions: if any(session.key == key for session in pruned_sessions): @@ -1269,6 +1274,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: incoming_turn_state is not None and incoming_turn_state.startswith("http_turn_") and not allow_forward_to_owner + and not force_new_turn_state_session ): _record_continuity_fail_closed( surface="http_bridge", @@ -1560,8 +1566,7 @@ def bind_account_neutral_recovery_owner(session: _HTTPBridgeSession) -> None: # restart_takeover means recovering a row whose previous # owner is genuinely gone. Every claim now advances the # epoch, so epoch > 1 alone would also count ordinary - # local successor claims (no pre-claim lookup, or a - # forced replace of a live local session). + # local successor claims (no pre-claim lookup, or a forced replace of a live local session). claim_kwargs["record_restart_takeover"] = True await self._claim_durable_http_bridge_session(created_session, **claim_kwargs) async with self._http_bridge_lock: diff --git a/app/modules/proxy/_service/http_bridge/streaming.py b/app/modules/proxy/_service/http_bridge/streaming.py index 7965bc3722..ce78e9dd4f 100644 --- a/app/modules/proxy/_service/http_bridge/streaming.py +++ b/app/modules/proxy/_service/http_bridge/streaming.py @@ -893,6 +893,7 @@ def stream_http_responses( enforce_openai_sdk_contract: bool = True, capacity_startup_wait_event: asyncio.Event | None = None, capacity_startup_ready_event: asyncio.Event | None = None, + verified_v1_goal_restart: bool = False, ) -> AsyncIterator[str]: _maybe_log_proxy_request_payload("stream_http", payload, headers) proxy_api_authorization = _header_value_case_insensitive(headers, "authorization") @@ -918,6 +919,7 @@ def stream_http_responses( enforce_openai_sdk_contract=enforce_openai_sdk_contract, capacity_startup_wait_event=capacity_startup_wait_event, capacity_startup_ready_event=capacity_startup_ready_event, + verified_v1_goal_restart=verified_v1_goal_restart, ) async def _stream_http_bridge_or_retry( @@ -943,6 +945,7 @@ async def _stream_http_bridge_or_retry( enforce_openai_sdk_contract: bool = True, capacity_startup_wait_event: asyncio.Event | None = None, capacity_startup_ready_event: asyncio.Event | None = None, + verified_v1_goal_restart: bool = False, ) -> AsyncIterator[str]: dashboard_settings = await _service_get_settings_cache().get() runtime_config = _http_bridge_runtime_config(dashboard_settings, _service_get_settings()) @@ -1028,6 +1031,7 @@ async def _stream_http_bridge_or_retry( enforce_openai_sdk_contract=enforce_openai_sdk_contract, capacity_startup_wait_event=capacity_startup_wait_event, capacity_startup_ready_event=capacity_startup_ready_event, + verified_v1_goal_restart=verified_v1_goal_restart, deferred_account_backoff_tracker=deferred_account_backoff_tracker, ): yield line @@ -1091,6 +1095,7 @@ async def _stream_via_http_bridge( enforce_openai_sdk_contract: bool = True, capacity_startup_wait_event: asyncio.Event | None = None, capacity_startup_ready_event: asyncio.Event | None = None, + verified_v1_goal_restart: bool = False, deferred_account_backoff_tracker: _DeferredAccountBackoffTracker | None = None, ) -> AsyncIterator[str]: del suppress_text_done_events @@ -2063,6 +2068,7 @@ def switch_to_account_neutral_replay( exclude_account_ids=fresh_replay_excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, + force_new_turn_state_session=verified_v1_goal_restart, ) except ProxyResponseError as exc: if not owner_unavailable_allows_account_neutral_replay(exc): @@ -2337,6 +2343,7 @@ def switch_to_account_neutral_replay( exclude_account_ids=request_state.excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, + force_new_turn_state_session=verified_v1_goal_restart, ) except ProxyResponseError as capacity_exc: if owner_unavailable_allows_account_neutral_replay(capacity_exc): @@ -2994,6 +3001,7 @@ def capture_verified_stale_anchor_quarantine_generation( exclude_account_ids=request_state.excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, + force_new_turn_state_session=verified_v1_goal_restart, ) except ProxyResponseError as capacity_exc: wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) @@ -3100,6 +3108,7 @@ def capture_verified_stale_anchor_quarantine_generation( exclude_account_ids=request_state.excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, + force_new_turn_state_session=verified_v1_goal_restart, ) except ProxyResponseError as capacity_exc: wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) @@ -3402,6 +3411,7 @@ def capture_verified_stale_anchor_quarantine_generation( exclude_account_ids=request_state.excluded_account_ids or None, deferred_account_backoff_lifecycle=request_state.deferred_account_backoff_lifecycle, defer_account_health_writes=request_state.api_key_reservation is not None, + force_new_turn_state_session=verified_v1_goal_restart, ) except ProxyResponseError as capacity_exc: wait_plan = _http_bridge_capacity_wait_plan(capacity_exc, request_deadline=request_deadline) @@ -4524,7 +4534,10 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: and request_state.error_http_status_override is not None and request_state.error_http_status_override >= 400 ): - if request_state.previous_response_not_found_rewritten: + if request_state.previous_response_not_found_rewritten and not ( + request_state.proxy_injected_previous_response_id + and _http_bridge_continuity_bound_without_safe_replay(request_state) + ): raise ProxyResponseError( request_state.error_http_status_override, openai_error( @@ -4532,10 +4545,11 @@ def stream_idle_keepalive(*, downstream_response_id: str) -> str | None: "Upstream websocket closed before response.completed", ), ) - raise ProxyResponseError( - request_state.error_http_status_override, - _openai_error_envelope_from_response_failed_payload(block_payload), - ) + if not request_state.previous_response_not_found_rewritten: + raise ProxyResponseError( + request_state.error_http_status_override, + _openai_error_envelope_from_response_failed_payload(block_payload), + ) yield event_block yielded_any = True finally: diff --git a/app/modules/proxy/api.py b/app/modules/proxy/api.py index 97340a6564..baa46055c9 100644 --- a/app/modules/proxy/api.py +++ b/app/modules/proxy/api.py @@ -698,6 +698,36 @@ def _has_explicit_openai_sdk_marker(request: Request) -> bool: return "openai" in user_agent +def _verified_v1_goal_restart_headers( + request: Request, + payload: ResponsesRequest, +) -> dict[str, str] | None: + """Rotate a proved native Goal restart away from its echoed HTTP turn.""" + if payload.stream is not True: + return None + if not _is_native_codex_request(request.headers) or _has_explicit_openai_sdk_marker(request): + return None + if proxy_affinity_module._codex_backend_identity(request.headers).thread_id is None: + return None + turn_state = proxy_affinity_module._sticky_key_from_turn_state_header(request.headers) + if ( + turn_state is None + or not turn_state.startswith("http_turn_") + or not proxy_affinity_module._is_synthesized_turn_state(turn_state) + ): + return None + if not proxy_affinity_module._request_allows_unavailable_legacy_owner_abandonment(payload): + return None + + forwarded_headers = dict(request.headers) + forwarded_headers["x-codex-turn-state"] = proxy_affinity_module.ensure_http_downstream_turn_state({}) + logger.info( + "v1_goal_restart_turn_state_rotated request_id=%s", + ensure_request_id(), + ) + return forwarded_headers + + def _is_openai_sdk_request( request: Request, payload: V1ResponsesRequest | Mapping[str, JsonValue] | None = None, @@ -1303,6 +1333,7 @@ async def v1_responses( service_tier_was_enforced=service_tier_was_enforced, ) if responses_payload.stream: + goal_restart_headers = _verified_v1_goal_restart_headers(request, responses_payload) response = await _stream_responses( request, responses_payload, @@ -1313,6 +1344,8 @@ async def v1_responses( prefer_http_bridge=True, api_key_policy_already_applied=True, prohibit_fast_mode=prohibit_fast_mode, + forwarded_headers=goal_restart_headers, + verified_v1_goal_restart=goal_restart_headers is not None, ) else: response = await _collect_responses( @@ -5541,6 +5574,7 @@ async def _stream_responses( native_codex_heartbeat: bool = False, api_key_policy_already_applied: bool = False, prohibit_fast_mode: bool = False, + verified_v1_goal_restart: bool = False, ) -> Response: # Owner-forwarded payloads have already passed API-key enforcement, # account-catalog fallback, reservation, and signing on the origin @@ -5779,6 +5813,7 @@ def build_response_stream() -> AsyncIterator[str]: enforce_openai_sdk_contract=enforce_openai_sdk_contract, capacity_startup_wait_event=capacity_wait_event, capacity_startup_ready_event=capacity_ready_event, + verified_v1_goal_restart=verified_v1_goal_restart, ) return context.service.stream_responses( payload, @@ -5832,6 +5867,7 @@ async def _retry() -> AsyncIterator[str]: enforce_openai_sdk_contract=enforce_openai_sdk_contract, capacity_startup_wait_event=capacity_wait_event, capacity_startup_ready_event=capacity_ready_event, + verified_v1_goal_restart=verified_v1_goal_restart, ) async for line in retry_stream: yield line diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/.openspec.yaml b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/.openspec.yaml new file mode 100644 index 0000000000..7f2cf9bc02 --- /dev/null +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-08-28 diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/design.md b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/design.md new file mode 100644 index 0000000000..ebc96662f1 --- /dev/null +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/design.md @@ -0,0 +1,121 @@ +## Context + +The production failure sequence used `/v1/responses` with an LB-generated +`http_turn_*` value and a durable thread session whose latest upstream anchor +was stale. The incoming automatic Goal request contained two or three +developer/user messages, no assistant output, no tool calls, and no file or +image state. It did not match the stored 13-item upstream prefix, so #1863 +correctly rejected it as a durable full-history replay. + +The Goal payload is nevertheless an explicit product-level restart. It carries +``, names the active objective, and says +that current worktree/external state is authoritative. The repository already +separates restart intent from replay safety: + +- `responses_request_contains_goal_continuation_context` proves intent. +- `responses_payload_is_account_neutral_fresh_replay` proves that the canonical + request carries no prior response, conversation, file/image, account-scoped + input, unresolved tool dependency, or unknown wire field. + +The remaining `/v1` problem is provenance. A marker alone is client-controlled, +and an arbitrary `x-codex-turn-state` must remain hard continuity. The route +therefore also requires the existing native Codex fingerprint, a logical +`thread-id`, and the exact LB-generated HTTP turn-state shape. + +## Goals / Non-Goals + +**Goals:** + +- Let a verified native Codex Goal restart establish a new continuation + without reading or modifying the stale durable anchor. +- Return a new turn state so later Desktop retries and turns naturally follow + the replacement continuation. +- Preserve ordinary delta-only fail-closed behavior and exactly-once fences. +- Stop the deterministic same-anchor local rebind after an explicit + `previous_response_not_found` rejection. +- Produce a structured startup error or a valid terminal SSE event instead of + an abnormal 200 stream disconnect. + +**Non-Goals:** + +- Reconstructing missing conversation history for arbitrary delta-only + requests. +- Deleting, clearing, rebinding, or migrating old durable sessions, aliases, + anchors, operations, journals, or retry circuits. +- Allowing files, images, conversations, prior responses, or account-bound + tool history to cross accounts. +- Changing WebSocket-native canonical error-code behavior or backporting the + unrelated source-SSE normalization change. +- Adding a setting or globally changing sticky/continuity behavior. + +## Decisions + +### 1. Verify the restart at the `/v1` boundary + +The route grants the escape only when all of the following are true: + +1. `stream=true`. +2. The request has the existing native Codex user-agent/originator proof and + no explicit OpenAI SDK fingerprint. +3. A nonblank logical `thread-id` is present. +4. The echoed turn state matches `http_turn_[0-9a-f]{32}`. +5. The canonical request contains the exact Goal marker. +6. The existing account-neutral fresh-replay classifier passes. + +The final classifier already rejects `previous_response_id`, `conversation`, +file/image state, unknown item types, account-scoped metadata, and unresolved +tool history. The route does not duplicate those rules. + +### 2. Rotate the turn state; do not mutate the old session + +For the one verified request, copy the effective headers and replace only +`x-codex-turn-state` with a newly synthesized HTTP turn state. The ordinary +bridge path then creates a fresh hard turn key and performs a normal unanchored +first dispatch. The response already echoes the effective turn state, so no +new protocol field is needed. + +This choice is smaller than adding a second recovery transaction and safer +than clearing the old anchor. The old durable row and aliases remain available +for audit or code rollback, while the new turn state is independently +addressable. + +### 3. Deliver the terminal event instead of raising it into local rebind + +The upstream-event path has already sanitized the stale response id, rewritten +the downstream response identity, settled the request, and persisted a terminal +`response.failed`. When the bridge consumer sees that specific rewritten event +with `propagate_http_errors=true`, it yields the event instead of raising +`bridge_previous_response_not_found`. + +If it arrives during the route startup probe, the public route can return a +structured non-200 error before headers. If it arrives after the probe window, +the public stream normalizer produces a valid terminal SSE sequence. Because +no exception reaches `_stream_via_http_bridge`, the same-anchor +`previous_response_recover_local` branch is not entered. + +Other first-event HTTP errors keep their existing startup behavior. + +## Risks / Trade-offs + +- **Risk:** A generic client forges the Goal XML. **Mitigation:** the marker is + only one of six independent predicates; native fingerprint, logical thread + identity, LB-generated turn-state shape, streaming mode, and the canonical + account-neutral classifier are also required. +- **Risk:** A client-supplied turn state happens to match the synthesized + shape. **Mitigation:** native Codex and `thread-id` proofs are additionally + required; every foreign/non-synthesized value remains hard-bound. +- **Risk:** A valid stale delta now ends immediately instead of getting one + local reconnect. **Mitigation:** an explicit `previous_response_not_found` + proves the same anchor cannot succeed on that upstream path, while unsafe + unanchored replay remains forbidden. Removing the identical redispatch + reduces load without weakening continuity. +- **Trade-off:** A failure that arrives inside the startup probe may be JSON + rather than SSE. Both forms are valid because headers have not been + committed; after commitment the contract requires and emits a terminal SSE + event. + +## Rollback + +Revert the code and deploy the prior immutable image. No data migration or +cleanup is required. Old durable state was never deleted or overwritten by the +escape path. diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/proposal.md b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/proposal.md new file mode 100644 index 0000000000..a988794577 --- /dev/null +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/proposal.md @@ -0,0 +1,73 @@ +# Recover verified `/v1` Goal restarts from a stale durable anchor + +Source sidechat: `01a045d1-ecf5-7840-b404-4c9b6098216d` + +Recovered from: `a347912b8334af52133bfc4436381fb78afb8888` + +## Why + +Codex Desktop custom providers use streaming `/v1/responses`. After the HTTP +bridge has returned an LB-generated `x-codex-turn-state`, Desktop echoes that +turn state on later requests. A synthetic persistent-Goal restart is a +self-contained request whose own instructions explicitly make the worktree and +external state authoritative, but `/v1` currently treats the echoed turn state +as hard continuity, resolves the old durable session, and injects its stale +`previous_response_id`. + +The deployed stale-anchor recovery correctly refuses to remove that anchor +when the incoming request is only a delta relative to the stored upstream +history. It therefore reports `fresh_replay_available=false`. The residual bug +is that the verified Goal restart never reaches a fresh bridge even though the +existing Goal marker and account-neutral replay classifier independently prove +that this particular request may start a new continuation. + +The same failure also exposes a terminal-delivery defect. An explicit upstream +`previous_response_not_found` is already rewritten and durably persisted as a +sanitized `response.failed`, but the public HTTP bridge raises that event as a +proxy exception, performs one local rebind with the same rejected anchor, and +can let the second exception escape after HTTP 200 SSE headers are committed. +Desktop then sees a disconnected stream instead of a deterministic terminal +failure. + +## What Changes + +- Recognize a streaming `/v1/responses` Goal restart only when the request is + from a native Codex client, carries a logical thread identity, echoes an + LB-synthesized `http_turn_<32 lowercase hex>` value, contains the exact Goal + marker, and passes the existing account-neutral self-contained replay + classifier. +- Replace the echoed stale turn state with a new LB-generated turn state for + that one verified request. This creates a fresh unanchored bridge through the + normal request path and returns the new turn state to the client without + deleting or overwriting the old durable session, aliases, anchor, or + operation ledger. +- Keep marker-only, generic `/v1`, non-streaming, foreign turn-state, + previous-response, conversation, file/image, and account-scoped requests on + the existing fail-closed path. +- When an explicit stale-anchor rejection has no verified replay, deliver the + already-sanitized terminal failure instead of performing a same-anchor local + rebind. Before headers this may become a structured startup error; after SSE + starts it remains a valid terminal `response.failed` event. +- Keep the stored anchor and durable session unchanged on this failure path. + +## Capabilities + +### New Capabilities + +- None. + +### Modified Capabilities + +- `responses-api-compat`: Adds a proof-gated fresh-continuation contract for + native Codex Goal restarts on streaming `/v1/responses`, and requires an + explicit stale-anchor rejection without a safe replay to terminate cleanly + without a same-anchor redispatch. + +## Impact + +- Public `/v1/responses` route classification and effective turn-state headers. +- Responses HTTP bridge terminal delivery for explicit stale-anchor rejection. +- Product-path integration tests for fresh Goal restart and fail-closed + terminal delivery. +- No schema, migration, setting, deployment prerequisite, dashboard, or + destructive data operation. diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/specs/responses-api-compat/spec.md b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/specs/responses-api-compat/spec.md new file mode 100644 index 0000000000..947d12ed36 --- /dev/null +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/specs/responses-api-compat/spec.md @@ -0,0 +1,98 @@ +## ADDED Requirements + +### Requirement: Verified native `/v1` Goal restarts establish a fresh continuation + +A streaming `/v1/responses` request MAY abandon its echoed HTTP turn-state +continuation only when the request is proven to be a native Codex Goal restart. +The proof MUST require the native Codex client fingerprint, a logical thread +identity, an echoed LB-synthesized `http_turn_<32 lowercase hex>` value, the +exact Goal-continuation marker, absence of an explicit OpenAI SDK fingerprint, +and successful classification by the existing account-neutral self-contained +fresh-replay predicate. + +For a proved request, the proxy MUST assign a new LB-synthesized HTTP turn +state and dispatch the request once without the old durable +`previous_response_id`. The response MUST return the new turn state, and a +follow-up that echoes it MUST continue on the new bridge. The proxy MUST NOT +delete, clear, overwrite, or rebind the old durable session, aliases, anchor, +operation ledger, recovery journal, or retry circuit in order to authorize the +restart. + +A marker without every other proof, a generic `/v1` client, a non-streaming +request, a missing logical thread identity, a foreign turn state, or a request +with previous-response, conversation, file/image, account-scoped, unknown, or +unresolved tool state MUST retain the ordinary continuity and fail-closed +behavior. + +#### Scenario: Verified Desktop Goal restart rotates away from a stale durable turn + +- **GIVEN** a native Codex Desktop `/v1/responses` stream echoes an + LB-synthesized HTTP turn state whose durable session contains a stale anchor +- **AND** the request carries a logical thread identity and the exact Goal + marker +- **AND** the canonical request is an account-neutral self-contained fresh + replay +- **WHEN** the request is admitted +- **THEN** the proxy assigns and returns a different LB-synthesized HTTP turn + state +- **AND** it dispatches the Goal request once without the old anchor +- **AND** the old durable session, aliases, anchor, and operations remain + unchanged +- **AND** a follow-up using the returned turn state continues on the new bridge + +#### Scenario: Goal marker with account-scoped state cannot rotate continuity + +- **GIVEN** a `/v1/responses` request contains the exact Goal marker +- **AND** it also contains a previous response, conversation, file/image, + account-scoped field, unknown input shape, or unresolved tool dependency +- **WHEN** the request is classified +- **THEN** the request MUST NOT receive a fresh turn state by this contract +- **AND** existing owner and continuity rules remain authoritative + +#### Scenario: Foreign or generic `/v1` clients cannot claim Goal restart authority + +- **GIVEN** a request lacks the native Codex fingerprint or logical thread + identity, or carries a turn state outside the LB-synthesized HTTP shape +- **WHEN** it includes a Goal-like marker +- **THEN** the marker MUST NOT authorize anchor bypass or fresh continuation +- **AND** the supplied turn state remains ordinary client continuity input + +### Requirement: Unreplayable stale HTTP continuations terminate without identical redispatch + +When the Responses HTTP bridge receives an explicit +`previous_response_not_found` rejection for a request that has no verified safe +fresh replay, the proxy MUST retain the durable anchor and MUST NOT dispatch a +second request carrying that same rejected anchor. The already-sanitized +failure MUST be delivered as a structured startup error before HTTP headers are +committed or as a valid terminal `response.failed` event after SSE streaming +has begun. + +The failure MUST use the current downstream response identity, MUST NOT expose +the raw upstream error envelope or stale response id, and MUST leave the old +durable session, aliases, operation ledger, recovery journal, and retry circuit +intact. Verified full-history recovery defined by the existing stale-anchor +contract remains unchanged. + +#### Scenario: Delta-only stale rejection ends once and preserves the anchor + +- **GIVEN** the bridge injects a durable previous-response anchor into a + delta-only request +- **AND** upstream explicitly rejects that anchor as not found before emitting + visible output +- **AND** no verified fresh replay is available +- **WHEN** the terminal failure is processed +- **THEN** exactly one anchored upstream dispatch occurs +- **AND** no `previous_response_recover_local` same-anchor redispatch occurs +- **AND** the client receives a structured startup error or valid terminal SSE + failure instead of an abnormal stream disconnect +- **AND** the old durable anchor and operation settlement remain intact + +#### Scenario: Verified full resend still uses the existing fenced recovery + +- **GIVEN** an explicit stale-anchor rejection has a verified durable full + resend and all required operation-fence and spool-reset proofs +- **WHEN** the existing stale-anchor recovery runs +- **THEN** this terminal-delivery requirement MUST NOT suppress its one bounded + unanchored replay +- **AND** all existing account-neutral, same-owner, file-pin, tool-settlement, + circuit-generation, and exactly-once constraints remain authoritative diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md new file mode 100644 index 0000000000..35ae566182 --- /dev/null +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md @@ -0,0 +1,52 @@ +## 1. Contract + +- [x] 1.1 Record the proof-gated `/v1` Goal restart and terminal-delivery + behavior in OpenSpec. +- [x] 1.2 Define negative cases for generic clients, foreign turn states, + missing thread identity, and account-scoped payloads. + +## 2. Regression coverage + +- [x] 2.1 Add a production-shape `/v1/responses` integration test that proves + a verified native Goal restart rotates turn state, dispatches unanchored + once, completes, and leaves the old durable session/anchor untouched. +- [x] 2.2 Prove a follow-up using the returned turn state continues on the new + bridge. +- [x] 2.3 Add negative product-path cases for missing marker, generic client, + missing thread identity, foreign turn state, client previous response, + conversation, and file/image/account-scoped input. +- [x] 2.4 Add an explicit stale-rejection test that proves one upstream + dispatch, no same-anchor local rebind, one terminal failure, and an + unchanged durable anchor. + +## 3. Implementation + +- [x] 3.1 Add the narrow `/v1` verified Goal-restart predicate and rotate only + its effective HTTP turn state. +- [x] 3.2 Deliver the already-rewritten stale-anchor terminal event instead of + raising it into the local same-anchor recovery branch. + +## 4. Validation + +- [x] 4.1 Run strict OpenSpec validation. +- [x] 4.2 Run focused integration and unit bridge tests. +- [ ] 4.3 Run Ruff format/check, `ty` on changed code, proxy architecture + checks, and PostgreSQL durable-session integration coverage. +- [x] 4.4 Verify the final diff preserves old durable state and introduces no + schema or setting change. + +> PostgreSQL durable-session coverage for 4.3 is still blocked locally: the +> project allowlisted test was run with a real `postgresql+asyncpg` URL, but +> `127.0.0.1:5432` refused the connection and no local PostgreSQL/Docker +> runtime is available. SQLite results are not counted as PostgreSQL evidence. + +## 5. Delivery + +- [ ] 5.1 Commit with sidechat and upstream-recovery traceability, push, and + open a focused PR. +- [ ] 5.2 Satisfy current-head CI, review, and mergeability gates; merge to + `origin/main`. +- [ ] 5.3 Build/promote an immutable image through GitOps and verify production + source SHA, digest, health, and migration head. +- [ ] 5.4 Run one no-tool verified Goal probe only after production readback + proves the new predicate and old-anchor bypass. diff --git a/tests/integration/test_http_responses_bridge.py b/tests/integration/test_http_responses_bridge.py index 2839b0b954..77fb6475e9 100644 --- a/tests/integration/test_http_responses_bridge.py +++ b/tests/integration/test_http_responses_bridge.py @@ -729,6 +729,34 @@ async def send_text(self, text: str) -> None: ) +class _CompleteThenPreviousResponseNotFoundUpstreamWebSocket(_FakeBridgeUpstreamWebSocket): + async def send_text(self, text: str) -> None: + if not self.sent_text: + await super().send_text(text) + return + self.sent_text.append(text) + payload = json.loads(text) + previous_response_id = payload.get("previous_response_id") + await self._messages.put( + _FakeUpstreamMessage( + "text", + text=json.dumps( + { + "type": "error", + "status": 400, + "error": { + "type": "invalid_request_error", + "code": "previous_response_not_found", + "message": f"Previous response with id '{previous_response_id}' not found.", + "param": "previous_response_id", + }, + }, + separators=(",", ":"), + ), + ) + ) + + class _PreviousResponseNotFoundAfterOutputUpstreamWebSocket(_PreviousResponseNotFoundUpstreamWebSocket): async def send_text(self, text: str) -> None: await self._messages.put( @@ -13801,7 +13829,155 @@ async def fake_connect_responses_websocket( @pytest.mark.asyncio -async def test_v1_responses_http_bridge_rebinds_after_upstream_previous_response_not_found( +async def test_v1_responses_verified_goal_restart_rotates_turn_state_and_preserves_stale_bridge( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_v1_goal_restart", + "http-bridge-v1-goal-restart@example.com", + ) + account = await _get_account(account_id) + stale_upstream = _CompleteThenPreviousResponseNotFoundUpstreamWebSocket("resp_v1_goal_stale") + replacement_upstream = _FakeBridgeUpstreamWebSocket("resp_v1_goal_replacement") + connected_upstreams: list[_FakeBridgeUpstreamWebSocket] = [] + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + upstream = stale_upstream if not connected_upstreams else replacement_upstream + connected_upstreams.append(upstream) + return upstream + + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_account_with_budget", + AsyncMock(return_value=AccountSelection(account=account, error_message=None, error_code=None)), + ) + monkeypatch.setattr( + proxy_module.ProxyService, + "_ensure_fresh_with_budget", + AsyncMock(return_value=account), + ) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + stale_turn_state = f"http_turn_{'a' * 32}" + request_headers = { + "user-agent": "codex_cli_rs/0.145.0", + "originator": "codex_cli_rs", + "x-codex-session-id": "v1-goal-restart-process", + "thread-id": "v1-goal-restart-thread", + "x-codex-turn-state": stale_turn_state, + } + first_events, first_headers = await _collect_sse_events_with_headers( + async_client, + "/v1/responses", + headers=request_headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": "prime the durable continuation", + "stream": True, + }, + ) + _assert_created_text_delta_completed(first_events) + assert first_headers["x-codex-turn-state"] == stale_turn_state + + service = get_proxy_service_for_app(app_instance) + async with service._http_bridge_lock: + stale_session = next( + session + for session in service._http_bridge_sessions.values() + if stale_turn_state in session.downstream_turn_state_aliases + ) + stale_response_id = stale_session.last_completed_response_id + assert stale_response_id == first_events[-1]["response"]["id"] + stale_lookup_before = await service._durable_bridge.lookup_request_targets( + session_key_kind=stale_session.key.affinity_kind, + session_key_value=stale_session.key.affinity_key, + api_key_id=None, + turn_state=stale_turn_state, + session_header=request_headers["x-codex-session-id"], + previous_response_id=stale_response_id, + ) + assert stale_lookup_before is not None + + goal_events, goal_headers = await _collect_sse_events_with_headers( + async_client, + "/v1/responses", + headers=request_headers, + json_body={ + "model": "gpt-5.1", + "instructions": ('\nContinue working toward the active thread goal.'), + "input": [ + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + }, + ) + + _assert_created_text_delta_completed(goal_events) + replacement_turn_state = goal_headers["x-codex-turn-state"] + assert replacement_turn_state != stale_turn_state + assert replacement_turn_state.startswith("http_turn_") + assert len(replacement_turn_state) == len("http_turn_") + 32 + assert len(connected_upstreams) == 2 + assert len(stale_upstream.sent_text) == 1 + assert len(replacement_upstream.sent_text) == 1 + replacement_payload = json.loads(replacement_upstream.sent_text[0]) + assert "previous_response_id" not in replacement_payload + assert stale_session.last_completed_response_id == stale_response_id + assert stale_session.closed is False + + stale_lookup_after = await service._durable_bridge.lookup_request_targets( + session_key_kind=stale_session.key.affinity_kind, + session_key_value=stale_session.key.affinity_key, + api_key_id=None, + turn_state=stale_turn_state, + session_header=request_headers["x-codex-session-id"], + previous_response_id=stale_response_id, + ) + assert stale_lookup_after is not None + assert stale_lookup_after.session_id == stale_lookup_before.session_id + assert stale_lookup_after.owner_epoch == stale_lookup_before.owner_epoch + assert stale_lookup_after.latest_response_id == stale_lookup_before.latest_response_id + + follow_up_headers = {**request_headers, "x-codex-turn-state": replacement_turn_state} + goal_input = [ + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ] + follow_up_events, follow_up_response_headers = await _collect_sse_events_with_headers( + async_client, + "/v1/responses", + headers=follow_up_headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": [ + *goal_input, + {"role": "user", "content": [{"type": "input_text", "text": "continue on the replacement bridge"}]}, + ], + "stream": True, + }, + ) + _assert_created_text_delta_completed(follow_up_events) + assert follow_up_response_headers["x-codex-turn-state"] == replacement_turn_state + assert len(connected_upstreams) == 2 + assert len(replacement_upstream.sent_text) == 2 + assert json.loads(replacement_upstream.sent_text[1])["previous_response_id"] == (goal_events[-1]["response"]["id"]) + + +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_terminates_after_upstream_previous_response_not_found( async_client, app_instance, monkeypatch, @@ -13916,6 +14092,105 @@ async def fake_connect_responses_websocket( assert connect_count == 2 +@pytest.mark.asyncio +async def test_v1_responses_http_bridge_stream_terminates_proxy_injected_stale_anchor_once( + async_client, + app_instance, + monkeypatch, +): + _install_bridge_settings(monkeypatch, enabled=True) + account_id = await _import_account( + async_client, + "acc_http_bridge_previous_response_stream_terminal", + "http-bridge-previous-response-stream-terminal@example.com", + ) + account = await _get_account(account_id) + stale_upstream = _CompleteThenPreviousResponseNotFoundUpstreamWebSocket("resp_stream_terminal_stale") + recovered_upstream = _FakeBridgeUpstreamWebSocket("resp_stream_terminal_recovered") + connect_count = 0 + + async def fake_connect_responses_websocket( + headers, + access_token, + account_id_header, + *, + base_url=None, + session=None, + ): + del headers, access_token, account_id_header, base_url, session + nonlocal connect_count + connect_count += 1 + return stale_upstream if connect_count == 1 else recovered_upstream + + monkeypatch.setattr( + proxy_module.ProxyService, + "_select_account_with_budget", + AsyncMock(return_value=AccountSelection(account=account, error_message=None, error_code=None)), + ) + monkeypatch.setattr( + proxy_module.ProxyService, + "_ensure_fresh_with_budget", + AsyncMock(return_value=account), + ) + monkeypatch.setattr(proxy_module, "connect_responses_websocket", fake_connect_responses_websocket) + + turn_state = f"http_turn_{'b' * 32}" + headers = {"x-codex-turn-state": turn_state} + historical_input = [ + { + "role": "user", + "content": [{"type": "input_text", "text": "prime the stale anchor"}], + } + ] + first_events = await _collect_sse_events( + async_client, + "/v1/responses", + headers=headers, + json_body={ + "model": "gpt-5.1", + "instructions": "Return exactly OK.", + "input": historical_input, + "stream": True, + }, + ) + _assert_created_text_delta_completed(first_events) + first_response_id = first_events[-1]["response"]["id"] + + service = get_proxy_service_for_app(app_instance) + async with service._http_bridge_lock: + bridge_session = next( + session + for session in service._http_bridge_sessions.values() + if turn_state in session.downstream_turn_state_aliases + ) + assert bridge_session.last_completed_response_id == first_response_id + + second = await async_client.post( + "/v1/responses", + headers=headers, + json={ + "model": "gpt-5.1", + "instructions": "Continue the existing task.", + "input": [ + *historical_input, + { + "role": "user", + "content": [{"type": "input_text", "text": "delta only"}], + }, + ], + "stream": True, + }, + ) + + assert second.status_code == 502 + assert second.json()["error"]["code"] == "stream_incomplete" + assert connect_count == 1 + assert len(stale_upstream.sent_text) == 2 + assert json.loads(stale_upstream.sent_text[1])["previous_response_id"] == first_response_id + assert len(recovered_upstream.sent_text) == 0 + assert bridge_session.last_completed_response_id == first_response_id + + @pytest.mark.parametrize( "replay_case", [ diff --git a/tests/unit/test_proxy_api_websocket_auth.py b/tests/unit/test_proxy_api_websocket_auth.py index 13517982fd..b765fb7288 100644 --- a/tests/unit/test_proxy_api_websocket_auth.py +++ b/tests/unit/test_proxy_api_websocket_auth.py @@ -811,3 +811,126 @@ def test_public_missing_tool_output_input_error_preserves_client_status(): assert error["code"] == "invalid_request_error" assert error["param"] == "input" assert "call_W3U0TC60cgB5OD7gVCyS0qIq" in masked.model_dump_json() + + +def _v1_goal_restart_request(headers: dict[str, str]) -> Request: + return Request( + { + "type": "http", + "method": "POST", + "path": "/v1/responses", + "headers": [(name.encode(), value.encode()) for name, value in headers.items()], + "client": ("127.0.0.1", 50001), + } + ) + + +def _v1_goal_restart_payload(**updates: object) -> ResponsesRequest: + payload: dict[str, object] = { + "model": "gpt-5.6-sol", + "instructions": "Continue the existing task.", + "input": [ + { + "role": "developer", + "content": '\nContinue working toward the active thread goal.', + }, + {"role": "user", "content": [{"type": "input_text", "text": "continue"}]}, + ], + "stream": True, + } + payload.update(updates) + return ResponsesRequest.model_validate(payload) + + +def test_verified_v1_goal_restart_headers_rotate_only_the_echoed_http_turn_state(): + old_turn_state = f"http_turn_{'a' * 32}" + request = _v1_goal_restart_request( + { + "user-agent": "codex_cli_rs/0.145.0", + "originator": "codex_cli_rs", + "thread-id": "verified-goal-thread", + "x-codex-session-id": "verified-goal-process", + "x-codex-turn-state": old_turn_state, + "x-request-trace": "keep-me", + } + ) + + forwarded_headers = proxy_api_module._verified_v1_goal_restart_headers( + request, + _v1_goal_restart_payload(), + ) + + assert forwarded_headers is not None + assert forwarded_headers["x-request-trace"] == "keep-me" + assert forwarded_headers["thread-id"] == "verified-goal-thread" + new_turn_state = forwarded_headers["x-codex-turn-state"] + assert new_turn_state != old_turn_state + assert new_turn_state.startswith("http_turn_") + assert len(new_turn_state) == len("http_turn_") + 32 + + +@pytest.mark.parametrize( + "rejected_case", + [ + "missing-goal-marker", + "generic-client", + "openai-sdk-marker", + "missing-thread-id", + "foreign-turn-state", + "non-http-synthesized-turn-state", + "client-previous-response", + "conversation", + "file-input", + "account-scoped-metadata", + "non-streaming", + ], +) +def test_verified_v1_goal_restart_headers_reject_unproved_or_account_scoped_requests(rejected_case: str): + headers = { + "user-agent": "codex_cli_rs/0.145.0", + "originator": "codex_cli_rs", + "thread-id": "verified-goal-thread", + "x-codex-session-id": "verified-goal-process", + "x-codex-turn-state": f"http_turn_{'b' * 32}", + } + payload_updates: dict[str, object] = {} + + if rejected_case == "missing-goal-marker": + payload_updates["input"] = "continue" + elif rejected_case == "generic-client": + headers["user-agent"] = "httpx/0.28" + headers.pop("originator") + elif rejected_case == "openai-sdk-marker": + headers["x-stainless-lang"] = "python" + elif rejected_case == "missing-thread-id": + headers.pop("thread-id") + elif rejected_case == "foreign-turn-state": + headers["x-codex-turn-state"] = "client-owned-turn-state" + elif rejected_case == "non-http-synthesized-turn-state": + headers["x-codex-turn-state"] = f"turn_{'c' * 32}" + elif rejected_case == "client-previous-response": + payload_updates["previous_response_id"] = "resp_client_previous" + elif rejected_case == "conversation": + payload_updates["conversation"] = "conv_account_scoped" + elif rejected_case == "file-input": + payload_updates["input"] = [ + { + "role": "developer", + "content": '\nContinue working toward the active thread goal.', + }, + {"type": "input_file", "file_id": "file_account_scoped"}, + ] + elif rejected_case == "account-scoped-metadata": + payload_updates["client_metadata"] = {"future_account_handle": "acct-a"} + elif rejected_case == "non-streaming": + payload_updates["stream"] = False + else: # pragma: no cover - the parameter list is exhaustive + raise AssertionError(rejected_case) + + assert ( + proxy_api_module._verified_v1_goal_restart_headers( + _v1_goal_restart_request(headers), + _v1_goal_restart_payload(**payload_updates), + ) + is None + ) diff --git a/tests/unit/test_proxy_http_bridge.py b/tests/unit/test_proxy_http_bridge.py index 6ec9806464..c3813cb0bc 100644 --- a/tests/unit/test_proxy_http_bridge.py +++ b/tests/unit/test_proxy_http_bridge.py @@ -17345,9 +17345,11 @@ async def fake_create_http_bridge_session( assert captured["key"] == key +@pytest.mark.parametrize("force_new_turn_state_session", [False, True]) @pytest.mark.asyncio async def test_get_or_create_http_bridge_session_falls_back_to_session_header_when_turn_state_alias_is_missing( monkeypatch: pytest.MonkeyPatch, + force_new_turn_state_session: bool, ) -> None: service = proxy_service.ProxyService(cast(Any, nullcontext())) requested_key = proxy_service._HTTPBridgeSessionKey("turn_state_header", "http_turn_generated", None) @@ -17424,12 +17426,13 @@ async def fake_create_http_bridge_session( request_model="gpt-5.4", idle_ttl_seconds=120.0, max_sessions=8, - previous_response_id="resp_prev_1", + previous_response_id=None if force_new_turn_state_session else "resp_prev_1", session_header_fallback_key=fallback_key, + force_new_turn_state_session=force_new_turn_state_session, ) assert resolved is created_session - assert captured["key"] == fallback_key + assert captured["key"] == (requested_key if force_new_turn_state_session else fallback_key) @pytest.mark.asyncio From 5fff712c1a817d73334676789758f5e2c03b37d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B4=AA=E6=B3=BD=E9=91=AB?= Date: Fri, 28 Aug 2026 11:03:28 +0800 Subject: [PATCH 2/2] docs(openspec): record stale-anchor PR delivery Source sidechat: 01a045d1-ecf5-7840-b404-4c9b6098216d Recovered from: a347912b8334af52133bfc4436381fb78afb8888 Record PR #48 creation and push evidence; leave CI, PostgreSQL, merge, deploy, and production probe tasks open. --- .../recover-v1-goal-restart-from-stale-durable-anchor/tasks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md index 35ae566182..3a6528b428 100644 --- a/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md +++ b/openspec/changes/recover-v1-goal-restart-from-stale-durable-anchor/tasks.md @@ -42,7 +42,7 @@ ## 5. Delivery -- [ ] 5.1 Commit with sidechat and upstream-recovery traceability, push, and +- [x] 5.1 Commit with sidechat and upstream-recovery traceability, push, and open a focused PR. - [ ] 5.2 Satisfy current-head CI, review, and mergeability gates; merge to `origin/main`.