From 849b0f077bfda43c52028a6f75ee2dc7be77577e Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Tue, 1 Sep 2026 14:49:12 +0200 Subject: [PATCH 1/4] feat(openrouter): allow pinning chat completions to named providers Provider routing existed only for embeddings; chat completions always used OpenRouter's ordinary marketplace routing, so a run could not be confined to a named set of hosts. That matters for benchmarking: two runs on the same model but different providers are not necessarily comparable, because quantization, context window and throughput all differ per host. Adds REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY (comma-separated or JSON list of provider slugs, e.g. z-ai,novita,deepinfra,gmicloud), mirroring the existing embedding parser. allow_fallbacks stays ON, which is not a loophole: 'only' already bounds the candidate pool, so failover moves between the listed providers and can never leave them. A long ingestion run would otherwise dead-letter a stage on one host's 5xx or 429. The allowlist is recorded in the selfhost attestation dict, because which hosts served a run is measurement provenance, not incidental configuration. Unset preserves today's behaviour exactly, so existing deployments are untouched. Tests cover the allowlist payload, the unset case, non-leakage into the embeddings path, and both env shapes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018eazkrM7j7scrwQUcUWbbS --- .env.example | 5 ++ compose.yaml | 1 + src/rememberstack/adapters/openrouter.py | 38 ++++++++ src/rememberstack/profiles/selfhost.py | 8 ++ src/tests/adapters/test_openrouter.py | 107 +++++++++++++++++++++++ 5 files changed, 159 insertions(+) diff --git a/.env.example b/.env.example index 99359f513..93fc97294 100644 --- a/.env.example +++ b/.env.example @@ -69,6 +69,11 @@ REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 # effort literals. Example: pin flash models to none while keeping high for a # reasoning model. # REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP={"z-ai/glm-4.7-flash":"none","openai/gpt-5.6-luna":"high"} + +# Restrict CHAT completions to named OpenRouter providers (allowlist of provider +# slugs, not quantization tags). `only` bounds the pool, so failover moves +# between these and can never leave them. Unset = ordinary marketplace routing. +# REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud # Debug only: retain raw schema-invalid completions under the private app-state # volume. Disabled by default because captured output may repeat source text. # REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/openrouter-invalid-completions diff --git a/compose.yaml b/compose.yaml index c899d803b..a7507805a 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,6 +21,7 @@ x-app: &app REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS: ${REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP:-} + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY: ${REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY:-} REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR: ${REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 6c0cad013..8b16e0aa6 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -115,6 +115,22 @@ class OpenRouterSettings(BaseSettings): endpoint labels; routing uses the base provider slug from the endpoints API. See ``design/operations/openrouter-embedding-routing.md``. """ + chat_provider_only: list[str] | None = None + """Allowlist of OpenRouter providers for CHAT completions (``provider.only``). + + Env: ``REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY`` as a comma-separated + list of *provider slugs* (not quantization tags): e.g. + ``z-ai,novita,deepinfra,gmicloud``. Tags like ``deepinfra/fp8`` are endpoint + labels; routing uses the base slug from the endpoints API. + + ``allow_fallbacks`` stays ON, which does not weaken the restriction: + ``only`` bounds the candidate pool, so fallback moves between the listed + providers and can never leave them. That matters for long ingestion runs, + where one host's 5xx or 429 would otherwise dead-letter a stage. + + Unset means OpenRouter's ordinary marketplace routing, which is the shipped + default and the behaviour every existing deployment keeps. + """ reasoning_effort: ReasoningEffort | None = None reasoning_effort_map: dict[str, ReasoningEffort] | None = None """Optional per-model effort overrides as a JSON object env var @@ -161,6 +177,12 @@ def parse_embedding_provider_order(cls, value: object) -> object: value=value, field_name="embedding_provider_order" ) + @field_validator("chat_provider_only", mode="before") + @classmethod + def parse_chat_provider_only(cls, value: object) -> object: + """Parse comma-separated or JSON list of OpenRouter provider slugs.""" + return _parse_provider_name_list(value=value, field_name="chat_provider_only") + @field_validator("max_completion_tokens", mode="before") @classmethod def default_empty_max_completion_tokens(cls, value: object) -> object: @@ -256,6 +278,9 @@ def generate( effort = self._reasoning_effort_for(request=request) if effort is not None: payload["reasoning"] = {"effort": effort} + provider = self._chat_provider_payload() + if provider is not None: + payload["provider"] = provider content, usage, body = self._completion_text( payload=payload, response_type=response_type, started_ns=started_ns @@ -434,6 +459,19 @@ def _completion_usage( " metadata did not recover it" ) from last_error + def _chat_provider_payload(self) -> dict[str, object] | None: + """Build OpenRouter provider routing for chat completions. + + ``only`` bounds the candidate pool to the configured slugs, so keeping + ``allow_fallbacks`` on lets a single host's 5xx/429 move to another + ALLOWED provider instead of dead-lettering the stage. It cannot route + outside the allowlist. + """ + allowed = self._settings.chat_provider_only + if not allowed: + return None + return {"only": list(allowed), "allow_fallbacks": True} + def _embedding_provider_payload(self) -> dict[str, object] | None: """Build OpenRouter provider routing for embedding requests. diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 01aee0c75..6105106d8 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -1341,6 +1341,14 @@ def _model_bindings() -> dict[str, str]: if openrouter.max_completion_tokens is not None else "unset" ), + # Which providers were allowed to serve chat completions is measurement + # provenance: two runs on the same model but different hosts are not + # necessarily comparable (quantization, context, throughput all differ). + "openrouter_chat_provider_only": ( + ",".join(openrouter.chat_provider_only) + if openrouter.chat_provider_only + else "unset" + ), "openrouter_reasoning_effort": openrouter.reasoning_effort or "auto", # Canonical (sorted-key) form so the effective per-model effort policy # is part of measurement provenance, not hidden behind the global pin. diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index a4d1fc3f0..bef87a537 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -967,6 +967,113 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert response.output.answer == "Prague" +def test_chat_provider_only_restricts_generation_to_the_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An allowlist bounds the pool; fallback may move only inside it.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings( + api_key="test-key", + chat_provider_only=["z-ai", "novita", "deepinfra", "gmicloud"], + ) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/chat/completions" + assert payload["provider"] == { + "only": ["z-ai", "novita", "deepinfra", "gmicloud"], + # ON is not a loophole: `only` already bounds the candidate pool, so + # this buys failover between allowed hosts, never outside them. + "allow_fallbacks": True, + } + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + response = provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + assert response.output.answer == "Prague" + + +def test_chat_provider_only_is_absent_when_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Existing deployments keep ordinary marketplace routing untouched.""" + provider = OpenRouterModelProvider(settings=OpenRouterSettings(api_key="test-key")) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert "provider" not in payload + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + +def test_chat_provider_only_is_not_forwarded_to_embeddings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Chat routing must not constrain independently hosted embedding models.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", chat_provider_only=["z-ai"]) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/embeddings" + assert "provider" not in payload + return { + "model": "qwen/qwen3-embedding-8b", + "usage": {"prompt_tokens": 2, "cost": "0.000001"}, + "data": [{"index": 0, "embedding": [0.1, 0.2]}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.embed( + request=EmbeddingRequest( + model="qwen/qwen3-embedding-8b", texts=("memory",), dimensions=2 + ) + ) + finally: + provider._client.close() + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ("z-ai,novita", ["z-ai", "novita"]), + (' ["z-ai", "deepinfra"] ', ["z-ai", "deepinfra"]), + ("", None), + ], +) +def test_chat_provider_only_parses_env_shapes( + configured: str, expected: list[str] | None +) -> None: + """Compose passes strings; both comma and JSON list forms must work.""" + settings = OpenRouterSettings.model_validate( + {"api_key": "test-key", "chat_provider_only": configured} + ) + assert settings.chat_provider_only == expected + + def _completion(*, content: object, finish: str = "stop", cost: str = "0.0001") -> dict: """One provider chat-completion body with the given message content.""" return { From fd197310172ca2b6aae83f9e5e283de70e0f5a29 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Tue, 1 Sep 2026 15:03:10 +0200 Subject: [PATCH 2/4] feat(benchmark): RS-LoCoMo-Full-v20 runs on z-ai/glm-5.3-flash Rolls the pinned protocol v19 -> v20 and moves every chat seat from openai/gpt-5.6-luna to z-ai/glm-5.3-flash, with chat completions restricted to the operator-named providers z-ai, novita, deepinfra and gmicloud. Embeddings stay on qwen/qwen3-embedding-8b via nebius. The model swap could not be a config change. The protocol ASSERTS its ingest bindings -- runner.py checks readiness.model_bindings == the expected map by exact dict equality -- so a deployment on a different model fails readiness and refuses to run. Rolling the protocol is the mechanism the repo already uses for this (v17 -> v18 -> v19), and the registry holds one current protocol at a time. The answer agent and judge move too, on operator instruction. That is worth stating plainly: the judge IS the scorer, so v20 numbers are NOT comparable with v19/v18 runs, and a score change cannot be attributed to the memory system rather than the grader. The protocol fingerprint changes accordingly (the golden is rolled to 76ac6811...), which is the intended signal. Also fixes a defect from the preceding commit: adding openrouter_chat_provider_only to the selfhost attestation dict broke the exact-equality readiness gate for every deployment. The key is now expected by the protocol and replayed by the shard runner, so a restored store reproduces the routing it was built with. Reasoning effort stays high for the ingest seats and none for answer/judge, exactly as v19 had it, so this run changes the model and the provider set and nothing else. 321 tests pass; ruff and pyright clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018eazkrM7j7scrwQUcUWbbS --- benchmarks/locomo/__init__.py | 2 +- benchmarks/locomo/cli.py | 2 +- benchmarks/locomo/dataset.py | 4 +- benchmarks/locomo/model.py | 18 ++++---- benchmarks/locomo/protocol.py | 43 ++++++++++--------- benchmarks/locomo/runner.py | 10 ++--- benchmarks/locomo/sharding/run_shard.sh | 28 ++++++------ benchmarks/locomo/sharding/store_backup.py | 2 + src/tests/benchmarks/test_locomo_protocol.py | 14 +++--- src/tests/benchmarks/test_locomo_runner.py | 12 +++--- .../benchmarks/test_locomo_store_backup.py | 10 ++--- 11 files changed, 75 insertions(+), 70 deletions(-) diff --git a/benchmarks/locomo/__init__.py b/benchmarks/locomo/__init__.py index 780fb253b..1a46ed3e8 100644 --- a/benchmarks/locomo/__init__.py +++ b/benchmarks/locomo/__init__.py @@ -1 +1 @@ -"""The pinned RS-LoCoMo-Full-v19 benchmark adapter.""" +"""The pinned RS-LoCoMo-Full-v20 benchmark adapter.""" diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index 65d47c2d3..f01cc26be 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -118,7 +118,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-Full-v19: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v20: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py index 75260bf6c..d29656056 100644 --- a/benchmarks/locomo/dataset.py +++ b/benchmarks/locomo/dataset.py @@ -140,10 +140,10 @@ def load_manifest(tier: str) -> QuestionManifest: ) if manifest.dataset_commit != DATASET_COMMIT: raise DatasetValidationError( - "manifest dataset commit is not RS-LoCoMo-Full-v19" + "manifest dataset commit is not RS-LoCoMo-Full-v20" ) if manifest.dataset_sha256 != DATASET_SHA256: - raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v19") + raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v20") actual = item_ids_hash(item_ids=manifest.item_ids) if actual != manifest.item_ids_sha256: raise DatasetValidationError( diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 6a9784e09..a0a872de3 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v19 protocol.""" +"""Typed values for the full-system RS-LoCoMo-Full-v20 protocol.""" from __future__ import annotations @@ -28,11 +28,11 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] -ProtocolKey = Literal["full-v19"] -ProtocolName = Literal["RS-LoCoMo-Full-v19"] +ProtocolKey = Literal["full-v20"] +ProtocolName = Literal["RS-LoCoMo-Full-v20"] SourceTimezoneBasis = Literal["assumed_utc"] -AnswerAgentModel = Literal["openai/gpt-5.6-luna"] -JudgeModel = Literal["openai/gpt-5.6-luna"] +AnswerAgentModel = Literal["z-ai/glm-5.3-flash"] +JudgeModel = Literal["z-ai/glm-5.3-flash"] FailureKind = Literal[ "readiness", "tool", "reader", "judge", "accounting", "invalid_response", "missing" ] @@ -120,7 +120,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v19" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -139,10 +139,10 @@ class RunConfiguration(FrozenModel): api_timeout_seconds: float = Field(default=60.0, gt=0) knowledge_mode: Literal["not_composed"] = "not_composed" document_binding_generation: Literal["document-t0-v1"] = "document-t0-v1" - answer_agent_model: AnswerAgentModel = "openai/gpt-5.6-luna" + answer_agent_model: AnswerAgentModel = "z-ai/glm-5.3-flash" answer_agent_reasoning_effort: Literal["none"] = "none" answer_word_cap: int | None = Field(default=None, ge=1) - judge_model: JudgeModel = "openai/gpt-5.6-luna" + judge_model: JudgeModel = "z-ai/glm-5.3-flash" judge_reasoning_effort: Literal["none"] = "none" answer_agent_temperature: float = Field(default=0.0, ge=0, le=2) judge_temperature: float = Field(default=0.0, ge=0, le=2) @@ -393,7 +393,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v19" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index 0048cfc05..4e5980567 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -29,9 +29,9 @@ from rememberstack.model import Envelope from rememberstack.model import ToolDescriptor -PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v19" -DEFAULT_PROTOCOL_KEY: Final = "full-v19" -ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-counterfactual-v19" +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v20" +DEFAULT_PROTOCOL_KEY: Final = "full-v20" +ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-glm53-pinned-providers-v20" MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 ANSWER_READER_RETRY_BUDGET: Final = 2 @@ -87,30 +87,31 @@ EXPECTED_INGEST_MODEL_BINDINGS: Final[Mapping[str, str]] = MappingProxyType( { "chunk_embedding": "qwen/qwen3-embedding-8b", - "claim_extraction": "openai/gpt-5.6-luna", - "context_prefix": "openai/gpt-5.6-luna", + "claim_extraction": "z-ai/glm-5.3-flash", + "context_prefix": "z-ai/glm-5.3-flash", "entity_observation_embedding": "qwen/qwen3-embedding-8b", - "fact_label": "openai/gpt-5.6-luna", - "observation_frontier": "openai/gpt-5.6-luna", - "observation_small": "openai/gpt-5.6-luna", + "fact_label": "z-ai/glm-5.3-flash", + "observation_frontier": "z-ai/glm-5.3-flash", + "observation_small": "z-ai/glm-5.3-flash", + "openrouter_chat_provider_only": "z-ai,novita,deepinfra,gmicloud", "openrouter_embedding_provider": "nebius", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "32000", "openrouter_reasoning_effort": "auto", - "openrouter_reasoning_effort_map": '{"openai/gpt-5.6-luna": "high"}', + "openrouter_reasoning_effort_map": '{"z-ai/glm-5.3-flash": "high"}', "p1_embedding": "qwen/qwen3-embedding-8b", - "relation_normalization": "openai/gpt-5.6-luna", - "section_role": "openai/gpt-5.6-luna", - "section_summary": "openai/gpt-5.6-luna", - "skeleton_check": "openai/gpt-5.6-luna", - "structure_fallback": "openai/gpt-5.6-luna", - "supersession_frontier": "openai/gpt-5.6-luna", - "supersession_small": "openai/gpt-5.6-luna", + "relation_normalization": "z-ai/glm-5.3-flash", + "section_role": "z-ai/glm-5.3-flash", + "section_summary": "z-ai/glm-5.3-flash", + "skeleton_check": "z-ai/glm-5.3-flash", + "structure_fallback": "z-ai/glm-5.3-flash", + "supersession_frontier": "z-ai/glm-5.3-flash", + "supersession_small": "z-ai/glm-5.3-flash", } ) -ANSWER_AGENT_MODEL: Final = "openai/gpt-5.6-luna" +ANSWER_AGENT_MODEL: Final = "z-ai/glm-5.3-flash" ANSWER_AGENT_REASONING_EFFORT: Final = "none" -JUDGE_MODEL: Final = "openai/gpt-5.6-luna" +JUDGE_MODEL: Final = "z-ai/glm-5.3-flash" JUDGE_REASONING_EFFORT: Final = "none" TEMPERATURE: Final = 0.0 @@ -206,8 +207,8 @@ class LoCoMoProtocol: answer_word_cap: int | None = None -_FULL_V19 = LoCoMoProtocol( - key="full-v19", +_FULL_V20 = LoCoMoProtocol( + key="full-v20", name=PROTOCOL_NAME, answer_agent_model=ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -229,7 +230,7 @@ class LoCoMoProtocol: ) PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( - {_FULL_V19.key: _FULL_V19} + {_FULL_V20.key: _FULL_V20} ) diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index 5d28f7efd..e8eb67abd 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -411,7 +411,7 @@ def ingest_sample( _require_current_ingest_bindings(model_bindings=build.model_bindings) if build.document_binding_generation != EXPECTED_DOCUMENT_BINDING_GENERATION: raise ExecutionGuardError( - "deployment document binding generation differs from RS-LoCoMo-Full-v19" + "deployment document binding generation differs from RS-LoCoMo-Full-v20" ) _require_current_query_surface(context=context, client=client) _require_exact_live_ingests( @@ -570,7 +570,7 @@ def answer_sample( ): raise ExecutionGuardError( "the deployment did not report the exact completed" - " RS-LoCoMo-Full-v19 pipeline, live graph, and fresh P3 projection" + " RS-LoCoMo-Full-v20 pipeline, live graph, and fresh P3 projection" ) _require_serving_revision(context=context, readiness=readiness) prior_readiness = context.state.readiness.get(sample_id) @@ -1177,7 +1177,7 @@ def _validate_run( """Recompute immutable run identity before any local or remote stage.""" selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v19") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v20") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -1187,7 +1187,7 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v19") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v20") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if _models_hash(values=documents) != configuration.documents_sha256: @@ -1445,7 +1445,7 @@ def _require_current_ingest_bindings(*, model_bindings: dict[str, str]) -> None: if model_bindings.get(name) != expected.get(name) ) raise ExecutionGuardError( - "deployment ingest model bindings differ from RS-LoCoMo-Full-v19: " + "deployment ingest model bindings differ from RS-LoCoMo-Full-v20: " + ", ".join(mismatches) ) diff --git a/benchmarks/locomo/sharding/run_shard.sh b/benchmarks/locomo/sharding/run_shard.sh index ab7c2bb32..83c14cd1b 100755 --- a/benchmarks/locomo/sharding/run_shard.sh +++ b/benchmarks/locomo/sharding/run_shard.sh @@ -52,26 +52,27 @@ export GOOGLE_API_USE_CLIENT_CERTIFICATE=true # RS-LoCoMo-Full-v19's non-secret ingest identity. Override ambient self-host # defaults so every shard runs the exact Luna/Qwen pipeline the protocol checks. -export REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_SKELETON_CHECK_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ROLE_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_SUMMARY_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_STRUCTURER_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_SKELETON_CHECK_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ROLE_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_SUMMARY_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_E1_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_E1_PREFIX_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_E2_EXTRACT_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_E3_NORMALIZE_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_E1_PREFIX_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_E2_EXTRACT_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_E3_NORMALIZE_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_OBS_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_OBS_SMALL_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_OBS_FRONTIER_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_OBS_SMALL_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_OBS_FRONTIER_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_P1_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_P1_LABEL_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_P1_LABEL_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius unset REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER export REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 unset REMEMBERSTACK_OPENROUTER_REASONING_EFFORT -export REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP='{"openai/gpt-5.6-luna":"high"}' +export REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP='{"z-ai/glm-5.3-flash":"high"}' +export REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud export REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/invalid-completions [[ -x "$python_bin" ]] || die "Python is not executable: $python_bin" @@ -139,6 +140,7 @@ attest_worker_environment() { REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL REMEMBERSTACK_P1_EMBEDDING_MODEL REMEMBERSTACK_P1_LABEL_MODEL + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS diff --git a/benchmarks/locomo/sharding/store_backup.py b/benchmarks/locomo/sharding/store_backup.py index 386faeff3..7bbde1c39 100644 --- a/benchmarks/locomo/sharding/store_backup.py +++ b/benchmarks/locomo/sharding/store_backup.py @@ -48,6 +48,7 @@ "fact_label": "REMEMBERSTACK_P1_LABEL_MODEL", "observation_frontier": "REMEMBERSTACK_OBS_FRONTIER_MODEL", "observation_small": "REMEMBERSTACK_OBS_SMALL_MODEL", + "openrouter_chat_provider_only": "REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY", "openrouter_embedding_provider": "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER", "openrouter_embedding_provider_order": ( "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER" @@ -69,6 +70,7 @@ "supersession_small": "REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL", } UNSET_MODEL_BINDINGS = { + "openrouter_chat_provider_only": "unset", "openrouter_embedding_provider": "auto", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "unset", diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 7fe02b62c..135c5cae7 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -241,8 +241,8 @@ def test_current_protocol_pins_manifest_and_complete_read_plane() -> None: def test_protocol_is_v19_and_answer_prompt_has_reasoning_and_loop_guards() -> None: """The current identity, bounded inference, and loop discipline are locked.""" - assert PROTOCOL_NAME == "RS-LoCoMo-Full-v19" - assert DEFAULT_PROTOCOL_KEY == "full-v19" + assert PROTOCOL_NAME == "RS-LoCoMo-Full-v20" + assert DEFAULT_PROTOCOL_KEY == "full-v20" prompt = ANSWER_AGENT_PROMPT_TEMPLATE normalized_prompt = " ".join(prompt.split()) assert ( @@ -269,11 +269,11 @@ def test_protocol_is_v19_and_answer_prompt_has_reasoning_and_loop_guards() -> No def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None: - assert tuple(PROTOCOL_REGISTRY) == ("full-v19",) - protocol = PROTOCOL_REGISTRY["full-v19"] + assert tuple(PROTOCOL_REGISTRY) == ("full-v20",) + protocol = PROTOCOL_REGISTRY["full-v20"] - assert protocol.name == "RS-LoCoMo-Full-v19" - assert protocol.answer_agent_model == "openai/gpt-5.6-luna" + assert protocol.name == "RS-LoCoMo-Full-v20" + assert protocol.answer_agent_model == "z-ai/glm-5.3-flash" assert protocol.answer_agent_reasoning_effort == "none" assert protocol.judge_reasoning_effort == "none" assert protocol.answer_reader_retry_budget == 2 @@ -310,7 +310,7 @@ def fake_prepare_run(**values: object) -> _Prepared: ) assert exit_code == 0 - assert selected == ["full-v19"] + assert selected == ["full-v20"] def test_summarize_cli_accepts_multiple_run_flags( diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index 9af538b5b..57fbf97b4 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -1033,7 +1033,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: assert answer.agent_call_count == 2 assert answer.reader_usage is not None assert answer.reader_usage.model_name == ( - "mixed:openai/gpt-5.6-luna|openai/not-luna" + "mixed:openai/not-luna|z-ai/glm-5.3-flash" ) assert answer.reader_usage.cost_usd == Decimal("0.02") assert state.evaluator_cost_usd == Decimal("0.02") @@ -1047,7 +1047,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: "invalid_first_step_completions", "invalid_reader_completions", ), - (("full-v19", "openai/gpt-5.6-luna", "none", 0, 2),), + (("full-v20", "z-ai/glm-5.3-flash", "none", 0, 2),), ) def test_staged_mock_run_uses_prepared_protocol_and_resumes( protocol: ProtocolKey, @@ -1906,8 +1906,8 @@ def test_single_run_summary_json_is_unchanged( serialized = summarize_run(run_dir=run_dir).model_dump_json() assert serialized == ( - '{"protocol_name":"RS-LoCoMo-Full-v19","protocol_fingerprint":' - '"099aee28603fab2db6f6c775095307b4e46a68e7e6913f05e87330183439c05d",' + '{"protocol_name":"RS-LoCoMo-Full-v20","protocol_fingerprint":' + '"76ac68117dbddd72a7e1f44734cde043180ad8b720109af9f9a839641c152784",' '"tier":"smoke","questions":1,"judge_correct":0,"judge_percent":0.0,' '"official_f1":0.0,"categories":[{"category":1,"questions":0,' '"judge_correct":0,"judge_percent":0.0,"official_f1":0.0},{"category":2,' @@ -2126,8 +2126,8 @@ def test_prepared_protocol_pins_current_surface_and_luna( dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir ) - assert prepared.protocol_name == "RS-LoCoMo-Full-v19" - assert prepared.answer_agent_model == "openai/gpt-5.6-luna" + assert prepared.protocol_name == "RS-LoCoMo-Full-v20" + assert prepared.answer_agent_model == "z-ai/glm-5.3-flash" assert prepared.answer_agent_reasoning_effort == "none" assert prepared.answer_reader_retry_budget == 2 assert prepared.surface_manifest_hash == EXPECTED_SURFACE_MANIFEST_HASH diff --git a/src/tests/benchmarks/test_locomo_store_backup.py b/src/tests/benchmarks/test_locomo_store_backup.py index 5538fac0e..4e16fa923 100644 --- a/src/tests/benchmarks/test_locomo_store_backup.py +++ b/src/tests/benchmarks/test_locomo_store_backup.py @@ -24,7 +24,7 @@ def _run_json(path: Path) -> None: (path / "run.json").write_text( json.dumps( { - "protocol_name": "RS-LoCoMo-Full-v19", + "protocol_name": "RS-LoCoMo-Full-v20", "protocol_fingerprint": "p" * 64, "repository_revision": "r" * 40, "prepared_at": "2026-08-11T00:00:00Z", @@ -660,7 +660,7 @@ def test_restore_validates_every_archive_before_running_docker( deployment_id="57000000-0000-0000-0000-000000000001", compose_project="rememberstack", run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v19", + protocol_name="RS-LoCoMo-Full-v20", protocol_fingerprint="p" * 64, repository_revision="r" * 40, prepared_at="2026-08-11T00:00:00Z", @@ -922,7 +922,7 @@ def test_runtime_validation_uses_the_image_revision_stamp( sample_id="conv-1", deployment_id=deployment_id, run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v19", + protocol_name="RS-LoCoMo-Full-v20", protocol_fingerprint="p" * 64, repository_revision=revision, prepared_at="2026-08-11T00:00:00Z", @@ -1019,8 +1019,8 @@ def test_shard_runner_guards_wipe_and_backs_up_before_scoring() -> None: assert "LOCOMO_BACKUP_DESTINATION must be" in script assert "LOCOMO_BACKUP_TOOL:-benchmarks/locomo/sharding/store_backup.py" in script assert 'compose=(docker compose --project-name "$compose_project")' in script - assert "REMEMBERSTACK_E2_EXTRACT_MODEL=openai/gpt-5.6-luna" in script - assert "REMEMBERSTACK_OBS_FRONTIER_MODEL=openai/gpt-5.6-luna" in script + assert "REMEMBERSTACK_E2_EXTRACT_MODEL=z-ai/glm-5.3-flash" in script + assert "REMEMBERSTACK_OBS_FRONTIER_MODEL=z-ai/glm-5.3-flash" in script assert "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius" in script assert "unset REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER" in script assert 'published=$("${compose[@]}" port api 8000 | head -n 1)' in script From 13b0724543c1b795d238af2cf45f41bcd83a0164 Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Wed, 2 Sep 2026 10:18:59 +0200 Subject: [PATCH 3/4] feat(openrouter): sort allowed chat providers, not just restrict them A conv-48 ingestion dead-lettered 28 adjudicate_observations items, every one a 429 from the SAME provider: provider_name: DeepInfra provider_error_code: engine_overloaded limit_source: upstream_provider_shared_pool is_byok: false Two things made that concentrate on one host. OpenRouter's default routing weights price, and DeepInfra was both the cheapest and the slowest of the four allowed providers (measured: $0.0001003 at 30.8 tok/s, against Z.AI's $0.0001581 at 43.9) -- so it was selected first on every call and on all three engine retries. And allow_fallbacks did not rescue it, despite the preceding commit arguing it would. The docs define it as allowing backups when the primary is 'unavailable' and never enumerate the codes; empirically a provider-returned 429 arrives as 'Provider returned error' and is not re-routed. That assumption is now corrected in the code comments and covered by a test. Adds REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT (price|throughput|latency) and sets throughput for the benchmark protocol. Sorting beats the obvious fix of dropping DeepInfra from the allowlist: a denylist hard-codes today's congested host, while sorting moves load off whichever host is slow at the time, and it preserves the operator's stated provider set instead of silently narrowing it. It is a bias, not a guarantee -- if every allowed provider is busy at once, calls still 429. Concurrency is part of the same decision: the identical allowlist ran 1,842 calls with one failure at 1 worker per stage and collapsed at extract 8 / normalize 6 / adjudicate 4 / embed 2. Written up in plan/analysis/openrouter_provider_routing_overload.md. 323 tests pass; ruff and pyright clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018eazkrM7j7scrwQUcUWbbS --- .env.example | 8 ++ benchmarks/locomo/protocol.py | 1 + benchmarks/locomo/sharding/run_shard.sh | 2 + benchmarks/locomo/sharding/store_backup.py | 2 + compose.yaml | 1 + .../openrouter_provider_routing_overload.md | 100 ++++++++++++++++++ src/rememberstack/adapters/openrouter.py | 28 ++++- src/rememberstack/profiles/selfhost.py | 1 + src/tests/adapters/test_openrouter.py | 69 ++++++++++++ 9 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 plan/analysis/openrouter_provider_routing_overload.md diff --git a/.env.example b/.env.example index 93fc97294..baaceeed1 100644 --- a/.env.example +++ b/.env.example @@ -74,6 +74,14 @@ REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 # slugs, not quantization tags). `only` bounds the pool, so failover moves # between these and can never leave them. Unset = ordinary marketplace routing. # REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud +# +# How OpenRouter orders those providers: price | throughput | latency. +# Unset uses OpenRouter's default, which weights price heavily -- and the +# cheapest endpoint is usually the most contended, so every call AND retry +# lands on it and dead-letters together when it is overloaded. Note that +# allow_fallbacks does NOT rescue this: a provider-returned 429 is surfaced +# as "Provider returned error", not treated as the provider being unavailable. +# REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT=throughput # Debug only: retain raw schema-invalid completions under the private app-state # volume. Disabled by default because captured output may repeat source text. # REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/openrouter-invalid-completions diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index f5ce0c7b9..06268e47e 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -96,6 +96,7 @@ "observation_frontier": "z-ai/glm-5.3-flash", "observation_small": "z-ai/glm-5.3-flash", "openrouter_chat_provider_only": "z-ai,novita,deepinfra,gmicloud", + "openrouter_chat_provider_sort": "throughput", "openrouter_embedding_provider": "nebius", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "32000", diff --git a/benchmarks/locomo/sharding/run_shard.sh b/benchmarks/locomo/sharding/run_shard.sh index 6049a5e42..3b904d544 100755 --- a/benchmarks/locomo/sharding/run_shard.sh +++ b/benchmarks/locomo/sharding/run_shard.sh @@ -73,6 +73,7 @@ export REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 unset REMEMBERSTACK_OPENROUTER_REASONING_EFFORT export REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP='{"z-ai/glm-5.3-flash":"high"}' export REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud +export REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT=throughput export REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/invalid-completions [[ -x "$python_bin" ]] || die "Python is not executable: $python_bin" @@ -141,6 +142,7 @@ attest_worker_environment() { REMEMBERSTACK_P1_EMBEDDING_MODEL REMEMBERSTACK_P1_LABEL_MODEL REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS diff --git a/benchmarks/locomo/sharding/store_backup.py b/benchmarks/locomo/sharding/store_backup.py index 7bbde1c39..e0f2bd780 100644 --- a/benchmarks/locomo/sharding/store_backup.py +++ b/benchmarks/locomo/sharding/store_backup.py @@ -49,6 +49,7 @@ "observation_frontier": "REMEMBERSTACK_OBS_FRONTIER_MODEL", "observation_small": "REMEMBERSTACK_OBS_SMALL_MODEL", "openrouter_chat_provider_only": "REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY", + "openrouter_chat_provider_sort": "REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT", "openrouter_embedding_provider": "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER", "openrouter_embedding_provider_order": ( "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER" @@ -71,6 +72,7 @@ } UNSET_MODEL_BINDINGS = { "openrouter_chat_provider_only": "unset", + "openrouter_chat_provider_sort": "default", "openrouter_embedding_provider": "auto", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "unset", diff --git a/compose.yaml b/compose.yaml index a7507805a..00066496c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -22,6 +22,7 @@ x-app: &app REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP:-} REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY: ${REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY:-} + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT: ${REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT:-} REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR: ${REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} diff --git a/plan/analysis/openrouter_provider_routing_overload.md b/plan/analysis/openrouter_provider_routing_overload.md new file mode 100644 index 000000000..c9f44be22 --- /dev/null +++ b/plan/analysis/openrouter_provider_routing_overload.md @@ -0,0 +1,100 @@ +# Price-first provider routing dead-letters long ingestion runs + +**Status:** analysis (non-binding). +**Date:** 2026-09-02. +**Evidence:** LoCoMo `conv-48` ingestion, protocol `RS-LoCoMo-Full-v21`, +`z-ai/glm-5.3-flash`, host `umc-locomo-bench-01`, deployment +`a5754c13-a721-4b85-9dfb-652f3fe94a46`. + +## What happened + +A conv-48 ingestion completed extraction cleanly (119/119) and then +dead-lettered **28 `adjudicate_observations` items**, each after exhausting +all three engine attempts. Every one of the 28 carried the identical +provider payload: + +```json +{"error":{"message":"Provider returned error","code":429, + "metadata":{ + "provider_name":"DeepInfra", + "provider_error_code":"engine_overloaded", + "limit_source":"upstream_provider_shared_pool", + "is_byok":false}}} +``` + +Three facts make this diagnosable rather than mysterious: + +1. **It is upstream, not us.** `limit_source: upstream_provider_shared_pool` + with `is_byok: false` means the ceiling is DeepInfra's own capacity, + shared across every OpenRouter user without a personal key. It is not an + OpenRouter account limit and not a spend cap. +2. **It is one provider, not four.** All 28 name DeepInfra. Z.AI, Novita and + GMICloud produced zero failures under the same concurrency. +3. **It is not a model-quality problem.** `engine_overloaded` says "busy". A + separate, single JSON-decode failure earlier in the run was retried and + succeeded; glm-5.3-flash's structured-output adherence was not the issue. + +## Why it concentrated on one host + +Measured on one identical extraction-shaped call at `reasoning=high`: + +| Provider | cost | throughput | +| --- | ---: | ---: | +| **DeepInfra** | **$0.0001003** | 30.8 tok/s | +| Novita | $0.0001246 | 40.0 tok/s | +| GMICloud | $0.0001333 | 39.5 tok/s | +| Z.AI | $0.0001581 | 43.9 tok/s | + +DeepInfra was the **cheapest and the slowest**. OpenRouter's default routing +weights price heavily, so it was selected first on essentially every call — +and, critically, on every retry too. Cheapest and slowest is exactly the +combination that is most contended. + +## The trap: `allow_fallbacks` does not cover this + +The run already had `allow_fallbacks: true`. The reasoning at the time was +sound-sounding: `only` bounds the pool, so fallback moves *between* allowed +providers and can never leave them. + +It did not rescue the run. OpenRouter's documentation defines +`allow_fallbacks` as *"whether to allow backup providers when the primary is +**unavailable**"* and **does not enumerate which HTTP codes count as +unavailable**. Empirically a provider-returned 429 is surfaced to the caller +as `"Provider returned error"` rather than treated as unavailability, so no +re-route happens. + +**Do not assume `allow_fallbacks` gives you 429 failover.** It does not. + +## What we did instead + +Added `REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT` (`provider.sort`: +`price | throughput | latency`) and set `throughput` for the benchmark. + +Sorting was preferred over the obvious alternative — dropping DeepInfra from +the allowlist — because: + +- Dropping a provider hard-codes *today's* congested host into a denylist. + Tomorrow a different one is busiest and the same failure returns. +- Sorting by throughput moves load off whichever host is slow *at the time*, + which is the actual property we care about. +- It preserves the operator's stated provider set rather than silently + narrowing it. + +## What this does not fix + +Sorting biases selection; it does not guarantee failover. If every allowed +provider is overloaded at once, calls will still 429 and still dead-letter. +The durable fixes for that are a personal provider key (`is_byok: true`, +which buys dedicated rather than shared limits), lower per-stage worker +concurrency, or a broader allowlist. + +Concurrency was a real contributor here: the failures appeared only after +scaling from 1 worker per stage to the reference topology (extract 8 / +normalize 6 / adjudicate 4 / embed 2). At 1 worker the same allowlist ran +1,842 calls with a single failure. + +## Rule of thumb + +For a long ingestion run against a shared-pool provider set: **sort by +throughput, not price**, and treat the number of concurrent workers as part +of the routing decision rather than an independent knob. diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 8b16e0aa6..34b0b7faa 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -10,6 +10,7 @@ import time from typing import Any from typing import Final +from typing import Literal from typing import TypeVar import httpx @@ -131,6 +132,22 @@ class OpenRouterSettings(BaseSettings): Unset means OpenRouter's ordinary marketplace routing, which is the shipped default and the behaviour every existing deployment keeps. """ + chat_provider_sort: Literal["price", "throughput", "latency"] | None = None + """How OpenRouter orders the allowed CHAT providers (``provider.sort``). + + Env: ``REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT``. + + Unset uses OpenRouter's default routing, which weights price heavily. That + default is a trap for long ingestion runs: the cheapest endpoint is also + the most contended, so every call AND every retry lands on the same host + and dead-letters together when it is overloaded. ``allow_fallbacks`` does + not save you -- a provider-returned 429 is surfaced as "Provider returned + error", not treated as the provider being unavailable. + + ``throughput`` prefers the fastest endpoint instead, which moves load off + whichever host is congested at the time rather than hard-coding today's + slowest one into a denylist. + """ reasoning_effort: ReasoningEffort | None = None reasoning_effort_map: dict[str, ReasoningEffort] | None = None """Optional per-model effort overrides as a JSON object env var @@ -150,6 +167,7 @@ class OpenRouterSettings(BaseSettings): @field_validator( "embedding_provider", + "chat_provider_sort", "reasoning_effort", "invalid_completion_capture_dir", mode="before", @@ -468,9 +486,15 @@ def _chat_provider_payload(self) -> dict[str, object] | None: outside the allowlist. """ allowed = self._settings.chat_provider_only - if not allowed: + sort = self._settings.chat_provider_sort + if not allowed and sort is None: return None - return {"only": list(allowed), "allow_fallbacks": True} + payload: dict[str, object] = {"allow_fallbacks": True} + if allowed: + payload["only"] = list(allowed) + if sort is not None: + payload["sort"] = sort + return payload def _embedding_provider_payload(self) -> dict[str, object] | None: """Build OpenRouter provider routing for embedding requests. diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 6105106d8..2f9029a6b 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -1349,6 +1349,7 @@ def _model_bindings() -> dict[str, str]: if openrouter.chat_provider_only else "unset" ), + "openrouter_chat_provider_sort": openrouter.chat_provider_sort or "default", "openrouter_reasoning_effort": openrouter.reasoning_effort or "auto", # Canonical (sorted-key) form so the effective per-model effort policy # is part of measurement provenance, not hidden behind the global pin. diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index bef87a537..ad9bf6396 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -1004,6 +1004,75 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert response.output.answer == "Prague" +def test_chat_provider_sort_moves_load_off_the_cheapest_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Regression: price-first routing sent every retry to one overloaded host. + + A conv-48 ingestion dead-lettered 28 items, all with + provider_error_code=engine_overloaded from the single CHEAPEST allowed + provider. OpenRouter's default routing weights price, so it was chosen + first on every call and on all three engine retries; `allow_fallbacks` + did not help because a provider-returned 429 arrives as "Provider + returned error" rather than as the provider being unavailable. + Sorting by throughput spreads load off whichever host is congested. + """ + provider = OpenRouterModelProvider( + settings=OpenRouterSettings( + api_key="test-key", + chat_provider_only=["z-ai", "novita", "deepinfra", "gmicloud"], + chat_provider_sort="throughput", + ) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert payload["provider"] == { + "allow_fallbacks": True, + "only": ["z-ai", "novita", "deepinfra", "gmicloud"], + "sort": "throughput", + } + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + +def test_chat_provider_sort_applies_without_an_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Sorting is useful on its own; it must not require pinning providers.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", chat_provider_sort="throughput") + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert payload["provider"] == {"allow_fallbacks": True, "sort": "throughput"} + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + def test_chat_provider_only_is_absent_when_unset( monkeypatch: pytest.MonkeyPatch, ) -> None: From 42989d896d168061df83ffef046520be2843fbec Mon Sep 17 00:00:00 2001 From: Jiri Puc Date: Thu, 3 Sep 2026 09:28:29 +0200 Subject: [PATCH 4/4] fix(openrouter): correct the root cause -- schema capability, not price The preceding commit blamed price-first routing: DeepInfra was the cheapest AND slowest of the four allowed providers, so the story went that it won selection on every call and every retry. That fit every observation and was wrong. It predicted that sort=throughput would fix the problem. After deploying the sort and replaying all 28 dead letters, all 28 failed again on DeepInfra -- which is what falsified it. The real cause is capability, not preference. Every chat completion here sends response_format json_schema strict, and of the four allowed providers only DeepInfra advertises structured_outputs for this model: DeepInfra structured_outputs=true Z.AI false Novita false GMICloud false So provider.only bounded the pool by NAME to four while the request bounded it by CAPABILITY to one. sort had nothing to reorder and allow_fallbacks had nowhere to fall back to. Isolated one parameter at a time: max_tokens alone and reasoning alone both route to Novita; the strict schema alone pins to DeepInfra and 429s. chat_provider_sort is kept -- biasing toward throughput is still sensible when more than one host is eligible -- but its docstring and test no longer claim it solves this, and the analysis records the wrong diagnosis alongside the right one so the next reader sees how it was caught. Practical rule now written down: when pinning providers, intersect each endpoint's supported_parameters with what the caller actually sends. An allowlist of four that supports one is a single point of failure in disguise. 323 tests pass; ruff and pyright clean. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018eazkrM7j7scrwQUcUWbbS --- .../openrouter_provider_routing_overload.md | 166 +++++++++--------- src/rememberstack/adapters/openrouter.py | 20 +-- src/tests/adapters/test_openrouter.py | 17 +- 3 files changed, 105 insertions(+), 98 deletions(-) diff --git a/plan/analysis/openrouter_provider_routing_overload.md b/plan/analysis/openrouter_provider_routing_overload.md index c9f44be22..db96e6501 100644 --- a/plan/analysis/openrouter_provider_routing_overload.md +++ b/plan/analysis/openrouter_provider_routing_overload.md @@ -1,4 +1,4 @@ -# Price-first provider routing dead-letters long ingestion runs +# A strict JSON schema can collapse a provider allowlist to one host **Status:** analysis (non-binding). **Date:** 2026-09-02. @@ -6,12 +6,16 @@ `z-ai/glm-5.3-flash`, host `umc-locomo-bench-01`, deployment `a5754c13-a721-4b85-9dfb-652f3fe94a46`. +> **Read this if you are pinning providers.** The headline is not "429s +> happen". It is that `provider.only` plus strict structured outputs can +> silently leave exactly **one** eligible provider, after which no amount of +> `sort` or `allow_fallbacks` helps. + ## What happened A conv-48 ingestion completed extraction cleanly (119/119) and then dead-lettered **28 `adjudicate_observations` items**, each after exhausting -all three engine attempts. Every one of the 28 carried the identical -provider payload: +its attempts. All 28 carried the same payload: ```json {"error":{"message":"Provider returned error","code":429, @@ -22,79 +26,83 @@ provider payload: "is_byok":false}}} ``` -Three facts make this diagnosable rather than mysterious: - -1. **It is upstream, not us.** `limit_source: upstream_provider_shared_pool` - with `is_byok: false` means the ceiling is DeepInfra's own capacity, - shared across every OpenRouter user without a personal key. It is not an - OpenRouter account limit and not a spend cap. -2. **It is one provider, not four.** All 28 name DeepInfra. Z.AI, Novita and - GMICloud produced zero failures under the same concurrency. -3. **It is not a model-quality problem.** `engine_overloaded` says "busy". A - separate, single JSON-decode failure earlier in the run was retried and - succeeded; glm-5.3-flash's structured-output adherence was not the issue. - -## Why it concentrated on one host - -Measured on one identical extraction-shaped call at `reasoning=high`: - -| Provider | cost | throughput | -| --- | ---: | ---: | -| **DeepInfra** | **$0.0001003** | 30.8 tok/s | -| Novita | $0.0001246 | 40.0 tok/s | -| GMICloud | $0.0001333 | 39.5 tok/s | -| Z.AI | $0.0001581 | 43.9 tok/s | - -DeepInfra was the **cheapest and the slowest**. OpenRouter's default routing -weights price heavily, so it was selected first on essentially every call — -and, critically, on every retry too. Cheapest and slowest is exactly the -combination that is most contended. - -## The trap: `allow_fallbacks` does not cover this - -The run already had `allow_fallbacks: true`. The reasoning at the time was -sound-sounding: `only` bounds the pool, so fallback moves *between* allowed -providers and can never leave them. - -It did not rescue the run. OpenRouter's documentation defines -`allow_fallbacks` as *"whether to allow backup providers when the primary is -**unavailable**"* and **does not enumerate which HTTP codes count as -unavailable**. Empirically a provider-returned 429 is surfaced to the caller -as `"Provider returned error"` rather than treated as unavailability, so no -re-route happens. - -**Do not assume `allow_fallbacks` gives you 429 failover.** It does not. - -## What we did instead - -Added `REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT` (`provider.sort`: -`price | throughput | latency`) and set `throughput` for the benchmark. - -Sorting was preferred over the obvious alternative — dropping DeepInfra from -the allowlist — because: - -- Dropping a provider hard-codes *today's* congested host into a denylist. - Tomorrow a different one is busiest and the same failure returns. -- Sorting by throughput moves load off whichever host is slow *at the time*, - which is the actual property we care about. -- It preserves the operator's stated provider set rather than silently - narrowing it. - -## What this does not fix - -Sorting biases selection; it does not guarantee failover. If every allowed -provider is overloaded at once, calls will still 429 and still dead-letter. -The durable fixes for that are a personal provider key (`is_byok: true`, -which buys dedicated rather than shared limits), lower per-stage worker -concurrency, or a broader allowlist. - -Concurrency was a real contributor here: the failures appeared only after -scaling from 1 worker per stage to the reference topology (extract 8 / -normalize 6 / adjudicate 4 / embed 2). At 1 worker the same allowlist ran -1,842 calls with a single failure. - -## Rule of thumb - -For a long ingestion run against a shared-pool provider set: **sort by -throughput, not price**, and treat the number of concurrent workers as part -of the routing decision rather than an independent knob. +The allowlist was `z-ai,novita,deepinfra,gmicloud`. Every failure named +DeepInfra; the other three produced none. + +## The actual root cause + +The engine sends **every** chat completion with +`response_format: {type: json_schema, strict: true}`. Of the four allowed +providers, only DeepInfra advertises strict structured-output support for +this model: + +| Provider | `structured_outputs` | `response_format` | +| --- | --- | --- | +| **DeepInfra** | **true** | true | +| Z.AI | false | true | +| Novita | false | true | +| GMICloud | false | true | + +So the effective pool was **one provider**, not four. `sort` had nothing to +reorder and `allow_fallbacks` had nowhere to fall back to. Each engine retry +returned to the same overloaded host. + +Isolated empirically — same allowlist, same `sort: throughput`, one +parameter at a time: + +| Request shape | Result | +| --- | --- | +| `max_tokens: 32000` only | OK, served by Novita | +| `reasoning: {effort: high}` only | OK, served by Novita | +| **strict `json_schema` only** | **429 from DeepInfra** | + +A trivial request with the identical provider block routes to Z.AI or Novita +and never to DeepInfra. Add the schema and it pins to DeepInfra every time. + +## A wrong diagnosis worth recording + +The first explanation was: OpenRouter's default routing weights price, +DeepInfra was the cheapest **and** slowest of the four (measured +\$0.0001003 at 30.8 tok/s against Z.AI's \$0.0001581 at 43.9), so it won +selection on every call and every retry. + +That story fit every observation and was still wrong. It predicted that +`sort: throughput` would fix the problem. It did not: after deploying the +sort and replaying all 28 dead letters, all 28 failed again on DeepInfra. +The correlation between "cheapest" and "the one that failed" was a +coincidence of DeepInfra also being the only schema-capable host. + +The lesson is about method rather than routing: **a hypothesis that explains +the failure is not the same as one that predicts the fix.** The replay was +what falsified it, and the one-parameter-at-a-time isolation is what found +the real cause. + +## What `allow_fallbacks` does and does not do + +Still true, and still worth knowing: OpenRouter documents `allow_fallbacks` +as allowing backups when the primary is *"unavailable"* without enumerating +which HTTP codes qualify, and a provider-returned 429 is surfaced as +`"Provider returned error"` rather than triggering a re-route. Do not rely +on it for rate-limit failover. + +But that was not what broke this run. Even perfect 429 failover would have +had nowhere to go. + +## What to do instead + +1. **Check capability, not just identity, when building an allowlist.** Query + `/models/{id}/endpoints` and intersect `supported_parameters` with what the + caller actually sends. An allowlist of four that supports one is a + single point of failure wearing a disguise. +2. **Widen the allowlist to schema-capable providers** for this model + (Fireworks, BaseTen, Cloudflare and others advertise it). +3. **Bring your own key** for the pinned provider (`is_byok: true`) to swap + the shared pool for dedicated limits. +4. **Treat concurrency as part of routing.** The same allowlist ran 1,842 + calls with one failure at 1 worker per stage, and collapsed at the + reference topology (extract 8 / normalize 6 / adjudicate 4 / embed 2). + Extraction survived only because it ran before the scale-up. + +`chat_provider_sort` remains a reasonable setting and is retained — biasing +toward throughput is sensible when there is genuinely more than one eligible +host. It simply cannot help when the eligible set has one member. diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 34b0b7faa..ffc538bc6 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -137,16 +137,16 @@ class OpenRouterSettings(BaseSettings): Env: ``REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT``. - Unset uses OpenRouter's default routing, which weights price heavily. That - default is a trap for long ingestion runs: the cheapest endpoint is also - the most contended, so every call AND every retry lands on the same host - and dead-letters together when it is overloaded. ``allow_fallbacks`` does - not save you -- a provider-returned 429 is surfaced as "Provider returned - error", not treated as the provider being unavailable. - - ``throughput`` prefers the fastest endpoint instead, which moves load off - whichever host is congested at the time rather than hard-coding today's - slowest one into a denylist. + Unset uses OpenRouter's default routing, which weights price heavily; + ``throughput`` prefers the fastest eligible endpoint instead. + + This only helps when more than one provider is actually eligible. + ``chat_provider_only`` bounds the pool by NAME while the request bounds + it by CAPABILITY: every chat call here sends a strict ``json_schema``, so + only providers advertising ``structured_outputs`` can serve it. An + allowlist can therefore collapse to a single host, after which neither + this setting nor ``allow_fallbacks`` has anywhere to go -- see + plan/analysis/openrouter_provider_routing_overload.md. """ reasoning_effort: ReasoningEffort | None = None reasoning_effort_map: dict[str, ReasoningEffort] | None = None diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index ad9bf6396..9745667b8 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -1007,15 +1007,14 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: def test_chat_provider_sort_moves_load_off_the_cheapest_host( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Regression: price-first routing sent every retry to one overloaded host. - - A conv-48 ingestion dead-lettered 28 items, all with - provider_error_code=engine_overloaded from the single CHEAPEST allowed - provider. OpenRouter's default routing weights price, so it was chosen - first on every call and on all three engine retries; `allow_fallbacks` - did not help because a provider-returned 429 arrives as "Provider - returned error" rather than as the provider being unavailable. - Sorting by throughput spreads load off whichever host is congested. + """Sorting biases routing toward the fastest ELIGIBLE provider. + + Deliberately modest about what this buys. A conv-48 ingestion + dead-lettered 28 items on one overloaded host and sorting did NOT fix it: + the strict json_schema every chat call sends left exactly one + schema-capable provider in the allowlist, so there was nothing to + reorder. Sorting helps only when the eligible set has more than one + member -- see plan/analysis/openrouter_provider_routing_overload.md. """ provider = OpenRouterModelProvider( settings=OpenRouterSettings(