diff --git a/.env.example b/.env.example index 99359f513..baaceeed1 100644 --- a/.env.example +++ b/.env.example @@ -69,6 +69,19 @@ REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 # effort literals. Example: pin flash models to none while keeping high for a # reasoning model. # REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP={"z-ai/glm-4.7-flash":"none","openai/gpt-5.6-luna":"high"} + +# Restrict CHAT completions to named OpenRouter providers (allowlist of provider +# slugs, not quantization tags). `only` bounds the pool, so failover moves +# between these and can never leave them. Unset = ordinary marketplace routing. +# REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud +# +# How OpenRouter orders those providers: price | throughput | latency. +# Unset uses OpenRouter's default, which weights price heavily -- and the +# cheapest endpoint is usually the most contended, so every call AND retry +# lands on it and dead-letters together when it is overloaded. Note that +# allow_fallbacks does NOT rescue this: a provider-returned 429 is surfaced +# as "Provider returned error", not treated as the provider being unavailable. +# REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT=throughput # Debug only: retain raw schema-invalid completions under the private app-state # volume. Disabled by default because captured output may repeat source text. # REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/openrouter-invalid-completions diff --git a/benchmarks/locomo/__init__.py b/benchmarks/locomo/__init__.py index 1a46ed3e8..d7dc5aa03 100644 --- a/benchmarks/locomo/__init__.py +++ b/benchmarks/locomo/__init__.py @@ -1 +1 @@ -"""The pinned RS-LoCoMo-Full-v20 benchmark adapter.""" +"""The pinned RS-LoCoMo-Full-v21 benchmark adapter.""" diff --git a/benchmarks/locomo/cli.py b/benchmarks/locomo/cli.py index f01cc26be..ece293744 100644 --- a/benchmarks/locomo/cli.py +++ b/benchmarks/locomo/cli.py @@ -118,7 +118,7 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="python -m benchmarks.locomo", description=( - "RS-LoCoMo-Full-v20: prepare is local; ingest/answer/judge require " + "RS-LoCoMo-Full-v21: prepare is local; ingest/answer/judge require " "explicit execution acknowledgements" ), ) diff --git a/benchmarks/locomo/dataset.py b/benchmarks/locomo/dataset.py index d29656056..cbca706ed 100644 --- a/benchmarks/locomo/dataset.py +++ b/benchmarks/locomo/dataset.py @@ -140,10 +140,10 @@ def load_manifest(tier: str) -> QuestionManifest: ) if manifest.dataset_commit != DATASET_COMMIT: raise DatasetValidationError( - "manifest dataset commit is not RS-LoCoMo-Full-v20" + "manifest dataset commit is not RS-LoCoMo-Full-v21" ) if manifest.dataset_sha256 != DATASET_SHA256: - raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v20") + raise DatasetValidationError("manifest dataset hash is not RS-LoCoMo-Full-v21") actual = item_ids_hash(item_ids=manifest.item_ids) if actual != manifest.item_ids_sha256: raise DatasetValidationError( diff --git a/benchmarks/locomo/model.py b/benchmarks/locomo/model.py index 33d5c9fb0..26bc67b32 100644 --- a/benchmarks/locomo/model.py +++ b/benchmarks/locomo/model.py @@ -1,4 +1,4 @@ -"""Typed values for the full-system RS-LoCoMo-Full-v20 protocol.""" +"""Typed values for the full-system RS-LoCoMo-Full-v21 protocol.""" from __future__ import annotations @@ -28,11 +28,11 @@ Category = Literal[1, 2, 3, 4, 5] RetainedCategory = Literal[1, 2, 3, 4] Tier = Literal["smoke", "development", "publication"] -ProtocolKey = Literal["full-v20"] -ProtocolName = Literal["RS-LoCoMo-Full-v20"] +ProtocolKey = Literal["full-v21"] +ProtocolName = Literal["RS-LoCoMo-Full-v21"] SourceTimezoneBasis = Literal["assumed_utc"] -AnswerAgentModel = Literal["openai/gpt-5.6-luna"] -JudgeModel = Literal["openai/gpt-5.6-luna"] +AnswerAgentModel = Literal["z-ai/glm-5.3-flash"] +JudgeModel = Literal["z-ai/glm-5.3-flash"] FailureKind = Literal[ "readiness", "tool", "reader", "judge", "accounting", "invalid_response", "missing" ] @@ -120,7 +120,7 @@ class QuestionManifest(FrozenModel): class RunConfiguration(FrozenModel): """Immutable identity of one prepared benchmark run.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" adapter_version: NonEmpty prepared_at: datetime repository_revision: NonEmpty @@ -139,10 +139,10 @@ class RunConfiguration(FrozenModel): api_timeout_seconds: float = Field(default=60.0, gt=0) knowledge_mode: Literal["not_composed"] = "not_composed" document_binding_generation: Literal["document-t0-v1"] = "document-t0-v1" - answer_agent_model: AnswerAgentModel = "openai/gpt-5.6-luna" + answer_agent_model: AnswerAgentModel = "z-ai/glm-5.3-flash" answer_agent_reasoning_effort: Literal["none"] = "none" answer_word_cap: int | None = Field(default=None, ge=1) - judge_model: JudgeModel = "openai/gpt-5.6-luna" + judge_model: JudgeModel = "z-ai/glm-5.3-flash" judge_reasoning_effort: Literal["none"] = "none" answer_agent_temperature: float = Field(default=0.0, ge=0, le=2) judge_temperature: float = Field(default=0.0, ge=0, le=2) @@ -393,7 +393,7 @@ class SessionDiagnosticSummary(FrozenModel): class RunSummary(FrozenModel): """Publication-ready local aggregate with no hidden denominator.""" - protocol_name: ProtocolName = "RS-LoCoMo-Full-v20" + protocol_name: ProtocolName = "RS-LoCoMo-Full-v21" protocol_fingerprint: NonEmpty tier: Tier questions: int = Field(ge=1) diff --git a/benchmarks/locomo/protocol.py b/benchmarks/locomo/protocol.py index 8859a1f3c..06268e47e 100644 --- a/benchmarks/locomo/protocol.py +++ b/benchmarks/locomo/protocol.py @@ -29,9 +29,11 @@ from rememberstack.model import Envelope from rememberstack.model import ToolDescriptor -PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v20" -DEFAULT_PROTOCOL_KEY: Final = "full-v20" -ADAPTER_VERSION: Final = "locomo-full-adapter-2026.09-complete-values-v20" +PROTOCOL_NAME: Final = "RS-LoCoMo-Full-v21" +DEFAULT_PROTOCOL_KEY: Final = "full-v21" +ADAPTER_VERSION: Final = ( + "locomo-full-adapter-2026.09-complete-values-glm53-pinned-providers-v21" +) MAX_TOOL_CALLS: Final = 8 MAX_AGENT_CALLS: Final = 9 ANSWER_READER_RETRY_BUDGET: Final = 2 @@ -87,30 +89,32 @@ EXPECTED_INGEST_MODEL_BINDINGS: Final[Mapping[str, str]] = MappingProxyType( { "chunk_embedding": "qwen/qwen3-embedding-8b", - "claim_extraction": "openai/gpt-5.6-luna", - "context_prefix": "openai/gpt-5.6-luna", + "claim_extraction": "z-ai/glm-5.3-flash", + "context_prefix": "z-ai/glm-5.3-flash", "entity_observation_embedding": "qwen/qwen3-embedding-8b", - "fact_label": "openai/gpt-5.6-luna", - "observation_frontier": "openai/gpt-5.6-luna", - "observation_small": "openai/gpt-5.6-luna", + "fact_label": "z-ai/glm-5.3-flash", + "observation_frontier": "z-ai/glm-5.3-flash", + "observation_small": "z-ai/glm-5.3-flash", + "openrouter_chat_provider_only": "z-ai,novita,deepinfra,gmicloud", + "openrouter_chat_provider_sort": "throughput", "openrouter_embedding_provider": "nebius", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "32000", "openrouter_reasoning_effort": "auto", - "openrouter_reasoning_effort_map": '{"openai/gpt-5.6-luna": "high"}', + "openrouter_reasoning_effort_map": '{"z-ai/glm-5.3-flash": "high"}', "p1_embedding": "qwen/qwen3-embedding-8b", - "relation_normalization": "openai/gpt-5.6-luna", - "section_role": "openai/gpt-5.6-luna", - "section_summary": "openai/gpt-5.6-luna", - "skeleton_check": "openai/gpt-5.6-luna", - "structure_fallback": "openai/gpt-5.6-luna", - "supersession_frontier": "openai/gpt-5.6-luna", - "supersession_small": "openai/gpt-5.6-luna", + "relation_normalization": "z-ai/glm-5.3-flash", + "section_role": "z-ai/glm-5.3-flash", + "section_summary": "z-ai/glm-5.3-flash", + "skeleton_check": "z-ai/glm-5.3-flash", + "structure_fallback": "z-ai/glm-5.3-flash", + "supersession_frontier": "z-ai/glm-5.3-flash", + "supersession_small": "z-ai/glm-5.3-flash", } ) -ANSWER_AGENT_MODEL: Final = "openai/gpt-5.6-luna" +ANSWER_AGENT_MODEL: Final = "z-ai/glm-5.3-flash" ANSWER_AGENT_REASONING_EFFORT: Final = "none" -JUDGE_MODEL: Final = "openai/gpt-5.6-luna" +JUDGE_MODEL: Final = "z-ai/glm-5.3-flash" JUDGE_REASONING_EFFORT: Final = "none" TEMPERATURE: Final = 0.0 @@ -211,8 +215,8 @@ class LoCoMoProtocol: answer_word_cap: int | None = None -_FULL_V20 = LoCoMoProtocol( - key="full-v20", +_FULL_V21 = LoCoMoProtocol( + key="full-v21", name=PROTOCOL_NAME, answer_agent_model=ANSWER_AGENT_MODEL, judge_model=JUDGE_MODEL, @@ -234,7 +238,7 @@ class LoCoMoProtocol: ) PROTOCOL_REGISTRY: Final[Mapping[ProtocolKey, LoCoMoProtocol]] = MappingProxyType( - {_FULL_V20.key: _FULL_V20} + {_FULL_V21.key: _FULL_V21} ) diff --git a/benchmarks/locomo/runner.py b/benchmarks/locomo/runner.py index e8eb67abd..60e037e33 100644 --- a/benchmarks/locomo/runner.py +++ b/benchmarks/locomo/runner.py @@ -411,7 +411,7 @@ def ingest_sample( _require_current_ingest_bindings(model_bindings=build.model_bindings) if build.document_binding_generation != EXPECTED_DOCUMENT_BINDING_GENERATION: raise ExecutionGuardError( - "deployment document binding generation differs from RS-LoCoMo-Full-v20" + "deployment document binding generation differs from RS-LoCoMo-Full-v21" ) _require_current_query_surface(context=context, client=client) _require_exact_live_ingests( @@ -570,7 +570,7 @@ def answer_sample( ): raise ExecutionGuardError( "the deployment did not report the exact completed" - " RS-LoCoMo-Full-v20 pipeline, live graph, and fresh P3 projection" + " RS-LoCoMo-Full-v21 pipeline, live graph, and fresh P3 projection" ) _require_serving_revision(context=context, readiness=readiness) prior_readiness = context.state.readiness.get(sample_id) @@ -1177,7 +1177,7 @@ def _validate_run( """Recompute immutable run identity before any local or remote stage.""" selected_protocol = protocol_for_name(configuration.protocol_name) if configuration.dataset_sha256 != DATASET_SHA256: - raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset hash is not RS-LoCoMo-Full-v21") if item_ids_hash(item_ids=manifest.item_ids) != manifest.item_ids_sha256: raise BenchmarkRunError("run manifest item hash changed") if manifest_bytes_hash(manifest=manifest) != configuration.manifest_sha256: @@ -1187,7 +1187,7 @@ def _validate_run( if manifest.tier != configuration.tier: raise BenchmarkRunError("run manifest tier changed") if configuration.dataset_commit != DATASET_COMMIT: - raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v20") + raise BenchmarkRunError("run dataset commit is not RS-LoCoMo-Full-v21") if configuration.adapter_version != ADAPTER_VERSION: raise BenchmarkRunError("run adapter version differs from current code") if _models_hash(values=documents) != configuration.documents_sha256: @@ -1445,7 +1445,7 @@ def _require_current_ingest_bindings(*, model_bindings: dict[str, str]) -> None: if model_bindings.get(name) != expected.get(name) ) raise ExecutionGuardError( - "deployment ingest model bindings differ from RS-LoCoMo-Full-v20: " + "deployment ingest model bindings differ from RS-LoCoMo-Full-v21: " + ", ".join(mismatches) ) diff --git a/benchmarks/locomo/sharding/run_shard.sh b/benchmarks/locomo/sharding/run_shard.sh index 12086c58e..3b904d544 100755 --- a/benchmarks/locomo/sharding/run_shard.sh +++ b/benchmarks/locomo/sharding/run_shard.sh @@ -25,7 +25,7 @@ run_dir=$2 dataset_path=$3 python_bin=${LOCOMO_PYTHON:-.venv/bin/python} tier=${LOCOMO_TIER:-publication} -protocol=${LOCOMO_PROTOCOL:-full-v20} +protocol=${LOCOMO_PROTOCOL:-full-v21} mount_root=${LOCOMO_MOUNT_ROOT:-$run_dir/.mounts} max_documents=${LOCOMO_MAX_DOCUMENTS:-100} max_questions=${LOCOMO_MAX_QUESTIONS:-1540} @@ -50,28 +50,30 @@ export GOOGLE_APPLICATION_CREDENTIALS=${LOCOMO_GCP_CREDENTIALS_FILE:-/etc/rememb export GOOGLE_API_CERTIFICATE_CONFIG=${LOCOMO_GCP_CERTIFICATE_CONFIG_FILE:-/etc/rememberstack/locomo-gcs/certificate-config.json} export GOOGLE_API_USE_CLIENT_CERTIFICATE=true -# RS-LoCoMo-Full-v20's non-secret ingest identity. Override ambient self-host +# RS-LoCoMo-Full-v21's non-secret ingest identity. Override ambient self-host # defaults so every shard runs the exact Luna/Qwen pipeline the protocol checks. -export REMEMBERSTACK_STRUCTURER_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_SKELETON_CHECK_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ROLE_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_SUMMARY_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_STRUCTURER_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_SKELETON_CHECK_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ROLE_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_SUMMARY_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_E1_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_E1_PREFIX_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_E2_EXTRACT_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_E3_NORMALIZE_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_E1_PREFIX_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_E2_EXTRACT_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_E3_NORMALIZE_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_OBS_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_OBS_SMALL_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_OBS_FRONTIER_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL=openai/gpt-5.6-luna -export REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_OBS_SMALL_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_OBS_FRONTIER_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL=z-ai/glm-5.3-flash +export REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_P1_EMBEDDING_MODEL=qwen/qwen3-embedding-8b -export REMEMBERSTACK_P1_LABEL_MODEL=openai/gpt-5.6-luna +export REMEMBERSTACK_P1_LABEL_MODEL=z-ai/glm-5.3-flash export REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius unset REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER export REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS=32000 unset REMEMBERSTACK_OPENROUTER_REASONING_EFFORT -export REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP='{"openai/gpt-5.6-luna":"high"}' +export REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP='{"z-ai/glm-5.3-flash":"high"}' +export REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY=z-ai,novita,deepinfra,gmicloud +export REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT=throughput export REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR=/var/lib/rememberstack/invalid-completions [[ -x "$python_bin" ]] || die "Python is not executable: $python_bin" @@ -139,6 +141,8 @@ attest_worker_environment() { REMEMBERSTACK_ADJUDICATOR_FRONTIER_MODEL REMEMBERSTACK_P1_EMBEDDING_MODEL REMEMBERSTACK_P1_LABEL_MODEL + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS diff --git a/benchmarks/locomo/sharding/store_backup.py b/benchmarks/locomo/sharding/store_backup.py index 386faeff3..e0f2bd780 100644 --- a/benchmarks/locomo/sharding/store_backup.py +++ b/benchmarks/locomo/sharding/store_backup.py @@ -48,6 +48,8 @@ "fact_label": "REMEMBERSTACK_P1_LABEL_MODEL", "observation_frontier": "REMEMBERSTACK_OBS_FRONTIER_MODEL", "observation_small": "REMEMBERSTACK_OBS_SMALL_MODEL", + "openrouter_chat_provider_only": "REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY", + "openrouter_chat_provider_sort": "REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT", "openrouter_embedding_provider": "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER", "openrouter_embedding_provider_order": ( "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER" @@ -69,6 +71,8 @@ "supersession_small": "REMEMBERSTACK_ADJUDICATOR_SMALL_MODEL", } UNSET_MODEL_BINDINGS = { + "openrouter_chat_provider_only": "unset", + "openrouter_chat_provider_sort": "default", "openrouter_embedding_provider": "auto", "openrouter_embedding_provider_order": "unset", "openrouter_max_completion_tokens": "unset", diff --git a/compose.yaml b/compose.yaml index c899d803b..00066496c 100644 --- a/compose.yaml +++ b/compose.yaml @@ -21,6 +21,8 @@ x-app: &app REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS: ${REMEMBERSTACK_OPENROUTER_MAX_COMPLETION_TOKENS:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT:-} REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP: ${REMEMBERSTACK_OPENROUTER_REASONING_EFFORT_MAP:-} + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY: ${REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY:-} + REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT: ${REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT:-} REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR: ${REMEMBERSTACK_OPENROUTER_INVALID_COMPLETION_CAPTURE_DIR:-} REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_ID} REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG: ${REMEMBERSTACK_SELFHOST_DEPLOYMENT_SLUG} diff --git a/plan/analysis/openrouter_provider_routing_overload.md b/plan/analysis/openrouter_provider_routing_overload.md new file mode 100644 index 000000000..db96e6501 --- /dev/null +++ b/plan/analysis/openrouter_provider_routing_overload.md @@ -0,0 +1,108 @@ +# A strict JSON schema can collapse a provider allowlist to one host + +**Status:** analysis (non-binding). +**Date:** 2026-09-02. +**Evidence:** LoCoMo `conv-48` ingestion, protocol `RS-LoCoMo-Full-v21`, +`z-ai/glm-5.3-flash`, host `umc-locomo-bench-01`, deployment +`a5754c13-a721-4b85-9dfb-652f3fe94a46`. + +> **Read this if you are pinning providers.** The headline is not "429s +> happen". It is that `provider.only` plus strict structured outputs can +> silently leave exactly **one** eligible provider, after which no amount of +> `sort` or `allow_fallbacks` helps. + +## What happened + +A conv-48 ingestion completed extraction cleanly (119/119) and then +dead-lettered **28 `adjudicate_observations` items**, each after exhausting +its attempts. All 28 carried the same payload: + +```json +{"error":{"message":"Provider returned error","code":429, + "metadata":{ + "provider_name":"DeepInfra", + "provider_error_code":"engine_overloaded", + "limit_source":"upstream_provider_shared_pool", + "is_byok":false}}} +``` + +The allowlist was `z-ai,novita,deepinfra,gmicloud`. Every failure named +DeepInfra; the other three produced none. + +## The actual root cause + +The engine sends **every** chat completion with +`response_format: {type: json_schema, strict: true}`. Of the four allowed +providers, only DeepInfra advertises strict structured-output support for +this model: + +| Provider | `structured_outputs` | `response_format` | +| --- | --- | --- | +| **DeepInfra** | **true** | true | +| Z.AI | false | true | +| Novita | false | true | +| GMICloud | false | true | + +So the effective pool was **one provider**, not four. `sort` had nothing to +reorder and `allow_fallbacks` had nowhere to fall back to. Each engine retry +returned to the same overloaded host. + +Isolated empirically — same allowlist, same `sort: throughput`, one +parameter at a time: + +| Request shape | Result | +| --- | --- | +| `max_tokens: 32000` only | OK, served by Novita | +| `reasoning: {effort: high}` only | OK, served by Novita | +| **strict `json_schema` only** | **429 from DeepInfra** | + +A trivial request with the identical provider block routes to Z.AI or Novita +and never to DeepInfra. Add the schema and it pins to DeepInfra every time. + +## A wrong diagnosis worth recording + +The first explanation was: OpenRouter's default routing weights price, +DeepInfra was the cheapest **and** slowest of the four (measured +\$0.0001003 at 30.8 tok/s against Z.AI's \$0.0001581 at 43.9), so it won +selection on every call and every retry. + +That story fit every observation and was still wrong. It predicted that +`sort: throughput` would fix the problem. It did not: after deploying the +sort and replaying all 28 dead letters, all 28 failed again on DeepInfra. +The correlation between "cheapest" and "the one that failed" was a +coincidence of DeepInfra also being the only schema-capable host. + +The lesson is about method rather than routing: **a hypothesis that explains +the failure is not the same as one that predicts the fix.** The replay was +what falsified it, and the one-parameter-at-a-time isolation is what found +the real cause. + +## What `allow_fallbacks` does and does not do + +Still true, and still worth knowing: OpenRouter documents `allow_fallbacks` +as allowing backups when the primary is *"unavailable"* without enumerating +which HTTP codes qualify, and a provider-returned 429 is surfaced as +`"Provider returned error"` rather than triggering a re-route. Do not rely +on it for rate-limit failover. + +But that was not what broke this run. Even perfect 429 failover would have +had nowhere to go. + +## What to do instead + +1. **Check capability, not just identity, when building an allowlist.** Query + `/models/{id}/endpoints` and intersect `supported_parameters` with what the + caller actually sends. An allowlist of four that supports one is a + single point of failure wearing a disguise. +2. **Widen the allowlist to schema-capable providers** for this model + (Fireworks, BaseTen, Cloudflare and others advertise it). +3. **Bring your own key** for the pinned provider (`is_byok: true`) to swap + the shared pool for dedicated limits. +4. **Treat concurrency as part of routing.** The same allowlist ran 1,842 + calls with one failure at 1 worker per stage, and collapsed at the + reference topology (extract 8 / normalize 6 / adjudicate 4 / embed 2). + Extraction survived only because it ran before the scale-up. + +`chat_provider_sort` remains a reasonable setting and is retained — biasing +toward throughput is sensible when there is genuinely more than one eligible +host. It simply cannot help when the eligible set has one member. diff --git a/src/rememberstack/adapters/openrouter.py b/src/rememberstack/adapters/openrouter.py index 6c0cad013..ffc538bc6 100644 --- a/src/rememberstack/adapters/openrouter.py +++ b/src/rememberstack/adapters/openrouter.py @@ -10,6 +10,7 @@ import time from typing import Any from typing import Final +from typing import Literal from typing import TypeVar import httpx @@ -115,6 +116,38 @@ class OpenRouterSettings(BaseSettings): endpoint labels; routing uses the base provider slug from the endpoints API. See ``design/operations/openrouter-embedding-routing.md``. """ + chat_provider_only: list[str] | None = None + """Allowlist of OpenRouter providers for CHAT completions (``provider.only``). + + Env: ``REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_ONLY`` as a comma-separated + list of *provider slugs* (not quantization tags): e.g. + ``z-ai,novita,deepinfra,gmicloud``. Tags like ``deepinfra/fp8`` are endpoint + labels; routing uses the base slug from the endpoints API. + + ``allow_fallbacks`` stays ON, which does not weaken the restriction: + ``only`` bounds the candidate pool, so fallback moves between the listed + providers and can never leave them. That matters for long ingestion runs, + where one host's 5xx or 429 would otherwise dead-letter a stage. + + Unset means OpenRouter's ordinary marketplace routing, which is the shipped + default and the behaviour every existing deployment keeps. + """ + chat_provider_sort: Literal["price", "throughput", "latency"] | None = None + """How OpenRouter orders the allowed CHAT providers (``provider.sort``). + + Env: ``REMEMBERSTACK_OPENROUTER_CHAT_PROVIDER_SORT``. + + Unset uses OpenRouter's default routing, which weights price heavily; + ``throughput`` prefers the fastest eligible endpoint instead. + + This only helps when more than one provider is actually eligible. + ``chat_provider_only`` bounds the pool by NAME while the request bounds + it by CAPABILITY: every chat call here sends a strict ``json_schema``, so + only providers advertising ``structured_outputs`` can serve it. An + allowlist can therefore collapse to a single host, after which neither + this setting nor ``allow_fallbacks`` has anywhere to go -- see + plan/analysis/openrouter_provider_routing_overload.md. + """ reasoning_effort: ReasoningEffort | None = None reasoning_effort_map: dict[str, ReasoningEffort] | None = None """Optional per-model effort overrides as a JSON object env var @@ -134,6 +167,7 @@ class OpenRouterSettings(BaseSettings): @field_validator( "embedding_provider", + "chat_provider_sort", "reasoning_effort", "invalid_completion_capture_dir", mode="before", @@ -161,6 +195,12 @@ def parse_embedding_provider_order(cls, value: object) -> object: value=value, field_name="embedding_provider_order" ) + @field_validator("chat_provider_only", mode="before") + @classmethod + def parse_chat_provider_only(cls, value: object) -> object: + """Parse comma-separated or JSON list of OpenRouter provider slugs.""" + return _parse_provider_name_list(value=value, field_name="chat_provider_only") + @field_validator("max_completion_tokens", mode="before") @classmethod def default_empty_max_completion_tokens(cls, value: object) -> object: @@ -256,6 +296,9 @@ def generate( effort = self._reasoning_effort_for(request=request) if effort is not None: payload["reasoning"] = {"effort": effort} + provider = self._chat_provider_payload() + if provider is not None: + payload["provider"] = provider content, usage, body = self._completion_text( payload=payload, response_type=response_type, started_ns=started_ns @@ -434,6 +477,25 @@ def _completion_usage( " metadata did not recover it" ) from last_error + def _chat_provider_payload(self) -> dict[str, object] | None: + """Build OpenRouter provider routing for chat completions. + + ``only`` bounds the candidate pool to the configured slugs, so keeping + ``allow_fallbacks`` on lets a single host's 5xx/429 move to another + ALLOWED provider instead of dead-lettering the stage. It cannot route + outside the allowlist. + """ + allowed = self._settings.chat_provider_only + sort = self._settings.chat_provider_sort + if not allowed and sort is None: + return None + payload: dict[str, object] = {"allow_fallbacks": True} + if allowed: + payload["only"] = list(allowed) + if sort is not None: + payload["sort"] = sort + return payload + def _embedding_provider_payload(self) -> dict[str, object] | None: """Build OpenRouter provider routing for embedding requests. diff --git a/src/rememberstack/profiles/selfhost.py b/src/rememberstack/profiles/selfhost.py index 01aee0c75..2f9029a6b 100644 --- a/src/rememberstack/profiles/selfhost.py +++ b/src/rememberstack/profiles/selfhost.py @@ -1341,6 +1341,15 @@ def _model_bindings() -> dict[str, str]: if openrouter.max_completion_tokens is not None else "unset" ), + # Which providers were allowed to serve chat completions is measurement + # provenance: two runs on the same model but different hosts are not + # necessarily comparable (quantization, context, throughput all differ). + "openrouter_chat_provider_only": ( + ",".join(openrouter.chat_provider_only) + if openrouter.chat_provider_only + else "unset" + ), + "openrouter_chat_provider_sort": openrouter.chat_provider_sort or "default", "openrouter_reasoning_effort": openrouter.reasoning_effort or "auto", # Canonical (sorted-key) form so the effective per-model effort policy # is part of measurement provenance, not hidden behind the global pin. diff --git a/src/tests/adapters/test_openrouter.py b/src/tests/adapters/test_openrouter.py index a4d1fc3f0..9745667b8 100644 --- a/src/tests/adapters/test_openrouter.py +++ b/src/tests/adapters/test_openrouter.py @@ -967,6 +967,181 @@ def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: assert response.output.answer == "Prague" +def test_chat_provider_only_restricts_generation_to_the_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """An allowlist bounds the pool; fallback may move only inside it.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings( + api_key="test-key", + chat_provider_only=["z-ai", "novita", "deepinfra", "gmicloud"], + ) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/chat/completions" + assert payload["provider"] == { + "only": ["z-ai", "novita", "deepinfra", "gmicloud"], + # ON is not a loophole: `only` already bounds the candidate pool, so + # this buys failover between allowed hosts, never outside them. + "allow_fallbacks": True, + } + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + response = provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + assert response.output.answer == "Prague" + + +def test_chat_provider_sort_moves_load_off_the_cheapest_host( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Sorting biases routing toward the fastest ELIGIBLE provider. + + Deliberately modest about what this buys. A conv-48 ingestion + dead-lettered 28 items on one overloaded host and sorting did NOT fix it: + the strict json_schema every chat call sends left exactly one + schema-capable provider in the allowlist, so there was nothing to + reorder. Sorting helps only when the eligible set has more than one + member -- see plan/analysis/openrouter_provider_routing_overload.md. + """ + provider = OpenRouterModelProvider( + settings=OpenRouterSettings( + api_key="test-key", + chat_provider_only=["z-ai", "novita", "deepinfra", "gmicloud"], + chat_provider_sort="throughput", + ) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert payload["provider"] == { + "allow_fallbacks": True, + "only": ["z-ai", "novita", "deepinfra", "gmicloud"], + "sort": "throughput", + } + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + +def test_chat_provider_sort_applies_without_an_allowlist( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Sorting is useful on its own; it must not require pinning providers.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", chat_provider_sort="throughput") + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert payload["provider"] == {"allow_fallbacks": True, "sort": "throughput"} + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + +def test_chat_provider_only_is_absent_when_unset( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Existing deployments keep ordinary marketplace routing untouched.""" + provider = OpenRouterModelProvider(settings=OpenRouterSettings(api_key="test-key")) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert "provider" not in payload + return { + "model": "z-ai/glm-5.3-flash", + "usage": {"prompt_tokens": 2, "completion_tokens": 1, "cost": "0"}, + "choices": [{"message": {"content": '{"answer":"Prague"}'}}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.generate( + request=ModelRequest(model="z-ai/glm-5.3-flash", prompt="Where?"), + response_type=_Answer, + ) + finally: + provider._client.close() + + +def test_chat_provider_only_is_not_forwarded_to_embeddings( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Chat routing must not constrain independently hosted embedding models.""" + provider = OpenRouterModelProvider( + settings=OpenRouterSettings(api_key="test-key", chat_provider_only=["z-ai"]) + ) + + def post(*, path: str, payload: dict[str, object]) -> dict[str, object]: + assert path == "/embeddings" + assert "provider" not in payload + return { + "model": "qwen/qwen3-embedding-8b", + "usage": {"prompt_tokens": 2, "cost": "0.000001"}, + "data": [{"index": 0, "embedding": [0.1, 0.2]}], + } + + monkeypatch.setattr(provider, "_post", post) + try: + provider.embed( + request=EmbeddingRequest( + model="qwen/qwen3-embedding-8b", texts=("memory",), dimensions=2 + ) + ) + finally: + provider._client.close() + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + ("z-ai,novita", ["z-ai", "novita"]), + (' ["z-ai", "deepinfra"] ', ["z-ai", "deepinfra"]), + ("", None), + ], +) +def test_chat_provider_only_parses_env_shapes( + configured: str, expected: list[str] | None +) -> None: + """Compose passes strings; both comma and JSON list forms must work.""" + settings = OpenRouterSettings.model_validate( + {"api_key": "test-key", "chat_provider_only": configured} + ) + assert settings.chat_provider_only == expected + + def _completion(*, content: object, finish: str = "stop", cost: str = "0.0001") -> dict: """One provider chat-completion body with the given message content.""" return { diff --git a/src/tests/benchmarks/test_locomo_protocol.py b/src/tests/benchmarks/test_locomo_protocol.py index 020addddc..00aeb3944 100644 --- a/src/tests/benchmarks/test_locomo_protocol.py +++ b/src/tests/benchmarks/test_locomo_protocol.py @@ -241,8 +241,8 @@ def test_current_protocol_pins_manifest_and_complete_read_plane() -> None: def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> None: """The current identity, bounded inference, and loop discipline are locked.""" - assert PROTOCOL_NAME == "RS-LoCoMo-Full-v20" - assert DEFAULT_PROTOCOL_KEY == "full-v20" + assert PROTOCOL_NAME == "RS-LoCoMo-Full-v21" + assert DEFAULT_PROTOCOL_KEY == "full-v21" prompt = ANSWER_AGENT_PROMPT_TEMPLATE normalized_prompt = " ".join(prompt.split()) assert ( @@ -274,11 +274,11 @@ def test_protocol_is_v20_and_answer_prompt_has_reasoning_and_loop_guards() -> No def test_typed_protocol_registry_pins_answer_agent_identity_and_effort() -> None: - assert tuple(PROTOCOL_REGISTRY) == ("full-v20",) - protocol = PROTOCOL_REGISTRY["full-v20"] + assert tuple(PROTOCOL_REGISTRY) == ("full-v21",) + protocol = PROTOCOL_REGISTRY["full-v21"] - assert protocol.name == "RS-LoCoMo-Full-v20" - assert protocol.answer_agent_model == "openai/gpt-5.6-luna" + assert protocol.name == "RS-LoCoMo-Full-v21" + assert protocol.answer_agent_model == "z-ai/glm-5.3-flash" assert protocol.answer_agent_reasoning_effort == "none" assert protocol.judge_reasoning_effort == "none" assert protocol.answer_reader_retry_budget == 2 @@ -315,7 +315,7 @@ def fake_prepare_run(**values: object) -> _Prepared: ) assert exit_code == 0 - assert selected == ["full-v20"] + assert selected == ["full-v21"] def test_summarize_cli_accepts_multiple_run_flags( diff --git a/src/tests/benchmarks/test_locomo_runner.py b/src/tests/benchmarks/test_locomo_runner.py index cb3b89f33..37fa363d9 100644 --- a/src/tests/benchmarks/test_locomo_runner.py +++ b/src/tests/benchmarks/test_locomo_runner.py @@ -1033,7 +1033,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: assert answer.agent_call_count == 2 assert answer.reader_usage is not None assert answer.reader_usage.model_name == ( - "mixed:openai/gpt-5.6-luna|openai/not-luna" + "mixed:openai/not-luna|z-ai/glm-5.3-flash" ) assert answer.reader_usage.cost_usd == Decimal("0.02") assert state.evaluator_cost_usd == Decimal("0.02") @@ -1047,7 +1047,7 @@ def test_answer_persists_usage_when_provider_drifts_after_tool_call() -> None: "invalid_first_step_completions", "invalid_reader_completions", ), - (("full-v20", "openai/gpt-5.6-luna", "none", 0, 2),), + (("full-v21", "z-ai/glm-5.3-flash", "none", 0, 2),), ) def test_staged_mock_run_uses_prepared_protocol_and_resumes( protocol: ProtocolKey, @@ -1906,8 +1906,8 @@ def test_single_run_summary_json_is_unchanged( serialized = summarize_run(run_dir=run_dir).model_dump_json() assert serialized == ( - '{"protocol_name":"RS-LoCoMo-Full-v20","protocol_fingerprint":' - '"e1a02aedf2adcf006168ab17c0dd0e9e963a5296db4c38bdaae0d977bfaa7d36",' + '{"protocol_name":"RS-LoCoMo-Full-v21","protocol_fingerprint":' + '"ba5ab840261a609f1dc2bccfc0936483991e12db34c7b1d96aebd5ed49925e09",' '"tier":"smoke","questions":1,"judge_correct":0,"judge_percent":0.0,' '"official_f1":0.0,"categories":[{"category":1,"questions":0,' '"judge_correct":0,"judge_percent":0.0,"official_f1":0.0},{"category":2,' @@ -2126,8 +2126,8 @@ def test_prepared_protocol_pins_current_surface_and_luna( dataset_path=tmp_path / "synthetic.json", tier="smoke", output=run_dir ) - assert prepared.protocol_name == "RS-LoCoMo-Full-v20" - assert prepared.answer_agent_model == "openai/gpt-5.6-luna" + assert prepared.protocol_name == "RS-LoCoMo-Full-v21" + assert prepared.answer_agent_model == "z-ai/glm-5.3-flash" assert prepared.answer_agent_reasoning_effort == "none" assert prepared.answer_reader_retry_budget == 2 assert prepared.surface_manifest_hash == EXPECTED_SURFACE_MANIFEST_HASH diff --git a/src/tests/benchmarks/test_locomo_store_backup.py b/src/tests/benchmarks/test_locomo_store_backup.py index a31767548..66d5dbd9a 100644 --- a/src/tests/benchmarks/test_locomo_store_backup.py +++ b/src/tests/benchmarks/test_locomo_store_backup.py @@ -24,7 +24,7 @@ def _run_json(path: Path) -> None: (path / "run.json").write_text( json.dumps( { - "protocol_name": "RS-LoCoMo-Full-v20", + "protocol_name": "RS-LoCoMo-Full-v21", "protocol_fingerprint": "p" * 64, "repository_revision": "r" * 40, "prepared_at": "2026-08-11T00:00:00Z", @@ -660,7 +660,7 @@ def test_restore_validates_every_archive_before_running_docker( deployment_id="57000000-0000-0000-0000-000000000001", compose_project="rememberstack", run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision="r" * 40, prepared_at="2026-08-11T00:00:00Z", @@ -922,7 +922,7 @@ def test_runtime_validation_uses_the_image_revision_stamp( sample_id="conv-1", deployment_id=deployment_id, run=store_backup.RunIdentity( - protocol_name="RS-LoCoMo-Full-v20", + protocol_name="RS-LoCoMo-Full-v21", protocol_fingerprint="p" * 64, repository_revision=revision, prepared_at="2026-08-11T00:00:00Z", @@ -1019,8 +1019,8 @@ def test_shard_runner_guards_wipe_and_backs_up_before_scoring() -> None: assert "LOCOMO_BACKUP_DESTINATION must be" in script assert "LOCOMO_BACKUP_TOOL:-benchmarks/locomo/sharding/store_backup.py" in script assert 'compose=(docker compose --project-name "$compose_project")' in script - assert "REMEMBERSTACK_E2_EXTRACT_MODEL=openai/gpt-5.6-luna" in script - assert "REMEMBERSTACK_OBS_FRONTIER_MODEL=openai/gpt-5.6-luna" in script + assert "REMEMBERSTACK_E2_EXTRACT_MODEL=z-ai/glm-5.3-flash" in script + assert "REMEMBERSTACK_OBS_FRONTIER_MODEL=z-ai/glm-5.3-flash" in script assert "REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER=nebius" in script assert "unset REMEMBERSTACK_OPENROUTER_EMBEDDING_PROVIDER_ORDER" in script assert 'published=$("${compose[@]}" port api 8000 | head -n 1)' in script