feat: route on the input modalities a request requires

README.md

@@ -99,14 +99,17 @@ production prompts before relying on the numbers.
 | Routing feature | Source |
 |---|---|
 | Task and complexity | The classifier; `low` leaves work on the cheapest capable model, `high` outweighs the specialization and cost terms. |
+| Required modality | `routing.modalities` lists the inputs a request needs (`text`, `image`, `audio`); a model whose card does not list them all is never a candidate. |
 | Structured-output requirement | `routing.structured` excludes any model whose card sets `supports_structured_output: false`. |
 | Quality by task and model | Mean benchmarked quality for the task from the catalog; the card's headline `quality` only for a task never measured. |
 | Current queue delay | An exponentially weighted average of what requests for that model actually waited, replacing the catalog estimate after the first observation. |
 | GPU capacity | Engine saturation — the worse of KV-cache occupancy and the share of admitted work not yet started — from the last metrics scrape, discarded after 30 seconds. |
 
 One gateway faces one engine, so saturation costs every local model equally: it can tip an
 eligible request to the approved external model, and it never reorders local models or overrides
-privacy. Required modality is not a routing feature yet; message content is text only.
+privacy. Modality is a routing restriction only: every model in the committed catalog is text-only
+and message content is text, so a request that requires `image` or `audio` is refused with `422`
+until a card declares that modality.
 
 Every response reports `task_source` (`declared`, `classifier`, `abstained`, `keyword`, or
 `cached`), `task_confidence`, and `complexity` beside the route reason.

src/llm_router/governance.py

@@ -184,6 +184,7 @@ def _model_version(card: ModelCard, artifact_root: str, checksum: str | None) ->
                 f"{hardware.count}x {hardware.accelerator}, "
                 f"{hardware.minimum_memory_gb} GB, tensor parallel {hardware.tensor_parallel_size}"
             ),
+            "catalog.modalities": ",".join(sorted(item.value for item in card.modalities)),
             "catalog.supported_tasks": ",".join(
                 sorted(task.value for task in card.supported_tasks)
             ),

src/llm_router/models.py

@@ -20,6 +20,15 @@ class PrivacyClass(StrEnum):
     RESTRICTED = "restricted"
 
 
+class Modality(StrEnum):
+    TEXT = "text"
+    IMAGE = "image"
+    AUDIO = "audio"
+
+
+TEXT_ONLY: frozenset[Modality] = frozenset({Modality.TEXT})
+
+
 class ChatMessage(BaseModel):
     role: Literal["system", "user", "assistant", "tool"]
     content: str
@@ -35,6 +44,9 @@ class RoutingOptions(BaseModel):
     # Declaring a structured-output requirement lets the gateway check the one
     # quality signal live traffic exposes: whether the response actually parsed.
     structured: bool = False
+    # Every input modality the request needs a model to accept. A model
+    # that lacks one is never a candidate, however it would have scored.
+    modalities: frozenset[Modality] = Field(default=TEXT_ONLY, min_length=1)
 
 
 class ChatCompletionRequest(BaseModel):
@@ -64,6 +76,7 @@ class ModelProfile(BaseModel):
     # `quality` figure is only the fallback for a task never measured.
     quality_by_task: dict[TaskClass, float] = Field(default_factory=dict)
     supports_structured_output: bool = True
+    modalities: frozenset[Modality] = TEXT_ONLY
 
     def quality_for(self, task: TaskClass) -> float:
         return self.quality_by_task.get(task, self.quality)

src/llm_router/registry.py

@@ -13,7 +13,7 @@
 import yaml
 from pydantic import BaseModel, Field, model_validator
 
-from llm_router.models import ModelProfile, PrivacyClass, TaskClass
+from llm_router.models import TEXT_ONLY, Modality, ModelProfile, PrivacyClass, TaskClass
 
 # Privacy classes are ordered so a tenant floor can be compared with what a
 # request declared; a floor may only ever raise the effective class.
@@ -77,6 +77,8 @@ class ModelCard(BaseModel):
     stage: LifecycleStage = LifecycleStage.DEVELOPMENT
     healthy: bool = True
     supports_structured_output: bool = True
+    # Input modalities the model accepts; text unless the card says more.
+    modalities: frozenset[Modality] = Field(default=TEXT_ONLY, min_length=1)
     intended_tasks: str
     limitations: str
     evaluation_references: tuple[str, ...] = ()
@@ -91,6 +93,7 @@ def to_profile(self, quality_by_task: dict[TaskClass, float] | None = None) -> M
         return ModelProfile(
             quality_by_task=quality_by_task or {},
             supports_structured_output=self.supports_structured_output,
+            modalities=self.modalities,
             id=self.id,
             revision=self.revision,
             local=self.local,

src/llm_router/routing.py

@@ -167,6 +167,9 @@ def select(
             # cannot produce structured output is never a candidate for a
             # request that requires it, whatever it would have scored.
             and (not request.routing.structured or profile.supports_structured_output)
+            # Likewise for modality: a model that cannot take an input the
+            # request carries cannot answer it at all.
+            and request.routing.modalities <= profile.modalities
             and self._privacy_allows(profile, effective_privacy)
             and self._external_allows(profile, request, tenant_allows_external)
             # A tenant entitlement is a hard restriction, like privacy: it is
@@ -182,8 +185,8 @@ def select(
 
         if not candidates:
             raise NoEligibleModelError(
-                "no healthy model satisfies capability, structured output, context, quality, "
-                "privacy, tenant entitlement, and fallback policy"
+                "no healthy model satisfies capability, modality, structured output, context, "
+                "quality, privacy, tenant entitlement, and fallback policy"
             )
 
         saturation = self.load.engine_saturation() if self.load is not None else 0.0

tests/unit/test_routing_features.py

@@ -1,4 +1,5 @@
 import pytest
+from pydantic import ValidationError
 
 from llm_router.classifier import DEFAULT_DATASET, load_classifier
 from llm_router.engine_stats import EngineStats
@@ -221,3 +222,35 @@ def test_quality_history_is_built_from_benchmarks_per_task() -> None:
     assert profiles["small-specialist"].quality_for(TaskClass.EXTRACTION) == pytest.approx(0.82)
     # No benchmark covers this task, so the card's headline figure applies.
     assert profiles["small-specialist"].quality_for(TaskClass.CLASSIFICATION) == small.quality
+
+
+def test_a_request_never_reaches_a_model_that_lacks_a_modality_it_needs() -> None:
+    profiles = (
+        profile("text-only", quality=0.99),
+        profile("multimodal", quality=0.80, modalities=frozenset({"text", "image"})),
+    )
+    router = Router(profiles)
+
+    plain = router.select(request_for("Extract the fields"))
+    with_image = router.select(request_for("Extract the fields", modalities=["text", "image"]))
+
+    # The text-only model scores far higher, and still is not a candidate.
+    assert plain.profile.id == "text-only"
+    assert with_image.profile.id == "multimodal"
+
+
+def test_a_modality_no_model_accepts_is_refused_and_an_empty_requirement_is_invalid() -> None:
+    router = Router((profile("text-only"),))
+
+    with pytest.raises(NoEligibleModelError, match="modality"):
+        router.select(request_for("Transcribe this", modalities=["audio"]))
+    with pytest.raises(ValidationError):
+        request_for("Extract the fields", modalities=[])
+
+
+def test_the_catalog_carries_modality_from_the_card_to_the_router() -> None:
+    catalog = load_registry("config/registry.yaml")
+
+    assert all(item.modalities == {"text"} for item in catalog.profiles())
+    card = catalog.models[0].model_copy(update={"modalities": frozenset({"text", "image"})})
+    assert card.to_profile().modalities == {"text", "image"}