diff --git a/README.md b/README.md index 42b43be..dafeaf7 100644 --- a/README.md +++ b/README.md @@ -99,6 +99,7 @@ production prompts before relying on the numbers. | Routing feature | Source | |---|---| | Task and complexity | The classifier; `low` leaves work on the cheapest capable model, `high` outweighs the specialization and cost terms. | +| Required modality | `routing.modalities` lists the inputs a request needs (`text`, `image`, `audio`); a model whose card does not list them all is never a candidate. | | Structured-output requirement | `routing.structured` excludes any model whose card sets `supports_structured_output: false`. | | Quality by task and model | Mean benchmarked quality for the task from the catalog; the card's headline `quality` only for a task never measured. | | Current queue delay | An exponentially weighted average of what requests for that model actually waited, replacing the catalog estimate after the first observation. | @@ -106,7 +107,9 @@ production prompts before relying on the numbers. One gateway faces one engine, so saturation costs every local model equally: it can tip an eligible request to the approved external model, and it never reorders local models or overrides -privacy. Required modality is not a routing feature yet; message content is text only. +privacy. Modality is a routing restriction only: every model in the committed catalog is text-only +and message content is text, so a request that requires `image` or `audio` is refused with `422` +until a card declares that modality. Every response reports `task_source` (`declared`, `classifier`, `abstained`, `keyword`, or `cached`), `task_confidence`, and `complexity` beside the route reason. diff --git a/src/llm_router/governance.py b/src/llm_router/governance.py index f78a11f..86ed7a0 100644 --- a/src/llm_router/governance.py +++ b/src/llm_router/governance.py @@ -184,6 +184,7 @@ def _model_version(card: ModelCard, artifact_root: str, checksum: str | None) -> f"{hardware.count}x {hardware.accelerator}, " f"{hardware.minimum_memory_gb} GB, tensor parallel {hardware.tensor_parallel_size}" ), + "catalog.modalities": ",".join(sorted(item.value for item in card.modalities)), "catalog.supported_tasks": ",".join( sorted(task.value for task in card.supported_tasks) ), diff --git a/src/llm_router/models.py b/src/llm_router/models.py index f2a621e..5a3b253 100644 --- a/src/llm_router/models.py +++ b/src/llm_router/models.py @@ -20,6 +20,15 @@ class PrivacyClass(StrEnum): RESTRICTED = "restricted" +class Modality(StrEnum): + TEXT = "text" + IMAGE = "image" + AUDIO = "audio" + + +TEXT_ONLY: frozenset[Modality] = frozenset({Modality.TEXT}) + + class ChatMessage(BaseModel): role: Literal["system", "user", "assistant", "tool"] content: str @@ -35,6 +44,9 @@ class RoutingOptions(BaseModel): # Declaring a structured-output requirement lets the gateway check the one # quality signal live traffic exposes: whether the response actually parsed. structured: bool = False + # Every input modality the request needs a model to accept. A model + # that lacks one is never a candidate, however it would have scored. + modalities: frozenset[Modality] = Field(default=TEXT_ONLY, min_length=1) class ChatCompletionRequest(BaseModel): @@ -64,6 +76,7 @@ class ModelProfile(BaseModel): # `quality` figure is only the fallback for a task never measured. quality_by_task: dict[TaskClass, float] = Field(default_factory=dict) supports_structured_output: bool = True + modalities: frozenset[Modality] = TEXT_ONLY def quality_for(self, task: TaskClass) -> float: return self.quality_by_task.get(task, self.quality) diff --git a/src/llm_router/registry.py b/src/llm_router/registry.py index 227829a..74eb985 100644 --- a/src/llm_router/registry.py +++ b/src/llm_router/registry.py @@ -13,7 +13,7 @@ import yaml from pydantic import BaseModel, Field, model_validator -from llm_router.models import ModelProfile, PrivacyClass, TaskClass +from llm_router.models import TEXT_ONLY, Modality, ModelProfile, PrivacyClass, TaskClass # Privacy classes are ordered so a tenant floor can be compared with what a # request declared; a floor may only ever raise the effective class. @@ -77,6 +77,8 @@ class ModelCard(BaseModel): stage: LifecycleStage = LifecycleStage.DEVELOPMENT healthy: bool = True supports_structured_output: bool = True + # Input modalities the model accepts; text unless the card says more. + modalities: frozenset[Modality] = Field(default=TEXT_ONLY, min_length=1) intended_tasks: str limitations: str evaluation_references: tuple[str, ...] = () @@ -91,6 +93,7 @@ def to_profile(self, quality_by_task: dict[TaskClass, float] | None = None) -> M return ModelProfile( quality_by_task=quality_by_task or {}, supports_structured_output=self.supports_structured_output, + modalities=self.modalities, id=self.id, revision=self.revision, local=self.local, diff --git a/src/llm_router/routing.py b/src/llm_router/routing.py index 7fe252f..a5bf611 100644 --- a/src/llm_router/routing.py +++ b/src/llm_router/routing.py @@ -167,6 +167,9 @@ def select( # cannot produce structured output is never a candidate for a # request that requires it, whatever it would have scored. and (not request.routing.structured or profile.supports_structured_output) + # Likewise for modality: a model that cannot take an input the + # request carries cannot answer it at all. + and request.routing.modalities <= profile.modalities and self._privacy_allows(profile, effective_privacy) and self._external_allows(profile, request, tenant_allows_external) # A tenant entitlement is a hard restriction, like privacy: it is @@ -182,8 +185,8 @@ def select( if not candidates: raise NoEligibleModelError( - "no healthy model satisfies capability, structured output, context, quality, " - "privacy, tenant entitlement, and fallback policy" + "no healthy model satisfies capability, modality, structured output, context, " + "quality, privacy, tenant entitlement, and fallback policy" ) saturation = self.load.engine_saturation() if self.load is not None else 0.0 diff --git a/tests/unit/test_routing_features.py b/tests/unit/test_routing_features.py index 2cda0d3..65feb74 100644 --- a/tests/unit/test_routing_features.py +++ b/tests/unit/test_routing_features.py @@ -1,4 +1,5 @@ import pytest +from pydantic import ValidationError from llm_router.classifier import DEFAULT_DATASET, load_classifier from llm_router.engine_stats import EngineStats @@ -221,3 +222,35 @@ def test_quality_history_is_built_from_benchmarks_per_task() -> None: assert profiles["small-specialist"].quality_for(TaskClass.EXTRACTION) == pytest.approx(0.82) # No benchmark covers this task, so the card's headline figure applies. assert profiles["small-specialist"].quality_for(TaskClass.CLASSIFICATION) == small.quality + + +def test_a_request_never_reaches_a_model_that_lacks_a_modality_it_needs() -> None: + profiles = ( + profile("text-only", quality=0.99), + profile("multimodal", quality=0.80, modalities=frozenset({"text", "image"})), + ) + router = Router(profiles) + + plain = router.select(request_for("Extract the fields")) + with_image = router.select(request_for("Extract the fields", modalities=["text", "image"])) + + # The text-only model scores far higher, and still is not a candidate. + assert plain.profile.id == "text-only" + assert with_image.profile.id == "multimodal" + + +def test_a_modality_no_model_accepts_is_refused_and_an_empty_requirement_is_invalid() -> None: + router = Router((profile("text-only"),)) + + with pytest.raises(NoEligibleModelError, match="modality"): + router.select(request_for("Transcribe this", modalities=["audio"])) + with pytest.raises(ValidationError): + request_for("Extract the fields", modalities=[]) + + +def test_the_catalog_carries_modality_from_the_card_to_the_router() -> None: + catalog = load_registry("config/registry.yaml") + + assert all(item.modalities == {"text"} for item in catalog.profiles()) + card = catalog.models[0].model_copy(update={"modalities": frozenset({"text", "image"})}) + assert card.to_profile().modalities == {"text", "image"}