Preserve native media and expose transport input support

crates/tinyinference-llm/Cargo.toml

@@ -12,6 +12,7 @@ keywords = ["llm", "inference", "streaming", "openai", "anthropic"]
 categories = ["api-bindings", "asynchronous"]
 
 [dependencies]
+base64 = "0.22"
 async-trait = { workspace = true }
 bytes = { workspace = true }
 futures = { workspace = true }

crates/tinyinference-llm/src/catalog/mod_tests.rs

@@ -87,6 +87,7 @@ fn null_is_empty_only_for_success_envelopes() {
 #[test]
 fn codex_hints_are_merged_without_duplicates() {
     let mut models = vec![ModelInfo {
+        input_modalities: None,
         id: "gpt-5.4".into(),
         owned_by: None,
         context_window: None,
@@ -117,3 +118,17 @@ fn blank_preferred_identifier_falls_back_to_the_next_usable_field() {
     assert_eq!(models[0].id, "usable-slug");
     assert_eq!(models[1].id, "usable-name");
 }
+
+#[test]
+fn catalog_preserves_advertised_input_modalities() {
+    let models = parse_models_response(&serde_json::json!({"data":[
+        {"id":"vision","architecture":{"input_modalities":["text","image"]}},
+        {"id":"unknown"}
+    ]}))
+    .unwrap();
+    assert_eq!(
+        models[0].input_modalities,
+        Some(vec!["text".into(), "image".into()])
+    );
+    assert_eq!(models[1].input_modalities, None);
+}

crates/tinyinference-llm/src/catalog/parsing.rs

@@ -128,6 +128,7 @@ pub fn merge_openai_codex_model_hints(models: &mut Vec<ModelInfo>) {
                 id: (*id).to_string(),
                 owned_by: Some("openai-codex".to_string()),
                 context_window: None,
+                input_modalities: None,
                 display_name: None,
                 input_per_1m: None,
                 output_per_1m: None,
@@ -142,6 +143,7 @@ fn model_info_from_catalog_item(item: &serde_json::Value) -> Option<ModelInfo> {
             id: id.to_string(),
             owned_by: None,
             context_window: None,
+            input_modalities: None,
             display_name: None,
             input_per_1m: None,
             output_per_1m: None,
@@ -184,6 +186,7 @@ fn model_info_from_catalog_item(item: &serde_json::Value) -> Option<ModelInfo> {
         id,
         owned_by,
         context_window,
+        input_modalities: crate::model::discover::input_modalities_from_entry(item),
         display_name,
         input_per_1m: price("inputPer1M"),
         output_per_1m: price("outputPer1M"),

crates/tinyinference-llm/src/catalog/types.rs

@@ -1,15 +1,18 @@
 //! `ModelInfo`: the typed representation of one `/models` catalog entry.
 
-use serde::Serialize;
+use serde::{Deserialize, Serialize};
 
-#[derive(Debug, Clone, PartialEq, Serialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 /// Normalized metadata for one model advertised by a provider catalog.
 pub struct ModelInfo {
     /// Provider model identifier used in inference requests.
     pub id: String,
     /// Provider or organization that owns the model, when advertised.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub owned_by: Option<String>,
+    /// Advertised input modalities; `None` means unknown, never assumed vision.
+    #[serde(default, skip_serializing_if = "Option::is_none")]
+    pub input_modalities: Option<Vec<String>>,
     /// Maximum context length in tokens, when advertised.
     #[serde(skip_serializing_if = "Option::is_none")]
     pub context_window: Option<u64>,

crates/tinyinference-llm/src/lib.rs

@@ -26,8 +26,8 @@ pub use failure::{
 };
 pub use message::{AssistantMessage, ContentBlock, Message, MessageDelta};
 pub use model::{
-    ChatModel, ModelRequest, ModelResponse, ModelStream, ModelStreamItem,
-    context_window_for_model_id, model_id_supports_vision,
+    ChatModel, InputModality, InputSource, ModelRequest, ModelResponse, ModelStream,
+    ModelStreamItem, context_window_for_model_id, model_id_supports_vision,
 };
 pub use network_guard::{allow_network_models, deny_network_models, network_models_denied};
 pub use providers::{MockModel, ProviderKind, ProviderSpec};

crates/tinyinference-llm/src/model/decorators.rs

@@ -93,6 +93,15 @@ impl<State: Send + Sync> ChatModel<State> for ProfileOverrideModel<State> {
         Some(&self.profile)
     }
 
+    fn supports_input(
+        &self,
+        modality: super::InputModality,
+        mime_type: &str,
+        source: super::InputSource,
+    ) -> bool {
+        self.inner.supports_input(modality, mime_type, source)
+    }
+
     fn cache_identity(&self) -> Option<String> {
         self.inner.cache_identity()
     }
@@ -144,6 +153,15 @@ impl<State: Send + Sync> ChatModel<State> for MaxTokensModel<State> {
         self.inner.profile()
     }
 
+    fn supports_input(
+        &self,
+        modality: super::InputModality,
+        mime_type: &str,
+        source: super::InputSource,
+    ) -> bool {
+        self.inner.supports_input(modality, mime_type, source)
+    }
+
     fn cache_identity(&self) -> Option<String> {
         self.inner.cache_identity()
     }
@@ -201,6 +219,15 @@ impl<State: Send + Sync> ChatModel<State> for RouteRecordingModel<State> {
         self.inner.profile()
     }
 
+    fn supports_input(
+        &self,
+        modality: super::InputModality,
+        mime_type: &str,
+        source: super::InputSource,
+    ) -> bool {
+        self.inner.supports_input(modality, mime_type, source)
+    }
+
     fn cache_identity(&self) -> Option<String> {
         self.inner.cache_identity()
     }
@@ -362,6 +389,15 @@ impl<State: Send + Sync> ChatModel<State> for ObservingModel<State> {
         self.inner.profile()
     }
 
+    fn supports_input(
+        &self,
+        modality: super::InputModality,
+        mime_type: &str,
+        source: super::InputSource,
+    ) -> bool {
+        self.inner.supports_input(modality, mime_type, source)
+    }
+
     fn cache_identity(&self) -> Option<String> {
         self.inner.cache_identity()
     }

crates/tinyinference-llm/src/model/discover/cache.rs

@@ -74,13 +74,15 @@ impl CachedLimits {
                 Some(ModelLimits {
                     context_window: Some(learned),
                     max_output_tokens: limits.max_output_tokens,
+                    input_modalities: limits.input_modalities,
                     source: LimitSource::LearnedFromOverflow,
                 })
             }
             (Some(limits), _) => Some(limits),
             (None, Some(learned)) => Some(ModelLimits {
                 context_window: Some(learned),
                 max_output_tokens: None,
+                input_modalities: None,
                 source: LimitSource::LearnedFromOverflow,
             }),
             (None, None) => None,
@@ -140,7 +142,9 @@ impl ModelLimitsCache {
 
     /// The fresh cached state for `(endpoint, model)` across every discovery
     /// variant: the smallest fresh discovered window, lowered by any learned
-    /// overflow window. Use [`Self::get_variant`] for one request's slot.
+    /// overflow window. Input modalities intersect fresh variant facts; any
+    /// unknown variant makes the aggregate unknown. Use [`Self::get_variant`]
+    /// for one request's slot.
     #[must_use]
     pub fn get(&self, endpoint: &str, model: &str) -> CachedLimits {
         self.get_at(endpoint, model, Instant::now())
@@ -159,9 +163,24 @@ impl ModelLimitsCache {
             // else a fresh negative, else nothing.
             let mut best: Option<ModelLimits> = None;
             let mut negative = false;
+            let mut modalities: Option<Vec<String>> = None;
+            let mut saw_modalities = false;
+            let mut unknown_modalities = false;
             for discovered in entry.discovered.values() {
                 match self.fresh(discovered, now) {
                     Some(Some(limits)) => {
+                        if let Some(values) = &limits.input_modalities {
+                            if saw_modalities {
+                                if let Some(common) = &mut modalities {
+                                    common.retain(|value| values.contains(value));
+                                }
+                            } else {
+                                modalities = Some(values.clone());
+                                saw_modalities = true;
+                            }
+                        } else {
+                            unknown_modalities = true;
+                        }
                         let smaller = best.as_ref().is_none_or(|current| {
                             limits.context_window.unwrap_or(u64::MAX)
                                 < current.context_window.unwrap_or(u64::MAX)
@@ -170,10 +189,24 @@ impl ModelLimitsCache {
                             best = Some(limits);
                         }
                     }
-                    Some(None) => negative = true,
+                    Some(None) => {
+                        negative = true;
+                        unknown_modalities = true;
+                    }
                     None => {}
                 }
             }
+            if let Some(best) = &mut best {
+                best.input_modalities = if unknown_modalities {
+                    None
+                } else {
+                    modalities.map(|mut values| {
+                        values.sort();
+                        values.dedup();
+                        values
+                    })
+                };
+            }
             CachedLimits {
                 discovered: best.map(Some).or(negative.then_some(None)),
                 learned_context_window: self.fresh_learned(entry, now),

crates/tinyinference-llm/src/model/discover/cache_tests.rs

@@ -7,6 +7,7 @@ fn limits(window: u64) -> ModelLimits {
     ModelLimits {
         context_window: Some(window),
         max_output_tokens: Some(8_192),
+        input_modalities: None,
         source: LimitSource::ProviderListing,
     }
 }
@@ -146,3 +147,104 @@ fn clear_drops_everything() {
     cache.clear();
     assert_eq!(cache.get(ENDPOINT, MODEL), CachedLimits::default());
 }
+
+fn variant(window: u64, modalities: Option<&[&str]>) -> ModelLimits {
+    ModelLimits {
+        input_modalities: modalities
+            .map(|values| values.iter().map(|value| (*value).to_owned()).collect()),
+        ..limits(window)
+    }
+}
+
+#[test]
+fn aggregate_modalities_intersect_independently_of_window_and_ties() {
+    for windows in [(32_000, 64_000), (64_000, 32_000), (32_000, 32_000)] {
+        let cache = cache();
+        let now = Instant::now();
+        cache.insert_discovered_variant_at(
+            ENDPOINT,
+            MODEL,
+            "image",
+            Some(variant(windows.0, Some(&["text", "image"]))),
+            now,
+        );
+        cache.insert_discovered_variant_at(
+            ENDPOINT,
+            MODEL,
+            "text",
+            Some(variant(windows.1, Some(&["text"]))),
+            now,
+        );
+        let aggregate = cache.get_at(ENDPOINT, MODEL, now).effective().unwrap();
+        assert_eq!(aggregate.context_window, Some(windows.0.min(windows.1)));
+        assert_eq!(aggregate.input_modalities, Some(vec!["text".into()]));
+        assert_eq!(
+            cache
+                .get_variant_at(ENDPOINT, MODEL, "image", now)
+                .effective()
+                .unwrap()
+                .input_modalities,
+            Some(vec!["text".into(), "image".into()])
+        );
+    }
+}
+
+#[test]
+fn aggregate_modalities_preserve_unknown_negative_empty_and_expiration() {
+    let cache = cache();
+    let now = Instant::now();
+    cache.insert_discovered_variant_at(
+        ENDPOINT,
+        MODEL,
+        "image",
+        Some(variant(32_000, Some(&["image", "text"]))),
+        now,
+    );
+    cache.insert_discovered_variant_at(
+        ENDPOINT,
+        MODEL,
+        "unknown",
+        Some(variant(64_000, None)),
+        now,
+    );
+    assert_eq!(
+        cache
+            .get_at(ENDPOINT, MODEL, now)
+            .effective()
+            .unwrap()
+            .input_modalities,
+        None
+    );
+    cache.insert_discovered_variant_at(
+        ENDPOINT,
+        MODEL,
+        "unknown",
+        Some(variant(64_000, Some(&[]))),
+        now,
+    );
+    assert_eq!(
+        cache
+            .get_at(ENDPOINT, MODEL, now)
+            .effective()
+            .unwrap()
+            .input_modalities,
+        Some(vec![])
+    );
+    cache.insert_discovered_variant_at(ENDPOINT, MODEL, "unknown", None, now);
+    assert_eq!(
+        cache
+            .get_at(ENDPOINT, MODEL, now)
+            .effective()
+            .unwrap()
+            .input_modalities,
+        None
+    );
+    assert_eq!(
+        cache
+            .get_at(ENDPOINT, MODEL, now + Duration::from_secs(10))
+            .effective()
+            .unwrap()
+            .input_modalities,
+        Some(vec!["image".into(), "text".into()])
+    );
+}

crates/tinyinference-llm/src/model/discover/fetch.rs

@@ -104,8 +104,9 @@ impl ModelListingFetcher for ReqwestListingFetcher {
 /// - A fresh cache entry (found or not found) answers without any request.
 /// - Otherwise the listing is read (`{endpoint}/models` or
 ///   `request.listing_url`); every entry it lists is cached, so one fetch
-///   serves every model on the endpoint. When the model is missing and
-///   `probe_single_model` is set, `{endpoint}/models/{id}` is tried. When
+///   serves every model on the endpoint. When the model or either token limit
+///   is missing and `probe_single_model` is set, `{endpoint}/models/{id}` is
+///   tried; missing fields are filled without discarding listing modalities. When
 ///   `pinned_providers` is non-empty, OpenRouter's
 ///   `{endpoint}/models/{id}/endpoints` narrows the window to those providers.
 /// - The whole probe is bounded by `request.timeout`; a timeout or any error
@@ -209,10 +210,27 @@ async fn fetch_limits(
         }
     };
 
-    if found.is_none() && request.probe_single_model {
+    if found
+        .as_ref()
+        .is_none_or(|limits| limits.context_window.is_none() || limits.max_output_tokens.is_none())
+        && request.probe_single_model
+    {
         let url = format!("{}/models/{}", request.endpoint, request.model);
         match fetcher.get_json(&url, &request.headers).await {
-            Ok(body) => found = parse_model_limits(&body, &request.model),
+            Ok(body) => {
+                if let Some(record) = parse_model_limits(&body, &request.model) {
+                    if let Some(listed) = &mut found {
+                        listed.context_window = listed.context_window.or(record.context_window);
+                        listed.max_output_tokens =
+                            listed.max_output_tokens.or(record.max_output_tokens);
+                        if listed.input_modalities.is_none() {
+                            listed.input_modalities = record.input_modalities;
+                        }
+                    } else {
+                        found = Some(record);
+                    }
+                }
+            }
             Err(error) => tracing::debug!(
                 endpoint = %request.endpoint,
                 model = %request.model,