diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a98d21..020f8b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,15 @@ `reasoning_effort` is omitted, since OpenRouter takes one or the other). Other OpenAI-compatible endpoints still drop the budget, and an explicit `reasoning` provider option still wins. +- OpenAI-compatible chat: adaptive parameter omission. A 400 whose message + names an optional field the request sent (`temperature`, `top_p`, `seed`, + `max_tokens`, `max_completion_tokens`, `reasoning_effort`) drops that field, + retries once, and remembers the omission for that endpoint and model. The + evidence rule and the process-wide store are public as + `providers::omission::{parameter_blamed_by, remember_omit, is_omitted}`. +- `providers::BearerSource` and `OpenAiModel::with_bearer_source`: a credential + read per request (honouring `AuthStyle`, invalidated on 401) so a rotating + token no longer forces rebuilding the model. - Anthropic: a request carrying both a reasoning `effort` and `budget_tokens` now keeps adaptive thinking with that effort, byte-identical to the effort-only request. Fixed-budget thinking is used only when no effort is diff --git a/crates/tinyinference-llm/src/providers/mod.rs b/crates/tinyinference-llm/src/providers/mod.rs index ca38127..08e53bb 100644 --- a/crates/tinyinference-llm/src/providers/mod.rs +++ b/crates/tinyinference-llm/src/providers/mod.rs @@ -41,6 +41,7 @@ mod types; // OpenAI-compatible endpoint. Providers with a different wire protocol would be // added behind their own Cargo feature. pub mod anthropic; +pub mod omission; pub mod openai; // #[cfg(feature = "ollama")] pub mod ollama; diff --git a/crates/tinyinference-llm/src/providers/omission.rs b/crates/tinyinference-llm/src/providers/omission.rs new file mode 100644 index 0000000..5aaac76 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/omission.rs @@ -0,0 +1,161 @@ +//! Learned request-parameter omission: what an endpoint has told us, by name, +//! that a model does not accept. +//! +//! A provider that rejects a request parameter with an HTTP 400 almost always +//! names it (`Unsupported parameter: 'max_tokens'`, `Unsupported value: +//! 'temperature' does not support 0.2 with this model`). An adapter can then drop +//! that one parameter, retry once, and [`remember_omit`] it so every later +//! request to the same endpoint and model leaves it off without spending the +//! round-trip again. [`OpenAiModel`](crate::providers::openai::OpenAiModel) does +//! this on its Chat Completions path; the functions are public so a host that +//! builds its own wire body can share the same evidence rule and the same store. +//! +//! This is what keeps static capability tables (such as +//! `OpenAiModel::with_temperature_unsupported_models` or the o-series +//! `max_completion_tokens` rename) an optimisation rather than a dependency: a +//! vendor that changes a restriction silently between releases is corrected by +//! one wasted round-trip (a 400 is billed nothing), not by a library release. +//! +//! Ported from OpenCompany's `company/inference/dialect.rs` learning layer. +//! +//! # Scope of what is learned +//! +//! * **Only parameters we sent.** [`parameter_blamed_by`] looks for the names +//! the caller says went out, never a vendor error format, so it cannot go +//! stale and a 400 about a field we did not send never drops one we did. +//! * **Keyed on endpoint and model.** A model id is not unique across gateways; +//! one gateway refusing `max_tokens` for `gpt-4o` says nothing about another. +//! The endpoint key keeps the base path (a path-routed gateway fronts several +//! upstreams) but drops the operation suffix, so `/chat/completions` and +//! `/responses` on one base share what was learnt. +//! * **Process-wide and in-memory.** It is a cache, not a record. Losing it on +//! restart costs one round-trip per model, and a vendor that lifts a +//! restriction is not remembered as broken forever. + +use std::collections::HashSet; +use std::sync::{OnceLock, RwLock}; + +/// Phrases that mark a provider error as a statement about the request's +/// *shape*. Without one, a body that merely mentions a parameter name (say, +/// model output echoed into an error) could talk us into dropping it. +const REJECTION_MARKERS: &[&str] = &[ + "unsupported", + "not supported", + "unsupported_value", + "unsupported_parameter", + "unknown parameter", + "unrecognized", + "not permitted", + "invalid_request_error", + "is deprecated", + "does not support", + "extra inputs", + "unexpected", +]; + +/// Operation suffixes stripped from an endpoint before it keys the store: +/// they name what was asked of a service, not which service answered. +const OPERATIONS: &[&str] = &[ + "/chat/completions", + "/completions", + "/responses", + "/messages", + "/models", +]; + +/// One learned omission: `(endpoint scope, lowercased model id, parameter)`. +type OmissionKey = (String, String, String); + +/// Which parameter, if any, a provider rejection blames — chosen only from the +/// parameters the request actually sent. +/// +/// `body` is the provider's error text (message, code, or raw body); `sent` is +/// the wire names that went out. Returns `None` when the text carries no +/// rejection marker (it is not about the request's shape) or names none of +/// `sent`. When several sent names appear, the longest wins, so a name that +/// contains another is not mistaken for it. +/// +/// Matching is against the *wire* names on purpose: after a rename +/// (`max_tokens` → `max_completion_tokens`) the field a vendor rejects is the +/// one that went out, not the one the caller asked with. +pub fn parameter_blamed_by<'a, S: AsRef>(body: &str, sent: &'a [S]) -> Option<&'a str> { + let body = body.to_ascii_lowercase(); + if !REJECTION_MARKERS.iter().any(|marker| body.contains(marker)) { + return None; + } + let mut candidates: Vec<&str> = sent.iter().map(AsRef::as_ref).collect(); + candidates.sort_by_key(|name| std::cmp::Reverse(name.len())); + candidates + .into_iter() + .find(|name| !name.is_empty() && body.contains(&name.to_ascii_lowercase())) +} + +/// Records that `model` at `endpoint` rejects `parameter`, so the next request +/// there omits it without spending a round-trip. +/// +/// Keyed on the endpoint as well as the model because that is who answered: +/// keyed on the model alone, one gateway's 400 would silently drop the +/// parameter for every gateway serving that id — and dropping an output cap is +/// a bill, not just a difference. The model id is compared case-insensitively. +pub fn remember_omit(endpoint: &str, model: &str, parameter: &str) { + tracing::debug!( + target: "tinyinference::omission", + model, + parameter, + "[providers][omission] remembering a rejected request parameter" + ); + learned_store() + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .insert(key(endpoint, model, parameter)); +} + +/// Whether `model` at `endpoint` is known to reject `parameter` — i.e. whether +/// [`remember_omit`] recorded it for the same service in this process. +pub fn is_omitted(endpoint: &str, model: &str, parameter: &str) -> bool { + learned_store() + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .contains(&key(endpoint, model, parameter)) +} + +fn key(endpoint: &str, model: &str, parameter: &str) -> OmissionKey { + ( + endpoint_scope(endpoint), + model.to_ascii_lowercase(), + parameter.to_string(), + ) +} + +/// The part of a URL that identifies the service: scheme, host, port and base +/// path, lowercased, with query, fragment and any [`OPERATIONS`] suffix removed. +/// +/// Anything that does not parse as `scheme://rest` is used whole: a key that is +/// too specific costs one round-trip, one that is too broad is the bug the +/// endpoint scoping exists to prevent. +pub(super) fn endpoint_scope(endpoint: &str) -> String { + let lowered = endpoint.trim().to_ascii_lowercase(); + let Some((scheme, rest)) = lowered.split_once("://") else { + return lowered; + }; + let authority_and_path = rest.split(['?', '#']).next().unwrap_or(rest); + let trimmed = authority_and_path.trim_end_matches('/'); + if trimmed.is_empty() { + return lowered; + } + let base = OPERATIONS + .iter() + .find_map(|operation| trimmed.strip_suffix(operation)) + .unwrap_or(trimmed); + format!("{scheme}://{}", base.trim_end_matches('/')) +} + +/// The single cell the reader and the writer share. +fn learned_store() -> &'static RwLock> { + static LEARNED: OnceLock>> = OnceLock::new(); + LEARNED.get_or_init(|| RwLock::new(HashSet::new())) +} + +#[cfg(test)] +#[path = "omission_tests.rs"] +mod tests; diff --git a/crates/tinyinference-llm/src/providers/omission_tests.rs b/crates/tinyinference-llm/src/providers/omission_tests.rs new file mode 100644 index 0000000..8214030 --- /dev/null +++ b/crates/tinyinference-llm/src/providers/omission_tests.rs @@ -0,0 +1,124 @@ +use super::*; + +/// One endpoint for the tests that do not exercise scoping. Every test uses +/// its own model id, because the learned store is process-wide. +const AT: &str = "https://api.example/v1"; + +#[test] +fn a_rejection_blames_only_a_parameter_we_actually_sent() { + let sent = ["temperature", "max_tokens"]; + assert_eq!( + parameter_blamed_by( + "Unsupported value: 'temperature' does not support 0.2 with this model.", + &sent + ), + Some("temperature") + ); + // A field we did not send is not ours to drop. + assert_eq!( + parameter_blamed_by("Unsupported parameter: 'logit_bias'", &sent), + None + ); + // And a 400 that is not about the request's shape teaches us nothing. + assert_eq!( + parameter_blamed_by("the temperature in Paris is 19 degrees", &sent), + None + ); +} + +#[test] +fn a_rejection_names_the_wire_name_that_actually_went_out() { + // After a rename the wire name is not the caller's name, so the blame is + // matched against what was sent. + let sent = ["temperature", "max_completion_tokens"]; + assert_eq!( + parameter_blamed_by( + "400: Unsupported parameter: 'max_completion_tokens' is not supported", + &sent + ), + Some("max_completion_tokens") + ); + // Nothing we sent is named, so there is nothing to drop and no retry. + assert_eq!( + parameter_blamed_by( + "400: Extra inputs are not permitted: 'reasoning_effort'", + &sent + ), + None + ); +} + +#[test] +fn the_longest_matching_name_is_blamed() { + // `max_completion_tokens` does not contain `max_tokens`, but a future pair + // that nests would otherwise blame the shorter name. + let sent = ["top", "top_p"]; + assert_eq!( + parameter_blamed_by("Unsupported parameter: 'top_p'", &sent), + Some("top_p") + ); +} + +#[test] +fn nothing_is_omitted_until_a_rejection_is_remembered() { + let model = "omission-learning-test-model-v1"; + assert!(!is_omitted(AT, model, "top_p")); + remember_omit(AT, model, "top_p"); + assert!(is_omitted(AT, model, "top_p")); + assert!( + !is_omitted(AT, model, "temperature"), + "only the rejected parameter is dropped" + ); +} + +#[test] +fn a_learned_omission_ignores_model_id_case() { + let model = "Omission-Case-Test-Model"; + remember_omit(AT, model, "seed"); + assert!(is_omitted(AT, &model.to_ascii_lowercase(), "seed")); +} + +#[test] +fn a_rejection_is_learned_about_the_endpoint_that_made_it() { + // A model id is not unique across endpoints: one gateway refusing a + // parameter says nothing about another serving the same id. + let model = "omission-endpoint-scope-test-model-v1"; + let one = "https://gateway-one.example/v1"; + let two = "https://gateway-two.example/v1"; + + remember_omit(one, model, "max_tokens"); + assert!(is_omitted(one, model, "max_tokens")); + assert!( + !is_omitted(two, model, "max_tokens"), + "the other gateway never rejected anything" + ); + + // Same service, different operation: one endpoint answering two ways. + for operation in [ + "https://gateway-one.example/v1/chat/completions", + "https://gateway-one.example/v1/responses", + "https://gateway-one.example/v1/", + "https://GATEWAY-ONE.example/v1?client_version=1", + ] { + assert!( + is_omitted(operation, model, "max_tokens"), + "{operation} is the same service" + ); + } + + // A path-routed gateway is several services behind one origin. + assert!( + !is_omitted( + "https://gateway-one.example/vendor-b/v1", + model, + "max_tokens" + ), + "another upstream behind the same host has not rejected anything" + ); +} + +#[test] +fn an_unparseable_endpoint_is_used_whole() { + assert_eq!(endpoint_scope(" Not A URL "), "not a url"); + assert_eq!(endpoint_scope("https://"), "https://"); +} diff --git a/crates/tinyinference-llm/src/providers/openai/README.md b/crates/tinyinference-llm/src/providers/openai/README.md index 18c2add..2f9dace 100644 --- a/crates/tinyinference-llm/src/providers/openai/README.md +++ b/crates/tinyinference-llm/src/providers/openai/README.md @@ -44,6 +44,14 @@ of the rest of TinyInference. Anthropic API), `groq`, `xai`, `openrouter`, `together`, `mistral`, `ollama`. Override the preset's default model with `.with_model(..)`. +**Rotating credentials.** `.with_bearer_source(Arc)` +replaces the constructor's static key with a value read on **every** request +(a projected platform token, a session JWT), so a rotation never needs a +rebuild and the connection pool and learned state survive it. The value is sent +per the configured `AuthStyle` exactly as the static key would be, `with_header` +headers are still attached, a source yielding `None` sends no credential header, +and a 401 calls `BearerSource::invalidate` so the next request re-reads. + Accessors: `.model()`, `.provider()`, `.base_url()`. ## Model discovery @@ -111,6 +119,33 @@ call, and only for the shape the request actually used. This means unmodified `from_env()` / preset instances "just work" against LM Studio and friends without any per-provider configuration. +## Adaptive parameter omission + +Static knowledge of which model takes which parameter +(`with_temperature_unsupported_models`, the o-series/`gpt-5` +`max_completion_tokens` rename) goes stale as vendors change models. When a +Chat Completions call fails with an HTTP 400 whose error **message** names one +of the optional fields that request actually sent — `temperature`, `top_p`, +`seed`, `max_tokens`, `max_completion_tokens`, `reasoning_effort` — next to a +rejection phrase (`unsupported`, `not supported`, `unknown parameter`, ...): + +1. that one field is dropped and the request is retried **once** (unary and + streaming paths alike; at most one retry per call, shared with the + request-shape degradation above, which takes precedence); +2. the omission is remembered process-wide for that endpoint and model, so every + later request there leaves the field off up front — across `OpenAiModel` + instances. + +Messages, tools, `stop` and `response_format` are never dropped, a +context-overflow 400 is never read as a rejection (those routinely name +`max_tokens`), and a field injected by the host's `on_payload` hook is never +stripped. The learning key is the endpoint's scheme, host, port and base path +(operation suffixes such as `/chat/completions` removed) plus the model id, so +two gateways serving the same id do not share what one of them rejected. The +evidence rule and the store live in `providers::omission` +(`parameter_blamed_by`, `remember_omit`, `is_omitted`) for hosts that build +their own wire bodies. + ## Inline reasoning-tag extraction Reasoning models served through OpenAI-compatible local runtimes (qwen3 and diff --git a/crates/tinyinference-llm/src/providers/openai/transport.rs b/crates/tinyinference-llm/src/providers/openai/transport.rs index 7e4389b..4e23c89 100644 --- a/crates/tinyinference-llm/src/providers/openai/transport.rs +++ b/crates/tinyinference-llm/src/providers/openai/transport.rs @@ -7,6 +7,8 @@ use super::responses; use super::*; use crate::model::effective_temperature; +use crate::providers::omission; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; /// How the provider expects the API credential to be sent on each request. @@ -43,6 +45,9 @@ pub struct OpenAiModel { api_key: String, /// How `api_key` is attached to each request (default [`AuthStyle::Bearer`]). auth: AuthStyle, + /// When set, read per request in place of `api_key`. See + /// [`Self::with_bearer_source`]. + bearer_source: Option>, /// Extra static headers attached to every request (e.g. provider /// attribution headers). Applied after the auth header. extra_headers: Vec<(String, String)>, @@ -148,6 +153,7 @@ impl std::fmt::Debug for OpenAiModel { formatter .debug_struct("OpenAiModel") .field("auth", &self.auth) + .field("bearer_source", &self.bearer_source.is_some()) .field("extra_header_names", &extra_header_names) .field("model", &self.model) .field("provider", &self.provider) @@ -363,6 +369,7 @@ impl OpenAiModel { .expect("default reqwest client builds"), api_key: api_key.into(), auth: AuthStyle::Bearer, + bearer_source: None, extra_headers: Vec::new(), temperature_unsupported: Vec::new(), temperature_override: None, @@ -453,6 +460,22 @@ impl OpenAiModel { self } + /// Reads the credential from `source` on every request instead of using the + /// static key passed to the constructor. + /// + /// For a token that rotates in place (a projected platform token, a session + /// JWT): the model is built once and keeps its connection pool and learned + /// state while the credential changes underneath it. The value is placed per + /// the configured [`AuthStyle`] exactly as the static key would be, static + /// [`with_header`](Self::with_header) headers are still attached, and a + /// source yielding `None` sends no credential header. A 401 response calls + /// [`BearerSource::invalidate`](crate::providers::BearerSource::invalidate) + /// so the next request re-reads rather than re-presents a refused token. + pub fn with_bearer_source(mut self, source: Arc) -> Self { + self.bearer_source = Some(source); + self + } + /// Attaches a static header to every request (repeatable). Applied after the /// auth header — e.g. provider attribution headers. pub fn with_header(mut self, name: impl Into, value: impl Into) -> Self { @@ -783,7 +806,11 @@ impl OpenAiModel { let url = format!("{}/models", self.base_url); let response = self - .send_checked(self.authorized(self.client.get(&url)), "request", &url) + .send_checked( + self.authorized(self.client.get(&url)).await?, + "request", + &url, + ) .await?; let text = response @@ -1026,12 +1053,13 @@ impl OpenAiModel { let endpoint = format!("{root}/api/show"); let builder = self .authorized(self.client.post(&endpoint)) + .await? .json(&ollama_show_body(&self.model)); (endpoint, builder) } LocalRuntimeKind::LmStudio => { let endpoint = format!("{root}/api/v0/models"); - let builder = self.authorized(self.client.get(&endpoint)); + let builder = self.authorized(self.client.get(&endpoint)).await?; (endpoint, builder) } LocalRuntimeKind::LlamaCpp | LocalRuntimeKind::Vllm => return Ok(LocalProbe::default()), @@ -1115,6 +1143,7 @@ impl OpenAiModel { self.keep_alive.as_deref(), ); self.authorized(self.client.post(&url)) + .await? .json(&body) .send() .await @@ -1383,12 +1412,27 @@ impl OpenAiModel { /// Attaches the provider's credential (per [`Self::auth`]) plus any static /// [`Self::extra_headers`] to an outbound request. /// - /// The single place auth is applied, shared by the chat and model-listing - /// calls. The header mapping itself lives in the pure [`auth_headers`] helper - /// so it is unit-testable without a network round-trip. - fn authorized(&self, mut builder: reqwest::RequestBuilder) -> reqwest::RequestBuilder { - for (name, value) in auth_headers(&self.auth, &self.api_key) { - builder = builder.header(name, value); + /// The single place auth is applied, shared by every call this adapter + /// makes. The credential is the [`Self::with_bearer_source`] value when one + /// is set — read here, per request — and the static key otherwise. The + /// header mapping itself lives in the pure [`auth_headers`] helper so it is + /// unit-testable without a network round-trip. + /// + /// # Errors + /// + /// Whatever the bearer source returns when it cannot produce a credential. + async fn authorized( + &self, + mut builder: reqwest::RequestBuilder, + ) -> Result { + let credential = match &self.bearer_source { + Some(source) => source.current().await?, + None => Some(self.api_key.clone()), + }; + if let Some(credential) = credential { + for (name, value) in auth_headers(&self.auth, &credential) { + builder = builder.header(name, value); + } } for (name, value) in &self.extra_headers { builder = builder.header(name.as_str(), value.as_str()); @@ -1399,7 +1443,7 @@ impl OpenAiModel { if !self.extra_query_params.is_empty() { builder = builder.query(&self.extra_query_params); } - builder + Ok(builder) } /// The `/responses` endpoint URL — a sibling of `/chat/completions` under the @@ -1572,7 +1616,7 @@ impl OpenAiModel { timeout_ms: Option, url: &str, ) -> Result { - let mut builder = self.authorized(self.client.post(url)).json(body); + let mut builder = self.authorized(self.client.post(url)).await?.json(body); if let Some(timeout) = request_timeout(timeout_ms, body.stream == Some(true)) { builder = builder.timeout(timeout); } @@ -1609,6 +1653,11 @@ impl OpenAiModel { })?; let status = response.status(); + if status == reqwest::StatusCode::UNAUTHORIZED + && let Some(source) = &self.bearer_source + { + source.invalidate(); + } if !status.is_success() { // Read the header before `text()` consumes the response: a 429's // `Retry-After` is the one wait the retry layer cannot guess. @@ -1624,32 +1673,68 @@ impl OpenAiModel { Ok(response) } - /// Issues an authenticated `POST {base_url}/chat/completions` with `body`, - /// applying the resolved per-request timeout, and returns the checked - /// response. + /// Issues an authenticated `POST {base_url}/chat/completions` with the + /// final wire `payload` (see [`Self::chat_payload`]), applying the resolved + /// per-request timeout, and returns the checked response. /// /// Shared by the unary ([`Self::invoke`]) and streaming ([`Self::stream`]) /// paths so URL construction, auth, timeout selection, and transport/status /// handling live in exactly one place. async fn post_json( &self, - body: &ChatCompletionRequest, + payload: &Value, timeout_ms: Option, streaming: bool, what: &str, ) -> Result { crate::network_guard::ensure_network_models_allowed()?; let url = format!("{}/chat/completions", self.base_url); - let mut payload = serde_json::to_value(body)?; - self.request_options.apply_payload(&mut payload); let client = self.request_options.http.as_ref().unwrap_or(&self.client); - let mut builder = self.authorized(client.post(&url)).json(&payload); + let mut builder = self.authorized(client.post(&url)).await?.json(payload); if let Some(timeout) = request_timeout(timeout_ms, streaming) { builder = builder.timeout(timeout); } self.send_checked(builder, what, &url).await } + /// Serializes `body` into the JSON actually sent: parameters this endpoint + /// already rejected for the body's model are left off (see + /// [`omission`](crate::providers::omission)), then the host's + /// [`on_payload`](crate::providers::ProviderRequestOptions::on_payload) hook + /// runs, so a field the host injects deliberately is never stripped. + fn chat_payload(&self, body: &ChatCompletionRequest) -> Result { + let mut payload = serde_json::to_value(body)?; + if let Some(object) = payload.as_object_mut() { + object.retain(|name, _| { + !(OMITTABLE_PARAMETERS.contains(&name.as_str()) + && omission::is_omitted(&self.base_url, &body.model, name)) + }); + } + self.request_options.apply_payload(&mut payload); + Ok(payload) + } + + /// The [`OMITTABLE_PARAMETERS`] member a 400 blames, if any, chosen only + /// from those present in the `payload` that was sent. + /// + /// Matched against the provider's error *message*, not its code or raw + /// body: OpenAI-shaped errors type nearly every 400 `invalid_request_error`, + /// which would turn any message that merely names a sent field into a + /// rejection of it. A context-overflow error is never a rejection — those + /// routinely name `max_tokens`, and dropping (then remembering) the cap over + /// one would uncap every later request to that model. + fn rejected_parameter(&self, err: &ProviderError, payload: &Value) -> Option { + if err.code.as_deref() == Some(CONTEXT_OVERFLOW_CODE) { + return None; + } + let sent: Vec<&str> = OMITTABLE_PARAMETERS + .iter() + .copied() + .filter(|name| payload.get(name).is_some_and(|value| !value.is_null())) + .collect(); + omission::parameter_blamed_by(&err.message, &sent).map(str::to_string) + } + /// Builds the chat-completions wire body for `request` under the given /// `degrade`, setting the streaming fields when `streaming` is `true`. fn build_chat_body( @@ -1716,8 +1801,9 @@ impl OpenAiModel { ) -> Result { let baseline = self.baseline_degrade(); let body = self.build_chat_body(request, baseline, streaming)?; + let payload = self.chat_payload(&body)?; match self - .post_json(&body, request.timeout_ms, streaming, what) + .post_json(&payload, request.timeout_ms, streaming, what) .await { Ok(response) => Ok(response), @@ -1730,6 +1816,24 @@ impl OpenAiModel { self.json_schema_strict.store(false, Ordering::Relaxed); } let retry = self.build_chat_body(request, degrade, streaming)?; + let retry = self.chat_payload(&retry)?; + self.post_json(&retry, request.timeout_ms, streaming, what) + .await + } else if let Some(parameter) = self.rejected_parameter(&err, &payload) { + // The model named a parameter we sent as one it does not + // take. Drop that one parameter, remember it for this + // endpoint and model, and try exactly once more — a 400 is + // billed nothing, so being wrong costs one round-trip. + tracing::debug!( + target: "tinyinference::openai", + model = %body.model, + parameter = %parameter, + "[openai] retrying without a parameter the endpoint rejected" + ); + omission::remember_omit(&self.base_url, &body.model, ¶meter); + // Rebuild so learned omissions are applied before the + // host hook, which must retain ownership of injected fields. + let retry = self.chat_payload(&body)?; self.post_json(&retry, request.timeout_ms, streaming, what) .await } else { @@ -1804,6 +1908,20 @@ impl OpenAiModel { } } +/// Optional request fields a Chat Completions endpoint may reject by name and +/// that can be dropped without changing what the request asks: sampling, +/// output-cap and reasoning knobs. Messages, tools, `stop` and response format +/// are never in this list — dropping one of those would answer a different +/// question. See [`OpenAiModel::rejected_parameter`]. +const OMITTABLE_PARAMETERS: &[&str] = &[ + "temperature", + "top_p", + "seed", + "max_tokens", + "max_completion_tokens", + "reasoning_effort", +]; + /// Request-shape degradations to apply when building an OpenAI wire body. /// /// Each field, when `true`, replaces a request shape that some local diff --git a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs index a8b01d7..8110112 100644 --- a/crates/tinyinference-llm/src/providers/openai/wire_tests.rs +++ b/crates/tinyinference-llm/src/providers/openai/wire_tests.rs @@ -7,6 +7,7 @@ use std::io::{Read, Write}; use std::net::TcpListener; +use std::sync::Arc; use serde_json::{Value, json}; @@ -125,37 +126,43 @@ fn explicit_reasoning_provider_option_wins_over_the_budget_on_openrouter() { assert_eq!(body["reasoning"], json!({ "effort": "low" }), "body={body}"); } +/// Reads one HTTP/1.1 request (head plus a `content-length` body) off `sock`. +fn read_request(sock: &mut std::net::TcpStream) -> Vec { + let mut buf = Vec::new(); + let mut tmp = [0u8; 4096]; + loop { + let n = sock.read(&mut tmp).unwrap(); + if n == 0 { + break; + } + buf.extend_from_slice(&tmp[..n]); + let Some(split) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { + continue; + }; + let head = String::from_utf8_lossy(&buf[..split]).to_ascii_lowercase(); + let len = head + .lines() + .find_map(|l| { + l.strip_prefix("content-length:")? + .trim() + .parse::() + .ok() + }) + .unwrap_or(0); + if buf.len() >= split + 4 + len { + break; + } + } + buf +} + /// Serves one canned chat completion and returns the raw request it received. fn serve_once() -> (String, std::thread::JoinHandle) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let base = format!("http://{}/v1", listener.local_addr().unwrap()); let handle = std::thread::spawn(move || { let (mut sock, _) = listener.accept().unwrap(); - let mut buf = Vec::new(); - let mut tmp = [0u8; 4096]; - loop { - let n = sock.read(&mut tmp).unwrap(); - if n == 0 { - break; - } - buf.extend_from_slice(&tmp[..n]); - let Some(split) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { - continue; - }; - let head = String::from_utf8_lossy(&buf[..split]).to_ascii_lowercase(); - let len = head - .lines() - .find_map(|l| { - l.strip_prefix("content-length:")? - .trim() - .parse::() - .ok() - }) - .unwrap_or(0); - if buf.len() >= split + 4 + len { - break; - } - } + let buf = read_request(&mut sock); let payload = json!({ "id": "chatcmpl-test", "object": "chat.completion", @@ -329,31 +336,7 @@ fn serve_error_once(status: u16, body: &'static str) -> String { let base = format!("http://{}/v1", listener.local_addr().unwrap()); std::thread::spawn(move || { let (mut sock, _) = listener.accept().unwrap(); - let mut buf = Vec::new(); - let mut tmp = [0u8; 4096]; - loop { - let n = sock.read(&mut tmp).unwrap(); - if n == 0 { - break; - } - buf.extend_from_slice(&tmp[..n]); - let Some(split) = buf.windows(4).position(|w| w == b"\r\n\r\n") else { - continue; - }; - let head = String::from_utf8_lossy(&buf[..split]).to_ascii_lowercase(); - let len = head - .lines() - .find_map(|l| { - l.strip_prefix("content-length:")? - .trim() - .parse::() - .ok() - }) - .unwrap_or(0); - if buf.len() >= split + 4 + len { - break; - } - } + read_request(&mut sock); let response = format!( "HTTP/1.1 {status} Error\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", body.len() @@ -431,3 +414,383 @@ async fn overflow_without_stamped_code_still_records_the_window() { .expect("window learned from the message itself"); assert_eq!(learned.context_window, Some(32_768)); } + +// --------------------------------------------------------------------------- +// Adaptive parameter omission (a 400 that names a parameter we sent) +// --------------------------------------------------------------------------- + +/// A canned chat completion body, shared by the multi-response server tests. +fn completion_body() -> String { + json!({ + "id": "chatcmpl-test", + "object": "chat.completion", + "model": "m", + "choices": [{ + "index": 0, + "message": { "role": "assistant", "content": "Hello!" }, + "finish_reason": "stop" + }] + }) + .to_string() +} + +/// Serves `responses` in order, one connection each, and returns the raw +/// requests received. The listener closes after the last one, so an extra +/// attempt surfaces as a transport error rather than a canned answer. +fn serve_sequence(responses: Vec<(u16, String)>) -> (String, std::thread::JoinHandle>) { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/v1", listener.local_addr().unwrap()); + let handle = std::thread::spawn(move || { + let mut received = Vec::new(); + for (status, body) in responses { + let (mut sock, _) = listener.accept().unwrap(); + received.push(String::from_utf8_lossy(&read_request(&mut sock)).into_owned()); + let response = format!( + "HTTP/1.1 {status} Canned\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}", + body.len() + ); + let _ = sock.write_all(response.as_bytes()); + } + received + }); + (base, handle) +} + +fn request_body(raw: &str) -> Value { + serde_json::from_str(raw.split("\r\n\r\n").nth(1).unwrap()).unwrap() +} + +fn unsupported(parameter: &str) -> String { + json!({ + "error": { + "message": format!("Unsupported parameter: '{parameter}' is not supported with this model."), + "type": "invalid_request_error", + "param": parameter, + "code": "unsupported_parameter" + } + }) + .to_string() +} + +#[tokio::test] +async fn a_rejected_parameter_is_dropped_and_the_request_retried_once() { + let (base, server) = serve_sequence(vec![ + (400, unsupported("temperature")), + (200, completion_body()), + ]); + let model = OpenAiModel::new("k").with_base_url(&base); + + let response = model + .invoke(&(), request("omission-retry-model", 0.7)) + .await + .unwrap(); + assert_eq!(response.text(), "Hello!"); + + let raw = server.join().unwrap(); + assert_eq!(raw.len(), 2, "one rejection, one retry"); + let first = request_body(&raw[0]); + let retry = request_body(&raw[1]); + assert_eq!(first["temperature"], json!(0.7)); + assert!(retry.get("temperature").is_none(), "retry={retry}"); + // Only the blamed parameter goes: the question asked is unchanged. + assert_eq!(retry["model"], first["model"]); + assert_eq!(retry["messages"], first["messages"]); + assert!(super::super::omission::is_omitted( + &base, + "omission-retry-model", + "temperature" + )); +} + +#[tokio::test] +async fn omission_retry_preserves_host_injected_parameters() { + let (base, server) = serve_sequence(vec![ + (400, unsupported("temperature")), + (200, completion_body()), + ]); + let options = crate::providers::ProviderRequestOptions { + on_payload: Some(Arc::new(|payload| { + payload["temperature"] = json!(0.25); + })), + ..Default::default() + }; + let model = OpenAiModel::new("k") + .with_base_url(&base) + .with_request_options(options); + + model + .invoke( + &(), + ModelRequest::new(vec![Message::user("hi")]).with_model("hook-retry-model"), + ) + .await + .unwrap(); + + let requests = server.join().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(request_body(&requests[0])["temperature"], json!(0.25)); + assert_eq!(request_body(&requests[1])["temperature"], json!(0.25)); +} + +#[tokio::test] +async fn a_learned_omission_is_applied_up_front_on_the_next_request() { + let (base, server) = serve_sequence(vec![ + (400, unsupported("max_tokens")), + (200, completion_body()), + (200, completion_body()), + ]); + let request = || request("omission-learned-model", 0.5).with_max_tokens(64); + + let first = OpenAiModel::new("k").with_base_url(&base); + first.invoke(&(), request()).await.unwrap(); + // A separate instance: what was learnt belongs to the endpoint and model, + // not to one adapter value. + let second = OpenAiModel::new("k").with_base_url(&base); + second.invoke(&(), request()).await.unwrap(); + + let raw = server.join().unwrap(); + assert_eq!(raw.len(), 3, "no round-trip is spent re-learning"); + let later = request_body(&raw[2]); + assert!(later.get("max_tokens").is_none(), "later={later}"); + assert_eq!(later["temperature"], json!(0.5), "nothing else is dropped"); +} + +#[tokio::test] +async fn only_one_omission_retry_is_spent() { + let (base, server) = serve_sequence(vec![ + (400, unsupported("temperature")), + (400, unsupported("seed")), + ]); + let model = OpenAiModel::new("k").with_base_url(&base); + let error = model + .invoke( + &(), + request("omission-single-retry-model", 0.3).with_seed(7), + ) + .await + .unwrap_err(); + + // The second rejection is what surfaces: had a third attempt been made it + // would have failed on the closed listener instead. + match error { + Error::Provider(error) => { + assert_eq!(error.status, Some(400)); + assert!(error.message.contains("'seed'"), "{}", error.message); + } + other => panic!("expected a provider error, got {other:?}"), + } + assert_eq!(server.join().unwrap().len(), 2); +} + +#[tokio::test] +async fn a_rejection_naming_nothing_we_sent_is_not_retried() { + let (base, server) = serve_sequence(vec![(400, unsupported("logit_bias"))]); + let model = OpenAiModel::new("k").with_base_url(&base); + assert!( + model + .invoke(&(), request("omission-unrelated-model", 0.3)) + .await + .is_err() + ); + assert_eq!(server.join().unwrap().len(), 1); + assert!(!super::super::omission::is_omitted( + &base, + "omission-unrelated-model", + "temperature" + )); +} + +#[tokio::test] +async fn a_context_overflow_never_drops_the_output_cap() { + // Overflow messages routinely name `max_tokens`; dropping (and + // remembering) the cap over one would uncap every later request. + let overflow = json!({ + "error": { + "message": "This model's maximum context length is 4096 tokens. 'max_tokens' is unsupported at 9000.", + "type": "invalid_request_error" + } + }) + .to_string(); + let (base, server) = serve_sequence(vec![(400, overflow)]); + let model = OpenAiModel::new("k").with_base_url(&base); + assert!( + model + .invoke( + &(), + request("omission-overflow-model", 0.3).with_max_tokens(9000) + ) + .await + .is_err() + ); + assert_eq!(server.join().unwrap().len(), 1); + assert!(!super::super::omission::is_omitted( + &base, + "omission-overflow-model", + "max_tokens" + )); +} + +#[tokio::test] +async fn the_streaming_path_retries_a_rejected_parameter_too() { + let (base, server) = + serve_sequence(vec![(400, unsupported("top_p")), (200, completion_body())]); + let model = OpenAiModel::new("k").with_base_url(&base); + let mut stream = model + .stream(&(), request("omission-stream-model", 0.3).with_top_p(0.9)) + .await + .unwrap(); + let mut completed = None; + while let Some(item) = stream.next().await { + if let crate::model::ModelStreamItem::Completed(response) = item { + completed = Some(response); + } + } + assert_eq!(completed.unwrap().text(), "Hello!"); + + let raw = server.join().unwrap(); + assert_eq!(raw.len(), 2); + assert!(request_body(&raw[1]).get("top_p").is_none()); +} + +// --------------------------------------------------------------------------- +// Per-request credentials (BearerSource) +// --------------------------------------------------------------------------- + +/// A rotating credential: yields `token-1`, `token-2`, ... and counts how +/// often it was told its value was rejected. +#[derive(Default)] +struct RotatingToken { + reads: std::sync::atomic::AtomicUsize, + invalidations: std::sync::atomic::AtomicUsize, +} + +#[async_trait::async_trait] +impl crate::providers::BearerSource for RotatingToken { + async fn current(&self) -> crate::Result> { + let n = self.reads.fetch_add(1, std::sync::atomic::Ordering::SeqCst) + 1; + Ok(Some(format!("token-{n}"))) + } + + fn invalidate(&self) { + self.invalidations + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + } +} + +/// A source that has nothing to present (a keyless endpoint). +struct NoToken; + +#[async_trait::async_trait] +impl crate::providers::BearerSource for NoToken { + async fn current(&self) -> crate::Result> { + Ok(None) + } +} + +/// A source that cannot produce a credential at all. +struct BrokenToken; + +#[async_trait::async_trait] +impl crate::providers::BearerSource for BrokenToken { + async fn current(&self) -> crate::Result> { + Err(Error::Model("token file unreadable".to_string())) + } +} + +#[tokio::test] +async fn a_bearer_source_is_read_on_every_request() { + let (base, server) = serve_sequence(vec![(200, completion_body()), (200, completion_body())]); + let model = OpenAiModel::new("static-key") + .with_base_url(&base) + .with_bearer_source(std::sync::Arc::new(RotatingToken::default())); + + model.invoke(&(), request("gpt-4o", 0.2)).await.unwrap(); + model.invoke(&(), request("gpt-4o", 0.2)).await.unwrap(); + + let raw = server.join().unwrap(); + assert_eq!( + header_value(&raw[0], "authorization").as_deref(), + Some("Bearer token-1") + ); + assert_eq!( + header_value(&raw[1], "authorization").as_deref(), + Some("Bearer token-2"), + "a rotated credential is picked up without rebuilding the model" + ); +} + +#[tokio::test] +async fn a_bearer_source_is_sent_in_the_configured_auth_style() { + let (base, server) = serve_sequence(vec![(200, completion_body())]); + let model = OpenAiModel::new("static-key") + .with_base_url(&base) + .with_auth_style(AuthStyle::XApiKey) + .with_header("x-extra", "kept") + .with_bearer_source(std::sync::Arc::new(RotatingToken::default())); + + model.invoke(&(), request("gpt-4o", 0.2)).await.unwrap(); + + let raw = server.join().unwrap(); + assert_eq!( + header_value(&raw[0], "x-api-key").as_deref(), + Some("token-1") + ); + assert_eq!(header_value(&raw[0], "authorization"), None); + assert_eq!(header_value(&raw[0], "x-extra").as_deref(), Some("kept")); +} + +#[tokio::test] +async fn a_bearer_source_with_nothing_to_present_sends_no_credentials() { + let (base, server) = serve_sequence(vec![(200, completion_body())]); + let model = OpenAiModel::new("static-key") + .with_base_url(&base) + .with_bearer_source(std::sync::Arc::new(NoToken)); + + model.invoke(&(), request("llama3", 0.2)).await.unwrap(); + + let raw = server.join().unwrap(); + assert_eq!(header_value(&raw[0], "authorization"), None, "{}", raw[0]); +} + +#[tokio::test] +async fn a_rejected_bearer_is_invalidated_at_its_source() { + let (base, server) = serve_sequence(vec![( + 401, + r#"{"error":{"message":"invalid token"}}"#.to_string(), + )]); + let source = std::sync::Arc::new(RotatingToken::default()); + let model = OpenAiModel::new("") + .with_base_url(&base) + .with_bearer_source(source.clone()); + + assert!(model.invoke(&(), request("gpt-4o", 0.2)).await.is_err()); + server.join().unwrap(); + assert_eq!( + source + .invalidations + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "the next request must re-read rather than re-present a refused token" + ); +} + +#[tokio::test] +async fn a_failing_bearer_source_fails_the_call() { + let model = OpenAiModel::new("static-key") + .with_base_url("http://127.0.0.1:9/v1") + .with_bearer_source(std::sync::Arc::new(BrokenToken)); + let error = model.invoke(&(), request("gpt-4o", 0.2)).await.unwrap_err(); + assert!( + error.to_string().contains("token file unreadable"), + "{error}" + ); +} + +#[test] +fn debug_output_reports_a_bearer_source_without_reading_it() { + let model = OpenAiModel::new("static-key") + .with_bearer_source(std::sync::Arc::new(RotatingToken::default())); + let debug = format!("{model:?}"); + assert!(debug.contains("bearer_source: true"), "{debug}"); + assert!(!debug.contains("static-key"), "{debug}"); +} diff --git a/crates/tinyinference-llm/src/providers/types.rs b/crates/tinyinference-llm/src/providers/types.rs index 0f08fb2..cbb8d6a 100644 --- a/crates/tinyinference-llm/src/providers/types.rs +++ b/crates/tinyinference-llm/src/providers/types.rs @@ -75,6 +75,42 @@ impl std::fmt::Debug for ProviderRequestOptions { } } +// --------------------------------------------------------------------------- +// Per-request credentials +// --------------------------------------------------------------------------- + +/// A credential read on **every** request instead of baked into an adapter at +/// construction time. +/// +/// A platform token that rotates in place (a projected service-account file, a +/// session JWT, an OAuth access token) changes value while the identity behind +/// it does not. Holding it as a `String` forces a host to rebuild the adapter — +/// and lose its connection pool and learned state — on every rotation. An +/// adapter given a source instead (see `OpenAiModel::with_bearer_source`) asks +/// it for the current value immediately before each call and places that +/// value exactly where its static key would go, honouring the adapter's +/// configured auth style (`Authorization: Bearer`, `x-api-key`, a custom +/// header). +/// +/// Implementations should cache cheaply-reused values themselves; the adapter +/// calls [`Self::current`] once per HTTP request. +#[async_trait::async_trait] +pub trait BearerSource: Send + Sync { + /// The credential to present on the next request, or `None` to send no + /// credential header at all (for example a keyless local endpoint). + /// + /// # Errors + /// + /// Any error fails the call before a request is sent; the adapter surfaces + /// it unchanged. + async fn current(&self) -> crate::Result>; + + /// Told that the endpoint rejected the last credential with HTTP 401, so a + /// cached value can be dropped and the next [`Self::current`] re-reads it + /// rather than re-presenting what was just refused. A no-op by default. + fn invalidate(&self) {} +} + // --------------------------------------------------------------------------- // Provider selection types // ---------------------------------------------------------------------------