diff --git a/AGENTS.md b/AGENTS.md index 463ba90c..ee8b8d4a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -140,7 +140,7 @@ current source and CI workflows for implementation details. The supported serving path is native Rust: ``` -HTTP request → switchyard-server → libsy Algorithm → LlmTarget/RoutedLlmClient +HTTP request → switchyard-server → libsy Algorithm → RoutedLlmClient → switchyard-translation → upstream model ``` diff --git a/crates/libsy-llm-client/README.md b/crates/libsy-llm-client/README.md index ed9d9918..30fef3a4 100644 --- a/crates/libsy-llm-client/README.md +++ b/crates/libsy-llm-client/README.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: Apache-2.0 # libsy-llm-client An HTTP client that speaks Switchyard's neutral IR directly. You hand it a -[`switchyard_protocol::Request`] and a model name; it looks up the configured backend, +[`switchyard_protocol::Request`] and a [`ModelId`]; it looks up the configured backend, encodes the request to that backend's wire format, adds auth and forwards your headers, makes the call with a shared `reqwest::Client`, and decodes the reply back into a [`switchyard_protocol::Response`] — buffered or streamed. @@ -20,16 +20,20 @@ It depends on `switchyard-libsy`, `switchyard-protocol`, and ## Concepts -- **Model configs.** A client is built from [`ModelConfig`] values. Each model has a - default [`Backend`] and can have additional backends for other wire formats. - [`TranslatingLlmClient::call_rewrite_model`] uses the request's metadata wire format - when set, otherwise the model's default backend. +- **Model configs.** A client is built from [`ModelConfig`] values, each keyed by a + [`ModelId`]. Each model has a default [`Backend`] and can have additional backends + for other wire formats. [`TranslatingLlmClient::call_rewrite_model`] uses the + request's metadata wire format when set, otherwise the model's default backend. +- **Model ids.** A [`ModelId`] is a newtype over `String` that behaves like one — it + derefs to `str`, compares against string literals, and converts from `&str` or + `String` with `.into()`. Anywhere below that takes `impl Into` accepts a + bare literal; the borrowed positions need a `ModelId` value to reference. - **Backends.** A [`Backend`] is one of `OpenAiChat`, `OpenAiResponses`, or `Anthropic`, each wrapping an [`HttpBackendConfig`] (`base_url`, `api_key`, static `extra_headers`, default `extra_body` fields, and `max_retries`). The variant fixes the URL path and auth scheme (Bearer vs `x-api-key` + `anthropic-version`). -- **Model rewrite.** The resolved model name is both the map key and the model id +- **Model rewrite.** The resolved [`ModelId`] is both the map key and the model id sent upstream — it overwrites whatever `model` the request arrived with. - **Streaming is chosen by the request.** If the encoded body has `stream: true` (i.e. `request.llm_request.stream`), you get `LlmResponse::Stream`; otherwise @@ -80,7 +84,7 @@ fn build_client() -> switchyard_llm_client::Result { ```rust use switchyard_llm_client::{LlmClientError, TranslatingLlmClient}; -use switchyard_protocol::{completion_text, text_request, LlmResponse, Request}; +use switchyard_protocol::{completion_text, text_request, LlmResponse, ModelId, Request}; async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result { let request = Request { @@ -90,9 +94,8 @@ async fn ask(client: &TranslatingLlmClient) -> switchyard_llm_client::Result Ok(completion_text(&agg)), @@ -110,7 +113,7 @@ Set `stream` on the IR request and drive the returned chunk stream: ```rust use futures_util::StreamExt; use switchyard_llm_client::TranslatingLlmClient; -use switchyard_protocol::{text_request, LlmResponse, LlmResponseChunk, Request}; +use switchyard_protocol::{text_request, LlmResponse, LlmResponseChunk, ModelId, Request}; async fn stream( client: &TranslatingLlmClient, @@ -119,16 +122,17 @@ async fn stream( llm_request.stream = true; let request = Request { llm_request, raw_request: None, metadata: None }; - let response = client - .call_rewrite_model(request, Some("gpt-4o-mini")) - .await?; - - if let LlmResponse::Stream(mut chunks) = response.llm_response { - while let Some(item) = chunks.next().await { - match item { - Ok(LlmResponseChunk::TextDelta { text, .. }) => print!("{text}"), - Ok(_) => {} // usage, tool-call deltas, message start/stop - Err(error) => return Err(error), + let model = ModelId::from("gpt-4o-mini"); + let response = client.call_rewrite_model(request, Some(&model)).await?; + + if let LlmResponse::Stream(mut events) = response.llm_response { + while let Some(item) = events.next().await { + // Each event carries zero or more provider-neutral chunks; the ones + // ignored here are usage, tool-call deltas, and message start/stop. + for chunk in item?.normalized() { + if let LlmResponseChunk::TextDelta { text, .. } = chunk { + print!("{text}"); + } } } } @@ -165,10 +169,27 @@ async fn route( ``` When targets are served by different clients — a judge on one provider and the serving -models on another, say — build the router from `model name -> client` with +models on another, say — build the router from `ModelId -> client` with [`ClientRouter::new`] instead. A model missing from that map fails with `LlmClientError::Configuration` rather than silently going to another provider. +```rust +use std::collections::HashMap; +use std::sync::Arc; +use switchyard_llm_client::ClientRouter; +use switchyard_protocol::{ModelId, RoutedLlmClient}; + +fn split_router( + judge: Arc, + serving: Arc, +) -> ClientRouter { + ClientRouter::new(HashMap::from([ + (ModelId::from("gpt-4o-mini"), judge), + (ModelId::from("claude-sonnet-4-5"), serving), + ])) +} +``` + A router is not itself a client: [`ClientRouter::route`] hands back a [`switchyard_protocol::RoutedLlmClient`] and the caller makes the call. @@ -248,5 +269,6 @@ and the retry budget plus capped `Retry-After` delays determines total latency. [`Backend`]: src/backend.rs [`HttpBackendConfig`]: src/backend.rs [`ModelConfig`]: src/client.rs +[`ModelId`]: ../protocol/src/model_id.rs [`TranslatingLlmClient::call_rewrite_model`]: src/client.rs [`LlmClientError`]: ../protocol/src/client.rs diff --git a/crates/libsy-llm-client/src/client.rs b/crates/libsy-llm-client/src/client.rs index d6ca1191..8928808e 100644 --- a/crates/libsy-llm-client/src/client.rs +++ b/crates/libsy-llm-client/src/client.rs @@ -13,7 +13,7 @@ use reqwest::RequestBuilder; use reqwest::header::{HeaderMap, RETRY_AFTER}; use serde_json::{Map, Value}; use switchyard_protocol::{ - Decision, LlmRequest, LlmResponse, Metadata, Request, Response, RoutedLlmClient, + Decision, LlmRequest, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient, }; use switchyard_translation::{ WireFormat, decode_aggregated_response, decode_request, decode_stream, @@ -56,7 +56,7 @@ const MAX_RETRY_AFTER: Duration = Duration::from_secs(60); /// pin a wire format, plus any `other_backends` reachable over additional formats. #[derive(Clone, Debug)] pub struct ModelConfig { - model_name: String, + model_name: ModelId, default_backend: Backend, other_backends: Option>, } @@ -65,7 +65,7 @@ impl ModelConfig { /// A model named `model_name` served by `default_backend`, optionally reachable /// over additional wire formats via `other_backends`. pub fn new( - model_name: impl Into, + model_name: impl Into, default_backend: Backend, other_backends: Option>, ) -> Self { @@ -86,7 +86,7 @@ impl ModelConfig { /// [`reqwest::Client`], and decodes the response back to the neutral IR (buffered /// or streamed). pub struct TranslatingLlmClient { - model_to_config: HashMap, + model_to_config: HashMap, client: reqwest::Client, } @@ -114,7 +114,7 @@ impl TranslatingLlmClient { /// The backend serving `model` over `format` — the default backend when its /// format matches, otherwise a matching entry in `other_backends`; `None` when /// the model is unknown or has no backend for `format`. - pub fn backend_for(&self, model: &str, format: WireFormat) -> Option<&Backend> { + pub fn backend_for(&self, model: &ModelId, format: WireFormat) -> Option<&Backend> { self.model_to_config.get(model).and_then(|config| { if config.default_backend.wire_format() == format { Some(&config.default_backend) @@ -128,7 +128,7 @@ impl TranslatingLlmClient { } /// Whether `model` has an Anthropic backend that supports token counting. - pub fn supports_count_tokens(&self, model: &str) -> bool { + pub fn supports_count_tokens(&self, model: &ModelId) -> bool { self.backend_for(model, WireFormat::AnthropicMessages) .is_some() } @@ -137,7 +137,7 @@ impl TranslatingLlmClient { /// /// Returns an error when the model has no Anthropic backend or the upstream /// request fails or returns invalid JSON. - pub async fn count_tokens(&self, model: &str, request: Request) -> Result { + pub async fn count_tokens(&self, model: &ModelId, request: Request) -> Result { let backend = self .backend_for(model, WireFormat::AnthropicMessages) .ok_or_else(|| LlmClientError::Configuration { @@ -188,7 +188,7 @@ impl TranslatingLlmClient { wire_format: WireFormat, mut llm_request: LlmRequest, metadata: Option<&Metadata>, - model: &str, + model: &ModelId, endpoint: UpstreamEndpoint, ) -> Result { // The resolved name is the upstream model id (per the crate contract). @@ -224,7 +224,7 @@ impl TranslatingLlmClient { let span = tracing::debug_span!( target: "libsy", "libsy.upstream_attempt", - model, + model = %model, wire_format = %wire_format, attempt = attempt + 1, max_attempts, @@ -276,7 +276,7 @@ impl TranslatingLlmClient { backend: &Backend, body: &Value, metadata: Option<&Metadata>, - model: &str, + model: &ModelId, streaming: bool, ) -> std::result::Result { let builder = self.client.post(url).json(body); @@ -336,7 +336,7 @@ impl TranslatingLlmClient { let error = if status == reqwest::StatusCode::BAD_REQUEST && backend.is_context_overflow(&body) { LlmClientError::ContextWindowExceeded { - model: model.to_string(), + model: model.clone(), message: body, } } else { @@ -363,7 +363,7 @@ impl TranslatingLlmClient { pub async fn call_rewrite_model( &self, request: Request, - model_name: Option<&str>, + model_name: Option<&ModelId>, ) -> Result { // Own the request's parts so the model can be set without a `mut` param // and without cloning the messages. `raw_request` is unused here. @@ -374,8 +374,8 @@ impl TranslatingLlmClient { } = request; let model = model_name - .map(str::to_string) - .or_else(|| llm_request.model.clone()) + .cloned() + .or_else(|| llm_request.model.clone().map(ModelId::from)) .ok_or_else(|| LlmClientError::InvalidRequest { message: "no model given".to_string(), })?; @@ -460,7 +460,7 @@ impl TranslatingLlmClient { &self, raw_http_request: Value, http_headers: Option, - model: Option<&str>, + model: Option<&ModelId>, wire_format: WireFormat, ) -> Result { let llm_request = decode_request(wire_format, &raw_http_request) @@ -469,7 +469,7 @@ impl TranslatingLlmClient { // one, else the request's own model. Mirrors `call_rewrite_model`'s own // resolution so the response names whoever answered. let served_model = model - .map(str::to_string) + .map(ModelId::to_string) .or_else(|| llm_request.model.clone()); let request = Request { @@ -1019,15 +1019,19 @@ mod tests { -> std::result::Result<(), Box> { let client = TranslatingLlmClient::new(&chat_map("https://example.test/v1"))?; // "gpt" is served over OpenAI Chat only; other formats and models miss. - assert!(client.backend_for("gpt", WireFormat::OpenAiChat).is_some()); assert!( client - .backend_for("gpt", WireFormat::AnthropicMessages) + .backend_for(&ModelId::from("gpt"), WireFormat::OpenAiChat) + .is_some() + ); + assert!( + client + .backend_for(&ModelId::from("gpt"), WireFormat::AnthropicMessages) .is_none() ); assert!( client - .backend_for("missing", WireFormat::OpenAiChat) + .backend_for(&ModelId::from("missing"), WireFormat::OpenAiChat) .is_none() ); Ok(()) @@ -1039,7 +1043,7 @@ mod tests { let client = TranslatingLlmClient::new(&[])?; // Arg "b" is looked up (and reported), not the request's "a". let Err(error) = client - .call_rewrite_model(request_for(Some("a"), false), Some("b")) + .call_rewrite_model(request_for(Some("a"), false), Some(&ModelId::from("b"))) .await else { panic!("expected an error"); @@ -1213,7 +1217,10 @@ mod tests { let client = TranslatingLlmClient::new(&chat_map(&format!("{}/v1", server.uri())))?; // Inbound model differs from the map key / resolved model. client - .call_rewrite_model(request_for(Some("switchyard"), false), Some("gpt")) + .call_rewrite_model( + request_for(Some("switchyard"), false), + Some(&ModelId::from("gpt")), + ) .await?; // The body_partial_json matcher asserts the upstream saw model "gpt". Ok(()) @@ -1258,7 +1265,12 @@ mod tests { }); client - .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await?; Ok(()) } @@ -1327,7 +1339,12 @@ mod tests { }); client - .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("claude")), + WireFormat::AnthropicMessages, + ) .await?; Ok(()) } @@ -1369,7 +1386,12 @@ mod tests { }); client - .call_rewrite_model_raw(raw, None, Some("claude"), WireFormat::AnthropicMessages) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("claude")), + WireFormat::AnthropicMessages, + ) .await?; Ok(()) } @@ -1431,7 +1453,12 @@ mod tests { }); let response = client - .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await?; assert!(matches!(response, RawResponse::Stream(_))); Ok(()) @@ -1641,7 +1668,7 @@ mod tests { let context_window = AttemptFailure { error: LlmClientError::ContextWindowExceeded { - model: "gpt".to_string(), + model: ModelId::from("gpt"), message: "too long".to_string(), }, status: Some(400), @@ -1821,7 +1848,12 @@ mod tests { -> std::result::Result<(), Box> { let client = TranslatingLlmClient::new(&[])?; let Err(error) = client - .call_rewrite_model_raw(json!("invalid"), None, Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + json!("invalid"), + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await else { panic!("expected request translation to fail"); @@ -1861,7 +1893,12 @@ mod tests { "messages": [{"role": "user", "content": "hi"}] }); let RawResponse::Buffered(body) = client - .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await? else { panic!("expected a buffered response"); @@ -1897,7 +1934,12 @@ mod tests { "stream": true }); let RawResponse::Stream(stream) = client - .call_rewrite_model_raw(raw, None, Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + raw, + None, + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await? else { panic!("expected a streamed response"); @@ -1945,7 +1987,12 @@ mod tests { // Matchers assert the forwarded x-request-id survives and reserved // authorization is the backend's, not the client's. client - .call_rewrite_model_raw(raw, Some(headers), Some("gpt"), WireFormat::OpenAiChat) + .call_rewrite_model_raw( + raw, + Some(headers), + Some(&ModelId::from("gpt")), + WireFormat::OpenAiChat, + ) .await?; Ok(()) } diff --git a/crates/libsy-llm-client/src/observation.rs b/crates/libsy-llm-client/src/observation.rs index 06352e25..cff08fff 100644 --- a/crates/libsy-llm-client/src/observation.rs +++ b/crates/libsy-llm-client/src/observation.rs @@ -6,13 +6,13 @@ use std::sync::Arc; use std::time::Duration; -use switchyard_protocol::Usage; +use switchyard_protocol::{ModelId, Usage}; /// One completed model call observed at the algorithm offload boundary. #[derive(Clone, Debug)] pub struct LlmCallObservation { /// Model selected for the completed call. - pub selected_model: String, + pub selected_model: ModelId, /// Whether this call generated an answer rather than a routing verdict. pub is_answer_call: bool, /// Whether the call completed successfully. diff --git a/crates/libsy-llm-client/src/run.rs b/crates/libsy-llm-client/src/run.rs index 4427fdf3..0324035a 100644 --- a/crates/libsy-llm-client/src/run.rs +++ b/crates/libsy-llm-client/src/run.rs @@ -17,7 +17,7 @@ use std::time::{Duration, Instant}; use parking_lot::Mutex; use switchyard_libsy::{Algorithm, CallModel, LibsyError, Result, drive}; -use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutedLlmClient}; +use switchyard_protocol::{Decision, LlmClientError, ModelId, Request, Response, RoutedLlmClient}; use crate::observation::{LlmCallObservation, RunObservation, RunObserver}; use crate::{metrics, observability}; @@ -109,12 +109,12 @@ impl RoutedCallWindows { fields( algorithm = call.algorithm, switchyard.algorithm = call.algorithm, - selected_model = call.decision.selected_model_id(), + selected_model = %call.decision.selected_model_id(), otel.kind = "client", otel.name = %format_args!("chat {}", call.decision.selected_model_id()), openinference.span.kind = "LLM", gen_ai.operation.name = "chat", - gen_ai.request.model = call.decision.selected_model_id(), + gen_ai.request.model = %call.decision.selected_model_id(), gen_ai.request.stream = tracing::field::Empty, gen_ai.request.temperature = tracing::field::Empty, gen_ai.request.top_p = tracing::field::Empty, @@ -157,7 +157,7 @@ async fn serve( } let request = call.request.clone(); let decision = call.decision.clone(); - let target = decision.selected_model_id().to_string(); + let target = decision.selected_model_id().clone(); let is_answer_call = decision.is_answer_call(); // Resolved before the clock starts: picking the client is Switchyard's work, not // the provider's, so it belongs in the routing overhead. @@ -174,7 +174,7 @@ async fn serve( let result = observability::observe_client_call(result); if let Some(observer) = observer { observer(RunObservation::LlmCall(LlmCallObservation { - selected_model: call.decision.selected_model_id().to_string(), + selected_model: call.decision.selected_model_id().clone(), is_answer_call, is_success: result.is_ok(), duration, @@ -209,12 +209,12 @@ enum Routing { /// One client serves every model. Single(Arc), /// Each model is served by the client configured for it. - ByModel(HashMap>), + ByModel(HashMap>), } impl ClientRouter { /// Build a router over `model name -> client`, for targets spread across providers. - pub fn new(by_model: HashMap>) -> Self { + pub fn new(by_model: HashMap>) -> Self { Self { routing: Arc::new(Routing::ByModel(by_model)), } @@ -237,7 +237,7 @@ impl ClientRouter { /// entry for this one, rather than silently sending the call to another provider. pub fn route( &self, - model: &str, + model: &ModelId, ) -> std::result::Result<&Arc, LlmClientError> { match self.routing.as_ref() { Routing::Single(client) => Ok(client), @@ -252,8 +252,8 @@ impl ClientRouter { } } -impl FromIterator<(String, Arc)> for ClientRouter { - fn from_iter)>>(iter: I) -> Self { +impl FromIterator<(ModelId, Arc)> for ClientRouter { + fn from_iter)>>(iter: I) -> Self { Self::new(iter.into_iter().collect()) } } diff --git a/crates/libsy-llm-client/tests/observability.rs b/crates/libsy-llm-client/tests/observability.rs index daa84dbf..d6f34be9 100644 --- a/crates/libsy-llm-client/tests/observability.rs +++ b/crates/libsy-llm-client/tests/observability.rs @@ -33,11 +33,11 @@ use tracing_subscriber::layer::{Context as LayerContext, SubscriberExt}; use tracing_subscriber::registry::LookupSpan; use switchyard_libsy::{ - AffinityRouter, Algorithm, Classifier, Driver, LibsyError, LlmClassifierConfig, LlmTarget, - LlmTargetSet, LlmTaskClassifier, PickerMode, StageRouter, StageRouterConfig, Step, - TaskClassifierConfig, + AffinityRouter, Algorithm, Classifier, Driver, LibsyError, LlmClassifierConfig, + LlmTaskClassifier, PickerMode, StageRouter, StageRouterConfig, Step, TaskClassifierConfig, }; use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver}; +use switchyard_protocol::ModelId; use switchyard_protocol::{ ContentBlock, Decision, LlmRequest, LlmResponse, Message, Metadata, Request, Response, Role, RoutedLlmClient, ToolCall, ToolResult, Usage, WireFormat, @@ -429,7 +429,7 @@ impl RoutedLlmClient for UsageClient { /// algorithm exercising both instrumented driver paths. struct SingleCallAlgo { name: String, - target_set: LlmTargetSet, + target_set: Vec, } #[async_trait] @@ -445,15 +445,10 @@ impl Algorithm for SingleCallAlgo { ) -> switchyard_libsy::Result { let target = self .target_set - .targets() .first() .ok_or(LibsyError::NoTargets)? .clone(); - let decision = Decision::new( - target.semantic_name.clone(), - Some(format!("picked '{}'", target.semantic_name)), - true, - ); + let decision = Decision::new(target.clone(), Some(format!("picked '{target}'")), true); driver.decide(decision.clone()).await?; driver.call_model(request, decision).await } @@ -478,9 +473,7 @@ fn request_with_metadata(session_id: &str, correlation_id: &str) -> Request { fn algo(name: &str, model: &str) -> Arc { Arc::new(SingleCallAlgo { name: name.to_string(), - target_set: LlmTargetSet::new(vec![LlmTarget { - semantic_name: model.to_string(), - }]), + target_set: vec![model.to_string()], }) } @@ -499,15 +492,11 @@ fn classifier_router( efficient_model: &str, capable_model: &str, ) -> switchyard_libsy::Result> { - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; - let targets = LlmTargetSet::new(vec![target(efficient_model), target(capable_model)]); Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Capability { - judge_target: target(judge_model), - efficient_target: targets.get_target(efficient_model)?, - capable_target: targets.get_target(capable_model)?, + judge_target: ModelId::from(judge_model), + efficient_target: ModelId::from(efficient_model), + capable_target: ModelId::from(capable_model), config: TaskClassifierConfig { base_threshold: 0.5, ..TaskClassifierConfig::default() @@ -862,12 +851,10 @@ async fn stage_router_records_algorithm_owned_metrics() -> switchyard_libsy::Res let (_, exporter, provider, _, _) = telemetry(); const STRONG: &str = "obs-stage-strong"; const WEAK: &str = "obs-stage-weak"; - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; + let target = |name: &str| name.to_string(); let algorithm = Arc::new(StageRouter::new( - target(STRONG), - target(WEAK), + target(STRONG).into(), + target(WEAK).into(), StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), )?) as Arc; let request = Request { diff --git a/crates/libsy/README.md b/crates/libsy/README.md index 5a2ea563..895c69f8 100644 --- a/crates/libsy/README.md +++ b/crates/libsy/README.md @@ -30,7 +30,7 @@ tokio = { version = "1", features = ["macros", "rt"] } ## How it fits together -[`LlmTarget`] names a routing destination. An [`Algorithm`] selects targets and +A target is a bare model id naming a routing destination. An [`Algorithm`] selects targets and records [`Decision`](switchyard_protocol::Decision)s, offloading every model call to its caller: [`Algorithm::run_stream`] yields a [`Step`] stream whose [`Step::CallModel`] items the host serves over its own transport. libsy makes no diff --git a/crates/libsy/src/algorithms/fall_through.rs b/crates/libsy/src/algorithms/fall_through.rs index ec9d48fb..391a24f3 100644 --- a/crates/libsy/src/algorithms/fall_through.rs +++ b/crates/libsy/src/algorithms/fall_through.rs @@ -28,13 +28,11 @@ use async_trait::async_trait; use parking_lot::Mutex; use tokio::sync::Mutex as AsyncMutex; -use crate::core::algorithm::{ - self, Algorithm, Driver, LlmTarget, LlmTargetSet, RoutingIdentity, SessionEvictions, -}; +use crate::core::algorithm::{self, Algorithm, Driver, RoutingIdentity, SessionEvictions}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::processor::{Event, Processor}; use crate::{LibsyError, Result}; -use switchyard_protocol::{Decision, Request, Response, RoutingFallbackReason}; +use switchyard_protocol::{Decision, ModelId, Request, Response, RoutingFallbackReason}; struct SessionState { state: Arc>, @@ -59,12 +57,12 @@ const SESSION_CLEANUP_INTERVAL: Duration = Duration::from_secs(60 * 60); /// decider that never does. Which target that is belongs to whoever assembles the /// cascade, not to the classifiers in it. pub struct DefaultTarget { - target: String, + target: ModelId, } impl DefaultTarget { /// Close a cascade with `target`. - pub fn new(target: impl Into) -> Self { + pub fn new(target: impl Into) -> Self { Self { target: target.into(), } @@ -98,7 +96,7 @@ pub struct FallThrough { decision_reason: fn(&str, &Score) -> String, processors: Vec>>, classifiers: Vec>>, - targets: LlmTargetSet, + targets: Vec, session_states: Option>>, cleanup_started: Once, session_evictions: SessionEvictions, @@ -106,7 +104,7 @@ pub struct FallThrough { impl FallThrough<()> { /// Creates an empty stateless router. - pub fn new(targets: LlmTargetSet) -> Self { + pub fn new(targets: Vec) -> Self { Self { name: "fall_through".to_string(), decision_reason: default_decision_reason, @@ -125,7 +123,7 @@ where S: Default + Send + 'static, { /// Creates a router that retains one private `S` per session. - pub fn new_with_state(targets: LlmTargetSet) -> Self { + pub fn new_with_state(targets: Vec) -> Self { Self { name: "fall_through".to_string(), decision_reason: default_decision_reason, @@ -258,8 +256,8 @@ where fn fallback_decision( &self, deciding: &dyn Classifier, - from: &LlmTarget, - to: &LlmTarget, + from: &ModelId, + to: &ModelId, reason: RoutingFallbackReason, ) -> Decision { let failure = match reason { @@ -267,15 +265,13 @@ where RoutingFallbackReason::Unavailable => "was unavailable", }; Decision::new( - to.semantic_name.clone(), + to.clone(), Some(with_routing_tier( format!( - "{} {failure}; fell back to {} (fallback reason: {})", - from.semantic_name, - to.semantic_name, + "{from} {failure}; fell back to {to} (fallback reason: {})", reason.as_str(), ), - deciding.routing_tier(&to.semantic_name), + deciding.routing_tier(to), )), true, ) @@ -298,15 +294,10 @@ where async fn route( &self, state: &mut S, - excluded: &HashSet, + excluded: &HashSet, driver: &Driver, request: &mut Request, - ) -> Result<( - LlmTarget, - Decision, - Option, - Arc>, - )> { + ) -> Result<(ModelId, Decision, Option, Arc>)> { // 1. Processor chain accumulates request-side facts into the composition's state. for processor in &self.processors { processor.process(state, Event::Request(request)).await?; @@ -332,21 +323,18 @@ where // 3. Resolve the target and publish the decision. When an excluded target sends // the request elsewhere, the reasoning describes where it actually went. - let target = self.targets.resolve_target(&score.target, excluded)?; - let reasoning = if target.semantic_name == score.target { + let target = algorithm::select_eligible_model(&self.targets, &score.target, excluded)?; + let reasoning = if target == score.target { (self.decision_reason)(&self.name, &score) } else { format!( "{} exceeded its context window; fell back to {}", - score.target, target.semantic_name + score.target, target ) }; let decision: Decision = Decision::new( - target.semantic_name.clone(), - Some(with_routing_tier( - reasoning, - deciding.routing_tier(&target.semantic_name), - )), + target.clone(), + Some(with_routing_tier(reasoning, deciding.routing_tier(&target))), true, ); driver.decide(decision.clone()).await?; @@ -512,15 +500,8 @@ mod tests { } } - fn target_set(names: &[&str]) -> LlmTargetSet { - LlmTargetSet::new( - names - .iter() - .map(|name| LlmTarget { - semantic_name: name.to_string(), - }) - .collect(), - ) + fn target_set(names: &[&str]) -> Vec { + names.iter().map(|name| ModelId::from(*name)).collect() } fn target_prompts() -> TargetPrompts { @@ -587,7 +568,7 @@ mod tests { fn score(target: &str, confidence: f64) -> Score { Score { confidence, - target: target.to_string(), + target: ModelId::from(target), } } @@ -659,8 +640,8 @@ mod tests { move |decision: Decision, _request: Request| { let calls = Arc::clone(&calls); async move { - let model = decision.selected_model_id().to_string(); - calls.lock().push(model.clone()); + let model = decision.selected_model_id().clone(); + calls.lock().push(model.to_string()); if overflowing.contains(&model.as_str()) { return Err(LlmClientError::ContextWindowExceeded { model, @@ -872,8 +853,11 @@ mod tests { #[async_trait] impl Classifier for TieredClassifier { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { - match selected_model_id { + fn routing_tier( + &self, + selected_model_id: &switchyard_protocol::ModelId, + ) -> Option<&'static str> { + match selected_model_id.as_str() { "weak" => Some("weak"), "strong" => Some("strong"), _ => None, diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index 80c173a8..ec5be595 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use async_trait::async_trait; use serde::{Deserialize, Deserializer}; use serde_json::Value; -use switchyard_protocol::{ContentBlock, Decision, Message, Role}; +use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role}; use super::fall_through::{DefaultTarget, FallThrough}; use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; @@ -21,7 +21,7 @@ use super::util::llm_judge::{ SerdeDecoder, StructuredJudge, }; use super::util::target_selector::TargetSelectorPolicy; -use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet}; +use crate::core::algorithm::{self, Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::{State, StateValue}; use crate::{LibsyError, Result}; @@ -165,16 +165,16 @@ impl ClassifierInput for TaskInput { type CapabilityJudge = StructuredJudge>; struct TaskClassifierPolicy { - efficient_target: String, - capable_target: String, + efficient_target: ModelId, + capable_target: ModelId, base_threshold: f64, threshold_step: f64, } impl TaskClassifierPolicy { fn new( - efficient_target: impl Into, - capable_target: impl Into, + efficient_target: impl Into, + capable_target: impl Into, config: &TaskClassifierConfig, ) -> Self { Self { @@ -433,8 +433,8 @@ impl JudgePolicy for CustomPolicyRuntime { struct TaskClassifier { classifier: JudgeClassifier, - efficient_target: String, - capable_target: String, + efficient_target: ModelId, + capable_target: ModelId, } // ── Escalation classifier ────────────────────────────────────────────────── @@ -449,9 +449,9 @@ fn streak(state: &State) -> u32 { } } -fn decisive(target: &str) -> Classification { +fn decisive(target: &ModelId) -> Classification { Classification::Scores(vec![Score { - target: target.to_string(), + target: target.clone(), confidence: 1.0, }]) } @@ -471,20 +471,20 @@ fn assistant_message(response: &AggLlmResponse) -> Message { /// not pay for a second model call. struct EscalationClassifier { judge: JudgeClassifier, - capable: LlmTarget, - efficient: LlmTarget, + capable: ModelId, + efficient: ModelId, /// Consecutive escalate verdicts required to latch. confirmations: u32, } #[async_trait] impl Classifier for EscalationClassifier { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { - if self.capable.semantic_name == self.efficient.semantic_name { + fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> { + if self.capable == self.efficient { None - } else if selected_model_id == self.capable.semantic_name { + } else if *selected_model_id == self.capable { Some("strong") - } else if selected_model_id == self.efficient.semantic_name { + } else if *selected_model_id == self.efficient { Some("weak") } else { None @@ -505,7 +505,7 @@ impl Classifier for EscalationClassifier { // A confirmed session stays capable without a judge call. if streak(state) >= self.confirmations { - return Ok((decisive(&self.capable.semantic_name), None)); + return Ok((decisive(&self.capable), None)); } // Call efficient model and buffer the response so the judge can read it. @@ -517,7 +517,7 @@ impl Classifier for EscalationClassifier { .call_model( request.clone(), Decision::new( - self.efficient.semantic_name.clone(), + self.efficient.clone(), Some("escalation classifier: efficient tier".into()), true, ), @@ -528,7 +528,7 @@ impl Classifier for EscalationClassifier { Err(LibsyError::ClientCall { source: LlmClientError::ContextWindowExceeded { .. }, .. - }) => return Ok((decisive(&self.capable.semantic_name), None)), + }) => return Ok((decisive(&self.capable), None)), Err(e) => return Err(e), }; let agg = efficient_response @@ -561,7 +561,7 @@ impl Classifier for EscalationClassifier { let held = streak(state); let best = classification.argmax(false)?; let (escalate, pending) = match &best { - Some(score) if score.target == self.capable.semantic_name => (true, held + 1), + Some(score) if score.target == self.capable => (true, held + 1), Some(_) => (false, 0), None => (false, held), }; @@ -571,13 +571,10 @@ impl Classifier for EscalationClassifier { if escalate && pending >= self.confirmations { // Streak confirmed: drop the efficient response, caller will serve capable. - return Ok((decisive(&self.capable.semantic_name), None)); + return Ok((decisive(&self.capable), None)); } - Ok(( - decisive(&self.efficient.semantic_name), - Some(efficient_response), - )) + Ok((decisive(&self.efficient), Some(efficient_response))) } } @@ -589,7 +586,7 @@ pub struct LlmTaskClassifier { } struct ClassifierRouteConfig { - default_target: String, + default_target: ModelId, session_affinity: bool, message_hash_fallback: bool, } @@ -600,22 +597,22 @@ pub enum LlmClassifierConfig { /// Routes between efficient and capable targets from a task-level verdict. Capability { /// Target that produces classifier verdicts. - judge_target: LlmTarget, + judge_target: ModelId, /// Target used when the efficient tier can handle the task. - efficient_target: LlmTarget, + efficient_target: ModelId, /// Target used when the task needs the capable tier. - capable_target: LlmTarget, + capable_target: ModelId, /// Capability classifier settings. config: TaskClassifierConfig, }, /// Judges efficient responses and escalates after a confirmed streak. Escalation { /// Target that produces escalation verdicts. - judge_target: LlmTarget, + judge_target: ModelId, /// Target called before each escalation decision. - efficient_target: LlmTarget, + efficient_target: ModelId, /// Target used after escalation is confirmed. - capable_target: LlmTarget, + capable_target: ModelId, /// Prompt and verdict contract settings for the escalation judge. contract: ClassifierContractConfig, /// Escalation policy settings. @@ -626,9 +623,9 @@ pub enum LlmClassifierConfig { /// Routes among named targets using a user-supplied schema and policy. Custom { /// Target that produces classifier verdicts. - judge_target: LlmTarget, + judge_target: ModelId, /// User-facing labels paired with their resolved routing targets. - targets: Vec<(String, LlmTarget)>, + targets: Vec<(String, ModelId)>, /// Label selected when the judge does not produce a usable verdict. default_target: String, /// Custom classifier settings. @@ -676,14 +673,14 @@ impl LlmTaskClassifier { } fn build_capability( - judge_target: LlmTarget, - efficient_target: LlmTarget, - capable_target: LlmTarget, + judge_target: ModelId, + efficient_target: ModelId, + capable_target: ModelId, config: TaskClassifierConfig, ) -> Result { config.validate()?; let contract = Self::load_capability_contract(&config.contract)?; - let targets = LlmTargetSet::new(vec![efficient_target.clone(), capable_target.clone()]); + let targets = vec![efficient_target.clone(), capable_target.clone()]; let session_affinity = config.session_affinity; let message_hash_fallback = config.message_hash_fallback; let classifier = Arc::new(TaskClassifier { @@ -698,13 +695,13 @@ impl LlmTaskClassifier { ), judge_target.clone(), TaskClassifierPolicy::new( - efficient_target.semantic_name.clone(), - capable_target.semantic_name.clone(), + efficient_target.clone(), + capable_target.clone(), &config, ), ), - efficient_target: efficient_target.semantic_name.clone(), - capable_target: capable_target.semantic_name.clone(), + efficient_target: efficient_target.clone(), + capable_target: capable_target.clone(), }); let inner: Arc> = classifier.clone(); Self::from_classifier( @@ -719,8 +716,8 @@ impl LlmTaskClassifier { } fn build_custom( - judge_target: LlmTarget, - targets: Vec<(String, LlmTarget)>, + judge_target: ModelId, + targets: Vec<(String, ModelId)>, default_target: String, config: CustomClassifierConfig, ) -> Result { @@ -732,7 +729,7 @@ impl LlmTaskClassifier { } let mut labels = BTreeSet::new(); - let mut semantic_names = BTreeSet::new(); + let mut resolved_names = BTreeSet::new(); let mut target_map = BTreeMap::new(); let mut resolved_targets = Vec::with_capacity(targets.len()); for (label, target) in targets { @@ -747,18 +744,15 @@ impl LlmTaskClassifier { message: format!("custom classifier target label {label:?} is duplicated"), }); } - if !semantic_names.insert(target.semantic_name.clone()) { + if !resolved_names.insert(target.clone()) { return Err(LibsyError::AlgorithmError { - message: format!( - "custom classifier resolved target {:?} is duplicated", - target.semantic_name - ), + message: format!("custom classifier resolved target {target:?} is duplicated"), }); } - target_map.insert(label, target.semantic_name.clone()); + target_map.insert(label, target.clone()); resolved_targets.push(target); } - let default_semantic_name = + let default_name = target_map .get(&default_target) .cloned() @@ -797,10 +791,10 @@ impl LlmTaskClassifier { )); Self::from_classifier( - LlmTargetSet::new(resolved_targets), + resolved_targets, classifier, ClassifierRouteConfig { - default_target: default_semantic_name, + default_target: default_name, session_affinity, message_hash_fallback, }, @@ -808,15 +802,15 @@ impl LlmTaskClassifier { } fn build_escalation( - judge_target: LlmTarget, - efficient_target: LlmTarget, - capable_target: LlmTarget, + judge_target: ModelId, + efficient_target: ModelId, + capable_target: ModelId, contract_config: ClassifierContractConfig, config: EscalationJudgeConfig, max_output_tokens: u64, ) -> Result { - let capable_name = capable_target.semantic_name.clone(); - let efficient_name = efficient_target.semantic_name.clone(); + let capable_name = capable_target.clone(); + let efficient_name = efficient_target.clone(); let confirmations = config.confirmations; let esc = Arc::new(EscalationClassifier { judge: escalation::build_judge( @@ -832,7 +826,7 @@ impl LlmTaskClassifier { confirmations, }); let inner: Arc> = esc.clone(); - let targets = LlmTargetSet::new(vec![capable_target, efficient_target]); + let targets = vec![capable_target, efficient_target]; Ok(Self { route: FallThrough::::new_with_state(targets) .with_name(ALGORITHM_NAME) @@ -848,11 +842,11 @@ impl LlmTaskClassifier { /// Keeps affinity and fallback ordering identical across judge-backed modes. fn from_classifier( - targets: LlmTargetSet, + targets: Vec, inner: Arc>, config: ClassifierRouteConfig, ) -> Result { - targets.get_target(&config.default_target)?; + algorithm::ensure_model_is_target(&targets, &config.default_target)?; if config.message_hash_fallback && !config.session_affinity { return Err(LibsyError::AlgorithmError { message: "message_hash_fallback requires session_affinity".to_string(), @@ -886,12 +880,12 @@ impl LlmTaskClassifier { #[async_trait] impl Classifier for TaskClassifier { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { + fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> { if self.efficient_target == self.capable_target { None - } else if selected_model_id == self.efficient_target { + } else if *selected_model_id == self.efficient_target { Some("weak") - } else if selected_model_id == self.capable_target { + } else if *selected_model_id == self.capable_target { Some("strong") } else { None @@ -910,7 +904,7 @@ impl Classifier for TaskClassifier { #[async_trait] impl Classifier for LlmTaskClassifier { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { + fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> { self.inner.routing_tier(selected_model_id) } @@ -981,7 +975,7 @@ mod tests { fn selected( policy: &TaskClassifierPolicy, verdict: Option<&TaskClassifierVerdict>, - ) -> Result { + ) -> Result { policy .to_classification(verdict) .argmax(false)? @@ -1079,14 +1073,11 @@ mod tests { } fn router() -> Result> { - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), config: test_config(TEST_THRESHOLD), }, )?)) @@ -1129,7 +1120,10 @@ mod tests { let (trace, response) = test_drive(router, classify_request(), unreachable_judge()).await?; - assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable")); + assert_eq!( + trace.last().map(|d| d.selected_model_id().as_str()), + Some("capable") + ); assert_eq!( response.llm_response.as_agg().map(completion_text), Some("answer from capable".to_string()) @@ -1165,13 +1159,10 @@ mod tests { #[tokio::test] async fn classifier_config_sets_the_judge_completion_cap() -> Result<()> { let recorder = Arc::new(Recorder::default()); - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), config: TaskClassifierConfig { max_output_tokens: 512, ..test_config(TEST_THRESHOLD) @@ -1187,13 +1178,10 @@ mod tests { #[tokio::test] async fn classifier_config_overrides_the_packaged_prompt() -> Result<()> { let recorder = Arc::new(Recorder::default()); - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), config: TaskClassifierConfig { contract: ClassifierContractConfig::default() .with_prompt("Custom capability rubric."), @@ -1212,13 +1200,10 @@ mod tests { #[tokio::test] async fn classifier_config_enables_session_affinity() -> Result<()> { let recorder = Arc::new(Recorder::default()); - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), config: TaskClassifierConfig { session_affinity: true, ..test_config(TEST_THRESHOLD) @@ -1236,13 +1221,10 @@ mod tests { #[tokio::test] async fn classifier_config_reuses_message_hash_affinity_for_a_follow_up() -> Result<()> { let recorder = Arc::new(Recorder::default()); - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), config: TaskClassifierConfig { session_affinity: true, message_hash_fallback: true, @@ -1301,15 +1283,12 @@ mod tests { #[test] fn invalid_classifier_config_is_rejected() -> Result<()> { - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; for bad in [1.5, -0.1, f64::NAN, f64::INFINITY] { assert!( LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("e"), - capable_target: target("c"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("e"), + capable_target: ModelId::from("c"), config: test_config(bad), }) .is_err(), @@ -1340,9 +1319,9 @@ mod tests { ] { assert!( LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("e"), - capable_target: target("c"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("e"), + capable_target: ModelId::from("c"), config, }) .is_err() @@ -1350,9 +1329,9 @@ mod tests { } for base_threshold in [0.0, 1.0] { LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: target("judge"), - efficient_target: target("e"), - capable_target: target("c"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("e"), + capable_target: ModelId::from("c"), config: test_config(base_threshold), })?; } @@ -1794,14 +1773,11 @@ mod tests { /// Builds a router with escalation enabled (`confirmations=1` latches on the first verdict). fn escalation_router() -> Result> { - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; Ok(Arc::new(LlmTaskClassifier::new( LlmClassifierConfig::Escalation { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), contract: ClassifierContractConfig::default(), config: EscalationJudgeConfig { confirmations: 1, @@ -1824,7 +1800,7 @@ mod tests { // The efficient model is the serving target, and the response comes from its call. assert_eq!( - trace.last().map(|d| d.selected_model_id()), + trace.last().map(|d| d.selected_model_id().as_str()), Some("efficient") ); assert!( @@ -1843,13 +1819,10 @@ mod tests { #[tokio::test] async fn escalation_config_overrides_the_packaged_prompt() -> Result<()> { let recorder = Arc::new(Recorder::default()); - let target = |name: &str| LlmTarget { - semantic_name: name.to_string(), - }; let router = Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: target("judge"), - efficient_target: target("efficient"), - capable_target: target("capable"), + judge_target: ModelId::from("judge"), + efficient_target: ModelId::from("efficient"), + capable_target: ModelId::from("capable"), contract: ClassifierContractConfig::default().with_prompt("Custom trajectory rubric."), config: EscalationJudgeConfig { confirmations: 1, @@ -1877,7 +1850,10 @@ mod tests { let (trace, response) = test_drive(router, classify_request(), queued(model, judge)).await?; - assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable")); + assert_eq!( + trace.last().map(|d| d.selected_model_id().as_str()), + Some("capable") + ); assert!( trace .last() @@ -1908,7 +1884,10 @@ mod tests { .await?; let (trace, _) = test_drive(router.clone(), session_request, queued(model, judge)).await?; - assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable")); + assert_eq!( + trace.last().map(|d| d.selected_model_id().as_str()), + Some("capable") + ); Ok(()) } @@ -1921,9 +1900,9 @@ mod tests { // Efficient overflows, capable answers, and the judge must never be called. let serve = |decision: Decision, _request: Request| async move { - match decision.selected_model_id() { + match decision.selected_model_id().as_str() { "efficient" => Err(LlmClientError::ContextWindowExceeded { - model: decision.selected_model_id().to_string(), + model: decision.selected_model_id().clone(), message: "prompt is too long".to_string(), }), "judge" => panic!("the judge must not be consulted when efficient overflows"), @@ -1933,7 +1912,10 @@ mod tests { let (trace, response) = test_drive(router, classify_request(), serve).await?; - assert_eq!(trace.last().map(|d| d.selected_model_id()), Some("capable")); + assert_eq!( + trace.last().map(|d| d.selected_model_id().as_str()), + Some("capable") + ); assert_eq!( response.llm_response.as_agg().map(completion_text), Some("capable answer".to_string()) diff --git a/crates/libsy/src/algorithms/passthrough.rs b/crates/libsy/src/algorithms/passthrough.rs index d80b16f1..0727175b 100644 --- a/crates/libsy/src/algorithms/passthrough.rs +++ b/crates/libsy/src/algorithms/passthrough.rs @@ -5,21 +5,23 @@ use std::sync::Arc; -use switchyard_protocol::{Request, Response}; +use switchyard_protocol::{ModelId, Request, Response}; use crate::Result; -use crate::core::algorithm::{Algorithm, Driver, LlmTarget}; +use crate::core::algorithm::{Algorithm, Driver}; use switchyard_protocol::Decision; /// Routing algorithm that always calls one configured target. pub struct Passthrough { - target: LlmTarget, + target: ModelId, } impl Passthrough { /// Creates an algorithm that always calls `target`. - pub fn new(target: LlmTarget) -> Self { - Passthrough { target } + pub fn new(target: impl Into) -> Self { + Passthrough { + target: target.into(), + } } } @@ -31,11 +33,8 @@ impl Algorithm for Passthrough { async fn route(self: Arc, driver: Driver, request: Request) -> Result { let decision: Decision = Decision::new( - self.target.semantic_name.clone(), - Some(format!( - "passthrough selected target '{}'", - self.target.semantic_name - )), + self.target.clone(), + Some(format!("passthrough selected target '{}'", self.target)), true, ); driver.decide(decision.clone()).await?; @@ -48,7 +47,7 @@ mod tests { use std::sync::Arc; use super::Passthrough; - use crate::core::algorithm::{Algorithm, LlmTarget}; + use crate::core::algorithm::Algorithm; use crate::core::testing::{echo, test_drive}; use switchyard_protocol::{Request, completion_text, text_request}; @@ -60,9 +59,7 @@ mod tests { raw_request: None, metadata: None, }; - let algorithm: Arc = Arc::new(Passthrough::new(LlmTarget { - semantic_name: MODEL_ID.to_string(), - })); + let algorithm: Arc = Arc::new(Passthrough::new(MODEL_ID)); let (trace, response) = test_drive(algorithm, request, echo()).await?; assert_eq!( diff --git a/crates/libsy/src/algorithms/rand.rs b/crates/libsy/src/algorithms/rand.rs index 2a7b61c8..bb52aa33 100644 --- a/crates/libsy/src/algorithms/rand.rs +++ b/crates/libsy/src/algorithms/rand.rs @@ -16,14 +16,14 @@ use rand::distr::{Distribution, weighted::WeightedIndex}; use rand::rngs::StdRng; use crate::algorithms::fall_through::FallThrough; -use crate::core::algorithm::{Algorithm, Driver, LlmTargetSet}; +use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier, Score}; use crate::{LibsyError, Result}; -use switchyard_protocol::{Request, Response}; +use switchyard_protocol::{ModelId, Request, Response}; /// Stateless weighted classifier used by random fall-through routing. pub struct RandomClassifier { - targets: Vec, + targets: Vec, distribution: WeightedIndex, rng: Mutex, } @@ -40,12 +40,16 @@ impl RandomClassifier { /// Returns an error when targets are empty or duplicated, or when explicit /// weights have the wrong length, are negative or non-finite, or contain no /// positive value. - pub fn new(targets: Vec, weights: Option>, seed: Option) -> Result { + pub fn new( + targets: Vec, + weights: Option>, + seed: Option, + ) -> Result { let target_count = targets.len(); if target_count == 0 { return Err(LibsyError::NoTargets); } - let unique_targets = targets.iter().map(String::as_str).collect::>(); + let unique_targets = targets.iter().map(ModelId::as_str).collect::>(); if unique_targets.len() != target_count { return Err(LibsyError::AlgorithmError { message: "random targets must be unique".to_string(), @@ -85,7 +89,7 @@ impl RandomClassifier { }) } - fn select_target(&self) -> String { + fn select_target(&self) -> ModelId { let mut rng = self.rng.lock(); let index = self.distribution.sample(&mut *rng); self.targets[index].clone() @@ -125,23 +129,18 @@ pub struct Random { } impl Random { - /// Creates a router over `target_set`. + /// Creates a router over `targets`. /// /// # Errors /// /// Returns an error when targets or weights are invalid for [`RandomClassifier`]. pub fn new( - target_set: LlmTargetSet, + targets: Vec, weights: Option>, seed: Option, ) -> Result { - let target_names = target_set - .targets() - .iter() - .map(|target| target.semantic_name.clone()) - .collect(); - let classifier = Arc::new(RandomClassifier::new(target_names, weights, seed)?); - let inner = FallThrough::<()>::new(target_set) + let classifier = Arc::new(RandomClassifier::new(targets.clone(), weights, seed)?); + let inner = FallThrough::<()>::new(targets) .with_name("random") .with_decision_reason(random_decision_reason) .with_classifier(classifier); @@ -172,7 +171,6 @@ mod tests { use switchyard_protocol::{Metadata, completion_text, text_request}; use crate::algorithms::util::affinity::AffinityRouter; - use crate::core::algorithm::LlmTarget; use crate::core::testing::{echo, test_drive}; use switchyard_protocol::Request; @@ -194,14 +192,8 @@ mod tests { } } - fn target_set(names: &[&str]) -> LlmTargetSet { - let targets = names - .iter() - .map(|name| LlmTarget { - semantic_name: (*name).to_string(), - }) - .collect(); - LlmTargetSet::new(targets) + fn target_set(names: &[&str]) -> Vec { + names.iter().map(|name| ModelId::from(*name)).collect() } fn algorithm(names: &[&str], weights: Option>, seed: Option) -> Result { @@ -324,7 +316,7 @@ mod tests { let names = ["a/model", "b/model"]; let affinity = Arc::new(AffinityRouter::new()); let random = Arc::new(RandomClassifier::new( - names.iter().map(|name| (*name).to_string()).collect(), + names.iter().map(|name| ModelId::from(*name)).collect(), None, Some(42), )?); @@ -353,7 +345,7 @@ mod tests { .argmax(false)?; assert_eq!( retained.map(|score| score.target), - Some(selected.to_string()) + Some(ModelId::from(selected.clone())) ); let (_, second) = test_drive(algorithm, request_for_session("session-1"), echo()).await?; diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index c2ee63c3..6512d763 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -25,11 +25,11 @@ use super::util::stage::{ record_decision_source, record_routing_decision, }; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; -use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet}; +use crate::core::algorithm::{Algorithm, Driver}; use crate::core::classifier::{Classification, Classifier}; use crate::core::state::State; use crate::{LibsyError, Result}; -use switchyard_protocol::{Request, Response}; +use switchyard_protocol::{ModelId, Request, Response}; /// Telemetry name for a router this module assembles. const STAGE_ROUTER: &str = "stage_router"; @@ -45,7 +45,7 @@ struct SourceStamp { #[async_trait] impl Classifier for SourceStamp { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { + fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> { self.inner.routing_tier(selected_model_id) } @@ -69,7 +69,7 @@ impl Classifier for SourceStamp { pub struct LlmFallback { /// Target the judge model is called through. It is not a routing /// destination, so it does not belong in the router's target set. - pub judge_target: LlmTarget, + pub judge_target: ModelId, /// Judge configuration. `recent_turn_window` is worth setting to this router's /// `recent_window` so the judge reads the same span the signal scorer scored. /// Note: `session_affinity` and `message_hash_fallback` have no effect here — @@ -126,11 +126,7 @@ impl StageRouter { /// routing destination. /// /// Errors if either threshold in `config` is outside `[0.0, 1.0]`. - pub fn new( - capable: LlmTarget, - efficient: LlmTarget, - config: StageRouterConfig, - ) -> Result { + pub fn new(capable: ModelId, efficient: ModelId, config: StageRouterConfig) -> Result { Ok(Self { route: build_route(capable, efficient, config)?, }) @@ -150,8 +146,8 @@ impl Algorithm for StageRouter { /// Wires the cascade the wrapper drives. fn build_route( - capable: LlmTarget, - efficient: LlmTarget, + capable: ModelId, + efficient: ModelId, config: StageRouterConfig, ) -> Result> { if !(0.0..=1.0).contains(&config.confidence_threshold) { @@ -164,10 +160,7 @@ fn build_route( } // The tiers are a fixed pair; their targets are whatever the deployment calls // them, and the classifier scores onto those names. - let targets = StageTargets::new( - capable.semantic_name.clone(), - efficient.semantic_name.clone(), - ); + let targets = StageTargets::new(capable.clone(), efficient.clone()); // The picker's mode fixes the fallback tier up front, so the terminal // classifier is a constant rather than a per-turn lookup. let fall_open = targets.name(config.mode.default_tier()).to_string(); @@ -179,7 +172,7 @@ fn build_route( let signals = ToolSignalProcessor { recent_window: config.recent_window.unwrap_or(DEFAULT_RECENT_WINDOW), }; - let target_set = LlmTargetSet::new(vec![capable.clone(), efficient.clone()]); + let target_set = vec![capable.clone(), efficient.clone()]; let mut router = FallThrough::::new_with_state(target_set) .with_name(STAGE_ROUTER) .with_processor(Arc::new(signals)) @@ -223,18 +216,11 @@ mod tests { use super::*; use crate::algorithms::util::stage::DECISION_SOURCE_KEY; - use crate::core::algorithm::LlmTarget; use crate::core::classifier::Score; use crate::core::state::StateValue; use crate::core::testing::{Serve, reply, test_drive}; use switchyard_protocol::{Decision, Metadata, Response}; - fn tier_target(name: &str) -> LlmTarget { - LlmTarget { - semantic_name: name.to_string(), - } - } - /// A classifier that always picks `target`, standing in for a cascade member. struct Fixed(&'static str); @@ -248,7 +234,7 @@ mod tests { ) -> Result<(Classification, Option)> { Ok(( Classification::Scores(vec![Score { - target: self.0.to_string(), + target: ModelId::from(self.0), confidence: 1.0, }]), None, @@ -311,7 +297,7 @@ mod tests { let mut config = config(); config.confidence_threshold = 1.5; assert!(matches!( - StageRouter::new(tier_target("strong"), tier_target("weak"), config), + StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config), Err(LibsyError::AlgorithmError { .. }) )); } @@ -320,23 +306,21 @@ mod tests { fn rejects_an_out_of_range_judge_threshold() { let mut config = config(); config.llm_fallback = Some(LlmFallback { - judge_target: LlmTarget { - semantic_name: "judge".to_string(), - }, + judge_target: ModelId::from("judge"), config: TaskClassifierConfig { base_threshold: -0.1, ..Default::default() }, }); assert!(matches!( - StageRouter::new(tier_target("strong"), tier_target("weak"), config), + StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config), Err(LibsyError::AlgorithmError { .. }) )); } #[test] fn builds_over_both_tiers() -> Result<()> { - let router = StageRouter::new(tier_target("strong"), tier_target("weak"), config())?; + let router = StageRouter::new(ModelId::from("strong"), ModelId::from("weak"), config())?; assert_eq!(router.name(), STAGE_ROUTER); Ok(()) } @@ -402,16 +386,10 @@ mod tests { } } - fn recording_target(name: &str) -> LlmTarget { - LlmTarget { - semantic_name: name.to_string(), - } - } - fn recording_router(config: StageRouterConfig) -> Result> { Ok(Arc::new(StageRouter::new( - recording_target("strong"), - recording_target("weak"), + ModelId::from("strong"), + ModelId::from("weak"), config, )?)) } @@ -426,7 +404,7 @@ mod tests { *recorder.judge_p_solve.lock() = p_solve; let mut c = config(); c.llm_fallback = Some(LlmFallback { - judge_target: recording_target(JUDGE), + judge_target: ModelId::from(JUDGE), config: TaskClassifierConfig { base_threshold: 0.5, recent_turn_window: Some(3), diff --git a/crates/libsy/src/algorithms/subagent_affinity_tests.rs b/crates/libsy/src/algorithms/subagent_affinity_tests.rs index 02b80893..4a390351 100644 --- a/crates/libsy/src/algorithms/subagent_affinity_tests.rs +++ b/crates/libsy/src/algorithms/subagent_affinity_tests.rs @@ -15,11 +15,11 @@ use super::fall_through::FallThrough; use super::util::affinity::AffinityRouter; use super::util::subagent::SubagentOverride; use crate::Result; -use crate::core::algorithm::{Driver, LlmTarget, LlmTargetSet}; +use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::testing::{echo, test_drive}; use switchyard_protocol::{ - Metadata, Request, Response, completion_text, slice_to_header_map, text_request, + Metadata, ModelId, Request, Response, completion_text, slice_to_header_map, text_request, }; /// The cascade's terminal classifier: always picks the orchestrator. @@ -36,22 +36,18 @@ impl Classifier for AlwaysOrchestrator { Ok(( Classification::Scores(vec![Score { confidence: 0.5, - target: "orchestrator".to_string(), + target: ModelId::from("orchestrator"), }]), None, )) } } -fn targets() -> LlmTargetSet { - LlmTargetSet::new( - ["orchestrator", "worker", "reviewer"] - .iter() - .map(|name| LlmTarget { - semantic_name: (*name).to_string(), - }) - .collect(), - ) +fn targets() -> Vec { + ["orchestrator", "worker", "reviewer"] + .iter() + .map(|name| ModelId::from(*name)) + .collect() } fn request(headers: &[(&str, &str)]) -> Request { diff --git a/crates/libsy/src/algorithms/util/affinity.rs b/crates/libsy/src/algorithms/util/affinity.rs index bef22d17..5ffc869c 100644 --- a/crates/libsy/src/algorithms/util/affinity.rs +++ b/crates/libsy/src/algorithms/util/affinity.rs @@ -23,7 +23,7 @@ use std::sync::atomic::{AtomicBool, Ordering}; use async_trait::async_trait; use parking_lot::Mutex; -use switchyard_protocol::{Request, Role}; +use switchyard_protocol::{ModelId, Request, Role}; use crate::core::algorithm::{Driver, RoutingIdentity}; use crate::core::classifier::{Classification, Classifier, Score}; @@ -46,7 +46,7 @@ const MAX_ASSIGNMENTS: usize = 4096; #[derive(Default)] pub struct AffinityRouter { /// When set, only these models are retained; a decision for any other model is not latched. - latch_only: Option>, + latch_only: Option>, /// Whether root-session requests should abstain instead of being retained. subagents_only: bool, /// In absence of headers, use the message hash based fallback key to do task based routing @@ -55,7 +55,7 @@ pub struct AffinityRouter { /// /// Held on the instance so the two roles share one process-local map through a /// single registered [`Arc`](std::sync::Arc); bounded by [`MAX_ASSIGNMENTS`]. - assignments: Mutex>, + assignments: Mutex>, /// Whether the "no identity to key on" warning has already been emitted. unkeyed_warning_emitted: AtomicBool, } @@ -84,7 +84,7 @@ impl AffinityRouter { /// Restricts latching to `models`; a decision for any other model routes but is not /// retained. - pub fn with_latch_only(mut self, models: impl IntoIterator>) -> Self { + pub fn with_latch_only(mut self, models: impl IntoIterator>) -> Self { self.latch_only = Some(models.into_iter().map(Into::into).collect()); self } @@ -153,7 +153,7 @@ where let mut assignments = self.assignments.lock(); if self.should_latch(model) && !assignments.contains_key(&key) { evict_if_full(&mut assignments); - assignments.insert(key, model.to_string()); + assignments.insert(key, model.clone()); } } Ok(()) @@ -179,7 +179,7 @@ impl Classifier for AffinityRouter where S: Send + 'static, { - fn target_unavailable(&self, request: &Request, target: &str) { + fn target_unavailable(&self, request: &Request, target: &ModelId) { let Some(key) = self.affinity_key(request) else { return; }; @@ -216,7 +216,7 @@ where } /// Evicts one arbitrary assignment when the map has reached [`MAX_ASSIGNMENTS`]. -fn evict_if_full(assignments: &mut HashMap) { +fn evict_if_full(assignments: &mut HashMap) { if assignments.len() >= MAX_ASSIGNMENTS && let Some(evicted) = assignments.keys().next().cloned() { diff --git a/crates/libsy/src/algorithms/util/escalation.rs b/crates/libsy/src/algorithms/util/escalation.rs index b8387f62..54ded690 100644 --- a/crates/libsy/src/algorithms/util/escalation.rs +++ b/crates/libsy/src/algorithms/util/escalation.rs @@ -8,14 +8,13 @@ //! lives with the assembled algorithm in [`crate::algorithms::escalation`]. use serde::Deserialize; -use switchyard_protocol::{ContentBlock, Message, Role}; +use switchyard_protocol::{ContentBlock, Message, ModelId, Role}; use super::classifier_contract::{ClassifierContract, ClassifierContractConfig}; use super::llm_judge::{ ClassifierInput, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, SerdeDecoder, StructuredJudge, }; -use crate::core::algorithm::LlmTarget; use crate::core::classifier::{Classification, Score}; use crate::core::state::State; use crate::{LibsyError, Result}; @@ -118,8 +117,8 @@ pub(crate) type EscalationJudge = StructuredJudge { judge: J, - target: LlmTarget, + target: ModelId, policy: P, } @@ -202,7 +203,7 @@ where P: JudgePolicy, { /// Combines a judge target with a verdict policy. - pub fn new(judge: J, target: LlmTarget, policy: P) -> Self { + pub fn new(judge: J, target: ModelId, policy: P) -> Self { Self { judge, target, @@ -223,13 +224,13 @@ where request: &Request, driver: &Driver, ) -> Option { - let judge_model = self.target.semantic_name.as_str(); + let judge_model = self.target.as_str(); let response = driver .call_model( self.judge.build_request(state, request), Decision::new( - self.target.semantic_name.to_string(), + self.target.to_string(), Some("llm judge consultation".to_string()), false, ), @@ -303,7 +304,7 @@ where return Err(LibsyError::AlgorithmError { message: format!( "judge classifier for target {:?} requires a driver to call it", - self.target.semantic_name + self.target ), }); }; @@ -375,20 +376,14 @@ mod tests { "no-verdict" }; Classification::Scores(vec![Score { - target: target.to_string(), + target: ModelId::from(target), confidence: 1.0, }]) } } fn classifier() -> JudgeClassifier { - JudgeClassifier::new( - TestJudge, - LlmTarget { - semantic_name: "judge".to_string(), - }, - TestPolicy, - ) + JudgeClassifier::new(TestJudge, ModelId::from("judge"), TestPolicy) } fn request() -> Request { @@ -450,7 +445,7 @@ mod tests { } } - fn selected(classification: Classification) -> Result { + fn selected(classification: Classification) -> Result { classification .argmax(false)? .map(|score| score.target) @@ -460,7 +455,7 @@ mod tests { } /// Serves the single offloaded judge call with `reply` through a standalone step receiver. - async fn score_served_with(reply: Result) -> Result { + async fn score_served_with(reply: Result) -> Result { let (driver, step_rx) = Driver::new("test"); let mut steps = tokio_stream::wrappers::ReceiverStream::new(step_rx); let classifier = classifier(); diff --git a/crates/libsy/src/algorithms/util/prompts.rs b/crates/libsy/src/algorithms/util/prompts.rs index 9d45e65d..e6fd6c67 100644 --- a/crates/libsy/src/algorithms/util/prompts.rs +++ b/crates/libsy/src/algorithms/util/prompts.rs @@ -26,7 +26,7 @@ use std::collections::BTreeMap; use async_trait::async_trait; -use switchyard_protocol::{ContentBlock, InstructionBlock, Message, Request, Role}; +use switchyard_protocol::{ContentBlock, InstructionBlock, Message, ModelId, Request, Role}; use crate::Result; use crate::core::processor::{Event, Processor}; @@ -74,18 +74,18 @@ fn drop_exact_replay(request: &mut Request) { /// untouched. #[derive(Clone, Debug, Default)] pub struct TargetPrompts { - by_target: BTreeMap, + by_target: BTreeMap, } impl TargetPrompts { /// Hand `target` this prompt on every turn it serves. - pub fn with(mut self, target: impl Into, prompt: impl Into) -> Self { + pub fn with(mut self, target: impl Into, prompt: impl Into) -> Self { self.by_target.insert(target.into(), prompt.into()); self } /// The prompt configured for `target`, if any. - pub fn get(&self, target: &str) -> Option<&str> { + pub fn get(&self, target: &ModelId) -> Option<&str> { self.by_target.get(target).map(String::as_str) } diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 1976f6b9..3f16334f 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -28,6 +28,7 @@ use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier, Score}; use crate::core::state::{State, StateValue}; use crate::observability::meter; +use switchyard_protocol::ModelId; use switchyard_protocol::Request; /// Turn depth below which stall signals stay quiet — early no-write turns are @@ -96,13 +97,13 @@ impl Tier { /// reaches the right model. #[derive(Clone, Debug)] pub struct StageTargets { - capable: String, - efficient: String, + capable: ModelId, + efficient: ModelId, } impl StageTargets { /// Name the targets the two tiers route to. - pub fn new(capable: impl Into, efficient: impl Into) -> Self { + pub fn new(capable: impl Into, efficient: impl Into) -> Self { Self { capable: capable.into(), efficient: efficient.into(), @@ -110,7 +111,7 @@ impl StageTargets { } /// The target `tier` routes to. - pub fn name(&self, tier: Tier) -> &str { + pub fn name(&self, tier: Tier) -> &ModelId { match tier { Tier::Capable => &self.capable, Tier::Efficient => &self.efficient, @@ -118,10 +119,10 @@ impl StageTargets { } /// The tier label for a routed target, or `None` for one outside the pair. - pub fn label_for(&self, target: &str) -> Option<&'static str> { - if target == self.capable { + pub fn label_for(&self, target: &ModelId) -> Option<&'static str> { + if *target == self.capable { Some(Tier::Capable.label()) - } else if target == self.efficient { + } else if *target == self.efficient { Some(Tier::Efficient.label()) } else { None @@ -542,7 +543,7 @@ impl StageClassifier { #[async_trait] impl Classifier for StageClassifier { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { + fn routing_tier(&self, selected_model_id: &ModelId) -> Option<&'static str> { self.targets.label_for(selected_model_id) } @@ -578,7 +579,7 @@ impl Classifier for StageClassifier { let conf = score.abs(); Ok(( Classification::Scores(vec![Score { - target: target.to_string(), + target: target.clone(), confidence: conf, }]), None, diff --git a/crates/libsy/src/algorithms/util/subagent.rs b/crates/libsy/src/algorithms/util/subagent.rs index 829d1cb2..c2077ad1 100644 --- a/crates/libsy/src/algorithms/util/subagent.rs +++ b/crates/libsy/src/algorithms/util/subagent.rs @@ -20,12 +20,13 @@ use async_trait::async_trait; use crate::Result; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier, Score}; +use switchyard_protocol::ModelId; use switchyard_protocol::{Metadata, Request, Response}; /// Scores a fixed worker target for delegated sub-agent work; abstains otherwise. pub struct SubagentOverride { /// Name of the worker target, resolved by the cascade against its target set. - worker: String, + worker: ModelId, } impl SubagentOverride { @@ -33,7 +34,7 @@ impl SubagentOverride { /// /// `worker` must name a target in the cascade's set, or routing a sub-agent request /// fails with [`LibsyError::TargetNotFound`](crate::LibsyError::TargetNotFound). - pub fn new(worker: impl Into) -> Self { + pub fn new(worker: impl Into) -> Self { Self { worker: worker.into(), } @@ -80,14 +81,14 @@ mod tests { let metadata = (!headers.is_empty()).then(|| Metadata::from_headers(&slice_to_header_map(headers))); Request { - llm_request: text_request(Some("auto".to_string()), "hi"), + llm_request: text_request(Some(ModelId::from("auto").to_string()), "hi"), raw_request: None, metadata, } } /// Scores `headers` through the override, returning the winning target if it scored. - async fn selected(headers: &[(&str, &str)]) -> Result> { + async fn selected(headers: &[(&str, &str)]) -> Result> { let mut state = (); let classification = SubagentOverride::new("worker") .score(&mut state, &mut request(headers), None) @@ -108,16 +109,16 @@ mod tests { ("x-claude-code-session-id", "root"), ("x-claude-code-agent-id", "child-1"), ]; - assert_eq!(selected(claude).await?, Some("worker".to_string())); + assert_eq!(selected(claude).await?, Some(ModelId::from("worker"))); // Codex delegated-work kinds. assert_eq!( selected(&[("x-openai-subagent", "review")]).await?, - Some("worker".to_string()) + Some(ModelId::from("worker")) ); assert_eq!( selected(&[("x-openai-subagent", "collab_spawn")]).await?, - Some("worker".to_string()) + Some(ModelId::from("worker")) ); Ok(()) } diff --git a/crates/libsy/src/algorithms/util/target_selector.rs b/crates/libsy/src/algorithms/util/target_selector.rs index da6c4659..4e36b648 100644 --- a/crates/libsy/src/algorithms/util/target_selector.rs +++ b/crates/libsy/src/algorithms/util/target_selector.rs @@ -11,18 +11,19 @@ use serde_json::Value; use super::llm_judge::JudgePolicy; use crate::core::classifier::{Classification, Score}; use crate::{LibsyError, Result}; +use switchyard_protocol::ModelId; /// Maps one string field in a validated verdict to a configured routing target. pub(crate) struct TargetSelectorPolicy { selector: PointerBuf, - targets: BTreeMap, + targets: BTreeMap, } impl TargetSelectorPolicy { /// Parses a JSON Pointer used to read validated verdicts. pub(crate) fn new( selector: impl Into, - targets: BTreeMap, + targets: BTreeMap, ) -> Result { let selector = PointerBuf::parse(selector.into()).map_err(|error| LibsyError::AlgorithmError { @@ -67,8 +68,8 @@ mod tests { let policy = TargetSelectorPolicy::new( "/decision/target", BTreeMap::from([ - ("opus".to_string(), "model/opus".to_string()), - ("sonnet".to_string(), "model/sonnet".to_string()), + ("opus".to_string(), ModelId::from("model/opus")), + ("sonnet".to_string(), ModelId::from("model/sonnet")), ]), )?; let classification = policy.to_classification(Some(&json!({ @@ -77,7 +78,7 @@ mod tests { assert_eq!( classification.argmax(false)?.map(|score| score.target), - Some("model/sonnet".to_string()) + Some(ModelId::from("model/sonnet")) ); Ok(()) } @@ -86,7 +87,7 @@ mod tests { fn a_missing_or_unknown_target_abstains() -> Result<()> { let policy = TargetSelectorPolicy::new( "/target", - BTreeMap::from([("sonnet".to_string(), "model/sonnet".to_string())]), + BTreeMap::from([("sonnet".to_string(), ModelId::from("model/sonnet"))]), )?; assert_eq!( diff --git a/crates/libsy/src/core/algorithm.rs b/crates/libsy/src/core/algorithm.rs index f63d6e10..6df5c98e 100644 --- a/crates/libsy/src/core/algorithm.rs +++ b/crates/libsy/src/core/algorithm.rs @@ -28,7 +28,9 @@ use tracing::Instrument; /// [`switchyard_protocol::LlmResponseStreamEvent`] is its host/algorithm envelope; and /// [`switchyard_protocol::LlmResponse`] carries either a live /// [`switchyard_protocol::LlmResponseStream`] or the terminal aggregate. -use switchyard_protocol::{Decision, LlmClientError, Request, Response, RoutingFallbackReason}; +use switchyard_protocol::{ + Decision, LlmClientError, ModelId, Request, Response, RoutingFallbackReason, +}; use crate::{DriverError, LibsyError, Result, observability}; @@ -139,7 +141,7 @@ impl Driver { skip_all, fields( algorithm = self.algorithm, - selected_model = decision.selected_model_id(), + selected_model = %decision.selected_model_id(), openinference.span.kind = "CHAIN", outcome = tracing::field::Empty, error = tracing::field::Empty, @@ -290,60 +292,36 @@ impl Drop for AbortOnDrop { } } -/// A named routing target an algorithm routes by. Serving its calls is the stream -/// consumer's concern: the selected identifier reaches the consumer as -/// `decision.selected_model_id()` on the offloaded [`CallModel`]. -#[derive(Clone)] -pub struct LlmTarget { - /// The routing label an algorithm selects this target by — a logical tier like - /// `"strong"`, or the model id when they coincide. Mapping it to a provider model - /// id is the consumer's concern, never the algorithm's. - pub semantic_name: String, -} - -/// The set of targets an algorithm may route among. An algorithm is constructed -/// with one and picks targets by position ([`targets`](Self::targets)) or by name -/// ([`get_target`](Self::get_target)). -#[derive(Clone)] -pub struct LlmTargetSet { - targets: Vec, +/// Errors unless `targets` contains `name`. +/// +/// Config target names must be resolved before an algorithm is built. This list contains +/// model IDs, not target names. +pub(crate) fn ensure_model_is_target(targets: &[ModelId], name: &ModelId) -> Result<()> { + targets + .iter() + .any(|target| target == name) + .then_some(()) + .ok_or_else(|| LibsyError::TargetNotFound { + target: name.clone(), + }) } -impl LlmTargetSet { - /// Build a target set from a list of targets. - pub fn new(targets: Vec) -> Self { - Self { targets } - } - - /// All targets in the set — e.g. for an algorithm to select among. - pub fn targets(&self) -> &[LlmTarget] { - &self.targets - } - - /// Look up a target by name; errors if no target has that name. - pub fn get_target(&self, name: &str) -> Result { - self.targets - .iter() - .find(|t| t.semantic_name == name) - .cloned() - .ok_or_else(|| LibsyError::TargetNotFound { - target: name.to_string(), - }) - } - - /// The named target, or the first one not in `excluded` when it has been barred. - /// Errors if every target is excluded. - pub fn resolve_target(&self, name: &str, excluded: &HashSet) -> Result { - let target = self.get_target(name)?; - if !excluded.contains(&target.semantic_name) { - return Ok(target); - } - self.targets - .iter() - .find(|t| !excluded.contains(&t.semantic_name)) - .cloned() - .ok_or(LibsyError::AllTargetsExcluded) +/// `name` itself, or the first target not in `excluded` when `name` has been barred. +/// Errors if `name` is unknown, or if every target is excluded. +pub(crate) fn select_eligible_model( + targets: &[ModelId], + name: &ModelId, + excluded: &HashSet, +) -> Result { + ensure_model_is_target(targets, name)?; + if !excluded.contains(name) { + return Ok(name.clone()); } + targets + .iter() + .find(|target| !excluded.contains(*target)) + .cloned() + .ok_or(LibsyError::AllTargetsExcluded) } /// Key for overflow history: a root request by its session, a child request by its session @@ -395,7 +373,7 @@ const MAX_EVICTION_IDENTITIES: usize = 1_024; /// without a routing identity are not tracked — there is nothing to remember them by. #[derive(Default)] pub(crate) struct SessionEvictions { - by_identity: Mutex>>, + by_identity: Mutex>>, } impl SessionEvictions { @@ -407,7 +385,7 @@ impl SessionEvictions { } /// The targets `identity` has already overflowed; empty for an untracked request. - fn evicted_for(&self, identity: Option<&RoutingIdentity>) -> Vec { + fn evicted_for(&self, identity: Option<&RoutingIdentity>) -> Vec { let Some(identity) = identity else { return Vec::new(); }; @@ -420,7 +398,7 @@ impl SessionEvictions { /// Remembers that `target` overflowed for `identity`, tracking at most /// [`MAX_EVICTION_IDENTITIES`] identities. - fn record(&self, identity: Option<&RoutingIdentity>, target: &str) { + fn record(&self, identity: Option<&RoutingIdentity>, target: &ModelId) { let Some(identity) = identity else { return }; let mut histories = self.by_identity.lock(); if histories.len() >= MAX_EVICTION_IDENTITIES @@ -432,24 +410,23 @@ impl SessionEvictions { histories .entry(identity.clone()) .or_default() - .insert(target.to_string()); + .insert(target.clone()); } } /// How many of `targets` this request is still allowed to reach. -fn eligible_targets(targets: &LlmTargetSet, excluded: &HashSet) -> usize { +fn eligible_targets(targets: &[ModelId], excluded: &HashSet) -> usize { targets - .targets() .iter() - .filter(|t| !excluded.contains(&t.semantic_name)) + .filter(|target| !excluded.contains(*target)) .count() } /// Bars the targets `identity` has already overflowed from this request, so routing does /// not select one that is certain to fail again. pub(crate) fn exclude_evicted( - excluded: &mut HashSet, - targets: &LlmTargetSet, + excluded: &mut HashSet, + targets: &[ModelId], evictions: &SessionEvictions, identity: Option<&RoutingIdentity>, ) { @@ -464,7 +441,7 @@ pub(crate) fn exclude_evicted( } /// Returns the failed target and routing fallback policy for a terminal client error. -fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason)> { +fn classify_fallback(error: &LibsyError) -> Option<(&ModelId, RoutingFallbackReason)> { let LibsyError::ClientCall { target, source } = error else { return None; }; @@ -492,16 +469,16 @@ fn classify_fallback(error: &LibsyError) -> Option<(&str, RoutingFallbackReason) /// overflows are recorded for `identity`; unavailable targets remain request-local. #[allow(clippy::too_many_arguments)] pub(crate) async fn call_model_with_fallback( - excluded: &mut HashSet, + excluded: &mut HashSet, driver: &Driver, - targets: &LlmTargetSet, - mut target: LlmTarget, + targets: &[ModelId], + mut target: ModelId, mut decision: Decision, request: Request, identity: Option<&RoutingIdentity>, evictions: &SessionEvictions, - target_unavailable: impl Fn(&Request, &str), - fallback_decision: impl Fn(&LlmTarget, &LlmTarget, RoutingFallbackReason) -> Decision, + target_unavailable: impl Fn(&Request, &ModelId), + fallback_decision: impl Fn(&ModelId, &ModelId, RoutingFallbackReason) -> Decision, ) -> Result { loop { let result = driver.call_model(request.clone(), decision.clone()).await; @@ -511,14 +488,14 @@ pub(crate) async fn call_model_with_fallback( }; // A target already excluded means the pool is spent; surface the client error // so the caller still sees the concrete upstream failure. - if !excluded.insert(failed.to_string()) { + if !excluded.insert(failed.clone()) { return Err(error); } match reason { RoutingFallbackReason::ContextWindow => evictions.record(identity, failed), RoutingFallbackReason::Unavailable => target_unavailable(&request, failed), } - let Ok(next) = targets.resolve_target(&target.semantic_name, excluded) else { + let Ok(next) = select_eligible_model(targets, &target, excluded) else { return Err(error); }; decision = fallback_decision(&target, &next, reason); @@ -627,7 +604,7 @@ mod tests { fn route_fallback_only_accepts_context_and_unavailable_failures() { assert_eq!( classified_client_error(LlmClientError::ContextWindowExceeded { - model: "target".to_string(), + model: ModelId::from("target"), message: "too long".to_string(), }), Some(RoutingFallbackReason::ContextWindow) @@ -675,14 +652,14 @@ mod tests { } /// Build a routed decision for orchestration tests. - fn test_decision(selected_model_id: String) -> Decision { + fn test_decision(selected_model_id: ModelId) -> Decision { Decision::new(selected_model_id, None, true) } /// Trivial algo used only to exercise the orchestrator: calls the first target /// and returns its response with a one-item trace. struct TestAlgo { - target_set: LlmTargetSet, + target_set: Vec, } #[async_trait] @@ -694,18 +671,17 @@ mod tests { async fn route(self: Arc, driver: Driver, request: Request) -> Result { let target = self .target_set - .targets() .first() .ok_or(LibsyError::NoTargets)? .clone(); - let decision = test_decision(target.semantic_name.clone()); + let decision = test_decision(target.clone()); driver.decide(decision.clone()).await?; driver.call_model(request, decision).await } } /// Build a shared `TestAlgo` over the given target set. - fn orch(target_set: LlmTargetSet) -> Arc { + fn orch(target_set: Vec) -> Arc { Arc::new(TestAlgo { target_set }) } @@ -717,14 +693,8 @@ mod tests { } } - fn target_set(names: &[&str]) -> LlmTargetSet { - let targets = names - .iter() - .map(|name| LlmTarget { - semantic_name: name.to_string(), - }) - .collect(); - LlmTargetSet::new(targets) + fn target_set(names: &[&str]) -> Vec { + names.iter().map(|name| ModelId::from(*name)).collect() } #[tokio::test] @@ -736,12 +706,12 @@ mod tests { let first_driver = driver.clone(); let mut first = tokio::spawn(async move { first_driver - .call_model(request(), test_decision("first".to_string())) + .call_model(request(), test_decision(ModelId::from("first"))) .await }); let second = tokio::spawn(async move { driver - .call_model(request(), test_decision("second".to_string())) + .call_model(request(), test_decision(ModelId::from("second"))) .await }); @@ -787,7 +757,7 @@ mod tests { let (driver, mut step_rx) = Driver::new("test"); let producer = tokio::spawn(async move { driver - .call_model(request(), test_decision("dropped".to_string())) + .call_model(request(), test_decision(ModelId::from("dropped"))) .await }); let step = step_rx.recv().await.ok_or(DriverError::StreamClosed)??; @@ -806,7 +776,7 @@ mod tests { // A standalone driver reports the typed step receiver disappearing at its next send. let (driver, step_rx) = Driver::new("test"); drop(step_rx); - let decision = test_decision("closed".to_string()); + let decision = test_decision(ModelId::from("closed")); let result = driver.decide(decision).await; assert!(matches!( result, @@ -825,7 +795,7 @@ mod tests { #[tokio::test] async fn into_parts_yields_the_call_without_answering_it() -> Result<()> { let (driver, mut step_rx) = Driver::new("test"); - let decision = test_decision("answer/model".to_string()); + let decision = test_decision(ModelId::from("answer/model")); let producer = tokio::spawn({ let decision = decision.clone(); async move { driver.call_model(request(), decision).await } @@ -864,7 +834,7 @@ mod tests { #[test] fn target_lookup_returns_the_missing_target() { - let error = target_set(&[]).get_target("missing").err(); + let error = ensure_model_is_target(&target_set(&[]), &ModelId::from("missing")).err(); assert!(matches!( error, Some(LibsyError::TargetNotFound { target }) if target == "missing" @@ -1307,8 +1277,8 @@ mod tests { /// Offloads two targets concurrently and returns the first to resolve, dropping the /// loser's call (first-wins hedging). struct Hedge { - winner: LlmTarget, - loser: LlmTarget, + winner: String, + loser: String, } #[async_trait] @@ -1318,8 +1288,8 @@ mod tests { } async fn route(self: Arc, driver: Driver, request: Request) -> Result { - let dec_w = test_decision(self.winner.semantic_name.clone()); - let dec_l = test_decision(self.loser.semantic_name.clone()); + let dec_w = test_decision(self.winner.clone().into()); + let dec_l = test_decision(self.loser.clone().into()); let win = driver.call_model(request.clone(), dec_w); let lose = driver.call_model(request, dec_l); // First to resolve wins; `select!` drops the losing future (and its promise). @@ -1336,12 +1306,8 @@ mod tests { fn hedge(loser_delay: Option) -> (Arc, impl Serve) { let started = Arc::new(tokio::sync::Notify::new()); let algo = Arc::new(Hedge { - winner: LlmTarget { - semantic_name: "winner".to_string(), - }, - loser: LlmTarget { - semantic_name: "loser".to_string(), - }, + winner: "winner".to_string(), + loser: "loser".to_string(), }); let serve = move |decision: Decision, _request: Request| { let started = started.clone(); @@ -1421,7 +1387,7 @@ mod tests { async fn route(self: Arc, driver: Driver, request: Request) -> Result { let offloads = futures::future::join_all((0..self.n).map(|i| { - let decision = test_decision(format!("m{i}")); + let decision = test_decision(format!("m{i}").into()); driver.call_model(request.clone(), decision) })); tokio::select! { diff --git a/crates/libsy/src/core/classifier.rs b/crates/libsy/src/core/classifier.rs index f277b293..f6212f67 100644 --- a/crates/libsy/src/core/classifier.rs +++ b/crates/libsy/src/core/classifier.rs @@ -4,7 +4,7 @@ use crate::core::algorithm::Driver; use crate::{LibsyError, Result}; use async_trait::async_trait; -use switchyard_protocol::{Request, Response}; +use switchyard_protocol::{ModelId, Request, Response}; /// One classifier's recommendation of a routing `target`, with a `[0.0, 1.0]` confidence. #[derive(Debug, Clone, PartialEq)] @@ -12,7 +12,7 @@ pub struct Score { /// `[0.0, 1.0]` confidence in `target`. pub confidence: f64, /// The target (model / tier) being recommended. - pub target: String, + pub target: ModelId, } /// A classifier's verdict for a request: a set of target [`Score`]s, flagged by how @@ -72,14 +72,14 @@ fn argmax(scores: &[Score]) -> Result> { #[async_trait] pub trait Classifier: Send + Sync { /// Stable tier represented by `selected_model_id`, when this classifier defines one. - fn routing_tier(&self, _selected_model_id: &str) -> Option<&'static str> { + fn routing_tier(&self, _selected_model_id: &ModelId) -> Option<&'static str> { None } /// Drops retained routing state when `target` was unavailable for `request`. /// /// Stateless classifiers do not need to implement this hook. - fn target_unavailable(&self, _request: &Request, _target: &str) {} + fn target_unavailable(&self, _request: &Request, _target: &ModelId) {} /// Score the classifier's targets given the current state and request. /// @@ -107,7 +107,7 @@ mod tests { /// Terse `Score` builder for the assertions below. fn score(target: &str, confidence: f64) -> Score { Score { - target: target.to_string(), + target: ModelId::from(target), confidence, } } @@ -125,7 +125,7 @@ mod tests { // Equal confidence: the earlier target in cascade order wins the tie. let scores = vec![score("first", 0.7), score("second", 0.7)]; let best = Classification::Scores(scores).argmax(false)?; - assert_eq!(best.map(|s| s.target), Some("first".to_string())); + assert_eq!(best.map(|s| s.target), Some(ModelId::from("first"))); Ok(()) } @@ -196,7 +196,7 @@ mod tests { let target = request.requested_model().unwrap_or("auto").to_string(); Ok(( Classification::Scores(vec![Score { - target, + target: target.into(), confidence: 1.0, }]), None, @@ -218,7 +218,7 @@ mod tests { .await?; assert_eq!( classification.argmax(false)?.map(|s| s.target), - Some("strong".to_string()) + Some(ModelId::from("strong")) ); assert!(state); Ok(()) @@ -238,7 +238,7 @@ mod tests { request.llm_request.model = Some("rewritten".to_string()); Ok(( Classification::Scores(vec![Score { - target: "rewritten".to_string(), + target: ModelId::from("rewritten"), confidence: 1.0, }]), None, diff --git a/crates/libsy/src/error.rs b/crates/libsy/src/error.rs index 4703cb44..aa5c63b4 100644 --- a/crates/libsy/src/error.rs +++ b/crates/libsy/src/error.rs @@ -5,7 +5,7 @@ use std::error::Error as StdError; -use switchyard_protocol::LlmClientError; +use switchyard_protocol::{LlmClientError, ModelId}; use thiserror::Error; /// Result type returned by libsy APIs. @@ -17,8 +17,8 @@ pub enum LibsyError { /// A named target was not present in the configured target set. #[error("target {target:?} was not found")] TargetNotFound { - /// Missing semantic target name. - target: String, + /// Missing target model id. + target: ModelId, }, /// Routing was attempted without any configured targets. @@ -44,7 +44,7 @@ pub enum LibsyError { #[error("client call to target {target:?} failed: {source}")] ClientCall { /// Target whose client failed. - target: String, + target: ModelId, /// Typed error supplied by the protocol-owned client trait. #[source] source: LlmClientError, @@ -67,7 +67,7 @@ pub enum LibsyError { impl LibsyError { /// Wrap an error returned by the protocol-owned client trait. - pub fn client_call(target: impl Into, source: LlmClientError) -> Self { + pub fn client_call(target: impl Into, source: LlmClientError) -> Self { Self::ClientCall { target: target.into(), source, diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 3a48885a..3b1ede9a 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -5,9 +5,7 @@ #![doc = include_str!("../README.md")] mod core; -pub use core::algorithm::{ - Algorithm, CallModel, Driver, LlmTarget, LlmTargetSet, Step, StepStream, drive, -}; +pub use core::algorithm::{Algorithm, CallModel, Driver, Step, StepStream, drive}; pub use core::classifier::{Classification, Classifier, Score}; pub use core::processor::{Event, Processor}; pub use core::state::{State, StateValue}; diff --git a/crates/libsy/src/observability.rs b/crates/libsy/src/observability.rs index ca093dae..2435cb02 100644 --- a/crates/libsy/src/observability.rs +++ b/crates/libsy/src/observability.rs @@ -292,7 +292,7 @@ pub(crate) fn record_decision(algorithm: &str, decision: &Decision) { tracing::debug!( target: TRACING_TARGET, algorithm, - selected_model, + selected_model = %selected_model, reasoning = decision.reasoning().unwrap_or(""), "routing decision" ); diff --git a/crates/protocol/src/client.rs b/crates/protocol/src/client.rs index cf137e84..21f2ac2e 100644 --- a/crates/protocol/src/client.rs +++ b/crates/protocol/src/client.rs @@ -13,7 +13,7 @@ use async_trait::async_trait; use thiserror::Error; -use crate::{Request, Response}; +use crate::{ModelId, Request, Response}; /// A boxed client-specific error preserved as the source of a routed call failure. pub type BoxError = Box; @@ -72,7 +72,7 @@ pub enum LlmClientError { #[error("context window exceeded for model {model}: {message}")] ContextWindowExceeded { /// Model whose context window was exceeded. - model: String, + model: ModelId, /// Upstream error message. message: String, }, @@ -131,7 +131,7 @@ impl RoutingFallbackReason { #[derive(Clone, Debug)] pub struct Decision { /// The model identifier selected for the call. - selected_model_id: String, + selected_model_id: ModelId, /// Why, for logs and traces. reasoning: Option, /// True for an answer-generating call. False for classifier and judge calls. @@ -141,7 +141,7 @@ pub struct Decision { impl Decision { /// Creates a decision and records whether its call produces the answer. pub fn new( - selected_model_id: impl Into, + selected_model_id: impl Into, reasoning: Option, is_answer_call: bool, ) -> Self { @@ -153,8 +153,8 @@ impl Decision { } /// The model identifier selected for the call. - pub fn selected_model_id(&self) -> &str { - self.selected_model_id.as_str() + pub fn selected_model_id(&self) -> &ModelId { + &self.selected_model_id } /// Why this decision was made. diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 2747a12e..15352b44 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -9,6 +9,7 @@ pub mod envelope; pub mod format; pub mod llm; pub mod metadata; +pub mod model_id; pub mod stream; pub use client::*; @@ -16,6 +17,7 @@ pub use envelope::*; pub use format::*; pub use llm::*; pub use metadata::*; +pub use model_id::*; pub use stream::*; /// Builds a single-turn request: one user message carrying `prompt`, for `model`. diff --git a/crates/protocol/src/model_id.rs b/crates/protocol/src/model_id.rs new file mode 100644 index 00000000..854e9d77 --- /dev/null +++ b/crates/protocol/src/model_id.rs @@ -0,0 +1,188 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Model identifier used by Switchyard. See ModelId docs for details. + +use std::borrow::Borrow; +use std::fmt; +use std::ops::Deref; + +use serde::{Deserialize, Serialize}; + +/// A model name used in a request or routing decision. +/// +/// It can name a provider model, such as `"openai/gpt-oss-20b"`. It can also name a +/// Switchyard route, such as `"switchyard/random"`. +/// +/// A target name from server config, such as `"capable"`, is not a model ID. The server +/// resolves target names to model IDs before routing. +/// +/// This type acts like a string. It can be printed, compared, and saved as a JSON string. +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct ModelId(String); + +/// Forwards to the wrapped string rather than deriving, so `{:?}` renders `"gpt-4"` +/// instead of `ModelId("gpt-4")`. +impl fmt::Debug for ModelId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + fmt::Debug::fmt(&self.0, formatter) + } +} + +impl ModelId { + /// Wraps a model identifier. + pub fn new(id: impl Into) -> Self { + Self(id.into()) + } + + /// The identifier as a string slice. + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Unwraps to the owned identifier. + pub fn into_string(self) -> String { + self.0 + } +} + +impl fmt::Display for ModelId { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +impl Deref for ModelId { + type Target = str; + + fn deref(&self) -> &str { + &self.0 + } +} + +impl AsRef for ModelId { + fn as_ref(&self) -> &str { + &self.0 + } +} + +/// Lets a `HashMap` or `HashSet` be looked up by `&str`, so +/// callers holding a borrowed id do not have to allocate one to query. +impl Borrow for ModelId { + fn borrow(&self) -> &str { + &self.0 + } +} + +impl From for ModelId { + fn from(id: String) -> Self { + Self(id) + } +} + +impl From<&str> for ModelId { + fn from(id: &str) -> Self { + Self(id.to_string()) + } +} + +/// Mirrors `From<&str> for String`, so a borrowed id reaches an owning +/// `impl Into` parameter without an explicit clone. +impl From<&ModelId> for ModelId { + fn from(id: &ModelId) -> Self { + id.clone() + } +} + +impl From for String { + fn from(id: ModelId) -> Self { + id.0 + } +} + +impl From<&ModelId> for String { + fn from(id: &ModelId) -> Self { + id.0.clone() + } +} + +// Comparison against bare strings, in both directions, so an id can be checked +// against a literal or a configured name without wrapping either side. +// This is likely excessive. We can reduce when things settle. + +impl PartialEq for ModelId { + fn eq(&self, other: &str) -> bool { + self.0 == other + } +} + +impl PartialEq<&str> for ModelId { + fn eq(&self, other: &&str) -> bool { + self.0 == *other + } +} + +impl PartialEq for ModelId { + fn eq(&self, other: &String) -> bool { + &self.0 == other + } +} + +impl PartialEq for str { + fn eq(&self, other: &ModelId) -> bool { + self == other.0 + } +} + +impl PartialEq for &str { + fn eq(&self, other: &ModelId) -> bool { + *self == other.0 + } +} + +impl PartialEq for String { + fn eq(&self, other: &ModelId) -> bool { + self == &other.0 + } +} + +#[cfg(test)] +mod tests { + use std::collections::HashMap; + + use super::*; + + #[test] + fn it_behaves_like_the_string_it_wraps() { + let id = ModelId::new("openai/gpt-oss-20b"); + + assert_eq!(id.to_string(), "openai/gpt-oss-20b"); + assert_eq!(id, "openai/gpt-oss-20b"); + assert_eq!("openai/gpt-oss-20b", id); + assert!(id.starts_with("openai/")); + assert_eq!(takes_str(&id), "openai/gpt-oss-20b"); + } + + /// Deref coercion means an `&ModelId` reaches a `&str` parameter unchanged. + fn takes_str(model: &str) -> &str { + model + } + + #[test] + fn a_map_of_ids_is_queryable_by_str() { + let by_model = HashMap::from([(ModelId::new("aws/anthropic/claude-opus-4-5"), 1)]); + + assert_eq!(by_model.get("aws/anthropic/claude-opus-4-5"), Some(&1)); + } + + #[test] + fn it_serializes_as_a_bare_string() -> serde_json::Result<()> { + let id = ModelId::new("openai/gpt-oss-20b"); + + let json = serde_json::to_string(&id)?; + assert_eq!(json, "\"openai/gpt-oss-20b\""); + assert_eq!(serde_json::from_str::(&json)?, id); + Ok(()) + } +} diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index 8be1bede..2bed6b40 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -13,12 +13,12 @@ use pyo3::prelude::*; use serde_json::{Value, json}; use switchyard_libsy::{ Algorithm, ClassifierContractConfig, HandoffNoteConfig, LibsyError as RustLibsyError, - LlmClassifierConfig, LlmFallback, LlmTarget, LlmTargetSet, LlmTaskClassifier, Noop, PickerMode, - Random, StageRouter, StageRouterConfig, TaskClassifierConfig, + LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, StageRouter, + StageRouterConfig, TaskClassifierConfig, }; use switchyard_llm_client::ClientRouter; use switchyard_protocol::{ - AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, + AggLlmResponse, Decision, LlmClientError, LlmResponse, Metadata, ModelId, Request, Response, RoutedLlmClient, }; @@ -78,10 +78,9 @@ struct PyLlmTarget { } impl PyLlmTarget { - fn clone_core(&self, _py: Python<'_>) -> LlmTarget { - LlmTarget { - semantic_name: self.name.clone(), - } + /// The bare model id libsy routes by; the client behind it stays with the bindings. + fn clone_core(&self, _py: Python<'_>) -> ModelId { + ModelId::new(self.name.clone()) } /// The `selected_model -> client` entry this target contributes to the algorithm's @@ -89,7 +88,7 @@ impl PyLlmTarget { /// mapping and serve the calls themselves. fn client_entry(&self, py: Python<'_>) -> ClientEntry { ( - self.name.clone(), + ModelId::new(self.name.clone()), Arc::new(PythonLlmClient { inner: self.client.clone_ref(py), }), @@ -229,7 +228,7 @@ struct PyAlgorithm { } /// One target's `selected_model -> client` mapping for an algorithm's router. -type ClientEntry = (String, Arc); +type ClientEntry = (ModelId, Arc); impl PyAlgorithm { fn new(inner: Arc, clients: impl IntoIterator) -> Self { @@ -310,13 +309,10 @@ fn random_algorithm( seed: Option, ) -> PyResult { let (cores, clients) = target_cores(py, &targets)?; - let algorithm = - Random::new(LlmTargetSet::new(cores), weights, seed).map_err(|error| match error { - RustLibsyError::NoTargets => { - PyValueError::new_err("random requires at least one target") - } - other => PyValueError::new_err(other.to_string()), - })?; + let algorithm = Random::new(cores, weights, seed).map_err(|error| match error { + RustLibsyError::NoTargets => PyValueError::new_err("random requires at least one target"), + other => PyValueError::new_err(other.to_string()), + })?; Ok(PyAlgorithm::new(Arc::new(algorithm), clients)) } @@ -325,7 +321,7 @@ fn random_algorithm( fn target_cores( py: Python<'_>, targets: &[Py], -) -> PyResult<(Vec, Vec)> { +) -> PyResult<(Vec, Vec)> { let mut cores = Vec::with_capacity(targets.len()); let mut clients = Vec::with_capacity(targets.len()); for target in targets { @@ -429,14 +425,10 @@ fn stage_router_algorithm( (None, None) => None, }; if let Some(prompt) = capable_system_prompt { - config.tier_prompts = config - .tier_prompts - .with(capable.semantic_name.clone(), prompt); + config.tier_prompts = config.tier_prompts.with(capable.clone(), prompt); } if let Some(prompt) = efficient_system_prompt { - config.tier_prompts = config - .tier_prompts - .with(efficient.semantic_name.clone(), prompt); + config.tier_prompts = config.tier_prompts.with(efficient.clone(), prompt); } // The judge is only reachable through the optional classifier fallback, so its client // joins the router only when a fallback is configured. diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index f7be8c64..83b4f3d9 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -10,9 +10,9 @@ use std::sync::Arc; use libsy::{ Algorithm, ClassifierContractConfig, CustomClassifierConfig, CustomClassifierPolicy, - EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTarget, - LlmTargetSet, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, - StageRouterConfig, TargetPrompts, TaskClassifierConfig, + EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, + Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts, + TaskClassifierConfig, }; use serde::Deserialize; use serde_json::Value; @@ -20,7 +20,7 @@ use switchyard_llm_client::{ Backend, ClientRouter, DEFAULT_MAX_RETRIES, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; -use switchyard_protocol::RoutedLlmClient; +use switchyard_protocol::{ModelId, RoutedLlmClient}; use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState}; @@ -102,7 +102,7 @@ impl ServerConfig { let client = self.build_client_router(config, &clients)?; let count_tokens_target = self.build_count_tokens_target(config, &clients); routes.push(( - config.id().to_string(), + config.id().clone(), algorithm, client, capabilities, @@ -133,7 +133,7 @@ impl ServerConfig { .get_mut(&target.llm_client) .ok_or_else(|| ServerError::new("validated llm client was not initialized"))?; model_configs.push(ModelConfig::new( - &target.id, + target.id.clone(), build_backend(&target.llm_client, client_config, &target.extra_body)?, None, )); @@ -150,18 +150,11 @@ impl ServerConfig { Ok(clients) } - fn build_targets(&self) -> ServerResult> { + fn build_targets(&self) -> ServerResult> { Ok(self .targets .iter() - .map(|(name, config)| { - ( - name.clone(), - LlmTarget { - semantic_name: config.id.clone(), - }, - ) - }) + .map(|(name, config)| (name.clone(), config.id.clone())) .collect()) } @@ -225,7 +218,7 @@ impl ServerConfig { } // Prefer known Claude families, then preserve the route's target order. -fn count_tokens_priority(target_name: &str, model_id: &str) -> usize { +fn count_tokens_priority(target_name: &str, model_id: &ModelId) -> usize { let target_name = target_name.to_ascii_lowercase(); let model_id = model_id.to_ascii_lowercase(); ["opus", "sonnet", "haiku"] @@ -249,7 +242,7 @@ struct LlmClientConfig { #[derive(Debug, Deserialize)] #[serde(deny_unknown_fields)] struct TargetConfig { - id: String, + id: ModelId, llm_client: String, #[serde(default)] extra_body: BTreeMap, @@ -333,7 +326,7 @@ struct CustomClassifierRouteConfig { #[serde(tag = "type", rename_all = "snake_case", deny_unknown_fields)] enum RouteConfig { Noop { - id: String, + id: ModelId, #[serde(default)] context_window: Option, #[serde(default)] @@ -342,7 +335,7 @@ enum RouteConfig { reasoning: Option, }, Random { - id: String, + id: ModelId, #[serde(default)] context_window: Option, #[serde(default)] @@ -354,7 +347,7 @@ enum RouteConfig { seed: Option, }, Passthrough { - id: String, + id: ModelId, #[serde(default)] context_window: Option, #[serde(default)] @@ -364,7 +357,7 @@ enum RouteConfig { target: String, }, LlmClassifier { - id: String, + id: ModelId, #[serde(default)] context_window: Option, #[serde(default)] @@ -404,7 +397,7 @@ enum RouteConfig { policy: Option, }, StageRouter { - id: String, + id: ModelId, #[serde(default)] context_window: Option, #[serde(default)] @@ -469,7 +462,7 @@ impl StageClassifierConfig { } impl RouteConfig { - fn id(&self) -> &str { + fn id(&self) -> &ModelId { use RouteConfig::*; match self { Noop { id, .. } @@ -821,7 +814,7 @@ const fn default_max_retries() -> u32 { fn build_algorithm( route_name: &str, config: &RouteConfig, - targets: &BTreeMap, + targets: &BTreeMap, ) -> ServerResult> { match config { RouteConfig::Noop { .. } => Ok(Arc::new(Noop {})), @@ -838,18 +831,19 @@ fn build_algorithm( Ok(Arc::new(algorithm)) } RouteConfig::Passthrough { target, .. } => { - let target = resolve_target(route_name, target, targets)?; + let target = resolve_target_model_id(route_name, target, targets)?; Ok(Arc::new(Passthrough::new(target))) } RouteConfig::LlmClassifier { classifier_target, .. } => { - let classifier = resolve_target(route_name, classifier_target, targets)?; + let classifier = resolve_target_model_id(route_name, classifier_target, targets)?; let mode = config.classifier_mode(route_name)?; let algorithm = match mode { LlmClassifierModeConfig::Capability(config) => { - let strong = resolve_target(route_name, &config.strong_target, targets)?; - let weak = resolve_target(route_name, &config.weak_target, targets)?; + let strong = + resolve_target_model_id(route_name, &config.strong_target, targets)?; + let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; let classifier_config = TaskClassifierConfig { base_threshold: config.base_threshold, threshold_step: config.threshold_step, @@ -867,8 +861,9 @@ fn build_algorithm( }) } LlmClassifierModeConfig::Escalation(config) => { - let strong = resolve_target(route_name, &config.strong_target, targets)?; - let weak = resolve_target(route_name, &config.weak_target, targets)?; + let strong = + resolve_target_model_id(route_name, &config.strong_target, targets)?; + let weak = resolve_target_model_id(route_name, &config.weak_target, targets)?; LlmTaskClassifier::new(LlmClassifierConfig::Escalation { judge_target: classifier, efficient_target: weak, @@ -883,7 +878,7 @@ fn build_algorithm( .targets .iter() .map(|name| { - resolve_target(route_name, name, targets) + resolve_target_model_id(route_name, name, targets) .map(|target| (name.clone(), target)) }) .collect::>>()?; @@ -933,15 +928,15 @@ fn build_algorithm( "stage_router route {route_name} uses picker \"capable_first\", which is experimental: published thresholds and routing results all come from \"efficient_first\", so there is no calibrated confidence_threshold for it and no measured accuracy or cost. Use \"efficient_first\" unless you are running your own calibration." ); } - let capable = resolve_target(route_name, capable_target, targets)?; - let efficient = resolve_target(route_name, efficient_target, targets)?; + let capable = resolve_target_model_id(route_name, capable_target, targets)?; + let efficient = resolve_target_model_id(route_name, efficient_target, targets)?; let mut config = StageRouterConfig::new(*picker, *confidence_threshold); config.recent_window = *recent_turn_window; config.handoff_notes = handoff_notes.clone(); config.tier_prompts = tier_prompts( - &capable.semantic_name, + &capable, capable_system_prompt.as_deref(), - &efficient.semantic_name, + &efficient, efficient_system_prompt.as_deref(), ); // The judge is called through its own target, so it is not a routing @@ -949,12 +944,12 @@ fn build_algorithm( config.llm_fallback = classifier .as_ref() .map(|classifier| { - resolve_target(route_name, &classifier.target, targets).map(|judge_target| { - LlmFallback { + resolve_target_model_id(route_name, &classifier.target, targets).map( + |judge_target| LlmFallback { judge_target, config: classifier.task_classifier_config(), - } - }) + }, + ) }) .transpose()?; let algorithm = StageRouter::new(capable, efficient, config).map_err(|error| { @@ -995,20 +990,19 @@ fn tier_prompts( fn resolve_targets<'a>( route_name: &str, names: impl IntoIterator, - targets: &BTreeMap, -) -> ServerResult { - let resolved = names + targets: &BTreeMap, +) -> ServerResult> { + names .into_iter() - .map(|name| resolve_target(route_name, name, targets)) - .collect::>>()?; - Ok(LlmTargetSet::new(resolved)) + .map(|name| resolve_target_model_id(route_name, name, targets)) + .collect() } -fn resolve_target( +fn resolve_target_model_id( route_name: &str, name: &str, - targets: &BTreeMap, -) -> ServerResult { + targets: &BTreeMap, +) -> ServerResult { targets.get(name).cloned().ok_or_else(|| { ServerError::new(format!( "route {route_name} references unknown target {name}" diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 7ad32d81..9c538ea0 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -37,7 +37,7 @@ use parking_lot::Mutex; use serde::Deserialize; use serde_json::{Value, json}; use switchyard_llm_client::{ClientRouter, RunObservation, RunObserver, TranslatingLlmClient}; -use switchyard_protocol::{Decision, LlmClientError, Metadata, Request, Usage}; +use switchyard_protocol::{Decision, LlmClientError, Metadata, ModelId, Request, Usage}; use tokio::net::{TcpListener, TcpSocket}; use tokio::task; use tracing::{Instrument, Level}; @@ -117,7 +117,7 @@ struct RouteEntry { /// Exact upstream model used by the server's Anthropic token-count endpoint. #[derive(Clone)] struct CountTokensTarget { - model: String, + model: ModelId, client: Arc, } @@ -130,7 +130,7 @@ impl CountTokensTarget { /// Shared server state used by all endpoint handlers. #[derive(Clone)] pub struct ServerState { - routes: Arc>, + routes: Arc>, metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, @@ -175,7 +175,7 @@ impl ServerState { /// Creates server state from route model IDs, their libsy algorithms, and the /// per-target client routing each route's calls are resolved through. pub fn new( - routes: impl IntoIterator, ClientRouter)>, + routes: impl IntoIterator, ClientRouter)>, ) -> ServerResult { Self::new_with_capabilities(routes.into_iter().map(|(model, algorithm, clients)| { ( @@ -191,7 +191,7 @@ impl ServerState { fn new_with_capabilities( routes: impl IntoIterator< Item = ( - String, + ModelId, Arc, ClientRouter, ModelCapabilities, @@ -201,7 +201,7 @@ impl ServerState { ) -> ServerResult { let mut entries = BTreeMap::new(); for (model, algorithm, target_clients, capabilities, count_tokens_target) in routes { - let model = model.trim(); + let model = ModelId::from(model.trim()); if model.is_empty() { return Err(ServerError::new("route model must not be empty")); } @@ -211,7 +211,7 @@ impl ServerState { capabilities, count_tokens_target, }; - if entries.insert(model.to_string(), entry).is_some() { + if entries.insert(model.clone(), entry).is_some() { return Err(ServerError::new(format!("duplicate route model {model}"))); } } @@ -240,7 +240,7 @@ impl ServerState { /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { - self.routes.keys().map(String::as_str) + self.routes.keys().map(ModelId::as_str) } fn route_for_model(&self, model: &str) -> Option<&RouteEntry> { @@ -1327,7 +1327,7 @@ mod tests { let call = |model: &str, is_answer_call: bool| { RunObservation::LlmCall(LlmCallObservation { - selected_model: model.to_string(), + selected_model: ModelId::from(model), is_answer_call, is_success: true, duration: Duration::from_millis(3), diff --git a/crates/switchyard-server/src/routing_log.rs b/crates/switchyard-server/src/routing_log.rs index 9060887e..8d44f46f 100644 --- a/crates/switchyard-server/src/routing_log.rs +++ b/crates/switchyard-server/src/routing_log.rs @@ -12,7 +12,7 @@ use std::time::SystemTime; use humantime::format_rfc3339_millis; use serde::{Deserialize, Serialize}; -use switchyard_protocol::Usage; +use switchyard_protocol::{ModelId, Usage}; use crate::usage_metrics::token_usage; use crate::{ServerError, ServerResult}; @@ -145,7 +145,7 @@ pub(crate) struct SessionStatsSnapshot { total_cached_tokens: u64, total_cache_creation_tokens: u64, total_completion_tokens: u64, - models: BTreeMap, + models: BTreeMap, } #[derive(Default, Serialize)] @@ -178,7 +178,7 @@ impl SessionStatsSnapshot { "" => "unknown", model => model, }; - let stats = self.models.entry(model.to_string()).or_default(); + let stats = self.models.entry(ModelId::from(model)).or_default(); stats.calls = stats.calls.saturating_add(1); stats.prompt_tokens = stats.prompt_tokens.saturating_add(record.prompt_tokens); stats.cached_tokens = stats.cached_tokens.saturating_add(record.cached_tokens); diff --git a/crates/switchyard-server/src/stats/accumulator.rs b/crates/switchyard-server/src/stats/accumulator.rs index 56d2df55..7ddfb06e 100644 --- a/crates/switchyard-server/src/stats/accumulator.rs +++ b/crates/switchyard-server/src/stats/accumulator.rs @@ -12,6 +12,7 @@ use serde::Serialize; use super::algorithms::{AlgorithmStats, AlgorithmStatsSnapshot}; use super::cache_eligibility::PrefixProbe; +use switchyard_protocol::ModelId; const MAX_LATENCY_SAMPLES: usize = 10_000; @@ -50,7 +51,7 @@ impl StatsAccumulator { } /// Records one successful routed backend call. - pub(crate) fn record_success(&self, model: impl Into, backend_latency_ms: f64) { + pub(crate) fn record_success(&self, model: impl Into, backend_latency_ms: f64) { let mut inner = self.lock(); inner.total_requests = inner.total_requests.saturating_add(1); let stats = inner.model_stats_mut(model.into()); @@ -59,7 +60,7 @@ impl StatsAccumulator { } /// Records one failed routed backend call. - pub(crate) fn record_error(&self, model: impl Into) { + pub(crate) fn record_error(&self, model: impl Into) { let mut inner = self.lock(); inner.total_requests = inner.total_requests.saturating_add(1); inner.total_errors = inner.total_errors.saturating_add(1); @@ -68,7 +69,7 @@ impl StatsAccumulator { } /// Records a stream failure after its routed call was already counted. - pub(crate) fn record_stream_error(&self, model: impl Into) { + pub(crate) fn record_stream_error(&self, model: impl Into) { let mut inner = self.lock(); inner.total_errors = inner.total_errors.saturating_add(1); let stats = inner.model_stats_mut(model.into()); @@ -78,7 +79,7 @@ impl StatsAccumulator { /// Records usage and terminal latency after a successful routed call. pub(crate) fn record_usage( &self, - model: impl Into, + model: impl Into, usage: TokenUsage, total_latency_ms: f64, ) { @@ -96,7 +97,7 @@ impl StatsAccumulator { /// Records one successful classifier or judge call. pub(crate) fn record_classifier_success( &self, - model: impl Into, + model: impl Into, usage: Option, latency_ms: f64, ) { @@ -112,7 +113,7 @@ impl StatsAccumulator { } /// Records one failed classifier or judge call. - pub(crate) fn record_classifier_error(&self, model: impl Into) { + pub(crate) fn record_classifier_error(&self, model: impl Into) { let mut inner = self.lock(); inner.classifier_requests = inner.classifier_requests.saturating_add(1); inner.classifier_errors = inner.classifier_errors.saturating_add(1); @@ -121,9 +122,9 @@ impl StatsAccumulator { } /// Returns the cache-eligible fraction for `model` and records the prefix as seen. - pub(crate) fn prefix_eligibility(&self, model: &str, probe: &PrefixProbe) -> f64 { + pub(crate) fn prefix_eligibility(&self, model: &ModelId, probe: &PrefixProbe) -> f64 { let mut inner = self.lock(); - let stats = inner.model_stats_mut(model.to_string()); + let stats = inner.model_stats_mut(model.clone()); let fraction = probe.eligible_fraction(&stats.seen_prefixes); if let Some(hash) = probe.full_hash() { stats.seen_prefixes.insert(hash); @@ -147,12 +148,12 @@ impl StatsAccumulator { } struct StatsAccumulatorInner { - by_model: BTreeMap, + by_model: BTreeMap, total_requests: u64, total_errors: u64, routing_overhead: LatencyHistogram, routing_fallbacks: RoutingFallbackStats, - by_classifier: BTreeMap, + by_classifier: BTreeMap, classifier_requests: u64, classifier_errors: u64, algorithm_stats: AlgorithmStats, @@ -173,11 +174,11 @@ impl StatsAccumulatorInner { } } - fn model_stats_mut(&mut self, model: String) -> &mut ModelStats { + fn model_stats_mut(&mut self, model: ModelId) -> &mut ModelStats { self.by_model.entry(model).or_default() } - fn classifier_stats_mut(&mut self, model: String) -> &mut ModelStats { + fn classifier_stats_mut(&mut self, model: ModelId) -> &mut ModelStats { self.by_classifier.entry(model).or_default() } @@ -315,7 +316,7 @@ pub(crate) struct StatsSnapshot { pub total_requests: u64, pub total_errors: u64, pub total_tokens: TokenTotals, - pub models: BTreeMap, + pub models: BTreeMap, pub routing_overhead: LatencyHistogramSnapshot, pub routing_fallbacks: RoutingFallbackStats, pub classifier: ClassifierStatsSnapshot, @@ -336,7 +337,7 @@ pub(crate) struct ClassifierStatsSnapshot { pub total_requests: u64, pub total_errors: u64, pub total_tokens: TokenTotals, - pub models: BTreeMap, + pub models: BTreeMap, } #[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize)] @@ -382,9 +383,9 @@ pub(crate) struct LatencyHistogramSnapshot { } fn build_model_snapshots( - by_model: &BTreeMap, + by_model: &BTreeMap, total_requests: u64, -) -> (BTreeMap, TokenTotals) { +) -> (BTreeMap, TokenTotals) { let mut totals = TokenTotals::default(); for stats in by_model.values() { totals.prompt = totals.prompt.saturating_add(stats.prompt_tokens); @@ -430,7 +431,7 @@ fn build_model_snapshots( } fn build_classifier_snapshot( - models: &BTreeMap, + models: &BTreeMap, total_requests: u64, total_errors: u64, ) -> ClassifierStatsSnapshot { @@ -536,12 +537,15 @@ mod tests { let probe = prefix_probe(&json!({ "messages": [{"role": "user", "content": "repeat me"}], })); - stats.prefix_eligibility("model/a", &probe); + stats.prefix_eligibility(&ModelId::from("model/a"), &probe); stats.reset(); assert_eq!(stats.snapshot(), StatsSnapshot::default()); - assert_eq!(stats.prefix_eligibility("model/a", &probe), 0.0); + assert_eq!( + stats.prefix_eligibility(&ModelId::from("model/a"), &probe), + 0.0 + ); } #[test] @@ -550,7 +554,7 @@ mod tests { let first = prefix_probe(&json!({ "messages": [{"role": "user", "content": "aaaa"}], })); - let first_eligible = stats.prefix_eligibility("model/a", &first); + let first_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &first); stats.record_usage( "model/a", TokenUsage { @@ -566,7 +570,7 @@ mod tests { {"role": "user", "content": "bbbb"}, ], })); - let second_eligible = stats.prefix_eligibility("model/a", &second); + let second_eligible = stats.prefix_eligibility(&ModelId::from("model/a"), &second); stats.record_usage( "model/a", TokenUsage { @@ -582,6 +586,9 @@ mod tests { stats.snapshot().models["model/a"].theoretical_cache_hit_rate, 0.25 ); - assert_eq!(stats.prefix_eligibility("model/b", &second), 0.0); + assert_eq!( + stats.prefix_eligibility(&ModelId::from("model/b"), &second), + 0.0 + ); } } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 65bd5ab7..a1241c59 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -17,11 +17,12 @@ use axum::response::{IntoResponse, Response as HttpResponse}; use axum::routing::post; use axum::{Json, Router}; use http_body_util::BodyExt; -use libsy::{Algorithm, LlmTarget, LlmTargetSet, Random}; +use libsy::{Algorithm, Random}; use serde_json::{Value, json}; use switchyard_llm_client::{ Backend, ClientRouter, HttpBackendConfig, ModelConfig, TranslatingLlmClient, }; +use switchyard_protocol::ModelId; use switchyard_protocol::RoutedLlmClient; use switchyard_server::config::load_server_state; use switchyard_server::{ServerState, build_switchyard_router}; @@ -227,17 +228,10 @@ fn random_state(base_url: &str, routes: &[(&str, &[&str])]) -> TestResult = Arc::new(Random::new(target_set, None, None)?); Ok(( - (*route_model).to_string(), + ModelId::from(*route_model), algorithm, ClientRouter::single(Arc::clone(&client)), ))