From 123378435217d29b1ad5041a67d4093c798587e6 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 12 Aug 2026 11:57:36 +1000 Subject: [PATCH 1/4] feat(server): opt-in cost savings endpoint and live dashboard Adds per-target pricing (USD per 1M tokens) to the server TOML and, when any target is priced, registers GET /v1/savings (JSON) and GET /dashboard (self-contained live HTML page). Savings compare actual routed spend against a baseline model - what the same traffic would have cost if every request had been served by the most capable target. Classifier calls are counted as routing overhead against the savings. Pricing semantics (base input / cache read / cache write / output buckets) match switchyard.cli.launchers.cost_estimator. Fully opt-in: no pricing in config means no new routes and no behaviour change. Signed-off-by: Michael Neale --- crates/switchyard-server/src/config.rs | 91 +++++- crates/switchyard-server/src/lib.rs | 34 ++ crates/switchyard-server/src/savings.rs | 302 ++++++++++++++++++ .../src/savings_dashboard.html | 205 ++++++++++++ 4 files changed, 630 insertions(+), 2 deletions(-) create mode 100644 crates/switchyard-server/src/savings.rs create mode 100644 crates/switchyard-server/src/savings_dashboard.html diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index f7be8c64..9c810e21 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -22,7 +22,10 @@ use switchyard_llm_client::{ }; use switchyard_protocol::RoutedLlmClient; -use crate::{CountTokensTarget, ModelCapabilities, ServerError, ServerResult, ServerState}; +use crate::{ + CountTokensTarget, ModelCapabilities, ModelPrice, SavingsConfig, ServerError, ServerResult, + ServerState, +}; const SUPPORTED_SCHEMA_VERSION: u32 = 1; const MAX_CONFIGURED_RETRIES: u32 = 10; @@ -55,6 +58,8 @@ struct ServerConfig { llm_clients: BTreeMap, targets: BTreeMap, routes: BTreeMap, + #[serde(default)] + savings: Option, } impl ServerConfig { @@ -109,7 +114,50 @@ impl ServerConfig { count_tokens_target, )); } - ServerState::new_with_capabilities(routes) + let state = ServerState::new_with_capabilities(routes)?; + self.apply_savings(state) + } + + /// Enables the savings endpoint when any target declares pricing. + /// + /// Stats are keyed by the model id the routed call selected, so the + /// pricing table is keyed by `target.id`, not the TOML target name. + fn apply_savings(&self, state: ServerState) -> ServerResult { + let mut pricing = BTreeMap::new(); + for target in self.targets.values() { + if let Some(config) = target.pricing { + pricing.insert(target.id.clone(), config.into_model_price()); + } + } + let baseline = match self + .savings + .as_ref() + .and_then(|s| s.baseline_target.as_ref()) + { + Some(name) => { + let target = self.targets.get(name).ok_or_else(|| { + ServerError::new(format!( + "savings baseline_target {name} does not match any [targets.*] entry" + )) + })?; + if target.pricing.is_none() { + return Err(ServerError::new(format!( + "savings baseline_target {name} has no pricing configured" + ))); + } + Some(target.id.clone()) + } + None => None, + }; + if pricing.is_empty() { + if self.savings.is_some() { + return Err(ServerError::new( + "[savings] requires at least one target with a [targets.*.pricing] section", + )); + } + return Ok(state); + } + Ok(state.with_savings(SavingsConfig::new(pricing, baseline))) } fn build_clients(&self) -> ServerResult>> { @@ -253,6 +301,45 @@ struct TargetConfig { llm_client: String, #[serde(default)] extra_body: BTreeMap, + #[serde(default)] + pricing: Option, +} + +/// Per-target pricing in USD per 1 million tokens. +/// +/// `cached` defaults to the cache-read discount being absent (0.0 means +/// cache reads are free only if explicitly priced so); `cache_write` +/// defaults to the base input rate when omitted, matching providers with +/// no cache-write premium. +#[derive(Clone, Copy, Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct PricingConfig { + input: f64, + output: f64, + #[serde(default)] + cached: Option, + #[serde(default)] + cache_write: Option, +} + +impl PricingConfig { + fn into_model_price(self) -> ModelPrice { + ModelPrice { + input: self.input, + output: self.output, + cached: self.cached.unwrap_or(self.input * 0.1), + cache_write: self.cache_write.unwrap_or(self.input), + } + } +} + +/// Optional `[savings]` section selecting the baseline comparison target. +#[derive(Debug, Default, Deserialize)] +#[serde(deny_unknown_fields)] +struct SavingsSectionConfig { + /// Target name (TOML key under `[targets.*]`) to price the baseline + /// against. Defaults to the most expensive priced target. + baseline_target: Option, } #[derive(Clone, Copy, Debug, Deserialize)] diff --git a/crates/switchyard-server/src/lib.rs b/crates/switchyard-server/src/lib.rs index 7ad32d81..47ad11fa 100644 --- a/crates/switchyard-server/src/lib.rs +++ b/crates/switchyard-server/src/lib.rs @@ -8,6 +8,7 @@ mod metrics; mod observability; mod response; mod routing_log; +mod savings; mod shutdown; mod sse; mod stats; @@ -45,6 +46,8 @@ use tracing::{Instrument, Level}; use switchyard_translation::{WireFormat, decode_request}; use crate::response::into_http_response; +use crate::savings::SavingsSnapshot; +pub use crate::savings::{ModelPrice, SavingsConfig}; use crate::stats::{StatsAccumulator, StatsSnapshot, prefix_probe, tracking_enabled_from_env}; pub use observability::{flush_observability, initialize_observability}; @@ -134,6 +137,7 @@ pub struct ServerState { metrics: prometheus::Registry, stats: StatsAccumulator, routing_log: Option, + savings: Option>, track_cache_eligibility: bool, } @@ -228,6 +232,7 @@ impl ServerState { metrics, stats, routing_log: None, + savings: None, track_cache_eligibility: tracking_enabled_from_env(), }) } @@ -238,6 +243,12 @@ impl ServerState { Ok(self) } + /// Enables the cost-savings endpoint and live dashboard. + pub fn with_savings(mut self, config: SavingsConfig) -> Self { + self.savings = Some(Arc::new(config)); + self + } + /// Returns the route model IDs served by the configured algorithms. pub fn models(&self) -> impl Iterator { self.routes.keys().map(String::as_str) @@ -471,6 +482,11 @@ pub fn build_switchyard_router(state: ServerState) -> Router { if state.routing_log.is_some() { router = router.route("/v1/routing/session-stats", get(get_session_stats)); } + if state.savings.is_some() { + router = router + .route("/v1/savings", get(get_savings)) + .route("/dashboard", get(savings_dashboard)); + } router .fallback(not_found) .layer(DefaultBodyLimit::max(DEFAULT_MAX_REQUEST_BODY_BYTES)) @@ -1023,6 +1039,24 @@ async fn get_stats(State(state): State) -> Json { Json(state.stats.snapshot()) } +async fn get_savings(State(state): State) -> Response { + let Some(savings) = &state.savings else { + // Unreachable: the route is only registered when savings is configured. + return not_found().await; + }; + let snapshot: SavingsSnapshot = savings.compute(&state.stats.snapshot()); + (StatusCode::OK, Json(snapshot)).into_response() +} + +/// Serves the self-contained live savings dashboard page. +async fn savings_dashboard() -> Response { + ( + [(CONTENT_TYPE, "text/html; charset=utf-8")], + include_str!("savings_dashboard.html"), + ) + .into_response() +} + async fn reset_stats(State(state): State) -> Json { state.stats.reset(); Json(json!({"status": "reset"})) diff --git a/crates/switchyard-server/src/savings.rs b/crates/switchyard-server/src/savings.rs new file mode 100644 index 00000000..04f09cf7 --- /dev/null +++ b/crates/switchyard-server/src/savings.rs @@ -0,0 +1,302 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Cost and savings accounting derived from the stats snapshot. +//! +//! Pricing is configured per model in the deployment TOML (USD per 1M +//! tokens). Savings compare the actual routed spend against a baseline: +//! what the same traffic would have cost if every request had been served +//! by the baseline (typically the most capable / expensive) model. + +use std::collections::BTreeMap; + +use serde::Serialize; + +use crate::stats::StatsSnapshot; + +/// Per-model pricing in USD per 1 million tokens. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct ModelPrice { + /// Base input price (fresh prompt tokens). + pub input: f64, + /// Output / completion price. + pub output: f64, + /// Cache-read price (prompt cache hit). + pub cached: f64, + /// Cache-write price (prompt cache creation). + pub cache_write: f64, +} + +/// Savings configuration: a pricing table plus an optional explicit baseline. +#[derive(Clone, Debug, Default)] +pub struct SavingsConfig { + pricing: BTreeMap, + baseline: Option, +} + +impl SavingsConfig { + /// Creates a savings config from per-model prices and an optional + /// baseline model. When `baseline` is `None`, the most expensive + /// priced model (by input + output rate) is used. + pub fn new(pricing: BTreeMap, baseline: Option) -> Self { + Self { pricing, baseline } + } + + /// True when no pricing has been configured at all. + pub fn is_empty(&self) -> bool { + self.pricing.is_empty() + } + + fn price_for(&self, model: &str) -> Option { + self.pricing.get(model).copied() + } + + fn baseline_model(&self) -> Option<(&str, ModelPrice)> { + if let Some(name) = &self.baseline { + return self.price_for(name).map(|price| (name.as_str(), price)); + } + self.pricing + .iter() + .max_by(|a, b| { + let ka = a.1.input + a.1.output; + let kb = b.1.input + b.1.output; + ka.partial_cmp(&kb).unwrap_or(std::cmp::Ordering::Equal) + }) + .map(|(name, price)| (name.as_str(), *price)) + } + + /// Computes a savings snapshot from the current stats snapshot. + pub(crate) fn compute(&self, stats: &StatsSnapshot) -> SavingsSnapshot { + let baseline = self.baseline_model(); + let mut models = BTreeMap::new(); + let mut actual_cost = 0.0; + let mut baseline_cost = 0.0; + let mut unpriced_models = Vec::new(); + + for (model, m) in &stats.models { + let tokens = TokenBuckets { + prompt: m.prompt_tokens, + completion: m.completion_tokens, + cached: m.cached_tokens, + cache_creation: m.cache_creation_tokens, + }; + let price = self.price_for(model); + if price.is_none() { + unpriced_models.push(model.clone()); + } + let cost = price.map(|p| tokens.cost(p)).unwrap_or(0.0); + let would_be = baseline.map(|(_, p)| tokens.cost(p)).unwrap_or(0.0); + actual_cost += cost; + baseline_cost += would_be; + models.insert( + model.clone(), + ModelSavings { + calls: m.calls, + prompt_tokens: m.prompt_tokens, + completion_tokens: m.completion_tokens, + cached_tokens: m.cached_tokens, + cost, + baseline_cost: would_be, + priced: price.is_some(), + }, + ); + } + + // Classifier / judge calls are pure routing overhead: they add to the + // actual spend but a baseline deployment would not make them at all. + let mut classifier_cost = 0.0; + for (model, m) in &stats.classifier.models { + let tokens = TokenBuckets { + prompt: m.prompt_tokens, + completion: m.completion_tokens, + cached: m.cached_tokens, + cache_creation: m.cache_creation_tokens, + }; + if let Some(price) = self.price_for(model) { + classifier_cost += tokens.cost(price); + } + } + actual_cost += classifier_cost; + + let saved = baseline_cost - actual_cost; + let saved_pct = if baseline_cost > 0.0 { + (saved / baseline_cost) * 100.0 + } else { + 0.0 + }; + + SavingsSnapshot { + total_requests: stats.total_requests, + actual_cost: round6(actual_cost), + baseline_cost: round6(baseline_cost), + classifier_cost: round6(classifier_cost), + saved: round6(saved), + saved_pct: round2(saved_pct), + baseline_model: baseline.map(|(name, _)| name.to_string()), + models, + unpriced_models, + } + } +} + +/// Token counters priced by [`TokenBuckets::cost`]. +#[derive(Clone, Copy, Debug, Default)] +struct TokenBuckets { + prompt: u64, + completion: u64, + cached: u64, + cache_creation: u64, +} + +impl TokenBuckets { + /// Splits prompt tokens into base / cached / cache-write buckets and + /// prices each, matching the Python `cost_estimator` semantics. + fn cost(&self, price: ModelPrice) -> f64 { + let base_input = self + .prompt + .saturating_sub(self.cached) + .saturating_sub(self.cache_creation); + (base_input as f64 / 1e6) * price.input + + (self.cached as f64 / 1e6) * price.cached + + (self.cache_creation as f64 / 1e6) * price.cache_write + + (self.completion as f64 / 1e6) * price.output + } +} + +/// Serialized savings response returned by `GET /v1/savings`. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct SavingsSnapshot { + pub total_requests: u64, + /// USD actually spent (routed calls + classifier overhead). + pub actual_cost: f64, + /// USD the same routed traffic would have cost on the baseline model. + pub baseline_cost: f64, + /// USD spent on classifier / judge routing calls. + pub classifier_cost: f64, + /// `baseline_cost - actual_cost`. + pub saved: f64, + /// Percentage of baseline cost saved. + pub saved_pct: f64, + /// Model the baseline comparison is priced against. + pub baseline_model: Option, + pub models: BTreeMap, + /// Models that served traffic but have no configured price (costed at 0). + pub unpriced_models: Vec, +} + +/// Per-model cost breakdown inside [`SavingsSnapshot`]. +#[derive(Clone, Debug, Serialize)] +pub(crate) struct ModelSavings { + pub calls: u64, + pub prompt_tokens: u64, + pub completion_tokens: u64, + pub cached_tokens: u64, + pub cost: f64, + pub baseline_cost: f64, + pub priced: bool, +} + +fn round6(v: f64) -> f64 { + (v * 1e6).round() / 1e6 +} + +fn round2(v: f64) -> f64 { + (v * 1e2).round() / 1e2 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::stats::StatsAccumulator; + use crate::stats::TokenUsage; + + fn price(input: f64, output: f64) -> ModelPrice { + ModelPrice { + input, + output, + cached: input * 0.1, + cache_write: input, + } + } + + fn usage(prompt: u64, completion: u64) -> TokenUsage { + TokenUsage { + prompt_tokens: prompt, + completion_tokens: completion, + ..TokenUsage::default() + } + } + + #[test] + fn savings_compare_cheap_traffic_against_expensive_baseline() { + let stats = StatsAccumulator::default(); + stats.record_success("cheap", 10.0); + stats.record_usage("cheap", usage(1_000_000, 1_000_000), 10.0); + + let mut pricing = BTreeMap::new(); + pricing.insert("cheap".to_string(), price(0.1, 0.4)); + pricing.insert("expensive".to_string(), price(3.0, 15.0)); + let config = SavingsConfig::new(pricing, None); + + let snapshot = config.compute(&stats.snapshot()); + assert_eq!(snapshot.baseline_model.as_deref(), Some("expensive")); + assert!((snapshot.actual_cost - 0.5).abs() < 1e-9); + assert!((snapshot.baseline_cost - 18.0).abs() < 1e-9); + assert!((snapshot.saved - 17.5).abs() < 1e-9); + assert!((snapshot.saved_pct - 97.22).abs() < 0.01); + } + + #[test] + fn explicit_baseline_wins_over_most_expensive() { + let stats = StatsAccumulator::default(); + stats.record_success("cheap", 1.0); + stats.record_usage("cheap", usage(1_000_000, 0), 1.0); + + let mut pricing = BTreeMap::new(); + pricing.insert("cheap".to_string(), price(0.1, 0.4)); + pricing.insert("mid".to_string(), price(1.0, 2.0)); + pricing.insert("expensive".to_string(), price(3.0, 15.0)); + let config = SavingsConfig::new(pricing, Some("mid".to_string())); + + let snapshot = config.compute(&stats.snapshot()); + assert_eq!(snapshot.baseline_model.as_deref(), Some("mid")); + assert!((snapshot.baseline_cost - 1.0).abs() < 1e-9); + } + + #[test] + fn unpriced_models_are_reported_not_priced() { + let stats = StatsAccumulator::default(); + stats.record_success("mystery", 1.0); + stats.record_usage("mystery", usage(500_000, 0), 1.0); + + let mut pricing = BTreeMap::new(); + pricing.insert("expensive".to_string(), price(3.0, 15.0)); + let config = SavingsConfig::new(pricing, None); + + let snapshot = config.compute(&stats.snapshot()); + assert_eq!(snapshot.unpriced_models, vec!["mystery".to_string()]); + assert!((snapshot.actual_cost - 0.0).abs() < 1e-9); + // Baseline still counts what that traffic would have cost. + assert!((snapshot.baseline_cost - 1.5).abs() < 1e-9); + } + + #[test] + fn classifier_calls_count_as_overhead_cost_only() { + let stats = StatsAccumulator::default(); + stats.record_success("cheap", 1.0); + stats.record_usage("cheap", usage(1_000_000, 0), 1.0); + stats.record_classifier_success("cheap", Some(usage(1_000_000, 0)), 1.0); + + let mut pricing = BTreeMap::new(); + pricing.insert("cheap".to_string(), price(1.0, 1.0)); + pricing.insert("expensive".to_string(), price(2.0, 2.0)); + let config = SavingsConfig::new(pricing, None); + + let snapshot = config.compute(&stats.snapshot()); + // 1.0 routed + 1.0 classifier overhead. + assert!((snapshot.actual_cost - 2.0).abs() < 1e-9); + assert!((snapshot.classifier_cost - 1.0).abs() < 1e-9); + // Baseline only reprices the routed traffic. + assert!((snapshot.baseline_cost - 2.0).abs() < 1e-9); + } +} diff --git a/crates/switchyard-server/src/savings_dashboard.html b/crates/switchyard-server/src/savings_dashboard.html new file mode 100644 index 00000000..e0bbdcfd --- /dev/null +++ b/crates/switchyard-server/src/savings_dashboard.html @@ -0,0 +1,205 @@ + + + + + +Switchyard — Live Savings + + + +

Switchyard — Live Savings

+
+ connecting… +  ·  comparing routed spend against the baseline model for the same traffic. +
+ +
+ +
+
+
Saved vs baseline
+
—
+
+
+
Dollars saved
+
—
+
+
+
Actual spend
+
—
+
+
+
Baseline would cost
+
—
+
+
+
Requests
+
—
+
+
+
Routing overhead spend
+
—
+
+
+ +
+

Routing distribution (by calls)

+
No requests yet.
+
+ +
+

Per-model cost

+
No requests yet.
+
+ +
+
+ + + baseline: — +
+
+ +
+ Auto-refreshes every 2s. Streaming and non-streaming requests are both counted. + Tracks cost, not answer quality. +
+ + + + From 161c0d9ad79f96ce995c1cec5bcdd4e0762e7473 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 12 Aug 2026 12:19:43 +1000 Subject: [PATCH 2/4] refactor(server): move pricing to a standalone [pricing] table keyed by model id Signed-off-by: Michael Neale --- crates/switchyard-server/src/config.rs | 67 ++++++++++++-------------- 1 file changed, 31 insertions(+), 36 deletions(-) diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 9c810e21..54b3b7ab 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -59,6 +59,8 @@ struct ServerConfig { targets: BTreeMap, routes: BTreeMap, #[serde(default)] + pricing: BTreeMap, + #[serde(default)] savings: Option, } @@ -118,45 +120,41 @@ impl ServerConfig { self.apply_savings(state) } - /// Enables the savings endpoint when any target declares pricing. + /// Enables the savings endpoint when a `[pricing]` table is present. /// - /// Stats are keyed by the model id the routed call selected, so the - /// pricing table is keyed by `target.id`, not the TOML target name. + /// Stats are keyed by the model id the routed call selected, so + /// `[pricing]` keys are model ids (`target.id`), not TOML target names. + /// Purely additive: no `[pricing]` table means no savings endpoints and + /// no behavior change. fn apply_savings(&self, state: ServerState) -> ServerResult { - let mut pricing = BTreeMap::new(); - for target in self.targets.values() { - if let Some(config) = target.pricing { - pricing.insert(target.id.clone(), config.into_model_price()); + if self.pricing.is_empty() { + if self.savings.is_some() { + return Err(ServerError::new( + "[savings] requires a [pricing] table with at least one model", + )); } + return Ok(state); } + let pricing: BTreeMap = self + .pricing + .iter() + .map(|(model, config)| (model.clone(), config.into_model_price())) + .collect(); let baseline = match self .savings .as_ref() - .and_then(|s| s.baseline_target.as_ref()) + .and_then(|s| s.baseline_model.as_ref()) { - Some(name) => { - let target = self.targets.get(name).ok_or_else(|| { - ServerError::new(format!( - "savings baseline_target {name} does not match any [targets.*] entry" - )) - })?; - if target.pricing.is_none() { + Some(model) => { + if !pricing.contains_key(model) { return Err(ServerError::new(format!( - "savings baseline_target {name} has no pricing configured" + "savings baseline_model {model} has no [pricing.\"{model}\"] entry" ))); } - Some(target.id.clone()) + Some(model.clone()) } None => None, }; - if pricing.is_empty() { - if self.savings.is_some() { - return Err(ServerError::new( - "[savings] requires at least one target with a [targets.*.pricing] section", - )); - } - return Ok(state); - } Ok(state.with_savings(SavingsConfig::new(pricing, baseline))) } @@ -301,16 +299,13 @@ struct TargetConfig { llm_client: String, #[serde(default)] extra_body: BTreeMap, - #[serde(default)] - pricing: Option, } -/// Per-target pricing in USD per 1 million tokens. +/// Per-model pricing in USD per 1 million tokens, keyed by model id. /// -/// `cached` defaults to the cache-read discount being absent (0.0 means -/// cache reads are free only if explicitly priced so); `cache_write` -/// defaults to the base input rate when omitted, matching providers with -/// no cache-write premium. +/// `cached` defaults to 10% of the base input rate (the common provider +/// discount); `cache_write` defaults to the base input rate, matching +/// providers with no cache-write premium. #[derive(Clone, Copy, Debug, Deserialize)] #[serde(deny_unknown_fields)] struct PricingConfig { @@ -333,13 +328,13 @@ impl PricingConfig { } } -/// Optional `[savings]` section selecting the baseline comparison target. +/// Optional `[savings]` section selecting the baseline comparison model. #[derive(Debug, Default, Deserialize)] #[serde(deny_unknown_fields)] struct SavingsSectionConfig { - /// Target name (TOML key under `[targets.*]`) to price the baseline - /// against. Defaults to the most expensive priced target. - baseline_target: Option, + /// Model id (a `[pricing]` key) to price the baseline against. + /// Defaults to the most expensive priced model. + baseline_model: Option, } #[derive(Clone, Copy, Debug, Deserialize)] From f744dafb9872acdd32bbc05d0dccee3093944f66 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 12 Aug 2026 13:14:17 +1000 Subject: [PATCH 3/4] docs(server): document cost savings reporting and add config tests Signed-off-by: Michael Neale --- CHANGELOG.md | 7 +++ crates/switchyard-server/src/config.rs | 27 +++++++++ docs/operations/cost_savings.md | 77 ++++++++++++++++++++++++++ mkdocs.yml | 1 + 4 files changed, 112 insertions(+) create mode 100644 docs/operations/cost_savings.md diff --git a/CHANGELOG.md b/CHANGELOG.md index de81bbc1..be889286 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,13 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **Cost savings reporting** — an optional `[pricing]` table in the server + TOML enables `GET /v1/savings` and a live `GET /dashboard` page comparing + routed spend against a baseline model. Purely additive: without pricing + the endpoints are not registered and behavior is unchanged. + ### Removed - **Deprecated Python server stack** — `switchyard serve`, YAML route bundles, diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 54b3b7ab..7952a3f7 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -1263,6 +1263,33 @@ target = "weak" ); } + #[test] + fn a_pricing_table_enables_savings() -> ServerResult<()> { + // Pricing is purely additive: without it the savings endpoints are + // not registered and behavior is unchanged. + let state = server_state_from_toml(VALID_CONFIG)?; + assert!(state.savings.is_none()); + + let priced = + format!("{VALID_CONFIG}\n[pricing.\"weak/model\"]\ninput = 1.0\noutput = 5.0\n"); + let state = server_state_from_toml(&priced)?; + assert!(state.savings.is_some()); + Ok(()) + } + + #[test] + fn savings_requires_pricing_and_a_priced_baseline() { + // A [savings] section without any priced model is a config mistake. + let orphan = format!("{VALID_CONFIG}\n[savings]\nbaseline_model = \"strong/model\"\n"); + assert!(error_message(&orphan).contains("requires a [pricing] table")); + + // The baseline must itself have a pricing entry. + let unpriced_baseline = format!( + "{VALID_CONFIG}\n[pricing.\"weak/model\"]\ninput = 1.0\noutput = 5.0\n\n[savings]\nbaseline_model = \"strong/model\"\n" + ); + assert!(error_message(&unpriced_baseline).contains("has no [pricing")); + } + #[test] fn rejects_unknown_fields_and_algorithm_types() { let unknown_field = diff --git a/docs/operations/cost_savings.md b/docs/operations/cost_savings.md new file mode 100644 index 00000000..8db7b0e7 --- /dev/null +++ b/docs/operations/cost_savings.md @@ -0,0 +1,77 @@ +# Cost Savings Reporting + +Switchyard can report, in real time, how much a routed deployment is saving +compared to sending every request to a single baseline model. The feature is +purely additive: it prices the token counters the server already records, and +it never influences routing decisions. + +## Enabling + +Add a `[pricing]` table to the deployment TOML. Keys are **model ids** (the +`id` of a target, not the TOML target name), and rates are USD per 1 million +tokens: + +```toml +[pricing."anthropic/claude-opus-4.7"] +input = 15.00 +output = 75.00 +cached = 1.50 # optional: cache-read rate, defaults to input x 0.1 +cache_write = 18.75 # optional: defaults to input (no cache-write premium) + +[pricing."moonshotai/kimi-k2.7-code"] +input = 0.60 +output = 2.50 + +[savings] +baseline_model = "anthropic/claude-opus-4.7" +``` + +The optional `[savings]` section selects the baseline. When omitted, the most +expensive priced model (by combined input and output rate) is used. A +`baseline_model` must have its own `[pricing]` entry, and a `[savings]` +section without any `[pricing]` table is rejected at startup. + +When no `[pricing]` table is present, the endpoints below are not registered +and the server behaves exactly as before. + +## Endpoints + +| Endpoint | Purpose | +|---|---| +| `GET /v1/savings` | JSON snapshot of actual spend, baseline spend, and savings | +| `GET /dashboard` | Self-contained live HTML dashboard (auto-refreshes) | + +`GET /v1/savings` prices every model that served completions and compares it +against serving the same token traffic with the baseline model: + +```json +{ + "total_requests": 11, + "actual_cost": 0.2864, + "baseline_cost": 0.4722, + "classifier_cost": 0.0026, + "saved": 0.1858, + "saved_pct": 39.35, + "baseline_model": "anthropic/claude-opus-4.7", + "models": { "...": { "calls": 7, "cost": 0.1256, "baseline_cost": 0.3141 } }, + "unpriced_models": [] +} +``` + +Classifier and judge traffic is priced separately as `classifier_cost` and +deducted from the savings, so routing overhead is charged against the result +rather than hidden. Models that served traffic without a pricing entry are +costed at zero and listed in `unpriced_models` so under-counting is visible. + +Counters reset together with the existing stats via `POST /v1/stats/reset`. + +## Semantics + +- Prompt tokens are split into base input, cache reads, and cache writes, and + each bucket is priced separately, matching the cost model of the + `switchyard.cli.launchers` cost estimator. +- The baseline cost re-prices each model's token traffic at the baseline + model's rates. It answers "what would this traffic have cost on the + baseline model", not "what would the baseline model have generated". +- Savings measure cost only. Whether the cheaper model's answers were good + enough is a quality question the report cannot answer. diff --git a/mkdocs.yml b/mkdocs.yml index d081dab5..74ba5885 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -31,6 +31,7 @@ nav: - Escalation-Router Routing: routing_algorithms/escalation_router_routing.md - Operations: - Context-Window Handling: operations/context_window.md + - Cost Savings Reporting: operations/cost_savings.md - Reference: - CLI Reference: cli_reference.md - TOML Schema: reference/toml_schema.md From da9703b21ffec22a44288bf0b49bcc142050e229 Mon Sep 17 00:00:00 2001 From: Michael Neale Date: Wed, 12 Aug 2026 15:34:53 +1000 Subject: [PATCH 4/4] fix(server): validate pricing rates and address savings review feedback Signed-off-by: Michael Neale --- crates/switchyard-server/src/config.rs | 37 ++++++++++++++++--- crates/switchyard-server/src/savings.rs | 20 +++++----- .../src/savings_dashboard.html | 17 +++++++-- dev-server/config.toml | 16 ++++++++ 4 files changed, 73 insertions(+), 17 deletions(-) diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 7952a3f7..c3702ca4 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -135,11 +135,27 @@ impl ServerConfig { } return Ok(state); } - let pricing: BTreeMap = self - .pricing - .iter() - .map(|(model, config)| (model.clone(), config.into_model_price())) - .collect(); + // Reject unusable rates at startup: a NaN or infinite rate would make + // the savings snapshot unserializable, and a negative rate produces + // negative spend. Matches the build-time validation of other numeric + // config in this file. + let mut pricing: BTreeMap = BTreeMap::new(); + for (model, config) in &self.pricing { + let price = config.into_model_price(); + for (field, rate) in [ + ("input", price.input), + ("output", price.output), + ("cached", price.cached), + ("cache_write", price.cache_write), + ] { + if !rate.is_finite() || rate < 0.0 { + return Err(ServerError::new(format!( + "[pricing.\"{model}\"] {field} must be a finite, non-negative rate" + ))); + } + } + pricing.insert(model.clone(), price); + } let baseline = match self .savings .as_ref() @@ -1277,6 +1293,17 @@ target = "weak" Ok(()) } + #[test] + fn pricing_rates_must_be_finite_and_non_negative() { + let negative = + format!("{VALID_CONFIG}\n[pricing.\"weak/model\"]\ninput = -1.0\noutput = 5.0\n"); + assert!(error_message(&negative).contains("finite, non-negative")); + + let non_finite = + format!("{VALID_CONFIG}\n[pricing.\"weak/model\"]\ninput = inf\noutput = 5.0\n"); + assert!(error_message(&non_finite).contains("finite, non-negative")); + } + #[test] fn savings_requires_pricing_and_a_priced_baseline() { // A [savings] section without any priced model is a config mistake. diff --git a/crates/switchyard-server/src/savings.rs b/crates/switchyard-server/src/savings.rs index 04f09cf7..bde5e3c1 100644 --- a/crates/switchyard-server/src/savings.rs +++ b/crates/switchyard-server/src/savings.rs @@ -42,11 +42,6 @@ impl SavingsConfig { Self { pricing, baseline } } - /// True when no pricing has been configured at all. - pub fn is_empty(&self) -> bool { - self.pricing.is_empty() - } - fn price_for(&self, model: &str) -> Option { self.pricing.get(model).copied() } @@ -95,8 +90,8 @@ impl SavingsConfig { prompt_tokens: m.prompt_tokens, completion_tokens: m.completion_tokens, cached_tokens: m.cached_tokens, - cost, - baseline_cost: would_be, + cost: round6(cost), + baseline_cost: round6(would_be), priced: price.is_some(), }, ); @@ -112,8 +107,15 @@ impl SavingsConfig { cached: m.cached_tokens, cache_creation: m.cache_creation_tokens, }; - if let Some(price) = self.price_for(model) { - classifier_cost += tokens.cost(price); + match self.price_for(model) { + Some(price) => classifier_cost += tokens.cost(price), + // An unpriced judge is under-counted the same way as an + // unpriced serving model; surface it rather than hide it. + None => { + if !unpriced_models.contains(model) { + unpriced_models.push(model.clone()); + } + } } } actual_cost += classifier_cost; diff --git a/crates/switchyard-server/src/savings_dashboard.html b/crates/switchyard-server/src/savings_dashboard.html index e0bbdcfd..cb3f7086 100644 --- a/crates/switchyard-server/src/savings_dashboard.html +++ b/crates/switchyard-server/src/savings_dashboard.html @@ -138,7 +138,7 @@

Per-model cost

return `
${esc(name)}
-
${m.calls} calls · ${pct.toFixed(1)}%
+
${fmt(m.calls)} calls · ${pct.toFixed(1)}%
`; }).join(""); } @@ -164,7 +164,10 @@

Per-model cost

${rows}`; } +let inFlight = false; async function refresh(){ + if(inFlight) return; + inFlight = true; try { const r = await fetch("/v1/savings"); if(!r.ok) throw new Error(await r.text()); @@ -189,17 +192,25 @@

Per-model cost

$("status").textContent = "live"; } catch(e) { $("status").textContent = "fetch failed"; + } finally { + inFlight = false; } } $("refresh").addEventListener("click", refresh); $("reset").addEventListener("click", async () => { - await fetch("/v1/stats/reset", {method:"POST"}); + try { + const r = await fetch("/v1/stats/reset", {method:"POST"}); + if(!r.ok) throw new Error(await r.text()); + } catch(e) { + $("status").textContent = "reset failed"; + return; + } refresh(); }); refresh(); -setInterval(refresh, 2000); +setInterval(() => { if(!document.hidden) refresh(); }, 2000); diff --git a/dev-server/config.toml b/dev-server/config.toml index 32178908..bcb49f83 100644 --- a/dev-server/config.toml +++ b/dev-server/config.toml @@ -77,3 +77,19 @@ base_threshold = 0.5 session_affinity = true message_hash_fallback = true + +# Optional cost savings reporting: pricing (USD per 1M tokens) enables +# GET /v1/savings and the live GET /dashboard page. Purely additive — +# remove this section and the server behaves exactly as before. +# See docs/operations/cost_savings.md. +# +# [pricing."aws/anthropic/bedrock-claude-opus-4-8"] +# input = 15.00 +# output = 75.00 +# +# [pricing."nvidia/zai-org/glm-5.2"] +# input = 0.60 +# output = 2.20 +# +# [savings] +# baseline_model = "aws/anthropic/bedrock-claude-opus-4-8"