From 76bcd19afe7446e17b1046a687459bcafe24d444 Mon Sep 17 00:00:00 2001
From: Yahoo <119646314+y4ho0@users.noreply.github.com>
Date: Thu, 13 Aug 2026 02:18:36 +0800
Subject: [PATCH] fix(metrics): extend LLM latency histogram buckets

Signed-off-by: Yahoo <119646314+y4ho0@users.noreply.github.com>
---
 crates/switchyard-server/src/metrics.rs  | 24 ++++++++++++++++++++++++
 crates/switchyard-server/tests/server.rs |  9 +++++++++
 2 files changed, 33 insertions(+)

diff --git a/crates/switchyard-server/src/metrics.rs b/crates/switchyard-server/src/metrics.rs
index 2469e0251..b6ba04f5c 100644
--- a/crates/switchyard-server/src/metrics.rs
+++ b/crates/switchyard-server/src/metrics.rs
@@ -19,6 +19,13 @@ const ROUTING_OVERHEAD_BUCKETS_MS: &[f64] = &[
     0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0, 2500.0, 5000.0,
 ];
 
+/// Bucket boundaries for model-call and end-to-end LLM latency histograms.
+/// Retains the SDK defaults through 10 seconds and extends them for long generations.
+const LLM_LATENCY_BUCKETS_MS: &[f64] = &[
+    0.0, 5.0, 10.0, 25.0, 50.0, 75.0, 100.0, 250.0, 500.0, 750.0, 1000.0, 2500.0, 5000.0, 7500.0,
+    10_000.0, 15_000.0, 30_000.0, 60_000.0, 120_000.0, 300_000.0,
+];
+
 struct Metrics {
     registry: Registry,
     provider: SdkMeterProvider,
@@ -43,6 +50,7 @@ fn initialize() -> Result<Metrics, String> {
     let mut builder = SdkMeterProvider::builder()
         .with_reader(exporter)
         .with_view(routing_overhead_buckets)
+        .with_view(llm_latency_buckets)
         .with_resource(crate::observability::resource());
     if crate::observability::otlp_enabled("METRICS") {
         let exporter = opentelemetry_otlp::MetricExporter::builder()
@@ -84,6 +92,22 @@ fn routing_overhead_buckets(instrument: &Instrument) -> Option<Stream> {
         .ok()
 }
 
+fn llm_latency_buckets(instrument: &Instrument) -> Option<Stream> {
+    if !matches!(
+        instrument.name(),
+        "switchyard.model_call_latency_ms" | "switchyard.total_latency_ms"
+    ) {
+        return None;
+    }
+    Stream::builder()
+        .with_aggregation(Aggregation::ExplicitBucketHistogram {
+            boundaries: LLM_LATENCY_BUCKETS_MS.to_vec(),
+            record_min_max: true,
+        })
+        .build()
+        .ok()
+}
+
 /// Make the metrics exist before they get a hit. Nicer for dashboards but not really necessary.
 /// The HTTP status codes we seed are somewhat arbitrary.
 fn seed_outcome_metrics() {
diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs
index a1241c59d..0d5ddf5a1 100644
--- a/crates/switchyard-server/tests/server.rs
+++ b/crates/switchyard-server/tests/server.rs
@@ -486,6 +486,15 @@ async fn metrics_exposes_switchyard_otel_instruments() -> TestResult {
         )
         .is_some()
     );
+    for metric in [
+        "switchyard_model_call_latency_ms_bucket",
+        "switchyard_total_latency_ms_bucket",
+    ] {
+        assert!(
+            metric_line(metrics, metric, &[("model", MODEL), ("le", "300000")]).is_some(),
+            "missing five-minute bucket for {metric}"
+        );
+    }
     assert!(
         metric_line(
             metrics,