Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions crates/libsy-llm-client/tests/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -441,7 +441,8 @@ impl Algorithm for SingleCallAlgo {
.first()
.ok_or(LibsyError::NoTargets)?
.clone();
let decision = Decision::new(target.clone(), Some(format!("picked '{target}'")), true);
tracing::info!("picked '{target}'");
let decision = Decision::new(target.clone(), true);
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
Expand Down Expand Up @@ -817,21 +818,15 @@ async fn successful_run_records_metrics_spans_and_decision_log() -> switchyard_l
Some("2")
);

// Structured debug event: the published decision with its reasoning.
// The algorithm logs why it made the decision.
let events = store.events();
assert!(
events.iter().any(|event| {
event.target == "libsy"
&& event.level == "DEBUG"
&& event.fields.get("selected_model").map(String::as_str) == Some(MODEL)
&& event
.fields
.get("reasoning")
.is_some_and(|reasoning| reasoning.contains("picked"))
event.level == "INFO"
&& event
.fields
.get("message")
.is_some_and(|message| message.contains("routing decision"))
.is_some_and(|message| message.contains("picked"))
}),
"no routing-decision log event for {MODEL} in {events:?}"
Comment thread
coderabbitai[bot] marked this conversation as resolved.
);
Expand Down Expand Up @@ -1229,11 +1224,11 @@ async fn classifier_metrics_count_only_the_final_routed_call() -> switchyard_lib

let (trace, _response) = run(router, client, classifier_request()).await?;

assert!(
assert_eq!(
trace
.last()
.and_then(|decision| decision.reasoning())
.is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
.map(|decision| decision.selected_model_id().as_str()),
Some("weak")
);

let snapshots = flushed_metrics(exporter, provider);
Expand Down
77 changes: 24 additions & 53 deletions crates/libsy/src/algorithms/fall_through.rs
Original file line number Diff line number Diff line change
Expand Up @@ -142,7 +142,7 @@ where
self
}

/// Sets the decision reasoning for an algorithm assembled from this cascade.
/// Sets the decision log message for an algorithm assembled from this cascade.
pub(crate) fn with_decision_reason(mut self, reason: fn(&str, &Score) -> String) -> Self {
self.decision_reason = reason;
self
Expand Down Expand Up @@ -264,17 +264,15 @@ where
RoutingFallbackReason::ContextWindow => "exceeded its context window",
RoutingFallbackReason::Unavailable => "was unavailable",
};
Decision::new(
to.clone(),
Some(with_routing_tier(
format!(
"{from} {failure}; fell back to {to} (fallback reason: {})",
reason.as_str(),
),
deciding.routing_tier(to),
)),
true,
)
let message = with_routing_tier(
format!(
"{from} {failure}; fell back to {to} (fallback reason: {})",
reason.as_str(),
),
deciding.routing_tier(to),
);
tracing::info!("{message}");
Decision::new(to.clone(), true)
}

/// Returns this request's retained state without holding the registry lock.
Expand Down Expand Up @@ -321,22 +319,20 @@ where
});
};

// 3. Resolve the target and publish the decision. When an excluded target sends
// the request elsewhere, the reasoning describes where it actually went.
// 3. Resolve the target, log the choice, and publish the decision. When an excluded
// target sends the request elsewhere, the log describes where it actually went.
let target = algorithm::select_eligible_model(&self.targets, &score.target, excluded)?;
let reasoning = if target == score.target {
let message = if target == score.target {
(self.decision_reason)(&self.name, &score)
} else {
format!(
"{} exceeded its context window; fell back to {}",
score.target, target
)
};
let decision: Decision = Decision::new(
target.clone(),
Some(with_routing_tier(reasoning, deciding.routing_tier(&target))),
true,
);
let message = with_routing_tier(message, deciding.routing_tier(&target));
tracing::info!("{message}");
let decision: Decision = Decision::new(target.clone(), true);
Comment thread
coderabbitai[bot] marked this conversation as resolved.
driver.decide(decision.clone()).await?;

// 4. Post-decision replay: every processor sees the decision so stateful ones
Expand Down Expand Up @@ -395,11 +391,11 @@ fn default_decision_reason(_name: &str, winner: &Score) -> String {
)
}

// Routing tier is not part of Decision struct, so keeping it in reasoning for now. Remove it later if needed from reasoning.
fn with_routing_tier(reasoning: String, tier: Option<&str>) -> String {
/// Appends the routing tier to a decision log message when the classifier supplies one.
fn with_routing_tier(message: String, tier: Option<&str>) -> String {
match tier {
Some(tier) => format!("{reasoning}; routing tier: {tier}"),
None => reasoning,
Some(tier) => format!("{message}; routing tier: {tier}"),
None => message,
}
}

Expand Down Expand Up @@ -836,34 +832,23 @@ mod tests {
for _ in 0..2 {
let (model, trace) = run_turn(&router, unavailable(&["weak"], calls.clone())).await?;
assert_eq!(model, "strong");
assert!(
assert_eq!(
trace
.last()
.and_then(|decision| decision.reasoning())
.is_some_and(|reasoning| reasoning.contains("fallback reason: unavailable"))
.map(|decision| decision.selected_model_id().as_str()),
Some("strong")
);
}
assert_eq!(&*calls.lock(), &["weak", "strong", "weak", "strong"]);
Ok(())
}

#[tokio::test]
async fn fallback_decision_preserves_tier_and_cause_in_reasoning() -> Result<()> {
async fn fallback_decision_preserves_answer_call_semantics() -> Result<()> {
struct TieredClassifier;

#[async_trait]
impl Classifier for TieredClassifier {
fn routing_tier(
&self,
selected_model_id: &switchyard_protocol::ModelId,
) -> Option<&'static str> {
match selected_model_id.as_str() {
"weak" => Some("weak"),
"strong" => Some("strong"),
_ => None,
}
}

async fn score(
&self,
_state: &mut (),
Expand All @@ -881,19 +866,10 @@ mod tests {
assert_eq!(trace.len(), 2);
assert_eq!(trace[0].selected_model_id(), "weak");
assert!(trace[0].is_answer_call());
assert!(
trace[0]
.reasoning()
.is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
);

let fallback = &trace[1];
assert_eq!(fallback.selected_model_id(), "strong");
assert!(fallback.is_answer_call());
assert!(fallback.reasoning().is_some_and(|reasoning| {
reasoning.contains("fallback reason: unavailable")
&& reasoning.contains("routing tier: strong")
}));
Ok(())
}

Expand Down Expand Up @@ -964,11 +940,6 @@ mod tests {

assert_eq!(text, "strong");
assert_eq!(trace[0].selected_model_id(), "strong");
assert!(
trace[0]
.reasoning()
.is_some_and(|r| r.contains("fell back to strong"))
);
Ok(())
}

Expand Down
25 changes: 5 additions & 20 deletions crates/libsy/src/algorithms/llm_class.rs
Original file line number Diff line number Diff line change
Expand Up @@ -513,15 +513,12 @@ impl Classifier<State> for EscalationClassifier {
// If the efficient model exceeds its context window, fall through to capable: returning
// `(decisive(capable), None)` tells FallThrough::execute to call
// call_model_with_fallback with the capable target instead of surfacing the error.
tracing::info!(
target = %self.efficient,
"escalation classifier selected efficient tier"
);
let efficient_response = match driver
.call_model(
request.clone(),
Decision::new(
self.efficient.clone(),
Some("escalation classifier: efficient tier".into()),
true,
),
)
.call_model(request.clone(), Decision::new(self.efficient.clone(), true))
.await
{
Ok(r) => r,
Expand Down Expand Up @@ -1822,12 +1819,6 @@ mod tests {
trace.last().map(|d| d.selected_model_id().as_str()),
Some("efficient")
);
assert!(
trace
.last()
.and_then(|decision| decision.reasoning())
.is_some_and(|reasoning| reasoning.contains("routing tier: weak"))
);
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("efficient answer".to_string())
Expand Down Expand Up @@ -1873,12 +1864,6 @@ mod tests {
trace.last().map(|d| d.selected_model_id().as_str()),
Some("capable")
);
assert!(
trace
.last()
.and_then(|decision| decision.reasoning())
.is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
);
assert_eq!(
response.llm_response.as_agg().map(completion_text),
Some("capable answer".to_string())
Expand Down
7 changes: 2 additions & 5 deletions crates/libsy/src/algorithms/noop.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,11 +27,8 @@ impl Algorithm for Noop {
let model_id = request
.model_id()
.unwrap_or_else(|| ModelId::from("switchyard/noop"));
let decision: Decision = Decision::new(
model_id.clone(),
Some("noop returned its synthetic response".to_string()),
true,
);
tracing::info!(target = %model_id, "noop returned its synthetic response");
let decision: Decision = Decision::new(model_id.clone(), true);
driver.decide(decision.clone()).await?;

let llm_response = LlmResponse::Agg(AggLlmResponse {
Expand Down
7 changes: 2 additions & 5 deletions crates/libsy/src/algorithms/passthrough.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,8 @@ impl Algorithm for Passthrough {
}

async fn route(self: Arc<Self>, driver: Driver, request: Request) -> Result<Response> {
let decision: Decision = Decision::new(
self.target.clone(),
Some(format!("passthrough selected target '{}'", self.target)),
true,
);
tracing::info!(target = %self.target, "passthrough selected target");
let decision: Decision = Decision::new(self.target.clone(), true);
driver.decide(decision.clone()).await?;
driver.call_model(request, decision).await
}
Expand Down
6 changes: 0 additions & 6 deletions crates/libsy/src/algorithms/rand.rs
Original file line number Diff line number Diff line change
Expand Up @@ -397,12 +397,6 @@ mod tests {
let decision = &trace[0];

assert_eq!(decision.selected_model_id(), "only/model");
assert!(
decision
.reasoning()
.unwrap_or_default()
.contains("only/model")
);
assert!(decision.is_answer_call());
Ok(())
}
Expand Down
6 changes: 3 additions & 3 deletions crates/libsy/src/algorithms/stage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -511,11 +511,11 @@ mod tests {
"the selected target should be recorded as an answer call"
);
drop(calls);
assert!(
assert_eq!(
trace
.last()
.and_then(|decision| decision.reasoning())
.is_some_and(|reasoning| reasoning.contains("routing tier: strong"))
.map(|decision| decision.selected_model_id().as_str()),
Some("strong")
);
Ok(())
}
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy/src/algorithms/util/affinity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ mod tests {
type BoxErr = Box<dyn std::error::Error + Send + Sync>;

fn fixed_decision(target: &str) -> Decision {
Decision::new(target, None, true)
Decision::new(target, true)
}

fn request(metadata: Metadata) -> Request {
Expand Down
7 changes: 2 additions & 5 deletions crates/libsy/src/algorithms/util/llm_judge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -226,14 +226,11 @@ where
) -> Option<J::Verdict> {
let judge_model = self.target.as_str();

tracing::info!(target = judge_model, "consulting llm judge");
let response = driver
.call_model(
self.judge.build_request(state, request),
Decision::new(
self.target.to_string(),
Some("llm judge consultation".to_string()),
false,
),
Decision::new(self.target.to_string(), false),
)
.await
.inspect_err(|error| report_fail_open(judge_model, error, libsy_error_reason(error)))
Expand Down
4 changes: 2 additions & 2 deletions crates/libsy/src/algorithms/util/prompts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -265,7 +265,7 @@ mod tests {
},
..Request::default()
};
let decision = Decision::new(target, None, true);
let decision = Decision::new(target, true);
processor
.process(
&mut (),
Expand Down Expand Up @@ -326,7 +326,7 @@ mod tests {
text: "you are a coding agent".to_string(),
}],
});
let decision = Decision::new("strong", None, true);
let decision = Decision::new("strong", true);

processor
.process(
Expand Down
6 changes: 3 additions & 3 deletions crates/libsy/src/core/algorithm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -192,8 +192,8 @@ impl Driver {
}

/// Publish a routing [`Decision`] as a [`Step::Decision`] on the stream.
/// Each successfully published decision is counted and logged with its
/// reasoning; a decision the stream never accepted is not recorded.
/// Each successfully published decision is counted and logged; a decision
/// the stream never accepted is not recorded.
pub async fn decide(&self, decision: Decision) -> Result<()> {
self.step_tx
.send(Ok(Step::Decision(decision.clone())))
Expand Down Expand Up @@ -660,7 +660,7 @@ mod tests {

/// Build a routed decision for orchestration tests.
fn test_decision(selected_model_id: ModelId) -> Decision {
Decision::new(selected_model_id, None, true)
Decision::new(selected_model_id, true)
}

/// Trivial algo used only to exercise the orchestrator: calls the first target
Expand Down
2 changes: 1 addition & 1 deletion crates/libsy/src/core/processor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ mod tests {
let mut state = TestState::default();
let mut req = request();
let response = text_response(None, "ok");
let decision = Decision::new("test/model", None, true);
let decision = Decision::new("test/model", true);
// Feed one of every event variant through the processor.
processor
.process(&mut state, Event::Request(&mut req))
Expand Down
4 changes: 1 addition & 3 deletions crates/libsy/src/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -285,15 +285,13 @@ pub(crate) fn record_llm_call(
}
}

/// Records one published routing decision: the decision counter plus a
/// structured debug event carrying the decision's reasoning.
/// Records one published routing decision: the decision counter plus a structured debug event.
pub(crate) fn record_decision(algorithm: &str, decision: &Decision) {
let selected_model = decision.selected_model_id();
tracing::debug!(
target: TRACING_TARGET,
algorithm,
selected_model = %selected_model,
reasoning = decision.reasoning().unwrap_or(""),
"routing decision"
);
meter().u64_counter("switchyard.decisions").build().add(
Expand Down
Loading
Loading