diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index b60ebea6..502921d4 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -14,7 +14,9 @@ use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role}; use super::fall_through::{DefaultTarget, FallThrough}; use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::AffinityRouter; -use super::util::classifier_contract::{ClassifierContract, ClassifierContractConfig}; +use super::util::classifier_contract::{ + ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, +}; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; use super::util::llm_judge::{ ClassifierInput, JsonSchemaDecoder, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, @@ -261,6 +263,8 @@ struct TaskClassifierConfigWire { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_judge_max_output_tokens")] max_output_tokens: u64, } @@ -275,6 +279,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { if let Some(prompt) = wire.prompt { contract = contract.with_prompt(prompt); } + contract = contract.with_response_format_type(wire.response_format_type); Ok(Self { base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, diff --git a/crates/libsy/src/algorithms/util/classifier_contract.rs b/crates/libsy/src/algorithms/util/classifier_contract.rs index d3298dcf..d597ed75 100644 --- a/crates/libsy/src/algorithms/util/classifier_contract.rs +++ b/crates/libsy/src/algorithms/util/classifier_contract.rs @@ -9,6 +9,17 @@ use serde_json::{Value, json}; use crate::{LibsyError, Result}; +/// Provider-side structured-output mode used by a classifier judge. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ClassifierResponseFormat { + /// Send the verdict schema through the provider's strict JSON Schema wrapper. + #[default] + JsonSchema, + /// Request a JSON object and enforce the verdict schema locally. + JsonObject, +} + /// User-configurable parts of a classifier's prompt and verdict contract. /// /// Fields are private so new contract settings can be added without breaking Rust struct literals. @@ -16,6 +27,8 @@ use crate::{LibsyError, Result}; pub struct ClassifierContractConfig { #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, } impl ClassifierContractConfig { @@ -29,6 +42,20 @@ impl ClassifierContractConfig { pub fn prompt(&self) -> Option<&str> { self.prompt.as_deref() } + + /// Selects the provider-side structured-output mode. + pub fn with_response_format_type( + mut self, + response_format_type: ClassifierResponseFormat, + ) -> Self { + self.response_format_type = response_format_type; + self + } + + /// Returns the configured provider-side structured-output mode. + pub fn response_format_type(&self) -> ClassifierResponseFormat { + self.response_format_type + } } /// Rendered prompt and response format for one classifier. @@ -42,8 +69,9 @@ pub(crate) struct ClassifierContract { impl ClassifierContract { /// Builds a contract from user settings and packaged defaults. /// - /// The response format must contain `json_schema.schema` and is retained separately for the - /// model request. Schemas are never copied into the system prompt. + /// The packaged response format must contain `json_schema.schema`. JSON Schema mode retains + /// that wrapper for the model request; JSON Object mode moves the schema into the prompt and + /// compiles it for local validation. pub(crate) fn from_config( config: &ClassifierContractConfig, default_prompt: &str, @@ -56,7 +84,31 @@ impl ClassifierContract { message: format!("response schema is invalid: {error}"), } })?; - Self::from_response_format(prompt_template, response_format, None) + let schema = response_format + .pointer("/json_schema/schema") + .ok_or_else(|| LibsyError::AlgorithmError { + message: "response schema has no json_schema.schema".to_string(), + })?; + match config.response_format_type() { + ClassifierResponseFormat::JsonSchema => { + Self::from_response_format(prompt_template, response_format, None) + } + ClassifierResponseFormat::JsonObject => { + validate_prompt(prompt_template)?; + let validator = compile_schema(schema)?; + let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| { + algorithm_error(format!("response schema could not be rendered: {error}")) + })?; + let system_prompt = format!( + "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}" + ); + Self::from_response_format( + &system_prompt, + json!({"type": "json_object"}), + Some(validator), + ) + } + } } /// Builds a provider response format around a user-supplied inner JSON Schema. @@ -88,21 +140,7 @@ impl ClassifierContract { response_format: Value, validator: Option, ) -> Result { - if prompt_template.trim().is_empty() { - return Err(LibsyError::AlgorithmError { - message: "classifier prompt must not be empty".to_string(), - }); - } - if prompt_template.contains("{{RESPONSE_SCHEMA}}") { - return Err(LibsyError::AlgorithmError { - message: "classifier prompt must not include {{RESPONSE_SCHEMA}}; the response schema is sent separately".to_string(), - }); - } - response_format - .pointer("/json_schema/schema") - .ok_or_else(|| LibsyError::AlgorithmError { - message: "response schema has no json_schema.schema".to_string(), - })?; + validate_prompt(prompt_template)?; Ok(Self { system_prompt: prompt_template.to_string(), @@ -119,6 +157,11 @@ impl ClassifierContract { &self.response_format } + /// Whether the provider response must be checked against the compiled schema locally. + pub(crate) fn validates_locally(&self) -> bool { + self.validator.is_some() + } + /// Validates a dynamic verdict when this contract carries a runtime schema validator. pub(crate) fn validate_verdict(&self, verdict: &Value) -> Result<()> { let Some(validator) = &self.validator else { @@ -132,6 +175,18 @@ impl ClassifierContract { } } +fn validate_prompt(prompt_template: &str) -> Result<()> { + if prompt_template.trim().is_empty() { + return Err(algorithm_error("classifier prompt must not be empty")); + } + if prompt_template.contains("{{RESPONSE_SCHEMA}}") { + return Err(algorithm_error( + "classifier prompt must not include {{RESPONSE_SCHEMA}}; remove the placeholder because Switchyard supplies the schema automatically", + )); + } + Ok(()) +} + fn compile_schema(schema: &Value) -> Result { if !schema.is_object() { return Err(algorithm_error("response_schema must be a JSON object")); @@ -223,7 +278,11 @@ mod tests { ) .expect_err("schema placeholders should be rejected"); - assert!(error.to_string().contains("schema is sent separately")); + assert!( + error + .to_string() + .contains("Switchyard supplies the schema automatically") + ); } #[test] diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index bb271ac5..6661cee1 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -62,9 +62,19 @@ where fn decode( &self, response: &AggLlmResponse, - _contract: &ClassifierContract, + contract: &ClassifierContract, ) -> Result { - parse_json_verdict(response) + if !contract.validates_locally() { + return parse_json_verdict(response); + } + let verdict = parse_json_verdict::(response)?; + contract.validate_verdict(&verdict)?; + serde_json::from_value(verdict).map_err(|error| LibsyError::AlgorithmError { + message: format!( + "judge reply did not parse as {}: {error}", + std::any::type_name::() + ), + }) } } @@ -353,6 +363,11 @@ mod tests { ok: bool, } + #[derive(Debug, Deserialize, PartialEq)] + struct ScoreVerdict { + score: f64, + } + struct TestJudge; impl Judge for TestJudge { @@ -416,6 +431,47 @@ mod tests { Ok(()) } + #[test] + fn typed_decoder_enforces_a_json_object_contract_locally() -> Result<()> { + use super::super::classifier_contract::{ + ClassifierContractConfig, ClassifierResponseFormat, + }; + + let config = ClassifierContractConfig::default() + .with_response_format_type(ClassifierResponseFormat::JsonObject); + let contract = ClassifierContract::from_config( + &config, + "Return one JSON score.", + r#"{ + "type": "json_schema", + "json_schema": { + "name": "ScoreVerdict", + "schema": { + "type": "object", + "properties": {"score": {"type": "number"}}, + "required": ["score"], + "additionalProperties": false + } + } + }"#, + )?; + let decoder = SerdeDecoder::::new(); + + let error = decoder + .decode( + &text_response(None, r#"{"score":0.5,"unexpected":true}"#), + &contract, + ) + .expect_err("an extra property should fail the local schema"); + + assert!(error.to_string().contains("did not match response_schema")); + assert_eq!( + decoder.decode(&text_response(None, r#"{"score":0.5}"#), &contract)?, + ScoreVerdict { score: 0.5 } + ); + Ok(()) + } + fn buffered(completion: &str) -> Response { Response { llm_response: LlmResponse::Agg(text_response(None, completion)), diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 3b1ede9a..583ce1f1 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -23,7 +23,9 @@ pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::util::affinity::AffinityRouter; -pub use algorithms::util::classifier_contract::ClassifierContractConfig; +pub use algorithms::util::classifier_contract::{ + ClassifierContractConfig, ClassifierResponseFormat, +}; pub use algorithms::util::escalation::EscalationJudgeConfig; pub use algorithms::util::prompts::{SystemPromptProcessor, TargetPrompts, append_note}; pub use algorithms::util::subagent::SubagentOverride; diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index ee2b70fd..a3cdc849 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -11,7 +11,7 @@ use http::header::{HeaderName, HeaderValue}; use pyo3::exceptions::{PyBaseException, PyStopAsyncIteration, PyTypeError, PyValueError}; use pyo3::prelude::*; use switchyard_libsy::{ - Algorithm, CallModel, ClassifierContractConfig, HandoffNoteConfig, + Algorithm, CallModel, ClassifierContractConfig, ClassifierResponseFormat, HandoffNoteConfig, LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, StageRouter, StageRouterConfig, Step as RustStep, StepStream, TaskClassifierConfig, @@ -68,7 +68,8 @@ impl PyTaskClassifierConfig { message_hash_fallback=false, recent_turn_window=None, max_output_tokens=4096, - prompt=None + prompt=None, + response_format_type="json_schema" ))] #[allow(clippy::too_many_arguments)] fn new( @@ -79,12 +80,23 @@ impl PyTaskClassifierConfig { recent_turn_window: Option, max_output_tokens: u64, prompt: Option, - ) -> Self { + response_format_type: &str, + ) -> PyResult { let mut contract = ClassifierContractConfig::default(); if let Some(prompt) = prompt { contract = contract.with_prompt(prompt); } - Self { + let response_format_type = match response_format_type { + "json_schema" => ClassifierResponseFormat::JsonSchema, + "json_object" => ClassifierResponseFormat::JsonObject, + other => { + return Err(PyValueError::new_err(format!( + "response_format_type must be 'json_schema' or 'json_object', got {other:?}" + ))); + } + }; + contract = contract.with_response_format_type(response_format_type); + Ok(Self { inner: TaskClassifierConfig { base_threshold, threshold_step, @@ -94,7 +106,7 @@ impl PyTaskClassifierConfig { contract, max_output_tokens, }, - } + }) } } diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index d2e59fa1..cdab0534 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -9,10 +9,10 @@ use std::path::Path; use std::sync::Arc; use libsy::{ - Algorithm, ClassifierContractConfig, CustomClassifierConfig, CustomClassifierPolicy, - EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, - Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts, - TaskClassifierConfig, + Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig, + CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, + LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, + StageRouterConfig, TargetPrompts, TaskClassifierConfig, }; use serde::Deserialize; use serde_json::Value; @@ -297,6 +297,7 @@ struct CapabilityClassifierRouteConfig { message_hash_fallback: bool, recent_turn_window: Option, prompt: Option, + response_format_type: ClassifierResponseFormat, max_output_tokens: u64, } @@ -305,6 +306,7 @@ struct EscalationClassifierRouteConfig { strong_target: String, weak_target: String, prompt: Option, + response_format_type: ClassifierResponseFormat, max_output_tokens: u64, judge: EscalationJudgeConfig, } @@ -383,6 +385,8 @@ enum RouteConfig { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_classifier_max_output_tokens")] max_output_tokens: u64, #[serde(default)] @@ -443,6 +447,8 @@ struct StageClassifierConfig { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_classifier_max_output_tokens")] max_output_tokens: u64, } @@ -455,7 +461,8 @@ impl StageClassifierConfig { session_affinity: self.session_affinity, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, - contract: classifier_contract(self.prompt.as_deref()), + contract: classifier_contract(self.prompt.as_deref()) + .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, } } @@ -581,6 +588,7 @@ impl RouteConfig { message_hash_fallback, recent_turn_window, prompt, + response_format_type, max_output_tokens, escalation, targets, @@ -638,6 +646,7 @@ impl RouteConfig { message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, prompt: prompt.clone(), + response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, }, )) @@ -675,6 +684,7 @@ impl RouteConfig { weak_target, )?, prompt: prompt.clone(), + response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, judge: required_classifier_field(route_name, "escalation", escalation)?, }, @@ -686,6 +696,7 @@ impl RouteConfig { || base_threshold.is_some() || threshold_step.is_some() || escalation.is_some() + || *response_format_type != ClassifierResponseFormat::JsonSchema { return Err(ServerError::new(format!( "llm_classifier route {route_name} mode custom cannot use capability or escalation fields" @@ -850,7 +861,8 @@ fn build_algorithm( session_affinity: config.session_affinity, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, - contract: classifier_contract(config.prompt.as_deref()), + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, }; LlmTaskClassifier::new(LlmClassifierConfig::Capability { @@ -868,7 +880,8 @@ fn build_algorithm( judge_target: classifier, efficient_target: weak, capable_target: strong, - contract: classifier_contract(config.prompt.as_deref()), + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), config: config.judge, max_output_tokens: config.max_output_tokens, }) @@ -1158,7 +1171,10 @@ target = "weak" "base_threshold = 0.5", "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"", ); - assert!(error_message(&schema_placeholder).contains("schema is sent separately")); + assert!( + error_message(&schema_placeholder) + .contains("Switchyard supplies the schema automatically") + ); Ok(()) } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index a1241c59..c6b1ac9b 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -164,6 +164,13 @@ async fn upstream_chat( .is_some_and(|content| content.contains("invalid verdict")) }) }); + let requests_schema_invalid_verdict = body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"] + .as_str() + .is_some_and(|content| content.contains("schema-invalid verdict")) + }) + }); let content = if model == "model/classifier" && custom_target_schema { if requests_invalid_verdict { r#"{"decision":{"target":"unknown"}}"# @@ -176,6 +183,8 @@ async fn upstream_chat( .is_some() { r#"{"escalate":false,"reason":"making progress"}"# + } else if model == "model/classifier" && requests_schema_invalid_verdict { + r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"# } else if model == "model/classifier" { r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"# } else { @@ -1026,7 +1035,7 @@ selector = "/decision/target" } #[tokio::test] -async fn classifier_prompt_overrides_reach_every_server_mode() -> TestResult { +async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; let state = load_test_config(&format!( r#" @@ -1057,6 +1066,7 @@ strong_target = "strong" weak_target = "weak" base_threshold = 0.5 prompt = "CUSTOM CAPABILITY" +response_format_type = "json_object" [routes.escalation] id = "switchyard/escalation" @@ -1066,6 +1076,7 @@ classifier_target = "classifier" strong_target = "strong" weak_target = "weak" prompt = "CUSTOM ESCALATION" +response_format_type = "json_object" escalation = {{ confirmations = 1 }} [routes.stage] @@ -1085,10 +1096,20 @@ prompt = "CUSTOM STAGE" ))?; let app = build_switchyard_router(state); - for (route, prompt_prefix, schema_field) in [ - ("switchyard/capability", "CUSTOM CAPABILITY", "p_solve"), - ("switchyard/escalation", "CUSTOM ESCALATION", "escalate"), - ("switchyard/stage", "CUSTOM STAGE", "p_solve"), + for (route, prompt_prefix, schema_field, json_object) in [ + ( + "switchyard/capability", + "CUSTOM CAPABILITY", + "p_solve", + true, + ), + ( + "switchyard/escalation", + "CUSTOM ESCALATION", + "escalate", + true, + ), + ("switchyard/stage", "CUSTOM STAGE", "p_solve", false), ] { upstream.calls.lock().await.clear(); let response = send( @@ -1112,16 +1133,118 @@ prompt = "CUSTOM STAGE" .as_str() .ok_or("classifier prompt was not text")?; assert!(prompt.starts_with(prompt_prefix), "{route}: {prompt}"); - assert!( - judge_call["response_format"]["json_schema"]["schema"]["properties"] - .get(schema_field) - .is_some(), - "{route}: missing {schema_field} in {judge_call}" - ); + if json_object { + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}), + "{route}: {judge_call}" + ); + assert!(prompt.contains("JSON Schema"), "{route}: {prompt}"); + assert!( + prompt.contains(&format!("\"{schema_field}\"")), + "{route}: missing {schema_field} in {prompt}" + ); + } else { + assert!( + judge_call["response_format"]["json_schema"]["schema"]["properties"] + .get(schema_field) + .is_some(), + "{route}: missing {schema_field} in {judge_call}" + ); + } } Ok(()) } +#[tokio::test] +async fn stage_classifier_can_request_json_object_output() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 + +[routes.stage.classifier] +target = "classifier" +base_threshold = 0.5 +response_format_type = "json_object" +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [{"role": "user", "content": "bounded task"}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("classifier target was not called")?; + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}) + ); + let prompt = judge_call["messages"][0]["content"] + .as_str() + .ok_or("classifier prompt was not text")?; + assert!(prompt.contains("JSON Schema"), "{prompt}"); + assert!(prompt.contains("\"p_solve\""), "{prompt}"); + + drop(calls); + let invalid_response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [{"role": "user", "content": "return a schema-invalid verdict"}] + })), + ) + .await?; + assert_eq!(invalid_response.status, StatusCode::OK); + assert_eq!( + invalid_response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/weak") + ); + Ok(()) +} + #[tokio::test] async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 6db279d8..2313e469 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -118,6 +118,7 @@ Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Yes | — | Target the judge is called through. Not a routing destination. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | +| `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | Capability mode classifies before serving. See [LLM Classifier Routing](../routing_algorithms/llm_classifier_routing.md). @@ -161,8 +162,9 @@ policy selector, and routes to any configured target label. | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `session_affinity`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard sends the -schema only through the provider's structured-output request. +Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies +the schema automatically: through the structured-output request in `json_schema` +mode, or in the prompt in `json_object` mode. ### `stage_router` @@ -179,6 +181,7 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | | `capable_system_prompt` | No | unset | System prompt handed to the capable tier. | | `efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. | +| `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | ## Validation Errors diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index e2246144..1ed9e808 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -48,8 +48,9 @@ clients send; the judge is not exposed as a client-selectable model. The route-level `prompt` key replaces the packaged trajectory-judge prompt. It uses the escalation verdict schema rather than the capability verdict schema. -Switchyard sends that schema separately through the provider's structured-output -request rather than copying it into the prompt. +Switchyard supplies that schema according to the route's `response_format_type`: +through the structured-output request in the default `json_schema` mode, or in +the prompt in `json_object` mode. ## How the decision works diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index e3f75097..a1cc43e4 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -78,6 +78,11 @@ Switchyard does not parse provider-specific reasoning fields such as to `strong_target` even when the judge request returned HTTP 200. With session affinity, that fallback can be reused without another judge call. +Capability and escalation routes use JSON Schema structured output by default. +For a provider that supports JSON Object mode but not JSON Schema, set +`response_format_type = "json_object"` on the route. Switchyard then adds the +verdict schema to the judge prompt and validates the returned object locally. + When a vLLM-compatible provider supports `enable_thinking`, configure it on the judge target through `extra_body`: @@ -104,13 +109,14 @@ for the server merge behavior. | `session_affinity` | `false` | Retains the first selected target for a session and reuses it on later requests. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `session_affinity = true`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | +| `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. | | `max_output_tokens` | `4096` | Maximum completion tokens available to the classifier verdict. Must be at least `1`. | ### Override the classifier prompt Set `prompt` on the route when the packaged capability rubric does not describe -your weak model. Switchyard sends the response schema separately through the -provider's structured-output request; do not copy it into the prompt. +your weak model. Do not copy the response schema into the prompt: Switchyard +supplies it according to `response_format_type`. ```toml [routes.smart] diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index c471a988..7bf0da7b 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -245,11 +245,14 @@ base_threshold = 0.5 # p_solve floor to route efficient; below this → ca threshold_step = 0.1 # adds 0.1 for uncertain and 0.2 for unsupported verdicts recent_turn_window = 3 # conversation span the judge sees prompt = "Estimate whether the efficient target can complete this request." +response_format_type = "json_object" # optional; default is "json_schema" ``` -`prompt` replaces the packaged capability-classifier prompt. The active schema -is sent separately through the structured-output request. The verdict schema -and routing thresholds remain unchanged. +`prompt` replaces the packaged capability-classifier prompt. In the default +`json_schema` mode, Switchyard sends the verdict schema through the structured-output +request. Set `response_format_type = "json_object"` for providers that only support +JSON Object mode; Switchyard then adds the schema to the judge prompt and validates +the returned object locally. The verdict schema and routing thresholds remain unchanged. Give the classifier its own LLM client or quota bucket where possible. Sharing one provider bucket with the efficient tier adds a request per classified turn diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 244b1dd0..fa4cef86 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -92,6 +92,7 @@ def __init__( recent_turn_window: int | None = None, max_output_tokens: int = 4096, prompt: str | None = None, + response_format_type: Literal["json_schema", "json_object"] = "json_schema", ) -> None: ... @final diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index eb1d6c8c..70f878b6 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -181,6 +181,66 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +async def test_classifier_config_accepts_json_object_output() -> None: + """Verify that Python can select JSON Object mode for a classifier judge.""" + + class JudgeClient(EchoClient): + async def call(self, request: dict[str, Any]) -> dict[str, Any]: + self.calls.append(request) + return { + "model": self.model, + "outputs": [ + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": ( + '{"crux":"bounded task","primary_rule":"SUP-1",' + '"capability_boundary":"supported","p_solve":0.9}' + ), + } + ], + "stop_reason": "end_turn", + } + ], + } + + judge = JudgeClient("judge") + weak = EchoClient("weak") + algorithm = algorithms.llm_task_classifier( + "judge", + "weak", + "strong", + config=TaskClassifierConfig(0.5, response_format_type="json_object"), + ) + + _, response = await run_algorithm( + algorithm, + { + "judge": judge, + "weak": weak, + "strong": EchoClient("strong"), + }, + ) + + assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"} + prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] + assert "JSON Schema" in prompt + assert '"p_solve"' in prompt + assert response["model"] == "weak" + + +def test_classifier_config_rejects_unknown_response_format() -> None: + invalid_response_format: Any = "yaml" + + with pytest.raises( + ValueError, + match="response_format_type must be 'json_schema' or 'json_object'", + ): + TaskClassifierConfig(0.5, response_format_type=invalid_response_format) + + async def test_random_weights_and_seed_are_reproducible() -> None: def algorithm(): return algorithms.random(