From fe1e24e160331dd4af9bc62bb265c37cf6844abc Mon Sep 17 00:00:00 2001 From: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:00:52 +0800 Subject: [PATCH 1/4] fix: support json_object classifier responses Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> --- crates/libsy/src/algorithms/llm_class.rs | 7 +- .../algorithms/util/classifier_contract.rs | 97 ++++++++++++++---- crates/libsy/src/algorithms/util/llm_judge.rs | 60 +++++++++++- crates/libsy/src/lib.rs | 4 +- crates/switchyard-py/src/libsy_bindings.rs | 26 +++-- crates/switchyard-server/src/config.rs | 18 ++-- crates/switchyard-server/tests/server.rs | 98 +++++++++++++++++++ docs/reference/toml_schema.md | 1 + .../stage_router_routing.md | 9 +- tests/test_libsy_minimal_bindings.py | 42 ++++++++ 10 files changed, 323 insertions(+), 39 deletions(-) diff --git a/crates/libsy/src/algorithms/llm_class.rs b/crates/libsy/src/algorithms/llm_class.rs index b60ebea63..502921d4b 100644 --- a/crates/libsy/src/algorithms/llm_class.rs +++ b/crates/libsy/src/algorithms/llm_class.rs @@ -14,7 +14,9 @@ use switchyard_protocol::{ContentBlock, Decision, Message, ModelId, Role}; use super::fall_through::{DefaultTarget, FallThrough}; use super::util::DEFAULT_JUDGE_MAX_OUTPUT_TOKENS; use super::util::affinity::AffinityRouter; -use super::util::classifier_contract::{ClassifierContract, ClassifierContractConfig}; +use super::util::classifier_contract::{ + ClassifierContract, ClassifierContractConfig, ClassifierResponseFormat, +}; use super::util::escalation::{self, EscalationJudge, EscalationJudgeConfig, EscalationPolicy}; use super::util::llm_judge::{ ClassifierInput, JsonSchemaDecoder, JudgeClassifier, JudgePolicy, JudgeRuntimeConfig, @@ -261,6 +263,8 @@ struct TaskClassifierConfigWire { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_judge_max_output_tokens")] max_output_tokens: u64, } @@ -275,6 +279,7 @@ impl<'de> Deserialize<'de> for TaskClassifierConfig { if let Some(prompt) = wire.prompt { contract = contract.with_prompt(prompt); } + contract = contract.with_response_format_type(wire.response_format_type); Ok(Self { base_threshold: wire.base_threshold, threshold_step: wire.threshold_step, diff --git a/crates/libsy/src/algorithms/util/classifier_contract.rs b/crates/libsy/src/algorithms/util/classifier_contract.rs index d3298dcf3..d597ed750 100644 --- a/crates/libsy/src/algorithms/util/classifier_contract.rs +++ b/crates/libsy/src/algorithms/util/classifier_contract.rs @@ -9,6 +9,17 @@ use serde_json::{Value, json}; use crate::{LibsyError, Result}; +/// Provider-side structured-output mode used by a classifier judge. +#[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum ClassifierResponseFormat { + /// Send the verdict schema through the provider's strict JSON Schema wrapper. + #[default] + JsonSchema, + /// Request a JSON object and enforce the verdict schema locally. + JsonObject, +} + /// User-configurable parts of a classifier's prompt and verdict contract. /// /// Fields are private so new contract settings can be added without breaking Rust struct literals. @@ -16,6 +27,8 @@ use crate::{LibsyError, Result}; pub struct ClassifierContractConfig { #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, } impl ClassifierContractConfig { @@ -29,6 +42,20 @@ impl ClassifierContractConfig { pub fn prompt(&self) -> Option<&str> { self.prompt.as_deref() } + + /// Selects the provider-side structured-output mode. + pub fn with_response_format_type( + mut self, + response_format_type: ClassifierResponseFormat, + ) -> Self { + self.response_format_type = response_format_type; + self + } + + /// Returns the configured provider-side structured-output mode. + pub fn response_format_type(&self) -> ClassifierResponseFormat { + self.response_format_type + } } /// Rendered prompt and response format for one classifier. @@ -42,8 +69,9 @@ pub(crate) struct ClassifierContract { impl ClassifierContract { /// Builds a contract from user settings and packaged defaults. /// - /// The response format must contain `json_schema.schema` and is retained separately for the - /// model request. Schemas are never copied into the system prompt. + /// The packaged response format must contain `json_schema.schema`. JSON Schema mode retains + /// that wrapper for the model request; JSON Object mode moves the schema into the prompt and + /// compiles it for local validation. pub(crate) fn from_config( config: &ClassifierContractConfig, default_prompt: &str, @@ -56,7 +84,31 @@ impl ClassifierContract { message: format!("response schema is invalid: {error}"), } })?; - Self::from_response_format(prompt_template, response_format, None) + let schema = response_format + .pointer("/json_schema/schema") + .ok_or_else(|| LibsyError::AlgorithmError { + message: "response schema has no json_schema.schema".to_string(), + })?; + match config.response_format_type() { + ClassifierResponseFormat::JsonSchema => { + Self::from_response_format(prompt_template, response_format, None) + } + ClassifierResponseFormat::JsonObject => { + validate_prompt(prompt_template)?; + let validator = compile_schema(schema)?; + let rendered_schema = serde_json::to_string_pretty(schema).map_err(|error| { + algorithm_error(format!("response schema could not be rendered: {error}")) + })?; + let system_prompt = format!( + "{prompt_template}\n\nReturn exactly one JSON object matching this JSON Schema:\n{rendered_schema}" + ); + Self::from_response_format( + &system_prompt, + json!({"type": "json_object"}), + Some(validator), + ) + } + } } /// Builds a provider response format around a user-supplied inner JSON Schema. @@ -88,21 +140,7 @@ impl ClassifierContract { response_format: Value, validator: Option, ) -> Result { - if prompt_template.trim().is_empty() { - return Err(LibsyError::AlgorithmError { - message: "classifier prompt must not be empty".to_string(), - }); - } - if prompt_template.contains("{{RESPONSE_SCHEMA}}") { - return Err(LibsyError::AlgorithmError { - message: "classifier prompt must not include {{RESPONSE_SCHEMA}}; the response schema is sent separately".to_string(), - }); - } - response_format - .pointer("/json_schema/schema") - .ok_or_else(|| LibsyError::AlgorithmError { - message: "response schema has no json_schema.schema".to_string(), - })?; + validate_prompt(prompt_template)?; Ok(Self { system_prompt: prompt_template.to_string(), @@ -119,6 +157,11 @@ impl ClassifierContract { &self.response_format } + /// Whether the provider response must be checked against the compiled schema locally. + pub(crate) fn validates_locally(&self) -> bool { + self.validator.is_some() + } + /// Validates a dynamic verdict when this contract carries a runtime schema validator. pub(crate) fn validate_verdict(&self, verdict: &Value) -> Result<()> { let Some(validator) = &self.validator else { @@ -132,6 +175,18 @@ impl ClassifierContract { } } +fn validate_prompt(prompt_template: &str) -> Result<()> { + if prompt_template.trim().is_empty() { + return Err(algorithm_error("classifier prompt must not be empty")); + } + if prompt_template.contains("{{RESPONSE_SCHEMA}}") { + return Err(algorithm_error( + "classifier prompt must not include {{RESPONSE_SCHEMA}}; remove the placeholder because Switchyard supplies the schema automatically", + )); + } + Ok(()) +} + fn compile_schema(schema: &Value) -> Result { if !schema.is_object() { return Err(algorithm_error("response_schema must be a JSON object")); @@ -223,7 +278,11 @@ mod tests { ) .expect_err("schema placeholders should be rejected"); - assert!(error.to_string().contains("schema is sent separately")); + assert!( + error + .to_string() + .contains("Switchyard supplies the schema automatically") + ); } #[test] diff --git a/crates/libsy/src/algorithms/util/llm_judge.rs b/crates/libsy/src/algorithms/util/llm_judge.rs index bb271ac5a..6661cee1f 100644 --- a/crates/libsy/src/algorithms/util/llm_judge.rs +++ b/crates/libsy/src/algorithms/util/llm_judge.rs @@ -62,9 +62,19 @@ where fn decode( &self, response: &AggLlmResponse, - _contract: &ClassifierContract, + contract: &ClassifierContract, ) -> Result { - parse_json_verdict(response) + if !contract.validates_locally() { + return parse_json_verdict(response); + } + let verdict = parse_json_verdict::(response)?; + contract.validate_verdict(&verdict)?; + serde_json::from_value(verdict).map_err(|error| LibsyError::AlgorithmError { + message: format!( + "judge reply did not parse as {}: {error}", + std::any::type_name::() + ), + }) } } @@ -353,6 +363,11 @@ mod tests { ok: bool, } + #[derive(Debug, Deserialize, PartialEq)] + struct ScoreVerdict { + score: f64, + } + struct TestJudge; impl Judge for TestJudge { @@ -416,6 +431,47 @@ mod tests { Ok(()) } + #[test] + fn typed_decoder_enforces_a_json_object_contract_locally() -> Result<()> { + use super::super::classifier_contract::{ + ClassifierContractConfig, ClassifierResponseFormat, + }; + + let config = ClassifierContractConfig::default() + .with_response_format_type(ClassifierResponseFormat::JsonObject); + let contract = ClassifierContract::from_config( + &config, + "Return one JSON score.", + r#"{ + "type": "json_schema", + "json_schema": { + "name": "ScoreVerdict", + "schema": { + "type": "object", + "properties": {"score": {"type": "number"}}, + "required": ["score"], + "additionalProperties": false + } + } + }"#, + )?; + let decoder = SerdeDecoder::::new(); + + let error = decoder + .decode( + &text_response(None, r#"{"score":0.5,"unexpected":true}"#), + &contract, + ) + .expect_err("an extra property should fail the local schema"); + + assert!(error.to_string().contains("did not match response_schema")); + assert_eq!( + decoder.decode(&text_response(None, r#"{"score":0.5}"#), &contract)?, + ScoreVerdict { score: 0.5 } + ); + Ok(()) + } + fn buffered(completion: &str) -> Response { Response { llm_response: LlmResponse::Agg(text_response(None, completion)), diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 3b1ede9ac..583ce1f13 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -23,7 +23,9 @@ pub use algorithms::passthrough::Passthrough; pub use algorithms::rand::{Random, RandomClassifier}; pub use algorithms::stage::{LlmFallback, StageRouter, StageRouterConfig}; pub use algorithms::util::affinity::AffinityRouter; -pub use algorithms::util::classifier_contract::ClassifierContractConfig; +pub use algorithms::util::classifier_contract::{ + ClassifierContractConfig, ClassifierResponseFormat, +}; pub use algorithms::util::escalation::EscalationJudgeConfig; pub use algorithms::util::prompts::{SystemPromptProcessor, TargetPrompts, append_note}; pub use algorithms::util::subagent::SubagentOverride; diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index d7ca40399..337e95954 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -12,9 +12,9 @@ use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use serde_json::{Value, json}; use switchyard_libsy::{ - Algorithm, ClassifierContractConfig, HandoffNoteConfig, LibsyError as RustLibsyError, - LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, PickerMode, Random, StageRouter, - StageRouterConfig, TaskClassifierConfig, + Algorithm, ClassifierContractConfig, ClassifierResponseFormat, HandoffNoteConfig, + LibsyError as RustLibsyError, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, Noop, + PickerMode, Random, StageRouter, StageRouterConfig, TaskClassifierConfig, }; use switchyard_llm_client::ClientRouter; use switchyard_protocol::{ @@ -147,7 +147,8 @@ impl PyTaskClassifierConfig { message_hash_fallback=false, recent_turn_window=None, max_output_tokens=4096, - prompt=None + prompt=None, + response_format_type="json_schema" ))] #[allow(clippy::too_many_arguments)] fn new( @@ -158,12 +159,23 @@ impl PyTaskClassifierConfig { recent_turn_window: Option, max_output_tokens: u64, prompt: Option, - ) -> Self { + response_format_type: &str, + ) -> PyResult { let mut contract = ClassifierContractConfig::default(); if let Some(prompt) = prompt { contract = contract.with_prompt(prompt); } - Self { + let response_format_type = match response_format_type { + "json_schema" => ClassifierResponseFormat::JsonSchema, + "json_object" => ClassifierResponseFormat::JsonObject, + other => { + return Err(PyValueError::new_err(format!( + "response_format_type must be 'json_schema' or 'json_object', got {other:?}" + ))); + } + }; + contract = contract.with_response_format_type(response_format_type); + Ok(Self { inner: TaskClassifierConfig { base_threshold, threshold_step, @@ -173,7 +185,7 @@ impl PyTaskClassifierConfig { contract, max_output_tokens, }, - } + }) } } diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index d2e59fa18..50b709628 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -9,10 +9,10 @@ use std::path::Path; use std::sync::Arc; use libsy::{ - Algorithm, ClassifierContractConfig, CustomClassifierConfig, CustomClassifierPolicy, - EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, LlmFallback, LlmTaskClassifier, - Noop, Passthrough, PickerMode, Random, StageRouter, StageRouterConfig, TargetPrompts, - TaskClassifierConfig, + Algorithm, ClassifierContractConfig, ClassifierResponseFormat, CustomClassifierConfig, + CustomClassifierPolicy, EscalationJudgeConfig, HandoffNoteConfig, LlmClassifierConfig, + LlmFallback, LlmTaskClassifier, Noop, Passthrough, PickerMode, Random, StageRouter, + StageRouterConfig, TargetPrompts, TaskClassifierConfig, }; use serde::Deserialize; use serde_json::Value; @@ -443,6 +443,8 @@ struct StageClassifierConfig { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_classifier_max_output_tokens")] max_output_tokens: u64, } @@ -455,7 +457,8 @@ impl StageClassifierConfig { session_affinity: self.session_affinity, message_hash_fallback: self.message_hash_fallback, recent_turn_window: self.recent_turn_window, - contract: classifier_contract(self.prompt.as_deref()), + contract: classifier_contract(self.prompt.as_deref()) + .with_response_format_type(self.response_format_type), max_output_tokens: self.max_output_tokens, } } @@ -1158,7 +1161,10 @@ target = "weak" "base_threshold = 0.5", "base_threshold = 0.5\nprompt = \"{{RESPONSE_SCHEMA}}\"", ); - assert!(error_message(&schema_placeholder).contains("schema is sent separately")); + assert!( + error_message(&schema_placeholder) + .contains("Switchyard supplies the schema automatically") + ); Ok(()) } diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index a1241c59d..20098eab2 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -164,6 +164,13 @@ async fn upstream_chat( .is_some_and(|content| content.contains("invalid verdict")) }) }); + let requests_schema_invalid_verdict = body["messages"].as_array().is_some_and(|messages| { + messages.iter().any(|message| { + message["content"] + .as_str() + .is_some_and(|content| content.contains("schema-invalid verdict")) + }) + }); let content = if model == "model/classifier" && custom_target_schema { if requests_invalid_verdict { r#"{"decision":{"target":"unknown"}}"# @@ -176,6 +183,8 @@ async fn upstream_chat( .is_some() { r#"{"escalate":false,"reason":"making progress"}"# + } else if model == "model/classifier" && requests_schema_invalid_verdict { + r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.1,"unexpected":true}"# } else if model == "model/classifier" { r#"{"crux":"bounded task","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"# } else { @@ -1122,6 +1131,95 @@ prompt = "CUSTOM STAGE" Ok(()) } +#[tokio::test] +async fn stage_classifier_can_request_json_object_output() -> TestResult { + let upstream = MockUpstream::start().await?; + let state = load_test_config(&format!( + r#" +schema_version = 1 + +[llm_clients.upstream] +format = "openai_chat" +base_url = "{base_url}" + +[targets.classifier] +id = "model/classifier" +llm_client = "upstream" + +[targets.strong] +id = "model/strong" +llm_client = "upstream" + +[targets.weak] +id = "model/weak" +llm_client = "upstream" + +[routes.stage] +id = "switchyard/stage" +type = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 1.0 + +[routes.stage.classifier] +target = "classifier" +base_threshold = 0.5 +response_format_type = "json_object" +"#, + base_url = upstream.base_url + ))?; + let app = build_switchyard_router(state); + + let response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [{"role": "user", "content": "bounded task"}] + })), + ) + .await?; + + assert_eq!(response.status, StatusCode::OK); + let calls = upstream.calls.lock().await; + let judge_call = calls + .iter() + .find(|call| call["model"] == "model/classifier") + .ok_or("classifier target was not called")?; + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}) + ); + let prompt = judge_call["messages"][0]["content"] + .as_str() + .ok_or("classifier prompt was not text")?; + assert!(prompt.contains("JSON Schema"), "{prompt}"); + assert!(prompt.contains("\"p_solve\""), "{prompt}"); + + drop(calls); + let invalid_response = send( + &app, + "POST", + "/v1/chat/completions", + Some(json!({ + "model": "switchyard/stage", + "messages": [{"role": "user", "content": "return a schema-invalid verdict"}] + })), + ) + .await?; + assert_eq!(invalid_response.status, StatusCode::OK); + assert_eq!( + invalid_response + .headers + .get("x-model-router-selected-model") + .and_then(|value| value.to_str().ok()), + Some("model/weak") + ); + Ok(()) +} + #[tokio::test] async fn count_tokens_forwards_to_configured_anthropic_target() -> TestResult { let upstream = MockUpstream::start().await?; diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 6db279d80..39aa3b278 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -179,6 +179,7 @@ optional `handoff_notes` and `classifier` tables and for tuning. | `recent_turn_window` | No | `3` | Trailing tool results the signals are computed over. | | `capable_system_prompt` | No | unset | System prompt handed to the capable tier. | | `efficient_system_prompt` | No | unset | System prompt handed to the efficient tier. | +| `classifier.response_format_type` | No | `json_schema` | Structured-output mode for the optional classifier judge. Use `json_object` when the classifier provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. | ## Validation Errors diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index c471a9882..7bf0da7bb 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -245,11 +245,14 @@ base_threshold = 0.5 # p_solve floor to route efficient; below this → ca threshold_step = 0.1 # adds 0.1 for uncertain and 0.2 for unsupported verdicts recent_turn_window = 3 # conversation span the judge sees prompt = "Estimate whether the efficient target can complete this request." +response_format_type = "json_object" # optional; default is "json_schema" ``` -`prompt` replaces the packaged capability-classifier prompt. The active schema -is sent separately through the structured-output request. The verdict schema -and routing thresholds remain unchanged. +`prompt` replaces the packaged capability-classifier prompt. In the default +`json_schema` mode, Switchyard sends the verdict schema through the structured-output +request. Set `response_format_type = "json_object"` for providers that only support +JSON Object mode; Switchyard then adds the schema to the judge prompt and validates +the returned object locally. The verdict schema and routing thresholds remain unchanged. Give the classifier its own LLM client or quota bucket where possible. Sharing one provider bucket with the efficient tier adds a request per classified turn diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 059ccf0aa..bb02162c1 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -110,6 +110,48 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +async def test_classifier_config_accepts_json_object_output() -> None: + """Verify that Python can select JSON Object mode for a classifier judge.""" + + class JudgeClient(EchoClient): + async def call(self, request: dict[str, Any]) -> dict[str, Any]: + self.calls.append(request) + return { + "model": self.model, + "outputs": [ + { + "role": "assistant", + "content": [ + { + "type": "text", + "text": ( + '{"crux":"bounded task","primary_rule":"SUP-1",' + '"capability_boundary":"supported","p_solve":0.9}' + ), + } + ], + "stop_reason": "end_turn", + } + ], + } + + judge = JudgeClient("judge") + algorithm = algorithms.llm_task_classifier( + LlmTarget("judge", judge), + LlmTarget("weak", EchoClient("weak")), + LlmTarget("strong", EchoClient("strong")), + config=TaskClassifierConfig(0.5, response_format_type="json_object"), + ) + + _, response = await algorithm.run(request_body()) + + assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"} + prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] + assert "JSON Schema" in prompt + assert '"p_solve"' in prompt + assert response["model"] == "weak" + + async def test_random_weights_and_seed_are_reproducible() -> None: def algorithm(): return algorithms.random( From ec2585d0610c09a78f7c89f53a95b4502b57a404 Mon Sep 17 00:00:00 2001 From: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> Date: Fri, 14 Aug 2026 02:58:41 +0800 Subject: [PATCH 2/4] fix(python): sync classifier response format signature Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> --- switchyard_rust/libsy.py | 1 + tests/test_libsy_minimal_bindings.py | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 244b1dd02..667c86f8e 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -92,6 +92,7 @@ def __init__( recent_turn_window: int | None = None, max_output_tokens: int = 4096, prompt: str | None = None, + response_format_type: str = "json_schema", ) -> None: ... @final diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index e5df7b880..699d6121f 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -207,14 +207,22 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: } judge = JudgeClient("judge") + weak = EchoClient("weak") algorithm = algorithms.llm_task_classifier( - LlmTarget("judge", judge), - LlmTarget("weak", EchoClient("weak")), - LlmTarget("strong", EchoClient("strong")), + "judge", + "weak", + "strong", config=TaskClassifierConfig(0.5, response_format_type="json_object"), ) - _, response = await algorithm.run(request_body()) + _, response = await run_algorithm( + algorithm, + { + "judge": judge, + "weak": weak, + "strong": EchoClient("strong"), + }, + ) assert judge.calls[0]["output"]["response_format"] == {"type": "json_object"} prompt = judge.calls[0]["instructions"][0]["content"][0]["text"] From c5d6d3f03ba58d5c632269feee15b1977b685e34 Mon Sep 17 00:00:00 2001 From: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> Date: Fri, 14 Aug 2026 20:34:19 +0800 Subject: [PATCH 3/4] fix(server): expose classifier response format Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> --- crates/switchyard-server/src/config.rs | 14 +++++- crates/switchyard-server/tests/server.rs | 47 ++++++++++++++----- docs/reference/toml_schema.md | 6 ++- .../escalation_router_routing.md | 5 +- .../llm_classifier_routing.md | 10 +++- 5 files changed, 63 insertions(+), 19 deletions(-) diff --git a/crates/switchyard-server/src/config.rs b/crates/switchyard-server/src/config.rs index 50b709628..cdab05349 100644 --- a/crates/switchyard-server/src/config.rs +++ b/crates/switchyard-server/src/config.rs @@ -297,6 +297,7 @@ struct CapabilityClassifierRouteConfig { message_hash_fallback: bool, recent_turn_window: Option, prompt: Option, + response_format_type: ClassifierResponseFormat, max_output_tokens: u64, } @@ -305,6 +306,7 @@ struct EscalationClassifierRouteConfig { strong_target: String, weak_target: String, prompt: Option, + response_format_type: ClassifierResponseFormat, max_output_tokens: u64, judge: EscalationJudgeConfig, } @@ -383,6 +385,8 @@ enum RouteConfig { recent_turn_window: Option, #[serde(default)] prompt: Option, + #[serde(default)] + response_format_type: ClassifierResponseFormat, #[serde(default = "default_classifier_max_output_tokens")] max_output_tokens: u64, #[serde(default)] @@ -584,6 +588,7 @@ impl RouteConfig { message_hash_fallback, recent_turn_window, prompt, + response_format_type, max_output_tokens, escalation, targets, @@ -641,6 +646,7 @@ impl RouteConfig { message_hash_fallback: *message_hash_fallback, recent_turn_window: *recent_turn_window, prompt: prompt.clone(), + response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, }, )) @@ -678,6 +684,7 @@ impl RouteConfig { weak_target, )?, prompt: prompt.clone(), + response_format_type: *response_format_type, max_output_tokens: *max_output_tokens, judge: required_classifier_field(route_name, "escalation", escalation)?, }, @@ -689,6 +696,7 @@ impl RouteConfig { || base_threshold.is_some() || threshold_step.is_some() || escalation.is_some() + || *response_format_type != ClassifierResponseFormat::JsonSchema { return Err(ServerError::new(format!( "llm_classifier route {route_name} mode custom cannot use capability or escalation fields" @@ -853,7 +861,8 @@ fn build_algorithm( session_affinity: config.session_affinity, message_hash_fallback: config.message_hash_fallback, recent_turn_window: config.recent_turn_window, - contract: classifier_contract(config.prompt.as_deref()), + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), max_output_tokens: config.max_output_tokens, }; LlmTaskClassifier::new(LlmClassifierConfig::Capability { @@ -871,7 +880,8 @@ fn build_algorithm( judge_target: classifier, efficient_target: weak, capable_target: strong, - contract: classifier_contract(config.prompt.as_deref()), + contract: classifier_contract(config.prompt.as_deref()) + .with_response_format_type(config.response_format_type), config: config.judge, max_output_tokens: config.max_output_tokens, }) diff --git a/crates/switchyard-server/tests/server.rs b/crates/switchyard-server/tests/server.rs index 20098eab2..c6b1ac9b3 100644 --- a/crates/switchyard-server/tests/server.rs +++ b/crates/switchyard-server/tests/server.rs @@ -1035,7 +1035,7 @@ selector = "/decision/target" } #[tokio::test] -async fn classifier_prompt_overrides_reach_every_server_mode() -> TestResult { +async fn classifier_contract_overrides_reach_every_server_mode() -> TestResult { let upstream = MockUpstream::start().await?; let state = load_test_config(&format!( r#" @@ -1066,6 +1066,7 @@ strong_target = "strong" weak_target = "weak" base_threshold = 0.5 prompt = "CUSTOM CAPABILITY" +response_format_type = "json_object" [routes.escalation] id = "switchyard/escalation" @@ -1075,6 +1076,7 @@ classifier_target = "classifier" strong_target = "strong" weak_target = "weak" prompt = "CUSTOM ESCALATION" +response_format_type = "json_object" escalation = {{ confirmations = 1 }} [routes.stage] @@ -1094,10 +1096,20 @@ prompt = "CUSTOM STAGE" ))?; let app = build_switchyard_router(state); - for (route, prompt_prefix, schema_field) in [ - ("switchyard/capability", "CUSTOM CAPABILITY", "p_solve"), - ("switchyard/escalation", "CUSTOM ESCALATION", "escalate"), - ("switchyard/stage", "CUSTOM STAGE", "p_solve"), + for (route, prompt_prefix, schema_field, json_object) in [ + ( + "switchyard/capability", + "CUSTOM CAPABILITY", + "p_solve", + true, + ), + ( + "switchyard/escalation", + "CUSTOM ESCALATION", + "escalate", + true, + ), + ("switchyard/stage", "CUSTOM STAGE", "p_solve", false), ] { upstream.calls.lock().await.clear(); let response = send( @@ -1121,12 +1133,25 @@ prompt = "CUSTOM STAGE" .as_str() .ok_or("classifier prompt was not text")?; assert!(prompt.starts_with(prompt_prefix), "{route}: {prompt}"); - assert!( - judge_call["response_format"]["json_schema"]["schema"]["properties"] - .get(schema_field) - .is_some(), - "{route}: missing {schema_field} in {judge_call}" - ); + if json_object { + assert_eq!( + judge_call["response_format"], + json!({"type": "json_object"}), + "{route}: {judge_call}" + ); + assert!(prompt.contains("JSON Schema"), "{route}: {prompt}"); + assert!( + prompt.contains(&format!("\"{schema_field}\"")), + "{route}: missing {schema_field} in {prompt}" + ); + } else { + assert!( + judge_call["response_format"]["json_schema"]["schema"]["properties"] + .get(schema_field) + .is_some(), + "{route}: missing {schema_field} in {judge_call}" + ); + } } Ok(()) } diff --git a/docs/reference/toml_schema.md b/docs/reference/toml_schema.md index 39aa3b278..2313e469d 100644 --- a/docs/reference/toml_schema.md +++ b/docs/reference/toml_schema.md @@ -118,6 +118,7 @@ Runs one of three judge-backed modes: `capability`, `escalation`, or `custom`. | `mode` | No | `capability` | Classifier behavior. Set it explicitly for new configurations. | | `classifier_target` | Yes | — | Target the judge is called through. Not a routing destination. | | `max_output_tokens` | No | `4096` | Maximum completion tokens for the judge verdict. Must be at least `1`. | +| `response_format_type` | No | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` when the provider does not support JSON Schema; Switchyard adds the schema to the prompt and validates the verdict locally. Custom mode always uses its configured JSON Schema. | Capability mode classifies before serving. See [LLM Classifier Routing](../routing_algorithms/llm_classifier_routing.md). @@ -161,8 +162,9 @@ policy selector, and routes to any configured target label. | `message_hash_fallback` | No | `false` | Keys affinity on the first user message. Requires `session_affinity`. | | `recent_turn_window` | No | unset | When unset, the judge sees the opening task and latest user follow-up, when present. When set, it also sees trailing turns. | -Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard sends the -schema only through the provider's structured-output request. +Classifier prompts must not contain `{{RESPONSE_SCHEMA}}`. Switchyard supplies +the schema automatically: through the structured-output request in `json_schema` +mode, or in the prompt in `json_object` mode. ### `stage_router` diff --git a/docs/routing_algorithms/escalation_router_routing.md b/docs/routing_algorithms/escalation_router_routing.md index e2246144e..1ed9e8085 100644 --- a/docs/routing_algorithms/escalation_router_routing.md +++ b/docs/routing_algorithms/escalation_router_routing.md @@ -48,8 +48,9 @@ clients send; the judge is not exposed as a client-selectable model. The route-level `prompt` key replaces the packaged trajectory-judge prompt. It uses the escalation verdict schema rather than the capability verdict schema. -Switchyard sends that schema separately through the provider's structured-output -request rather than copying it into the prompt. +Switchyard supplies that schema according to the route's `response_format_type`: +through the structured-output request in the default `json_schema` mode, or in +the prompt in `json_object` mode. ## How the decision works diff --git a/docs/routing_algorithms/llm_classifier_routing.md b/docs/routing_algorithms/llm_classifier_routing.md index e3f750974..a1cc43e45 100644 --- a/docs/routing_algorithms/llm_classifier_routing.md +++ b/docs/routing_algorithms/llm_classifier_routing.md @@ -78,6 +78,11 @@ Switchyard does not parse provider-specific reasoning fields such as to `strong_target` even when the judge request returned HTTP 200. With session affinity, that fallback can be reused without another judge call. +Capability and escalation routes use JSON Schema structured output by default. +For a provider that supports JSON Object mode but not JSON Schema, set +`response_format_type = "json_object"` on the route. Switchyard then adds the +verdict schema to the judge prompt and validates the returned object locally. + When a vLLM-compatible provider supports `enable_thinking`, configure it on the judge target through `extra_body`: @@ -104,13 +109,14 @@ for the server merge behavior. | `session_affinity` | `false` | Retains the first selected target for a session and reuses it on later requests. | | `message_hash_fallback` | `false` | When session metadata is absent, keys affinity from the first user-message text. Requires `session_affinity = true`. | | `prompt` | packaged capability prompt | Replaces the classifier's system prompt. The packaged verdict schema and routing policy remain active. | +| `response_format_type` | `json_schema` | Structured-output mode for capability and escalation judges. Use `json_object` for providers without JSON Schema support. | | `max_output_tokens` | `4096` | Maximum completion tokens available to the classifier verdict. Must be at least `1`. | ### Override the classifier prompt Set `prompt` on the route when the packaged capability rubric does not describe -your weak model. Switchyard sends the response schema separately through the -provider's structured-output request; do not copy it into the prompt. +your weak model. Do not copy the response schema into the prompt: Switchyard +supplies it according to `response_format_type`. ```toml [routes.smart] From 9e3dcbeee4680edff0b7241732ebc7ecfa1d639b Mon Sep 17 00:00:00 2001 From: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> Date: Sat, 15 Aug 2026 02:35:13 +0800 Subject: [PATCH 4/4] fix(py): tighten classifier response format Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com> --- switchyard_rust/libsy.py | 2 +- tests/test_libsy_minimal_bindings.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 667c86f8e..fa4cef868 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -92,7 +92,7 @@ def __init__( recent_turn_window: int | None = None, max_output_tokens: int = 4096, prompt: str | None = None, - response_format_type: str = "json_schema", + response_format_type: Literal["json_schema", "json_object"] = "json_schema", ) -> None: ... @final diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index 699d6121f..70f878b61 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -231,6 +231,16 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: assert response["model"] == "weak" +def test_classifier_config_rejects_unknown_response_format() -> None: + invalid_response_format: Any = "yaml" + + with pytest.raises( + ValueError, + match="response_format_type must be 'json_schema' or 'json_object'", + ): + TaskClassifierConfig(0.5, response_format_type=invalid_response_format) + + async def test_random_weights_and_seed_are_reproducible() -> None: def algorithm(): return algorithms.random(