feat(examples): update LiteLLM for run stream

crates/switchyard-py/src/errors.rs

@@ -8,12 +8,17 @@ use pyo3::exceptions::PyRuntimeError;
 use pyo3::prelude::*;
 
 create_exception!(_switchyard_rust, LibsyError, PyRuntimeError);
+create_exception!(_switchyard_rust, ContextWindowExceededError, PyRuntimeError);
 
 /// Converts libsy execution failures into one stable Python exception.
 pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr {
     LibsyError::new_err(error.to_string())
 }
 
 pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
-    module.add("LibsyError", module.py().get_type::<LibsyError>())
+    module.add("LibsyError", module.py().get_type::<LibsyError>())?;
+    module.add(
+        "ContextWindowExceededError",
+        module.py().get_type::<ContextWindowExceededError>(),
+    )
 }

crates/switchyard-py/src/libsy_bindings.rs

@@ -21,7 +21,7 @@ use switchyard_protocol::{
 };
 use tokio::sync::Mutex;
 
-use crate::errors::py_libsy_error;
+use crate::errors::{ContextWindowExceededError, py_libsy_error};
 use crate::py_serde::{from_python, to_python};
 
 /// Convert Python-owned headers into the request metadata expected by libsy.
@@ -250,8 +250,15 @@ impl PyModelCall {
         }
         let call = self.take()?;
         let target = call.decision.selected_model_id().clone();
-        let source = LlmClientError::Ffi {
-            source: Box::new(PyErr::from_value(error.clone())),
+        let source = if error.is_instance_of::<ContextWindowExceededError>() {
+            LlmClientError::ContextWindowExceeded {
+                model: target.clone(),
+                message: error.str()?.to_string_lossy().into_owned(),
+            }
+        } else {
+            LlmClientError::Ffi {
+                source: Box::new(PyErr::from_value(error.clone())),
+            }
         };
         call.respond(Err(RustLibsyError::client_call(target, source)))
             .map_err(py_libsy_error)
@@ -505,6 +512,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> {
         &libsy_module
     )?)?;
     libsy_module.add_function(wrap_pyfunction!(stage_router_algorithm, &libsy_module)?)?;
+    libsy_module.add(
+        "ContextWindowExceededError",
+        module.getattr("ContextWindowExceededError")?,
+    )?;
     libsy_module.add("LibsyError", module.getattr("LibsyError")?)?;
     module.add_submodule(&libsy_module)?;
     Ok(())

examples/experimental/litellm/README.md

@@ -11,8 +11,8 @@ point for your own application.
 LiteLLM provides the OpenAI-compatible gateway, model aliases, and OpenRouter
 provider integration. Switchyard's Stage router makes the routing decision from
 the coding agent's recent tool history.
-`LiteLLMSyClient` connects Switchyard's normalized libsy requests to the
-selected alias through LiteLLM's asynchronous Completion API and the
+`LiteLLMSyClient` connects Switchyard's normalized libsy requests to their
+selected aliases through LiteLLM's asynchronous Completion API and the
 Dockerized gateway. Together, they let an application keep routing policy in
 Switchyard while LiteLLM owns model access and sends Chat Completions inference
 through OpenRouter.
@@ -107,7 +107,7 @@ normalized request shape and routing policy:
 ```python
 import asyncio
 
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Step, algorithms
 from switchyard_litellm import LiteLLMSyClient
 
 
@@ -150,22 +150,30 @@ async def main() -> None:
         "reasoning": {"effort": "low"},
         "output": {"max_output_tokens": 64},
     }
-    strong = LiteLLMSyClient("strong")
-    fast = LiteLLMSyClient("fast")
+    client = LiteLLMSyClient()
     router = algorithms.stage_router(
-        LlmTarget("strong", strong),
-        LlmTarget("fast", fast),
+        "strong",
+        "fast",
         picker="efficient_first",
         confidence_threshold=0.5,
         recent_window=3,
     )
     try:
-        decisions, response = await router.run(request)
-        print(decisions)
-        print(response)
+        async for step in router.run_stream(request):
+            match step:
+                case Step.Decision(decision):
+                    print("Decision:", decision.selected_model_id, decision.reasoning)
+                case Step.CallModel(call):
+                    try:
+                        response = await client.call(call.request)
+                    except Exception as error:
+                        call.fail(error)
+                    else:
+                        call.respond(response)
+                case Step.Done(response):
+                    print("Response:", response)
     finally:
-        await strong.aclose()
-        await fast.aclose()
+        await client.aclose()
 
 
 asyncio.run(main())

examples/experimental/litellm/example.py

@@ -8,7 +8,7 @@
 
 from switchyard_litellm import LiteLLMSyClient
 
-from switchyard.libsy import LlmTarget, algorithms
+from switchyard.libsy import Step, algorithms
 
 
 def sy_request() -> dict[str, object]:
@@ -60,21 +60,30 @@ def sy_request() -> dict[str, object]:
 
 async def main() -> None:
     """Run the Stage router and print its normalized result."""
-    strong_client = LiteLLMSyClient("strong")
-    fast_client = LiteLLMSyClient("fast")
+    client = LiteLLMSyClient()
     router = algorithms.stage_router(
-        LlmTarget("strong", strong_client),
-        LlmTarget("fast", fast_client),
+        "strong",
+        "fast",
         picker="efficient_first",
         confidence_threshold=0.5,
         recent_window=3,
     )
     try:
-        decisions, response = await router.run(sy_request())
-        print("Stage router:", decisions, response)
+        async for step in router.run_stream(sy_request()):
+            match step:
+                case Step.Decision(decision):
+                    print("Decision:", decision.selected_model_id, decision.reasoning)
+                case Step.CallModel(call):
+                    try:
+                        response = await client.call(call.request)
+                    except Exception as error:
+                        call.fail(error)
+                    else:
+                        call.respond(response)
+                case Step.Done(response):
+                    print("Response:", response)
     finally:
-        await strong_client.aclose()
-        await fast_client.aclose()
+        await client.aclose()
 
 
 if __name__ == "__main__":

examples/experimental/litellm/src/switchyard_litellm/client.py

@@ -10,6 +10,11 @@
 from typing import Any, cast
 
 from litellm import ModelResponse, acompletion
+from litellm.exceptions import (
+    ContextWindowExceededError as LiteLLMContextWindowExceededError,
+)
+
+from switchyard.libsy import ContextWindowExceededError
 
 _TEXT_ROLES = {"system", "developer", "user"}
 _STOP_REASONS = {
@@ -208,7 +213,10 @@ def _optional_mapping(
     return _mapping(value, field)
 
 
-def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:
+async def _payload(request: Mapping[str, object]) -> dict[str, Any]:
+    model = request.get("model")
+    if not isinstance(model, str) or not model:
+        raise ValueError("model must be a non-empty string")
     if request.get("stream") is True:
         raise ValueError("stream=True is not supported")
     _reject_sequence_payload(request, "instructions")
@@ -226,7 +234,9 @@ def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]:
         raise ValueError("reasoning.raw is not supported")
 
     payload: dict[str, Any] = {
-        "model": model,
+        # LiteLLM needs this prefix to select its OpenAI-compatible transport,
+        # then strips it before sending the selected alias to the gateway.
+        "model": f"openai/{model}",
         "messages": _messages(request),
         "stream": False,
     }
@@ -307,18 +317,14 @@ def _response(response: ModelResponse) -> dict[str, object]:
 
 
 class LiteLLMSyClient:
-    """Call a LiteLLM gateway alias for a libsy target."""
+    """Call the LiteLLM gateway alias selected in a libsy request."""
 
     def __init__(
         self,
-        model: str,
         *,
         base_url: str = "http://127.0.0.1:4000/v1",
         api_key: str = "not-needed",
     ) -> None:
-        if not model:
-            raise ValueError("model must not be empty")
-        self.model = model
         self._base_url = base_url
         self._api_key = api_key
 
@@ -327,17 +333,21 @@ async def call(
         sy_request: Mapping[str, object],
     ) -> Mapping[str, object]:
         """Send one normalized, buffered text request through LiteLLM."""
-        response = await acompletion(
-            **_payload(sy_request, f"openai/{self.model}"),
-            api_base=self._base_url,
-            api_key=self._api_key,
-            num_retries=0,
-            # LiteLLM otherwise imports its optional proxy MCP stack for ordinary
-            # OpenAI function tools before it determines that they are not MCP tools.
-            _skip_mcp_handler=True,
-            allowed_openai_params=["reasoning_effort"],
-            extra_body={"allowed_openai_params": ["reasoning_effort"]},
-        )
+        payload = await _payload(sy_request)
+        try:
+            response = await acompletion(
+                **payload,
+                api_base=self._base_url,
+                api_key=self._api_key,
+                num_retries=0,
+                # LiteLLM otherwise imports its optional proxy MCP stack for ordinary
+                # OpenAI function tools before it determines that they are not MCP tools.
+                _skip_mcp_handler=True,
+                allowed_openai_params=["reasoning_effort"],
+                extra_body={"allowed_openai_params": ["reasoning_effort"]},
+            )
+        except LiteLLMContextWindowExceededError as error:
+            raise ContextWindowExceededError(str(error)) from error
         return _response(cast(ModelResponse, response))
 
     async def aclose(self) -> None: