diff --git a/crates/switchyard-py/src/errors.rs b/crates/switchyard-py/src/errors.rs index 9e0beb775..8f5e44e47 100644 --- a/crates/switchyard-py/src/errors.rs +++ b/crates/switchyard-py/src/errors.rs @@ -8,6 +8,7 @@ use pyo3::exceptions::PyRuntimeError; use pyo3::prelude::*; create_exception!(_switchyard_rust, LibsyError, PyRuntimeError); +create_exception!(_switchyard_rust, ContextWindowExceededError, PyRuntimeError); /// Converts libsy execution failures into one stable Python exception. pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr { @@ -15,5 +16,9 @@ pub(crate) fn py_libsy_error(error: impl std::fmt::Display) -> PyErr { } pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { - module.add("LibsyError", module.py().get_type::()) + module.add("LibsyError", module.py().get_type::())?; + module.add( + "ContextWindowExceededError", + module.py().get_type::(), + ) } diff --git a/crates/switchyard-py/src/libsy_bindings.rs b/crates/switchyard-py/src/libsy_bindings.rs index ee2b70fd9..ce303d5e6 100644 --- a/crates/switchyard-py/src/libsy_bindings.rs +++ b/crates/switchyard-py/src/libsy_bindings.rs @@ -21,7 +21,7 @@ use switchyard_protocol::{ }; use tokio::sync::Mutex; -use crate::errors::py_libsy_error; +use crate::errors::{ContextWindowExceededError, py_libsy_error}; use crate::py_serde::{from_python, to_python}; /// Convert Python-owned headers into the request metadata expected by libsy. @@ -250,8 +250,15 @@ impl PyModelCall { } let call = self.take()?; let target = call.decision.selected_model_id().clone(); - let source = LlmClientError::Ffi { - source: Box::new(PyErr::from_value(error.clone())), + let source = if error.is_instance_of::() { + LlmClientError::ContextWindowExceeded { + model: target.clone(), + message: error.str()?.to_string_lossy().into_owned(), + } + } else { + LlmClientError::Ffi { + source: Box::new(PyErr::from_value(error.clone())), + } }; call.respond(Err(RustLibsyError::client_call(target, source))) .map_err(py_libsy_error) @@ -505,6 +512,10 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { &libsy_module )?)?; libsy_module.add_function(wrap_pyfunction!(stage_router_algorithm, &libsy_module)?)?; + libsy_module.add( + "ContextWindowExceededError", + module.getattr("ContextWindowExceededError")?, + )?; libsy_module.add("LibsyError", module.getattr("LibsyError")?)?; module.add_submodule(&libsy_module)?; Ok(()) diff --git a/examples/experimental/litellm/README.md b/examples/experimental/litellm/README.md index 1f2294e9a..836cca915 100644 --- a/examples/experimental/litellm/README.md +++ b/examples/experimental/litellm/README.md @@ -11,8 +11,8 @@ point for your own application. LiteLLM provides the OpenAI-compatible gateway, model aliases, and OpenRouter provider integration. Switchyard's Stage router makes the routing decision from the coding agent's recent tool history. -`LiteLLMSyClient` connects Switchyard's normalized libsy requests to the -selected alias through LiteLLM's asynchronous Completion API and the +`LiteLLMSyClient` connects Switchyard's normalized libsy requests to their +selected aliases through LiteLLM's asynchronous Completion API and the Dockerized gateway. Together, they let an application keep routing policy in Switchyard while LiteLLM owns model access and sends Chat Completions inference through OpenRouter. @@ -107,7 +107,7 @@ normalized request shape and routing policy: ```python import asyncio -from switchyard.libsy import LlmTarget, algorithms +from switchyard.libsy import Step, algorithms from switchyard_litellm import LiteLLMSyClient @@ -150,22 +150,30 @@ async def main() -> None: "reasoning": {"effort": "low"}, "output": {"max_output_tokens": 64}, } - strong = LiteLLMSyClient("strong") - fast = LiteLLMSyClient("fast") + client = LiteLLMSyClient() router = algorithms.stage_router( - LlmTarget("strong", strong), - LlmTarget("fast", fast), + "strong", + "fast", picker="efficient_first", confidence_threshold=0.5, recent_window=3, ) try: - decisions, response = await router.run(request) - print(decisions) - print(response) + async for step in router.run_stream(request): + match step: + case Step.Decision(decision): + print("Decision:", decision.selected_model_id, decision.reasoning) + case Step.CallModel(call): + try: + response = await client.call(call.request) + except Exception as error: + call.fail(error) + else: + call.respond(response) + case Step.Done(response): + print("Response:", response) finally: - await strong.aclose() - await fast.aclose() + await client.aclose() asyncio.run(main()) diff --git a/examples/experimental/litellm/example.py b/examples/experimental/litellm/example.py index bb67c27ea..df2691865 100644 --- a/examples/experimental/litellm/example.py +++ b/examples/experimental/litellm/example.py @@ -8,7 +8,7 @@ from switchyard_litellm import LiteLLMSyClient -from switchyard.libsy import LlmTarget, algorithms +from switchyard.libsy import Step, algorithms def sy_request() -> dict[str, object]: @@ -60,21 +60,30 @@ def sy_request() -> dict[str, object]: async def main() -> None: """Run the Stage router and print its normalized result.""" - strong_client = LiteLLMSyClient("strong") - fast_client = LiteLLMSyClient("fast") + client = LiteLLMSyClient() router = algorithms.stage_router( - LlmTarget("strong", strong_client), - LlmTarget("fast", fast_client), + "strong", + "fast", picker="efficient_first", confidence_threshold=0.5, recent_window=3, ) try: - decisions, response = await router.run(sy_request()) - print("Stage router:", decisions, response) + async for step in router.run_stream(sy_request()): + match step: + case Step.Decision(decision): + print("Decision:", decision.selected_model_id, decision.reasoning) + case Step.CallModel(call): + try: + response = await client.call(call.request) + except Exception as error: + call.fail(error) + else: + call.respond(response) + case Step.Done(response): + print("Response:", response) finally: - await strong_client.aclose() - await fast_client.aclose() + await client.aclose() if __name__ == "__main__": diff --git a/examples/experimental/litellm/src/switchyard_litellm/client.py b/examples/experimental/litellm/src/switchyard_litellm/client.py index dc0286c1f..dd83b5b0a 100644 --- a/examples/experimental/litellm/src/switchyard_litellm/client.py +++ b/examples/experimental/litellm/src/switchyard_litellm/client.py @@ -10,6 +10,11 @@ from typing import Any, cast from litellm import ModelResponse, acompletion +from litellm.exceptions import ( + ContextWindowExceededError as LiteLLMContextWindowExceededError, +) + +from switchyard.libsy import ContextWindowExceededError _TEXT_ROLES = {"system", "developer", "user"} _STOP_REASONS = { @@ -208,7 +213,10 @@ def _optional_mapping( return _mapping(value, field) -def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]: +async def _payload(request: Mapping[str, object]) -> dict[str, Any]: + model = request.get("model") + if not isinstance(model, str) or not model: + raise ValueError("model must be a non-empty string") if request.get("stream") is True: raise ValueError("stream=True is not supported") _reject_sequence_payload(request, "instructions") @@ -226,7 +234,9 @@ def _payload(request: Mapping[str, object], model: str) -> dict[str, Any]: raise ValueError("reasoning.raw is not supported") payload: dict[str, Any] = { - "model": model, + # LiteLLM needs this prefix to select its OpenAI-compatible transport, + # then strips it before sending the selected alias to the gateway. + "model": f"openai/{model}", "messages": _messages(request), "stream": False, } @@ -307,18 +317,14 @@ def _response(response: ModelResponse) -> dict[str, object]: class LiteLLMSyClient: - """Call a LiteLLM gateway alias for a libsy target.""" + """Call the LiteLLM gateway alias selected in a libsy request.""" def __init__( self, - model: str, *, base_url: str = "http://127.0.0.1:4000/v1", api_key: str = "not-needed", ) -> None: - if not model: - raise ValueError("model must not be empty") - self.model = model self._base_url = base_url self._api_key = api_key @@ -327,17 +333,21 @@ async def call( sy_request: Mapping[str, object], ) -> Mapping[str, object]: """Send one normalized, buffered text request through LiteLLM.""" - response = await acompletion( - **_payload(sy_request, f"openai/{self.model}"), - api_base=self._base_url, - api_key=self._api_key, - num_retries=0, - # LiteLLM otherwise imports its optional proxy MCP stack for ordinary - # OpenAI function tools before it determines that they are not MCP tools. - _skip_mcp_handler=True, - allowed_openai_params=["reasoning_effort"], - extra_body={"allowed_openai_params": ["reasoning_effort"]}, - ) + payload = await _payload(sy_request) + try: + response = await acompletion( + **payload, + api_base=self._base_url, + api_key=self._api_key, + num_retries=0, + # LiteLLM otherwise imports its optional proxy MCP stack for ordinary + # OpenAI function tools before it determines that they are not MCP tools. + _skip_mcp_handler=True, + allowed_openai_params=["reasoning_effort"], + extra_body={"allowed_openai_params": ["reasoning_effort"]}, + ) + except LiteLLMContextWindowExceededError as error: + raise ContextWindowExceededError(str(error)) from error return _response(cast(ModelResponse, response)) async def aclose(self) -> None: diff --git a/examples/experimental/litellm/tests/test_client.py b/examples/experimental/litellm/tests/test_client.py index 5e3654816..157b35c05 100644 --- a/examples/experimental/litellm/tests/test_client.py +++ b/examples/experimental/litellm/tests/test_client.py @@ -7,15 +7,24 @@ import httpx import pytest import respx -from litellm import AuthenticationError, ModelResponse, RateLimitError +from litellm import ( + AuthenticationError, + ModelResponse, + RateLimitError, +) +from litellm.exceptions import ( + ContextWindowExceededError as LiteLLMContextWindowExceededError, +) from switchyard_litellm import LiteLLMSyClient +from switchyard.libsy import ContextWindowExceededError + BASE_URL = "http://gateway.test/v1" def request_body() -> dict[str, object]: return { - "model": "auto", + "model": "fast", "instructions": [], "messages": [ { @@ -87,7 +96,7 @@ async def fake_acompletion(**kwargs: object) -> ModelResponse: return ModelResponse(**gateway_response()) monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: await client.call(request_body()) finally: @@ -108,7 +117,7 @@ async def test_call_translates_request_and_normalizes_response() -> None: ) request = request_body() original = copy.deepcopy(request) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: response = await client.call(request) finally: @@ -200,7 +209,7 @@ async def test_call_translates_function_tool_history() -> None: ] request["tool_choice"] = {"type": "tool", "data": {"name": "Bash"}} - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: await client.call(request) finally: @@ -255,7 +264,7 @@ async def fake_acompletion(**_: object) -> ModelResponse: return ModelResponse(**gateway_tool_response()) monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: response = await client.call(request_body()) finally: @@ -298,7 +307,7 @@ async def fake_acompletion(**_: object) -> ModelResponse: ], } ] - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match=r"messages\[0\]\.content\[0\]\.arguments"): await client.call(request) @@ -317,7 +326,7 @@ async def fake_acompletion(**_: object) -> ModelResponse: return ModelResponse(**payload) monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match="invalid tool-call arguments"): await client.call(request_body()) @@ -340,7 +349,7 @@ async def fake_acompletion(**kwargs: object) -> ModelResponse: request = request_body() request["tools"] = [{"name": "Bash", "description": None, "parameters": {}}] request["tool_choice"] = {"type": choice} - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: await client.call(request) finally: @@ -442,7 +451,7 @@ async def test_call_rejects_unsupported_normalized_fields( router.post(f"{BASE_URL}/chat/completions").mock( return_value=httpx.Response(200, json=gateway_response()) ) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match=match): await client.call(request) @@ -451,10 +460,10 @@ async def test_call_rejects_unsupported_normalized_fields( async def test_call_rejects_missing_messages() -> None: - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match="messages"): - await client.call({}) + await client.call({"model": "fast"}) finally: await client.aclose() @@ -466,7 +475,7 @@ async def test_call_rejects_a_response_without_text() -> None: respx.post(f"{BASE_URL}/chat/completions").mock( return_value=httpx.Response(200, json=payload) ) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match="no text content"): await client.call(request_body()) @@ -484,7 +493,7 @@ async def fake_acompletion(**_: object) -> ModelResponse: return ModelResponse(**payload) monkeypatch.setattr("switchyard_litellm.client.acompletion", fake_acompletion) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(ValueError, match="no choices"): await client.call(request_body()) @@ -506,7 +515,7 @@ async def test_litellm_errors_propagate() -> None: }, ) ) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(AuthenticationError): await client.call(request_body()) @@ -528,7 +537,7 @@ async def test_retryable_litellm_error_is_not_retried() -> None: }, ) ) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: with pytest.raises(RateLimitError): await client.call(request_body()) @@ -538,6 +547,27 @@ async def test_retryable_litellm_error_is_not_retried() -> None: assert route.call_count == 1 +async def test_context_window_error_is_preserved( + monkeypatch: pytest.MonkeyPatch, +) -> None: + async def fail_with_context_window(**_: object) -> ModelResponse: + raise LiteLLMContextWindowExceededError( + message="request is too large", + model="fast", + llm_provider="openai", + ) + + monkeypatch.setattr( + "switchyard_litellm.client.acompletion", fail_with_context_window + ) + client = LiteLLMSyClient(base_url=BASE_URL) + try: + with pytest.raises(ContextWindowExceededError, match="request is too large"): + await client.call(request_body()) + finally: + await client.aclose() + + @respx.mock async def test_cached_token_count_preserves_explicit_zero() -> None: payload = gateway_response() @@ -545,7 +575,7 @@ async def test_cached_token_count_preserves_explicit_zero() -> None: respx.post(f"{BASE_URL}/chat/completions").mock( return_value=httpx.Response(200, json=payload) ) - client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) try: response = await client.call(request_body()) finally: diff --git a/examples/experimental/litellm/tests/test_e2e.py b/examples/experimental/litellm/tests/test_e2e.py index d4a733357..ad5504681 100644 --- a/examples/experimental/litellm/tests/test_e2e.py +++ b/examples/experimental/litellm/tests/test_e2e.py @@ -11,7 +11,7 @@ import pytest from switchyard_litellm import LiteLLMSyClient -from switchyard.libsy import LlmTarget, algorithms +from switchyard.libsy import Algorithm, Decision, Step, algorithms PACKAGE_ROOT = Path(__file__).resolve().parents[1] @@ -143,28 +143,45 @@ def _request(*, critical_error: bool = False) -> dict[str, object]: } +async def _run_router( + router: Algorithm, + request: dict[str, object], + client: LiteLLMSyClient, +) -> tuple[list[Decision], dict[str, object]]: + decisions: list[Decision] = [] + async for step in router.run_stream(request): + match step: + case Step.Decision(decision): + decisions.append(decision) + case Step.CallModel(call): + call.respond(await client.call(call.request)) + case Step.Done(response): + return decisions, response + raise AssertionError("algorithm stream ended without a response") + + @pytest.mark.e2e async def test_stage_router_calls_both_real_openrouter_models( litellm_base_url: str, ) -> None: - strong_client = LiteLLMSyClient("strong", base_url=litellm_base_url) - fast_client = LiteLLMSyClient("fast", base_url=litellm_base_url) + client = LiteLLMSyClient(base_url=litellm_base_url) router = algorithms.stage_router( - LlmTarget("strong", strong_client), - LlmTarget("fast", fast_client), + "strong", + "fast", picker="efficient_first", confidence_threshold=0.5, recent_window=3, ) try: - fast_trace, fast_response = await router.run(_request()) - strong_trace, strong_response = await router.run(_request(critical_error=True)) + fast_trace, fast_response = await _run_router(router, _request(), client) + strong_trace, strong_response = await _run_router( + router, _request(critical_error=True), client + ) finally: - await strong_client.aclose() - await fast_client.aclose() + await client.aclose() - assert [item["selected_model"] for item in strong_trace] == ["strong"] - assert [item["selected_model"] for item in fast_trace] == ["fast"] + assert [item.selected_model_id for item in strong_trace] == ["strong"] + assert [item.selected_model_id for item in fast_trace] == ["fast"] for response in (strong_response, fast_response): text = response["outputs"][0]["content"][0]["text"] assert isinstance(text, str) diff --git a/examples/experimental/litellm/tests/test_stage_routing.py b/examples/experimental/litellm/tests/test_stage_routing.py index 1e8bf4202..54a1e051c 100644 --- a/examples/experimental/litellm/tests/test_stage_routing.py +++ b/examples/experimental/litellm/tests/test_stage_routing.py @@ -7,7 +7,7 @@ import respx from switchyard_litellm import LiteLLMSyClient -from switchyard.libsy import LlmTarget, algorithms +from switchyard.libsy import Algorithm, Decision, Step, algorithms BASE_URL = "http://gateway.test/v1" @@ -73,8 +73,25 @@ def gateway_response(model: str) -> dict[str, object]: } +async def _run_router( + router: Algorithm, + request: dict[str, object], + client: LiteLLMSyClient, +) -> tuple[list[Decision], dict[str, object]]: + decisions: list[Decision] = [] + async for step in router.run_stream(request): + match step: + case Step.Decision(decision): + decisions.append(decision) + case Step.CallModel(call): + call.respond(await client.call(call.request)) + case Step.Done(response): + return decisions, response + raise AssertionError("algorithm stream ended without a response") + + @respx.mock -async def test_stage_router_drives_both_litellm_targets() -> None: +async def test_stage_router_drives_both_litellm_models() -> None: seen: list[str] = [] def respond(request: httpx.Request) -> httpx.Response: @@ -83,26 +100,26 @@ def respond(request: httpx.Request) -> httpx.Response: return httpx.Response(200, json=gateway_response(model)) respx.post(f"{BASE_URL}/chat/completions").mock(side_effect=respond) - strong_client = LiteLLMSyClient("strong", base_url=BASE_URL) - fast_client = LiteLLMSyClient("fast", base_url=BASE_URL) + client = LiteLLMSyClient(base_url=BASE_URL) router = algorithms.stage_router( - LlmTarget("strong", strong_client), - LlmTarget("fast", fast_client), + "strong", + "fast", picker="efficient_first", confidence_threshold=0.5, recent_window=3, ) try: - fast_decisions, fast_response = await router.run(request_body()) - strong_decisions, strong_response = await router.run( - request_body(critical_error=True) + fast_decisions, fast_response = await _run_router( + router, request_body(), client + ) + strong_decisions, strong_response = await _run_router( + router, request_body(critical_error=True), client ) finally: - await strong_client.aclose() - await fast_client.aclose() + await client.aclose() - assert [item["selected_model"] for item in fast_decisions] == ["fast"] - assert [item["selected_model"] for item in strong_decisions] == ["strong"] + assert [item.selected_model_id for item in fast_decisions] == ["fast"] + assert [item.selected_model_id for item in strong_decisions] == ["strong"] assert fast_response["outputs"][0]["content"][0]["text"] == "fast" assert strong_response["outputs"][0]["content"][0]["text"] == "strong" assert seen == ["fast", "strong"] diff --git a/examples/experimental/litellm/uv.lock b/examples/experimental/litellm/uv.lock index b7b6e9702..656b5b6a8 100644 --- a/examples/experimental/litellm/uv.lock +++ b/examples/experimental/litellm/uv.lock @@ -69,25 +69,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, ] -[[package]] -name = "anthropic" -version = "0.120.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "distro" }, - { name = "docstring-parser" }, - { name = "httpx" }, - { name = "jiter" }, - { name = "pydantic" }, - { name = "sniffio" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d7/10/4ca013cb166f226bd89e0aeb0fcaff94f45ddf716d4925ce89475d3c587b/anthropic-0.120.2.tar.gz", hash = "sha256:9722efc10c27a30a69f5338ddacdb35bc6a64297a4e4ba729bf83af873d5fb3a", size = 1008421, upload-time = "2026-07-28T17:38:26.986Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/63/af/0f5db57b9397a0f3b7fc204cbef143401a7cadaf982330f97f1ce3d39f34/anthropic-0.120.2-py3-none-any.whl", hash = "sha256:0f0bc2b381dc0eb41c8d886b815d79c2041cd2374f83aed36f574b6dc9c579c1", size = 1022851, upload-time = "2026-07-28T17:38:25.466Z" }, -] - [[package]] name = "anyio" version = "4.14.2" @@ -171,15 +152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "docstring-parser" -version = "0.18.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, -] - [[package]] name = "fastuuid" version = "0.14.0" @@ -484,46 +456,24 @@ wheels = [ name = "nemo-switchyard" version = "0.2.0" source = { editable = "../../../" } -dependencies = [ - { name = "anthropic" }, - { name = "httpx" }, - { name = "openai" }, - { name = "pydantic" }, -] [package.metadata] -requires-dist = [ - { name = "anthropic", specifier = ">=0.99.0,<1.0" }, - { name = "ddtrace", marker = "extra == 'tracing'", specifier = ">=2.9,<4" }, - { name = "fastapi", marker = "extra == 'server'", specifier = ">=0.136.1,<1.0" }, - { name = "httpx", specifier = ">=0.28.1,<1.0" }, - { name = "nemo-switchyard", extras = ["server", "cli", "tracing", "affinity-redis"], marker = "extra == 'all'" }, - { name = "openai", specifier = ">=2.7,<3.0" }, - { name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }, - { name = "pydantic", specifier = ">=2.13.3,<3.0" }, - { name = "redis", marker = "extra == 'affinity-redis'", specifier = ">=5.0.1,<6" }, - { name = "sse-starlette", marker = "extra == 'server'", specifier = ">=3.4.1,<4.0" }, - { name = "uvicorn", extras = ["standard"], marker = "extra == 'server'", specifier = ">=0.46.0,<1.0" }, -] -provides-extras = ["server", "cli", "tracing", "affinity-redis", "all"] +requires-dist = [{ name = "prompt-toolkit", marker = "extra == 'cli'", specifier = ">=3.0.52,<4.0" }] +provides-extras = ["cli"] [package.metadata.requires-dev] dev = [ { name = "harbor", marker = "python_full_version >= '3.12'", git = "https://github.com/harbor-framework/harbor.git?rev=v0.6.4" }, - { name = "httpx", specifier = ">=0.28.1,<1.0" }, { name = "maturin", specifier = ">=1.9,<2.0" }, { name = "mypy", specifier = ">=1.20.2,<2.0" }, - { name = "nemo-switchyard", extras = ["server"] }, - { name = "prometheus-client", specifier = ">=0.21.0,<1.0" }, + { name = "nemo-switchyard", extras = ["cli"] }, { name = "pytest", specifier = ">=9.0.3,<10.0" }, { name = "pytest-asyncio", specifier = ">=1.3.0,<2.0" }, - { name = "pytest-cov", specifier = ">=7.1.0,<8.0" }, { name = "pytest-markdown-docs", specifier = ">=0.9.2" }, - { name = "pytest-mock", specifier = ">=3.15.1,<4.0" }, { name = "pytest-timeout", specifier = ">=2.4.0,<3.0" }, - { name = "respx", specifier = ">=0.23.1,<1.0" }, + { name = "pyyaml", specifier = ">=6.0.3,<7.0" }, { name = "ruff", specifier = ">=0.15.12,<1.0" }, - { name = "socksio", specifier = ">=1.0.0,<2.0" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0,<3.0" }, ] docs = [ { name = "mkdocs", specifier = ">=1.6.0,<2.0" }, diff --git a/switchyard/libsy/__init__.py b/switchyard/libsy/__init__.py index 6d2b5ba74..78a535d01 100644 --- a/switchyard/libsy/__init__.py +++ b/switchyard/libsy/__init__.py @@ -5,6 +5,7 @@ from switchyard_rust.libsy import ( Algorithm, + ContextWindowExceededError, Decision, LibsyError, LlmFallback, @@ -17,6 +18,7 @@ __all__ = [ "Algorithm", + "ContextWindowExceededError", "Decision", "LibsyError", "LlmFallback", diff --git a/switchyard_rust/libsy.py b/switchyard_rust/libsy.py index 244b1dd02..5dd607566 100644 --- a/switchyard_rust/libsy.py +++ b/switchyard_rust/libsy.py @@ -13,6 +13,7 @@ _EXPORTS = frozenset( { "Algorithm", + "ContextWindowExceededError", "Decision", "LibsyError", "LlmFallback", @@ -32,6 +33,8 @@ class LibsyError(RuntimeError): ... + class ContextWindowExceededError(RuntimeError): ... + @final class Decision: """A semantic routing choice produced by an algorithm.""" diff --git a/tests/test_libsy_minimal_bindings.py b/tests/test_libsy_minimal_bindings.py index eb1d6c8ce..988454635 100644 --- a/tests/test_libsy_minimal_bindings.py +++ b/tests/test_libsy_minimal_bindings.py @@ -9,6 +9,7 @@ from switchyard.libsy import ( Algorithm, + ContextWindowExceededError, Decision, LibsyError, Step, @@ -273,3 +274,23 @@ async def call(self, request: dict[str, Any]) -> dict[str, Any]: with pytest.raises(LibsyError, match="client failed"): await run_algorithm(algorithm, {"broken": FailingClient()}) + + +async def test_context_window_failure_falls_back_to_the_next_model() -> None: + class OverflowClient: + async def call(self, request: dict[str, Any]) -> dict[str, Any]: + raise ContextWindowExceededError("request exceeds context window") + + algorithm = algorithms.stage_router( + "strong", + "fast", + picker="efficient_first", + confidence_threshold=0.5, + ) + decisions, response = await run_algorithm( + algorithm, + {"fast": OverflowClient(), "strong": EchoClient("strong")}, + ) + + assert [decision.selected_model_id for decision in decisions] == ["fast", "strong"] + assert response["model"] == "strong"