From 6db7275d2025f205a136cb775f16556b0eb22405 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Mon, 10 Aug 2026 12:17:50 -0600 Subject: [PATCH 1/9] feat(relay): add isolated Switchyard routing plugin Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 22 + README.md | 2 + .../switchyard-nemo-relay-plugin/Cargo.lock | 2483 +++++++++++++++++ .../switchyard-nemo-relay-plugin/Cargo.toml | 34 + crates/switchyard-nemo-relay-plugin/README.md | 426 +++ .../config.schema.json | 217 ++ .../relay-plugin.toml | 31 + .../scripts/package_bundle.py | 62 + .../src/client.rs | 469 ++++ .../src/config.rs | 1186 ++++++++ .../src/executor.rs | 137 + .../switchyard-nemo-relay-plugin/src/ffi.rs | 444 +++ .../switchyard-nemo-relay-plugin/src/lib.rs | 442 +++ .../src/runtime.rs | 1793 ++++++++++++ .../src/translation.rs | 116 + docs/index.md | 2 + 16 files changed, 7866 insertions(+) create mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.lock create mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.toml create mode 100644 crates/switchyard-nemo-relay-plugin/README.md create mode 100644 crates/switchyard-nemo-relay-plugin/config.schema.json create mode 100644 crates/switchyard-nemo-relay-plugin/relay-plugin.toml create mode 100644 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py create mode 100644 crates/switchyard-nemo-relay-plugin/src/client.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/config.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/executor.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/ffi.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/lib.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/translation.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index ee3e55528..c4151729b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + +- **NeMo Relay native plugin** — a dynamically loaded integration that runs + libsy's weighted-random, LLM-classifier, escalation, and stage-router + algorithms in process while Switchyard owns provider HTTP dispatch, + credentials, translation, retries, and fallback. Managed calls require NeMo + Relay 0.7 or newer and do not depend on `switchyard-server`. + +- **NeMo Relay routing-model usage marks** — classifier judges, escalation + judges and discarded weak candidates, and failed routing candidates now emit + `switchyard.routing.llm_call` ATOF marks with normalized token usage and + latency. The final serving call remains represented only by Relay's outer LLM + lifecycle event to prevent double-counting. + ### Removed - **Latency-aware router** — the `latency_service` route type and its @@ -31,6 +45,14 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed +- **NeMo Relay stage and escalation integration** — preserved same-protocol + request bodies are now re-encoded after tier prompts or handoff notes mutate + the normalized request; Relay's synthetic `gateway-gateway` identity no + longer shares escalation latch state across unrelated raw gateway requests; + stage decision marks now retain picker-default tiers, decision sources, and + hard-override confidence. Target bindings also accept non-secret + `extra_body` defaults for provider-specific judge controls. + - **Response `model` now names the model that actually served the request**, on every serving path and wire format. Streamed Anthropic and Responses replies, and every libsy-served reply, previously echoed the model id the client diff --git a/README.md b/README.md index 7ead4efd3..160ff78cb 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ algorithm you write yourself. - **Protocol Translation**: convert between OpenAI Chat, Anthropic Messages, and OpenAI Responses formats - **Multi-Backend Routing**: random routing, LLM-as-classifier routing, signal-driven stage-router, or your own algorithm - **Operational Metrics**: Prometheus metrics cover requests, errors, latency, tokens, and routing overhead +- **NeMo Relay Plugin**: run random, classifier, escalation, or stage routing in Relay while Switchyard owns provider HTTP dispatch ## Maturity @@ -154,6 +155,7 @@ configured LLM client selects one upstream format. - **[`switchyard-libsy`](crates/libsy/README.md)**: embed routing algorithms in a Rust application - **[`switchyard-protocol`](crates/protocol/README.md)**: provider-neutral request, response, and streaming types - **[`switchyard-translation`](crates/switchyard-translation/README.md)**: request, response, and stream translation +- **[`switchyard-nemo-relay-plugin`](crates/switchyard-nemo-relay-plugin/README.md)**: install Switchyard as a native NeMo Relay plugin ## Community diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.lock b/crates/switchyard-nemo-relay-plugin/Cargo.lock new file mode 100644 index 000000000..72f37d26a --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.lock @@ -0,0 +1,2483 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "async-trait" +version = "0.1.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "aws-lc-rs" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" +dependencies = [ + "aws-lc-sys", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", + "pkg-config", +] + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] + +[[package]] +name = "borrow-or-share" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" + +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "bytecount" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cc" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "chacha20" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" +dependencies = [ + "cfg-if", + "cpufeatures", + "rand_core", +] + +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", + "serde", +] + +[[package]] +name = "cmake" +version = "0.1.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" +dependencies = [ + "cc", +] + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] + +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" +dependencies = [ + "libc", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + +[[package]] +name = "data-encoding" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" + +[[package]] +name = "displaydoc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + +[[package]] +name = "email_address" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" +dependencies = [ + "serde", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + +[[package]] +name = "fancy-regex" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "find-msvc-tools" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" + +[[package]] +name = "fluent-uri" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" +dependencies = [ + "borrow-or-share", + "ref-cast", + "serde", +] + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "form_urlencoded" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "fraction" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" +dependencies = [ + "lazy_static", + "num", +] + +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + +[[package]] +name = "futures" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" + +[[package]] +name = "futures-executor" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" + +[[package]] +name = "futures-macro" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "futures-sink" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" + +[[package]] +name = "futures-task" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" + +[[package]] +name = "futures-util" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "wasi", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 5.3.0", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi 6.0.0", + "rand_core", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", +] + +[[package]] +name = "icu_collections" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" +dependencies = [ + "displaydoc", + "potential_utf", + "utf8_iter", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locale_core" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_normalizer" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" +dependencies = [ + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" + +[[package]] +name = "icu_properties" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" +dependencies = [ + "icu_collections", + "icu_locale_core", + "icu_properties_data", + "icu_provider", + "zerotrie", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" + +[[package]] +name = "icu_provider" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" +dependencies = [ + "displaydoc", + "icu_locale_core", + "writeable", + "yoke", + "zerofrom", + "zerotrie", + "zerovec", +] + +[[package]] +name = "idna" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ipnet" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys", + "log", + "simd_cesu8", + "thiserror", + "walkdir", + "windows-link", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.119", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.119", +] + +[[package]] +name = "jobserver" +version = "0.1.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +dependencies = [ + "getrandom 0.4.3", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + +[[package]] +name = "jsonptr" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85019623956752c8dd1f04a7b05d066187e0c9b217454246d8397d2a4893cc83" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "jsonschema" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" +dependencies = [ + "ahash", + "bytecount", + "data-encoding", + "email_address", + "fancy-regex", + "fraction", + "getrandom 0.3.4", + "idna", + "itoa", + "jsonschema-regex", + "jsonschema-value", + "num-cmp", + "num-traits", + "percent-encoding", + "referencing", + "regex", + "serde", + "serde_json", + "strum", + "unicode-general-category", + "uuid-simd", +] + +[[package]] +name = "jsonschema-regex" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" +dependencies = [ + "regex-syntax", +] + +[[package]] +name = "jsonschema-value" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" +dependencies = [ + "ahash", + "bytecount", + "fraction", + "num-cmp", + "num-traits", + "serde_json", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "litemap" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" + +[[package]] +name = "lru-slab" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "micromap" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" + +[[package]] +name = "mio" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "nemo-relay-plugin" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df37bebc79a6d757a7cb18d6c94ca0825fcb6c8e51ab4e13900e7d5853e0de8b" +dependencies = [ + "nemo-relay-types", + "serde", + "serde_json", +] + +[[package]] +name = "nemo-relay-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ea078c95f9e0804a77a0beb5d86f003ff4a9315c27e1a7afb2cee98bd4d7fd" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-cmp" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "opentelemetry" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" +dependencies = [ + "futures-core", + "futures-sink", + "js-sys", + "pin-project-lite", + "thiserror", +] + +[[package]] +name = "outref" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" + +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "percent-encoding" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "pkg-config" +version = "0.3.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" + +[[package]] +name = "potential_utf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" +dependencies = [ + "zerovec", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quinn" +version = "0.11.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" +dependencies = [ + "bytes", + "cfg_aliases", + "pin-project-lite", + "quinn-proto", + "quinn-udp", + "rustc-hash", + "rustls", + "socket2", + "thiserror", + "tokio", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-proto" +version = "0.11.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" +dependencies = [ + "aws-lc-rs", + "bytes", + "getrandom 0.4.3", + "lru-slab", + "rand", + "rand_pcg", + "ring", + "rustc-hash", + "rustls", + "rustls-pki-types", + "slab", + "thiserror", + "tinyvec", + "tracing", + "web-time", +] + +[[package]] +name = "quinn-udp" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" +dependencies = [ + "cfg_aliases", + "libc", + "once_cell", + "socket2", + "tracing", + "windows-sys 0.61.2", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "chacha20", + "getrandom 0.4.3", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "ref-cast" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "referencing" +version = "0.49.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" +dependencies = [ + "ahash", + "fluent-uri", + "getrandom 0.3.4", + "hashbrown", + "itoa", + "micromap", + "parking_lot", + "percent-encoding", + "serde_json", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "reqwest" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" +dependencies = [ + "base64", + "bytes", + "futures-core", + "futures-util", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-util", + "js-sys", + "log", + "percent-encoding", + "pin-project-lite", + "quinn", + "rustls", + "rustls-pki-types", + "rustls-platform-verifier", + "serde", + "serde_json", + "sync_wrapper", + "tokio", + "tokio-rustls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver", +] + +[[package]] +name = "rustls" +version = "0.23.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pki-types" +version = "1.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" +dependencies = [ + "web-time", + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +dependencies = [ + "core-foundation", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sharded-slab" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" +dependencies = [ + "lazy_static", +] + +[[package]] +name = "shlex" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "socket2" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "strum" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" +dependencies = [ + "strum_macros", +] + +[[package]] +name = "strum_macros" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "switchyard-libsy" +version = "0.2.0" +dependencies = [ + "async-trait", + "futures", + "jsonptr", + "jsonschema", + "opentelemetry", + "parking_lot", + "rand", + "serde", + "serde_json", + "switchyard-protocol", + "thiserror", + "tokio", + "tokio-stream", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "switchyard-llm-client" +version = "0.2.0" +dependencies = [ + "async-trait", + "futures", + "futures-util", + "http", + "httpdate", + "opentelemetry", + "parking_lot", + "reqwest", + "serde_json", + "switchyard-libsy", + "switchyard-protocol", + "switchyard-translation", + "tokio", + "tracing", + "tracing-opentelemetry", +] + +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +dependencies = [ + "async-channel", + "async-trait", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "switchyard-translation", + "tokio", +] + +[[package]] +name = "switchyard-protocol" +version = "0.2.0" +dependencies = [ + "async-trait", + "futures", + "http", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "switchyard-translation" +version = "0.2.0" +dependencies = [ + "async-stream", + "futures", + "serde", + "serde_json", + "switchyard-protocol", + "thiserror", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "thiserror" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "thread_local" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "tinystr" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "tinyvec" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "tokio" +version = "1.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" +dependencies = [ + "bytes", + "libc", + "mio", + "parking_lot", + "pin-project-lite", + "signal-hook-registry", + "socket2", + "tokio-macros", + "windows-sys 0.61.2", +] + +[[package]] +name = "tokio-macros" +version = "2.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.3", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-stream" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" +dependencies = [ + "bitflags", + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", + "url", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", + "valuable", +] + +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + +[[package]] +name = "tracing-opentelemetry" +version = "0.33.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" +dependencies = [ + "js-sys", + "opentelemetry", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + +[[package]] +name = "tracing-subscriber" +version = "0.3.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" +dependencies = [ + "sharded-slab", + "thread_local", + "tracing-core", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "unicode-general-category" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "url" +version = "2.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + +[[package]] +name = "uuid-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "valuable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "wasip2" +version = "1.0.4+wasi-0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.77" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn 2.0.119", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.127" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-streams" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "web-sys" +version = "0.3.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-root-certs" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys 0.61.2", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "wit-bindgen" +version = "0.57.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" + +[[package]] +name = "writeable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" + +[[package]] +name = "yoke" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" +dependencies = [ + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.56" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" + +[[package]] +name = "zerotrie" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", +] + +[[package]] +name = "zerovec" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml new file mode 100644 index 000000000..6a169f6fb --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[package] +name = "switchyard-nemo-relay-plugin" +version = "0.1.0" +description = "Switchyard-owned HTTP routing plugin for NeMo Relay" +authors = ["NVIDIA Corporation"] +edition = "2024" +license = "Apache-2.0" +repository = "https://github.com/NVIDIA-NeMo/Switchyard" +rust-version = "1.96.1" +publish = false + +[lib] +crate-type = ["cdylib"] + +[dependencies] +async-channel = "2" +async-trait = "0.1" +futures-util = "0.3" +http = "1" +nemo-relay-plugin = "=0.7.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +switchyard-libsy = { path = "../libsy", version = "0.2.0" } +switchyard-llm-client = { path = "../libsy-llm-client", version = "0.2.0" } +switchyard-protocol = { path = "../protocol", version = "0.2.0" } +switchyard-translation = { path = "../switchyard-translation", version = "0.2.0" } +tokio = { version = "1", features = ["full"] } + +# Keep the dynamic plugin self-contained so adding it does not modify the root +# Switchyard workspace manifest or lockfile. +[workspace] diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md new file mode 100644 index 000000000..0182f4de2 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -0,0 +1,426 @@ + + +# Switchyard NeMo Relay Dynamic Plugin + +This crate builds the external `nvidia.switchyard` native plugin. It embeds +`switchyard-libsy`, drives it through `switchyard-llm-client::run`, and uses +`switchyard-llm-client` for provider HTTP calls. Managed calls use Relay's +completion-based asynchronous middleware hooks and do not require a targeted +provider continuation from Relay. + +The plugin uses NeMo Relay native API v1. It depends on the small +`nemo-relay-plugin` authoring SDK, not the Relay runtime, and does not start +`switchyard-server`. Managed provider calls do not use Relay's provider +continuation. + +## Ownership boundary + +For a managed LLM call: + +1. Relay invokes the native LLM execution intercept. +2. The plugin decodes the caller JSON through `switchyard-translation`. +3. The plugin passes the configured algorithm and its target-to-client map to + `switchyard-llm-client::run`, using the library's public execution and + observation boundary. +4. For every routed call, the selected target client translates the neutral + request, applies its URL and credentials, and performs the HTTP request. +5. `switchyard-llm-client` drives libsy to its final response while the plugin + records decisions and routing-only model usage. +6. The plugin encodes the final neutral response into the caller's protocol. + +Relay still owns the outer LLM lifecycle, dynamic-plugin loading, plugin +configuration, and event substrate. Relay's downstream LLM continuation is +used only for calls whose inbound protocol is not managed by this plugin. + +```mermaid +flowchart LR + A["Caller JSON"] --> B["Relay LLM execution intercept"] + B --> C["Switchyard decode"] + C --> D["switchyard-llm-client run"] + D --> E["libsy algorithm"] + E --> F["target ClientRouter"] + F --> G["Provider HTTP endpoint"] + G --> H["Switchyard response or event decode"] + H --> I["libsy final response"] + I --> J["routing observations"] + J --> K["Switchyard encode"] + K --> A + + U["Unmanaged profile"] -.-> V["Relay v1 continuation"] +``` + +This boundary has two important consequences: + +- Managed provider calls do not traverse Relay middleware registered after the + Switchyard intercept and do not use the host's provider callback. Provider + transport activity is therefore not represented as nested Relay LLM + lifecycle events. Relay records the outer managed call and the plugin emits + Switchyard routing marks; bridging Switchyard transport spans into Relay is + future work. The adapter captures the active Relay scope before returning + `Pending`, so asynchronous routing marks retain their event parent. +- Switchyard owns provider URLs, credentials, HTTP retry behavior, and + translation for managed calls. Relay neither validates nor transports those + target details. + +## Native API v1 and asynchronous execution + +The manifest remains `compat.native_api = "1"`, but the plugin requires the +generic host-table v3 extension shipped by Relay 0.7. It registers through +v3's completion-based buffered and incremental streaming hooks, returns +`Pending` immediately, and performs libsy and provider HTTP work on a +plugin-owned Tokio runtime. Relay workers therefore do not wait synchronously +for provider I/O. + +The stream adapter forwards the plugin's bounded 32-message channel into +Relay's bounded output queue. It retries a logical event when the host queue is +full and checks cancellation between attempts. Managed HTTP work is selected +against Relay caller cancellation, so cancelling a buffered or streaming call +drops its in-flight provider future. + +Unmanaged profiles use the same v3 continuation hooks for pass-through. V3's +downstream stream callback has continue/cancel control but no asynchronous +acknowledgement, so the adapter uses a nonblocking bridge capped at 8 MiB of +queued encoded payloads and 256 events. A pass-through stream that outruns +either bound is rejected rather than consuming unbounded memory. + +This is a raw C boundary: Switchyard contains a small ownership adapter for +host strings, completion and stream handles, continuation handles, and captured +scope handles because Relay 0.7 does not expose a safe Rust facade for its +generic asynchronous surface. The HTTP, routing, and translation behavior +remains in Switchyard. The adapter can be replaced with the safe typed surface +when Relay exposes equivalent asynchronous callbacks and cancellation. + +## Supported routers + +The plugin supports four libsy routing modes: + +- seeded, weighted `random` routing; and +- capability-based `llm_classifier` routing, where a judge selects the weak or + strong target before the final provider call; +- escalation-mode `llm_classifier` routing, where a judge evaluates the weak + model's completed turn and latches a session to the strong target after a + configured confirmation streak; and +- signal-driven `stage_router` routing, with optional handoff notes, tier + prompts, and a capability-classifier fallback for ambiguous turns. + +Unsupported algorithm kinds are rejected instead of being approximated. + +## Compatibility Matrix + +The following matrix describes the algorithm behavior implemented by the +plugin. `Conditional` means that the feature is implemented with the constraint +shown in the table; it does not mean that the feature falls back to a different +algorithm. + +| Compatibility Area | `random` | `llm_classifier` (`capability`) | `llm_classifier` (`escalation`) | `stage_router` | +|---|---|---|---|---| +| Version-2 configuration and static validation | Supported | Supported | Supported | Supported | +| Caller protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | +| Serving-target protocols | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | OpenAI Chat, OpenAI Responses, Anthropic Messages | +| Structured-output judge protocols | Not applicable | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses | OpenAI Chat or OpenAI Responses for the optional classifier | +| Buffered responses | Supported | Supported | Supported | Supported | +| Streaming responses | Supported | Supported after the judge selects a target | Conditional: an unlatched weak stream is aggregated before the judge runs | Supported after the signal cascade selects a target | +| Retained routing state | No selection affinity; context-overflow eviction can use session identity | Optional session affinity and message-hash fallback | Confirmation streak and strong latch require stable session identity | No classifier affinity; context-overflow eviction can use session identity | +| Router-specific prompts | Not applicable | Optional judge prompt | Optional escalation-judge prompt | Optional tier prompts, handoff notes, and classifier prompt | +| Relay decision marks | Algorithm, attempt, selected target, and identity; routing tier is `null` | Algorithm, attempt, selected target, weak or strong routing tier, and identity | Algorithm, attempt, selected target, weak or strong routing tier, and identity | Algorithm, attempt, selected target, routing tier, decision source, and identity | +| ATOF routing-LLM usage | Not applicable unless a failed candidate is replaced | Judge calls, plus failed candidates | Judge calls and discarded weak candidates | Optional classifier judge calls, plus failed candidates | + +Anthropic Messages is supported for callers and serving targets, but not for a +structured-output judge. That restriction is intentional and fails during +static configuration loading. Same-protocol streaming preserves parsed provider +events when the router does not aggregate or replace them; raw SSE bytes and +framing are not part of the compatibility contract. + +### Known issue: OpenAI Responses structured-output judges + +OpenAI Responses targets are accepted for structured-output judges, but the +shared Responses request encoder currently emits the Chat-compatible JSON +Schema object directly under `text.format`. This places `name`, `schema`, and +`strict` under `text.format.json_schema`; conforming Responses endpoints expect +those fields directly under `text.format`. InferenceHub therefore returns HTTP +400 with `Missing required parameter: 'text.format.name'`, and the affected +router follows its existing judge-failure or fall-open path. + +A hosted Relay process run passed 20 of 23 router-matrix cases. The only +failures were the Responses-judge variants of the capability classifier, +escalation router, and stage classifier fallback. OpenAI Responses remains +verified as a caller and ordinary serving-target protocol. Until the shared +`switchyard-translation` encoder is corrected, configure structured-output +judges with `protocol = "openai_chat"`. Follow-up work must add the inverse of +the existing Responses-to-neutral schema conversion plus core and +process-level regression coverage for all three affected router paths. + +The following integration components have not reached complete compatibility. +`Not built` identifies missing integration work rather than a hidden or +best-effort runtime path. + +| Compatibility Component | Status | Current Boundary | +|---|---|---| +| Pinned Relay process-level acceptance harness | Not built | The plugin's standalone tests run through its nested Cargo workspace. Relay 0.7.1 plus Ollama smoke coverage is manual and currently covers OpenAI Chat paths for escalation and stage routing. | +| OpenAI Responses and Anthropic Messages process-level routing matrix | Not built | Translation and in-process runtime coverage exists, including stage signals across all three caller protocols, but no automated Relay gateway matrix exercises every algorithm and target combination. | +| Real coding-agent acceptance harness | Not built | Codex, Claude Code, and Hermes sessions are not driven automatically through the packaged plugin. | +| Hosted-provider certification | Not built | No automated suite qualifies structured-output judges, serving targets, latency, or token cost against hosted provider APIs. | +| Native bundle platform and Relay-version matrix | Partial | The plugin was smoked manually on macOS arm64 with Relay 0.7.1. Dedicated packaged-plugin jobs do not yet cover Linux and Windows bundles or every supported Relay 0.7 release. | +| Dynamic-plugin lifecycle automation | Partial | Manifest validation, registration, enablement, execution, and unload were exercised manually; they are not part of a repeatable CI acceptance test. | +| Nested Relay lifecycle telemetry for managed provider calls | Not built | Relay records the outer serving call. Routing-only model calls emit `switchyard.routing.llm_call` marks with normalized usage, but Switchyard provider HTTP spans are not bridged into nested Relay LLM lifecycle events. | +| Provider `Retry-After` propagation | Not built | The outer routing loop uses bounded exponential backoff because the client error contract does not expose `Retry-After`. | +| Cross-protocol streaming loss diagnostics | Not built | Cross-protocol streams use normalized chunks, but the stream adapter does not surface the buffered translation engine's reject-lossy diagnostics. | +| Safe typed Relay asynchronous host adapter | Blocked on host API | The plugin uses its tested raw C ownership adapter until Relay exposes an equivalent safe asynchronous Rust facade. | + +Managed inner provider calls also do not re-enter Relay's downstream provider +middleware. This behavior is part of the current ownership boundary, not an +automatic compatibility fallback. + +Each completed routing-only model call emits a +`switchyard.routing.llm_call` ATOF mark. Its data identifies the algorithm, +attempt, call order, target, routing tier, role (`judge` or discarded +`candidate`), outcome, latency, and normalized provider token `usage`. The +successful call that serves the caller is deliberately excluded because +Relay's outer LLM end event already records that usage. A failed call, or a +provider response that omits usage, has `usage = null`. Consumers can therefore +add these marks to the outer LLM usage to measure total request compute without +double-counting the serving model. + +The plugin owns the outer routing retry loop. Each retry starts a fresh libsy +run. Random routing draws again; an algorithm configured with persistent state, +such as classifier session affinity, may intentionally retain its assignment. +Each target's built-in HTTP retry count is set to zero to avoid retrying a +failed target invisibly before reselection. A random target with `weight = 0` +is fallback-only and is not considered by the algorithm. Trusted fallback is +attempted at most once and, for streaming responses, only before the first +caller event is emitted. Outer routing retries use exponential backoff starting +at 250 milliseconds and capped at 2 seconds. They do not currently honor +provider `Retry-After` headers because the client error contract does not expose +that metadata to the routing loop. + +## Translation and stream fidelity + +`switchyard-translation` is the only request, response, and event translation +layer. It decodes caller JSON into Switchyard's neutral protocol, encodes each +selected call for the target protocol, decodes provider results, and encodes +`ReturnToAgent` back to the caller protocol. Relay codecs are not used. + +The streaming contract carries each parsed provider JSON event in a preservation +envelope alongside its normalized `LlmResponseChunk` representation. +Same-protocol routes replay the preserved JSON unchanged, including +provider-specific fields; this preserves parsed events, not raw SSE bytes or +framing. Cross-protocol routes encode only normalized chunks, and the streaming +helpers still do not expose the buffered translation engine's reject-lossy +diagnostics, so unsupported fields may be normalized or omitted. Replacing +normalized stream content or folding a stream into an aggregate drops the +per-event preservation envelope. + +## Configuration + +The manifest declares `compat.native_api = "1"` and Relay `>=0.7.0,<0.8`, and +the Rust SDK uses the exact published `0.7.0` crate. The manifest API value +selects Relay's released native plugin contract; the binary also requires the +v3 C host table shipped on the Relay 0.7 line. Rebuild the bundle when changing +SDK versions rather than assuming Rust dynamic-library compatibility from the +manifest value alone. + +A Relay project can configure a seeded weighted-random router as follows: + +```toml +version = 1 + +[[plugins.dynamic]] +manifest = "/opt/switchyard-relay-plugin/relay-plugin.toml" + +[plugins.dynamic.config] +version = 2 +priority = 0 +max_retries = 3 + +[plugins.dynamic.config.algorithm] +kind = "random" +seed = 42 + +[plugins.dynamic.config.default_targets] +openai_chat = "fast" + +[plugins.dynamic.config.targets.fast] +model = "provider/model" +protocol = "openai_chat" +endpoint = "/v1/chat/completions" +base_url = "https://provider.example.com" +weight = 1 +drop_caller_extra_body = true + +[plugins.dynamic.config.targets.fast.header_env] +authorization = "PROVIDER_AUTHORIZATION" +``` + +Target map keys such as `fast` are stable semantic names visible to libsy. The +target binding is authoritative for the provider model, protocol, endpoint, +base URL, weight, and environment-backed headers. Each `default_targets` key +both enables that inbound protocol and names its trusted fallback. + +`header_env` is the only custom provider-header source. It resolves values in +the plugin process at registration time so literal header values never appear +in configuration. Environment values must not appear in errors, routing marks, +spans, or debug output. The plugin does not inherit caller credentials for +managed calls. Each variable supplies the complete header value, so an +`authorization` value must include its scheme, such as `Bearer`. Literal +`headers` configuration is rejected; non-secret routing or tenancy headers must +also use `header_env`. + +Relay may intercept an OpenAI SDK call before the SDK materializes its +`extra_body` option into a provider request. Targets that reject this +caller-specific wrapper can set `drop_caller_extra_body = true`. The plugin +then drops the wrapper and its contents; it does not promote those values to +top-level provider fields. The default is `false` so lossless same-format +forwarding remains unchanged for targets that consume the extension. + +`extra_body` supplies non-secret provider defaults for a target. It is useful +for provider-specific controls such as disabling reasoning on a dedicated +judge model. Fields already present on the caller's request take precedence. +Do not put credentials in `extra_body`; use `header_env` for secrets. + +For `kind = "llm_classifier"`, the classifier target must use `openai_chat` or +`openai_responses`; libsy's judge request uses a JSON-schema response format +that cannot be represented losslessly by Anthropic Messages. Omitting `mode` +selects `capability`, preserving the original version-2 configuration shape. + +Escalation mode evaluates the weak model's completed response before returning +it or replacing it with a strong-model response: + +```toml +[plugins.dynamic.config.algorithm] +kind = "llm_classifier" +mode = "escalation" +classifier_target = "judge" +weak_target = "weak" +strong_target = "strong" +prompt = "Judge whether the weak model is stuck." +max_output_tokens = 512 + +[plugins.dynamic.config.algorithm.escalation] +confirmations = 2 +recent_turn_window = 28 +window_message_chars = 500 +``` + +`judge`, `weak`, and `strong` are keys in +`plugins.dynamic.config.targets`, configured with the same model, protocol, +URL, and `header_env` fields shown above. The judge must use `openai_chat` or +`openai_responses`; the serving targets may use any supported protocol. + +Use a dedicated, non-reasoning model for the judge when possible. Providers +that expose a reasoning switch can configure it on that target, for example: + +```toml +[plugins.dynamic.config.targets.judge] +model = "provider/non-reasoning-judge" +protocol = "openai_chat" +base_url = "https://provider.example.com" +extra_body = { think = false } +``` + +The packaged escalation rubric is intentionally detailed and can consume +roughly two thousand or more input tokens depending on the tokenizer. Every +unlatched request also pays for a complete judge call. A custom `prompt` can +reduce that cost, but should be evaluated against representative trajectories +before deployment. Reasoning models may spend `max_output_tokens` on hidden or +visible reasoning before returning the structured verdict; disable reasoning +with provider-supported `extra_body` controls or raise the cap after measuring. + +An unlatched streaming escalation request is intentionally buffered. Libsy must +read the complete weak response before asking the judge, so caller first-token +delivery waits for the weak call and judge verdict. A declined escalation is +reconstructed as a stream from the aggregate response, which drops the +provider-event preservation envelope. A confirmed escalation discards that +weak response and serves the strong target. + +The default `confirmations = 2` retains a streak per Switchyard session. Callers +must send a stable `x-switchyard-session-id` header for the streak and strong +latch to survive across turns. Without session identity each request has +isolated state and a multi-confirmation escalation cannot latch. + +A full stage router can combine tool-result signals, model-specific prompts, +handoff notes, and an optional judge for ambiguous turns: + +```toml +[plugins.dynamic.config.algorithm] +kind = "stage_router" +capable_target = "strong" +efficient_target = "weak" +picker = "efficient_first" +confidence_threshold = 0.5 +recent_turn_window = 3 +capable_system_prompt = "Diagnose before editing." +efficient_system_prompt = "Follow the settled plan." + +[plugins.dynamic.config.algorithm.handoff_notes] +escalation_note = "The previous model was stalling; pick up the diagnosis." +deescalation_note = "The task is settled; continue with the mechanical work." +only_on_wrong_signal_escalation = true + +[plugins.dynamic.config.algorithm.classifier] +target = "judge" +base_threshold = 0.5 +threshold_step = 0.1 +recent_turn_window = 3 +prompt = "Estimate whether the efficient target can finish this turn." +max_output_tokens = 512 +``` + +Stage routing reads normalized tool calls and tool results from OpenAI Chat, +OpenAI Responses, and Anthropic Messages traffic. When the signals do not cross +`confidence_threshold`, the optional classifier decides; if it is absent or +cannot decide, the configured picker's default tier serves the turn. The +classifier target has the same structured-output protocol restriction as the +standalone classifier. + +Ambiguous turns that reach the optional classifier add one judge call; +decisive tool signals do not. Decision marks include `routing_tier` for signal, +classifier, and picker-default paths, plus `decision_source` (`override`, +`tests_passed`, `dimensions`, `llm-classifier`, or `fall_open`) for stage-router +explainability. + +Version-1 service configuration, decision-only execution, and observe-only +mode are rejected. + +## Build and bundle + +The crate is a nested standalone Cargo workspace with `publish = false`. This +keeps the integration from changing Switchyard's root workspace manifest or +lockfile. Operators install a binary bundle rather than a Rust crate: + +```bash +cargo build --release \ + --manifest-path crates/switchyard-nemo-relay-plugin/Cargo.toml +python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ + --library crates/switchyard-nemo-relay-plugin/target/release/libswitchyard_nemo_relay_plugin.so \ + --output dist/switchyard-nemo-relay-plugin-linux-x86_64 +``` + +On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle +builder creates the minimal Relay package: the shared library, a materialized +manifest with Relay's inline SHA-256 integrity digest, and the JSON schema. + +Install the materialized bundle with Relay's normal lifecycle commands: + +```bash +nemo-relay plugins validate /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins add /opt/switchyard-relay-plugin/relay-plugin.toml +nemo-relay plugins enable nvidia.switchyard +nemo-relay plugins inspect nvidia.switchyard +``` + +## Validation expectations + +Before release, validate both routers against buffered and streaming OpenAI +Chat, OpenAI Responses, and Anthropic Messages providers. The acceptance suite +must cover same- and supported cross-protocol routes, deterministic weighted +routing, independent runs, classifier weak and strong selections, retry +reselection, exhaustion, exactly-once fallback, stream commitment, empty +streams, late errors, cancellation, credential privacy, and unmanaged +pass-through. + +The tests must also prove that managed target traffic reaches the provider +through `switchyard-llm-client`, never through Relay's provider continuation, +and that no Switchyard service or health endpoint is involved. diff --git a/crates/switchyard-nemo-relay-plugin/config.schema.json b/crates/switchyard-nemo-relay-plugin/config.schema.json new file mode 100644 index 000000000..63db7b40c --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/config.schema.json @@ -0,0 +1,217 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Switchyard NeMo Relay Plugin", + "description": "In-process Switchyard routing with Switchyard-owned provider HTTP dispatch.", + "type": "object", + "additionalProperties": false, + "required": ["version", "algorithm", "targets", "default_targets"], + "properties": { + "version": { + "const": 2, + "description": "Library-only Switchyard configuration version." + }, + "priority": { + "type": "integer", + "default": 0 + }, + "max_retries": { + "type": "integer", + "minimum": 0, + "maximum": 10, + "default": 3, + "description": "Routing retries after the initial libsy run. Every retry starts a fresh run." + }, + "algorithm": { + "description": "In-process random, capability, escalation, or stage-router configuration.", + "oneOf": [ + { + "type": "object", + "additionalProperties": false, + "required": ["kind"], + "properties": { + "kind": { "const": "random" }, + "seed": { "type": ["integer", "null"], "minimum": 0 } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "classifier_target", + "weak_target", + "strong_target", + "base_threshold" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "mode": { "const": "capability", "default": "capability" }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the judge. The target must use openai_chat or openai_responses because the judge requires a JSON-schema response format." + }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0 + }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, + "prompt": { "type": "string" }, + "session_affinity": { "type": "boolean", "default": false }, + "message_hash_fallback": { "type": "boolean", "default": false } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "mode", + "classifier_target", + "weak_target", + "strong_target", + "escalation" + ], + "properties": { + "kind": { "const": "llm_classifier" }, + "mode": { "const": "escalation" }, + "classifier_target": { + "type": "string", + "minLength": 1, + "description": "Semantic target name for the trajectory judge. The target must use openai_chat or openai_responses." + }, + "weak_target": { "type": "string", "minLength": 1 }, + "strong_target": { "type": "string", "minLength": 1 }, + "prompt": { "type": "string" }, + "max_output_tokens": { + "type": "integer", + "minimum": 1, + "default": 4096 + }, + "escalation": { + "type": "object", + "additionalProperties": false, + "properties": { + "confirmations": { "type": "integer", "minimum": 1, "default": 2 }, + "recent_turn_window": { "type": "integer", "minimum": 1, "default": 28 }, + "window_message_chars": { "type": "integer", "minimum": 50, "default": 500 } + } + } + } + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "kind", + "capable_target", + "efficient_target", + "picker", + "confidence_threshold" + ], + "properties": { + "kind": { "const": "stage_router" }, + "capable_target": { "type": "string", "minLength": 1 }, + "efficient_target": { "type": "string", "minLength": 1 }, + "picker": { "enum": ["capable_first", "efficient_first"] }, + "confidence_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "recent_turn_window": { + "type": ["integer", "null"], + "minimum": 0, + "description": "Trailing tool results scored for the current turn. Null uses libsy's default window." + }, + "capable_system_prompt": { "type": "string" }, + "efficient_system_prompt": { "type": "string" }, + "handoff_notes": { + "type": "object", + "additionalProperties": false, + "required": ["escalation_note"], + "properties": { + "escalation_note": { "type": "string", "minLength": 1 }, + "deescalation_note": { "type": ["string", "null"], "minLength": 1 }, + "only_on_wrong_signal_escalation": { "type": "boolean", "default": true } + } + }, + "classifier": { + "type": "object", + "additionalProperties": false, + "required": ["target", "base_threshold"], + "properties": { + "target": { + "type": "string", + "minLength": 1, + "description": "Judge target used only when stage signals are ambiguous. It must use openai_chat or openai_responses." + }, + "base_threshold": { "type": "number", "minimum": 0, "maximum": 1 }, + "threshold_step": { "type": "number", "minimum": 0, "default": 0 }, + "recent_turn_window": { "type": ["integer", "null"], "minimum": 0 }, + "prompt": { "type": "string" }, + "max_output_tokens": { "type": "integer", "minimum": 1, "default": 4096 } + } + } + } + } + ] + }, + "targets": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["model", "protocol", "base_url"], + "properties": { + "model": { "type": "string", "minLength": 1 }, + "protocol": { + "enum": ["openai_chat", "openai_responses", "anthropic_messages"] + }, + "endpoint": { + "type": "string", + "pattern": "^$|^/", + "description": "Optional provider endpoint override. The resolved URL must end in the canonical route for the selected protocol." + }, + "base_url": { + "type": "string", + "pattern": "^https?://" + }, + "weight": { "type": "number", "minimum": 0, "default": 1 }, + "drop_caller_extra_body": { + "type": "boolean", + "default": false, + "description": "Drop an intercepted OpenAI SDK extra_body wrapper instead of forwarding it to targets that reject caller-specific extensions." + }, + "extra_body": { + "type": "object", + "default": {}, + "description": "Non-secret provider request defaults, such as judge reasoning controls. Caller-provided fields take precedence.", + "additionalProperties": true + }, + "header_env": { + "type": "object", + "description": "Sole custom provider-header source. Maps header names to environment-variable names resolved by the plugin process so literal values are never stored in configuration.", + "additionalProperties": { "type": "string", "minLength": 1 } + } + } + } + }, + "default_targets": { + "type": "object", + "description": "Maps each managed inbound protocol to its trusted fallback target.", + "minProperties": 1, + "additionalProperties": false, + "properties": { + "openai_chat": { "type": "string", "minLength": 1 }, + "openai_responses": { "type": "string", "minLength": 1 }, + "anthropic_messages": { "type": "string", "minLength": 1 } + } + } + } +} diff --git a/crates/switchyard-nemo-relay-plugin/relay-plugin.toml b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml new file mode 100644 index 000000000..920decc31 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/relay-plugin.toml @@ -0,0 +1,31 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +manifest_version = 1 + +[plugin] +id = "nvidia.switchyard" +kind = "rust_dynamic" + +[compat] +relay = ">=0.7.0,<0.8" +native_api = "1" + +[defaults] +enabled = false + +[capabilities] +items = ["plugin_native", "config_schema"] + +[config_schema] +path = "config.schema.json" + +[source] +artifact = "" + +[integrity] +sha256 = "sha256:" + +[load] +library = "" +symbol = "nemo_relay_register_plugin" diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py new file mode 100644 index 000000000..550f836ed --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -0,0 +1,62 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Materialize the minimal Relay plugin bundle from a compiled cdylib.""" + +from __future__ import annotations + +import argparse +import hashlib +import shutil +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] + + +def digest(path: Path) -> str: + """Return the lowercase SHA-256 digest for a file.""" + value = hashlib.sha256() + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + value.update(chunk) + return value.hexdigest() + + +def main() -> None: + """Materialize a Relay-loadable plugin bundle in an empty directory.""" + parser = argparse.ArgumentParser() + parser.add_argument("--library", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + args = parser.parse_args() + + library = args.library.resolve() + if not library.is_file(): + parser.error(f"compiled plugin library does not exist: {library}") + + manifest = (CRATE_ROOT / "relay-plugin.toml").read_text(encoding="utf-8") + placeholders = ("", "") + missing = [placeholder for placeholder in placeholders if placeholder not in manifest] + if missing: + parser.error(f"plugin manifest is missing placeholders: {', '.join(missing)}") + + output = args.output.resolve() + if output.exists() and not output.is_dir(): + parser.error(f"bundle output exists and is not a directory: {output}") + if output.is_dir() and any(output.iterdir()): + parser.error(f"bundle output directory must be empty: {output}") + output.mkdir(parents=True, exist_ok=True) + + artifact = output / library.name + shutil.copy2(library, artifact) + shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") + + artifact_digest = digest(artifact) + manifest = manifest.replace("", artifact.name) + manifest = manifest.replace("", artifact_digest) + (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") + + print(output) + + +if __name__ == "__main__": + main() diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs new file mode 100644 index 000000000..c468f949d --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -0,0 +1,469 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Switchyard-owned HTTP clients bound to one semantic routing target. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::Value as Json; +use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; +use switchyard_protocol::{ + ContentBlock, Context, Decision, LlmClientError, Message, Request, Response, Role, + RoutedLlmClient, ToolCall, ToolResult, WireFormat, +}; +use switchyard_translation::TranslationEngine; + +use crate::translation; + +/// A provider client bound to one configured Switchyard target. +/// +/// libsy routes with a stable semantic name (for example `fast`). The provider +/// still expects its own model id (for example `meta/llama-3.1-8b-instruct`). +/// Keeping that mapping here prevents an algorithm's semantic labels from +/// leaking into provider requests. +pub(crate) struct TargetClient { + provider_model: String, + target_format: WireFormat, + drop_caller_extra_body: bool, + inner: TranslatingLlmClient, + translation: TranslationEngine, +} + +impl TargetClient { + pub(crate) fn new( + provider_model: String, + target_format: WireFormat, + dispatch_url: String, + headers: BTreeMap, + extra_body: BTreeMap, + drop_caller_extra_body: bool, + ) -> Result { + let backend_config = HttpBackendConfig { + // `dispatch_url` is already resolved by configuration. Backend URL + // joining accepts a complete canonical endpoint as well as a base + // URL/prefix. + base_url: dispatch_url, + api_key: None, + extra_headers: headers, + extra_body, + // Routing retries belong to the plugin: every retry must start a + // fresh libsy run and obtain a fresh decision. + max_retries: 0, + }; + let backend = match target_format { + WireFormat::OpenAiChat => Backend::OpenAiChat(backend_config), + WireFormat::OpenAiResponses => Backend::OpenAiResponses(backend_config), + WireFormat::AnthropicMessages => Backend::Anthropic(backend_config), + }; + let model = ModelConfig::new(provider_model.clone(), backend, None); + let inner = TranslatingLlmClient::new(&[model])?; + Ok(Self { + provider_model, + target_format, + drop_caller_extra_body, + inner, + translation: TranslationEngine::default(), + }) + } + + /// Retargets only the provider-facing transport metadata. + /// + /// Correlation and agent identity remain available to libsy, while inbound + /// HTTP headers are deliberately removed. Provider credentials come solely + /// from this target's `header_env` configuration. + fn prepare_request(&self, mut request: Request, decision: &Decision) -> Request { + if !decision.is_answer_call() { + sanitize_judge_request(&mut request); + } + if decision.reasoning() == Some("escalation classifier: efficient tier") { + // Escalation always buffers this draft before judging it. Asking the + // provider for a buffered response preserves normalized usage for ATOF; + // libsy reconstructs a caller stream when the weak draft wins. + request.llm_request.stream = false; + request.llm_request.preservation.requests.clear(); + } + let metadata = request.metadata.get_or_insert_default(); + metadata.wire_format = Some(self.target_format); + metadata.http_headers = None; + if self.drop_caller_extra_body { + request.llm_request.extensions.fields.remove("extra_body"); + for preserved in request.llm_request.preservation.requests.values_mut() { + if let Some(body) = preserved.as_object_mut() { + body.remove("extra_body"); + } + } + } + request + } +} + +#[async_trait] +impl RoutedLlmClient for TargetClient { + async fn call( + &self, + ctx: Context, + request: Request, + decision: Arc, + ) -> Result { + let request = self.prepare_request(request, decision.as_ref()); + translation::validate_target_request( + &self.translation, + self.target_format, + &request.llm_request, + ) + .map_err(LlmClientError::RequestEncoding)?; + self.inner + .call_rewrite_model(ctx, request, Some(&self.provider_model)) + .await + } +} + +/// Maximum plain-text context retained from one native tool block in a judge request. +const MAX_JUDGE_TOOL_CONTEXT_CHARS: usize = 4_096; + +/// Keep judge requests provider-neutral. Native tool turns without their original +/// definitions are rejected by some OpenAI-compatible Bedrock gateways, while the +/// text evidence is still valuable to the classifier. +fn sanitize_judge_request(request: &mut Request) { + request.llm_request.messages = request + .llm_request + .messages + .drain(..) + .map(|message| Message { + role: if message.role == Role::Tool { + Role::User + } else { + message.role + }, + content: message + .content + .into_iter() + .map(|block| match block { + ContentBlock::ToolCall(call) => ContentBlock::Text { + text: bounded_tool_context(tool_call_text(call)), + }, + ContentBlock::ToolResult(result) => ContentBlock::Text { + text: bounded_tool_context(tool_result_text(result)), + }, + ordinary => ordinary, + }) + .collect(), + }) + .collect(); + request.llm_request.tools.clear(); + request.llm_request.tool_choice = None; + if let Some(response_format) = request.llm_request.output.response_format.as_mut() { + remove_numeric_schema_bounds(response_format); + } + request.llm_request.preservation.requests.clear(); +} + +fn tool_call_text(call: ToolCall) -> String { + format!( + "[tool call]\nid: {}\nname: {}\narguments: {}", + Json::String(call.id), + Json::String(call.name), + call.arguments + ) +} + +fn tool_result_text(result: ToolResult) -> String { + let content = result + .content + .into_iter() + .map(tool_content_text) + .collect::>() + .join("\n"); + format!( + "[tool result]\ncall_id: {}\nis_error: {}\ncontent:\n{}", + Json::String(result.tool_call_id), + result + .is_error + .map_or_else(|| "unknown".to_string(), |value| value.to_string()), + content + ) +} + +fn tool_content_text(block: ContentBlock) -> String { + match block { + ContentBlock::Text { text } + | ContentBlock::Reasoning { text, .. } + | ContentBlock::Refusal { text } => text, + ContentBlock::ToolCall(call) => tool_call_text(call), + ContentBlock::ToolResult(result) => tool_result_text(result), + ContentBlock::Image { .. } => "[image omitted]".to_string(), + ContentBlock::Audio { .. } => "[audio omitted]".to_string(), + ContentBlock::Video { .. } => "[video omitted]".to_string(), + ContentBlock::File { .. } => "[file omitted]".to_string(), + ContentBlock::Unknown { provider, .. } => { + format!("[unsupported {provider} content omitted]") + } + } +} + +fn bounded_tool_context(text: String) -> String { + const TRUNCATED: &str = "\n[truncated]"; + let keep = MAX_JUDGE_TOOL_CONTEXT_CHARS - TRUNCATED.chars().count(); + let mut chars = text.chars(); + let prefix = chars.by_ref().take(keep).collect::(); + if chars.next().is_some() { + prefix + TRUNCATED + } else { + prefix + } +} + +fn remove_numeric_schema_bounds(value: &mut Json) { + match value { + Json::Object(object) => { + for key in ["minimum", "maximum", "exclusiveMinimum", "exclusiveMaximum"] { + object.remove(key); + } + for child in object.values_mut() { + remove_numeric_schema_bounds(child); + } + } + Json::Array(values) => { + for child in values { + remove_numeric_schema_bounds(child); + } + } + _ => {} + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use switchyard_protocol::{ + LlmRequest, Metadata, PreservationMetadata, ProviderExtensions, ToolChoice, ToolDefinition, + }; + + fn decision() -> Decision { + Decision::new("target", None, true) + } + + fn client(format: WireFormat) -> TargetClient { + TargetClient::new( + "provider/model".into(), + format, + match format { + WireFormat::OpenAiChat => "https://provider.example/v1/chat/completions".into(), + WireFormat::OpenAiResponses => "https://provider.example/v1/responses".into(), + WireFormat::AnthropicMessages => "https://provider.example/v1/messages".into(), + }, + BTreeMap::new(), + BTreeMap::new(), + false, + ) + .unwrap() + } + + #[test] + fn target_preparation_forces_format_and_removes_inbound_headers() { + let client = client(WireFormat::AnthropicMessages); + let request = Request { + metadata: Some(Metadata { + correlation_id: Some("request-123".into()), + wire_format: Some(WireFormat::OpenAiChat), + http_headers: Some(http::HeaderMap::from_iter([ + ( + http::HeaderName::from_static("authorization"), + http::HeaderValue::from_static("Bearer caller-secret"), + ), + ( + http::HeaderName::from_static("x-caller-only"), + http::HeaderValue::from_static("must-not-forward"), + ), + ])), + ..Metadata::default() + }), + ..Request::default() + }; + + let prepared = client.prepare_request(request, &decision()); + let metadata = prepared.metadata.unwrap(); + assert_eq!(metadata.wire_format, Some(WireFormat::AnthropicMessages)); + assert_eq!(metadata.correlation_id.as_deref(), Some("request-123")); + assert!(metadata.http_headers.is_none()); + } + + #[test] + fn missing_metadata_is_created_for_the_target_format() { + let client = client(WireFormat::OpenAiResponses); + let prepared = client.prepare_request(Request::default(), &decision()); + assert_eq!( + prepared.metadata.and_then(|metadata| metadata.wire_format), + Some(WireFormat::OpenAiResponses) + ); + } + + #[test] + fn configured_target_drops_intercepted_caller_extra_body() { + let client = TargetClient::new( + "provider/model".into(), + WireFormat::OpenAiChat, + "https://provider.example/v1/chat/completions".into(), + BTreeMap::new(), + BTreeMap::new(), + true, + ) + .unwrap(); + let request = Request { + llm_request: LlmRequest { + extensions: ProviderExtensions { + fields: serde_json::Map::from_iter([( + "extra_body".into(), + json!({"reasoning": {"effort": "medium"}}), + )]), + }, + preservation: PreservationMetadata { + requests: BTreeMap::from([( + WireFormat::OpenAiChat.into(), + json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "extra_body": { + "reasoning": {"effort": "medium"}, + "session_id": "hermes-session" + } + }), + )]), + ..PreservationMetadata::default() + }, + ..LlmRequest::default() + }, + ..Request::default() + }; + + let prepared = client.prepare_request(request, &decision()); + assert!( + !prepared + .llm_request + .extensions + .fields + .contains_key("extra_body") + ); + assert!( + prepared + .llm_request + .preservation + .requests + .values() + .all(|body| body.get("extra_body").is_none()) + ); + } + + #[test] + fn judge_preparation_sanitizes_tool_history_and_schema_dialect() { + let client = client(WireFormat::OpenAiChat); + let request = Request { + llm_request: LlmRequest { + messages: vec![ + Message::text(Role::User, "inspect the workspace"), + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: "call-1".into(), + name: "terminal".into(), + arguments: json!({"command": "pwd"}), + })], + }, + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".into(), + content: vec![ContentBlock::Text { + text: format!("result {} TAIL", "x".repeat(5_000)), + }], + is_error: Some(false), + })], + }, + ], + tools: vec![ToolDefinition { + name: "terminal".into(), + description: None, + parameters: json!({"type": "object"}), + strict: None, + }], + tool_choice: Some(ToolChoice::Required), + output: switchyard_protocol::OutputParams { + max_output_tokens: Some(64), + response_format: Some(json!({ + "type": "json_schema", + "json_schema": { + "schema": { + "properties": { + "p_solve": { + "type": "number", + "minimum": 0.0, + "maximum": 1.0 + } + } + } + } + })), + }, + ..LlmRequest::default() + }, + ..Request::default() + }; + + let prepared = client.prepare_request( + request, + &Decision::new("judge", Some("structured judge".into()), false), + ); + + assert!(prepared.llm_request.tools.is_empty()); + assert_eq!(prepared.llm_request.tool_choice, None); + assert!( + prepared + .llm_request + .messages + .iter() + .all(|message| message.role != Role::Tool) + ); + let text = prepared + .llm_request + .messages + .iter() + .filter_map(|message| message.text_content("\n")) + .collect::>() + .join("\n"); + assert!(text.contains("[tool call]")); + assert!(text.contains("terminal")); + assert!(text.contains("[tool result]")); + assert!(text.contains("[truncated]")); + assert!(!text.contains("TAIL")); + let schema = prepared.llm_request.output.response_format.unwrap(); + let p_solve = schema + .pointer("/json_schema/schema/properties/p_solve") + .unwrap(); + assert!(p_solve.get("minimum").is_none()); + assert!(p_solve.get("maximum").is_none()); + } + + #[test] + fn escalation_candidate_is_buffered_for_usage_accounting() { + let client = client(WireFormat::OpenAiChat); + let mut request = Request::default(); + request.llm_request.stream = true; + request.llm_request.preservation.requests.insert( + WireFormat::OpenAiChat.into(), + json!({"model": "route", "stream": true}), + ); + let decision = Decision::new( + "weak", + Some("escalation classifier: efficient tier".into()), + true, + ); + + let prepared = client.prepare_request(request, &decision); + + assert!(!prepared.llm_request.stream); + assert!(prepared.llm_request.preservation.requests.is_empty()); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs new file mode 100644 index 000000000..435718fcb --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -0,0 +1,1186 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use http::Uri; +use http::header::{HeaderName, HeaderValue}; +use serde::Deserialize; +use serde_json::Value as Json; +use switchyard_libsy::{ + Algorithm, ClassifierContractConfig, EscalationJudgeConfig, HandoffNoteConfig, + LlmClassifierConfig, LlmFallback, LlmTarget, LlmTargetSet, LlmTaskClassifier, PickerMode, + Random, StageRouter, StageRouterConfig, TargetPrompts, TaskClassifierConfig, +}; +use switchyard_protocol::{RoutedLlmClient, WireFormat}; + +use crate::client::TargetClient; + +pub(crate) fn protocol_from_call(name: &str) -> Option { + match name { + "openai.chat_completions" => Some(WireFormat::OpenAiChat), + "openai.responses" => Some(WireFormat::OpenAiResponses), + "anthropic.messages" => Some(WireFormat::AnthropicMessages), + _ => None, + } +} + +const fn default_endpoint(protocol: WireFormat) -> &'static str { + match protocol { + WireFormat::OpenAiChat => "/v1/chat/completions", + WireFormat::OpenAiResponses => "/v1/responses", + WireFormat::AnthropicMessages => "/v1/messages", + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct TargetBinding { + model: String, + protocol: WireFormat, + #[serde(default)] + endpoint: String, + base_url: String, + #[serde(default = "default_weight")] + weight: f64, + #[serde(default)] + drop_caller_extra_body: bool, + #[serde(default)] + header_env: BTreeMap, + #[serde(default)] + extra_body: BTreeMap, +} + +impl TargetBinding { + fn dispatch_url(&self) -> String { + let base = self.base_url.trim_end_matches('/'); + let default = default_endpoint(self.protocol); + if self.endpoint.is_empty() && base.ends_with(default) { + return base.to_string(); + } + let endpoint = if self.endpoint.is_empty() { + default + } else { + &self.endpoint + }; + let endpoint = if base.ends_with("/v1") && endpoint.starts_with("/v1/") { + &endpoint[3..] + } else { + endpoint + }; + format!("{base}{endpoint}") + } + + fn validate(&self, name: &str) -> Result<(), String> { + if self.model.trim().is_empty() { + return Err(format!("target {name:?} model must be non-empty")); + } + if !self.endpoint.is_empty() && !self.endpoint.starts_with('/') { + return Err(format!( + "target {name:?} endpoint must be empty or begin with '/'" + )); + } + if !self.weight.is_finite() || self.weight < 0.0 { + return Err(format!( + "target {name:?} weight must be finite and nonnegative" + )); + } + validate_dispatch_url(name, self.protocol, &self.dispatch_url())?; + self.validate_headers(name) + } + + fn validate_headers(&self, target_name: &str) -> Result<(), String> { + let mut normalized = BTreeSet::new(); + for (name, variable) in &self.header_env { + let canonical = validate_header_name(name)?; + if !normalized.insert(canonical) { + return Err(format!( + "target {target_name:?} configures header {name:?} more than once (header names are case-insensitive)" + )); + } + if variable.trim().is_empty() { + return Err(format!( + "environment variable name for target header {name:?} must not be empty" + )); + } + if variable.as_bytes().contains(&b'=') || variable.as_bytes().contains(&b'\0') { + return Err(format!( + "environment variable name for target header {name:?} must not contain '=' or NUL" + )); + } + } + Ok(()) + } + + fn prepare(&self) -> Result { + let mut headers = BTreeMap::new(); + for (name, variable) in &self.header_env { + let value = std::env::var(variable) + .map_err(|_| format!("environment variable {variable:?} is not set"))?; + validate_header(name, &value)?; + headers.insert(name.clone(), value); + } + let dispatch_url = self.dispatch_url(); + let client = TargetClient::new( + self.model.clone(), + self.protocol, + dispatch_url, + headers, + self.extra_body.clone(), + self.drop_caller_extra_body, + ) + .map_err(|error| format!("failed to create target HTTP client: {error}"))?; + Ok(PreparedTargetBinding { + client: Arc::new(client), + }) + } +} + +pub(crate) struct PreparedTargetBinding { + pub(crate) client: Arc, +} + +#[derive(Clone, Copy, Default, Deserialize)] +#[serde(rename_all = "snake_case")] +enum LlmClassifierMode { + #[default] + Capability, + Escalation, +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct LlmClassifierAlgorithmConfig { + #[serde(default)] + mode: LlmClassifierMode, + classifier_target: String, + weak_target: String, + strong_target: String, + #[serde(default)] + base_threshold: Option, + #[serde(default)] + threshold_step: Option, + #[serde(default)] + session_affinity: Option, + #[serde(default)] + message_hash_fallback: Option, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + prompt: Option, + #[serde(default = "default_classifier_max_output_tokens")] + max_output_tokens: u64, + #[serde(default)] + escalation: Option, +} + +impl LlmClassifierAlgorithmConfig { + fn capability_config(&self) -> Result { + if self.escalation.is_some() { + return Err( + "llm_classifier capability mode does not accept escalation settings".into(), + ); + } + let base_threshold = self + .base_threshold + .ok_or_else(|| "llm_classifier capability mode requires base_threshold".to_string())?; + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + Ok(TaskClassifierConfig { + base_threshold, + threshold_step: self.threshold_step.unwrap_or_default(), + session_affinity: self.session_affinity.unwrap_or_default(), + message_hash_fallback: self.message_hash_fallback.unwrap_or_default(), + recent_turn_window: self.recent_turn_window, + contract, + max_output_tokens: self.max_output_tokens, + }) + } + + fn escalation_config( + &self, + ) -> Result<(ClassifierContractConfig, EscalationJudgeConfig), String> { + if self.base_threshold.is_some() + || self.threshold_step.is_some() + || self.session_affinity.is_some() + || self.message_hash_fallback.is_some() + || self.recent_turn_window.is_some() + { + return Err( + "llm_classifier escalation mode does not accept capability settings".into(), + ); + } + let config = self.escalation.clone().ok_or_else(|| { + "llm_classifier escalation mode requires escalation settings".to_string() + })?; + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + Ok((contract, config)) + } +} + +#[derive(Clone, Deserialize)] +#[serde(deny_unknown_fields)] +struct StageFallbackConfig { + target: String, + base_threshold: f64, + #[serde(default)] + threshold_step: f64, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + prompt: Option, + #[serde(default = "default_classifier_max_output_tokens")] + max_output_tokens: u64, +} + +impl StageFallbackConfig { + fn classifier_config(&self) -> TaskClassifierConfig { + let mut contract = ClassifierContractConfig::default(); + if let Some(prompt) = &self.prompt { + contract = contract.with_prompt(prompt.clone()); + } + TaskClassifierConfig { + base_threshold: self.base_threshold, + threshold_step: self.threshold_step, + session_affinity: false, + message_hash_fallback: false, + recent_turn_window: self.recent_turn_window, + contract, + max_output_tokens: self.max_output_tokens, + } + } +} + +#[derive(Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum AlgorithmConfig { + Random { + #[serde(default)] + seed: Option, + }, + LlmClassifier { + #[serde(flatten)] + config: LlmClassifierAlgorithmConfig, + }, + StageRouter { + capable_target: String, + efficient_target: String, + picker: PickerMode, + confidence_threshold: f64, + #[serde(default)] + recent_turn_window: Option, + #[serde(default)] + capable_system_prompt: Option, + #[serde(default)] + efficient_system_prompt: Option, + #[serde(default)] + handoff_notes: Option, + #[serde(default)] + classifier: Option, + }, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SwitchyardConfig { + version: u32, + #[serde(default)] + pub(crate) priority: i32, + #[serde(default = "default_max_retries")] + max_retries: u32, + algorithm: AlgorithmConfig, + targets: BTreeMap, + default_targets: BTreeMap, +} + +pub(crate) struct PreparedConfig { + pub(crate) max_retries: u32, + pub(crate) algorithm: Arc, + pub(crate) targets: BTreeMap, + pub(crate) default_targets: BTreeMap, + pub(crate) target_tiers: BTreeMap, + pub(crate) stage_marks: Option, +} + +#[derive(Clone)] +pub(crate) struct StageMarkConfig { + pub(crate) picker: PickerMode, + pub(crate) confidence_threshold: f64, + pub(crate) recent_turn_window: Option, + pub(crate) classifier_enabled: bool, +} + +impl SwitchyardConfig { + pub(crate) fn validate(&self) -> Result<(), String> { + self.validate_structure()?; + self.build_algorithm(None).map(drop) + } + + fn validate_structure(&self) -> Result<(), String> { + if self.version != 2 { + return Err(format!( + "unsupported Switchyard config version {}; version 1 used switchyard-server; migrate to version = 2", + self.version + )); + } + if self.max_retries > 10 { + return Err("max_retries must not exceed 10".into()); + } + if self.targets.is_empty() { + return Err("targets must not be empty".into()); + } + if self.default_targets.is_empty() { + return Err("default_targets must not be empty".into()); + } + for (name, target) in &self.targets { + if name.trim().is_empty() { + return Err("target names must be non-empty".into()); + } + target.validate(name)?; + } + for (protocol, fallback) in &self.default_targets { + let target = self + .targets + .get(fallback) + .ok_or_else(|| format!("default target {fallback:?} is not configured"))?; + if target.protocol != *protocol { + return Err(format!( + "default target {fallback:?} must use protocol {}", + protocol.as_str() + )); + } + } + Ok(()) + } + + pub(crate) fn prepare(self) -> Result { + self.validate_structure()?; + let (target_tiers, stage_marks) = self.routing_mark_config(); + let targets = self + .targets + .iter() + .map(|(name, target)| target.prepare().map(|prepared| (name.clone(), prepared))) + .collect::, _>>()?; + let algorithm = self.build_algorithm(Some(&targets))?; + Ok(PreparedConfig { + max_retries: self.max_retries, + algorithm, + targets, + default_targets: self.default_targets, + target_tiers, + stage_marks, + }) + } + + fn routing_mark_config(&self) -> (BTreeMap, Option) { + match &self.algorithm { + AlgorithmConfig::Random { .. } => (BTreeMap::new(), None), + AlgorithmConfig::LlmClassifier { config } => ( + BTreeMap::from([ + (config.weak_target.clone(), "weak"), + (config.strong_target.clone(), "strong"), + ]), + None, + ), + AlgorithmConfig::StageRouter { + capable_target, + efficient_target, + picker, + confidence_threshold, + recent_turn_window, + classifier, + .. + } => ( + BTreeMap::from([ + (capable_target.clone(), "strong"), + (efficient_target.clone(), "weak"), + ]), + Some(StageMarkConfig { + picker: *picker, + confidence_threshold: *confidence_threshold, + recent_turn_window: *recent_turn_window, + classifier_enabled: classifier.is_some(), + }), + ), + } + } + + fn build_algorithm( + &self, + prepared: Option<&BTreeMap>, + ) -> Result, String> { + let target = |name: &str| { + if !self.targets.contains_key(name) { + return Err(format!("algorithm target {name:?} is not configured")); + } + Ok(match prepared { + Some(targets) => { + targets + .get(name) + .ok_or_else(|| format!("algorithm target {name:?} was not prepared"))?; + LlmTarget { + semantic_name: name.to_string(), + } + } + None => LlmTarget { + semantic_name: name.to_string(), + }, + }) + }; + + match &self.algorithm { + AlgorithmConfig::Random { seed } => { + let routable = self + .targets + .iter() + .filter(|(_, binding)| binding.weight > 0.0) + .collect::>(); + if routable.is_empty() { + return Err( + "random routing requires at least one positive target weight".into(), + ); + } + let targets = routable + .iter() + .map(|(name, _)| target(name)) + .collect::, _>>()?; + let weights = routable + .iter() + .map(|(_, binding)| binding.weight) + .collect::>(); + Random::new(LlmTargetSet::new(targets), Some(weights), *seed) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::LlmClassifier { config } => { + self.validate_judge_target(&config.classifier_target)?; + let algorithm = match config.mode { + LlmClassifierMode::Capability => LlmClassifierConfig::Capability { + judge_target: target(&config.classifier_target)?, + efficient_target: target(&config.weak_target)?, + capable_target: target(&config.strong_target)?, + config: config.capability_config()?, + }, + LlmClassifierMode::Escalation => { + let (contract, escalation) = config.escalation_config()?; + LlmClassifierConfig::Escalation { + judge_target: target(&config.classifier_target)?, + efficient_target: target(&config.weak_target)?, + capable_target: target(&config.strong_target)?, + contract, + config: escalation, + max_output_tokens: config.max_output_tokens, + } + } + }; + LlmTaskClassifier::new(algorithm) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + AlgorithmConfig::StageRouter { + capable_target, + efficient_target, + picker, + confidence_threshold, + recent_turn_window, + capable_system_prompt, + efficient_system_prompt, + handoff_notes, + classifier, + } => { + let capable = target(capable_target)?; + let efficient = target(efficient_target)?; + let mut config = StageRouterConfig::new(*picker, *confidence_threshold); + config.recent_window = *recent_turn_window; + config.handoff_notes = handoff_notes.clone(); + let mut prompts = TargetPrompts::default(); + if let Some(prompt) = capable_system_prompt { + prompts = prompts.with(capable_target, prompt); + } + if let Some(prompt) = efficient_system_prompt { + prompts = prompts.with(efficient_target, prompt); + } + config.tier_prompts = prompts; + if let Some(classifier) = classifier { + self.validate_judge_target(&classifier.target)?; + config.llm_fallback = Some(LlmFallback { + judge_target: target(&classifier.target)?, + config: classifier.classifier_config(), + }); + } + StageRouter::new(capable, efficient, config) + .map(|algorithm| Arc::new(algorithm) as Arc) + .map_err(|error| error.to_string()) + } + } + } + + fn validate_judge_target(&self, name: &str) -> Result<(), String> { + let binding = self + .targets + .get(name) + .ok_or_else(|| format!("algorithm target {name:?} is not configured"))?; + if binding.protocol == WireFormat::AnthropicMessages { + return Err(format!( + "classifier target {name:?} uses anthropic_messages, which cannot encode the required JSON-schema response format without loss; use an openai_chat or openai_responses target" + )); + } + Ok(()) + } +} + +fn validate_dispatch_url( + target_name: &str, + protocol: WireFormat, + dispatch_url: &str, +) -> Result<(), String> { + let uri = dispatch_url + .parse::() + .map_err(|error| format!("target {target_name:?} has invalid URL: {error}"))?; + if !matches!(uri.scheme_str(), Some("http" | "https")) { + return Err(format!( + "target {target_name:?} base_url must use http or https" + )); + } + let authority = uri + .authority() + .ok_or_else(|| format!("target {target_name:?} URL must include a host"))?; + if authority.host().is_empty() { + return Err(format!("target {target_name:?} URL must include a host")); + } + if authority.as_str().contains('@') { + return Err(format!( + "target {target_name:?} URL must not contain embedded credentials" + )); + } + if uri.query().is_some() { + return Err(format!( + "target {target_name:?} URL query parameters are not supported" + )); + } + + // The current switchyard-llm-client accepts provider base URLs and complete + // canonical endpoints. Reject a custom terminal route to avoid + // allowing Backend::url() to append another provider suffix silently. + let expected_suffix = match protocol { + WireFormat::OpenAiChat => "/chat/completions", + WireFormat::OpenAiResponses => "/responses", + WireFormat::AnthropicMessages => "/v1/messages", + }; + if !uri.path().ends_with(expected_suffix) { + return Err(format!( + "target {target_name:?} endpoint must resolve to a canonical {protocol} route ending in {expected_suffix:?}" + )); + } + Ok(()) +} + +fn validate_header_name(name: &str) -> Result { + let parsed = HeaderName::from_bytes(name.as_bytes()) + .map_err(|error| format!("invalid target header name {name:?}: {error}"))?; + let canonical = parsed.as_str().to_ascii_lowercase(); + if is_forbidden_target_header(&canonical) { + return Err(format!( + "target header {name:?} is controlled by the HTTP transport and cannot be configured" + )); + } + Ok(canonical) +} + +fn validate_header(name: &str, value: &str) -> Result { + let canonical = validate_header_name(name)?; + HeaderValue::from_str(value) + .map_err(|error| format!("invalid target header value for {name:?}: {error}"))?; + Ok(canonical) +} + +fn is_forbidden_target_header(name: &str) -> bool { + matches!( + name, + "connection" + | "content-length" + | "host" + | "keep-alive" + | "proxy-connection" + | "proxy-authenticate" + | "proxy-authorization" + | "te" + | "trailer" + | "transfer-encoding" + | "upgrade" + ) || name.starts_with("x-nemo-relay-internal-") +} + +const fn default_max_retries() -> u32 { + 3 +} + +const fn default_weight() -> f64 { + 1.0 +} + +fn default_classifier_max_output_tokens() -> u64 { + TaskClassifierConfig::default().max_output_tokens +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::{Value, json}; + + fn binding(protocol: WireFormat, model: &str) -> TargetBinding { + TargetBinding { + model: model.into(), + protocol, + endpoint: String::new(), + base_url: "https://provider.example/v1".into(), + weight: 1.0, + drop_caller_extra_body: false, + header_env: BTreeMap::new(), + extra_body: BTreeMap::new(), + } + } + + fn config() -> SwitchyardConfig { + SwitchyardConfig { + version: 2, + priority: 0, + max_retries: 3, + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([ + ( + "chat".into(), + binding(WireFormat::OpenAiChat, "provider/chat"), + ), + ( + "responses".into(), + binding(WireFormat::OpenAiResponses, "provider/responses"), + ), + ( + "anthropic".into(), + binding(WireFormat::AnthropicMessages, "provider/anthropic"), + ), + ]), + default_targets: BTreeMap::from([ + (WireFormat::OpenAiChat, "chat".into()), + (WireFormat::OpenAiResponses, "responses".into()), + (WireFormat::AnthropicMessages, "anthropic".into()), + ]), + } + } + + #[test] + fn version_two_random_configuration_builds_clients_without_a_service() { + let config = config(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "random"); + assert_eq!(prepared.targets.len(), 3); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); + } + + #[test] + fn version_one_reports_the_service_to_library_migration() { + let mut config = config(); + config.version = 1; + let error = config.validate().unwrap_err(); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("version = 2")); + } + + #[test] + fn target_endpoints_must_be_canonical_for_the_current_http_client() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); + let error = config.validate().unwrap_err(); + assert!(error.contains("ending in \"/chat/completions\"")); + + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); + config.validate().unwrap(); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/custom/chat/completions" + ); + } + + #[test] + fn complete_provider_endpoint_is_not_appended_twice() { + let mut config = config(); + let chat = config.targets.get_mut("chat").unwrap(); + chat.base_url = "https://provider.example/v1/chat/completions/".into(); + assert_eq!( + chat.dispatch_url(), + "https://provider.example/v1/chat/completions" + ); + config.validate().unwrap(); + } + + #[test] + fn absolute_urls_cannot_embed_credentials_or_query_parameters() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().base_url = + "https://user:password@provider.example/v1".into(); + assert!( + config + .validate() + .unwrap_err() + .contains("embedded credentials") + ); + + config.targets.get_mut("chat").unwrap().base_url = + "https://provider.example/v1?api-version=1".into(); + assert!(config.validate().unwrap_err().contains("query parameters")); + } + + #[test] + fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { + let mut host_header_config = config(); + let chat = host_header_config.targets.get_mut("chat").unwrap(); + chat.header_env.insert("Host".into(), "TARGET_HOST".into()); + assert!( + host_header_config + .validate() + .unwrap_err() + .contains("HTTP transport") + ); + + let mut duplicate_config = config(); + let chat = duplicate_config.targets.get_mut("chat").unwrap(); + chat.header_env + .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); + chat.header_env + .insert("x-tenant".into(), "TARGET_TENANT_B".into()); + assert!( + duplicate_config + .validate() + .unwrap_err() + .contains("more than once") + ); + } + + #[test] + fn only_canonical_relay_execution_names_resolve_protocols() { + assert_eq!( + protocol_from_call("openai.chat_completions"), + Some(WireFormat::OpenAiChat) + ); + assert_eq!( + protocol_from_call("openai.responses"), + Some(WireFormat::OpenAiResponses) + ); + assert_eq!( + protocol_from_call("anthropic.messages"), + Some(WireFormat::AnthropicMessages) + ); + assert_eq!(protocol_from_call("openai_chat"), None); + } + + #[test] + fn schema_required_contract_fields_do_not_default_during_deserialization() { + let base = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + for field in ["version", "algorithm", "default_targets"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + let error = serde_json::from_value::(value) + .err() + .expect("required field must not default"); + assert!(error.to_string().contains(field), "field={field}: {error}"); + } + } + + #[test] + fn unknown_target_fields_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "unexpected_setting": true + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("unknown target field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); + } + + #[test] + fn literal_target_headers_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "headers": {"x-provider-token": "plaintext-secret"} + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("literal target headers must be rejected") + .to_string(); + assert!(error.contains("unknown field `headers`")); + assert!(!error.contains("plaintext-secret")); + } + + #[test] + fn unknown_algorithm_fields_are_rejected() { + let error = serde_json::from_value::(json!({ + "kind": "random", + "seed": 42, + "unexpected_setting": true + })) + .err() + .expect("unknown algorithm field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); + } + + #[test] + fn classifier_prepares_clients_for_judge_and_routed_targets() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "recent_turn_window": 4, + "max_output_tokens": 512 + })) + .unwrap(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); + } + + #[test] + fn target_provider_defaults_are_accepted_for_judge_controls() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().extra_body = + BTreeMap::from([("think".into(), json!(false))]); + + config.validate().unwrap(); + config.prepare().unwrap(); + } + + #[test] + fn classifier_rejects_anthropic_judge_targets_before_dispatch() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "anthropic", + "weak_target": "responses", + "strong_target": "chat", + "base_threshold": 0.5 + })) + .unwrap(); + + let error = config.validate().unwrap_err(); + assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); + } + + #[test] + fn validation_does_not_resolve_environment_backed_headers() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( + "authorization".into(), + "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), + )]); + + config.validate().unwrap(); + let error = config + .prepare() + .err() + .expect("preparation must resolve headers"); + assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); + } + + #[test] + fn invalid_environment_variable_names_are_rejected_before_resolution() { + for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = + BTreeMap::from([("authorization".into(), variable.into())]); + + let error = config.validate().unwrap_err(); + assert!(error.contains("must not contain '=' or NUL")); + } + } + + #[test] + fn static_validation_preserves_algorithm_constructor_checks() { + let mut random = config(); + for target in random.targets.values_mut() { + target.weight = 0.0; + } + assert!( + random + .validate() + .unwrap_err() + .contains("at least one positive target weight") + ); + + let mut classifier = config(); + classifier.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 1.1 + })) + .unwrap(); + assert!( + classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); + } + + #[test] + fn escalation_classifier_builds_with_defaulted_policy_settings() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "prompt": "Judge the completed trajectory.", + "max_output_tokens": 256, + "escalation": {} + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); + } + + #[test] + fn classifier_modes_reject_mixed_or_missing_settings() { + let mut capability = config(); + capability.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + capability + .validate() + .unwrap_err() + .contains("capability mode does not accept escalation") + ); + + let mut escalation = config(); + escalation.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + escalation + .validate() + .unwrap_err() + .contains("escalation mode does not accept capability") + ); + + let mut missing = config(); + missing.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic" + })) + .unwrap(); + assert!( + missing + .validate() + .unwrap_err() + .contains("requires escalation settings") + ); + } + + #[test] + fn escalation_settings_are_validated_by_the_libsy_constructor() { + for (settings, expected) in [ + ( + json!({"confirmations": 0}), + "confirmations must be at least 1", + ), + ( + json!({"recent_turn_window": 0}), + "recent_turn_window must be at least 1", + ), + ( + json!({"window_message_chars": 49}), + "window_message_chars must be at least 50", + ), + ] { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "escalation": settings + })) + .unwrap(); + assert!(config.validate().unwrap_err().contains(expected)); + } + } + + #[test] + fn full_stage_router_configuration_builds_all_clients() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "efficient_first", + "confidence_threshold": 0.5, + "recent_turn_window": 3, + "capable_system_prompt": "Diagnose before editing.", + "efficient_system_prompt": "Follow the settled plan.", + "handoff_notes": { + "escalation_note": "The previous model was stalling.", + "deescalation_note": "The task is settled.", + "only_on_wrong_signal_escalation": true + }, + "classifier": { + "target": "chat", + "base_threshold": 0.5, + "threshold_step": 0.1, + "recent_turn_window": 3, + "prompt": "Can the efficient tier finish this turn?", + "max_output_tokens": 256 + } + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "stage_router"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); + } + + #[test] + fn stage_router_validates_threshold_targets_and_judge_protocol() { + let stage = |classifier: Value, threshold: f64| { + serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": threshold, + "classifier": classifier + })) + .unwrap() + }; + + let mut invalid_threshold = config(); + invalid_threshold.algorithm = stage(Value::Null, 1.1); + assert!( + invalid_threshold + .validate() + .unwrap_err() + .contains("confidence_threshold must be between 0 and 1") + ); + + let mut missing_target = config(); + missing_target.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "missing", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": 0.5 + })) + .unwrap(); + assert!( + missing_target + .validate() + .unwrap_err() + .contains("algorithm target \"missing\" is not configured") + ); + + let mut anthropic_judge = config(); + anthropic_judge.algorithm = + stage(json!({"target": "anthropic", "base_threshold": 0.5}), 0.5); + assert!( + anthropic_judge + .validate() + .unwrap_err() + .contains("classifier target \"anthropic\" uses anthropic_messages") + ); + } + + #[test] + fn zero_weight_random_targets_are_fallback_only() { + let mut config = config(); + config.targets.get_mut("anthropic").unwrap().weight = 0.0; + let prepared = config.prepare().unwrap(); + + assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["chat"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["responses"].client), 1); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/executor.rs b/crates/switchyard-nemo-relay-plugin/src/executor.rs new file mode 100644 index 000000000..206c7aa99 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/executor.rs @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::future::Future; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; + +use tokio::runtime::{Builder, Handle}; +use tokio::sync::oneshot; +use tokio::task::AbortHandle; + +/// Plugin-owned async executor. +/// +/// The public native-plugin SDK uses synchronous Rust callbacks and pull-based +/// iterators at the dynamic-library boundary. Switchyard performs provider I/O +/// on this dedicated runtime rather than entering Relay's Tokio runtime from a +/// separately linked cdylib. +#[derive(Clone)] +pub(crate) struct PluginExecutor { + inner: Arc, +} + +struct ExecutorInner { + handle: Handle, + shutdown: Mutex>>, + thread: Mutex>>, +} + +impl PluginExecutor { + pub(crate) fn new() -> Result { + let (ready_tx, ready_rx) = mpsc::sync_channel(1); + let thread = thread::Builder::new() + .name("switchyard-relay-http".into()) + .spawn(move || { + let runtime = match Builder::new_multi_thread() + .worker_threads(2) + .thread_name("switchyard-relay-http-worker") + .enable_all() + .build() + { + Ok(runtime) => runtime, + Err(error) => { + let _ = ready_tx.send(Err(error.to_string())); + return; + } + }; + let (shutdown_tx, shutdown_rx) = oneshot::channel(); + if ready_tx + .send(Ok((runtime.handle().clone(), shutdown_tx))) + .is_err() + { + return; + } + runtime.block_on(async { + let _ = shutdown_rx.await; + }); + }) + .map_err(|error| format!("failed to start Switchyard HTTP runtime: {error}"))?; + let (handle, shutdown) = ready_rx + .recv() + .map_err(|_| "Switchyard HTTP runtime stopped during startup".to_string())??; + Ok(Self { + inner: Arc::new(ExecutorInner { + handle, + shutdown: Mutex::new(Some(shutdown)), + thread: Mutex::new(Some(thread)), + }), + }) + } + + pub(crate) fn spawn(&self, future: F) -> AbortHandle + where + F: Future + Send + 'static, + { + self.inner.handle.spawn(future).abort_handle() + } +} + +impl Drop for ExecutorInner { + fn drop(&mut self) { + if let Some(shutdown) = self + .shutdown + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + let _ = shutdown.send(()); + } + if let Some(thread) = self + .thread + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .take() + { + if std::thread::current() + .name() + .is_some_and(|name| name.starts_with("switchyard-relay-http-worker")) + { + // The runtime owner will join this worker after the current + // task returns. Waiting here would deadlock that shutdown. + drop(thread); + } else { + let _ = thread.join(); + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn executor_spawns_work() { + let executor = PluginExecutor::new().unwrap(); + let (sender, receiver) = mpsc::sync_channel(1); + executor.spawn(async move { + sender.send("done").unwrap(); + }); + assert_eq!(receiver.recv().unwrap(), "done"); + } + + #[test] + fn last_reference_can_drop_on_a_worker() { + let executor = PluginExecutor::new().unwrap(); + let worker_reference = executor.clone(); + let (sender, receiver) = mpsc::sync_channel(1); + executor.spawn(async move { + drop(worker_reference); + sender.send(()).unwrap(); + }); + drop(executor); + receiver + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("dropping the executor on its own worker must not deadlock"); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/ffi.rs b/crates/switchyard-nemo-relay-plugin/src/ffi.rs new file mode 100644 index 000000000..f5ce094c1 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/ffi.rs @@ -0,0 +1,444 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Small ownership wrapper around Relay's generic C host-table v3 hooks. +//! +//! The plugin manifest remains native API v1. Relay 0.7 supplies the appended +//! v3 host table to rebuilt v1 plugins, which lets this crate return `Pending` +//! and settle work from its own runtime without a targeted-continuation ABI. + +use std::ffi::c_void; +use std::ptr; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::time::Duration; + +use nemo_relay_plugin::{ + Json, LlmRequest, NemoRelayNativeAsyncCompletion, NemoRelayNativeAsyncNext, + NemoRelayNativeAsyncNextStreamCb, NemoRelayNativeAsyncStream, NemoRelayNativeHostApiV1, + NemoRelayNativeHostApiV3, NemoRelayNativeScopeHandle, NemoRelayNativeString, NemoRelayStatus, +}; +use serde::Serialize; +use tokio::sync::{mpsc, oneshot}; + +const BACKPRESSURE_POLL: Duration = Duration::from_millis(1); +const CANCELLATION_POLL: Duration = Duration::from_millis(10); +const MAX_PASSTHROUGH_BUFFER_BYTES: usize = 8 * 1024 * 1024; +const MAX_PASSTHROUGH_BUFFER_EVENTS: usize = 256; + +pub(crate) struct HostString { + host: NemoRelayNativeHostApiV1, + ptr: *mut NemoRelayNativeString, +} + +// Host strings are immutable allocations owned by Relay's thread-safe host table. +unsafe impl Send for HostString {} + +impl HostString { + pub(crate) fn json( + host: &NemoRelayNativeHostApiV1, + value: &impl Serialize, + ) -> Result { + let value = serde_json::to_string(value).map_err(|error| error.to_string())?; + Self::text(host, &value) + } + + pub(crate) fn text(host: &NemoRelayNativeHostApiV1, value: &str) -> Result { + let mut ptr = ptr::null_mut(); + let status = unsafe { (host.string_new)(value.as_ptr(), value.len(), &mut ptr) }; + if status == NemoRelayStatus::Ok && !ptr.is_null() { + Ok(Self { host: *host, ptr }) + } else { + Err(format!("Relay host string allocation failed: {status:?}")) + } + } + + pub(crate) fn as_ptr(&self) -> *const NemoRelayNativeString { + self.ptr + } +} + +impl Drop for HostString { + fn drop(&mut self) { + unsafe { (self.host.string_free)(self.ptr) }; + } +} + +pub(crate) fn read_string( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + if value.is_null() { + return Err("Relay passed a null native string".into()); + } + let len = unsafe { (host.string_len)(value) }; + let data = unsafe { (host.string_data)(value) }; + if data.is_null() && len != 0 { + return Err("Relay passed an invalid native string".into()); + } + let bytes = if len == 0 { + &[][..] + } else { + unsafe { std::slice::from_raw_parts(data, len) } + }; + std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|error| error.to_string()) +} + +pub(crate) fn read_json( + host: &NemoRelayNativeHostApiV1, + value: *const NemoRelayNativeString, +) -> Result { + serde_json::from_str(&read_string(host, value)?).map_err(|error| error.to_string()) +} + +/// Captures the current Relay scope as an explicit event parent. +/// +/// Async plugin work runs on a plugin-owned thread, so relying on thread-local +/// scope state would orphan its marks. The host handle is a cloned scope handle +/// and remains valid until this guard is dropped. +pub(crate) struct ParentScope { + host: NemoRelayNativeHostApiV1, + ptr: *mut NemoRelayNativeScopeHandle, +} + +unsafe impl Send for ParentScope {} +unsafe impl Sync for ParentScope {} + +impl ParentScope { + pub(crate) fn capture(host: &NemoRelayNativeHostApiV1) -> Option { + let mut ptr = ptr::null_mut(); + let status = unsafe { (host.scope_get_current)(&mut ptr) }; + (status == NemoRelayStatus::Ok && !ptr.is_null()).then_some(Self { host: *host, ptr }) + } + + pub(crate) fn emit_mark(&self, name: &str, data: &Json, metadata: &Json) -> Result<(), String> { + let name = HostString::text(&self.host, name)?; + let data = HostString::json(&self.host, data)?; + let metadata = HostString::json(&self.host, metadata)?; + let status = unsafe { + (self.host.emit_mark)( + name.as_ptr(), + self.ptr, + data.as_ptr(), + metadata.as_ptr(), + ptr::null(), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "Relay rejected Switchyard routing mark: {status:?}" + )) + } + } +} + +impl Drop for ParentScope { + fn drop(&mut self) { + unsafe { (self.host.scope_handle_free)(self.ptr) }; + } +} + +pub(crate) fn invoke_next_buffered( + host: &NemoRelayNativeHostApiV3, + next: usize, + completion: usize, + request: &LlmRequest, +) -> Result<(), String> { + let request = HostString::json(&host.v1, request)?; + let status = unsafe { + (host.async_next_invoke)( + next as *const NemoRelayNativeAsyncNext, + request.as_ptr(), + completion as *const NemoRelayNativeAsyncCompletion, + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!("Relay rejected buffered pass-through: {status:?}")) + } +} + +enum DownstreamStreamItem { + Chunk { value: Json, encoded_bytes: usize }, +} + +struct DownstreamStreamState { + host: NemoRelayNativeHostApiV1, + sender: mpsc::Sender, + terminal: Option>>, + queued_bytes: Arc, +} + +pub(crate) async fn invoke_next_stream( + host: &NemoRelayNativeHostApiV3, + next: usize, + output: usize, + request: &LlmRequest, +) -> Result<(), String> { + let request = HostString::json(&host.v1, request)?; + let (sender, mut receiver) = mpsc::channel(MAX_PASSTHROUGH_BUFFER_EVENTS); + let (terminal, terminal_result) = oneshot::channel(); + let queued_bytes = Arc::new(AtomicUsize::new(0)); + let state = Box::into_raw(Box::new(DownstreamStreamState { + host: host.v1, + sender, + terminal: Some(terminal), + queued_bytes: Arc::clone(&queued_bytes), + })) + .cast::(); + let status = unsafe { + (host.async_next_invoke_stream)( + next as *const NemoRelayNativeAsyncNext, + request.as_ptr(), + output as *const NemoRelayNativeAsyncStream, + downstream_stream_result as NemoRelayNativeAsyncNextStreamCb, + state, + ) + }; + if status != NemoRelayStatus::Ok { + unsafe { drop(Box::from_raw(state.cast::())) }; + return Err(format!("Relay rejected streaming pass-through: {status:?}")); + } + + while let Some(item) = receiver.recv().await { + match item { + DownstreamStreamItem::Chunk { + value, + encoded_bytes, + } => { + let result = push_stream(host, output, &value).await; + queued_bytes.fetch_sub(encoded_bytes, Ordering::AcqRel); + result?; + } + } + } + terminal_result + .await + .unwrap_or_else(|_| Err("Relay dropped the streaming pass-through callback".into())) +} + +unsafe extern "C" fn downstream_stream_result( + user_data: *mut c_void, + chunk_json: *const NemoRelayNativeString, + error: *const NemoRelayNativeString, + done: bool, +) -> bool { + if !error.is_null() { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + let error = read_string(&state.host, error) + .unwrap_or_else(|_| "Relay streaming pass-through failed".into()); + settle_downstream_stream(state, Err(error)); + return false; + } + if done { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Ok(())); + return false; + } + + let state = unsafe { &*user_data.cast::() }; + let parsed = read_string(&state.host, chunk_json).and_then(|encoded| { + let encoded_bytes = encoded.len(); + let value = serde_json::from_str(&encoded).map_err(|error| error.to_string())?; + Ok((value, encoded_bytes)) + }); + let (value, encoded_bytes) = match parsed { + Ok(parsed) => parsed, + Err(error) => { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Err(error)); + return false; + } + }; + if !reserve_buffer_bytes(&state.queued_bytes, encoded_bytes) { + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream( + state, + Err(format!( + "Relay streaming pass-through exceeded its {}-byte queued payload limit", + MAX_PASSTHROUGH_BUFFER_BYTES + )), + ); + return false; + } + + match state.sender.try_send(DownstreamStreamItem::Chunk { + value, + encoded_bytes, + }) { + Ok(()) => true, + Err(error) => { + let (item, message) = match error { + mpsc::error::TrySendError::Full(item) => ( + item, + format!( + "Relay streaming pass-through exceeded its {MAX_PASSTHROUGH_BUFFER_EVENTS}-event queue" + ), + ), + mpsc::error::TrySendError::Closed(item) => ( + item, + "Relay dropped the streaming pass-through receiver".into(), + ), + }; + let encoded_bytes = item.encoded_bytes(); + state + .queued_bytes + .fetch_sub(encoded_bytes, Ordering::AcqRel); + let state = unsafe { Box::from_raw(user_data.cast::()) }; + settle_downstream_stream(state, Err(message)); + false + } + } +} + +impl DownstreamStreamItem { + fn encoded_bytes(&self) -> usize { + match self { + Self::Chunk { encoded_bytes, .. } => *encoded_bytes, + } + } +} + +fn settle_downstream_stream(mut state: Box, result: Result<(), String>) { + if let Some(terminal) = state.terminal.take() { + let _ = terminal.send(result); + } +} + +fn reserve_buffer_bytes(queued: &AtomicUsize, encoded_bytes: usize) -> bool { + queued + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + current + .checked_add(encoded_bytes) + .filter(|next| *next <= MAX_PASSTHROUGH_BUFFER_BYTES) + }) + .is_ok() +} + +pub(crate) async fn wait_for_completion_cancellation( + host: &NemoRelayNativeHostApiV3, + completion: usize, +) { + while !completion_cancelled(host, completion as *const NemoRelayNativeAsyncCompletion) { + tokio::time::sleep(CANCELLATION_POLL).await; + } +} + +pub(crate) async fn wait_for_stream_cancellation(host: &NemoRelayNativeHostApiV3, stream: usize) { + while !unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + tokio::time::sleep(CANCELLATION_POLL).await; + } +} + +pub(crate) fn completion_cancelled( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, +) -> bool { + unsafe { (host.async_completion_is_cancelled)(completion) } +} + +pub(crate) fn resolve_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, + value: &Json, +) -> NemoRelayStatus { + match HostString::json(&host.v1, value) { + Ok(value) => unsafe { (host.async_completion_resolve_json)(completion, value.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub(crate) fn reject_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, + message: &str, +) -> NemoRelayStatus { + match HostString::text(&host.v1, message) { + Ok(message) => unsafe { (host.async_completion_reject)(completion, message.as_ptr()) }, + Err(_) => NemoRelayStatus::Internal, + } +} + +pub(crate) async fn push_stream( + host: &NemoRelayNativeHostApiV3, + stream: usize, + value: &Json, +) -> Result<(), String> { + let value = HostString::json(&host.v1, value)?; + loop { + if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + return Err("Relay caller cancelled the output stream".into()); + } + match unsafe { + (host.async_stream_push_json)( + stream as *const NemoRelayNativeAsyncStream, + value.as_ptr(), + ) + } { + NemoRelayStatus::Ok => return Ok(()), + // Native API v1 reports its bounded queue's WouldBlock state as Internal. + NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, + status => return Err(format!("Relay rejected output stream event: {status:?}")), + } + } +} + +pub(crate) fn finish_stream( + host: &NemoRelayNativeHostApiV3, + stream: *const NemoRelayNativeAsyncStream, +) -> NemoRelayStatus { + unsafe { (host.async_stream_finish)(stream) } +} + +pub(crate) async fn reject_stream( + host: &NemoRelayNativeHostApiV3, + stream: usize, + message: &str, +) -> NemoRelayStatus { + let Ok(message) = HostString::text(&host.v1, message) else { + return NemoRelayStatus::Internal; + }; + loop { + if unsafe { (host.async_stream_is_cancelled)(stream as *const NemoRelayNativeAsyncStream) } + { + return NemoRelayStatus::InvalidArg; + } + match unsafe { + (host.async_stream_reject)( + stream as *const NemoRelayNativeAsyncStream, + message.as_ptr(), + ) + } { + NemoRelayStatus::Internal => tokio::time::sleep(BACKPRESSURE_POLL).await, + status => return status, + } + } +} + +pub(crate) unsafe fn release_completion( + host: &NemoRelayNativeHostApiV3, + completion: *const NemoRelayNativeAsyncCompletion, +) { + unsafe { (host.async_completion_release)(completion) }; +} + +pub(crate) unsafe fn release_next( + host: &NemoRelayNativeHostApiV3, + next: *const NemoRelayNativeAsyncNext, +) { + unsafe { (host.async_next_release)(next) }; +} + +pub(crate) unsafe fn release_stream( + host: &NemoRelayNativeHostApiV3, + stream: *const NemoRelayNativeAsyncStream, +) { + unsafe { (host.async_stream_release)(stream) }; +} diff --git a/crates/switchyard-nemo-relay-plugin/src/lib.rs b/crates/switchyard-nemo-relay-plugin/src/lib.rs new file mode 100644 index 000000000..f3a2fd7c6 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/lib.rs @@ -0,0 +1,442 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +mod client; +mod config; +mod executor; +mod ffi; +mod runtime; +mod translation; + +use std::ffi::c_void; +use std::mem; +use std::panic::AssertUnwindSafe; +use std::sync::Arc; + +use futures_util::FutureExt; +use nemo_relay_plugin::{ + ConfigDiagnostic, DiagnosticLevel, Json, NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE, + NativePlugin, NemoRelayNativeAsyncCallbackState, NemoRelayNativeAsyncCompletion, + NemoRelayNativeAsyncMiddlewareKind, NemoRelayNativeAsyncNext, NemoRelayNativeAsyncStream, + NemoRelayNativeHostApiV3, NemoRelayNativeString, NemoRelayStatus, PluginContext, +}; +use serde::Deserialize; +use serde_json::Map; + +use crate::config::SwitchyardConfig; +use crate::executor::PluginExecutor; +use crate::runtime::{RoutingMark, StreamMessage, SwitchyardRuntime}; + +#[derive(Deserialize)] +struct Invocation { + name: String, + request: nemo_relay_plugin::LlmRequest, +} + +struct CallbackState { + host: NemoRelayNativeHostApiV3, + runtime: Arc, + executor: PluginExecutor, +} + +#[derive(Default)] +struct SwitchyardPlugin; + +impl NativePlugin for SwitchyardPlugin { + fn plugin_kind(&self) -> &str { + "nvidia.switchyard" + } + + fn allows_multiple_components(&self) -> bool { + false + } + + fn validate(&self, plugin_config: &Map) -> Vec { + match parse_config(plugin_config).and_then(|config| config.validate()) { + Ok(()) => Vec::new(), + Err(message) => vec![ConfigDiagnostic { + level: DiagnosticLevel::Error, + code: "switchyard.invalid_config".into(), + component: Some("nvidia.switchyard".into()), + field: Some("config".into()), + message, + }], + } + } + + fn register( + &mut self, + plugin_config: &Map, + ctx: &mut PluginContext<'_>, + ) -> nemo_relay_plugin::Result<()> { + let host_v1 = ctx.host_api(); + if host_v1.abi_version < NEMO_RELAY_NATIVE_ABI_VERSION_ASYNC_MIDDLEWARE + || host_v1.struct_size < mem::size_of::() + { + return Err( + "Switchyard requires Relay 0.7 or newer with the generic asynchronous native host table" + .into(), + ); + } + let host = unsafe { *(host_v1 as *const _ as *const NemoRelayNativeHostApiV3) }; + let config = parse_config(plugin_config)?; + let priority = config.priority; + let state = Arc::new(CallbackState { + host, + runtime: Arc::new(SwitchyardRuntime::new(config)?), + executor: PluginExecutor::new()?, + }); + + register_buffered(ctx, priority, Arc::clone(&state))?; + register_stream(ctx, priority, state)?; + Ok(()) + } +} + +fn register_buffered( + ctx: &mut PluginContext<'_>, + priority: i32, + state: Arc, +) -> Result<(), String> { + let user_data = Box::into_raw(Box::new(state)).cast::(); + let status = unsafe { + ctx.register_async_middleware_raw( + NemoRelayNativeAsyncMiddlewareKind::LlmExecutionIntercept, + "switchyard.run_stream.buffered", + priority, + false, + buffered_callback, + user_data, + Some(free_callback_state), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "failed to register Switchyard buffered execution: {status:?}" + )) + } +} + +fn register_stream( + ctx: &mut PluginContext<'_>, + priority: i32, + state: Arc, +) -> Result<(), String> { + let user_data = Box::into_raw(Box::new(state)).cast::(); + let status = unsafe { + ctx.register_async_stream_middleware_raw( + "switchyard.run_stream.streaming", + priority, + stream_callback, + user_data, + Some(free_callback_state), + ) + }; + if status == NemoRelayStatus::Ok { + Ok(()) + } else { + Err(format!( + "failed to register Switchyard streaming execution: {status:?}" + )) + } +} + +fn parse_config(plugin_config: &Map) -> Result { + match plugin_config.get("version").and_then(Json::as_u64) { + Some(2) => {} + Some(version) => { + return Err(format!( + "unsupported Switchyard config version {version}; version 1 used switchyard-server; migrate to version = 2" + )); + } + None => { + return Err("invalid Switchyard configuration: version must be the integer 2".into()); + } + } + serde_json::from_value(Json::Object(plugin_config.clone())) + .map_err(|error| format!("invalid Switchyard configuration: {error}")) +} + +fn emit_marks(parent: Option<&ffi::ParentScope>, marks: Vec) { + let Some(parent) = parent else { + return; + }; + for mark in marks { + if let Err(error) = parent.emit_mark(&mark.name, &mark.data, &mark.metadata) { + eprintln!( + "Switchyard could not emit routing mark {:?}: {error}", + mark.name + ); + } + } +} + +async fn execute_managed_stream( + state: &CallbackState, + output: usize, + inbound: switchyard_protocol::WireFormat, + request: switchyard_protocol::Request, + parent: Option<&ffi::ParentScope>, +) -> Result<(), String> { + let (sender, receiver) = async_channel::bounded(32); + let runtime = Arc::clone(&state.runtime); + let execution = async move { runtime.execute_stream(inbound, request, &sender).await }; + let forwarding = async { + while let Ok(message) = receiver.recv().await { + match message { + StreamMessage::Mark(mark) => emit_marks(parent, vec![mark]), + StreamMessage::Event(event) => { + ffi::push_stream(&state.host, output, &event).await? + } + } + } + Ok(()) + }; + tokio::try_join!(execution, forwarding)?; + Ok(()) +} + +unsafe extern "C" fn free_callback_state(user_data: *mut c_void) { + if !user_data.is_null() { + unsafe { drop(Box::from_raw(user_data.cast::>())) }; + } +} + +unsafe extern "C" fn buffered_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + completion: *const NemoRelayNativeAsyncCompletion, +) -> u32 { + if user_data.is_null() || completion.is_null() || next.is_null() { + return NemoRelayNativeAsyncCallbackState::Complete as u32; + } + let state = unsafe { &*user_data.cast::>() }.clone(); + let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|error| error.to_string()) + }); + let next = next as usize; + let completion = completion as usize; + let invocation = match invocation { + Ok(invocation) => invocation, + Err(error) => { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &format!("invalid Relay LLM invocation: {error}"), + ); + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + } + }; + let Some(inbound) = state.runtime.managed_protocol(&invocation.name) else { + if let Err(error) = + ffi::invoke_next_buffered(&state.host, next, completion, &invocation.request) + { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &error, + ); + } + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + }; + let request = match state + .runtime + .decode_request(inbound, &invocation.request, false) + { + Ok(request) => request, + Err(error) => { + let _ = ffi::reject_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + &error, + ); + unsafe { + ffi::release_next(&state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion( + &state.host, + completion as *const NemoRelayNativeAsyncCompletion, + ); + } + return NemoRelayNativeAsyncCallbackState::Pending as u32; + } + }; + let parent = ffi::ParentScope::capture(&state.host.v1); + let task_state = Arc::clone(&state); + state.executor.spawn(async move { + let execution = AssertUnwindSafe(async { + let mut marks = Vec::new(); + let result = task_state + .runtime + .execute_buffered(inbound, request, &mut marks) + .await; + emit_marks(parent.as_ref(), marks); + result + }) + .catch_unwind(); + tokio::pin!(execution); + let result = tokio::select! { + biased; + () = ffi::wait_for_completion_cancellation(&task_state.host, completion) => None, + result = &mut execution => Some( + result.unwrap_or_else(|_| Err("Switchyard buffered execution panicked".into())) + ), + }; + + let completion_ptr = completion as *const NemoRelayNativeAsyncCompletion; + if let Some(result) = result { + match result { + Ok(response) => { + let _ = ffi::resolve_completion(&task_state.host, completion_ptr, &response); + } + Err(error) => { + let _ = ffi::reject_completion(&task_state.host, completion_ptr, &error); + } + } + } + unsafe { + ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_completion(&task_state.host, completion_ptr); + } + }); + NemoRelayNativeAsyncCallbackState::Pending as u32 +} + +unsafe extern "C" fn stream_callback( + user_data: *mut c_void, + invocation_json: *const NemoRelayNativeString, + next: *const NemoRelayNativeAsyncNext, + output: *const NemoRelayNativeAsyncStream, +) -> u32 { + if user_data.is_null() || output.is_null() || next.is_null() { + return NemoRelayNativeAsyncCallbackState::Complete as u32; + } + let state = unsafe { &*user_data.cast::>() }.clone(); + let invocation = ffi::read_json(&state.host.v1, invocation_json).and_then(|value| { + serde_json::from_value::(value).map_err(|error| error.to_string()) + }); + let managed_protocol = invocation + .as_ref() + .ok() + .and_then(|invocation| state.runtime.managed_protocol(&invocation.name)); + let parent = managed_protocol.and_then(|_| ffi::ParentScope::capture(&state.host.v1)); + let next = next as usize; + let output = output as usize; + let task_state = Arc::clone(&state); + state.executor.spawn(async move { + let execution = AssertUnwindSafe(async { + match invocation { + Ok(invocation) => { + if let Some(inbound) = managed_protocol { + match task_state + .runtime + .decode_request(inbound, &invocation.request, true) + { + Ok(request) => { + execute_managed_stream( + &task_state, + output, + inbound, + request, + parent.as_ref(), + ) + .await + } + Err(error) => Err(error), + } + } else { + ffi::invoke_next_stream(&task_state.host, next, output, &invocation.request) + .await + } + } + Err(error) => Err(format!("invalid Relay LLM stream invocation: {error}")), + } + }) + .catch_unwind(); + tokio::pin!(execution); + let result = tokio::select! { + biased; + () = ffi::wait_for_stream_cancellation(&task_state.host, output) => None, + result = &mut execution => Some( + result.unwrap_or_else(|_| Err("Switchyard streaming execution panicked".into())) + ), + }; + + match result { + Some(Ok(())) => { + let _ = ffi::finish_stream( + &task_state.host, + output as *const NemoRelayNativeAsyncStream, + ); + } + Some(Err(error)) => { + let _ = ffi::reject_stream(&task_state.host, output, &error).await; + } + None => {} + } + unsafe { + ffi::release_next(&task_state.host, next as *const NemoRelayNativeAsyncNext); + ffi::release_stream( + &task_state.host, + output as *const NemoRelayNativeAsyncStream, + ); + } + }); + NemoRelayNativeAsyncCallbackState::Pending as u32 +} + +nemo_relay_plugin::nemo_relay_plugin!(nemo_relay_register_plugin, SwitchyardPlugin::default); + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::*; + + #[test] + fn version_one_service_config_gets_a_migration_error_before_v2_deserialization() { + let value = json!({ + "version": 1, + "service_url": "http://127.0.0.1:8080", + "health_endpoint": "/healthz" + }); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("version one must be rejected"); + assert!(error.contains("version 1 used switchyard-server")); + assert!(error.contains("migrate to version = 2")); + assert!(!error.contains("unknown field")); + } + + #[test] + fn version_must_be_an_integer() { + let value = json!({"version": "2"}); + let plugin_config = value.as_object().unwrap(); + + let error = parse_config(plugin_config) + .err() + .expect("non-integer versions must be rejected"); + assert_eq!( + error, + "invalid Switchyard configuration: version must be the integer 2" + ); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs new file mode 100644 index 000000000..ab7f8d08e --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -0,0 +1,1793 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::{BTreeMap, HashMap}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use futures_util::{StreamExt, stream}; +use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; +use serde_json::{Map, json}; +use switchyard_libsy::{Algorithm, LibsyError, PickOutcome, ToolSignals, pick_tier}; +use switchyard_llm_client::{ClientRouter, LlmCallObservation, RunObservation, RunObserver, run}; +use switchyard_protocol::{ + Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, WireFormat, +}; +use switchyard_translation::{TranslationEngine, encode_stream}; + +use crate::config::{PreparedTargetBinding, StageMarkConfig, SwitchyardConfig, protocol_from_call}; +use crate::translation; + +const INITIAL_RETRY_BACKOFF: Duration = Duration::from_millis(250); +const MAX_RETRY_BACKOFF: Duration = Duration::from_secs(2); + +#[derive(Debug)] +pub(crate) struct RoutingMark { + pub(crate) name: String, + pub(crate) data: Json, + pub(crate) metadata: Json, +} + +#[derive(Debug)] +pub(crate) enum StreamMessage { + Mark(RoutingMark), + Event(Json), +} + +pub(crate) struct SwitchyardRuntime { + max_retries: u32, + algorithm: Arc, + targets: BTreeMap, + default_targets: BTreeMap, + target_tiers: BTreeMap, + stage_marks: Option, + translation: TranslationEngine, +} + +impl SwitchyardRuntime { + pub(crate) fn new(config: SwitchyardConfig) -> Result { + let prepared = config.prepare()?; + Ok(Self { + max_retries: prepared.max_retries, + algorithm: prepared.algorithm, + targets: prepared.targets, + default_targets: prepared.default_targets, + target_tiers: prepared.target_tiers, + stage_marks: prepared.stage_marks, + translation: TranslationEngine::default(), + }) + } + + pub(crate) fn managed_protocol(&self, name: &str) -> Option { + protocol_from_call(name).filter(|protocol| self.default_targets.contains_key(protocol)) + } + + pub(crate) fn decode_request( + &self, + inbound: WireFormat, + request: &RelayRequest, + streaming: bool, + ) -> Result { + let mut llm_request = translation::decode_request(&self.translation, inbound, request)?; + llm_request.stream = streaming; + let headers = string_headers(&request.headers); + let mut metadata = Metadata::from_headers(&headers); + let relay_gateway_placeholder = !headers.contains_key("x-switchyard-session-id") + && headers + .get("x-nemo-relay-source") + .and_then(|value| value.to_str().ok()) + == Some("gateway") + && metadata.session_id.as_deref() == Some("gateway-gateway"); + if relay_gateway_placeholder { + metadata.session_id = None; + } + // Keep identity/routing metadata, but target clients deliberately clear + // these caller headers before HTTP dispatch. + metadata.http_headers = Some(headers); + metadata.wire_format = Some(inbound); + Ok(Request { + llm_request, + raw_request: Some(request.content.clone()), + metadata: Some(metadata), + }) + } + + pub(crate) async fn execute_buffered( + &self, + inbound: WireFormat, + request: Request, + marks: &mut Vec, + ) -> Result { + let metadata = identity_metadata(request.metadata.as_ref()); + let max_attempts = self.max_retries + 1; + let mut attempt = 1; + loop { + self.mark( + marks, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + &metadata, + ); + let result = self + .drive(request.clone(), attempt, marks, &metadata) + .await + .and_then(|response| { + finalize_buffered_response(&self.translation, inbound, response) + .map_err(|source| LibsyError::client_call("return_to_agent", source)) + }); + match result { + Ok(response) => return Ok(response), + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { + self.mark( + marks, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + sleep_before_retry(attempt).await; + attempt += 1; + } + Err(failure) => { + self.mark( + marks, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let response = self + .fallback_response(inbound, request, marks, &metadata) + .await?; + return finalize_buffered_response(&self.translation, inbound, response) + .map_err(|error| { + public_response_failure("trusted fallback response", &error) + }); + } + } + } + } + + pub(crate) async fn execute_stream( + &self, + inbound: WireFormat, + request: Request, + output: &async_channel::Sender, + ) -> Result<(), String> { + let metadata = identity_metadata(request.metadata.as_ref()); + let max_attempts = self.max_retries + 1; + let mut attempt = 1; + let mut marks = Vec::new(); + 'attempts: loop { + self.mark( + &mut marks, + "switchyard.routing.requested", + json!({"algorithm": self.algorithm.name(), "attempt": attempt}), + &metadata, + ); + let (response, mut fallback_used) = match self + .drive(request.clone(), attempt, &mut marks, &metadata) + .await + { + Ok(response) => (response, false), + Err(failure) if libsy_error_retryable(&failure) && attempt < max_attempts => { + self.mark( + &mut marks, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; + continue; + } + Err(failure) => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + (fallback?, true) + } + }; + send_marks(output, &mut marks).await?; + + let mut events = match returned_events(response, inbound).await { + Ok(events) => events, + Err(failure) + if !fallback_used + && libsy_error_retryable(&failure) + && attempt < max_attempts => + { + self.mark( + &mut marks, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; + continue; + } + Err(failure) if !fallback_used => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + fallback_used = true; + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + let fallback = fallback?; + returned_events(fallback, inbound) + .await + .map_err(|error| public_libsy_failure("trusted fallback stream", &error))? + } + Err(failure) => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + }; + + let mut committed = false; + while let Some(item) = events.next().await { + match item { + Ok(event) => { + send_event(output, event).await?; + committed = true; + } + Err(failure) + if !fallback_used + && !committed + && libsy_error_retryable(&failure) + && attempt < max_attempts => + { + self.mark( + &mut marks, + "switchyard.routing.retry", + failure_mark_data(attempt, &failure), + &metadata, + ); + send_marks(output, &mut marks).await?; + sleep_before_retry(attempt).await; + attempt += 1; + continue 'attempts; + } + Err(failure) if !fallback_used && !committed => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + let fallback = self + .fallback_response(inbound, request.clone(), &mut marks, &metadata) + .await; + send_marks(output, &mut marks).await?; + let fallback = fallback?; + let mut fallback = + returned_events(fallback, inbound).await.map_err(|error| { + public_libsy_failure("trusted fallback stream", &error) + })?; + while let Some(item) = fallback.next().await { + let event = item.map_err(|error| { + public_libsy_failure("trusted fallback stream", &error) + })?; + send_event(output, event).await?; + } + return Ok(()); + } + Err(failure) if !committed => { + return Err(public_libsy_failure("trusted fallback stream", &failure)); + } + Err(failure) => { + self.mark( + &mut marks, + "switchyard.routing.error", + failure_mark_data(attempt, &failure), + &metadata, + ); + send_marks(output, &mut marks).await?; + return Err(public_libsy_failure( + "Switchyard stream failed after response commitment", + &failure, + )); + } + } + } + if committed { + return Ok(()); + } + return Err("Switchyard response stream produced no caller events".into()); + } + } + + async fn drive( + &self, + request: Request, + attempt: u32, + marks: &mut Vec, + mark_metadata: &Json, + ) -> Result { + let context = context_from_metadata(request.metadata.as_ref()); + let stage_request = self.stage_marks.as_ref().map(|_| request.clone()); + let observations = Arc::new(Mutex::new(Vec::new())); + let observed_calls = observations.clone(); + let observer: RunObserver = Arc::new(move |observation| { + if let RunObservation::LlmCall(call) = observation { + observed_calls + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .push(call); + } + }); + let clients = ClientRouter::new( + self.targets + .iter() + .map(|(name, target)| (name.clone(), target.client.clone())) + .collect::>(), + ); + match run( + self.algorithm.clone(), + clients, + context, + request, + Some(observer), + ) + .await + { + Ok((decisions, response)) => { + for decision in decisions { + self.emit_decision( + marks, + decision.as_ref(), + stage_request.as_ref(), + attempt, + mark_metadata, + ); + } + self.emit_routing_llm_calls( + marks, + take_observed_calls(&observations), + attempt, + mark_metadata, + true, + ); + Ok(response) + } + Err(error) => { + self.emit_routing_llm_calls( + marks, + take_observed_calls(&observations), + attempt, + mark_metadata, + false, + ); + Err(error) + } + } + } + + async fn fallback_response( + &self, + inbound: WireFormat, + request: Request, + marks: &mut Vec, + metadata: &Json, + ) -> Result { + let target_name = self.default_target(inbound)?; + let target = self.target(target_name)?; + self.mark( + marks, + "switchyard.routing.fallback", + json!({"selected_target": target_name}), + metadata, + ); + let decision = Arc::new(Decision::new( + target_name, + Some("trusted fallback target".into()), + true, + )); + let context = context_from_metadata(request.metadata.as_ref()); + target + .client + .call(context, request, decision) + .await + .map_err(|error| public_client_failure("trusted fallback", &error)) + } + + fn target(&self, name: &str) -> Result<&PreparedTargetBinding, String> { + self.targets + .get(name) + .ok_or_else(|| format!("libsy selected unknown target {name:?}")) + } + + fn default_target(&self, protocol: WireFormat) -> Result<&str, String> { + self.default_targets + .get(&protocol) + .map(String::as_str) + .ok_or_else(|| format!("managed protocol {protocol} has no default target")) + } + + fn mark(&self, marks: &mut Vec, name: &str, data: Json, metadata: &Json) { + marks.push(RoutingMark { + name: name.to_string(), + data, + metadata: metadata.clone(), + }); + } + + fn emit_decision( + &self, + marks: &mut Vec, + decision: &Decision, + request: Option<&Request>, + attempt: u32, + metadata: &Json, + ) { + let decision_source = + request.and_then(|request| self.stage_decision_source(request, decision)); + let routing_tier = self.target_tiers.get(decision.selected_model_id()).copied(); + self.mark( + marks, + "switchyard.routing.decision", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "selected_target": decision.selected_model_id(), + "reasoning": decision.reasoning(), + "routing_tier": routing_tier, + "decision_source": decision_source, + "is_routed_call": decision.is_answer_call(), + }), + metadata, + ); + } + + fn stage_decision_source( + &self, + request: &Request, + decision: &Decision, + ) -> Option<&'static str> { + let config = self.stage_marks.as_ref()?; + let signals = ToolSignals::from_request(request, config.recent_turn_window); + match pick_tier(&signals, config.picker, config.confidence_threshold) { + PickOutcome::Resolved { source, .. } => Some(source.as_str()), + PickOutcome::ConsultClassifier { .. } => { + let classifier_decided = config.classifier_enabled + && decision + .reasoning() + .and_then(decision_confidence) + .is_some_and(|confidence| confidence > 0.0); + Some(if classifier_decided { + "llm-classifier" + } else { + "fall_open" + }) + } + } + } + + fn emit_routing_llm_calls( + &self, + marks: &mut Vec, + mut calls: Vec, + attempt: u32, + metadata: &Json, + successful_run: bool, + ) { + // The last successful routed call produced the response represented by Relay's + // outer LLM lifecycle event. Keep it out of these marks so consumers can add + // routing overhead without counting the serving call twice. Earlier routed calls + // are discarded candidates (for example, escalation's weak draft). + if successful_run + && let Some(position) = calls + .iter() + .rposition(|call| call.is_answer_call && call.is_success) + { + calls.remove(position); + } + + for (index, call) in calls.into_iter().enumerate() { + let routing_tier = self.target_tiers.get(&call.selected_model).copied(); + self.mark( + marks, + "switchyard.routing.llm_call", + json!({ + "algorithm": self.algorithm.name(), + "attempt": attempt, + "call_index": index + 1, + "selected_target": call.selected_model, + "routing_tier": routing_tier, + "call_role": if call.is_answer_call { "candidate" } else { "judge" }, + "outcome": if call.is_success { "ok" } else { "error" }, + "latency_ms": call.duration.as_secs_f64() * 1_000.0, + "usage": call.usage, + "contributes_to_routing_overhead": true, + }), + metadata, + ); + } + } +} + +fn take_observed_calls(observations: &Mutex>) -> Vec { + std::mem::take( + &mut *observations + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()), + ) +} + +fn decision_confidence(reasoning: &str) -> Option { + let (_, suffix) = reasoning.rsplit_once("confidence ")?; + let numeric = suffix + .trim_start_matches(|character: char| { + !character.is_ascii_digit() && !matches!(character, '.' | '-' | '+') + }) + .chars() + .take_while(|character| { + character.is_ascii_digit() || matches!(character, '.' | '-' | '+' | 'e' | 'E') + }) + .collect::(); + numeric.parse().ok() +} + +async fn send_marks( + output: &async_channel::Sender, + marks: &mut Vec, +) -> Result<(), String> { + for mark in marks.drain(..) { + output + .send(StreamMessage::Mark(mark)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string())?; + } + Ok(()) +} + +async fn send_event( + output: &async_channel::Sender, + event: Json, +) -> Result<(), String> { + output + .send(StreamMessage::Event(event)) + .await + .map_err(|_| "Relay cancelled the Switchyard response stream".to_string()) +} + +type ReturnedEventStream = + std::pin::Pin> + Send>>; + +fn finalize_buffered_response( + translation_engine: &TranslationEngine, + inbound: WireFormat, + response: Response, +) -> Result { + let LlmResponse::Agg(response) = response.llm_response else { + return Err(LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::other( + "libsy returned a stream for a buffered request", + )), + }); + }; + translation::encode_response(translation_engine, inbound, &response) + .map_err(LlmClientError::ResponseTranslation) +} + +async fn returned_events( + response: Response, + inbound: WireFormat, +) -> Result { + let chunks = match response.llm_response { + LlmResponse::Agg(response) => response.into_stream(), + LlmResponse::Stream(mut chunks) => { + let Some(first) = chunks.next().await else { + return Err(LibsyError::client_call( + "return_to_agent", + LlmClientError::InvalidResponse { + source: Box::new(std::io::Error::new( + std::io::ErrorKind::UnexpectedEof, + "provider returned an empty stream", + )), + }, + )); + }; + Box::pin(stream::once(async move { first }).chain(chunks)) + } + }; + let events = encode_stream(chunks, inbound, None) + .map_err(|error| LibsyError::client_call("return_to_agent", error))?; + Ok(Box::pin(events.map(|item| { + item.map_err(|source| match source.downcast::() { + Ok(source) => LibsyError::client_call("return_to_agent", *source), + Err(source) => LibsyError::client_call( + "return_to_agent", + LlmClientError::ResponseTranslation(source.to_string()), + ), + }) + }))) +} + +fn libsy_error_retryable(error: &LibsyError) -> bool { + let LibsyError::ClientCall { source, .. } = error else { + return false; + }; + match source { + LlmClientError::UpstreamHttp { status, .. } => { + matches!(*status, 408 | 425 | 429 | 500 | 502 | 503 | 504) + } + LlmClientError::Transport { .. } | LlmClientError::Timeout { .. } => true, + _ => false, + } +} + +fn retry_backoff(attempt: u32) -> Duration { + let exponent = attempt.saturating_sub(1).min(3); + INITIAL_RETRY_BACKOFF + .saturating_mul(1_u32 << exponent) + .min(MAX_RETRY_BACKOFF) +} + +async fn sleep_before_retry(attempt: u32) { + tokio::time::sleep(retry_backoff(attempt)).await; +} + +fn failure_mark_data(attempt: u32, failure: &LibsyError) -> Json { + let mut data = Map::from_iter([ + ("attempt".into(), Json::from(attempt)), + ( + "retryable".into(), + Json::from(libsy_error_retryable(failure)), + ), + ]); + match failure { + LibsyError::ClientCall { + source: LlmClientError::UpstreamHttp { status, .. }, + .. + } => { + data.insert("failure_kind".into(), Json::from("http")); + data.insert("http_status".into(), Json::from(*status)); + } + LibsyError::ClientCall { source, .. } => { + data.insert("failure_kind".into(), Json::from("non_http")); + data.insert( + "non_http_kind".into(), + Json::from(client_error_label(source)), + ); + } + _ => { + data.insert("failure_kind".into(), Json::from("algorithm")); + } + } + Json::Object(data) +} + +fn client_error_label(error: &LlmClientError) -> &'static str { + match error { + LlmClientError::InvalidRequest { .. } => "invalid_request", + LlmClientError::RequestTranslation(_) => "request_translation", + LlmClientError::RequestEncoding(_) => "request_encoding", + LlmClientError::ResponseTranslation(_) => "response_translation", + LlmClientError::Configuration { .. } => "configuration", + LlmClientError::Transport { .. } => "transport", + LlmClientError::Timeout { .. } => "timeout", + LlmClientError::ContextWindowExceeded { .. } => "context_window_exceeded", + LlmClientError::UpstreamHttp { .. } => "http", + LlmClientError::InvalidResponse { .. } => "invalid_response", + LlmClientError::Ffi { .. } => "ffi", + LlmClientError::General(_) => "general", + _ => "unknown", + } +} + +fn public_libsy_failure(prefix: &str, error: &LibsyError) -> String { + match error { + LibsyError::ClientCall { source, .. } => public_client_failure(prefix, source), + _ => format!("{prefix}: Switchyard algorithm failure"), + } +} + +fn public_response_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::InvalidResponse { .. } => format!("{prefix}: invalid response"), + LlmClientError::ResponseTranslation(_) => { + format!("{prefix}: response translation failure") + } + _ => format!("{prefix}: response finalization failure"), + } +} + +fn public_client_failure(prefix: &str, error: &LlmClientError) -> String { + match error { + LlmClientError::UpstreamHttp { status, .. } => { + format!("{prefix}: provider returned HTTP {status}") + } + _ => format!("{prefix}: provider {} failure", client_error_label(error)), + } +} + +fn string_headers(headers: &Map) -> http::HeaderMap { + let mut parsed = http::HeaderMap::with_capacity(headers.len()); + for (name, value) in headers { + let Some(value) = value.as_str() else { + continue; + }; + let (Ok(name), Ok(value)) = ( + http::HeaderName::from_bytes(name.as_bytes()), + http::HeaderValue::from_str(value), + ) else { + continue; + }; + parsed.insert(name, value); + } + parsed +} + +fn identity_metadata(metadata: Option<&Metadata>) -> Json { + json!({ + "session_id": metadata.and_then(|value| value.session_id.as_deref()), + "agent_id": metadata.and_then(|value| value.agent_id.as_deref()), + "parent_agent_id": metadata.and_then(|value| value.parent_agent_id.as_deref()), + "task_id": metadata.and_then(|value| value.task_id.as_deref()), + "turn_id": metadata.and_then(|value| value.turn_id.as_deref()), + "correlation_id": metadata.and_then(|value| value.correlation_id.as_deref()), + }) +} + +fn context_from_metadata(metadata: Option<&Metadata>) -> Context { + let Some(metadata) = metadata else { + return Context::default(); + }; + let mut values = std::collections::HashMap::new(); + for (name, value) in [ + ("session_id", metadata.session_id.as_deref()), + ("agent_id", metadata.agent_id.as_deref()), + ("parent_agent_id", metadata.parent_agent_id.as_deref()), + ("agent_kind", metadata.agent_kind.as_deref()), + ("agent_role", metadata.agent_role.as_deref()), + ("task_id", metadata.task_id.as_deref()), + ("task_kind", metadata.task_kind.as_deref()), + ("turn_id", metadata.turn_id.as_deref()), + ("correlation_id", metadata.correlation_id.as_deref()), + ] { + if let Some(value) = value { + values.insert(name.to_string(), value.to_string()); + } + } + values.insert("is_subagent".into(), metadata.is_subagent.to_string()); + values.insert( + "is_delegated_work".into(), + metadata.is_delegated_work.to_string(), + ); + if let Some(session_final) = metadata.session_final { + values.insert("session_final".into(), session_final.to_string()); + } + if let Some(extra) = &metadata.extra_metadata { + for (name, value) in extra { + values.entry(name.clone()).or_insert_with(|| value.clone()); + } + } + let mut context = Context::default(); + context.values = values; + context +} + +#[cfg(test)] +mod tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use switchyard_libsy::{ + ClassifierContractConfig, EscalationJudgeConfig, LlmClassifierConfig, LlmFallback, + LlmTarget, LlmTaskClassifier, Passthrough, PickerMode, StageRouter, StageRouterConfig, + TaskClassifierConfig, + }; + use switchyard_protocol::{ + ContentBlock, LlmRequest, LlmResponseStream, Message, Role, RoutedLlmClient, ToolCall, + ToolResult, Usage, text_request, text_response, + }; + + use super::*; + + #[derive(Clone, Copy)] + enum StreamBehavior { + Empty, + Failing, + CallFailure, + } + + struct StreamClient { + behavior: StreamBehavior, + calls: AtomicUsize, + } + + struct BufferedClient { + calls: AtomicUsize, + } + + enum FixedBehavior { + Text(&'static str), + TransportFailure, + } + + struct FixedClient { + behavior: FixedBehavior, + calls: AtomicUsize, + } + + #[async_trait::async_trait] + impl RoutedLlmClient for StreamClient { + async fn call( + &self, + _ctx: Context, + _request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + let stream: LlmResponseStream = match self.behavior { + StreamBehavior::Empty => Box::pin(stream::empty()), + StreamBehavior::Failing => Box::pin(stream::once(async { + Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback stream failed")), + }) + })), + StreamBehavior::CallFailure => { + return Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback call failed")), + }); + } + }; + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } + } + + #[async_trait::async_trait] + impl RoutedLlmClient for BufferedClient { + async fn call( + &self, + _ctx: Context, + _request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + Ok(Response { + llm_response: LlmResponse::Agg(Default::default()), + metadata: None, + }) + } + } + + #[async_trait::async_trait] + impl RoutedLlmClient for FixedClient { + async fn call( + &self, + _ctx: Context, + request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + match self.behavior { + FixedBehavior::Text(text) => { + let mut response = text_response(None, text); + response.usage = Usage { + input_tokens: Some(11), + output_tokens: Some(7), + total_tokens: Some(18), + ..Usage::default() + }; + Ok(Response { + llm_response: LlmResponse::Agg(response), + metadata: request.metadata, + }) + } + FixedBehavior::TransportFailure => Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("scripted failure")), + }), + } + } + } + + fn fixed_target(name: &str, _client: Arc) -> LlmTarget { + LlmTarget { + semantic_name: name.to_string(), + } + } + + fn runtime_with_algorithm( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, + ) -> SwitchyardRuntime { + runtime_with_algorithm_clients(algorithm, fallback, protocol, Vec::new()) + } + + fn runtime_with_algorithm_clients( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, + clients: Vec<(&str, Arc)>, + ) -> SwitchyardRuntime { + let is_stage = algorithm.name() == "stage_router"; + let mut targets = BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback as Arc, + }, + )]); + for (name, client) in clients { + targets.insert( + name.to_string(), + PreparedTargetBinding { + client: client as Arc, + }, + ); + } + SwitchyardRuntime { + max_retries: 0, + algorithm, + targets, + default_targets: BTreeMap::from([(protocol, "fallback".into())]), + target_tiers: BTreeMap::from([("weak".into(), "weak"), ("strong".into(), "strong")]), + stage_marks: is_stage.then_some(StageMarkConfig { + picker: PickerMode::CapableFirst, + confidence_threshold: 0.5, + recent_turn_window: None, + classifier_enabled: true, + }), + translation: TranslationEngine::default(), + } + } + + fn request_with_session(protocol: WireFormat, session: Option<&str>) -> Request { + Request { + llm_request: text_request(Some("auto".into()), "fix the build"), + raw_request: None, + metadata: Some(Metadata { + wire_format: Some(protocol), + session_id: session.map(str::to_string), + ..Metadata::default() + }), + } + } + + fn stage_signal_request(protocol: WireFormat) -> Request { + Request { + llm_request: LlmRequest { + model: Some("auto".into()), + messages: vec![ + Message::text(Role::User, "fix the build"), + Message { + role: Role::Assistant, + content: vec![ContentBlock::ToolCall(ToolCall { + id: "call-1".into(), + name: "bash".into(), + arguments: json!({"cmd": "cargo test"}), + })], + }, + Message { + role: Role::Tool, + content: vec![ContentBlock::ToolResult(ToolResult { + tool_call_id: "call-1".into(), + content: vec![ContentBlock::Text { + text: "fatal runtime error: out of memory".into(), + }], + is_error: Some(true), + })], + }, + ], + ..LlmRequest::default() + }, + raw_request: None, + metadata: Some(Metadata { + wire_format: Some(protocol), + session_id: Some(format!("stage-{}", protocol.as_str())), + ..Metadata::default() + }), + } + } + + #[test] + fn relay_gateway_placeholder_session_is_not_retained() { + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ("x-dynamo-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!(decoded.metadata.unwrap().session_id, None); + } + + #[test] + fn explicit_switchyard_session_overrides_relay_gateway_placeholder() { + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-switchyard-session-id".into(), json!("caller-session")), + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!( + decoded.metadata.unwrap().session_id.as_deref(), + Some("caller-session") + ); + } + + #[tokio::test] + async fn buffered_finalization_failure_uses_fallback_once() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(BufferedClient { + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 1, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) + .await + .expect("the buffered fallback response should be encoded"); + + assert!(response.is_object()); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.retry") + ); + let error = marks + .iter() + .find(|mark| mark.name == "switchyard.routing.error") + .expect("finalization failure should emit an error mark"); + assert_eq!(error.data["retryable"], false); + assert_eq!(error.data["non_http_kind"], "invalid_response"); + assert_eq!( + marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.fallback") + .count(), + 1 + ); + } + + #[tokio::test] + async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { + let content = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }); + let terminal = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop" + }] + }); + let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); + let stream = switchyard_translation::decode_stream( + stream::once(async move { Ok::<_, LlmClientError>(body) }), + WireFormat::OpenAiChat, + ) + .expect("provider SSE should decode"); + let response = Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }; + + let replayed = returned_events(response, WireFormat::OpenAiChat) + .await + .expect("return stream should encode") + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("return stream should not fail"); + + assert_eq!(replayed, vec![content, terminal]); + } + + #[tokio::test] + async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(StreamClient { + behavior: StreamBehavior::Failing, + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let (output, _messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback stream must fail the request"); + + assert_eq!(error, "trusted fallback stream: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + } + + #[tokio::test] + async fn failing_fallback_call_flushes_error_and_fallback_marks() { + let selected = Arc::new(StreamClient { + behavior: StreamBehavior::Empty, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(StreamClient { + behavior: StreamBehavior::CallFailure, + calls: AtomicUsize::new(0), + }); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let (output, messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback call must fail the request"); + + assert_eq!(error, "trusted fallback: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + let mut terminal_marks = Vec::new(); + while let Ok(message) = messages.try_recv() { + if let StreamMessage::Mark(mark) = message + && matches!( + mark.name.as_str(), + "switchyard.routing.error" | "switchyard.routing.fallback" + ) + { + terminal_marks.push(mark.name); + } + } + assert_eq!( + terminal_marks, + ["switchyard.routing.error", "switchyard.routing.fallback"] + ); + } + + #[test] + fn retry_backoff_increases_exponentially_and_is_capped() { + assert_eq!(retry_backoff(1), Duration::from_millis(250)); + assert_eq!(retry_backoff(2), Duration::from_millis(500)); + assert_eq!(retry_backoff(3), Duration::from_secs(1)); + assert_eq!(retry_backoff(4), Duration::from_secs(2)); + assert_eq!(retry_backoff(u32::MAX), Duration::from_secs(2)); + } + + #[tokio::test] + async fn capability_classifier_emits_judge_usage_without_serving_usage() { + let weak = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak answer"), + calls: AtomicUsize::new(0), + }); + let strong = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong answer"), + calls: AtomicUsize::new(0), + }); + let judge = Arc::new(FixedClient { + behavior: FixedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + ), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: fixed_target("judge", judge.clone()), + efficient_target: fixed_target("weak", weak.clone()), + capable_target: fixed_target("strong", strong.clone()), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("capability")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); + } + + #[tokio::test] + async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { + let weak = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak draft"), + calls: AtomicUsize::new(0), + }); + let strong = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong answer"), + calls: AtomicUsize::new(0), + }); + let judge = Arc::new(FixedClient { + behavior: FixedBehavior::Text(r#"{"escalate":true,"reason":"stuck"}"#), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge", judge.clone()), + efficient_target: fixed_target("weak", weak.clone()), + capable_target: fixed_target("strong", strong.clone()), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + let mut first = request_with_session(WireFormat::OpenAiChat, Some("session-1")); + first.llm_request.stream = true; + let (output, messages) = async_channel::bounded(32); + runtime + .execute_stream(WireFormat::OpenAiChat, first, &output) + .await + .unwrap(); + let mut streamed = Vec::new(); + let mut routing_calls = Vec::new(); + while let Ok(message) = messages.try_recv() { + match message { + StreamMessage::Event(event) => streamed.push(event), + StreamMessage::Mark(mark) if mark.name == "switchyard.routing.llm_call" => { + routing_calls.push(mark.data) + } + StreamMessage::Mark(_) => {} + } + } + assert!(!streamed.is_empty()); + assert!( + streamed + .iter() + .any(|event| event.to_string().contains("strong answer")) + ); + assert_eq!(routing_calls.len(), 2); + assert_eq!(routing_calls[0]["selected_target"], "weak"); + assert_eq!(routing_calls[0]["call_role"], "candidate"); + assert_eq!(routing_calls[0]["usage"]["total_tokens"], 18); + assert_eq!(routing_calls[1]["selected_target"], "judge"); + assert_eq!(routing_calls[1]["call_role"], "judge"); + assert_eq!(routing_calls[1]["usage"]["total_tokens"], 18); + assert!( + routing_calls + .iter() + .all(|call| call["selected_target"] != "strong") + ); + + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("strong answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 2); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.llm_call") + ); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == "strong" + && mark.data["routing_tier"] == "strong" + && mark.metadata["session_id"] == "session-1" + })); + } + + #[tokio::test] + async fn escalation_judge_failure_falls_open_to_the_buffered_weak_response() { + let weak = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak answer"), + calls: AtomicUsize::new(0), + }); + let strong = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong answer"), + calls: AtomicUsize::new(0), + }); + let judge = Arc::new(FixedClient { + behavior: FixedBehavior::TransportFailure, + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge", judge.clone()), + efficient_target: fixed_target("weak", weak.clone()), + capable_target: fixed_target("strong", strong.clone()), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig::default(), + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + + assert!(response.to_string().contains("weak answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "judge"); + assert_eq!(routing_calls[0].data["outcome"], "error"); + assert!(routing_calls[0].data["usage"].is_null()); + } + + #[tokio::test] + async fn escalation_without_session_identity_cannot_accumulate_confirmations() { + let weak = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak answer"), + calls: AtomicUsize::new(0), + }); + let strong = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong answer"), + calls: AtomicUsize::new(0), + }); + let judge = Arc::new(FixedClient { + behavior: FixedBehavior::Text(r#"{"escalate":true,"reason":"stuck"}"#), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge", judge.clone()), + efficient_target: fixed_target("weak", weak.clone()), + capable_target: fixed_target("strong", strong.clone()), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 2, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + for _ in 0..2 { + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("weak answer")); + } + assert_eq!(weak.calls.load(Ordering::Relaxed), 2); + assert_eq!(judge.calls.load(Ordering::Relaxed), 2); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + } + + #[tokio::test] + async fn stage_router_uses_tool_signals_for_every_managed_protocol() { + for protocol in [ + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + WireFormat::AnthropicMessages, + ] { + let capable = Arc::new(FixedClient { + behavior: FixedBehavior::Text("capable answer"), + calls: AtomicUsize::new(0), + }); + let efficient = Arc::new(FixedClient { + behavior: FixedBehavior::Text("efficient answer"), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = StageRouter::new( + fixed_target("strong", capable.clone()), + fixed_target("weak", efficient.clone()), + StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + protocol, + vec![("strong", capable.clone()), ("weak", efficient.clone())], + ); + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered(protocol, stage_signal_request(protocol), &mut marks) + .await + .unwrap(); + + assert!(response.to_string().contains("capable answer")); + assert_eq!(capable.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["algorithm"] == "stage_router" + && mark.data["selected_target"] == "strong" + && mark.data["routing_tier"] == "strong" + && mark.data["decision_source"] == "override" + && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) + })); + } + } + + #[tokio::test] + async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { + for (picker, expected) in [ + (PickerMode::CapableFirst, "strong"), + (PickerMode::EfficientFirst, "weak"), + ] { + let capable = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong"), + calls: AtomicUsize::new(0), + }); + let efficient = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak"), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let algorithm = StageRouter::new( + fixed_target("strong", capable.clone()), + fixed_target("weak", efficient.clone()), + StageRouterConfig::new(picker, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![("strong", capable), ("weak", efficient)], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == expected + && mark.data["routing_tier"] == expected + && mark.data["decision_source"] == "fall_open" + })); + } + } + + #[tokio::test] + async fn stage_router_classifier_resolves_an_ambiguous_turn() { + let capable = Arc::new(FixedClient { + behavior: FixedBehavior::Text("strong"), + calls: AtomicUsize::new(0), + }); + let efficient = Arc::new(FixedClient { + behavior: FixedBehavior::Text("weak"), + calls: AtomicUsize::new(0), + }); + let judge = Arc::new(FixedClient { + behavior: FixedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + ), + calls: AtomicUsize::new(0), + }); + let fallback = Arc::new(FixedClient { + behavior: FixedBehavior::Text("fallback"), + calls: AtomicUsize::new(0), + }); + let mut config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5); + config.llm_fallback = Some(LlmFallback { + judge_target: fixed_target("judge", judge.clone()), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }); + let algorithm = StageRouter::new( + fixed_target("strong", capable.clone()), + fixed_target("weak", efficient.clone()), + config, + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("strong", capable.clone()), + ("weak", efficient.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("stage-classifier")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 1); + assert_eq!(capable.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "judge"); + assert_eq!(routing_calls[0].data["outcome"], "ok"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == "weak" + && mark.data["routing_tier"] == "weak" + && mark.data["decision_source"] == "llm-classifier" + })); + } + + #[test] + fn context_carries_identity_without_http_headers() { + let context = context_from_metadata(Some(&Metadata { + session_id: Some("session-1".into()), + agent_id: Some("agent-1".into()), + is_subagent: true, + extra_metadata: Some(BTreeMap::from([("tenant".into(), "blue".into())])), + http_headers: Some(http::HeaderMap::from_iter([( + http::HeaderName::from_static("authorization"), + http::HeaderValue::from_static("Bearer caller-secret"), + )])), + ..Metadata::default() + })); + + assert_eq!( + context.values.get("session_id").map(String::as_str), + Some("session-1") + ); + assert_eq!( + context.values.get("agent_id").map(String::as_str), + Some("agent-1") + ); + assert_eq!( + context.values.get("is_subagent").map(String::as_str), + Some("true") + ); + assert_eq!( + context.values.get("tenant").map(String::as_str), + Some("blue") + ); + assert!(!context.values.contains_key("authorization")); + } +} diff --git a/crates/switchyard-nemo-relay-plugin/src/translation.rs b/crates/switchyard-nemo-relay-plugin/src/translation.rs new file mode 100644 index 000000000..6f6f965fe --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/translation.rs @@ -0,0 +1,116 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use nemo_relay_plugin::LlmRequest as RelayRequest; +use serde_json::Value as Json; +use switchyard_protocol::{AggLlmResponse, LlmRequest, WireFormat}; +use switchyard_translation::{ + DeterministicIdPolicy, DiagnosticSeverity, LossyConversionPolicy, PreservationPolicy, + TargetCapabilities, TranslationDiagnostic, TranslationEngine, TranslationPolicy, + UnknownFieldPolicy, +}; + +pub(crate) fn decode_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &RelayRequest, +) -> Result { + let output = engine + .decode_request(protocol, &request.content, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.request) +} + +pub(crate) fn validate_target_request( + engine: &TranslationEngine, + protocol: WireFormat, + request: &LlmRequest, +) -> Result<(), String> { + let output = engine + .encode_request(protocol, request, &request_policy(protocol)) + .map_err(error)?; + safe(&output.diagnostics) +} + +pub(crate) fn encode_response( + engine: &TranslationEngine, + protocol: WireFormat, + response: &AggLlmResponse, +) -> Result { + let output = engine + .encode_response(protocol, response, &policy()) + .map_err(error)?; + safe(&output.diagnostics)?; + Ok(output.body) +} + +fn policy() -> TranslationPolicy { + TranslationPolicy { + unknown_field_policy: UnknownFieldPolicy::Preserve, + lossy_conversion_policy: LossyConversionPolicy::Reject, + deterministic_ids: DeterministicIdPolicy::GenerateStable { + prefix: "relay".into(), + }, + preservation: PreservationPolicy::InMemory, + target_capabilities: TargetCapabilities::default(), + } +} + +fn request_policy(protocol: WireFormat) -> TranslationPolicy { + let mut policy = policy(); + if protocol == WireFormat::AnthropicMessages { + policy + .target_capabilities + .supports_json_schema_response_format = Some(false); + } + policy +} + +fn safe(diagnostics: &[TranslationDiagnostic]) -> Result<(), String> { + let unsafe_diagnostics = diagnostics + .iter() + .filter(|diagnostic| diagnostic.severity != DiagnosticSeverity::Info) + .collect::>(); + if unsafe_diagnostics.is_empty() { + Ok(()) + } else { + Err(format!( + "Switchyard translation was not lossless: {unsafe_diagnostics:?}" + )) + } +} + +fn error(error: switchyard_translation::TranslationError) -> String { + format!("Switchyard translation failed: {error}") +} + +#[cfg(test)] +mod tests { + use serde_json::{Map, json}; + + use super::*; + + #[test] + fn same_protocol_request_preserves_unknown_fields() { + let request = RelayRequest { + headers: Map::new(), + content: json!({ + "model": "route", + "messages": [{"role": "user", "content": "hello"}], + "provider_extension": {"exact": true} + }), + }; + let engine = TranslationEngine::default(); + let decoded = decode_request(&engine, WireFormat::OpenAiChat, &request).unwrap(); + validate_target_request(&engine, WireFormat::OpenAiChat, &decoded).unwrap(); + assert_eq!( + decoded + .preservation + .requests + .get(&WireFormat::OpenAiChat.into()) + .and_then(|body| body.get("provider_extension")), + Some(&json!({"exact": true})) + ); + } +} diff --git a/docs/index.md b/docs/index.md index 1dc5056cd..93814e9cc 100644 --- a/docs/index.md +++ b/docs/index.md @@ -10,6 +10,7 @@ It supports OpenAI Chat Completions, OpenAI Responses, and Anthropic Messages. | Run Claude Code, Codex, or OpenClaw through Switchyard | Launcher Path | [Install and launch an agent](getting_started.md#launcher-path) | | Run Switchyard as a standalone proxy for API clients | Server Path | [Build and run the Rust server](getting_started.md#server-path) | | Add Switchyard routing to a Rust application | Library Path | [`switchyard-libsy`](../crates/libsy/README.md) | +| Add Switchyard routing to NeMo Relay | Native Plugin Path | [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md) | The Launcher Path installs the `switchyard` CLI and hosts the native Rust server through its packaged PyO3 binding. The Server Path builds and runs the @@ -30,3 +31,4 @@ standalone `switchyard-server` binary. - [`switchyard-libsy`](reference/rust_api.md#switchyard-libsy): embeddable routing algorithms - [`switchyard-protocol`](reference/rust_api.md#switchyard-protocol): provider-neutral API types - [`switchyard-translation`](../crates/switchyard-translation/README.md): protocol translation +- [`switchyard-nemo-relay-plugin`](../crates/switchyard-nemo-relay-plugin/README.md): native NeMo Relay integration From 2aa4c6be2182be5cbd141b6e97f734baec60d6f3 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 11:47:42 -0600 Subject: [PATCH 2/9] docs(relay): trim transient plugin status Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 16 +++---- crates/switchyard-nemo-relay-plugin/README.md | 45 +++---------------- 2 files changed, 12 insertions(+), 49 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4151729b..0cdd90812 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,17 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). libsy's weighted-random, LLM-classifier, escalation, and stage-router algorithms in process while Switchyard owns provider HTTP dispatch, credentials, translation, retries, and fallback. Managed calls require NeMo - Relay 0.7 or newer and do not depend on `switchyard-server`. + Relay 0.7 or newer and do not depend on `switchyard-server`. Target bindings + accept non-secret `extra_body` provider defaults, preserved requests are + re-encoded after routing mutations, and synthetic Relay gateway identities do + not become shared router session state. - **NeMo Relay routing-model usage marks** — classifier judges, escalation judges and discarded weak candidates, and failed routing candidates now emit `switchyard.routing.llm_call` ATOF marks with normalized token usage and latency. The final serving call remains represented only by Relay's outer LLM - lifecycle event to prevent double-counting. + lifecycle event to prevent double-counting. Stage decision marks retain + picker-default tiers, decision sources, and hard-override confidence. ### Removed @@ -45,14 +49,6 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixed -- **NeMo Relay stage and escalation integration** — preserved same-protocol - request bodies are now re-encoded after tier prompts or handoff notes mutate - the normalized request; Relay's synthetic `gateway-gateway` identity no - longer shares escalation latch state across unrelated raw gateway requests; - stage decision marks now retain picker-default tiers, decision sources, and - hard-override confidence. Target bindings also accept non-secret - `extra_body` defaults for provider-specific judge controls. - - **Response `model` now names the model that actually served the request**, on every serving path and wire format. Streamed Anthropic and Responses replies, and every libsy-served reply, previously echoed the model id the client diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 0182f4de2..203e4b23d 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -144,31 +144,12 @@ those fields directly under `text.format`. InferenceHub therefore returns HTTP 400 with `Missing required parameter: 'text.format.name'`, and the affected router follows its existing judge-failure or fall-open path. -A hosted Relay process run passed 20 of 23 router-matrix cases. The only -failures were the Responses-judge variants of the capability classifier, -escalation router, and stage classifier fallback. OpenAI Responses remains -verified as a caller and ordinary serving-target protocol. Until the shared -`switchyard-translation` encoder is corrected, configure structured-output -judges with `protocol = "openai_chat"`. Follow-up work must add the inverse of -the existing Responses-to-neutral schema conversion plus core and -process-level regression coverage for all three affected router paths. - -The following integration components have not reached complete compatibility. -`Not built` identifies missing integration work rather than a hidden or -best-effort runtime path. - -| Compatibility Component | Status | Current Boundary | -|---|---|---| -| Pinned Relay process-level acceptance harness | Not built | The plugin's standalone tests run through its nested Cargo workspace. Relay 0.7.1 plus Ollama smoke coverage is manual and currently covers OpenAI Chat paths for escalation and stage routing. | -| OpenAI Responses and Anthropic Messages process-level routing matrix | Not built | Translation and in-process runtime coverage exists, including stage signals across all three caller protocols, but no automated Relay gateway matrix exercises every algorithm and target combination. | -| Real coding-agent acceptance harness | Not built | Codex, Claude Code, and Hermes sessions are not driven automatically through the packaged plugin. | -| Hosted-provider certification | Not built | No automated suite qualifies structured-output judges, serving targets, latency, or token cost against hosted provider APIs. | -| Native bundle platform and Relay-version matrix | Partial | The plugin was smoked manually on macOS arm64 with Relay 0.7.1. Dedicated packaged-plugin jobs do not yet cover Linux and Windows bundles or every supported Relay 0.7 release. | -| Dynamic-plugin lifecycle automation | Partial | Manifest validation, registration, enablement, execution, and unload were exercised manually; they are not part of a repeatable CI acceptance test. | -| Nested Relay lifecycle telemetry for managed provider calls | Not built | Relay records the outer serving call. Routing-only model calls emit `switchyard.routing.llm_call` marks with normalized usage, but Switchyard provider HTTP spans are not bridged into nested Relay LLM lifecycle events. | -| Provider `Retry-After` propagation | Not built | The outer routing loop uses bounded exponential backoff because the client error contract does not expose `Retry-After`. | -| Cross-protocol streaming loss diagnostics | Not built | Cross-protocol streams use normalized chunks, but the stream adapter does not surface the buffered translation engine's reject-lossy diagnostics. | -| Safe typed Relay asynchronous host adapter | Blocked on host API | The plugin uses its tested raw C ownership adapter until Relay exposes an equivalent safe asynchronous Rust facade. | +OpenAI Responses remains supported as a caller and ordinary serving-target +protocol. Until the shared `switchyard-translation` encoder is corrected, +configure structured-output judges with `protocol = "openai_chat"`. Follow-up +work must add the inverse of the existing Responses-to-neutral schema conversion +plus core and process-level regression coverage for all three affected router +paths. Managed inner provider calls also do not re-enter Relay's downstream provider middleware. This behavior is part of the current ownership boundary, not an @@ -410,17 +391,3 @@ nemo-relay plugins add /opt/switchyard-relay-plugin/relay-plugin.toml nemo-relay plugins enable nvidia.switchyard nemo-relay plugins inspect nvidia.switchyard ``` - -## Validation expectations - -Before release, validate both routers against buffered and streaming OpenAI -Chat, OpenAI Responses, and Anthropic Messages providers. The acceptance suite -must cover same- and supported cross-protocol routes, deterministic weighted -routing, independent runs, classifier weak and strong selections, retry -reselection, exhaustion, exactly-once fallback, stream commitment, empty -streams, late errors, cancellation, credential privacy, and unmanaged -pass-through. - -The tests must also prove that managed target traffic reaches the provider -through `switchyard-llm-client`, never through Relay's provider continuation, -and that no Switchyard service or health endpoint is involved. From 17d8cb026f22c7b61871d8065f300786d4013db8 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 12:47:05 -0600 Subject: [PATCH 3/9] test(relay): streamline plugin test modules Signed-off-by: Bryan Bednarski --- .../src/config.rs | 555 +-------- .../src/config/tests.rs | 545 +++++++++ .../src/runtime.rs | 1011 +---------------- .../src/runtime/tests.rs | 920 +++++++++++++++ 4 files changed, 1467 insertions(+), 1564 deletions(-) create mode 100644 crates/switchyard-nemo-relay-plugin/src/config/tests.rs create mode 100644 crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 435718fcb..5b3722763 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -630,557 +630,4 @@ fn default_classifier_max_output_tokens() -> u64 { } #[cfg(test)] -mod tests { - use super::*; - use serde_json::{Value, json}; - - fn binding(protocol: WireFormat, model: &str) -> TargetBinding { - TargetBinding { - model: model.into(), - protocol, - endpoint: String::new(), - base_url: "https://provider.example/v1".into(), - weight: 1.0, - drop_caller_extra_body: false, - header_env: BTreeMap::new(), - extra_body: BTreeMap::new(), - } - } - - fn config() -> SwitchyardConfig { - SwitchyardConfig { - version: 2, - priority: 0, - max_retries: 3, - algorithm: AlgorithmConfig::Random { seed: Some(42) }, - targets: BTreeMap::from([ - ( - "chat".into(), - binding(WireFormat::OpenAiChat, "provider/chat"), - ), - ( - "responses".into(), - binding(WireFormat::OpenAiResponses, "provider/responses"), - ), - ( - "anthropic".into(), - binding(WireFormat::AnthropicMessages, "provider/anthropic"), - ), - ]), - default_targets: BTreeMap::from([ - (WireFormat::OpenAiChat, "chat".into()), - (WireFormat::OpenAiResponses, "responses".into()), - (WireFormat::AnthropicMessages, "anthropic".into()), - ]), - } - } - - #[test] - fn version_two_random_configuration_builds_clients_without_a_service() { - let config = config(); - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "random"); - assert_eq!(prepared.targets.len(), 3); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); - } - - #[test] - fn version_one_reports_the_service_to_library_migration() { - let mut config = config(); - config.version = 1; - let error = config.validate().unwrap_err(); - assert!(error.contains("version 1 used switchyard-server")); - assert!(error.contains("version = 2")); - } - - #[test] - fn target_endpoints_must_be_canonical_for_the_current_http_client() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); - let error = config.validate().unwrap_err(); - assert!(error.contains("ending in \"/chat/completions\"")); - - config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); - config.validate().unwrap(); - assert_eq!( - config.targets["chat"].dispatch_url(), - "https://provider.example/v1/custom/chat/completions" - ); - } - - #[test] - fn complete_provider_endpoint_is_not_appended_twice() { - let mut config = config(); - let chat = config.targets.get_mut("chat").unwrap(); - chat.base_url = "https://provider.example/v1/chat/completions/".into(); - assert_eq!( - chat.dispatch_url(), - "https://provider.example/v1/chat/completions" - ); - config.validate().unwrap(); - } - - #[test] - fn absolute_urls_cannot_embed_credentials_or_query_parameters() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().base_url = - "https://user:password@provider.example/v1".into(); - assert!( - config - .validate() - .unwrap_err() - .contains("embedded credentials") - ); - - config.targets.get_mut("chat").unwrap().base_url = - "https://provider.example/v1?api-version=1".into(); - assert!(config.validate().unwrap_err().contains("query parameters")); - } - - #[test] - fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { - let mut host_header_config = config(); - let chat = host_header_config.targets.get_mut("chat").unwrap(); - chat.header_env.insert("Host".into(), "TARGET_HOST".into()); - assert!( - host_header_config - .validate() - .unwrap_err() - .contains("HTTP transport") - ); - - let mut duplicate_config = config(); - let chat = duplicate_config.targets.get_mut("chat").unwrap(); - chat.header_env - .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); - chat.header_env - .insert("x-tenant".into(), "TARGET_TENANT_B".into()); - assert!( - duplicate_config - .validate() - .unwrap_err() - .contains("more than once") - ); - } - - #[test] - fn only_canonical_relay_execution_names_resolve_protocols() { - assert_eq!( - protocol_from_call("openai.chat_completions"), - Some(WireFormat::OpenAiChat) - ); - assert_eq!( - protocol_from_call("openai.responses"), - Some(WireFormat::OpenAiResponses) - ); - assert_eq!( - protocol_from_call("anthropic.messages"), - Some(WireFormat::AnthropicMessages) - ); - assert_eq!(protocol_from_call("openai_chat"), None); - } - - #[test] - fn schema_required_contract_fields_do_not_default_during_deserialization() { - let base = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1" - } - }, - "default_targets": {"openai_chat": "chat"} - }); - for field in ["version", "algorithm", "default_targets"] { - let mut value = base.clone(); - value.as_object_mut().unwrap().remove(field); - let error = serde_json::from_value::(value) - .err() - .expect("required field must not default"); - assert!(error.to_string().contains(field), "field={field}: {error}"); - } - } - - #[test] - fn unknown_target_fields_are_rejected() { - let value = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1", - "unexpected_setting": true - } - }, - "default_targets": {"openai_chat": "chat"} - }); - let error = serde_json::from_value::(value) - .err() - .expect("unknown target field must be rejected"); - assert!(error.to_string().contains("unexpected_setting")); - } - - #[test] - fn literal_target_headers_are_rejected() { - let value = json!({ - "version": 2, - "algorithm": {"kind": "random"}, - "targets": { - "chat": { - "model": "provider/chat", - "protocol": "openai_chat", - "base_url": "https://provider.example/v1", - "headers": {"x-provider-token": "plaintext-secret"} - } - }, - "default_targets": {"openai_chat": "chat"} - }); - let error = serde_json::from_value::(value) - .err() - .expect("literal target headers must be rejected") - .to_string(); - assert!(error.contains("unknown field `headers`")); - assert!(!error.contains("plaintext-secret")); - } - - #[test] - fn unknown_algorithm_fields_are_rejected() { - let error = serde_json::from_value::(json!({ - "kind": "random", - "seed": 42, - "unexpected_setting": true - })) - .err() - .expect("unknown algorithm field must be rejected"); - assert!(error.to_string().contains("unexpected_setting")); - } - - #[test] - fn classifier_prepares_clients_for_judge_and_routed_targets() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "recent_turn_window": 4, - "max_output_tokens": 512 - })) - .unwrap(); - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); - } - - #[test] - fn target_provider_defaults_are_accepted_for_judge_controls() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().extra_body = - BTreeMap::from([("think".into(), json!(false))]); - - config.validate().unwrap(); - config.prepare().unwrap(); - } - - #[test] - fn classifier_rejects_anthropic_judge_targets_before_dispatch() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "anthropic", - "weak_target": "responses", - "strong_target": "chat", - "base_threshold": 0.5 - })) - .unwrap(); - - let error = config.validate().unwrap_err(); - assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); - } - - #[test] - fn validation_does_not_resolve_environment_backed_headers() { - let mut config = config(); - config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( - "authorization".into(), - "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), - )]); - - config.validate().unwrap(); - let error = config - .prepare() - .err() - .expect("preparation must resolve headers"); - assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); - } - - #[test] - fn invalid_environment_variable_names_are_rejected_before_resolution() { - for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { - let mut config = config(); - config.targets.get_mut("chat").unwrap().header_env = - BTreeMap::from([("authorization".into(), variable.into())]); - - let error = config.validate().unwrap_err(); - assert!(error.contains("must not contain '=' or NUL")); - } - } - - #[test] - fn static_validation_preserves_algorithm_constructor_checks() { - let mut random = config(); - for target in random.targets.values_mut() { - target.weight = 0.0; - } - assert!( - random - .validate() - .unwrap_err() - .contains("at least one positive target weight") - ); - - let mut classifier = config(); - classifier.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 1.1 - })) - .unwrap(); - assert!( - classifier - .validate() - .unwrap_err() - .contains("base_threshold must be between 0 and 1") - ); - } - - #[test] - fn escalation_classifier_builds_with_defaulted_policy_settings() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "prompt": "Judge the completed trajectory.", - "max_output_tokens": 256, - "escalation": {} - })) - .unwrap(); - - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); - } - - #[test] - fn classifier_modes_reject_mixed_or_missing_settings() { - let mut capability = config(); - capability.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "escalation": {} - })) - .unwrap(); - assert!( - capability - .validate() - .unwrap_err() - .contains("capability mode does not accept escalation") - ); - - let mut escalation = config(); - escalation.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "base_threshold": 0.5, - "escalation": {} - })) - .unwrap(); - assert!( - escalation - .validate() - .unwrap_err() - .contains("escalation mode does not accept capability") - ); - - let mut missing = config(); - missing.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic" - })) - .unwrap(); - assert!( - missing - .validate() - .unwrap_err() - .contains("requires escalation settings") - ); - } - - #[test] - fn escalation_settings_are_validated_by_the_libsy_constructor() { - for (settings, expected) in [ - ( - json!({"confirmations": 0}), - "confirmations must be at least 1", - ), - ( - json!({"recent_turn_window": 0}), - "recent_turn_window must be at least 1", - ), - ( - json!({"window_message_chars": 49}), - "window_message_chars must be at least 50", - ), - ] { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "llm_classifier", - "mode": "escalation", - "classifier_target": "chat", - "weak_target": "responses", - "strong_target": "anthropic", - "escalation": settings - })) - .unwrap(); - assert!(config.validate().unwrap_err().contains(expected)); - } - } - - #[test] - fn full_stage_router_configuration_builds_all_clients() { - let mut config = config(); - config.algorithm = serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "anthropic", - "efficient_target": "responses", - "picker": "efficient_first", - "confidence_threshold": 0.5, - "recent_turn_window": 3, - "capable_system_prompt": "Diagnose before editing.", - "efficient_system_prompt": "Follow the settled plan.", - "handoff_notes": { - "escalation_note": "The previous model was stalling.", - "deescalation_note": "The task is settled.", - "only_on_wrong_signal_escalation": true - }, - "classifier": { - "target": "chat", - "base_threshold": 0.5, - "threshold_step": 0.1, - "recent_turn_window": 3, - "prompt": "Can the efficient tier finish this turn?", - "max_output_tokens": 256 - } - })) - .unwrap(); - - config.validate().unwrap(); - let prepared = config.prepare().unwrap(); - assert_eq!(prepared.algorithm.name(), "stage_router"); - assert!( - prepared - .targets - .values() - .all(|target| Arc::strong_count(&target.client) == 1) - ); - } - - #[test] - fn stage_router_validates_threshold_targets_and_judge_protocol() { - let stage = |classifier: Value, threshold: f64| { - serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "anthropic", - "efficient_target": "responses", - "picker": "capable_first", - "confidence_threshold": threshold, - "classifier": classifier - })) - .unwrap() - }; - - let mut invalid_threshold = config(); - invalid_threshold.algorithm = stage(Value::Null, 1.1); - assert!( - invalid_threshold - .validate() - .unwrap_err() - .contains("confidence_threshold must be between 0 and 1") - ); - - let mut missing_target = config(); - missing_target.algorithm = serde_json::from_value(json!({ - "kind": "stage_router", - "capable_target": "missing", - "efficient_target": "responses", - "picker": "capable_first", - "confidence_threshold": 0.5 - })) - .unwrap(); - assert!( - missing_target - .validate() - .unwrap_err() - .contains("algorithm target \"missing\" is not configured") - ); - - let mut anthropic_judge = config(); - anthropic_judge.algorithm = - stage(json!({"target": "anthropic", "base_threshold": 0.5}), 0.5); - assert!( - anthropic_judge - .validate() - .unwrap_err() - .contains("classifier target \"anthropic\" uses anthropic_messages") - ); - } - - #[test] - fn zero_weight_random_targets_are_fallback_only() { - let mut config = config(); - config.targets.get_mut("anthropic").unwrap().weight = 0.0; - let prepared = config.prepare().unwrap(); - - assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); - assert_eq!(Arc::strong_count(&prepared.targets["chat"].client), 1); - assert_eq!(Arc::strong_count(&prepared.targets["responses"].client), 1); - } -} +mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/config/tests.rs b/crates/switchyard-nemo-relay-plugin/src/config/tests.rs new file mode 100644 index 000000000..1a1153e9f --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/config/tests.rs @@ -0,0 +1,545 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use super::*; +use serde_json::{Value, json}; + +fn binding(protocol: WireFormat, model: &str) -> TargetBinding { + TargetBinding { + model: model.into(), + protocol, + endpoint: String::new(), + base_url: "https://provider.example/v1".into(), + weight: 1.0, + drop_caller_extra_body: false, + header_env: BTreeMap::new(), + extra_body: BTreeMap::new(), + } +} + +fn config() -> SwitchyardConfig { + SwitchyardConfig { + version: 2, + priority: 0, + max_retries: 3, + algorithm: AlgorithmConfig::Random { seed: Some(42) }, + targets: BTreeMap::from([ + ( + "chat".into(), + binding(WireFormat::OpenAiChat, "provider/chat"), + ), + ( + "responses".into(), + binding(WireFormat::OpenAiResponses, "provider/responses"), + ), + ( + "anthropic".into(), + binding(WireFormat::AnthropicMessages, "provider/anthropic"), + ), + ]), + default_targets: BTreeMap::from([ + (WireFormat::OpenAiChat, "chat".into()), + (WireFormat::OpenAiResponses, "responses".into()), + (WireFormat::AnthropicMessages, "anthropic".into()), + ]), + } +} + +#[test] +fn version_two_random_configuration_builds_clients_without_a_service() { + let config = config(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "random"); + assert_eq!(prepared.targets.len(), 3); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn target_endpoints_must_be_canonical_for_the_current_http_client() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat".into(); + let error = config.validate().unwrap_err(); + assert!(error.contains("ending in \"/chat/completions\"")); + + config.targets.get_mut("chat").unwrap().endpoint = "/custom/chat/completions".into(); + config.validate().unwrap(); + assert_eq!( + config.targets["chat"].dispatch_url(), + "https://provider.example/v1/custom/chat/completions" + ); +} + +#[test] +fn complete_provider_endpoint_is_not_appended_twice() { + let mut config = config(); + let chat = config.targets.get_mut("chat").unwrap(); + chat.base_url = "https://provider.example/v1/chat/completions/".into(); + assert_eq!( + chat.dispatch_url(), + "https://provider.example/v1/chat/completions" + ); + config.validate().unwrap(); +} + +#[test] +fn absolute_urls_cannot_embed_credentials_or_query_parameters() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().base_url = + "https://user:password@provider.example/v1".into(); + assert!( + config + .validate() + .unwrap_err() + .contains("embedded credentials") + ); + + config.targets.get_mut("chat").unwrap().base_url = + "https://provider.example/v1?api-version=1".into(); + assert!(config.validate().unwrap_err().contains("query parameters")); +} + +#[test] +fn transport_owned_and_case_duplicate_environment_headers_are_rejected() { + let mut host_header_config = config(); + let chat = host_header_config.targets.get_mut("chat").unwrap(); + chat.header_env.insert("Host".into(), "TARGET_HOST".into()); + assert!( + host_header_config + .validate() + .unwrap_err() + .contains("HTTP transport") + ); + + let mut duplicate_config = config(); + let chat = duplicate_config.targets.get_mut("chat").unwrap(); + chat.header_env + .insert("X-Tenant".into(), "TARGET_TENANT_A".into()); + chat.header_env + .insert("x-tenant".into(), "TARGET_TENANT_B".into()); + assert!( + duplicate_config + .validate() + .unwrap_err() + .contains("more than once") + ); +} + +#[test] +fn only_canonical_relay_execution_names_resolve_protocols() { + assert_eq!( + protocol_from_call("openai.chat_completions"), + Some(WireFormat::OpenAiChat) + ); + assert_eq!( + protocol_from_call("openai.responses"), + Some(WireFormat::OpenAiResponses) + ); + assert_eq!( + protocol_from_call("anthropic.messages"), + Some(WireFormat::AnthropicMessages) + ); + assert_eq!(protocol_from_call("openai_chat"), None); +} + +#[test] +fn schema_required_contract_fields_do_not_default_during_deserialization() { + let base = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1" + } + }, + "default_targets": {"openai_chat": "chat"} + }); + for field in ["version", "algorithm", "default_targets"] { + let mut value = base.clone(); + value.as_object_mut().unwrap().remove(field); + let error = serde_json::from_value::(value) + .err() + .expect("required field must not default"); + assert!(error.to_string().contains(field), "field={field}: {error}"); + } +} + +#[test] +fn unknown_target_fields_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "unexpected_setting": true + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("unknown target field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); +} + +#[test] +fn literal_target_headers_are_rejected() { + let value = json!({ + "version": 2, + "algorithm": {"kind": "random"}, + "targets": { + "chat": { + "model": "provider/chat", + "protocol": "openai_chat", + "base_url": "https://provider.example/v1", + "headers": {"x-provider-token": "plaintext-secret"} + } + }, + "default_targets": {"openai_chat": "chat"} + }); + let error = serde_json::from_value::(value) + .err() + .expect("literal target headers must be rejected") + .to_string(); + assert!(error.contains("unknown field `headers`")); + assert!(!error.contains("plaintext-secret")); +} + +#[test] +fn unknown_algorithm_fields_are_rejected() { + let error = serde_json::from_value::(json!({ + "kind": "random", + "seed": 42, + "unexpected_setting": true + })) + .err() + .expect("unknown algorithm field must be rejected"); + assert!(error.to_string().contains("unexpected_setting")); +} + +#[test] +fn classifier_prepares_clients_for_judge_and_routed_targets() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "recent_turn_window": 4, + "max_output_tokens": 512 + })) + .unwrap(); + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn target_provider_defaults_are_accepted_for_judge_controls() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().extra_body = + BTreeMap::from([("think".into(), json!(false))]); + + config.validate().unwrap(); + config.prepare().unwrap(); +} + +#[test] +fn classifier_rejects_anthropic_judge_targets_before_dispatch() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "anthropic", + "weak_target": "responses", + "strong_target": "chat", + "base_threshold": 0.5 + })) + .unwrap(); + + let error = config.validate().unwrap_err(); + assert!(error.contains("classifier target \"anthropic\" uses anthropic_messages")); +} + +#[test] +fn validation_does_not_resolve_environment_backed_headers() { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = BTreeMap::from([( + "authorization".into(), + "SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET".into(), + )]); + + config.validate().unwrap(); + let error = config + .prepare() + .err() + .expect("preparation must resolve headers"); + assert!(error.contains("SWITCHYARD_TEST_ENVIRONMENT_VARIABLE_THAT_IS_NOT_SET")); +} + +#[test] +fn invalid_environment_variable_names_are_rejected_before_resolution() { + for variable in ["INVALID=VARIABLE", "INVALID\0VARIABLE"] { + let mut config = config(); + config.targets.get_mut("chat").unwrap().header_env = + BTreeMap::from([("authorization".into(), variable.into())]); + + let error = config.validate().unwrap_err(); + assert!(error.contains("must not contain '=' or NUL")); + } +} + +#[test] +fn static_validation_preserves_algorithm_constructor_checks() { + let mut random = config(); + for target in random.targets.values_mut() { + target.weight = 0.0; + } + assert!( + random + .validate() + .unwrap_err() + .contains("at least one positive target weight") + ); + + let mut classifier = config(); + classifier.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 1.1 + })) + .unwrap(); + assert!( + classifier + .validate() + .unwrap_err() + .contains("base_threshold must be between 0 and 1") + ); +} + +#[test] +fn escalation_classifier_builds_with_defaulted_policy_settings() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "prompt": "Judge the completed trajectory.", + "max_output_tokens": 256, + "escalation": {} + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "llm_task_classifier"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn classifier_modes_reject_mixed_or_missing_settings() { + let mut capability = config(); + capability.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + capability + .validate() + .unwrap_err() + .contains("capability mode does not accept escalation") + ); + + let mut escalation = config(); + escalation.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "base_threshold": 0.5, + "escalation": {} + })) + .unwrap(); + assert!( + escalation + .validate() + .unwrap_err() + .contains("escalation mode does not accept capability") + ); + + let mut missing = config(); + missing.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic" + })) + .unwrap(); + assert!( + missing + .validate() + .unwrap_err() + .contains("requires escalation settings") + ); +} + +#[test] +fn escalation_settings_are_validated_by_the_libsy_constructor() { + for (settings, expected) in [ + ( + json!({"confirmations": 0}), + "confirmations must be at least 1", + ), + ( + json!({"recent_turn_window": 0}), + "recent_turn_window must be at least 1", + ), + ( + json!({"window_message_chars": 49}), + "window_message_chars must be at least 50", + ), + ] { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "llm_classifier", + "mode": "escalation", + "classifier_target": "chat", + "weak_target": "responses", + "strong_target": "anthropic", + "escalation": settings + })) + .unwrap(); + assert!(config.validate().unwrap_err().contains(expected)); + } +} + +#[test] +fn full_stage_router_configuration_builds_all_clients() { + let mut config = config(); + config.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "efficient_first", + "confidence_threshold": 0.5, + "recent_turn_window": 3, + "capable_system_prompt": "Diagnose before editing.", + "efficient_system_prompt": "Follow the settled plan.", + "handoff_notes": { + "escalation_note": "The previous model was stalling.", + "deescalation_note": "The task is settled.", + "only_on_wrong_signal_escalation": true + }, + "classifier": { + "target": "chat", + "base_threshold": 0.5, + "threshold_step": 0.1, + "recent_turn_window": 3, + "prompt": "Can the efficient tier finish this turn?", + "max_output_tokens": 256 + } + })) + .unwrap(); + + config.validate().unwrap(); + let prepared = config.prepare().unwrap(); + assert_eq!(prepared.algorithm.name(), "stage_router"); + assert!( + prepared + .targets + .values() + .all(|target| Arc::strong_count(&target.client) == 1) + ); +} + +#[test] +fn stage_router_validates_threshold_targets_and_judge_protocol() { + let stage = |classifier: Value, threshold: f64| { + serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "anthropic", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": threshold, + "classifier": classifier + })) + .unwrap() + }; + + let mut invalid_threshold = config(); + invalid_threshold.algorithm = stage(Value::Null, 1.1); + assert!( + invalid_threshold + .validate() + .unwrap_err() + .contains("confidence_threshold must be between 0 and 1") + ); + + let mut missing_target = config(); + missing_target.algorithm = serde_json::from_value(json!({ + "kind": "stage_router", + "capable_target": "missing", + "efficient_target": "responses", + "picker": "capable_first", + "confidence_threshold": 0.5 + })) + .unwrap(); + assert!( + missing_target + .validate() + .unwrap_err() + .contains("algorithm target \"missing\" is not configured") + ); + + let mut anthropic_judge = config(); + anthropic_judge.algorithm = stage(json!({"target": "anthropic", "base_threshold": 0.5}), 0.5); + assert!( + anthropic_judge + .validate() + .unwrap_err() + .contains("classifier target \"anthropic\" uses anthropic_messages") + ); +} + +#[test] +fn zero_weight_random_targets_are_fallback_only() { + let mut config = config(); + config.targets.get_mut("anthropic").unwrap().weight = 0.0; + let prepared = config.prepare().unwrap(); + + assert_eq!(Arc::strong_count(&prepared.targets["anthropic"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["chat"].client), 1); + assert_eq!(Arc::strong_count(&prepared.targets["responses"].client), 1); +} diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index ab7f8d08e..73cd3fa51 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -781,1013 +781,4 @@ fn context_from_metadata(metadata: Option<&Metadata>) -> Context { } #[cfg(test)] -mod tests { - use std::sync::atomic::{AtomicUsize, Ordering}; - - use switchyard_libsy::{ - ClassifierContractConfig, EscalationJudgeConfig, LlmClassifierConfig, LlmFallback, - LlmTarget, LlmTaskClassifier, Passthrough, PickerMode, StageRouter, StageRouterConfig, - TaskClassifierConfig, - }; - use switchyard_protocol::{ - ContentBlock, LlmRequest, LlmResponseStream, Message, Role, RoutedLlmClient, ToolCall, - ToolResult, Usage, text_request, text_response, - }; - - use super::*; - - #[derive(Clone, Copy)] - enum StreamBehavior { - Empty, - Failing, - CallFailure, - } - - struct StreamClient { - behavior: StreamBehavior, - calls: AtomicUsize, - } - - struct BufferedClient { - calls: AtomicUsize, - } - - enum FixedBehavior { - Text(&'static str), - TransportFailure, - } - - struct FixedClient { - behavior: FixedBehavior, - calls: AtomicUsize, - } - - #[async_trait::async_trait] - impl RoutedLlmClient for StreamClient { - async fn call( - &self, - _ctx: Context, - _request: Request, - _decision: Arc, - ) -> Result { - self.calls.fetch_add(1, Ordering::Relaxed); - let stream: LlmResponseStream = match self.behavior { - StreamBehavior::Empty => Box::pin(stream::empty()), - StreamBehavior::Failing => Box::pin(stream::once(async { - Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other("fallback stream failed")), - }) - })), - StreamBehavior::CallFailure => { - return Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other("fallback call failed")), - }); - } - }; - Ok(Response { - llm_response: LlmResponse::Stream(stream), - metadata: None, - }) - } - } - - #[async_trait::async_trait] - impl RoutedLlmClient for BufferedClient { - async fn call( - &self, - _ctx: Context, - _request: Request, - _decision: Arc, - ) -> Result { - self.calls.fetch_add(1, Ordering::Relaxed); - Ok(Response { - llm_response: LlmResponse::Agg(Default::default()), - metadata: None, - }) - } - } - - #[async_trait::async_trait] - impl RoutedLlmClient for FixedClient { - async fn call( - &self, - _ctx: Context, - request: Request, - _decision: Arc, - ) -> Result { - self.calls.fetch_add(1, Ordering::Relaxed); - match self.behavior { - FixedBehavior::Text(text) => { - let mut response = text_response(None, text); - response.usage = Usage { - input_tokens: Some(11), - output_tokens: Some(7), - total_tokens: Some(18), - ..Usage::default() - }; - Ok(Response { - llm_response: LlmResponse::Agg(response), - metadata: request.metadata, - }) - } - FixedBehavior::TransportFailure => Err(LlmClientError::Transport { - source: Box::new(std::io::Error::other("scripted failure")), - }), - } - } - } - - fn fixed_target(name: &str, _client: Arc) -> LlmTarget { - LlmTarget { - semantic_name: name.to_string(), - } - } - - fn runtime_with_algorithm( - algorithm: Arc, - fallback: Arc, - protocol: WireFormat, - ) -> SwitchyardRuntime { - runtime_with_algorithm_clients(algorithm, fallback, protocol, Vec::new()) - } - - fn runtime_with_algorithm_clients( - algorithm: Arc, - fallback: Arc, - protocol: WireFormat, - clients: Vec<(&str, Arc)>, - ) -> SwitchyardRuntime { - let is_stage = algorithm.name() == "stage_router"; - let mut targets = BTreeMap::from([( - "fallback".into(), - PreparedTargetBinding { - client: fallback as Arc, - }, - )]); - for (name, client) in clients { - targets.insert( - name.to_string(), - PreparedTargetBinding { - client: client as Arc, - }, - ); - } - SwitchyardRuntime { - max_retries: 0, - algorithm, - targets, - default_targets: BTreeMap::from([(protocol, "fallback".into())]), - target_tiers: BTreeMap::from([("weak".into(), "weak"), ("strong".into(), "strong")]), - stage_marks: is_stage.then_some(StageMarkConfig { - picker: PickerMode::CapableFirst, - confidence_threshold: 0.5, - recent_turn_window: None, - classifier_enabled: true, - }), - translation: TranslationEngine::default(), - } - } - - fn request_with_session(protocol: WireFormat, session: Option<&str>) -> Request { - Request { - llm_request: text_request(Some("auto".into()), "fix the build"), - raw_request: None, - metadata: Some(Metadata { - wire_format: Some(protocol), - session_id: session.map(str::to_string), - ..Metadata::default() - }), - } - } - - fn stage_signal_request(protocol: WireFormat) -> Request { - Request { - llm_request: LlmRequest { - model: Some("auto".into()), - messages: vec![ - Message::text(Role::User, "fix the build"), - Message { - role: Role::Assistant, - content: vec![ContentBlock::ToolCall(ToolCall { - id: "call-1".into(), - name: "bash".into(), - arguments: json!({"cmd": "cargo test"}), - })], - }, - Message { - role: Role::Tool, - content: vec![ContentBlock::ToolResult(ToolResult { - tool_call_id: "call-1".into(), - content: vec![ContentBlock::Text { - text: "fatal runtime error: out of memory".into(), - }], - is_error: Some(true), - })], - }, - ], - ..LlmRequest::default() - }, - raw_request: None, - metadata: Some(Metadata { - wire_format: Some(protocol), - session_id: Some(format!("stage-{}", protocol.as_str())), - ..Metadata::default() - }), - } - } - - #[test] - fn relay_gateway_placeholder_session_is_not_retained() { - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let runtime = runtime_with_algorithm( - Arc::new(Passthrough::new(LlmTarget { - semantic_name: "selected".into(), - })), - fallback, - WireFormat::OpenAiChat, - ); - let request = RelayRequest { - headers: Map::from_iter([ - ("x-nemo-relay-source".into(), json!("gateway")), - ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), - ("x-dynamo-session-id".into(), json!("gateway-gateway")), - ]), - content: json!({ - "model": "router", - "messages": [{"role": "user", "content": "hello"}] - }), - }; - - let decoded = runtime - .decode_request(WireFormat::OpenAiChat, &request, false) - .unwrap(); - - assert_eq!(decoded.metadata.unwrap().session_id, None); - } - - #[test] - fn explicit_switchyard_session_overrides_relay_gateway_placeholder() { - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let runtime = runtime_with_algorithm( - Arc::new(Passthrough::new(LlmTarget { - semantic_name: "selected".into(), - })), - fallback, - WireFormat::OpenAiChat, - ); - let request = RelayRequest { - headers: Map::from_iter([ - ("x-switchyard-session-id".into(), json!("caller-session")), - ("x-nemo-relay-source".into(), json!("gateway")), - ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), - ]), - content: json!({ - "model": "router", - "messages": [{"role": "user", "content": "hello"}] - }), - }; - - let decoded = runtime - .decode_request(WireFormat::OpenAiChat, &request, false) - .unwrap(); - - assert_eq!( - decoded.metadata.unwrap().session_id.as_deref(), - Some("caller-session") - ); - } - - #[tokio::test] - async fn buffered_finalization_failure_uses_fallback_once() { - let selected = Arc::new(StreamClient { - behavior: StreamBehavior::Empty, - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(BufferedClient { - calls: AtomicUsize::new(0), - }); - let runtime = SwitchyardRuntime { - max_retries: 1, - algorithm: Arc::new(Passthrough::new(LlmTarget { - semantic_name: "selected".into(), - })), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, - translation: TranslationEngine::default(), - }; - let mut marks = Vec::new(); - - let response = runtime - .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) - .await - .expect("the buffered fallback response should be encoded"); - - assert!(response.is_object()); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); - assert!( - !marks - .iter() - .any(|mark| mark.name == "switchyard.routing.retry") - ); - let error = marks - .iter() - .find(|mark| mark.name == "switchyard.routing.error") - .expect("finalization failure should emit an error mark"); - assert_eq!(error.data["retryable"], false); - assert_eq!(error.data["non_http_kind"], "invalid_response"); - assert_eq!( - marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.fallback") - .count(), - 1 - ); - } - - #[tokio::test] - async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { - let content = json!({ - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "gpt-4o", - "system_fingerprint": "fp_provider_specific", - "choices": [{ - "index": 0, - "delta": {"content": "Hi"}, - "finish_reason": null - }] - }); - let terminal = json!({ - "id": "chatcmpl-test", - "object": "chat.completion.chunk", - "model": "gpt-4o", - "choices": [{ - "index": 0, - "delta": {}, - "finish_reason": "stop" - }] - }); - let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); - let stream = switchyard_translation::decode_stream( - stream::once(async move { Ok::<_, LlmClientError>(body) }), - WireFormat::OpenAiChat, - ) - .expect("provider SSE should decode"); - let response = Response { - llm_response: LlmResponse::Stream(stream), - metadata: None, - }; - - let replayed = returned_events(response, WireFormat::OpenAiChat) - .await - .expect("return stream should encode") - .collect::>() - .await - .into_iter() - .collect::, _>>() - .expect("return stream should not fail"); - - assert_eq!(replayed, vec![content, terminal]); - } - - #[tokio::test] - async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { - let selected = Arc::new(StreamClient { - behavior: StreamBehavior::Empty, - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(StreamClient { - behavior: StreamBehavior::Failing, - calls: AtomicUsize::new(0), - }); - let runtime = SwitchyardRuntime { - max_retries: 0, - algorithm: Arc::new(Passthrough::new(LlmTarget { - semantic_name: "selected".into(), - })), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, - translation: TranslationEngine::default(), - }; - let (output, _messages) = async_channel::bounded(32); - - let error = runtime - .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) - .await - .expect_err("the failing fallback stream must fail the request"); - - assert_eq!(error, "trusted fallback stream: provider transport failure"); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); - } - - #[tokio::test] - async fn failing_fallback_call_flushes_error_and_fallback_marks() { - let selected = Arc::new(StreamClient { - behavior: StreamBehavior::Empty, - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(StreamClient { - behavior: StreamBehavior::CallFailure, - calls: AtomicUsize::new(0), - }); - let runtime = SwitchyardRuntime { - max_retries: 0, - algorithm: Arc::new(Passthrough::new(LlmTarget { - semantic_name: "selected".into(), - })), - targets: BTreeMap::from([ - ( - "selected".into(), - PreparedTargetBinding { - client: selected.clone(), - }, - ), - ( - "fallback".into(), - PreparedTargetBinding { - client: fallback.clone(), - }, - ), - ]), - default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, - translation: TranslationEngine::default(), - }; - let (output, messages) = async_channel::bounded(32); - - let error = runtime - .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) - .await - .expect_err("the failing fallback call must fail the request"); - - assert_eq!(error, "trusted fallback: provider transport failure"); - assert_eq!(selected.calls.load(Ordering::Relaxed), 1); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); - let mut terminal_marks = Vec::new(); - while let Ok(message) = messages.try_recv() { - if let StreamMessage::Mark(mark) = message - && matches!( - mark.name.as_str(), - "switchyard.routing.error" | "switchyard.routing.fallback" - ) - { - terminal_marks.push(mark.name); - } - } - assert_eq!( - terminal_marks, - ["switchyard.routing.error", "switchyard.routing.fallback"] - ); - } - - #[test] - fn retry_backoff_increases_exponentially_and_is_capped() { - assert_eq!(retry_backoff(1), Duration::from_millis(250)); - assert_eq!(retry_backoff(2), Duration::from_millis(500)); - assert_eq!(retry_backoff(3), Duration::from_secs(1)); - assert_eq!(retry_backoff(4), Duration::from_secs(2)); - assert_eq!(retry_backoff(u32::MAX), Duration::from_secs(2)); - } - - #[tokio::test] - async fn capability_classifier_emits_judge_usage_without_serving_usage() { - let weak = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak answer"), - calls: AtomicUsize::new(0), - }); - let strong = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong answer"), - calls: AtomicUsize::new(0), - }); - let judge = Arc::new(FixedClient { - behavior: FixedBehavior::Text( - r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, - ), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { - judge_target: fixed_target("judge", judge.clone()), - efficient_target: fixed_target("weak", weak.clone()), - capable_target: fixed_target("strong", strong.clone()), - config: TaskClassifierConfig { - base_threshold: 0.5, - ..TaskClassifierConfig::default() - }, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback, - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("capability")), - &mut marks, - ) - .await - .unwrap(); - - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); - } - - #[tokio::test] - async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { - let weak = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak draft"), - calls: AtomicUsize::new(0), - }); - let strong = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong answer"), - calls: AtomicUsize::new(0), - }); - let judge = Arc::new(FixedClient { - behavior: FixedBehavior::Text(r#"{"escalate":true,"reason":"stuck"}"#), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge", judge.clone()), - efficient_target: fixed_target("weak", weak.clone()), - capable_target: fixed_target("strong", strong.clone()), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 1, - ..EscalationJudgeConfig::default() - }, - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - - let mut first = request_with_session(WireFormat::OpenAiChat, Some("session-1")); - first.llm_request.stream = true; - let (output, messages) = async_channel::bounded(32); - runtime - .execute_stream(WireFormat::OpenAiChat, first, &output) - .await - .unwrap(); - let mut streamed = Vec::new(); - let mut routing_calls = Vec::new(); - while let Ok(message) = messages.try_recv() { - match message { - StreamMessage::Event(event) => streamed.push(event), - StreamMessage::Mark(mark) if mark.name == "switchyard.routing.llm_call" => { - routing_calls.push(mark.data) - } - StreamMessage::Mark(_) => {} - } - } - assert!(!streamed.is_empty()); - assert!( - streamed - .iter() - .any(|event| event.to_string().contains("strong answer")) - ); - assert_eq!(routing_calls.len(), 2); - assert_eq!(routing_calls[0]["selected_target"], "weak"); - assert_eq!(routing_calls[0]["call_role"], "candidate"); - assert_eq!(routing_calls[0]["usage"]["total_tokens"], 18); - assert_eq!(routing_calls[1]["selected_target"], "judge"); - assert_eq!(routing_calls[1]["call_role"], "judge"); - assert_eq!(routing_calls[1]["usage"]["total_tokens"], 18); - assert!( - routing_calls - .iter() - .all(|call| call["selected_target"] != "strong") - ); - - let mut marks = Vec::new(); - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("session-1")), - &mut marks, - ) - .await - .unwrap(); - assert!(response.to_string().contains("strong answer")); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 2); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - assert!( - !marks - .iter() - .any(|mark| mark.name == "switchyard.routing.llm_call") - ); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["selected_target"] == "strong" - && mark.data["routing_tier"] == "strong" - && mark.metadata["session_id"] == "session-1" - })); - } - - #[tokio::test] - async fn escalation_judge_failure_falls_open_to_the_buffered_weak_response() { - let weak = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak answer"), - calls: AtomicUsize::new(0), - }); - let strong = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong answer"), - calls: AtomicUsize::new(0), - }); - let judge = Arc::new(FixedClient { - behavior: FixedBehavior::TransportFailure, - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge", judge.clone()), - efficient_target: fixed_target("weak", weak.clone()), - capable_target: fixed_target("strong", strong.clone()), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig::default(), - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("session-1")), - &mut marks, - ) - .await - .unwrap(); - - assert!(response.to_string().contains("weak answer")); - assert_eq!(weak.calls.load(Ordering::Relaxed), 1); - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["call_role"], "judge"); - assert_eq!(routing_calls[0].data["outcome"], "error"); - assert!(routing_calls[0].data["usage"].is_null()); - } - - #[tokio::test] - async fn escalation_without_session_identity_cannot_accumulate_confirmations() { - let weak = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak answer"), - calls: AtomicUsize::new(0), - }); - let strong = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong answer"), - calls: AtomicUsize::new(0), - }); - let judge = Arc::new(FixedClient { - behavior: FixedBehavior::Text(r#"{"escalate":true,"reason":"stuck"}"#), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { - judge_target: fixed_target("judge", judge.clone()), - efficient_target: fixed_target("weak", weak.clone()), - capable_target: fixed_target("strong", strong.clone()), - contract: ClassifierContractConfig::default(), - config: EscalationJudgeConfig { - confirmations: 2, - ..EscalationJudgeConfig::default() - }, - max_output_tokens: 128, - }) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("weak", weak.clone()), - ("strong", strong.clone()), - ("judge", judge.clone()), - ], - ); - - for _ in 0..2 { - let mut marks = Vec::new(); - let response = runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, None), - &mut marks, - ) - .await - .unwrap(); - assert!(response.to_string().contains("weak answer")); - } - assert_eq!(weak.calls.load(Ordering::Relaxed), 2); - assert_eq!(judge.calls.load(Ordering::Relaxed), 2); - assert_eq!(strong.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - } - - #[tokio::test] - async fn stage_router_uses_tool_signals_for_every_managed_protocol() { - for protocol in [ - WireFormat::OpenAiChat, - WireFormat::OpenAiResponses, - WireFormat::AnthropicMessages, - ] { - let capable = Arc::new(FixedClient { - behavior: FixedBehavior::Text("capable answer"), - calls: AtomicUsize::new(0), - }); - let efficient = Arc::new(FixedClient { - behavior: FixedBehavior::Text("efficient answer"), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = StageRouter::new( - fixed_target("strong", capable.clone()), - fixed_target("weak", efficient.clone()), - StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), - ) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - protocol, - vec![("strong", capable.clone()), ("weak", efficient.clone())], - ); - let mut marks = Vec::new(); - - let response = runtime - .execute_buffered(protocol, stage_signal_request(protocol), &mut marks) - .await - .unwrap(); - - assert!(response.to_string().contains("capable answer")); - assert_eq!(capable.calls.load(Ordering::Relaxed), 1); - assert_eq!(efficient.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["algorithm"] == "stage_router" - && mark.data["selected_target"] == "strong" - && mark.data["routing_tier"] == "strong" - && mark.data["decision_source"] == "override" - && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) - })); - } - } - - #[tokio::test] - async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { - for (picker, expected) in [ - (PickerMode::CapableFirst, "strong"), - (PickerMode::EfficientFirst, "weak"), - ] { - let capable = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong"), - calls: AtomicUsize::new(0), - }); - let efficient = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak"), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let algorithm = StageRouter::new( - fixed_target("strong", capable.clone()), - fixed_target("weak", efficient.clone()), - StageRouterConfig::new(picker, 0.5), - ) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback, - WireFormat::OpenAiChat, - vec![("strong", capable), ("weak", efficient)], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, None), - &mut marks, - ) - .await - .unwrap(); - - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["selected_target"] == expected - && mark.data["routing_tier"] == expected - && mark.data["decision_source"] == "fall_open" - })); - } - } - - #[tokio::test] - async fn stage_router_classifier_resolves_an_ambiguous_turn() { - let capable = Arc::new(FixedClient { - behavior: FixedBehavior::Text("strong"), - calls: AtomicUsize::new(0), - }); - let efficient = Arc::new(FixedClient { - behavior: FixedBehavior::Text("weak"), - calls: AtomicUsize::new(0), - }); - let judge = Arc::new(FixedClient { - behavior: FixedBehavior::Text( - r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, - ), - calls: AtomicUsize::new(0), - }); - let fallback = Arc::new(FixedClient { - behavior: FixedBehavior::Text("fallback"), - calls: AtomicUsize::new(0), - }); - let mut config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5); - config.llm_fallback = Some(LlmFallback { - judge_target: fixed_target("judge", judge.clone()), - config: TaskClassifierConfig { - base_threshold: 0.5, - ..TaskClassifierConfig::default() - }, - }); - let algorithm = StageRouter::new( - fixed_target("strong", capable.clone()), - fixed_target("weak", efficient.clone()), - config, - ) - .unwrap(); - let runtime = runtime_with_algorithm_clients( - Arc::new(algorithm), - fallback.clone(), - WireFormat::OpenAiChat, - vec![ - ("strong", capable.clone()), - ("weak", efficient.clone()), - ("judge", judge.clone()), - ], - ); - let mut marks = Vec::new(); - - runtime - .execute_buffered( - WireFormat::OpenAiChat, - request_with_session(WireFormat::OpenAiChat, Some("stage-classifier")), - &mut marks, - ) - .await - .unwrap(); - - assert_eq!(judge.calls.load(Ordering::Relaxed), 1); - assert_eq!(efficient.calls.load(Ordering::Relaxed), 1); - assert_eq!(capable.calls.load(Ordering::Relaxed), 0); - assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); - let routing_calls = marks - .iter() - .filter(|mark| mark.name == "switchyard.routing.llm_call") - .collect::>(); - assert_eq!(routing_calls.len(), 1); - assert_eq!(routing_calls[0].data["selected_target"], "judge"); - assert_eq!(routing_calls[0].data["call_role"], "judge"); - assert_eq!(routing_calls[0].data["outcome"], "ok"); - assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); - assert!(marks.iter().any(|mark| { - mark.name == "switchyard.routing.decision" - && mark.data["selected_target"] == "weak" - && mark.data["routing_tier"] == "weak" - && mark.data["decision_source"] == "llm-classifier" - })); - } - - #[test] - fn context_carries_identity_without_http_headers() { - let context = context_from_metadata(Some(&Metadata { - session_id: Some("session-1".into()), - agent_id: Some("agent-1".into()), - is_subagent: true, - extra_metadata: Some(BTreeMap::from([("tenant".into(), "blue".into())])), - http_headers: Some(http::HeaderMap::from_iter([( - http::HeaderName::from_static("authorization"), - http::HeaderValue::from_static("Bearer caller-secret"), - )])), - ..Metadata::default() - })); - - assert_eq!( - context.values.get("session_id").map(String::as_str), - Some("session-1") - ); - assert_eq!( - context.values.get("agent_id").map(String::as_str), - Some("agent-1") - ); - assert_eq!( - context.values.get("is_subagent").map(String::as_str), - Some("true") - ); - assert_eq!( - context.values.get("tenant").map(String::as_str), - Some("blue") - ); - assert!(!context.values.contains_key("authorization")); - } -} +mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs new file mode 100644 index 000000000..1145f03fc --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs @@ -0,0 +1,920 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::sync::atomic::{AtomicUsize, Ordering}; + +use switchyard_libsy::{ + ClassifierContractConfig, EscalationJudgeConfig, LlmClassifierConfig, LlmFallback, LlmTarget, + LlmTaskClassifier, Passthrough, PickerMode, StageRouter, StageRouterConfig, + TaskClassifierConfig, +}; +use switchyard_protocol::{LlmResponseStream, RoutedLlmClient, Usage, text_request, text_response}; + +use super::*; + +enum ScriptedBehavior { + Text(&'static str), + EmptyBuffered, + EmptyStream, + FailingStream, + TransportFailure(&'static str), +} + +struct ScriptedClient { + behavior: ScriptedBehavior, + calls: AtomicUsize, +} + +fn scripted(behavior: ScriptedBehavior) -> Arc { + Arc::new(ScriptedClient { + behavior, + calls: AtomicUsize::new(0), + }) +} + +#[async_trait::async_trait] +impl RoutedLlmClient for ScriptedClient { + async fn call( + &self, + _ctx: Context, + request: Request, + _decision: Arc, + ) -> Result { + self.calls.fetch_add(1, Ordering::Relaxed); + match self.behavior { + ScriptedBehavior::Text(text) => { + let mut response = text_response(None, text); + response.usage = Usage { + input_tokens: Some(11), + output_tokens: Some(7), + total_tokens: Some(18), + ..Usage::default() + }; + Ok(Response { + llm_response: LlmResponse::Agg(response), + metadata: request.metadata, + }) + } + ScriptedBehavior::EmptyBuffered => Ok(Response { + llm_response: LlmResponse::Agg(Default::default()), + metadata: None, + }), + ScriptedBehavior::EmptyStream => Ok(Response { + llm_response: LlmResponse::Stream(Box::pin(stream::empty())), + metadata: None, + }), + ScriptedBehavior::FailingStream => { + let stream: LlmResponseStream = Box::pin(stream::once(async { + Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other("fallback stream failed")), + }) + })); + Ok(Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }) + } + ScriptedBehavior::TransportFailure(message) => Err(LlmClientError::Transport { + source: Box::new(std::io::Error::other(message)), + }), + } + } +} + +fn fixed_target(name: &str) -> LlmTarget { + LlmTarget { + semantic_name: name.to_string(), + } +} + +fn runtime_with_algorithm( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, +) -> SwitchyardRuntime { + runtime_with_algorithm_clients(algorithm, fallback, protocol, Vec::new()) +} + +fn runtime_with_algorithm_clients( + algorithm: Arc, + fallback: Arc, + protocol: WireFormat, + clients: Vec<(&str, Arc)>, +) -> SwitchyardRuntime { + let is_stage = algorithm.name() == "stage_router"; + let mut targets = BTreeMap::from([( + "fallback".into(), + PreparedTargetBinding { + client: fallback as Arc, + }, + )]); + for (name, client) in clients { + targets.insert( + name.to_string(), + PreparedTargetBinding { + client: client as Arc, + }, + ); + } + SwitchyardRuntime { + max_retries: 0, + algorithm, + targets, + default_targets: BTreeMap::from([(protocol, "fallback".into())]), + target_tiers: BTreeMap::from([("weak".into(), "weak"), ("strong".into(), "strong")]), + stage_marks: is_stage.then_some(StageMarkConfig { + picker: PickerMode::CapableFirst, + confidence_threshold: 0.5, + recent_turn_window: None, + classifier_enabled: true, + }), + translation: TranslationEngine::default(), + } +} + +fn request_with_session(protocol: WireFormat, session: Option<&str>) -> Request { + Request { + llm_request: text_request(Some("auto".into()), "fix the build"), + raw_request: None, + metadata: Some(Metadata { + wire_format: Some(protocol), + session_id: session.map(str::to_string), + ..Metadata::default() + }), + } +} + +fn stage_signal_relay_request(protocol: WireFormat) -> RelayRequest { + let content = match protocol { + WireFormat::OpenAiChat => json!({ + "model": "auto", + "messages": [ + {"role": "user", "content": "fix the build"}, + { + "role": "assistant", + "content": null, + "tool_calls": [{ + "id": "call-1", + "type": "function", + "function": { + "name": "bash", + "arguments": "{\"cmd\":\"cargo test\"}" + } + }] + }, + { + "role": "tool", + "tool_call_id": "call-1", + "content": "fatal runtime error: out of memory" + } + ] + }), + WireFormat::OpenAiResponses => json!({ + "model": "auto", + "input": [ + {"type": "message", "role": "user", "content": "fix the build"}, + { + "type": "function_call", + "call_id": "call-1", + "name": "bash", + "arguments": "{\"cmd\":\"cargo test\"}" + }, + { + "type": "function_call_output", + "call_id": "call-1", + "output": "fatal runtime error: out of memory" + } + ] + }), + WireFormat::AnthropicMessages => json!({ + "model": "auto", + "max_tokens": 128, + "messages": [ + {"role": "user", "content": "fix the build"}, + { + "role": "assistant", + "content": [{ + "type": "tool_use", + "id": "call-1", + "name": "bash", + "input": {"cmd": "cargo test"} + }] + }, + { + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "call-1", + "content": "fatal runtime error: out of memory", + "is_error": true + }] + } + ] + }), + }; + + RelayRequest { + headers: Map::from_iter([( + "x-switchyard-session-id".into(), + json!(format!("stage-{}", protocol.as_str())), + )]), + content, + } +} + +#[test] +fn relay_gateway_placeholder_session_is_not_retained() { + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ("x-dynamo-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!(decoded.metadata.unwrap().session_id, None); +} + +#[test] +fn explicit_switchyard_session_overrides_relay_gateway_placeholder() { + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let runtime = runtime_with_algorithm( + Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + fallback, + WireFormat::OpenAiChat, + ); + let request = RelayRequest { + headers: Map::from_iter([ + ("x-switchyard-session-id".into(), json!("caller-session")), + ("x-nemo-relay-source".into(), json!("gateway")), + ("x-nemo-relay-session-id".into(), json!("gateway-gateway")), + ]), + content: json!({ + "model": "router", + "messages": [{"role": "user", "content": "hello"}] + }), + }; + + let decoded = runtime + .decode_request(WireFormat::OpenAiChat, &request, false) + .unwrap(); + + assert_eq!( + decoded.metadata.unwrap().session_id.as_deref(), + Some("caller-session") + ); +} + +#[tokio::test] +async fn buffered_finalization_failure_uses_fallback_once() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::EmptyBuffered); + let runtime = SwitchyardRuntime { + max_retries: 1, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered(WireFormat::OpenAiChat, Request::default(), &mut marks) + .await + .expect("the buffered fallback response should be encoded"); + + assert!(response.is_object()); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.retry") + ); + let error = marks + .iter() + .find(|mark| mark.name == "switchyard.routing.error") + .expect("finalization failure should emit an error mark"); + assert_eq!(error.data["retryable"], false); + assert_eq!(error.data["non_http_kind"], "invalid_response"); + assert_eq!( + marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.fallback") + .count(), + 1 + ); +} + +#[tokio::test] +async fn returned_events_replays_preserved_openai_chat_without_duplicate_terminal() { + let content = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "system_fingerprint": "fp_provider_specific", + "choices": [{ + "index": 0, + "delta": {"content": "Hi"}, + "finish_reason": null + }] + }); + let terminal = json!({ + "id": "chatcmpl-test", + "object": "chat.completion.chunk", + "model": "gpt-4o", + "choices": [{ + "index": 0, + "delta": {}, + "finish_reason": "stop" + }] + }); + let body = format!("data: {content}\n\ndata: {terminal}\n\ndata: [DONE]\n\n").into_bytes(); + let stream = switchyard_translation::decode_stream( + stream::once(async move { Ok::<_, LlmClientError>(body) }), + WireFormat::OpenAiChat, + ) + .expect("provider SSE should decode"); + let response = Response { + llm_response: LlmResponse::Stream(stream), + metadata: None, + }; + + let replayed = returned_events(response, WireFormat::OpenAiChat) + .await + .expect("return stream should encode") + .collect::>() + .await + .into_iter() + .collect::, _>>() + .expect("return stream should not fail"); + + assert_eq!(replayed, vec![content, terminal]); +} + +#[tokio::test] +async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::FailingStream); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let (output, _messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback stream must fail the request"); + + assert_eq!(error, "trusted fallback stream: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); +} + +#[tokio::test] +async fn failing_fallback_call_flushes_error_and_fallback_marks() { + let selected = scripted(ScriptedBehavior::EmptyStream); + let fallback = scripted(ScriptedBehavior::TransportFailure("fallback call failed")); + let runtime = SwitchyardRuntime { + max_retries: 0, + algorithm: Arc::new(Passthrough::new(LlmTarget { + semantic_name: "selected".into(), + })), + targets: BTreeMap::from([ + ( + "selected".into(), + PreparedTargetBinding { + client: selected.clone(), + }, + ), + ( + "fallback".into(), + PreparedTargetBinding { + client: fallback.clone(), + }, + ), + ]), + default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), + target_tiers: BTreeMap::new(), + stage_marks: None, + translation: TranslationEngine::default(), + }; + let (output, messages) = async_channel::bounded(32); + + let error = runtime + .execute_stream(WireFormat::OpenAiChat, Request::default(), &output) + .await + .expect_err("the failing fallback call must fail the request"); + + assert_eq!(error, "trusted fallback: provider transport failure"); + assert_eq!(selected.calls.load(Ordering::Relaxed), 1); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 1); + let mut terminal_marks = Vec::new(); + while let Ok(message) = messages.try_recv() { + if let StreamMessage::Mark(mark) = message + && matches!( + mark.name.as_str(), + "switchyard.routing.error" | "switchyard.routing.fallback" + ) + { + terminal_marks.push(mark.name); + } + } + assert_eq!( + terminal_marks, + ["switchyard.routing.error", "switchyard.routing.fallback"] + ); +} + +#[test] +fn retry_backoff_increases_exponentially_and_is_capped() { + assert_eq!(retry_backoff(1), Duration::from_millis(250)); + assert_eq!(retry_backoff(2), Duration::from_millis(500)); + assert_eq!(retry_backoff(3), Duration::from_secs(1)); + assert_eq!(retry_backoff(4), Duration::from_secs(2)); + assert_eq!(retry_backoff(u32::MAX), Duration::from_secs(2)); +} + +#[tokio::test] +async fn capability_classifier_emits_judge_usage_without_serving_usage() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Capability { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("capability")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); +} + +#[tokio::test] +async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { + let weak = scripted(ScriptedBehavior::Text("weak draft")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"escalate":true,"reason":"stuck"}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 1, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + let mut first = request_with_session(WireFormat::OpenAiChat, Some("session-1")); + first.llm_request.stream = true; + let (output, messages) = async_channel::bounded(32); + runtime + .execute_stream(WireFormat::OpenAiChat, first, &output) + .await + .unwrap(); + let mut streamed = Vec::new(); + let mut routing_calls = Vec::new(); + while let Ok(message) = messages.try_recv() { + match message { + StreamMessage::Event(event) => streamed.push(event), + StreamMessage::Mark(mark) if mark.name == "switchyard.routing.llm_call" => { + routing_calls.push(mark.data) + } + StreamMessage::Mark(_) => {} + } + } + assert!(!streamed.is_empty()); + assert!( + streamed + .iter() + .any(|event| event.to_string().contains("strong answer")) + ); + assert_eq!(routing_calls.len(), 2); + assert_eq!(routing_calls[0]["selected_target"], "weak"); + assert_eq!(routing_calls[0]["call_role"], "candidate"); + assert_eq!(routing_calls[0]["usage"]["total_tokens"], 18); + assert_eq!(routing_calls[1]["selected_target"], "judge"); + assert_eq!(routing_calls[1]["call_role"], "judge"); + assert_eq!(routing_calls[1]["usage"]["total_tokens"], 18); + assert!( + routing_calls + .iter() + .all(|call| call["selected_target"] != "strong") + ); + + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("strong answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 2); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!( + !marks + .iter() + .any(|mark| mark.name == "switchyard.routing.llm_call") + ); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == "strong" + && mark.data["routing_tier"] == "strong" + && mark.metadata["session_id"] == "session-1" + })); +} + +#[tokio::test] +async fn escalation_judge_failure_falls_open_to_the_buffered_weak_response() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::TransportFailure("scripted failure")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig::default(), + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("session-1")), + &mut marks, + ) + .await + .unwrap(); + + assert!(response.to_string().contains("weak answer")); + assert_eq!(weak.calls.load(Ordering::Relaxed), 1); + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "judge"); + assert_eq!(routing_calls[0].data["outcome"], "error"); + assert!(routing_calls[0].data["usage"].is_null()); +} + +#[tokio::test] +async fn escalation_without_session_identity_cannot_accumulate_confirmations() { + let weak = scripted(ScriptedBehavior::Text("weak answer")); + let strong = scripted(ScriptedBehavior::Text("strong answer")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"escalate":true,"reason":"stuck"}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = LlmTaskClassifier::new(LlmClassifierConfig::Escalation { + judge_target: fixed_target("judge"), + efficient_target: fixed_target("weak"), + capable_target: fixed_target("strong"), + contract: ClassifierContractConfig::default(), + config: EscalationJudgeConfig { + confirmations: 2, + ..EscalationJudgeConfig::default() + }, + max_output_tokens: 128, + }) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("weak", weak.clone()), + ("strong", strong.clone()), + ("judge", judge.clone()), + ], + ); + + for _ in 0..2 { + let mut marks = Vec::new(); + let response = runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + assert!(response.to_string().contains("weak answer")); + } + assert_eq!(weak.calls.load(Ordering::Relaxed), 2); + assert_eq!(judge.calls.load(Ordering::Relaxed), 2); + assert_eq!(strong.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); +} + +#[tokio::test] +async fn stage_router_uses_tool_signals_for_every_managed_protocol() { + for protocol in [ + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + WireFormat::AnthropicMessages, + ] { + let capable = scripted(ScriptedBehavior::Text("capable answer")); + let efficient = scripted(ScriptedBehavior::Text("efficient answer")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = StageRouter::new( + fixed_target("strong"), + fixed_target("weak"), + StageRouterConfig::new(PickerMode::EfficientFirst, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + protocol, + vec![("strong", capable.clone()), ("weak", efficient.clone())], + ); + let mut marks = Vec::new(); + let relay_request = stage_signal_relay_request(protocol); + let request = runtime + .decode_request(protocol, &relay_request, false) + .unwrap(); + + let response = runtime + .execute_buffered(protocol, request, &mut marks) + .await + .unwrap(); + + assert!(response.to_string().contains("capable answer")); + assert_eq!(capable.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["algorithm"] == "stage_router" + && mark.data["selected_target"] == "strong" + && mark.data["routing_tier"] == "strong" + && mark.data["decision_source"] == "override" + && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) + })); + } +} + +#[tokio::test] +async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { + for (picker, expected) in [ + (PickerMode::CapableFirst, "strong"), + (PickerMode::EfficientFirst, "weak"), + ] { + let capable = scripted(ScriptedBehavior::Text("strong")); + let efficient = scripted(ScriptedBehavior::Text("weak")); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let algorithm = StageRouter::new( + fixed_target("strong"), + fixed_target("weak"), + StageRouterConfig::new(picker, 0.5), + ) + .unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback, + WireFormat::OpenAiChat, + vec![("strong", capable), ("weak", efficient)], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, None), + &mut marks, + ) + .await + .unwrap(); + + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == expected + && mark.data["routing_tier"] == expected + && mark.data["decision_source"] == "fall_open" + })); + } +} + +#[tokio::test] +async fn stage_router_classifier_resolves_an_ambiguous_turn() { + let capable = scripted(ScriptedBehavior::Text("strong")); + let efficient = scripted(ScriptedBehavior::Text("weak")); + let judge = scripted(ScriptedBehavior::Text( + r#"{"crux":"bounded","primary_rule":"SUP-1","capability_boundary":"supported","p_solve":0.9}"#, + )); + let fallback = scripted(ScriptedBehavior::Text("fallback")); + let mut config = StageRouterConfig::new(PickerMode::CapableFirst, 0.5); + config.llm_fallback = Some(LlmFallback { + judge_target: fixed_target("judge"), + config: TaskClassifierConfig { + base_threshold: 0.5, + ..TaskClassifierConfig::default() + }, + }); + let algorithm = StageRouter::new(fixed_target("strong"), fixed_target("weak"), config).unwrap(); + let runtime = runtime_with_algorithm_clients( + Arc::new(algorithm), + fallback.clone(), + WireFormat::OpenAiChat, + vec![ + ("strong", capable.clone()), + ("weak", efficient.clone()), + ("judge", judge.clone()), + ], + ); + let mut marks = Vec::new(); + + runtime + .execute_buffered( + WireFormat::OpenAiChat, + request_with_session(WireFormat::OpenAiChat, Some("stage-classifier")), + &mut marks, + ) + .await + .unwrap(); + + assert_eq!(judge.calls.load(Ordering::Relaxed), 1); + assert_eq!(efficient.calls.load(Ordering::Relaxed), 1); + assert_eq!(capable.calls.load(Ordering::Relaxed), 0); + assert_eq!(fallback.calls.load(Ordering::Relaxed), 0); + let routing_calls = marks + .iter() + .filter(|mark| mark.name == "switchyard.routing.llm_call") + .collect::>(); + assert_eq!(routing_calls.len(), 1); + assert_eq!(routing_calls[0].data["selected_target"], "judge"); + assert_eq!(routing_calls[0].data["call_role"], "judge"); + assert_eq!(routing_calls[0].data["outcome"], "ok"); + assert_eq!(routing_calls[0].data["usage"]["total_tokens"], 18); + assert!(marks.iter().any(|mark| { + mark.name == "switchyard.routing.decision" + && mark.data["selected_target"] == "weak" + && mark.data["routing_tier"] == "weak" + && mark.data["decision_source"] == "llm-classifier" + })); +} + +#[test] +fn context_carries_identity_without_http_headers() { + let context = context_from_metadata(Some(&Metadata { + session_id: Some("session-1".into()), + agent_id: Some("agent-1".into()), + is_subagent: true, + extra_metadata: Some(BTreeMap::from([("tenant".into(), "blue".into())])), + http_headers: Some(http::HeaderMap::from_iter([( + http::HeaderName::from_static("authorization"), + http::HeaderValue::from_static("Bearer caller-secret"), + )])), + ..Metadata::default() + })); + + assert_eq!( + context.values.get("session_id").map(String::as_str), + Some("session-1") + ); + assert_eq!( + context.values.get("agent_id").map(String::as_str), + Some("agent-1") + ); + assert_eq!( + context.values.get("is_subagent").map(String::as_str), + Some("true") + ); + assert_eq!( + context.values.get("tenant").map(String::as_str), + Some("blue") + ); + assert!(!context.values.contains_key("authorization")); +} From 047624a166cdf7d2cac0085ee1c6b76031e642d0 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 13:27:36 -0600 Subject: [PATCH 4/9] refactor(relay): emit libsy decisions directly Signed-off-by: Bryan Bednarski --- CHANGELOG.md | 3 +- crates/libsy/src/algorithms/stage.rs | 123 +----------------- crates/libsy/src/algorithms/util/stage.rs | 48 ++----- crates/libsy/src/lib.rs | 5 +- crates/switchyard-nemo-relay-plugin/README.md | 12 +- .../src/config.rs | 46 ------- .../src/runtime.rs | 65 +-------- .../src/runtime/tests.rs | 30 ++--- .../stage_router_routing.md | 13 -- 9 files changed, 36 insertions(+), 309 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0cdd90812..adfd24fa5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,8 +21,7 @@ adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). judges and discarded weak candidates, and failed routing candidates now emit `switchyard.routing.llm_call` ATOF marks with normalized token usage and latency. The final serving call remains represented only by Relay's outer LLM - lifecycle event to prevent double-counting. Stage decision marks retain - picker-default tiers, decision sources, and hard-override confidence. + lifecycle event to prevent double-counting. ### Removed diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 5091d1d3f..592547199 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -20,13 +20,9 @@ use async_trait::async_trait; use super::fall_through::{DefaultTarget, FallThrough}; use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig}; use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; -use super::util::stage::{ - DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, - record_decision_source, -}; +use super::util::stage::{HandoffNoteConfig, PickerMode, StageClassifier, StageTargets}; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet}; -use crate::core::classifier::{Classification, Classifier}; use crate::core::state::State; use crate::{LibsyError, Result}; use switchyard_protocol::{Context, Request, Response}; @@ -34,36 +30,6 @@ use switchyard_protocol::{Context, Request, Response}; /// Telemetry name for a router this module assembles. const STAGE_ROUTER: &str = "stage_router"; -/// Attributes a turn to the classifier it wraps, when that classifier decides it. -/// -/// The classifiers themselves are composition-agnostic and write no state; only -/// this router knows where each sits in its cascade. -struct SourceStamp { - inner: Arc>, - source: DecisionSource, -} - -#[async_trait] -impl Classifier for SourceStamp { - fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { - self.inner.routing_tier(selected_model_id) - } - - async fn score( - &self, - state: &mut State, - request: &mut Request, - driver: Option<&Driver>, - ) -> Result<(Classification, Option)> { - let (classification, served) = self.inner.score(state, request, driver).await?; - // An abstaining classifier passes the turn on, so it is not its to claim. - if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) { - record_decision_source(state, self.source); - } - Ok((classification, served)) - } -} - /// The capability judge a stage router falls through to. pub struct LlmFallback { /// Target the judge model is called through. It is not a routing @@ -192,22 +158,18 @@ fn build_route( if let Some(fallback) = config.llm_fallback { // The capability judge takes its tiers in the same order the capability // route passes them: efficient first, capable second. - router = router.with_classifier(Arc::new(SourceStamp { - inner: Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { + router = router.with_classifier(Arc::new(LlmTaskClassifier::new( + LlmClassifierConfig::Capability { judge_target: fallback.judge_target, efficient_target: efficient, capable_target: capable, config: fallback.config, - })?), - source: DecisionSource::LlmClassifier, - })); + }, + )?)); } // Nothing behind this, so the turn lands on the picker's default tier — // including when the judge could not tell. - router = router.with_classifier(Arc::new(SourceStamp { - inner: Arc::new(DefaultTarget::new(fall_open)), - source: DecisionSource::FallOpen, - })); + router = router.with_classifier(Arc::new(DefaultTarget::new(fall_open))); // Runs on the post-decision hook, so it applies to the target the cascade // settled on, whichever classifier picked it. With no prompts configured it // is a no-op, so there is nothing to branch on. @@ -219,7 +181,6 @@ fn build_route( mod tests { use std::sync::Arc; - use async_trait::async_trait; use parking_lot::Mutex; use serde_json::json; use switchyard_protocol::{ @@ -227,12 +188,9 @@ mod tests { }; use super::*; - use crate::algorithms::util::stage::DECISION_SOURCE_KEY; use crate::core::algorithm::LlmTarget; - use crate::core::classifier::Score; - use crate::core::state::StateValue; use crate::core::testing::{Serve, reply, test_drive}; - use switchyard_protocol::{Context, Decision, Metadata, Response}; + use switchyard_protocol::{Context, Decision, Metadata}; fn tier_target(name: &str) -> LlmTarget { LlmTarget { @@ -240,73 +198,6 @@ mod tests { } } - /// A classifier that always picks `target`, standing in for a cascade member. - struct Fixed(&'static str); - - #[async_trait] - impl Classifier for Fixed { - async fn score( - &self, - _state: &mut State, - _request: &mut Request, - _driver: Option<&Driver>, - ) -> Result<(Classification, Option)> { - Ok(( - Classification::Scores(vec![Score { - target: self.0.to_string(), - confidence: 1.0, - }]), - None, - )) - } - } - - /// A classifier that never decides. - struct Abstains; - - #[async_trait] - impl Classifier for Abstains { - async fn score( - &self, - _state: &mut State, - _request: &mut Request, - _driver: Option<&Driver>, - ) -> Result<(Classification, Option)> { - Ok((Classification::Ambiguous(vec![]), None)) - } - } - - async fn stamped(inner: Arc>) -> Result> { - let stamp = SourceStamp { - inner, - source: DecisionSource::LlmClassifier, - }; - let mut state = State::default(); - stamp - .score(&mut state, &mut Request::default(), None) - .await?; - Ok(match state.extra.get(DECISION_SOURCE_KEY) { - Some(StateValue::String(source)) => Some(source.clone()), - _ => None, - }) - } - - #[tokio::test] - async fn a_deciding_classifier_is_credited_with_the_turn() -> Result<()> { - assert_eq!( - stamped(Arc::new(Fixed("strong"))).await?.as_deref(), - Some("llm-classifier") - ); - Ok(()) - } - - #[tokio::test] - async fn an_abstaining_classifier_claims_nothing() -> Result<()> { - // It passed the turn on, so the next classifier is the one that decided. - assert_eq!(stamped(Arc::new(Abstains)).await?, None); - Ok(()) - } - fn config() -> StageRouterConfig { StageRouterConfig::new(PickerMode::EfficientFirst, 0.5) } diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 3105c1102..1765d987d 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -25,7 +25,7 @@ use super::tool_signals::ToolSignals; use crate::Result; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier, Score}; -use crate::core::state::{State, StateValue}; +use crate::core::state::State; use switchyard_protocol::Request; /// Turn depth below which stall signals stay quiet — early no-write turns are @@ -126,23 +126,7 @@ impl PickerMode { } } -/// `State.extra` key under which the turn's [`DecisionSource`] is recorded. -pub const DECISION_SOURCE_KEY: &str = "decision_source"; - -/// Record which component decided the turn. -pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) { - state.extra.insert( - DECISION_SOURCE_KEY.to_string(), - StateValue::String(source.as_str().to_string()), - ); -} - -/// What produced a decision — for stats and explainability. -/// -/// Each is stamped by the component that knows it, so a turn's final label names -/// whoever actually decided it. [`Ambiguous`](Self::Ambiguous) is the exception: -/// the signal scorer records it on its way out, and whichever classifier resolves -/// the turn overwrites it. +/// What produced a signal-resolved picker outcome. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum DecisionSource { /// Hard override (critical severity or context compaction). @@ -160,7 +144,7 @@ pub enum DecisionSource { } impl DecisionSource { - /// Stable lowercase label used in stats. + /// Stable lowercase label for picker consumers. pub fn as_str(self) -> &'static str { match self { Self::Override => "override", @@ -459,8 +443,7 @@ impl StageClassifier { /// The signals could not decide, so this turn belongs to whatever the cascade /// has behind this classifier. - fn abstain(state: &mut State) -> Classification { - record_decision_source(state, DecisionSource::Ambiguous); + fn abstain() -> Classification { Classification::Ambiguous(Vec::new()) } @@ -490,7 +473,7 @@ impl Classifier for StageClassifier { let Some(signal) = tool_signals else { // No tool activity yet — nothing to score, so the signals have no // opinion, same as a below-threshold turn. - return Ok((Self::abstain(state), None)); + return Ok((Self::abstain(), None)); }; let outcome = pick_tier(signal, self.mode, self.confidence_threshold); @@ -502,7 +485,6 @@ impl Classifier for StageClassifier { .. } => { let target = self.targets.name(tier); - record_decision_source(state, source); // Only a resolved turn routes on this classifier's target, so it // is the only branch whose tier the signals actually chose — an // ambiguous turn is decided further down the cascade. @@ -516,7 +498,7 @@ impl Classifier for StageClassifier { None, )) } - PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(state), None)), + PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(), None)), } } } @@ -621,10 +603,6 @@ mod tests { .score(&mut state, &mut Request::default(), None) .await?; assert!(classification.0.argmax(false)?.is_none()); - assert!(matches!( - state.extra.get(DECISION_SOURCE_KEY), - Some(StateValue::String(source)) if source == "ambiguous" - )); Ok(()) } @@ -646,11 +624,6 @@ mod tests { } _ => panic!("expected a definite classification"), } - // The decision source travels downstream (for handoff-note gating). - assert!(matches!( - state.extra.get(DECISION_SOURCE_KEY), - Some(StateValue::String(source)) if source == "override" - )); Ok(()) } @@ -679,18 +652,13 @@ mod tests { } #[tokio::test] - async fn classifier_falls_open_to_default_and_records_it() -> Result<()> { - // A quiet signal corroborates neither axis, so the scorer abstains and - // records why. + async fn classifier_abstains_on_an_ambiguous_signal() -> Result<()> { + // A quiet signal corroborates neither axis, so the scorer abstains. let mut state = state_with(ToolSignals::default()); let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; assert!(classification.0.argmax(false)?.is_none()); - assert!(matches!( - state.extra.get(DECISION_SOURCE_KEY), - Some(StateValue::String(source)) if source == "ambiguous" - )); Ok(()) } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index d40328136..15c675c7c 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -35,9 +35,8 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals}; // Stage-router scoring and tier selection — the shared signal-driven routing // core (scorer, picker, and the `StageClassifier`). pub use algorithms::util::stage::{ - CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome, - PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal, - pick_tier, score_signal, + CodingAgentDimensions, DecisionSource, HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult, + StageClassifier, StageTargets, Tier, dimensions_from_signal, pick_tier, score_signal, }; mod observability; diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 203e4b23d..93f22cdbd 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -125,7 +125,7 @@ algorithm. | Streaming responses | Supported | Supported after the judge selects a target | Conditional: an unlatched weak stream is aggregated before the judge runs | Supported after the signal cascade selects a target | | Retained routing state | No selection affinity; context-overflow eviction can use session identity | Optional session affinity and message-hash fallback | Confirmation streak and strong latch require stable session identity | No classifier affinity; context-overflow eviction can use session identity | | Router-specific prompts | Not applicable | Optional judge prompt | Optional escalation-judge prompt | Optional tier prompts, handoff notes, and classifier prompt | -| Relay decision marks | Algorithm, attempt, selected target, and identity; routing tier is `null` | Algorithm, attempt, selected target, weak or strong routing tier, and identity | Algorithm, attempt, selected target, weak or strong routing tier, and identity | Algorithm, attempt, selected target, routing tier, decision source, and identity | +| Relay decision marks | Algorithm, attempt, selected target, reasoning, answer-call status, and identity | Algorithm, attempt, selected target, reasoning, answer-call status, and identity | Algorithm, attempt, selected target, reasoning, answer-call status, and identity | Algorithm, attempt, selected target, reasoning, answer-call status, and identity | | ATOF routing-LLM usage | Not applicable unless a failed candidate is replaced | Judge calls, plus failed candidates | Judge calls and discarded weak candidates | Optional classifier judge calls, plus failed candidates | Anthropic Messages is supported for callers and serving targets, but not for a @@ -157,8 +157,8 @@ automatic compatibility fallback. Each completed routing-only model call emits a `switchyard.routing.llm_call` ATOF mark. Its data identifies the algorithm, -attempt, call order, target, routing tier, role (`judge` or discarded -`candidate`), outcome, latency, and normalized provider token `usage`. The +attempt, call order, target, role (`judge` or discarded `candidate`), outcome, +latency, and normalized provider token `usage`. The successful call that serves the caller is deliberately excluded because Relay's outer LLM end event already records that usage. A failed call, or a provider response that omits usage, has `usage = null`. Consumers can therefore @@ -357,10 +357,8 @@ classifier target has the same structured-output protocol restriction as the standalone classifier. Ambiguous turns that reach the optional classifier add one judge call; -decisive tool signals do not. Decision marks include `routing_tier` for signal, -classifier, and picker-default paths, plus `decision_source` (`override`, -`tests_passed`, `dimensions`, `llm-classifier`, or `fall_open`) for stage-router -explainability. +decisive tool signals do not. Decision marks report the selected model, +reasoning, and answer-call status exposed by libsy's `Decision` API. Version-1 service configuration, decision-only execution, and observe-only mode are rejected. diff --git a/crates/switchyard-nemo-relay-plugin/src/config.rs b/crates/switchyard-nemo-relay-plugin/src/config.rs index 5b3722763..ff3668150 100644 --- a/crates/switchyard-nemo-relay-plugin/src/config.rs +++ b/crates/switchyard-nemo-relay-plugin/src/config.rs @@ -304,16 +304,6 @@ pub(crate) struct PreparedConfig { pub(crate) algorithm: Arc, pub(crate) targets: BTreeMap, pub(crate) default_targets: BTreeMap, - pub(crate) target_tiers: BTreeMap, - pub(crate) stage_marks: Option, -} - -#[derive(Clone)] -pub(crate) struct StageMarkConfig { - pub(crate) picker: PickerMode, - pub(crate) confidence_threshold: f64, - pub(crate) recent_turn_window: Option, - pub(crate) classifier_enabled: bool, } impl SwitchyardConfig { @@ -361,7 +351,6 @@ impl SwitchyardConfig { pub(crate) fn prepare(self) -> Result { self.validate_structure()?; - let (target_tiers, stage_marks) = self.routing_mark_config(); let targets = self .targets .iter() @@ -373,44 +362,9 @@ impl SwitchyardConfig { algorithm, targets, default_targets: self.default_targets, - target_tiers, - stage_marks, }) } - fn routing_mark_config(&self) -> (BTreeMap, Option) { - match &self.algorithm { - AlgorithmConfig::Random { .. } => (BTreeMap::new(), None), - AlgorithmConfig::LlmClassifier { config } => ( - BTreeMap::from([ - (config.weak_target.clone(), "weak"), - (config.strong_target.clone(), "strong"), - ]), - None, - ), - AlgorithmConfig::StageRouter { - capable_target, - efficient_target, - picker, - confidence_threshold, - recent_turn_window, - classifier, - .. - } => ( - BTreeMap::from([ - (capable_target.clone(), "strong"), - (efficient_target.clone(), "weak"), - ]), - Some(StageMarkConfig { - picker: *picker, - confidence_threshold: *confidence_threshold, - recent_turn_window: *recent_turn_window, - classifier_enabled: classifier.is_some(), - }), - ), - } - } - fn build_algorithm( &self, prepared: Option<&BTreeMap>, diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index 73cd3fa51..c5604613f 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -8,14 +8,14 @@ use std::time::Duration; use futures_util::{StreamExt, stream}; use nemo_relay_plugin::{Json, LlmRequest as RelayRequest}; use serde_json::{Map, json}; -use switchyard_libsy::{Algorithm, LibsyError, PickOutcome, ToolSignals, pick_tier}; +use switchyard_libsy::{Algorithm, LibsyError}; use switchyard_llm_client::{ClientRouter, LlmCallObservation, RunObservation, RunObserver, run}; use switchyard_protocol::{ Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, WireFormat, }; use switchyard_translation::{TranslationEngine, encode_stream}; -use crate::config::{PreparedTargetBinding, StageMarkConfig, SwitchyardConfig, protocol_from_call}; +use crate::config::{PreparedTargetBinding, SwitchyardConfig, protocol_from_call}; use crate::translation; const INITIAL_RETRY_BACKOFF: Duration = Duration::from_millis(250); @@ -39,8 +39,6 @@ pub(crate) struct SwitchyardRuntime { algorithm: Arc, targets: BTreeMap, default_targets: BTreeMap, - target_tiers: BTreeMap, - stage_marks: Option, translation: TranslationEngine, } @@ -52,8 +50,6 @@ impl SwitchyardRuntime { algorithm: prepared.algorithm, targets: prepared.targets, default_targets: prepared.default_targets, - target_tiers: prepared.target_tiers, - stage_marks: prepared.stage_marks, translation: TranslationEngine::default(), }) } @@ -317,7 +313,6 @@ impl SwitchyardRuntime { mark_metadata: &Json, ) -> Result { let context = context_from_metadata(request.metadata.as_ref()); - let stage_request = self.stage_marks.as_ref().map(|_| request.clone()); let observations = Arc::new(Mutex::new(Vec::new())); let observed_calls = observations.clone(); let observer: RunObserver = Arc::new(move |observation| { @@ -345,13 +340,7 @@ impl SwitchyardRuntime { { Ok((decisions, response)) => { for decision in decisions { - self.emit_decision( - marks, - decision.as_ref(), - stage_request.as_ref(), - attempt, - mark_metadata, - ); + self.emit_decision(marks, decision.as_ref(), attempt, mark_metadata); } self.emit_routing_llm_calls( marks, @@ -428,13 +417,9 @@ impl SwitchyardRuntime { &self, marks: &mut Vec, decision: &Decision, - request: Option<&Request>, attempt: u32, metadata: &Json, ) { - let decision_source = - request.and_then(|request| self.stage_decision_source(request, decision)); - let routing_tier = self.target_tiers.get(decision.selected_model_id()).copied(); self.mark( marks, "switchyard.routing.decision", @@ -443,38 +428,12 @@ impl SwitchyardRuntime { "attempt": attempt, "selected_target": decision.selected_model_id(), "reasoning": decision.reasoning(), - "routing_tier": routing_tier, - "decision_source": decision_source, - "is_routed_call": decision.is_answer_call(), + "is_answer_call": decision.is_answer_call(), }), metadata, ); } - fn stage_decision_source( - &self, - request: &Request, - decision: &Decision, - ) -> Option<&'static str> { - let config = self.stage_marks.as_ref()?; - let signals = ToolSignals::from_request(request, config.recent_turn_window); - match pick_tier(&signals, config.picker, config.confidence_threshold) { - PickOutcome::Resolved { source, .. } => Some(source.as_str()), - PickOutcome::ConsultClassifier { .. } => { - let classifier_decided = config.classifier_enabled - && decision - .reasoning() - .and_then(decision_confidence) - .is_some_and(|confidence| confidence > 0.0); - Some(if classifier_decided { - "llm-classifier" - } else { - "fall_open" - }) - } - } - } - fn emit_routing_llm_calls( &self, marks: &mut Vec, @@ -496,7 +455,6 @@ impl SwitchyardRuntime { } for (index, call) in calls.into_iter().enumerate() { - let routing_tier = self.target_tiers.get(&call.selected_model).copied(); self.mark( marks, "switchyard.routing.llm_call", @@ -505,7 +463,6 @@ impl SwitchyardRuntime { "attempt": attempt, "call_index": index + 1, "selected_target": call.selected_model, - "routing_tier": routing_tier, "call_role": if call.is_answer_call { "candidate" } else { "judge" }, "outcome": if call.is_success { "ok" } else { "error" }, "latency_ms": call.duration.as_secs_f64() * 1_000.0, @@ -526,20 +483,6 @@ fn take_observed_calls(observations: &Mutex>) -> Vec Option { - let (_, suffix) = reasoning.rsplit_once("confidence ")?; - let numeric = suffix - .trim_start_matches(|character: char| { - !character.is_ascii_digit() && !matches!(character, '.' | '-' | '+') - }) - .chars() - .take_while(|character| { - character.is_ascii_digit() || matches!(character, '.' | '-' | '+' | 'e' | 'E') - }) - .collect::(); - numeric.parse().ok() -} - async fn send_marks( output: &async_channel::Sender, marks: &mut Vec, diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs index 1145f03fc..517da1c5d 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs @@ -101,7 +101,6 @@ fn runtime_with_algorithm_clients( protocol: WireFormat, clients: Vec<(&str, Arc)>, ) -> SwitchyardRuntime { - let is_stage = algorithm.name() == "stage_router"; let mut targets = BTreeMap::from([( "fallback".into(), PreparedTargetBinding { @@ -121,13 +120,6 @@ fn runtime_with_algorithm_clients( algorithm, targets, default_targets: BTreeMap::from([(protocol, "fallback".into())]), - target_tiers: BTreeMap::from([("weak".into(), "weak"), ("strong".into(), "strong")]), - stage_marks: is_stage.then_some(StageMarkConfig { - picker: PickerMode::CapableFirst, - confidence_threshold: 0.5, - recent_turn_window: None, - classifier_enabled: true, - }), translation: TranslationEngine::default(), } } @@ -307,8 +299,6 @@ async fn buffered_finalization_failure_uses_fallback_once() { ), ]), default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, translation: TranslationEngine::default(), }; let mut marks = Vec::new(); @@ -411,8 +401,6 @@ async fn invalid_selected_stream_does_not_invoke_failing_fallback_twice() { ), ]), default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, translation: TranslationEngine::default(), }; let (output, _messages) = async_channel::bounded(32); @@ -451,8 +439,6 @@ async fn failing_fallback_call_flushes_error_and_fallback_marks() { ), ]), default_targets: BTreeMap::from([(WireFormat::OpenAiChat, "fallback".into())]), - target_tiers: BTreeMap::new(), - stage_marks: None, translation: TranslationEngine::default(), }; let (output, messages) = async_channel::bounded(32); @@ -632,7 +618,8 @@ async fn escalation_buffers_weak_stream_then_latches_the_session_to_strong() { assert!(marks.iter().any(|mark| { mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == "strong" - && mark.data["routing_tier"] == "strong" + && mark.data["is_answer_call"] == true + && mark.data["reasoning"].is_string() && mark.metadata["session_id"] == "session-1" })); } @@ -778,9 +765,10 @@ async fn stage_router_uses_tool_signals_for_every_managed_protocol() { assert!(marks.iter().any(|mark| { mark.name == "switchyard.routing.decision" && mark.data["algorithm"] == "stage_router" + && mark.data["attempt"] == 1 && mark.data["selected_target"] == "strong" - && mark.data["routing_tier"] == "strong" - && mark.data["decision_source"] == "override" + && mark.data["reasoning"].is_string() + && mark.data["is_answer_call"] == true && mark.metadata["session_id"] == format!("stage-{}", protocol.as_str()) })); } @@ -821,8 +809,8 @@ async fn stage_router_falls_open_to_each_picker_default_without_tool_history() { assert!(marks.iter().any(|mark| { mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == expected - && mark.data["routing_tier"] == expected - && mark.data["decision_source"] == "fall_open" + && mark.data["reasoning"].is_string() + && mark.data["is_answer_call"] == true })); } } @@ -881,8 +869,8 @@ async fn stage_router_classifier_resolves_an_ambiguous_turn() { assert!(marks.iter().any(|mark| { mark.name == "switchyard.routing.decision" && mark.data["selected_target"] == "weak" - && mark.data["routing_tier"] == "weak" - && mark.data["decision_source"] == "llm-classifier" + && mark.data["reasoning"].is_string() + && mark.data["is_answer_call"] == true })); } diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index 207586134..a49636de0 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -276,19 +276,6 @@ Each response carries two routing headers: | `x-model-router-selected-model` | The model ID the turn was routed to. | | `x-model-router-rationale` | Human-readable routing reason (e.g. `stage_router selected weak (confidence 0.612)`). | -### Decision sources - -The router records an internal `decision_source` for each turn to distinguish the -paths through its cascade: - -| Source | When | -|---|---| -| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | -| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — landed the turn on the efficient tier. | -| `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | -| `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | -| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured; the default tier was used. | - ## When *not* to use stage-router - **Single-model deployments.** Use a `passthrough` route instead. From fa0d23f4698d743dc0684aa8eca0682a59fb8a3d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 13:51:15 -0600 Subject: [PATCH 5/9] refactor(relay): keep plugin changes isolated Signed-off-by: Bryan Bednarski --- crates/libsy/src/algorithms/stage.rs | 123 ++++++++++++++++++++-- crates/libsy/src/algorithms/util/stage.rs | 48 +++++++-- crates/libsy/src/lib.rs | 5 +- 3 files changed, 159 insertions(+), 17 deletions(-) diff --git a/crates/libsy/src/algorithms/stage.rs b/crates/libsy/src/algorithms/stage.rs index 592547199..5091d1d3f 100644 --- a/crates/libsy/src/algorithms/stage.rs +++ b/crates/libsy/src/algorithms/stage.rs @@ -20,9 +20,13 @@ use async_trait::async_trait; use super::fall_through::{DefaultTarget, FallThrough}; use super::llm_class::{LlmClassifierConfig, LlmTaskClassifier, TaskClassifierConfig}; use super::util::prompts::{SystemPromptProcessor, TargetPrompts}; -use super::util::stage::{HandoffNoteConfig, PickerMode, StageClassifier, StageTargets}; +use super::util::stage::{ + DecisionSource, HandoffNoteConfig, PickerMode, StageClassifier, StageTargets, + record_decision_source, +}; use super::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignalProcessor}; use crate::core::algorithm::{Algorithm, Driver, LlmTarget, LlmTargetSet}; +use crate::core::classifier::{Classification, Classifier}; use crate::core::state::State; use crate::{LibsyError, Result}; use switchyard_protocol::{Context, Request, Response}; @@ -30,6 +34,36 @@ use switchyard_protocol::{Context, Request, Response}; /// Telemetry name for a router this module assembles. const STAGE_ROUTER: &str = "stage_router"; +/// Attributes a turn to the classifier it wraps, when that classifier decides it. +/// +/// The classifiers themselves are composition-agnostic and write no state; only +/// this router knows where each sits in its cascade. +struct SourceStamp { + inner: Arc>, + source: DecisionSource, +} + +#[async_trait] +impl Classifier for SourceStamp { + fn routing_tier(&self, selected_model_id: &str) -> Option<&'static str> { + self.inner.routing_tier(selected_model_id) + } + + async fn score( + &self, + state: &mut State, + request: &mut Request, + driver: Option<&Driver>, + ) -> Result<(Classification, Option)> { + let (classification, served) = self.inner.score(state, request, driver).await?; + // An abstaining classifier passes the turn on, so it is not its to claim. + if matches!(&classification, Classification::Scores(scores) if !scores.is_empty()) { + record_decision_source(state, self.source); + } + Ok((classification, served)) + } +} + /// The capability judge a stage router falls through to. pub struct LlmFallback { /// Target the judge model is called through. It is not a routing @@ -158,18 +192,22 @@ fn build_route( if let Some(fallback) = config.llm_fallback { // The capability judge takes its tiers in the same order the capability // route passes them: efficient first, capable second. - router = router.with_classifier(Arc::new(LlmTaskClassifier::new( - LlmClassifierConfig::Capability { + router = router.with_classifier(Arc::new(SourceStamp { + inner: Arc::new(LlmTaskClassifier::new(LlmClassifierConfig::Capability { judge_target: fallback.judge_target, efficient_target: efficient, capable_target: capable, config: fallback.config, - }, - )?)); + })?), + source: DecisionSource::LlmClassifier, + })); } // Nothing behind this, so the turn lands on the picker's default tier — // including when the judge could not tell. - router = router.with_classifier(Arc::new(DefaultTarget::new(fall_open))); + router = router.with_classifier(Arc::new(SourceStamp { + inner: Arc::new(DefaultTarget::new(fall_open)), + source: DecisionSource::FallOpen, + })); // Runs on the post-decision hook, so it applies to the target the cascade // settled on, whichever classifier picked it. With no prompts configured it // is a no-op, so there is nothing to branch on. @@ -181,6 +219,7 @@ fn build_route( mod tests { use std::sync::Arc; + use async_trait::async_trait; use parking_lot::Mutex; use serde_json::json; use switchyard_protocol::{ @@ -188,9 +227,12 @@ mod tests { }; use super::*; + use crate::algorithms::util::stage::DECISION_SOURCE_KEY; use crate::core::algorithm::LlmTarget; + use crate::core::classifier::Score; + use crate::core::state::StateValue; use crate::core::testing::{Serve, reply, test_drive}; - use switchyard_protocol::{Context, Decision, Metadata}; + use switchyard_protocol::{Context, Decision, Metadata, Response}; fn tier_target(name: &str) -> LlmTarget { LlmTarget { @@ -198,6 +240,73 @@ mod tests { } } + /// A classifier that always picks `target`, standing in for a cascade member. + struct Fixed(&'static str); + + #[async_trait] + impl Classifier for Fixed { + async fn score( + &self, + _state: &mut State, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result<(Classification, Option)> { + Ok(( + Classification::Scores(vec![Score { + target: self.0.to_string(), + confidence: 1.0, + }]), + None, + )) + } + } + + /// A classifier that never decides. + struct Abstains; + + #[async_trait] + impl Classifier for Abstains { + async fn score( + &self, + _state: &mut State, + _request: &mut Request, + _driver: Option<&Driver>, + ) -> Result<(Classification, Option)> { + Ok((Classification::Ambiguous(vec![]), None)) + } + } + + async fn stamped(inner: Arc>) -> Result> { + let stamp = SourceStamp { + inner, + source: DecisionSource::LlmClassifier, + }; + let mut state = State::default(); + stamp + .score(&mut state, &mut Request::default(), None) + .await?; + Ok(match state.extra.get(DECISION_SOURCE_KEY) { + Some(StateValue::String(source)) => Some(source.clone()), + _ => None, + }) + } + + #[tokio::test] + async fn a_deciding_classifier_is_credited_with_the_turn() -> Result<()> { + assert_eq!( + stamped(Arc::new(Fixed("strong"))).await?.as_deref(), + Some("llm-classifier") + ); + Ok(()) + } + + #[tokio::test] + async fn an_abstaining_classifier_claims_nothing() -> Result<()> { + // It passed the turn on, so the next classifier is the one that decided. + assert_eq!(stamped(Arc::new(Abstains)).await?, None); + Ok(()) + } + fn config() -> StageRouterConfig { StageRouterConfig::new(PickerMode::EfficientFirst, 0.5) } diff --git a/crates/libsy/src/algorithms/util/stage.rs b/crates/libsy/src/algorithms/util/stage.rs index 1765d987d..3105c1102 100644 --- a/crates/libsy/src/algorithms/util/stage.rs +++ b/crates/libsy/src/algorithms/util/stage.rs @@ -25,7 +25,7 @@ use super::tool_signals::ToolSignals; use crate::Result; use crate::core::algorithm::Driver; use crate::core::classifier::{Classification, Classifier, Score}; -use crate::core::state::State; +use crate::core::state::{State, StateValue}; use switchyard_protocol::Request; /// Turn depth below which stall signals stay quiet — early no-write turns are @@ -126,7 +126,23 @@ impl PickerMode { } } -/// What produced a signal-resolved picker outcome. +/// `State.extra` key under which the turn's [`DecisionSource`] is recorded. +pub const DECISION_SOURCE_KEY: &str = "decision_source"; + +/// Record which component decided the turn. +pub(crate) fn record_decision_source(state: &mut State, source: DecisionSource) { + state.extra.insert( + DECISION_SOURCE_KEY.to_string(), + StateValue::String(source.as_str().to_string()), + ); +} + +/// What produced a decision — for stats and explainability. +/// +/// Each is stamped by the component that knows it, so a turn's final label names +/// whoever actually decided it. [`Ambiguous`](Self::Ambiguous) is the exception: +/// the signal scorer records it on its way out, and whichever classifier resolves +/// the turn overwrites it. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum DecisionSource { /// Hard override (critical severity or context compaction). @@ -144,7 +160,7 @@ pub enum DecisionSource { } impl DecisionSource { - /// Stable lowercase label for picker consumers. + /// Stable lowercase label used in stats. pub fn as_str(self) -> &'static str { match self { Self::Override => "override", @@ -443,7 +459,8 @@ impl StageClassifier { /// The signals could not decide, so this turn belongs to whatever the cascade /// has behind this classifier. - fn abstain() -> Classification { + fn abstain(state: &mut State) -> Classification { + record_decision_source(state, DecisionSource::Ambiguous); Classification::Ambiguous(Vec::new()) } @@ -473,7 +490,7 @@ impl Classifier for StageClassifier { let Some(signal) = tool_signals else { // No tool activity yet — nothing to score, so the signals have no // opinion, same as a below-threshold turn. - return Ok((Self::abstain(), None)); + return Ok((Self::abstain(state), None)); }; let outcome = pick_tier(signal, self.mode, self.confidence_threshold); @@ -485,6 +502,7 @@ impl Classifier for StageClassifier { .. } => { let target = self.targets.name(tier); + record_decision_source(state, source); // Only a resolved turn routes on this classifier's target, so it // is the only branch whose tier the signals actually chose — an // ambiguous turn is decided further down the cascade. @@ -498,7 +516,7 @@ impl Classifier for StageClassifier { None, )) } - PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(), None)), + PickOutcome::ConsultClassifier { .. } => Ok((Self::abstain(state), None)), } } } @@ -603,6 +621,10 @@ mod tests { .score(&mut state, &mut Request::default(), None) .await?; assert!(classification.0.argmax(false)?.is_none()); + assert!(matches!( + state.extra.get(DECISION_SOURCE_KEY), + Some(StateValue::String(source)) if source == "ambiguous" + )); Ok(()) } @@ -624,6 +646,11 @@ mod tests { } _ => panic!("expected a definite classification"), } + // The decision source travels downstream (for handoff-note gating). + assert!(matches!( + state.extra.get(DECISION_SOURCE_KEY), + Some(StateValue::String(source)) if source == "override" + )); Ok(()) } @@ -652,13 +679,18 @@ mod tests { } #[tokio::test] - async fn classifier_abstains_on_an_ambiguous_signal() -> Result<()> { - // A quiet signal corroborates neither axis, so the scorer abstains. + async fn classifier_falls_open_to_default_and_records_it() -> Result<()> { + // A quiet signal corroborates neither axis, so the scorer abstains and + // records why. let mut state = state_with(ToolSignals::default()); let classification = StageClassifier::new(tiers(), PickerMode::EfficientFirst, 0.5) .score(&mut state, &mut Request::default(), None) .await?; assert!(classification.0.argmax(false)?.is_none()); + assert!(matches!( + state.extra.get(DECISION_SOURCE_KEY), + Some(StateValue::String(source)) if source == "ambiguous" + )); Ok(()) } diff --git a/crates/libsy/src/lib.rs b/crates/libsy/src/lib.rs index 15c675c7c..d40328136 100644 --- a/crates/libsy/src/lib.rs +++ b/crates/libsy/src/lib.rs @@ -35,8 +35,9 @@ pub use algorithms::util::tool_signals::{DEFAULT_RECENT_WINDOW, ToolSignals}; // Stage-router scoring and tier selection — the shared signal-driven routing // core (scorer, picker, and the `StageClassifier`). pub use algorithms::util::stage::{ - CodingAgentDimensions, DecisionSource, HandoffNoteConfig, PickOutcome, PickerMode, ScoreResult, - StageClassifier, StageTargets, Tier, dimensions_from_signal, pick_tier, score_signal, + CodingAgentDimensions, DECISION_SOURCE_KEY, DecisionSource, HandoffNoteConfig, PickOutcome, + PickerMode, ScoreResult, StageClassifier, StageTargets, Tier, dimensions_from_signal, + pick_tier, score_signal, }; mod observability; From e48563a3abd0dd48aec9c8508045e70d63018b94 Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 13:58:36 -0600 Subject: [PATCH 6/9] docs: restore stage-router decision sources Signed-off-by: Bryan Bednarski --- docs/routing_algorithms/stage_router_routing.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/routing_algorithms/stage_router_routing.md b/docs/routing_algorithms/stage_router_routing.md index a49636de0..207586134 100644 --- a/docs/routing_algorithms/stage_router_routing.md +++ b/docs/routing_algorithms/stage_router_routing.md @@ -276,6 +276,19 @@ Each response carries two routing headers: | `x-model-router-selected-model` | The model ID the turn was routed to. | | `x-model-router-rationale` | Human-readable routing reason (e.g. `stage_router selected weak (confidence 0.612)`). | +### Decision sources + +The router records an internal `decision_source` for each turn to distinguish the +paths through its cascade: + +| Source | When | +|---|---| +| `override` | A critical-error severity (or a context-compaction marker) forced the capable tier. | +| `tests_passed` | A settled run — a recent test pass with a recent write and no windowed error — landed the turn on the efficient tier. | +| `dimensions` | The corroborative scorer crossed `confidence_threshold` and picked the tier by the sign of the score. | +| `llm-classifier` | The signals were ambiguous and the classifier returned a verdict. | +| `fall_open` | The signals were ambiguous and the classifier failed or wasn't configured; the default tier was used. | + ## When *not* to use stage-router - **Single-model deployments.** Use a `passthrough` route instead. From 9d57ae55247caa2f203a1aae2443a581d796804e Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 19:55:36 -0600 Subject: [PATCH 7/9] fix(relay): follow current client API Signed-off-by: Bryan Bednarski --- .../src/client.rs | 16 ++--- .../src/runtime.rs | 62 ++----------------- .../src/runtime/tests.rs | 36 +---------- 3 files changed, 11 insertions(+), 103 deletions(-) diff --git a/crates/switchyard-nemo-relay-plugin/src/client.rs b/crates/switchyard-nemo-relay-plugin/src/client.rs index c468f949d..bc197bfba 100644 --- a/crates/switchyard-nemo-relay-plugin/src/client.rs +++ b/crates/switchyard-nemo-relay-plugin/src/client.rs @@ -4,14 +4,13 @@ //! Switchyard-owned HTTP clients bound to one semantic routing target. use std::collections::BTreeMap; -use std::sync::Arc; use async_trait::async_trait; use serde_json::Value as Json; use switchyard_llm_client::{Backend, HttpBackendConfig, ModelConfig, TranslatingLlmClient}; use switchyard_protocol::{ - ContentBlock, Context, Decision, LlmClientError, Message, Request, Response, Role, - RoutedLlmClient, ToolCall, ToolResult, WireFormat, + ContentBlock, Decision, LlmClientError, Message, Request, Response, Role, RoutedLlmClient, + ToolCall, ToolResult, WireFormat, }; use switchyard_translation::TranslationEngine; @@ -101,13 +100,8 @@ impl TargetClient { #[async_trait] impl RoutedLlmClient for TargetClient { - async fn call( - &self, - ctx: Context, - request: Request, - decision: Arc, - ) -> Result { - let request = self.prepare_request(request, decision.as_ref()); + async fn call(&self, request: Request, decision: Decision) -> Result { + let request = self.prepare_request(request, &decision); translation::validate_target_request( &self.translation, self.target_format, @@ -115,7 +109,7 @@ impl RoutedLlmClient for TargetClient { ) .map_err(LlmClientError::RequestEncoding)?; self.inner - .call_rewrite_model(ctx, request, Some(&self.provider_model)) + .call_rewrite_model(request, Some(&self.provider_model)) .await } } diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime.rs b/crates/switchyard-nemo-relay-plugin/src/runtime.rs index c5604613f..9615e91d8 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime.rs @@ -11,7 +11,7 @@ use serde_json::{Map, json}; use switchyard_libsy::{Algorithm, LibsyError}; use switchyard_llm_client::{ClientRouter, LlmCallObservation, RunObservation, RunObserver, run}; use switchyard_protocol::{ - Context, Decision, LlmClientError, LlmResponse, Metadata, Request, Response, WireFormat, + Decision, LlmClientError, LlmResponse, Metadata, Request, Response, WireFormat, }; use switchyard_translation::{TranslationEngine, encode_stream}; @@ -312,7 +312,6 @@ impl SwitchyardRuntime { marks: &mut Vec, mark_metadata: &Json, ) -> Result { - let context = context_from_metadata(request.metadata.as_ref()); let observations = Arc::new(Mutex::new(Vec::new())); let observed_calls = observations.clone(); let observer: RunObserver = Arc::new(move |observation| { @@ -329,18 +328,10 @@ impl SwitchyardRuntime { .map(|(name, target)| (name.clone(), target.client.clone())) .collect::>(), ); - match run( - self.algorithm.clone(), - clients, - context, - request, - Some(observer), - ) - .await - { + match run(self.algorithm.clone(), clients, request, Some(observer)).await { Ok((decisions, response)) => { for decision in decisions { - self.emit_decision(marks, decision.as_ref(), attempt, mark_metadata); + self.emit_decision(marks, &decision, attempt, mark_metadata); } self.emit_routing_llm_calls( marks, @@ -379,15 +370,10 @@ impl SwitchyardRuntime { json!({"selected_target": target_name}), metadata, ); - let decision = Arc::new(Decision::new( - target_name, - Some("trusted fallback target".into()), - true, - )); - let context = context_from_metadata(request.metadata.as_ref()); + let decision = Decision::new(target_name, Some("trusted fallback target".into()), true); target .client - .call(context, request, decision) + .call(request, decision) .await .map_err(|error| public_client_failure("trusted fallback", &error)) } @@ -685,43 +671,5 @@ fn identity_metadata(metadata: Option<&Metadata>) -> Json { }) } -fn context_from_metadata(metadata: Option<&Metadata>) -> Context { - let Some(metadata) = metadata else { - return Context::default(); - }; - let mut values = std::collections::HashMap::new(); - for (name, value) in [ - ("session_id", metadata.session_id.as_deref()), - ("agent_id", metadata.agent_id.as_deref()), - ("parent_agent_id", metadata.parent_agent_id.as_deref()), - ("agent_kind", metadata.agent_kind.as_deref()), - ("agent_role", metadata.agent_role.as_deref()), - ("task_id", metadata.task_id.as_deref()), - ("task_kind", metadata.task_kind.as_deref()), - ("turn_id", metadata.turn_id.as_deref()), - ("correlation_id", metadata.correlation_id.as_deref()), - ] { - if let Some(value) = value { - values.insert(name.to_string(), value.to_string()); - } - } - values.insert("is_subagent".into(), metadata.is_subagent.to_string()); - values.insert( - "is_delegated_work".into(), - metadata.is_delegated_work.to_string(), - ); - if let Some(session_final) = metadata.session_final { - values.insert("session_final".into(), session_final.to_string()); - } - if let Some(extra) = &metadata.extra_metadata { - for (name, value) in extra { - values.entry(name.clone()).or_insert_with(|| value.clone()); - } - } - let mut context = Context::default(); - context.values = values; - context -} - #[cfg(test)] mod tests; diff --git a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs index 517da1c5d..54d383278 100644 --- a/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs +++ b/crates/switchyard-nemo-relay-plugin/src/runtime/tests.rs @@ -36,9 +36,8 @@ fn scripted(behavior: ScriptedBehavior) -> Arc { impl RoutedLlmClient for ScriptedClient { async fn call( &self, - _ctx: Context, request: Request, - _decision: Arc, + _decision: Decision, ) -> Result { self.calls.fetch_add(1, Ordering::Relaxed); match self.behavior { @@ -873,36 +872,3 @@ async fn stage_router_classifier_resolves_an_ambiguous_turn() { && mark.data["is_answer_call"] == true })); } - -#[test] -fn context_carries_identity_without_http_headers() { - let context = context_from_metadata(Some(&Metadata { - session_id: Some("session-1".into()), - agent_id: Some("agent-1".into()), - is_subagent: true, - extra_metadata: Some(BTreeMap::from([("tenant".into(), "blue".into())])), - http_headers: Some(http::HeaderMap::from_iter([( - http::HeaderName::from_static("authorization"), - http::HeaderValue::from_static("Bearer caller-secret"), - )])), - ..Metadata::default() - })); - - assert_eq!( - context.values.get("session_id").map(String::as_str), - Some("session-1") - ); - assert_eq!( - context.values.get("agent_id").map(String::as_str), - Some("agent-1") - ); - assert_eq!( - context.values.get("is_subagent").map(String::as_str), - Some("true") - ); - assert_eq!( - context.values.get("tenant").map(String::as_str), - Some("blue") - ); - assert!(!context.values.contains_key("authorization")); -} From 5af4f1621cf3e3a2c60675ddb8dd20491e5f145d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Tue, 11 Aug 2026 19:55:45 -0600 Subject: [PATCH 8/9] chore(relay): prepare plugin release bundle Signed-off-by: Bryan Bednarski --- .../switchyard-nemo-relay-plugin/Cargo.lock | 2 +- .../switchyard-nemo-relay-plugin/Cargo.toml | 2 +- crates/switchyard-nemo-relay-plugin/README.md | 20 +++-- .../scripts/package_bundle.py | 39 +++++++++- .../tests/test_package_bundle.py | 76 +++++++++++++++++++ 5 files changed, 131 insertions(+), 8 deletions(-) create mode 100644 crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.lock b/crates/switchyard-nemo-relay-plugin/Cargo.lock index 72f37d26a..4b21b2fea 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.lock +++ b/crates/switchyard-nemo-relay-plugin/Cargo.lock @@ -1718,7 +1718,7 @@ dependencies = [ [[package]] name = "switchyard-nemo-relay-plugin" -version = "0.1.0" +version = "0.2.0" dependencies = [ "async-channel", "async-trait", diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 6a169f6fb..77590987e 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -3,7 +3,7 @@ [package] name = "switchyard-nemo-relay-plugin" -version = "0.1.0" +version = "0.2.0" description = "Switchyard-owned HTTP routing plugin for NeMo Relay" authors = ["NVIDIA Corporation"] edition = "2024" diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index 93f22cdbd..e021fccf2 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -90,8 +90,9 @@ This is a raw C boundary: Switchyard contains a small ownership adapter for host strings, completion and stream handles, continuation handles, and captured scope handles because Relay 0.7 does not expose a safe Rust facade for its generic asynchronous surface. The HTTP, routing, and translation behavior -remains in Switchyard. The adapter can be replaced with the safe typed surface -when Relay exposes equivalent asynchronous callbacks and cancellation. +remains in Switchyard. NeMo Relay plans to provide the equivalent safe typed +surface in 0.8.0; once that is available, the raw-FFI compatibility adapter +should be removable. ## Supported routers @@ -374,12 +375,21 @@ cargo build --release \ --manifest-path crates/switchyard-nemo-relay-plugin/Cargo.toml python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ --library crates/switchyard-nemo-relay-plugin/target/release/libswitchyard_nemo_relay_plugin.so \ - --output dist/switchyard-nemo-relay-plugin-linux-x86_64 + --output build/switchyard-nemo-relay-plugin-linux-x86_64 \ + --archive dist/switchyard-nemo-relay-plugin-0.2.0-linux-x86_64.tar.gz ``` On macOS the library suffix is `.dylib`; Windows builds use `.dll`. The bundle -builder creates the minimal Relay package: the shared library, a materialized -manifest with Relay's inline SHA-256 integrity digest, and the JSON schema. +builder creates the Relay package: the shared library, a materialized manifest +with Relay's inline SHA-256 integrity digest, the JSON schema, and the project +license files. Use `.tar.gz` archives on Linux and macOS and `.zip` on Windows. +The archive's top-level directory is always `switchyard-nemo-relay-plugin`. + +The release archive convention is +`switchyard-nemo-relay-plugin--.`. A future Actions +matrix should upload each archive under the artifact name +`switchyard-nemo-relay-plugin-`, matching Switchyard's existing +platform-qualified artifact convention. Install the materialized bundle with Relay's normal lifecycle commands: diff --git a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py index 550f836ed..c5ea9e946 100644 --- a/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py +++ b/crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py @@ -8,9 +8,13 @@ import argparse import hashlib import shutil +import tarfile +import zipfile from pathlib import Path CRATE_ROOT = Path(__file__).resolve().parents[1] +REPOSITORY_ROOT = CRATE_ROOT.parents[1] +PACKAGE_NAME = "switchyard-nemo-relay-plugin" def digest(path: Path) -> str: @@ -22,11 +26,33 @@ def digest(path: Path) -> str: return value.hexdigest() +def archive_bundle(bundle: Path, archive: Path) -> None: + """Archive a materialized bundle under the stable package directory name.""" + archive.parent.mkdir(parents=True, exist_ok=True) + if archive.exists(): + raise ValueError(f"bundle archive already exists: {archive}") + + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive, "w:gz") as stream: + stream.add(bundle, arcname=PACKAGE_NAME) + return + + if archive.suffix == ".zip": + with zipfile.ZipFile(archive, "w", compression=zipfile.ZIP_DEFLATED) as stream: + for path in sorted(bundle.rglob("*")): + if path.is_file(): + stream.write(path, Path(PACKAGE_NAME) / path.relative_to(bundle)) + return + + raise ValueError("bundle archive must end in .tar.gz or .zip") + + def main() -> None: """Materialize a Relay-loadable plugin bundle in an empty directory.""" parser = argparse.ArgumentParser() parser.add_argument("--library", required=True, type=Path) parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--archive", type=Path) args = parser.parse_args() library = args.library.resolve() @@ -49,13 +75,24 @@ def main() -> None: artifact = output / library.name shutil.copy2(library, artifact) shutil.copy2(CRATE_ROOT / "config.schema.json", output / "config.schema.json") + for filename in ("LICENSE", "NOTICE"): + shutil.copy2(REPOSITORY_ROOT / filename, output / filename) artifact_digest = digest(artifact) manifest = manifest.replace("", artifact.name) manifest = manifest.replace("", artifact_digest) (output / "relay-plugin.toml").write_text(manifest, encoding="utf-8") - print(output) + if args.archive is None: + print(output) + return + + archive = args.archive.resolve() + try: + archive_bundle(output, archive) + except ValueError as error: + parser.error(str(error)) + print(archive) if __name__ == "__main__": diff --git a/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py new file mode 100644 index 000000000..296cbc4b8 --- /dev/null +++ b/crates/switchyard-nemo-relay-plugin/tests/test_package_bundle.py @@ -0,0 +1,76 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the native Relay plugin bundle packager.""" + +from __future__ import annotations + +import hashlib +import subprocess +import sys +import tarfile +import tempfile +import unittest +import zipfile +from pathlib import Path + +CRATE_ROOT = Path(__file__).resolve().parents[1] +PACKAGER = CRATE_ROOT / "scripts" / "package_bundle.py" +PACKAGE_NAME = "switchyard-nemo-relay-plugin" + + +class PackageBundleTest(unittest.TestCase): + """Verify materialized and archived plugin bundle contents.""" + + def test_materializes_and_archives_supported_formats(self) -> None: + for archive_suffix in (".tar.gz", ".zip"): + with self.subTest(archive_suffix=archive_suffix), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + library = root / "libswitchyard_nemo_relay_plugin.so" + library.write_bytes(b"compiled plugin") + output = root / "bundle" + archive = root / f"{PACKAGE_NAME}-0.2.0-linux-x86_64{archive_suffix}" + + subprocess.run( + [ + sys.executable, + str(PACKAGER), + "--library", + str(library), + "--output", + str(output), + "--archive", + str(archive), + ], + check=True, + capture_output=True, + text=True, + ) + + expected = { + "LICENSE", + "NOTICE", + "config.schema.json", + library.name, + "relay-plugin.toml", + } + self.assertEqual({path.name for path in output.iterdir()}, expected) + manifest = (output / "relay-plugin.toml").read_text(encoding="utf-8") + self.assertIn(f'artifact = "{library.name}"', manifest) + self.assertIn(hashlib.sha256(library.read_bytes()).hexdigest(), manifest) + self.assertNotIn("", manifest) + self.assertNotIn("", manifest) + self.assertEqual(self.archive_members(archive), {f"{PACKAGE_NAME}/{name}" for name in expected}) + + @staticmethod + def archive_members(archive: Path) -> set[str]: + """Return regular-file paths from a supported bundle archive.""" + if archive.name.endswith(".tar.gz"): + with tarfile.open(archive) as stream: + return {member.name for member in stream.getmembers() if member.isfile()} + with zipfile.ZipFile(archive) as stream: + return {member.filename for member in stream.infolist() if not member.is_dir()} + + +if __name__ == "__main__": + unittest.main() From 322461d99f522e23f51df6b4d08bb4705a47879d Mon Sep 17 00:00:00 2001 From: Bryan Bednarski Date: Wed, 12 Aug 2026 11:05:03 -0700 Subject: [PATCH 9/9] refactor: add NeMo Relay plugin to workspace Signed-off-by: Bryan Bednarski --- Cargo.lock | 141 + Cargo.toml | 3 + .../switchyard-nemo-relay-plugin/Cargo.lock | 2483 ----------------- .../switchyard-nemo-relay-plugin/Cargo.toml | 40 +- crates/switchyard-nemo-relay-plugin/README.md | 7 +- 5 files changed, 165 insertions(+), 2509 deletions(-) delete mode 100644 crates/switchyard-nemo-relay-plugin/Cargo.lock diff --git a/Cargo.lock b/Cargo.lock index 39871f0e9..b318bdbe3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -106,6 +106,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "async-channel" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" +dependencies = [ + "concurrent-queue", + "event-listener-strategy", + "futures-core", + "pin-project-lite", +] + [[package]] name = "async-stream" version = "0.3.6" @@ -274,6 +286,9 @@ name = "bitflags" version = "2.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +dependencies = [ + "serde_core", +] [[package]] name = "borrow-or-share" @@ -334,6 +349,16 @@ dependencies = [ "rand_core 0.10.1", ] +[[package]] +name = "chrono" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" +dependencies = [ + "num-traits", + "serde", +] + [[package]] name = "clap" version = "4.6.2" @@ -399,6 +424,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "concurrent-queue" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "core-foundation" version = "0.10.1" @@ -424,6 +458,12 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-utils" +version = "0.8.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" + [[package]] name = "data-encoding" version = "2.11.1" @@ -502,6 +542,26 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "event-listener" +version = "5.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" +dependencies = [ + "parking", + "pin-project-lite", +] + +[[package]] +name = "event-listener-strategy" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" +dependencies = [ + "event-listener", + "pin-project-lite", +] + [[package]] name = "fancy-regex" version = "0.18.0" @@ -1226,6 +1286,31 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "nemo-relay-plugin" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df37bebc79a6d757a7cb18d6c94ca0825fcb6c8e51ab4e13900e7d5853e0de8b" +dependencies = [ + "nemo-relay-types", + "serde", + "serde_json", +] + +[[package]] +name = "nemo-relay-types" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ea078c95f9e0804a77a0beb5d86f003ff4a9315c27e1a7afb2cee98bd4d7fd" +dependencies = [ + "bitflags", + "chrono", + "serde", + "serde_json", + "typed-builder", + "uuid", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1431,6 +1516,12 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" +[[package]] +name = "parking" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" + [[package]] name = "parking_lot" version = "0.12.5" @@ -2311,6 +2402,24 @@ dependencies = [ "wiremock", ] +[[package]] +name = "switchyard-nemo-relay-plugin" +version = "0.2.0" +dependencies = [ + "async-channel", + "async-trait", + "futures-util", + "http", + "nemo-relay-plugin", + "serde", + "serde_json", + "switchyard-libsy", + "switchyard-llm-client", + "switchyard-protocol", + "switchyard-translation", + "tokio", +] + [[package]] name = "switchyard-protocol" version = "0.2.0" @@ -2766,6 +2875,26 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "typed-builder" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" +dependencies = [ + "typed-builder-macro", +] + +[[package]] +name = "typed-builder-macro" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "unicode-general-category" version = "1.1.0" @@ -2808,6 +2937,18 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" +dependencies = [ + "getrandom 0.3.4", + "js-sys", + "serde", + "wasm-bindgen", +] + [[package]] name = "uuid-simd" version = "0.8.0" diff --git a/Cargo.toml b/Cargo.toml index 07d133bb3..82caff84c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,6 +8,7 @@ members = [ "crates/libsy-llm-client", "crates/switchyard-py", "crates/protocol", + "crates/switchyard-nemo-relay-plugin", "crates/switchyard-server", "crates/switchyard-skill-distillation", "crates/switchyard-translation", @@ -22,6 +23,7 @@ repository = "https://github.com/NVIDIA-NeMo/Switchyard" rust-version = "1.96.1" [workspace.dependencies] +async-channel = "2" async-stream = "0.3" async-trait = "0.1" futures = "0.3" @@ -30,6 +32,7 @@ http = "1" httpdate = "1" jsonschema = { version = "0.49.4", default-features = false } jsonptr = { version = "0.8.1", default-features = false, features = ["std", "json", "resolve"] } +nemo-relay-plugin = "=0.7.0" parking_lot = "0.12" rand = "0.10" reqwest = { version = "0.13.4", default-features = false, features = ["json", "rustls", "stream"] } diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.lock b/crates/switchyard-nemo-relay-plugin/Cargo.lock deleted file mode 100644 index 4b21b2fea..000000000 --- a/crates/switchyard-nemo-relay-plugin/Cargo.lock +++ /dev/null @@ -1,2483 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "ahash" -version = "0.8.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" -dependencies = [ - "cfg-if", - "getrandom 0.3.4", - "once_cell", - "serde", - "version_check", - "zerocopy", -] - -[[package]] -name = "aho-corasick" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" -dependencies = [ - "memchr", -] - -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "async-channel" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2" -dependencies = [ - "concurrent-queue", - "event-listener-strategy", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" -dependencies = [ - "async-stream-impl", - "futures-core", - "pin-project-lite", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "async-trait" -version = "0.1.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "atomic-waker" -version = "1.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "aws-lc-rs" -version = "1.18.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce2b2dcc879c3bae0d371e77c99f2238400ef24ec001394befa67b6e543add9e" -dependencies = [ - "aws-lc-sys", - "zeroize", -] - -[[package]] -name = "aws-lc-sys" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09fae7be8bb3174e05c6afdb34199e6dc0c7c04ba9fa237b1967adfbde27483" -dependencies = [ - "cc", - "cmake", - "dunce", - "fs_extra", - "pkg-config", -] - -[[package]] -name = "base64" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - -[[package]] -name = "bit-set" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" -dependencies = [ - "bit-vec", -] - -[[package]] -name = "bit-vec" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" - -[[package]] -name = "bitflags" -version = "2.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" -dependencies = [ - "serde_core", -] - -[[package]] -name = "borrow-or-share" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0b364ead1874514c8c2855ab558056ebfeb775653e7ae45ff72f28f8f3166c" - -[[package]] -name = "bumpalo" -version = "3.20.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" - -[[package]] -name = "bytecount" -version = "0.6.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "175812e0be2bccb6abe50bb8d566126198344f707e304f45c648fd8f2cc0365e" - -[[package]] -name = "bytes" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" - -[[package]] -name = "cc" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d262e149917187838d5b42777c8253bcb64500067342904e7d429499a6f277e" -dependencies = [ - "find-msvc-tools", - "jobserver", - "libc", - "shlex", -] - -[[package]] -name = "cfg-if" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "chacha20" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81" -dependencies = [ - "cfg-if", - "cpufeatures", - "rand_core", -] - -[[package]] -name = "chrono" -version = "0.4.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aa79e62e7697b8e29b513a68abacf485adcd1fe8284a4316c5ae868e6633327" -dependencies = [ - "num-traits", - "serde", -] - -[[package]] -name = "cmake" -version = "0.1.58" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0f78a02292a74a88ac736019ab962ece0bc380e3f977bf72e376c5d78ff0678" -dependencies = [ - "cc", -] - -[[package]] -name = "combine" -version = "4.6.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" -dependencies = [ - "bytes", - "memchr", -] - -[[package]] -name = "concurrent-queue" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "core-foundation-sys" -version = "0.8.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" - -[[package]] -name = "cpufeatures" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201" -dependencies = [ - "libc", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "data-encoding" -version = "2.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4583a4551df46e2792f82ceeac45e850d2e2d5debba0b91f102385cda5b11f06" - -[[package]] -name = "displaydoc" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "dunce" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" - -[[package]] -name = "email_address" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e079f19b08ca6239f47f8ba8509c11cf3ea30095831f7fed61441475edd8c449" -dependencies = [ - "serde", -] - -[[package]] -name = "equivalent" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "event-listener" -version = "5.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a23add41df1562121a9393cb065eab5146a1242410f23a644851e90cfd669d2" -dependencies = [ - "parking", - "pin-project-lite", -] - -[[package]] -name = "event-listener-strategy" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" -dependencies = [ - "event-listener", - "pin-project-lite", -] - -[[package]] -name = "fancy-regex" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "476de73bddf2ef8490aa4ee8f1cf40b430bf1d56c48c22080e5186952cd580e6" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "find-msvc-tools" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b73573e6edcd2af0cdf47bd6cb58f0b3839491263c314eaad1ccf24430e1de" - -[[package]] -name = "fluent-uri" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc74ac4d8359ae70623506d512209619e5cf8f347124910440dbc221714b328e" -dependencies = [ - "borrow-or-share", - "ref-cast", - "serde", -] - -[[package]] -name = "foldhash" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - -[[package]] -name = "form_urlencoded" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "fraction" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e076045bb43dac435333ed5f04caf35c7463631d0dae2deb2638d94dd0a5b872" -dependencies = [ - "lazy_static", - "num", -] - -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - -[[package]] -name = "futures" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a88cf1f829d945f548cf8fec32c61b1f202b6d93b45848602fc02af4b12ad218" -dependencies = [ - "futures-channel", - "futures-core", - "futures-executor", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-channel" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "262590f4fe6afeb0bc83be1daa64e52657fe185690a958af7f3ad0e92085c5ae" -dependencies = [ - "futures-core", - "futures-sink", -] - -[[package]] -name = "futures-core" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cd50c473c80f6d7c3670a752354b8e569b1a7cbfdc0419ec88e5edad85e0dc7" - -[[package]] -name = "futures-executor" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6754879cc9f2c66f88c6e5c35344bb0bdb0708b0352b1201815667c7eabc7458" -dependencies = [ - "futures-core", - "futures-task", - "futures-util", -] - -[[package]] -name = "futures-io" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4577ecaa3c4f96589d473f679a71b596316f6641bc350038b962a5daf0085d7a" - -[[package]] -name = "futures-macro" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6d3cde68c518367be28956066ddfef33813991b77a55005a69dae04bf3b10b" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "futures-sink" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e34418ac499d6305c2fb5ad0ed2f6ac998c5f8ca209b4510f7f94242c647e307" - -[[package]] -name = "futures-task" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b231ed28831efb4a61a08580c4bc233ec56bc009f4cd8f52da2c3cb97df0c109" - -[[package]] -name = "futures-util" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a77a90a256fce34da66415271e30f94ee91c57b04b8a2c042d9cf3220179deaa" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-macro", - "futures-sink", - "futures-task", - "memchr", - "pin-project-lite", - "slab", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "wasi", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 5.3.0", - "wasip2", - "wasm-bindgen", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "js-sys", - "libc", - "r-efi 6.0.0", - "rand_core", - "wasm-bindgen", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "allocator-api2", - "equivalent", - "foldhash", -] - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "http" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "918d3568bebf352712bc2ef3d46a8bcf1a75b373be6539de198e9105cbbf9ce0" -dependencies = [ - "bytes", - "itoa", -] - -[[package]] -name = "http-body" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2a8f2913ee65f60facd6a5905613afaa448497a0230cc41ce022d93290bc2c" -dependencies = [ - "bytes", - "http", -] - -[[package]] -name = "http-body-util" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f41fd6a08e4d4ec69df65976da761afd5ad5e58a9d4acb46bd1c953a9e3ff2" -dependencies = [ - "bytes", - "futures-core", - "http", - "http-body", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72" -dependencies = [ - "atomic-waker", - "bytes", - "futures-channel", - "futures-core", - "http", - "http-body", - "httparse", - "itoa", - "pin-project-lite", - "smallvec", - "tokio", - "want", -] - -[[package]] -name = "hyper-rustls" -version = "0.27.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" -dependencies = [ - "http", - "hyper", - "hyper-util", - "rustls", - "tokio", - "tokio-rustls", - "tower-service", -] - -[[package]] -name = "hyper-util" -version = "0.1.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" -dependencies = [ - "base64", - "bytes", - "futures-channel", - "futures-util", - "http", - "http-body", - "hyper", - "ipnet", - "libc", - "percent-encoding", - "pin-project-lite", - "socket2", - "tokio", - "tower-service", - "tracing", -] - -[[package]] -name = "icu_collections" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2984d1cd16c883d7935b9e07e44071dca8d917fd52ecc02c04d5fa0b5a3f191c" -dependencies = [ - "displaydoc", - "potential_utf", - "utf8_iter", - "yoke", - "zerofrom", - "zerovec", -] - -[[package]] -name = "icu_locale_core" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92219b62b3e2b4d88ac5119f8904c10f8f61bf7e95b640d25ba3075e6cac2c29" -dependencies = [ - "displaydoc", - "litemap", - "tinystr", - "writeable", - "zerovec", -] - -[[package]] -name = "icu_normalizer" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c56e5ee99d6e3d33bd91c5d85458b6005a22140021cc324cea84dd0e72cff3b4" -dependencies = [ - "icu_collections", - "icu_normalizer_data", - "icu_properties", - "icu_provider", - "smallvec", - "zerovec", -] - -[[package]] -name = "icu_normalizer_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da3be0ae77ea334f4da67c12f149704f19f81d1adf7c51cf482943e84a2bad38" - -[[package]] -name = "icu_properties" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bee3b67d0ea5c2cca5003417989af8996f8604e34fb9ddf96208a033901e70de" -dependencies = [ - "icu_collections", - "icu_locale_core", - "icu_properties_data", - "icu_provider", - "zerotrie", - "zerovec", -] - -[[package]] -name = "icu_properties_data" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e2bbb201e0c04f7b4b3e14382af113e17ba4f63e2c9d2ee626b720cbce54a14" - -[[package]] -name = "icu_provider" -version = "2.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "139c4cf31c8b5f33d7e199446eff9c1e02decfc2f0eec2c8d71f65befa45b421" -dependencies = [ - "displaydoc", - "icu_locale_core", - "writeable", - "yoke", - "zerofrom", - "zerotrie", - "zerovec", -] - -[[package]] -name = "idna" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" -dependencies = [ - "idna_adapter", - "smallvec", - "utf8_iter", -] - -[[package]] -name = "idna_adapter" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb68373c0d6620ef8105e855e7745e18b0d00d3bdb07fb532e434244cdb9a714" -dependencies = [ - "icu_normalizer", - "icu_properties", -] - -[[package]] -name = "ipnet" -version = "2.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a756c3fac73139e83f14c2d742155dd2b78d3ee56597b419a0579b7bdd6dd78" - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "jni" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" -dependencies = [ - "cfg-if", - "combine", - "jni-macros", - "jni-sys", - "log", - "simd_cesu8", - "thiserror", - "walkdir", - "windows-link", -] - -[[package]] -name = "jni-macros" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn 2.0.119", -] - -[[package]] -name = "jni-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" -dependencies = [ - "jni-sys-macros", -] - -[[package]] -name = "jni-sys-macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" -dependencies = [ - "quote", - "syn 2.0.119", -] - -[[package]] -name = "jobserver" -version = "0.1.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" -dependencies = [ - "getrandom 0.4.3", - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" -dependencies = [ - "cfg-if", - "futures-util", - "wasm-bindgen", -] - -[[package]] -name = "jsonptr" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85019623956752c8dd1f04a7b05d066187e0c9b217454246d8397d2a4893cc83" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "jsonschema" -version = "0.49.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59ec8a241beed129f06114aa68007e905ca350e7baeb6e17a7631bb7978d91b2" -dependencies = [ - "ahash", - "bytecount", - "data-encoding", - "email_address", - "fancy-regex", - "fraction", - "getrandom 0.3.4", - "idna", - "itoa", - "jsonschema-regex", - "jsonschema-value", - "num-cmp", - "num-traits", - "percent-encoding", - "referencing", - "regex", - "serde", - "serde_json", - "strum", - "unicode-general-category", - "uuid-simd", -] - -[[package]] -name = "jsonschema-regex" -version = "0.49.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91994f45017ed5e66aa8e59b8415f4cb033a6380d7200387b7cf117595fbdf85" -dependencies = [ - "regex-syntax", -] - -[[package]] -name = "jsonschema-value" -version = "0.49.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ec7637f83e510868ae6ed625f7ebfbbde4554ee8ce49854caa5126a8b9b9ecb" -dependencies = [ - "ahash", - "bytecount", - "fraction", - "num-cmp", - "num-traits", - "serde_json", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.189" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" - -[[package]] -name = "litemap" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92daf443525c4cce67b150400bc2316076100ce0b3686209eb8cf3c31612e6f0" - -[[package]] -name = "lock_api" -version = "0.4.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" -dependencies = [ - "scopeguard", -] - -[[package]] -name = "log" -version = "0.4.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" - -[[package]] -name = "lru-slab" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "micromap" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a86d3146ed3995b5913c414f6664344b9617457320782e64f0bb44afd49d74" - -[[package]] -name = "mio" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d65c71f1ce40ab09135ce117d742b9f8a19ff91a41a8b57ed50bc2de59c427" -dependencies = [ - "libc", - "wasi", - "windows-sys 0.61.2", -] - -[[package]] -name = "nemo-relay-plugin" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df37bebc79a6d757a7cb18d6c94ca0825fcb6c8e51ab4e13900e7d5853e0de8b" -dependencies = [ - "nemo-relay-types", - "serde", - "serde_json", -] - -[[package]] -name = "nemo-relay-types" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99ea078c95f9e0804a77a0beb5d86f003ff4a9315c27e1a7afb2cee98bd4d7fd" -dependencies = [ - "bitflags", - "chrono", - "serde", - "serde_json", - "typed-builder", - "uuid", -] - -[[package]] -name = "num" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" -dependencies = [ - "num-bigint", - "num-complex", - "num-integer", - "num-iter", - "num-rational", - "num-traits", -] - -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-cmp" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63335b2e2c34fae2fb0aa2cecfd9f0832a1e24b3b32ecec612c3426d46dc8aaa" - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-integer" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" -dependencies = [ - "num-traits", -] - -[[package]] -name = "num-iter" -version = "0.1.46" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c92800bd69a1eac91786bcfe9da64a897eb72911b8dc3095decbd07429e8048b" -dependencies = [ - "num-integer", - "num-traits", -] - -[[package]] -name = "num-rational" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" -dependencies = [ - "num-bigint", - "num-integer", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "opentelemetry" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0142c63252a9e054e68a4c61a5778f7b14f576274d593f8ce883d191a099682" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "pin-project-lite", - "thiserror", -] - -[[package]] -name = "outref" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e" - -[[package]] -name = "parking" -version = "2.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" - -[[package]] -name = "parking_lot" -version = "0.12.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-link", -] - -[[package]] -name = "percent-encoding" -version = "2.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - -[[package]] -name = "pin-project-lite" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" - -[[package]] -name = "pkg-config" -version = "0.3.33" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" - -[[package]] -name = "potential_utf" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0103b1cef7ec0cf76490e969665504990193874ea05c85ff9bab8b911d0a0564" -dependencies = [ - "zerovec", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "quinn" -version = "0.11.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c1a41e437b6bbd489372cd4971de128e85c855f56c57f283d20ff016cf7c0a8" -dependencies = [ - "bytes", - "cfg_aliases", - "pin-project-lite", - "quinn-proto", - "quinn-udp", - "rustc-hash", - "rustls", - "socket2", - "thiserror", - "tokio", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-proto" -version = "0.11.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560" -dependencies = [ - "aws-lc-rs", - "bytes", - "getrandom 0.4.3", - "lru-slab", - "rand", - "rand_pcg", - "ring", - "rustc-hash", - "rustls", - "rustls-pki-types", - "slab", - "thiserror", - "tinyvec", - "tracing", - "web-time", -] - -[[package]] -name = "quinn-udp" -version = "0.5.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35a133f956daabe89a61a685c2649f13d82d5aa4bd5d12d1277e1072a21c0694" -dependencies = [ - "cfg_aliases", - "libc", - "once_cell", - "socket2", - "tracing", - "windows-sys 0.61.2", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "5.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" -dependencies = [ - "chacha20", - "getrandom 0.4.3", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" - -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core", -] - -[[package]] -name = "redox_syscall" -version = "0.5.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" -dependencies = [ - "bitflags", -] - -[[package]] -name = "ref-cast" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" -dependencies = [ - "ref-cast-impl", -] - -[[package]] -name = "ref-cast-impl" -version = "1.0.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "referencing" -version = "0.49.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6efa2154ea6f5ce0fdecdd2a8d18f2fa1a39a8fbba91564f555a592e4dce8278" -dependencies = [ - "ahash", - "fluent-uri", - "getrandom 0.3.4", - "hashbrown", - "itoa", - "micromap", - "parking_lot", - "percent-encoding", - "serde_json", -] - -[[package]] -name = "regex" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" - -[[package]] -name = "reqwest" -version = "0.13.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" -dependencies = [ - "base64", - "bytes", - "futures-core", - "futures-util", - "http", - "http-body", - "http-body-util", - "hyper", - "hyper-rustls", - "hyper-util", - "js-sys", - "log", - "percent-encoding", - "pin-project-lite", - "quinn", - "rustls", - "rustls-pki-types", - "rustls-platform-verifier", - "serde", - "serde_json", - "sync_wrapper", - "tokio", - "tokio-rustls", - "tokio-util", - "tower", - "tower-http", - "tower-service", - "url", - "wasm-bindgen", - "wasm-bindgen-futures", - "wasm-streams", - "web-sys", -] - -[[package]] -name = "ring" -version = "0.17.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" -dependencies = [ - "cc", - "cfg-if", - "getrandom 0.2.17", - "libc", - "untrusted", - "windows-sys 0.52.0", -] - -[[package]] -name = "rustc-hash" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" - -[[package]] -name = "rustc_version" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" -dependencies = [ - "semver", -] - -[[package]] -name = "rustls" -version = "0.23.43" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0283386ce02abc0151e1761d08802dfe86c173b0b494af5cbc086574e453da06" -dependencies = [ - "aws-lc-rs", - "once_cell", - "rustls-pki-types", - "rustls-webpki", - "subtle", - "zeroize", -] - -[[package]] -name = "rustls-native-certs" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" -dependencies = [ - "openssl-probe", - "rustls-pki-types", - "schannel", - "security-framework", -] - -[[package]] -name = "rustls-pki-types" -version = "1.15.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f4925028c7eb5d1fcdaf196971378ed9d2c1c4efc7dc5d011256f76c99c0a96" -dependencies = [ - "web-time", - "zeroize", -] - -[[package]] -name = "rustls-platform-verifier" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" -dependencies = [ - "core-foundation", - "core-foundation-sys", - "jni", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.61.2", -] - -[[package]] -name = "rustls-platform-verifier-android" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" - -[[package]] -name = "rustls-webpki" -version = "0.103.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" -dependencies = [ - "aws-lc-rs", - "ring", - "rustls-pki-types", - "untrusted", -] - -[[package]] -name = "rustversion" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "scopeguard" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" - -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - -[[package]] -name = "semver" -version = "1.0.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "serde_json" -version = "1.0.151" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "sharded-slab" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6" -dependencies = [ - "lazy_static", -] - -[[package]] -name = "shlex" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8fadd59c855ef2080decdef8ff161eb6661b86933c9d82e5ba29dc602a55aba" - -[[package]] -name = "signal-hook-registry" -version = "1.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" -dependencies = [ - "errno", - "libc", -] - -[[package]] -name = "simd_cesu8" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" -dependencies = [ - "rustc_version", - "simdutf8", -] - -[[package]] -name = "simdutf8" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" - -[[package]] -name = "slab" -version = "0.4.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" - -[[package]] -name = "smallvec" -version = "1.15.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" - -[[package]] -name = "socket2" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" -dependencies = [ - "libc", - "windows-sys 0.61.2", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "strum" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9628de9b8791db39ceda2b119bbe13134770b56c138ec1d3af810d045c04f9bd" -dependencies = [ - "strum_macros", -] - -[[package]] -name = "strum_macros" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab85eea0270ee17587ed4156089e10b9e6880ee688791d45a905f5b1ca36f664" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "subtle" -version = "2.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" - -[[package]] -name = "switchyard-libsy" -version = "0.2.0" -dependencies = [ - "async-trait", - "futures", - "jsonptr", - "jsonschema", - "opentelemetry", - "parking_lot", - "rand", - "serde", - "serde_json", - "switchyard-protocol", - "thiserror", - "tokio", - "tokio-stream", - "tracing", - "tracing-opentelemetry", -] - -[[package]] -name = "switchyard-llm-client" -version = "0.2.0" -dependencies = [ - "async-trait", - "futures", - "futures-util", - "http", - "httpdate", - "opentelemetry", - "parking_lot", - "reqwest", - "serde_json", - "switchyard-libsy", - "switchyard-protocol", - "switchyard-translation", - "tokio", - "tracing", - "tracing-opentelemetry", -] - -[[package]] -name = "switchyard-nemo-relay-plugin" -version = "0.2.0" -dependencies = [ - "async-channel", - "async-trait", - "futures-util", - "http", - "nemo-relay-plugin", - "serde", - "serde_json", - "switchyard-libsy", - "switchyard-llm-client", - "switchyard-protocol", - "switchyard-translation", - "tokio", -] - -[[package]] -name = "switchyard-protocol" -version = "0.2.0" -dependencies = [ - "async-trait", - "futures", - "http", - "serde", - "serde_json", - "thiserror", -] - -[[package]] -name = "switchyard-translation" -version = "0.2.0" -dependencies = [ - "async-stream", - "futures", - "serde", - "serde_json", - "switchyard-protocol", - "thiserror", -] - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "sync_wrapper" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" -dependencies = [ - "futures-core", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "thiserror" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "2.0.20" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "thread_local" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ad99c4c6d32803332c548b1af0540b357b3f5fc0be8f6c6bfe8b2e6ae784070" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "tinystr" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8323304221c2a851516f22236c5722a72eaa19749016521d6dff0824447d96d" -dependencies = [ - "displaydoc", - "zerovec", -] - -[[package]] -name = "tinyvec" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb4ebadaa0af04fab11ae01eb5f9fdb5f9c5b875506e210e71c07873528baa7f" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "tokio" -version = "1.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "202caea871b69668250d242070849eb495be178ed697a3e98aebce5bc81a0bed" -dependencies = [ - "bytes", - "libc", - "mio", - "parking_lot", - "pin-project-lite", - "signal-hook-registry", - "socket2", - "tokio-macros", - "windows-sys 0.61.2", -] - -[[package]] -name = "tokio-macros" -version = "2.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.3", -] - -[[package]] -name = "tokio-rustls" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" -dependencies = [ - "rustls", - "tokio", -] - -[[package]] -name = "tokio-stream" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3d06f0b082ba57c26b79407372e57cf2a1e28124f78e9479fe80322cf53420b" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494815d09bf52b5548659851081238f0ca39ff638363907596da739561c62c52" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tower" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" -dependencies = [ - "futures-core", - "futures-util", - "pin-project-lite", - "sync_wrapper", - "tokio", - "tower-layer", - "tower-service", -] - -[[package]] -name = "tower-http" -version = "0.6.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" -dependencies = [ - "bitflags", - "bytes", - "futures-util", - "http", - "http-body", - "pin-project-lite", - "tower", - "tower-layer", - "tower-service", - "url", -] - -[[package]] -name = "tower-layer" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" - -[[package]] -name = "tower-service" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" - -[[package]] -name = "tracing" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" -dependencies = [ - "pin-project-lite", - "tracing-attributes", - "tracing-core", -] - -[[package]] -name = "tracing-attributes" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7490cfa5ec963746568740651ac6781f701c9c5ea257c58e057f3ba8cf69e8da" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "tracing-core" -version = "0.1.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" -dependencies = [ - "once_cell", - "valuable", -] - -[[package]] -name = "tracing-log" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" -dependencies = [ - "log", - "once_cell", - "tracing-core", -] - -[[package]] -name = "tracing-opentelemetry" -version = "0.33.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adbc64cba7137545b8044cb1fe9814f7aacf3c6b5f9b45be8bb5db538befdb26" -dependencies = [ - "js-sys", - "opentelemetry", - "smallvec", - "tracing", - "tracing-core", - "tracing-log", - "tracing-subscriber", - "web-time", -] - -[[package]] -name = "tracing-subscriber" -version = "0.3.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" -dependencies = [ - "sharded-slab", - "thread_local", - "tracing-core", -] - -[[package]] -name = "try-lock" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" - -[[package]] -name = "typed-builder" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31aa81521b70f94402501d848ccc0ecaa8f93c8eb6999eb9747e72287757ffda" -dependencies = [ - "typed-builder-macro", -] - -[[package]] -name = "typed-builder-macro" -version = "0.23.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076a02dc54dd46795c2e9c8282ed40bcfb1e22747e955de9389a1de28190fb26" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "unicode-general-category" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b993bddc193ae5bd0d623b49ec06ac3e9312875fdae725a975c51db1cc1677f" - -[[package]] -name = "unicode-ident" -version = "1.0.24" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" - -[[package]] -name = "untrusted" -version = "0.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" - -[[package]] -name = "url" -version = "2.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", - "serde", -] - -[[package]] -name = "utf8_iter" -version = "1.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - -[[package]] -name = "uuid" -version = "1.18.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" -dependencies = [ - "getrandom 0.3.4", - "js-sys", - "serde", - "wasm-bindgen", -] - -[[package]] -name = "uuid-simd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b082222b4f6619906941c17eb2297fff4c2fb96cb60164170522942a200bd8" -dependencies = [ - "outref", - "vsimd", -] - -[[package]] -name = "valuable" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" - -[[package]] -name = "version_check" -version = "0.9.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - -[[package]] -name = "vsimd" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "wasip2" -version = "1.0.4+wasi-0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487" -dependencies = [ - "wit-bindgen", -] - -[[package]] -name = "wasm-bindgen" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" -dependencies = [ - "cfg-if", - "once_cell", - "rustversion", - "wasm-bindgen-macro", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.77" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b7777d5cc23d0e91404e53ce2d5e8ec7acae3026b16233dba62cd3246457950" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" -dependencies = [ - "bumpalo", - "proc-macro2", - "quote", - "syn 2.0.119", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.127" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "wasm-streams" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d1ec4f6517c9e11ae630e200b2b65d193279042e28edd4a2cda233e46670bbb" -dependencies = [ - "futures-util", - "js-sys", - "wasm-bindgen", - "wasm-bindgen-futures", - "web-sys", -] - -[[package]] -name = "web-sys" -version = "0.3.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c435338968042f4f59a557f690a253676d47ce13ceb55d70100e7facf6620a30" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "web-time" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "webpki-root-certs" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b" -dependencies = [ - "rustls-pki-types", -] - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys 0.61.2", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "windows-targets" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" - -[[package]] -name = "wit-bindgen" -version = "0.57.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ebf944e87a7c253233ad6766e082e3cd714b5d03812acc24c318f549614536e" - -[[package]] -name = "writeable" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" - -[[package]] -name = "yoke" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5" -dependencies = [ - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.8.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de844c262c8848816172cef550288e7dc6c7b7814b4ee56b3e1553f275f1858e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zerocopy" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.56" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11532158c46691caf0f2593ea8358fed6bbf68a0315e80aae9bd41fbade684a1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure", -] - -[[package]] -name = "zeroize" -version = "1.9.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" - -[[package]] -name = "zerotrie" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f9152d31db0792fa83f70fb2f83148effb5c1f5b8c7686c3459e361d9bc20bf" -dependencies = [ - "displaydoc", - "yoke", - "zerofrom", -] - -[[package]] -name = "zerovec" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90f911cbc359ab6af17377d242225f4d75119aec87ea711a880987b18cd7b239" -dependencies = [ - "yoke", - "zerofrom", - "zerovec-derive", -] - -[[package]] -name = "zerovec-derive" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625dc425cab0dca6dc3c3319506e6593dcb08a9f387ea3b284dbd52a92c40555" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/crates/switchyard-nemo-relay-plugin/Cargo.toml b/crates/switchyard-nemo-relay-plugin/Cargo.toml index 77590987e..243e66cc0 100644 --- a/crates/switchyard-nemo-relay-plugin/Cargo.toml +++ b/crates/switchyard-nemo-relay-plugin/Cargo.toml @@ -3,32 +3,28 @@ [package] name = "switchyard-nemo-relay-plugin" -version = "0.2.0" +version.workspace = true description = "Switchyard-owned HTTP routing plugin for NeMo Relay" -authors = ["NVIDIA Corporation"] -edition = "2024" -license = "Apache-2.0" -repository = "https://github.com/NVIDIA-NeMo/Switchyard" -rust-version = "1.96.1" +authors.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true publish = false [lib] crate-type = ["cdylib"] [dependencies] -async-channel = "2" -async-trait = "0.1" -futures-util = "0.3" -http = "1" -nemo-relay-plugin = "=0.7.0" -serde = { version = "1", features = ["derive"] } -serde_json = "1" -switchyard-libsy = { path = "../libsy", version = "0.2.0" } -switchyard-llm-client = { path = "../libsy-llm-client", version = "0.2.0" } -switchyard-protocol = { path = "../protocol", version = "0.2.0" } -switchyard-translation = { path = "../switchyard-translation", version = "0.2.0" } -tokio = { version = "1", features = ["full"] } - -# Keep the dynamic plugin self-contained so adding it does not modify the root -# Switchyard workspace manifest or lockfile. -[workspace] +async-channel.workspace = true +async-trait.workspace = true +futures-util.workspace = true +http.workspace = true +nemo-relay-plugin.workspace = true +serde.workspace = true +serde_json.workspace = true +switchyard-libsy.workspace = true +switchyard-llm-client.workspace = true +switchyard-protocol.workspace = true +switchyard-translation.workspace = true +tokio.workspace = true diff --git a/crates/switchyard-nemo-relay-plugin/README.md b/crates/switchyard-nemo-relay-plugin/README.md index e021fccf2..d718b4a68 100644 --- a/crates/switchyard-nemo-relay-plugin/README.md +++ b/crates/switchyard-nemo-relay-plugin/README.md @@ -366,15 +366,14 @@ mode are rejected. ## Build and bundle -The crate is a nested standalone Cargo workspace with `publish = false`. This -keeps the integration from changing Switchyard's root workspace manifest or -lockfile. Operators install a binary bundle rather than a Rust crate: +The crate is a non-publishable member of the Switchyard Cargo workspace. +Operators install a binary bundle rather than a Rust crate: ```bash cargo build --release \ --manifest-path crates/switchyard-nemo-relay-plugin/Cargo.toml python3 crates/switchyard-nemo-relay-plugin/scripts/package_bundle.py \ - --library crates/switchyard-nemo-relay-plugin/target/release/libswitchyard_nemo_relay_plugin.so \ + --library target/release/libswitchyard_nemo_relay_plugin.so \ --output build/switchyard-nemo-relay-plugin-linux-x86_64 \ --archive dist/switchyard-nemo-relay-plugin-0.2.0-linux-x86_64.tar.gz ```