Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions crates/libsy-llm-client/tests/observability.rs
Original file line number Diff line number Diff line change
Expand Up @@ -568,6 +568,8 @@ async fn affinity_warns_once_when_request_has_no_usable_identity() -> switchyard
content: vec![ContentBlock::Reasoning {
text: "provider reasoning".to_string(),
signature: None,
details: Vec::new(),
fallback_text: None,
}],
}],
..LlmRequest::default()
Expand Down
2 changes: 2 additions & 0 deletions crates/libsy/src/algorithms/util/affinity.rs
Original file line number Diff line number Diff line change
Expand Up @@ -520,6 +520,8 @@ mod tests {
ContentBlock::Reasoning {
text: "Internal provider reasoning.".to_string(),
signature: Some("provider-signature".to_string()),
details: Vec::new(),
fallback_text: None,
},
],
});
Expand Down
2 changes: 2 additions & 0 deletions crates/libsy/src/algorithms/util/llm_judge.rs
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,8 @@ mod tests {
ContentBlock::Reasoning {
text: r#"{"ok":false}"#.to_string(),
signature: None,
details: Vec::new(),
fallback_text: None,
},
);
}
Expand Down
6 changes: 6 additions & 0 deletions crates/protocol/src/llm.rs
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ pub enum ContentBlock {
text: String,
/// Provider signature used to validate or continue the reasoning block.
signature: Option<String>,
/// Structured reasoning details that must be replayed without modification.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
details: Vec<Value>,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you give an example in the comment where reasoning is not a String?

/// Plaintext fallback when structured details contain no displayable text.
#[serde(default, skip_serializing_if = "Option::is_none")]
fallback_text: Option<String>,
},
/// Image content.
Image {
Expand Down
93 changes: 86 additions & 7 deletions crates/protocol/src/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -193,11 +193,24 @@ impl AggLlmResponse {
text,
});
}
ContentBlock::Reasoning { text, .. } => {
chunks.push(LlmResponseChunk::ReasoningDelta {
index: output_index,
text,
});
ContentBlock::Reasoning {
text,
details,
fallback_text,
..
} => {
if !details.is_empty() {
chunks.push(LlmResponseChunk::ReasoningDetailsDelta {
index: output_index,
details,
fallback_text,
});
} else {
chunks.push(LlmResponseChunk::ReasoningDelta {
index: output_index,
text,
});
}
}
ContentBlock::ToolCall(tool) => {
let args = serde_json::to_string(&tool.arguments).unwrap_or_default();
Expand Down Expand Up @@ -272,6 +285,15 @@ pub enum LlmResponseChunk {
/// Reasoning fragment.
text: String,
},
/// Adds structured reasoning details to one output index.
ReasoningDetailsDelta {
/// Provider output index.
index: usize,
/// Reasoning detail objects in provider order.
details: Vec<Value>,
/// Plaintext fallback when the details contain no displayable text.
fallback_text: Option<String>,
},
/// Adds or updates a tool call at one index.
ToolCallDelta {
/// Tool-call index within the response.
Expand Down Expand Up @@ -322,6 +344,8 @@ pub struct ResponseAccumulator {
model: Option<String>,
text: String,
reasoning: Option<String>,
reasoning_details: Vec<Value>,
reasoning_fallback: Option<String>,
tool_calls: BTreeMap<usize, PartialToolCall>,
usage: Usage,
stop_reason: Option<StopReason>,
Expand Down Expand Up @@ -359,6 +383,21 @@ impl ResponseAccumulator {
.get_or_insert_with(String::new)
.push_str(&text);
}
LlmResponseChunk::ReasoningDetailsDelta {
details,
fallback_text,
..
} => {
self.reasoning_details.extend(details);
if let Some(text) = fallback_text {
self.reasoning
.get_or_insert_with(String::new)
.push_str(&text);
self.reasoning_fallback
.get_or_insert_with(String::new)
.push_str(&text);
}
}
LlmResponseChunk::ToolCallDelta {
index,
id,
Expand Down Expand Up @@ -388,10 +427,12 @@ impl ResponseAccumulator {
/// tool calls (by ascending delta index) — a single assistant output.
pub fn finish(self) -> AggLlmResponse {
let mut content = Vec::new();
if let Some(reasoning) = self.reasoning {
if self.reasoning.is_some() || !self.reasoning_details.is_empty() {
content.push(ContentBlock::Reasoning {
text: reasoning,
text: self.reasoning.unwrap_or_default(),
signature: None,
details: self.reasoning_details,
fallback_text: self.reasoning_fallback,
});
}
if !self.text.is_empty() {
Expand Down Expand Up @@ -611,6 +652,8 @@ mod tests {
ContentBlock::Reasoning {
text: "think".to_string(),
signature: None,
details: Vec::new(),
fallback_text: None,
},
ContentBlock::Text {
text: "answer".to_string(),
Expand Down Expand Up @@ -649,6 +692,42 @@ mod tests {
assert_eq!(recovered.outputs[0].content, original.outputs[0].content);
}

#[test]
fn into_stream_retains_encrypted_reasoning_and_fallback_text() {
let details = vec![json!({
"type": "reasoning.encrypted",
"data": "opaque-encrypted-reasoning"
})];
let original = AggLlmResponse {
outputs: vec![ResponseOutput {
role: Role::Assistant,
content: vec![ContentBlock::Reasoning {
text: "fallback reasoning".to_string(),
signature: None,
details: details.clone(),
fallback_text: Some("fallback reasoning".to_string()),
}],
stop_reason: Some(StopReason::EndTurn),
}],
..AggLlmResponse::default()
};

let recovered = block_on(LlmResponse::Stream(original.into_stream()).into_agg())
.expect("into_agg failed");
let ContentBlock::Reasoning {
text,
details: recovered_details,
fallback_text,
..
} = &recovered.outputs[0].content[0]
else {
panic!("expected reasoning block");
};
assert_eq!(text, "fallback reasoning");
assert_eq!(recovered_details, &details);
assert_eq!(fallback_text.as_deref(), Some("fallback reasoning"));
}

#[test]
fn into_agg_preserves_stream_item_error() {
let response = LlmResponse::Stream(Box::pin(stream::once(async {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -525,6 +525,8 @@ fn decode_anthropic_content_block(
.and_then(Value::as_str)
.filter(|signature| !signature.is_empty())
.map(ToOwned::to_owned),
details: Vec::new(),
fallback_text: None,
}],
Some("tool_use") => vec![ContentBlock::ToolCall(ToolCall {
id: block
Expand Down Expand Up @@ -794,6 +796,7 @@ fn encode_one_anthropic_response_block(block: &ContentBlock) -> Vec<Value> {
ContentBlock::Reasoning {
text,
signature: None,
..
} => vec![json!({
"type": "thinking",
"thinking": text,
Expand All @@ -812,6 +815,7 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
ContentBlock::Reasoning {
text,
signature: Some(signature),
..
} if !signature.is_empty() => vec![json!({
"type": "thinking",
"thinking": text,
Expand Down
17 changes: 17 additions & 0 deletions crates/switchyard-translation/src/codecs/anthropic/stream.rs
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
use serde_json::{Map, Value, json};

use crate::LlmResponseChunk;
use crate::codecs::common::reasoning_text_from_details;
use crate::codecs::stream::{
StreamCodec, StreamTranslationState, record_source_identity,
target_message_id_or_source_message_id, target_model_or_source_model,
Expand Down Expand Up @@ -192,6 +193,22 @@ fn encode_anthropic_stream(
}));
out
}
LlmResponseChunk::ReasoningDetailsDelta {
details,
fallback_text,
..
} => {
let Some(text) = reasoning_text_from_details(&details).or(fallback_text) else {
return Vec::new();
};
let mut out = ensure_anthropic_reasoning_block(state);
out.push(json!({
"type": "content_block_delta",
"index": state.reasoning_block_index.unwrap_or(0),
"delta": {"type": "thinking_delta", "thinking": text},
}));
out
}
LlmResponseChunk::ToolCallDelta {
index,
id,
Expand Down
23 changes: 22 additions & 1 deletion crates/switchyard-translation/src/codecs/common.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

//! Provider-agnostic helpers shared by buffered wire-format codecs.
//! Provider-agnostic helpers shared by wire-format codecs.

use serde_json::{Map, Value};

Expand Down Expand Up @@ -41,6 +41,27 @@ pub(crate) fn reasoning_text_from_blocks(content: &[ContentBlock], separator: &s
.join(separator)
}

/// Extracts displayable text from structured reasoning details.
pub(crate) fn reasoning_text_from_details(details: &[Value]) -> Option<String> {

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Like this! Very nice.

let parts = details
.iter()
.filter_map(Value::as_object)
.filter_map(|detail| {
detail
.get("text")
.and_then(Value::as_str)
.filter(|text| !text.is_empty())
.or_else(|| {
detail
.get("summary")
.and_then(Value::as_str)
.filter(|summary| !summary.is_empty())
})
})
.collect::<Vec<_>>();
(!parts.is_empty()).then(|| parts.join("\n"))
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.

/// Copies unknown provider fields into the IR extension map.
pub(crate) fn provider_extensions(
object: &Map<String, Value>,
Expand Down
Loading