diff --git a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs index 7a8e79c9e..fe5573039 100644 --- a/crates/switchyard-translation/src/codecs/anthropic/buffered.rs +++ b/crates/switchyard-translation/src/codecs/anthropic/buffered.rs @@ -554,20 +554,18 @@ fn decode_anthropic_content_block( content: decode_tool_result_content(block.get("content").unwrap_or(&Value::Null)), is_error: block.get("is_error").and_then(Value::as_bool), })], - Some("image") => { - let source = block - .get("source") - .cloned() - .map(ImageSource::Raw) - .unwrap_or_else(|| ImageSource::Raw(Value::Object(block.clone()))); - vec![ContentBlock::Image { source }] - } + Some("image") => vec![ContentBlock::Image { + source: ImageSource::Raw(Value::Object(block.clone())), + }], Some("input_image") | Some("image_url") => decode_image_source(block) .map(|source| vec![ContentBlock::Image { source }]) .unwrap_or_default(), Some("input_file") | Some("file") => vec![ContentBlock::File { source: decode_file_source(block), }], + Some("document") => vec![ContentBlock::File { + source: decode_anthropic_file_source(block), + }], _ => vec![ContentBlock::Unknown { provider: WireFormat::AnthropicMessages.into(), raw: Value::Object(block.clone()), @@ -575,30 +573,36 @@ fn decode_anthropic_content_block( }) } -// Converts Anthropic tool-result content into text-like IR blocks. +// Preserves supported Anthropic tool-result blocks in the neutral IR. fn decode_tool_result_content(value: &Value) -> Vec { match value { Value::String(text) => vec![ContentBlock::Text { text: text.clone() }], Value::Array(blocks) => { - let mut text = Vec::new(); + let mut content = Vec::new(); for block in blocks { if let Some(block) = block.as_object() { - if block.get("type").and_then(Value::as_str) == Some("text") { - text.push( - block + match block.get("type").and_then(Value::as_str) { + Some("text") => content.push(ContentBlock::Text { + text: block .get("text") .and_then(Value::as_str) .unwrap_or_default() .to_string(), - ); - } else { - text.push(json_string(&Value::Object(block.clone()))); + }), + Some("image") => content.push(ContentBlock::Image { + source: ImageSource::Raw(Value::Object(block.clone())), + }), + Some("document") => content.push(ContentBlock::File { + source: decode_anthropic_file_source(block), + }), + _ => content.push(ContentBlock::Unknown { + provider: WireFormat::AnthropicMessages.into(), + raw: Value::Object(block.clone()), + }), } } } - vec![ContentBlock::Text { - text: text.join(" "), - }] + content } Value::Null => vec![ContentBlock::Text { text: String::new(), @@ -609,6 +613,11 @@ fn decode_tool_result_content(value: &Value) -> Vec { } } +// Keeps Anthropic document fields together for same-format re-encoding. +fn decode_anthropic_file_source(block: &Map) -> FileSource { + FileSource::Raw(Value::Object(block.clone())) +} + // Decodes Anthropic tool definitions into normalized tool definitions. fn decode_anthropic_tools(value: Option<&Value>) -> Vec { value @@ -815,11 +824,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec { "name": call.name, "input": anthropic_tool_input(&call.arguments), })], - ContentBlock::ToolResult(result) => vec![json!({ - "type": "tool_result", - "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id), - "content": text_from_blocks(&result.content, " "), - })], + ContentBlock::ToolResult(result) => { + let content = if result.content.iter().all(|block| { + matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) + }) { + Value::String(text_from_blocks(&result.content, " ")) + } else { + Value::Array( + result + .content + .iter() + .flat_map(encode_one_anthropic_tool_result_block) + .collect(), + ) + }; + vec![json!({ + "type": "tool_result", + "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id), + "content": content, + })] + } ContentBlock::Image { source } => vec![match source { ImageSource::Url { url, .. } => { json!({"type": "image", "source": {"type": "url", "url": url}}) @@ -880,6 +907,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec { } } +// Encodes only provider-safe block shapes inside Anthropic tool results. +fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec { + match block { + ContentBlock::Text { .. } + | ContentBlock::Refusal { .. } + | ContentBlock::Image { .. } + | ContentBlock::File { .. } => encode_one_anthropic_block(block), + ContentBlock::Unknown { provider, raw } + if provider.as_str() == WireFormat::AnthropicMessages.as_str() => + { + vec![raw.clone()] + } + ContentBlock::Unknown { raw, .. } => { + vec![json!({"type": "text", "text": json_string(raw)})] + } + ContentBlock::Reasoning { .. } + | ContentBlock::Audio { .. } + | ContentBlock::Video { .. } + | ContentBlock::ToolCall(_) + | ContentBlock::ToolResult(_) => Vec::new(), + } +} + // Anthropic requires `tool_use.input` to be object-shaped, while OpenAI and // Responses commonly carry function-call arguments as JSON strings. fn anthropic_tool_input(arguments: &Value) -> Value { diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index dbe77ca2a..32d0de1a2 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -700,18 +700,30 @@ fn encode_message_with_tool_results_to_openai( for block in &message.content { if let ContentBlock::ToolResult(result) = block { - push_pending_openai_message( - &mut out, - message.role, - &mut pending_content, - diagnostics, - policy, - )?; out.push(json!({ "role": "tool", "tool_call_id": result.tool_call_id, "content": text_from_blocks(&result.content, " "), })); + let non_text = result + .content + .iter() + .filter(|block| { + !matches!( + block, + ContentBlock::Text { .. } | ContentBlock::Refusal { .. } + ) + }) + .cloned() + .collect::>(); + if !non_text.is_empty() { + push_lossy( + diagnostics, + policy, + "OpenAI Chat tool messages only support text; non-text tool-result content was moved to a user message", + )?; + pending_content.extend(non_text); + } } else { pending_content.push(block.clone()); } @@ -956,6 +968,18 @@ fn openai_image_part(source: &ImageSource) -> Option { // Recognizes common raw image shapes emitted by Anthropic and Responses. fn openai_raw_image_part(raw: &Value) -> Option { let object = raw.as_object()?; + let object = if object.get("type").and_then(Value::as_str) == Some("image") { + let source = object.get("source").and_then(Value::as_object)?; + if !matches!( + source.get("type").and_then(Value::as_str), + Some("base64" | "url") + ) { + return None; + } + source + } else { + object + }; if let Some(url) = object.get("url").and_then(Value::as_str) { return Some(json!({"type": "image_url", "image_url": {"url": url}})); } @@ -999,8 +1023,26 @@ fn openai_file_part(source: &FileSource) -> Option { } Some(json!({"type": "file", "file": file})) } - FileSource::Raw(_) => None, + FileSource::Raw(raw) => openai_raw_file_part(raw), + } +} + +// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs. +fn openai_raw_file_part(raw: &Value) -> Option { + let block = raw.as_object()?; + if block.get("type").and_then(Value::as_str) != Some("document") { + return None; + } + let source = block.get("source").and_then(Value::as_object)?; + if source.get("type").and_then(Value::as_str) != Some("base64") { + return None; + } + let data = source.get("data").and_then(Value::as_str)?; + let mut file = json!({"file_data": data}); + if let Some(title) = block.get("title").and_then(Value::as_str) { + file["filename"] = Value::String(title.to_string()); } + Some(json!({"type": "file", "file": file})) } // Converts file sources to deterministic text fallback content. diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 71c765d01..e14d38e9b 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -5,7 +5,9 @@ use pretty_assertions::assert_eq; use serde_json::{Value, json}; -use switchyard_translation::{TranslationEngine, TranslationPolicy, WireFormat}; +use switchyard_translation::{ + LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat, +}; type TestResult = std::result::Result<(), Box>; @@ -310,6 +312,211 @@ fn anthropic_tool_result_followup_text_splits_to_openai_messages() -> TestResult Ok(()) } +// Verifies Anthropic multimodal blocks retain provider fields inside tool results. +#[test] +fn anthropic_tool_result_multimodal_blocks_round_trip_complete() -> TestResult { + let engine = TranslationEngine::default(); + let policy = TranslationPolicy { + preservation: switchyard_translation::PreservationPolicy::Disabled, + ..TranslationPolicy::default() + }; + let document = json!({ + "type": "document", + "title": "report.pdf", + "context": "Quarterly results", + "citations": {"enabled": true}, + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "ZG9jdW1lbnQ=" + } + }); + let image = json!({ + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aW1hZ2U=" + } + }); + let body = json!({ + "model": "claude-sonnet-4-20250514", + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_document", + "content": [ + {"type": "text", "text": "content ready"}, + image.clone(), + document.clone() + ] + }] + }], + "max_tokens": 1024 + }); + + let output = engine + .translate_request( + WireFormat::AnthropicMessages, + WireFormat::AnthropicMessages, + &body, + &policy, + )? + .body; + + assert_eq!( + output["messages"][0]["content"][0]["content"], + json!([ + {"type": "text", "text": "content ready"}, + image, + document + ]) + ); + Ok(()) +} + +// Verifies provider-managed Anthropic file IDs are not reused as OpenAI file IDs. +#[test] +fn anthropic_tool_result_file_id_does_not_become_openai_file_id() -> TestResult { + let engine = TranslationEngine::default(); + let document = json!({ + "type": "document", + "source": { + "type": "file", + "file_id": "file_anthropic_123" + } + }); + let body = json!({ + "model": "claude-sonnet-4-20250514", + "messages": [{ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": "toolu_document", + "content": [document.clone()] + }] + }], + "max_tokens": 1024 + }); + + let translated = engine.translate_request( + WireFormat::AnthropicMessages, + WireFormat::OpenAiChat, + &body, + &TranslationPolicy::default(), + )?; + + assert_eq!(translated.body["messages"][1]["content"][0]["type"], "text"); + let recovered: Value = serde_json::from_str( + translated.body["messages"][1]["content"][0]["text"] + .as_str() + .ok_or("file fallback should be text")?, + )?; + assert_eq!(recovered, document); + assert!(translated.diagnostics.iter().any(|diagnostic| { + diagnostic.message == "OpenAI Chat codec could not map file content" + })); + Ok(()) +} + +// Verifies parallel tool results preserve ordering and obey strict conversion policy. +#[test] +fn anthropic_parallel_multimodal_tool_results_preserve_order_and_policy() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "claude-sonnet-4-20250514", + "messages": [{ + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_image", + "content": [ + {"type": "text", "text": "image ready"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "aW1hZ2U=" + } + } + ] + }, + { + "type": "tool_result", + "tool_use_id": "toolu_document", + "content": [ + {"type": "text", "text": "document ready"}, + { + "type": "document", + "title": "report.pdf", + "source": { + "type": "base64", + "media_type": "application/pdf", + "data": "ZG9jdW1lbnQ=" + } + } + ] + } + ] + }], + "max_tokens": 1024 + }); + + let output = engine + .translate_request( + WireFormat::AnthropicMessages, + WireFormat::OpenAiChat, + &body, + &TranslationPolicy::default(), + )? + .body; + + assert_eq!( + output["messages"], + json!([ + {"role": "tool", "tool_call_id": "toolu_image", "content": "image ready"}, + { + "role": "tool", + "tool_call_id": "toolu_document", + "content": "document ready" + }, + { + "role": "user", + "content": [ + { + "type": "image_url", + "image_url": {"url": "data:image/png;base64,aW1hZ2U="} + }, + { + "type": "file", + "file": {"file_data": "ZG9jdW1lbnQ=", "filename": "report.pdf"} + } + ] + } + ]) + ); + let policy = TranslationPolicy { + lossy_conversion_policy: LossyConversionPolicy::Reject, + ..TranslationPolicy::default() + }; + + let error = match engine.translate_request( + WireFormat::AnthropicMessages, + WireFormat::OpenAiChat, + &body, + &policy, + ) { + Ok(_) => panic!("multimodal tool result should be rejected by strict policy"), + Err(error) => error, + }; + + assert_eq!(error.kind(), "LossyConversion"); + Ok(()) +} + // Verifies structured Anthropic system blocks remain separated in OpenAI system text. #[test] fn anthropic_structured_system_blocks_preserve_boundaries_for_openai_chat() -> TestResult {