fix(translation): preserve multimodal tool results (#389)

* fix(translation): preserve multimodal tool results

Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com>

* fix(translation): preserve raw document wrappers

Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com>

* fix(translation): preserve Anthropic multimodal wrappers

Signed-off-by: Elias Shieh <32212900+ting-hong-shieh@users.noreply.github.com>

---------

Signed-off-by: Ting-Hong Shieh <32212900+ting-hong-shieh@users.noreply.github.com>
Signed-off-by: Elias Shieh <32212900+ting-hong-shieh@users.noreply.github.com>

Author: ting-hong-shieh

crates/switchyard-translation/src/codecs/anthropic/buffered.rs

@@ -554,51 +554,55 @@ fn decode_anthropic_content_block(
             content: decode_tool_result_content(block.get("content").unwrap_or(&Value::Null)),
             is_error: block.get("is_error").and_then(Value::as_bool),
         })],
-        Some("image") => {
-            let source = block
-                .get("source")
-                .cloned()
-                .map(ImageSource::Raw)
-                .unwrap_or_else(|| ImageSource::Raw(Value::Object(block.clone())));
-            vec![ContentBlock::Image { source }]
-        }
+        Some("image") => vec![ContentBlock::Image {
+            source: ImageSource::Raw(Value::Object(block.clone())),
+        }],
         Some("input_image") | Some("image_url") => decode_image_source(block)
             .map(|source| vec![ContentBlock::Image { source }])
             .unwrap_or_default(),
         Some("input_file") | Some("file") => vec![ContentBlock::File {
             source: decode_file_source(block),
         }],
+        Some("document") => vec![ContentBlock::File {
+            source: decode_anthropic_file_source(block),
+        }],
         _ => vec![ContentBlock::Unknown {
             provider: WireFormat::AnthropicMessages.into(),
             raw: Value::Object(block.clone()),
         }],
     })
 }
 
-// Converts Anthropic tool-result content into text-like IR blocks.
+// Preserves supported Anthropic tool-result blocks in the neutral IR.
 fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {
     match value {
         Value::String(text) => vec![ContentBlock::Text { text: text.clone() }],
         Value::Array(blocks) => {
-            let mut text = Vec::new();
+            let mut content = Vec::new();
             for block in blocks {
                 if let Some(block) = block.as_object() {
-                    if block.get("type").and_then(Value::as_str) == Some("text") {
-                        text.push(
-                            block
+                    match block.get("type").and_then(Value::as_str) {
+                        Some("text") => content.push(ContentBlock::Text {
+                            text: block
                                 .get("text")
                                 .and_then(Value::as_str)
                                 .unwrap_or_default()
                                 .to_string(),
-                        );
-                    } else {
-                        text.push(json_string(&Value::Object(block.clone())));
+                        }),
+                        Some("image") => content.push(ContentBlock::Image {
+                            source: ImageSource::Raw(Value::Object(block.clone())),
+                        }),
+                        Some("document") => content.push(ContentBlock::File {
+                            source: decode_anthropic_file_source(block),
+                        }),
+                        _ => content.push(ContentBlock::Unknown {
+                            provider: WireFormat::AnthropicMessages.into(),
+                            raw: Value::Object(block.clone()),
+                        }),
                     }
                 }
             }
-            vec![ContentBlock::Text {
-                text: text.join(" "),
-            }]
+            content
         }
         Value::Null => vec![ContentBlock::Text {
             text: String::new(),
@@ -609,6 +613,11 @@ fn decode_tool_result_content(value: &Value) -> Vec<ContentBlock> {
     }
 }
 
+// Keeps Anthropic document fields together for same-format re-encoding.
+fn decode_anthropic_file_source(block: &Map<String, Value>) -> FileSource {
+    FileSource::Raw(Value::Object(block.clone()))
+}
+
 // Decodes Anthropic tool definitions into normalized tool definitions.
 fn decode_anthropic_tools(value: Option<&Value>) -> Vec<ToolDefinition> {
     value
@@ -815,11 +824,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
             "name": call.name,
             "input": anthropic_tool_input(&call.arguments),
         })],
-        ContentBlock::ToolResult(result) => vec![json!({
-            "type": "tool_result",
-            "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),
-            "content": text_from_blocks(&result.content, " "),
-        })],
+        ContentBlock::ToolResult(result) => {
+            let content = if result.content.iter().all(|block| {
+                matches!(
+                    block,
+                    ContentBlock::Text { .. } | ContentBlock::Refusal { .. }
+                )
+            }) {
+                Value::String(text_from_blocks(&result.content, " "))
+            } else {
+                Value::Array(
+                    result
+                        .content
+                        .iter()
+                        .flat_map(encode_one_anthropic_tool_result_block)
+                        .collect(),
+                )
+            };
+            vec![json!({
+                "type": "tool_result",
+                "tool_use_id": sanitize_anthropic_tool_use_id(&result.tool_call_id),
+                "content": content,
+            })]
+        }
         ContentBlock::Image { source } => vec![match source {
             ImageSource::Url { url, .. } => {
                 json!({"type": "image", "source": {"type": "url", "url": url}})
@@ -880,6 +907,29 @@ fn encode_one_anthropic_block(block: &ContentBlock) -> Vec<Value> {
     }
 }
 
+// Encodes only provider-safe block shapes inside Anthropic tool results.
+fn encode_one_anthropic_tool_result_block(block: &ContentBlock) -> Vec<Value> {
+    match block {
+        ContentBlock::Text { .. }
+        | ContentBlock::Refusal { .. }
+        | ContentBlock::Image { .. }
+        | ContentBlock::File { .. } => encode_one_anthropic_block(block),
+        ContentBlock::Unknown { provider, raw }
+            if provider.as_str() == WireFormat::AnthropicMessages.as_str() =>
+        {
+            vec![raw.clone()]
+        }
+        ContentBlock::Unknown { raw, .. } => {
+            vec![json!({"type": "text", "text": json_string(raw)})]
+        }
+        ContentBlock::Reasoning { .. }
+        | ContentBlock::Audio { .. }
+        | ContentBlock::Video { .. }
+        | ContentBlock::ToolCall(_)
+        | ContentBlock::ToolResult(_) => Vec::new(),
+    }
+}
+
 // Anthropic requires `tool_use.input` to be object-shaped, while OpenAI and
 // Responses commonly carry function-call arguments as JSON strings.
 fn anthropic_tool_input(arguments: &Value) -> Value {

crates/switchyard-translation/src/codecs/openai_chat/buffered.rs

@@ -700,18 +700,30 @@ fn encode_message_with_tool_results_to_openai(
 
     for block in &message.content {
         if let ContentBlock::ToolResult(result) = block {
-            push_pending_openai_message(
-                &mut out,
-                message.role,
-                &mut pending_content,
-                diagnostics,
-                policy,
-            )?;
             out.push(json!({
                 "role": "tool",
                 "tool_call_id": result.tool_call_id,
                 "content": text_from_blocks(&result.content, " "),
             }));
+            let non_text = result
+                .content
+                .iter()
+                .filter(|block| {
+                    !matches!(
+                        block,
+                        ContentBlock::Text { .. } | ContentBlock::Refusal { .. }
+                    )
+                })
+                .cloned()
+                .collect::<Vec<_>>();
+            if !non_text.is_empty() {
+                push_lossy(
+                    diagnostics,
+                    policy,
+                    "OpenAI Chat tool messages only support text; non-text tool-result content was moved to a user message",
+                )?;
+                pending_content.extend(non_text);
+            }
         } else {
             pending_content.push(block.clone());
         }
@@ -956,6 +968,18 @@ fn openai_image_part(source: &ImageSource) -> Option<Value> {
 // Recognizes common raw image shapes emitted by Anthropic and Responses.
 fn openai_raw_image_part(raw: &Value) -> Option<Value> {
     let object = raw.as_object()?;
+    let object = if object.get("type").and_then(Value::as_str) == Some("image") {
+        let source = object.get("source").and_then(Value::as_object)?;
+        if !matches!(
+            source.get("type").and_then(Value::as_str),
+            Some("base64" | "url")
+        ) {
+            return None;
+        }
+        source
+    } else {
+        object
+    };
     if let Some(url) = object.get("url").and_then(Value::as_str) {
         return Some(json!({"type": "image_url", "image_url": {"url": url}}));
     }
@@ -999,8 +1023,26 @@ fn openai_file_part(source: &FileSource) -> Option<Value> {
             }
             Some(json!({"type": "file", "file": file}))
         }
-        FileSource::Raw(_) => None,
+        FileSource::Raw(raw) => openai_raw_file_part(raw),
+    }
+}
+
+// Maps portable fields from raw Anthropic documents without forwarding provider-managed IDs.
+fn openai_raw_file_part(raw: &Value) -> Option<Value> {
+    let block = raw.as_object()?;
+    if block.get("type").and_then(Value::as_str) != Some("document") {
+        return None;
+    }
+    let source = block.get("source").and_then(Value::as_object)?;
+    if source.get("type").and_then(Value::as_str) != Some("base64") {
+        return None;
+    }
+    let data = source.get("data").and_then(Value::as_str)?;
+    let mut file = json!({"file_data": data});
+    if let Some(title) = block.get("title").and_then(Value::as_str) {
+        file["filename"] = Value::String(title.to_string());
     }
+    Some(json!({"type": "file", "file": file}))
 }
 
 // Converts file sources to deterministic text fallback content.