fix(translation): classify OpenAI audio and video inputs

Signed-off-by: Todd Fisher <todd.fisher@gmail.com>

Author: taf2

crates/switchyard-translation/src/codecs/openai_chat/buffered.rs

@@ -480,6 +480,16 @@ pub(crate) fn decode_openai_content(
                             source: decode_file_source(block),
                         });
                     }
+                    Some("audio") | Some("audio_url") | Some("input_audio") => {
+                        content.push(ContentBlock::Audio {
+                            source: decode_media_source(block, "audio"),
+                        });
+                    }
+                    Some("video") | Some("video_url") | Some("input_video") => {
+                        content.push(ContentBlock::Video {
+                            source: decode_media_source(block, "video"),
+                        });
+                    }
                     _ => content.push(ContentBlock::Unknown {
                         provider: provider.into(),
                         raw: Value::Object(block.clone()),
@@ -561,6 +571,76 @@ pub(crate) fn decode_file_source(block: &Map<String, Value>) -> FileSource {
     FileSource::Raw(Value::Object(block.clone()))
 }
 
+/// Decodes common OpenAI audio and video block shapes into normalized media sources.
+pub(crate) fn decode_media_source(block: &Map<String, Value>, kind: &str) -> MediaSource {
+    let url_key = format!("{kind}_url");
+    if let Some(value) = block.get(&url_key) {
+        if let Some(url) = value.as_str() {
+            return MediaSource::Url {
+                url: url.to_string(),
+                media_type: media_type(block, kind),
+            };
+        }
+        if let Some(payload) = value.as_object()
+            && let Some(url) = payload.get("url").and_then(Value::as_str)
+        {
+            return MediaSource::Url {
+                url: url.to_string(),
+                media_type: media_type(payload, kind).or_else(|| media_type(block, kind)),
+            };
+        }
+    }
+
+    let input_key = format!("input_{kind}");
+    for key in [input_key.as_str(), kind] {
+        let Some(value) = block.get(key) else {
+            continue;
+        };
+        if let Some(data) = value.as_str() {
+            return MediaSource::Base64 {
+                media_type: media_type(block, kind),
+                data: data.to_string(),
+            };
+        }
+        if let Some(payload) = value.as_object() {
+            if let Some(url) = payload.get("url").and_then(Value::as_str) {
+                return MediaSource::Url {
+                    url: url.to_string(),
+                    media_type: media_type(payload, kind).or_else(|| media_type(block, kind)),
+                };
+            }
+            if let Some(data) = payload.get("data").and_then(Value::as_str) {
+                return MediaSource::Base64 {
+                    media_type: media_type(payload, kind).or_else(|| media_type(block, kind)),
+                    data: data.to_string(),
+                };
+            }
+        }
+    }
+
+    if let Some(data) = block.get("data").and_then(Value::as_str) {
+        return MediaSource::Base64 {
+            media_type: media_type(block, kind),
+            data: data.to_string(),
+        };
+    }
+    MediaSource::Raw(Value::Object(block.clone()))
+}
+
+fn media_type(object: &Map<String, Value>, kind: &str) -> Option<String> {
+    object
+        .get("media_type")
+        .or_else(|| object.get("format"))
+        .and_then(Value::as_str)
+        .map(|value| {
+            if value.contains('/') {
+                value.to_string()
+            } else {
+                format!("{kind}/{value}")
+            }
+        })
+}
+
 /// Decodes one OpenAI tool call into a normalized tool call.
 pub(crate) fn decode_openai_tool_call(
     tool_call: &Value,

crates/switchyard-translation/src/codecs/openai_chat/mod.rs

@@ -9,4 +9,4 @@ mod stream;
 pub use buffered::OpenAiChatCodec;
 pub use stream::OpenAiChatStreamCodec;
 
-pub(crate) use buffered::{decode_file_source, decode_image_source};
+pub(crate) use buffered::{decode_file_source, decode_image_source, decode_media_source};

crates/switchyard-translation/src/codecs/responses/buffered.rs

@@ -10,7 +10,7 @@ use serde_json::{Map, Value, json};
 use crate::codecs::common::{
     is_known_role_name, provider_extensions, reasoning_text_from_blocks, text_from_blocks,
 };
-use crate::codecs::openai_chat::{decode_file_source, decode_image_source};
+use crate::codecs::openai_chat::{decode_file_source, decode_image_source, decode_media_source};
 use crate::codecs::{
     DecodedRequest, DecodedResponse, EncodedRequest, EncodedResponse, FormatCodec,
 };
@@ -659,6 +659,16 @@ fn decode_responses_content(value: &Value) -> Vec<ContentBlock> {
                     Some("input_file") => out.push(ContentBlock::File {
                         source: decode_file_source(block),
                     }),
+                    Some("audio") | Some("audio_url") | Some("input_audio") => {
+                        out.push(ContentBlock::Audio {
+                            source: decode_media_source(block, "audio"),
+                        });
+                    }
+                    Some("video") | Some("video_url") | Some("input_video") => {
+                        out.push(ContentBlock::Video {
+                            source: decode_media_source(block, "video"),
+                        });
+                    }
                     _ => out.push(ContentBlock::Unknown {
                         provider: WireFormat::OpenAiResponses.into(),
                         raw: Value::Object(block.clone()),

crates/switchyard-translation/tests/request_translation.rs

@@ -6,11 +6,55 @@
 use pretty_assertions::assert_eq;
 use serde_json::{Value, json};
 use switchyard_translation::{
-    LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,
+    InputModality, LossyConversionPolicy, TranslationEngine, TranslationPolicy, WireFormat,
 };
 
 type TestResult = std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>;
 
+#[test]
+fn openai_request_media_blocks_are_typed_for_modality_routing() -> TestResult {
+    let engine = TranslationEngine::default();
+    let chat = engine.decode_request(
+        WireFormat::OpenAiChat,
+        &json!({
+            "model": "gpt",
+            "messages": [{
+                "role": "user",
+                "content": [{
+                    "type": "input_audio",
+                    "input_audio": {"data": "AAAA", "format": "wav"}
+                }]
+            }]
+        }),
+        &TranslationPolicy::default(),
+    )?;
+    assert_eq!(
+        chat.request.input_modalities(),
+        [InputModality::Audio].into_iter().collect()
+    );
+
+    let responses = engine.decode_request(
+        WireFormat::OpenAiResponses,
+        &json!({
+            "model": "gpt",
+            "input": [{
+                "type": "message",
+                "role": "user",
+                "content": [{
+                    "type": "input_video",
+                    "video": {"media_type": "video/mp4", "data": "AAAA"}
+                }]
+            }]
+        }),
+        &TranslationPolicy::default(),
+    )?;
+    assert_eq!(
+        responses.request.input_modalities(),
+        [InputModality::Video].into_iter().collect()
+    );
+    Ok(())
+}
+
 // Verifies Anthropic-only request fields are dropped or mapped for OpenAI Chat.
 #[test]
 fn anthropic_request_translates_to_openai_chat_without_anthropic_only_fields() -> TestResult {