diff --git a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs index 4b0394f43..879aa3cdc 100644 --- a/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs +++ b/crates/switchyard-translation/src/codecs/openai_chat/buffered.rs @@ -604,9 +604,21 @@ pub(crate) fn decode_file_source(block: &Map) -> FileSource { } return FileSource::Raw(Value::Object(file.clone())); } + // A Responses `input_file` carries the payload directly on the block instead + // of nesting it under `file`, so read that shape too. Chat blocks never reach + // here with these keys, having matched the nested branch above. if let Some(file_id) = block.get("file_id").and_then(Value::as_str) { return FileSource::FileId(file_id.to_string()); } + if let Some(file_data) = block.get("file_data").and_then(Value::as_str) { + return FileSource::FileData { + data: file_data.to_string(), + filename: block + .get("filename") + .and_then(Value::as_str) + .map(ToOwned::to_owned), + }; + } FileSource::Raw(Value::Object(block.clone())) } diff --git a/crates/switchyard-translation/src/codecs/responses/buffered.rs b/crates/switchyard-translation/src/codecs/responses/buffered.rs index 03e11f15b..04392264f 100644 --- a/crates/switchyard-translation/src/codecs/responses/buffered.rs +++ b/crates/switchyard-translation/src/codecs/responses/buffered.rs @@ -18,9 +18,9 @@ use crate::diagnostic::TranslationDiagnostic; use crate::error::{Result, TranslationError}; use crate::format::{FormatId, WireFormat}; use crate::llm::{ - AggLlmResponse, ContentBlock, InstructionBlock, LlmRequest, MediaSource, Message, OutputParams, - ProviderExtensions, ReasoningParams, ResponseOutput, Role, SamplingParams, StopReason, - ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, + AggLlmResponse, ContentBlock, FileSource, ImageSource, InstructionBlock, LlmRequest, + MediaSource, Message, OutputParams, ProviderExtensions, ReasoningParams, ResponseOutput, Role, + SamplingParams, StopReason, ToolCall, ToolChoice, ToolDefinition, ToolResult, Usage, }; use crate::policy::{DeterministicIdPolicy, TranslationPolicy}; use crate::util::{ @@ -1159,9 +1159,18 @@ fn encode_responses_content( ContentBlock::Refusal { text } => { blocks.push(json!({"type": "refusal", "refusal": text})); } - ContentBlock::Image { source } => { - blocks.push(json!({"type": "input_image", "image_url": source})); - } + ContentBlock::Image { source } => match responses_image_part(source) { + Some(part) => blocks.push(part), + None => { + push_lossy( + diagnostics, + policy, + "Responses codec could not map image content", + )?; + let raw = serde_json::to_value(source).unwrap_or_default(); + blocks.push(json!({"type": "input_text", "text": json_string(&raw)})); + } + }, ContentBlock::Audio { source } => blocks.push(match source { MediaSource::Raw(raw) => json!({"type": "input_text", "text": json_string(raw)}), MediaSource::Url { url, media_type } => json!({ @@ -1187,7 +1196,7 @@ fn encode_responses_content( }), }), ContentBlock::File { source } => { - blocks.push(json!({"type": "input_file", "file": source})); + blocks.push(responses_file_part(source)); } ContentBlock::Unknown { raw, .. } => { push_lossy( @@ -1205,6 +1214,106 @@ fn encode_responses_content( Ok(Value::Array(blocks)) } +// Encodes a normalized image source as a Responses `input_image` part. +// +// `ImageSource` is adjacently tagged (`#[serde(tag = "type", content = "data")]`), +// so serializing it inline emits `{"type": "url", "data": {..}}` where the +// Responses API requires `image_url` to be a bare URL or data-URI string. The +// Chat and Anthropic codecs destructure it for this reason; this mirrors +// `openai_chat::openai_image_part`. +fn responses_image_part(source: &ImageSource) -> Option { + match source { + ImageSource::Url { url, detail } => { + let mut part = json!({"type": "input_image", "image_url": url}); + if let Some(detail) = detail { + part["detail"] = Value::String(detail.clone()); + } + Some(part) + } + ImageSource::Base64 { media_type, data } => media_type.as_ref().map(|media_type| { + json!({ + "type": "input_image", + "image_url": format!("data:{media_type};base64,{data}"), + }) + }), + ImageSource::Raw(raw) => responses_raw_image_part(raw), + } +} + +// Recovers a Responses `input_image` part from a provider image source that has +// no normalized representation. +fn responses_raw_image_part(raw: &Value) -> Option { + if let Some(url) = raw.as_str() { + return Some(json!({"type": "input_image", "image_url": url})); + } + let object = raw.as_object()?; + // An Anthropic image arrives as the whole `{"type": "image", "source": {..}}` + // block, so the payload lives one level down. + let object = if object.get("type").and_then(Value::as_str) == Some("image") { + let source = object.get("source").and_then(Value::as_object)?; + if !matches!( + source.get("type").and_then(Value::as_str), + Some("base64" | "url") + ) { + return None; + } + source + } else { + object + }; + for key in ["url", "image_url"] { + if let Some(url) = object.get(key).and_then(Value::as_str) { + return Some(json!({"type": "input_image", "image_url": url})); + } + } + let data = object.get("data").and_then(Value::as_str)?; + let media_type = object + .get("media_type") + .and_then(Value::as_str) + .unwrap_or("application/octet-stream"); + Some(json!({ + "type": "input_image", + "image_url": format!("data:{media_type};base64,{data}"), + })) +} + +// Encodes a normalized file source as a Responses `input_file` part. +// +// `FileSource` carries the same adjacent tagging as `ImageSource`, so it cannot +// be serialized inline. Unlike OpenAI Chat, which nests the payload under a +// `file` object, the Responses wire carries `file_id`, `file_data` and +// `filename` directly on the part -- nesting them produces a block upstream +// cannot read, so the file is accepted and then silently ignored, the same +// failure this commit fixes for images. +fn responses_file_part(source: &FileSource) -> Value { + let mut part = json!({"type": "input_file"}); + match source { + FileSource::FileId(file_id) => { + part["file_id"] = Value::String(file_id.clone()); + } + FileSource::FileData { data, filename } => { + part["file_data"] = Value::String(data.clone()); + if let Some(filename) = filename { + part["filename"] = Value::String(filename.clone()); + } + } + // A raw source is whatever the decoder could not normalize. It is an + // object in every path that builds one, so lift its keys onto the part + // rather than re-nesting them; `type` is already set above. + FileSource::Raw(raw) => match raw.as_object() { + Some(object) => { + for (key, value) in object { + if key != "type" { + part[key.as_str()] = value.clone(); + } + } + } + None => part["file_data"] = raw.clone(), + }, + } + part +} + // Encodes normalized tool definitions into Responses tool JSON. // Encodes normalized tools as Responses entries. // diff --git a/crates/switchyard-translation/tests/request_translation.rs b/crates/switchyard-translation/tests/request_translation.rs index 49ae5184a..65b3fd585 100644 --- a/crates/switchyard-translation/tests/request_translation.rs +++ b/crates/switchyard-translation/tests/request_translation.rs @@ -2301,3 +2301,277 @@ fn anthropic_thinking_is_dropped_from_responses_input() -> TestResult { ); Ok(()) } + +// Verifies a Responses `input_image` keeps `image_url` as a bare string. +// +// Codex sends `{"type": "input_image", "image_url": "data:image/png;base64,.."}`. +// `ImageSource` is adjacently tagged, so encoding it inline produced +// `image_url: {"type": "url", "data": {..}}`, which upstream cannot read -- the +// image was accepted, silently unreadable, and the model saw no image at all. +#[test] +fn responses_input_image_keeps_image_url_as_string() -> TestResult { + let engine = TranslationEngine::default(); + let data_uri = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="; + let body = json!({ + "model": "gpt-5.4-mini", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "What is in this image?"}, + {"type": "input_image", "image_url": data_uri} + ] + }] + }); + + let output = engine + .translate_request( + WireFormat::OpenAiResponses, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["input"][0]["content"] + .as_array() + .ok_or("Responses content should be an array")?; + let image = content + .iter() + .find(|block| block["type"] == "input_image") + .ok_or("input_image block should survive translation")?; + assert_eq!( + image["image_url"], + Value::String(data_uri.to_string()), + "image_url must be a bare string, not a tagged ImageSource" + ); + Ok(()) +} + +// Verifies `detail` survives as a sibling key of `image_url`. +#[test] +fn responses_input_image_preserves_detail() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "gpt-5.4-mini", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "look"}, + { + "type": "input_image", + "image_url": "https://example.test/image.png", + "detail": "high" + } + ] + }] + }); + + let output = engine + .translate_request( + WireFormat::OpenAiResponses, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["input"][0]["content"] + .as_array() + .ok_or("Responses content should be an array")?; + let image = content + .iter() + .find(|block| block["type"] == "input_image") + .ok_or("input_image block should survive translation")?; + assert_eq!(image["image_url"], "https://example.test/image.png"); + assert_eq!(image["detail"], "high"); + Ok(()) +} + +// Verifies an Anthropic base64 image encodes to a Responses data-URI string. +#[test] +fn anthropic_image_encodes_as_responses_input_image_string() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "claude-sonnet-4-20250514", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "describe"}, + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "iVBORw0KGgoAAAANSUhEUg==" + } + } + ] + }], + "max_tokens": 1024 + }); + + let output = engine + .translate_request( + WireFormat::AnthropicMessages, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["input"][0]["content"] + .as_array() + .ok_or("Responses content should be an array")?; + let image = content + .iter() + .find(|block| block["type"] == "input_image") + .ok_or("input_image block should survive translation")?; + let url = image["image_url"] + .as_str() + .ok_or("image_url must be a bare string, not a tagged ImageSource")?; + assert_eq!(url, "data:image/png;base64,iVBORw0KGgoAAAANSUhEUg=="); + Ok(()) +} + +// Verifies a file crossing INTO Responses carries its payload on the part itself. +// +// OpenAI Chat nests `file_id`/`file_data`/`filename` under a `file` object; the +// Responses wire carries them as siblings of `type`. Emitting the Chat shape onto +// a Responses request reproduced the image failure exactly -- accepted, unreadable, +// ignored. +// +// ⚠ This must be a CROSS-format translation. A same-format Responses request +// replays its preserved body verbatim via `exact_preserved_request`, so the +// encoder never runs and a Responses->Responses test passes against the nested +// shape too -- it asserts the input, not the encoder. +#[test] +fn chat_file_id_encodes_onto_the_responses_part() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "gpt-5.4-mini", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "summarize"}, + {"type": "file", "file": {"file_id": "file-abc123"}} + ] + }] + }); + + let output = engine + .translate_request( + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["input"][0]["content"] + .as_array() + .ok_or("Responses content should be an array")?; + let file = content + .iter() + .find(|block| block["type"] == "input_file") + .ok_or("input_file block should survive translation")?; + assert_eq!( + file["file_id"], "file-abc123", + "file_id must sit directly on the part, not under `file`" + ); + assert!( + file.get("file").is_none(), + "the Chat-style nested `file` object must not reach a Responses request" + ); + Ok(()) +} + +// Verifies base64 file content and its filename cross into Responses as direct fields. +#[test] +fn chat_file_data_encodes_onto_the_responses_part() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "gpt-5.4-mini", + "messages": [{ + "role": "user", + "content": [ + {"type": "text", "text": "read this"}, + { + "type": "file", + "file": {"file_data": "JVBERi0xLjQK", "filename": "report.pdf"} + } + ] + }] + }); + + let output = engine + .translate_request( + WireFormat::OpenAiChat, + WireFormat::OpenAiResponses, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["input"][0]["content"] + .as_array() + .ok_or("Responses content should be an array")?; + let file = content + .iter() + .find(|block| block["type"] == "input_file") + .ok_or("input_file block should survive translation")?; + assert_eq!(file["file_data"], "JVBERi0xLjQK"); + assert_eq!(file["filename"], "report.pdf"); + assert!( + file.get("file").is_none(), + "the Chat-style nested `file` object must not reach a Responses request" + ); + Ok(()) +} + +// Verifies a flat Responses `input_file` decodes instead of being dropped. +// +// `decode_file_source` read a direct `file_id` but not a direct `file_data`, so a +// Responses file fell through to `FileSource::Raw`. Chat's raw encoder maps only +// Anthropic `document` blocks and returns `None` for anything else -- so the file +// did not merely lose its filename, it vanished from the request entirely. +#[test] +fn responses_flat_file_data_survives_into_chat() -> TestResult { + let engine = TranslationEngine::default(); + let body = json!({ + "model": "gpt-5.4-mini", + "input": [{ + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "read this"}, + { + "type": "input_file", + "file_data": "JVBERi0xLjQK", + "filename": "report.pdf" + } + ] + }] + }); + + let output = engine + .translate_request( + WireFormat::OpenAiResponses, + WireFormat::OpenAiChat, + &body, + &TranslationPolicy::default(), + )? + .body; + + let content = output["messages"][0]["content"] + .as_array() + .ok_or("Chat content should be an array")?; + let file = content + .iter() + .find(|block| block["type"] == "file") + .ok_or("the file must survive into Chat, not be dropped as unmappable raw")?; + assert_eq!(file["file"]["file_data"], "JVBERi0xLjQK"); + assert_eq!(file["file"]["filename"], "report.pdf"); + Ok(()) +}