From 186c2adb326050587f4577f09e7dedcaafc982d8 Mon Sep 17 00:00:00 2001 From: Awais Qureshi Date: Tue, 17 Mar 2026 10:38:16 +0500 Subject: [PATCH] fix(gemini): support images in tool_results for /v1/messages routing (#23724) * fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(fireworks): skip #transform=inline for base64 data URLs Closes #23583 --- .../prompt_templates/factory.py | 59 ++++-- ...llm_core_utils_prompt_templates_factory.py | 172 ++++++++++++++++++ 2 files changed, 219 insertions(+), 12 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 47272b38ad..6c4c98ebf9 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1498,17 +1498,49 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 from litellm.types.llms.vertex_ai import BlobType content_str: str = "" - inline_data: Optional[BlobType] = None + inline_data_list: List[BlobType] = [] if "content" in message: if isinstance(message["content"], str): content_str = message["content"] + # Detect data-URL images (e.g. from Anthropic tool_result with a single image block + # that was serialised as a plain string by translate_anthropic_messages_to_openai) + # and promote them to inline_data so Gemini receives actual image bytes. + if content_str.startswith("data:") and ";base64," in content_str: + try: + mime_rest = content_str[5:].split(";base64,", 1) + if len(mime_rest) == 2 and mime_rest[0].startswith("image/"): + # Strip any extra parameters (e.g. ";charset=UTF-8") from the MIME segment + clean_mime = mime_rest[0].split(";")[0].strip() + inline_data_list.append( + BlobType(data=mime_rest[1], mime_type=clean_mime) + ) + content_str = "" + except Exception as e: + verbose_logger.warning( + f"Failed to parse data URL in tool response: {e}" + ) elif isinstance(message["content"], List): content_list = message["content"] for content in content_list: content_type = content.get("type", "") if content_type == "text": content_str += content.get("text", "") + elif content_type == "image": + # Anthropic-native image block: {"type": "image", "source": {"type": "base64", ...}} + source = content.get("source", {}) + if isinstance(source, dict) and source.get("type") == "base64": + try: + inline_data_list.append( + BlobType( + data=source.get("data", ""), + mime_type=source.get("media_type", "image/jpeg"), + ) + ) + except Exception as e: + verbose_logger.warning( + f"Failed to process Anthropic image block in tool response: {e}" + ) elif content_type in ("input_image", "image_url"): # Extract image for inline_data (for Computer Use screenshots and tool results) image_url_data = content.get("image_url", "") @@ -1524,9 +1556,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 image_obj = convert_to_anthropic_image_obj( image_url, format=None ) - inline_data = BlobType( - data=image_obj["data"], - mime_type=image_obj["media_type"], + inline_data_list.append( + BlobType( + data=image_obj["data"], + mime_type=image_obj["media_type"], + ) ) except Exception as e: verbose_logger.warning( @@ -1551,9 +1585,11 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 file_obj = convert_to_anthropic_image_obj( file_data, format=None ) - inline_data = BlobType( - data=file_obj["data"], - mime_type=file_obj["media_type"], + inline_data_list.append( + BlobType( + data=file_obj["data"], + mime_type=file_obj["media_type"], + ) ) except Exception as e: verbose_logger.warning( @@ -1607,13 +1643,12 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 # Create part with function_response, and optionally inline_data for images (Computer Use) _part: VertexPartType = {"function_response": _function_response} - # For Computer Use, if we have an image, we need separate parts: + # For Computer Use, if we have images/files, we need separate parts: # - One part with function_response - # - One part with inline_data + # - One part per inline_data item # Gemini's PartType is a oneof, so we can't have both in the same part - if inline_data: - image_part: VertexPartType = {"inline_data": inline_data} - return [_part, image_part] + if inline_data_list: + return [_part] + [{"inline_data": d} for d in inline_data_list] return _part diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 8d68539564..5c5cd5bdc3 100644 --- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -1,3 +1,4 @@ +import base64 import json from unittest.mock import MagicMock, patch @@ -9,9 +10,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( BedrockConverseMessagesProcessor, BedrockImageProcessor, _convert_to_bedrock_tool_call_invoke, + convert_to_gemini_tool_call_result, ollama_pt, sanitize_messages_for_tool_calling, ) +from litellm.types.llms.openai import ChatCompletionToolMessage def test_ollama_pt_simple_messages(): @@ -550,6 +553,175 @@ def test_convert_gemini_tool_call_result_with_image_url(): assert isinstance(result2, list) and any("inline_data" in p for p in result2) +def test_convert_gemini_tool_call_result_with_anthropic_image_block(): + """ + Test that Anthropic-native image blocks in tool_result list content are + converted to Gemini inline_data instead of being silently dropped. + Fixes: https://github.com/BerriAI/litellm/issues/23712 + """ + tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode() + + message = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_123", + content=[ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": tiny_png_b64, + }, + } + ], + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_123", + "type": "function", + "index": 0, + "function": {"name": "read_file", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result, list), "expected a list of parts" + inline_parts = [p for p in result if "inline_data" in p] + assert len(inline_parts) == 1, "expected exactly one inline_data part" + assert inline_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64 + + +def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks(): + """ + Test that multiple Anthropic-native image blocks in a single tool_result + are all preserved as separate inline_data parts instead of only the last + one being kept. + Fixes: https://github.com/BerriAI/litellm/issues/23712 + """ + png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode() + jpeg_b64 = base64.b64encode(b"JPEG_PLACEHOLDER").decode() + + message = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_multi", + content=[ + {"type": "text", "text": "here are two images"}, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": png_b64}, + }, + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": jpeg_b64}, + }, + ], + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_multi", + "type": "function", + "index": 0, + "function": {"name": "screenshot", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result, list), "expected a list of parts" + inline_parts = [p for p in result if "inline_data" in p] + assert len(inline_parts) == 2, f"expected 2 inline_data parts, got {len(inline_parts)}" + mime_types = {p["inline_data"]["mime_type"] for p in inline_parts} + assert mime_types == {"image/png", "image/jpeg"} + + +def test_convert_gemini_tool_call_result_with_data_url_string(): + """ + Test that a data-URL string in tool_result content is converted to + Gemini inline_data instead of being passed as plain text. + Fixes: https://github.com/BerriAI/litellm/issues/23712 + """ + tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode() + + message = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_456", + content=f"data:image/png;base64,{tiny_png_b64}", + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_456", + "type": "function", + "index": 0, + "function": {"name": "read_file", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result, list), "expected a list of parts" + inline_parts = [p for p in result if "inline_data" in p] + assert len(inline_parts) == 1, "data-URL image string was not converted to inline_data" + assert inline_parts[0]["inline_data"]["mime_type"] == "image/png" + assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64 + + +def test_convert_gemini_tool_call_result_with_data_url_extra_params(): + """ + Test that a data-URL with extra MIME parameters (e.g. charset) produces + a clean mime_type without the extra parameters. + """ + tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode() + + message = ChatCompletionToolMessage( + role="tool", + tool_call_id="call_extra", + content=f"data:image/png;charset=UTF-8;base64,{tiny_png_b64}", + ) + last_message_with_tool_calls = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": "call_extra", + "type": "function", + "index": 0, + "function": {"name": "read_file", "arguments": "{}"}, + } + ], + } + + result = convert_to_gemini_tool_call_result( + message=message, + last_message_with_tool_calls=last_message_with_tool_calls, + ) + assert isinstance(result, list), "expected a list of parts" + inline_parts = [p for p in result if "inline_data" in p] + assert len(inline_parts) == 1 + assert inline_parts[0]["inline_data"]["mime_type"] == "image/png", ( + f"expected clean 'image/png', got '{inline_parts[0]['inline_data']['mime_type']}'" + ) + + def test_bedrock_tools_unpack_defs(): """ Test that the unpack_defs method handles nested $ref inside anyOf items correctly