fix(gemini): support images in tool_results for /v1/messages routing (#23724)

* fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(gemini): support images in tool_results for /v1/messages routing convert_to_gemini_tool_call_result() dropped images in two cases: - data-URL strings (data:image/...;base64,...) treated as plain text - Anthropic image blocks in list content skipped Add detection and convert both to Gemini inline_data BlobType so image bytes are preserved. Fixes #23712. * fix(fireworks): skip #transform=inline for base64 data URLs Closes #23583
2026-08-16 14:23:12 +00:00 · 2026-03-16 22:38:16 -07:00
parent e9291a97c3
commit 186c2adb32
2 changed files with 219 additions and 12 deletions
@@ -1498,17 +1498,49 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
    from litellm.types.llms.vertex_ai import BlobType

    content_str: str = ""
-    inline_data: Optional[BlobType] = None
+    inline_data_list: List[BlobType] = []

    if "content" in message:
        if isinstance(message["content"], str):
            content_str = message["content"]
+            # Detect data-URL images (e.g. from Anthropic tool_result with a single image block
+            # that was serialised as a plain string by translate_anthropic_messages_to_openai)
+            # and promote them to inline_data so Gemini receives actual image bytes.
+            if content_str.startswith("data:") and ";base64," in content_str:
+                try:
+                    mime_rest = content_str[5:].split(";base64,", 1)
+                    if len(mime_rest) == 2 and mime_rest[0].startswith("image/"):
+                        # Strip any extra parameters (e.g. ";charset=UTF-8") from the MIME segment
+                        clean_mime = mime_rest[0].split(";")[0].strip()
+                        inline_data_list.append(
+                            BlobType(data=mime_rest[1], mime_type=clean_mime)
+                        )
+                        content_str = ""
+                except Exception as e:
+                    verbose_logger.warning(
+                        f"Failed to parse data URL in tool response: {e}"
+                    )
        elif isinstance(message["content"], List):
            content_list = message["content"]
            for content in content_list:
                content_type = content.get("type", "")
                if content_type == "text":
                    content_str += content.get("text", "")
+                elif content_type == "image":
+                    # Anthropic-native image block: {"type": "image", "source": {"type": "base64", ...}}
+                    source = content.get("source", {})
+                    if isinstance(source, dict) and source.get("type") == "base64":
+                        try:
+                            inline_data_list.append(
+                                BlobType(
+                                    data=source.get("data", ""),
+                                    mime_type=source.get("media_type", "image/jpeg"),
+                                )
+                            )
+                        except Exception as e:
+                            verbose_logger.warning(
+                                f"Failed to process Anthropic image block in tool response: {e}"
+                            )
                elif content_type in ("input_image", "image_url"):
                    # Extract image for inline_data (for Computer Use screenshots and tool results)
                    image_url_data = content.get("image_url", "")
@@ -1524,9 +1556,11 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
                            image_obj = convert_to_anthropic_image_obj(
                                image_url, format=None
                            )
-                            inline_data = BlobType(
-                                data=image_obj["data"],
-                                mime_type=image_obj["media_type"],
+                            inline_data_list.append(
+                                BlobType(
+                                    data=image_obj["data"],
+                                    mime_type=image_obj["media_type"],
+                                )
                            )
                        except Exception as e:
                            verbose_logger.warning(
@@ -1551,9 +1585,11 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
                            file_obj = convert_to_anthropic_image_obj(
                                file_data, format=None
                            )
-                            inline_data = BlobType(
-                                data=file_obj["data"],
-                                mime_type=file_obj["media_type"],
+                            inline_data_list.append(
+                                BlobType(
+                                    data=file_obj["data"],
+                                    mime_type=file_obj["media_type"],
+                                )
                            )
                        except Exception as e:
                            verbose_logger.warning(
@@ -1607,13 +1643,12 @@ def convert_to_gemini_tool_call_result(  # noqa: PLR0915
    # Create part with function_response, and optionally inline_data for images (Computer Use)
    _part: VertexPartType = {"function_response": _function_response}

-    # For Computer Use, if we have an image, we need separate parts:
+    # For Computer Use, if we have images/files, we need separate parts:
    # - One part with function_response
-    # - One part with inline_data
+    # - One part per inline_data item
    # Gemini's PartType is a oneof, so we can't have both in the same part
-    if inline_data:
-        image_part: VertexPartType = {"inline_data": inline_data}
-        return [_part, image_part]
+    if inline_data_list:
+        return [_part] + [{"inline_data": d} for d in inline_data_list]

    return _part

@@ -1,3 +1,4 @@
+import base64
 import json
 from unittest.mock import MagicMock, patch

@@ -9,9 +10,11 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
    BedrockConverseMessagesProcessor,
    BedrockImageProcessor,
    _convert_to_bedrock_tool_call_invoke,
+    convert_to_gemini_tool_call_result,
    ollama_pt,
    sanitize_messages_for_tool_calling,
 )
+from litellm.types.llms.openai import ChatCompletionToolMessage


 def test_ollama_pt_simple_messages():
@@ -550,6 +553,175 @@ def test_convert_gemini_tool_call_result_with_image_url():
    assert isinstance(result2, list) and any("inline_data" in p for p in result2)


+def test_convert_gemini_tool_call_result_with_anthropic_image_block():
+    """
+    Test that Anthropic-native image blocks in tool_result list content are
+    converted to Gemini inline_data instead of being silently dropped.
+    Fixes: https://github.com/BerriAI/litellm/issues/23712
+    """
+    tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode()
+
+    message = ChatCompletionToolMessage(
+        role="tool",
+        tool_call_id="call_123",
+        content=[
+            {
+                "type": "image",
+                "source": {
+                    "type": "base64",
+                    "media_type": "image/png",
+                    "data": tiny_png_b64,
+                },
+            }
+        ],
+    )
+    last_message_with_tool_calls = {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+            {
+                "id": "call_123",
+                "type": "function",
+                "index": 0,
+                "function": {"name": "read_file", "arguments": "{}"},
+            }
+        ],
+    }
+
+    result = convert_to_gemini_tool_call_result(
+        message=message,
+        last_message_with_tool_calls=last_message_with_tool_calls,
+    )
+    assert isinstance(result, list), "expected a list of parts"
+    inline_parts = [p for p in result if "inline_data" in p]
+    assert len(inline_parts) == 1, "expected exactly one inline_data part"
+    assert inline_parts[0]["inline_data"]["mime_type"] == "image/png"
+    assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64
+
+
+def test_convert_gemini_tool_call_result_with_multiple_anthropic_image_blocks():
+    """
+    Test that multiple Anthropic-native image blocks in a single tool_result
+    are all preserved as separate inline_data parts instead of only the last
+    one being kept.
+    Fixes: https://github.com/BerriAI/litellm/issues/23712
+    """
+    png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode()
+    jpeg_b64 = base64.b64encode(b"JPEG_PLACEHOLDER").decode()
+
+    message = ChatCompletionToolMessage(
+        role="tool",
+        tool_call_id="call_multi",
+        content=[
+            {"type": "text", "text": "here are two images"},
+            {
+                "type": "image",
+                "source": {"type": "base64", "media_type": "image/png", "data": png_b64},
+            },
+            {
+                "type": "image",
+                "source": {"type": "base64", "media_type": "image/jpeg", "data": jpeg_b64},
+            },
+        ],
+    )
+    last_message_with_tool_calls = {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+            {
+                "id": "call_multi",
+                "type": "function",
+                "index": 0,
+                "function": {"name": "screenshot", "arguments": "{}"},
+            }
+        ],
+    }
+
+    result = convert_to_gemini_tool_call_result(
+        message=message,
+        last_message_with_tool_calls=last_message_with_tool_calls,
+    )
+    assert isinstance(result, list), "expected a list of parts"
+    inline_parts = [p for p in result if "inline_data" in p]
+    assert len(inline_parts) == 2, f"expected 2 inline_data parts, got {len(inline_parts)}"
+    mime_types = {p["inline_data"]["mime_type"] for p in inline_parts}
+    assert mime_types == {"image/png", "image/jpeg"}
+
+
+def test_convert_gemini_tool_call_result_with_data_url_string():
+    """
+    Test that a data-URL string in tool_result content is converted to
+    Gemini inline_data instead of being passed as plain text.
+    Fixes: https://github.com/BerriAI/litellm/issues/23712
+    """
+    tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode()
+
+    message = ChatCompletionToolMessage(
+        role="tool",
+        tool_call_id="call_456",
+        content=f"data:image/png;base64,{tiny_png_b64}",
+    )
+    last_message_with_tool_calls = {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+            {
+                "id": "call_456",
+                "type": "function",
+                "index": 0,
+                "function": {"name": "read_file", "arguments": "{}"},
+            }
+        ],
+    }
+
+    result = convert_to_gemini_tool_call_result(
+        message=message,
+        last_message_with_tool_calls=last_message_with_tool_calls,
+    )
+    assert isinstance(result, list), "expected a list of parts"
+    inline_parts = [p for p in result if "inline_data" in p]
+    assert len(inline_parts) == 1, "data-URL image string was not converted to inline_data"
+    assert inline_parts[0]["inline_data"]["mime_type"] == "image/png"
+    assert inline_parts[0]["inline_data"]["data"] == tiny_png_b64
+
+
+def test_convert_gemini_tool_call_result_with_data_url_extra_params():
+    """
+    Test that a data-URL with extra MIME parameters (e.g. charset) produces
+    a clean mime_type without the extra parameters.
+    """
+    tiny_png_b64 = base64.b64encode(b"PNG_PLACEHOLDER").decode()
+
+    message = ChatCompletionToolMessage(
+        role="tool",
+        tool_call_id="call_extra",
+        content=f"data:image/png;charset=UTF-8;base64,{tiny_png_b64}",
+    )
+    last_message_with_tool_calls = {
+        "role": "assistant",
+        "content": "",
+        "tool_calls": [
+            {
+                "id": "call_extra",
+                "type": "function",
+                "index": 0,
+                "function": {"name": "read_file", "arguments": "{}"},
+            }
+        ],
+    }
+
+    result = convert_to_gemini_tool_call_result(
+        message=message,
+        last_message_with_tool_calls=last_message_with_tool_calls,
+    )
+    assert isinstance(result, list), "expected a list of parts"
+    inline_parts = [p for p in result if "inline_data" in p]
+    assert len(inline_parts) == 1
+    assert inline_parts[0]["inline_data"]["mime_type"] == "image/png", (
+        f"expected clean 'image/png', got '{inline_parts[0]['inline_data']['mime_type']}'"
+    )
+
+
 def test_bedrock_tools_unpack_defs():
    """
    Test that the unpack_defs method handles nested $ref inside anyOf items correctly