fix(vertex_ai): convert image URLs to base64 for Vertex AI Anthropic

Fixes #18430

  - Pass custom_llm_provider to anthropic_messages_pt instead of hardcoded 'anthropic'
  - Add check for vertex_ai provider to force base64 conversion for image URLs
  - Add tests to verify behavior for both Vertex AI and regular Anthropic

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -930,7 +930,8 @@ def create_anthropic_image_param(
     
     # Check if the image URL is an HTTP/HTTPS URL
     if image_url.startswith("http://") or image_url.startswith("https://"):
-        # For Bedrock invoke, always convert URLs to base64 (Bedrock invoke doesn't support URLs)
+        # For Bedrock invoke and Vertex AI Anthropic, always convert URLs to base64
+        # as these providers don't support URL sources for images
         if is_bedrock_invoke or image_url.startswith("http://"):
             base64_url = convert_url_to_base64(url=image_url)
             image_chunk = convert_to_anthropic_image_obj(
@@ -1914,9 +1915,12 @@ def anthropic_messages_pt(  # noqa: PLR0915
                                     "format": image_url_value.get("format"),
                                 }
                             # Bedrock invoke models have format: invoke/...
+                            # Vertex AI Anthropic also doesn't support URL sources for images
                             is_bedrock_invoke = model.lower().startswith("invoke/")
+                            is_vertex_ai = llm_provider.startswith("vertex_ai") if llm_provider else False
+                            force_base64 = is_bedrock_invoke or is_vertex_ai
                             _anthropic_content_element = create_anthropic_image_param(
-                                image_url_input, format=format, is_bedrock_invoke=is_bedrock_invoke
+                                image_url_input, format=format, is_bedrock_invoke=force_base64
                             ) 
                             _content_element = add_cache_control_to_content(
                                 anthropic_content_element=_anthropic_content_element,

litellm/llms/anthropic/chat/transformation.py

@@ -994,7 +994,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             anthropic_messages = anthropic_messages_pt(
                 model=model,
                 messages=messages,
-                llm_provider="anthropic",
+                llm_provider=self.custom_llm_provider or "anthropic",
             )
         except Exception as e:
             raise AnthropicError(

tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_anthropic_image_url_handling.py

@@ -0,0 +1,179 @@
+"""
+Tests for Vertex AI Anthropic image URL handling.
+
+Issue: https://github.com/BerriAI/litellm/issues/18430
+Vertex AI Anthropic models don't support URL sources for images.
+LiteLLM should convert image URLs to base64 when using Vertex AI Anthropic.
+"""
+import os
+import sys
+from unittest.mock import patch, MagicMock
+
+import pytest
+
+sys.path.insert(
+    0, os.path.abspath("../../../../../..")
+)  # Adds the parent directory to the system path
+
+from litellm.litellm_core_utils.prompt_templates.factory import (
+    anthropic_messages_pt,
+    create_anthropic_image_param,
+)
+
+
+class TestVertexAIAnthropicImageURLHandling:
+    """Test that Vertex AI Anthropic converts image URLs to base64."""
+
+    @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+    def test_vertex_ai_anthropic_converts_https_url_to_base64(
+        self, mock_convert_url: MagicMock
+    ):
+        """
+        Test that HTTPS image URLs are converted to base64 for Vertex AI Anthropic.
+
+        For regular Anthropic, HTTPS URLs are passed through as URL type.
+        For Vertex AI Anthropic, HTTPS URLs should be converted to base64.
+        """
+        mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRgABAQAAAQ=="
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Describe this image"},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": "https://example.com/image.jpg"},
+                    },
+                ],
+            }
+        ]
+
+        # For Vertex AI, image URLs should be converted to base64
+        result = anthropic_messages_pt(
+            messages=messages,
+            model="claude-sonnet-4",
+            llm_provider="vertex_ai",
+        )
+
+        # Verify convert_url_to_base64 was called
+        mock_convert_url.assert_called_once_with(url="https://example.com/image.jpg")
+
+        # Check the result has base64 source type
+        user_message = result[0]
+        assert user_message["role"] == "user"
+        image_content = user_message["content"][1]
+        assert image_content["type"] == "image"
+        assert image_content["source"]["type"] == "base64"
+
+    @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+    def test_regular_anthropic_uses_url_type_for_https(
+        self, mock_convert_url: MagicMock
+    ):
+        """
+        Test that regular Anthropic API uses URL type for HTTPS images.
+
+        This confirms the original behavior is preserved for non-Vertex AI.
+        """
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "Describe this image"},
+                    {
+                        "type": "image_url",
+                        "image_url": {"url": "https://example.com/image.jpg"},
+                    },
+                ],
+            }
+        ]
+
+        # For regular Anthropic, HTTPS URLs should NOT be converted
+        result = anthropic_messages_pt(
+            messages=messages,
+            model="claude-sonnet-4",
+            llm_provider="anthropic",
+        )
+
+        # convert_url_to_base64 should NOT be called for regular Anthropic with HTTPS
+        mock_convert_url.assert_not_called()
+
+        # Check the result has URL source type
+        user_message = result[0]
+        assert user_message["role"] == "user"
+        image_content = user_message["content"][1]
+        assert image_content["type"] == "image"
+        assert image_content["source"]["type"] == "url"
+        assert image_content["source"]["url"] == "https://example.com/image.jpg"
+
+    @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+    def test_vertex_ai_beta_also_converts_to_base64(
+        self, mock_convert_url: MagicMock
+    ):
+        """
+        Test that vertex_ai_beta provider also converts image URLs to base64.
+        """
+        mock_convert_url.return_value = "data:image/png;base64,iVBORw0KGgo="
+
+        messages = [
+            {
+                "role": "user",
+                "content": [
+                    {"type": "text", "text": "What is in this image?"},
+                    {
+                        "type": "image_url",
+                        "image_url": "https://example.com/photo.png",
+                    },
+                ],
+            }
+        ]
+
+        result = anthropic_messages_pt(
+            messages=messages,
+            model="claude-3-opus",
+            llm_provider="vertex_ai_beta",
+        )
+
+        # Verify convert_url_to_base64 was called
+        mock_convert_url.assert_called_once()
+
+        # Check the result has base64 source type
+        user_message = result[0]
+        image_content = user_message["content"][1]
+        assert image_content["source"]["type"] == "base64"
+
+
+class TestCreateAnthropicImageParam:
+    """Test the create_anthropic_image_param function directly."""
+
+    @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+    def test_force_base64_converts_https_url(self, mock_convert_url: MagicMock):
+        """
+        Test that is_bedrock_invoke=True (used for both Bedrock and Vertex AI)
+        forces conversion of HTTPS URLs to base64.
+        """
+        mock_convert_url.return_value = "data:image/jpeg;base64,/9j/4AAQSkZJRg=="
+
+        result = create_anthropic_image_param(
+            image_url_input="https://example.com/image.jpg",
+            format=None,
+            is_bedrock_invoke=True,  # This flag is set for both Bedrock and Vertex AI
+        )
+
+        mock_convert_url.assert_called_once_with(url="https://example.com/image.jpg")
+        assert result["source"]["type"] == "base64"
+
+    @patch("litellm.litellm_core_utils.prompt_templates.factory.convert_url_to_base64")
+    def test_no_force_uses_url_type(self, mock_convert_url: MagicMock):
+        """
+        Test that without force, HTTPS URLs use URL type.
+        """
+        result = create_anthropic_image_param(
+            image_url_input="https://example.com/image.jpg",
+            format=None,
+            is_bedrock_invoke=False,
+        )
+
+        mock_convert_url.assert_not_called()
+        assert result["source"]["type"] == "url"
+        assert result["source"]["url"] == "https://example.com/image.jpg"