[Fix] - Reliability fix OOMs with image url handling (#19257)

* fix MAX_IMAGE_URL_DOWNLOAD_SIZE_MB

* test_image_exceeds_size_limit_with_content_length

* fix: _process_image_response

* add constants 50MB

* fix convert_to_anthropic_image_obj image handling

* test_gemini_image_size_limit_exceeded

* MAX_IMAGE_URL_DOWNLOAD_SIZE_MB fix

* MAX_IMAGE_URL_DOWNLOAD_SIZE_MB

* test_image_size_limit_disabled

* async_convert_url_to_base64

* docs fix

* code QA check

* fix Exception

docs/my-website/docs/proxy/config_settings.md

@@ -755,6 +755,7 @@ router_settings:
 | LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS | Cooldown time in seconds before allowing another aggressive clear operation when the queue is full. Default is 0.5 
 | MAX_STRING_LENGTH_PROMPT_IN_DB | Maximum length for strings in spend logs when sanitizing request bodies. Strings longer than this will be truncated. Default is 1000
 | MAX_IN_MEMORY_QUEUE_FLUSH_COUNT | Maximum count for in-memory queue flush operations. Default is 1000
+| MAX_IMAGE_URL_DOWNLOAD_SIZE_MB | Maximum size in MB for downloading images from URLs. Prevents memory issues from downloading very large images. Images exceeding this limit will be rejected before download. Set to 0 to completely disable image URL handling (all image_url requests will be blocked). Default is 50MB (matching [OpenAI's limit](https://platform.openai.com/docs/guides/images-vision?api-mode=chat#image-input-requirements))
 | MAX_LONG_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the long side of high-resolution images. Default is 2000
 | MAX_REDIS_BUFFER_DEQUEUE_COUNT | Maximum count for Redis buffer dequeue operations. Default is 100
 | MAX_SHORT_SIDE_FOR_IMAGE_HIGH_RES | Maximum length for the short side of high-resolution images. Default is 768

litellm/constants.py

@@ -48,6 +48,11 @@ DEFAULT_REPLICATE_POLLING_DELAY_SECONDS = int(
 DEFAULT_IMAGE_TOKEN_COUNT = int(os.getenv("DEFAULT_IMAGE_TOKEN_COUNT", 250))
 DEFAULT_IMAGE_WIDTH = int(os.getenv("DEFAULT_IMAGE_WIDTH", 300))
 DEFAULT_IMAGE_HEIGHT = int(os.getenv("DEFAULT_IMAGE_HEIGHT", 300))
+# Maximum size for image URL downloads in MB (default 50MB, set to 0 to disable limit)
+# This prevents memory issues from downloading very large images
+# Maps to OpenAI's 50 MB payload limit - requests with images exceeding this size will be rejected
+# Set MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0 to disable image URL handling entirely
+MAX_IMAGE_URL_DOWNLOAD_SIZE_MB = float(os.getenv("MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 50))
 MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
     os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 1024)
 )  # 1MB = 1024KB

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -903,11 +903,11 @@ def convert_to_anthropic_image_obj(
             media_type=media_type,
             data=base64_data,
         )
+    except litellm.ImageFetchError:
+        raise
     except Exception as e:
-        if "Error: Unable to fetch image from URL" in str(e):
-            raise e
         raise Exception(
-            """Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{base64_image}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']."""
+            f"""Image url not in expected format. Example Expected input - "image_url": "data:image/jpeg;base64,{{base64_image}}". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {str(e)}"""
         )
 
 
@@ -1555,8 +1555,6 @@ def convert_to_gemini_tool_call_result(
     # For Computer Use, the response should contain structured data like {"url": "..."}
     response_data: dict
     try:
-        import json
-
         if content_str.strip().startswith("{") or content_str.strip().startswith("["):
             # Try to parse as JSON (for Computer Use structured responses)
             parsed = json.loads(content_str)
@@ -1672,7 +1670,7 @@ def convert_to_anthropic_tool_result(
                     anthropic_content_element=_anthropic_image_param,
                     original_content_element=content,
                 )
-                anthropic_content_list.append(_anthropic_image_param)
+                anthropic_content_list.append(cast(AnthropicMessagesImageParam, _anthropic_image_param))
 
         anthropic_content = anthropic_content_list
     anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None

litellm/litellm_core_utils/prompt_templates/image_handling.py

@@ -9,6 +9,7 @@ from httpx import Response
 import litellm
 from litellm import verbose_logger
 from litellm.caching.caching import InMemoryCache
+from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB
 
 MAX_IMGS_IN_MEMORY = 10
 
@@ -21,7 +22,25 @@ def _process_image_response(response: Response, url: str) -> str:
             f"Error: Unable to fetch image from URL. Status code: {response.status_code}, url={url}"
         )
 
+    # Check size before downloading if Content-Length header is present
+    content_length = response.headers.get("Content-Length")
+    if content_length is not None:
+        size_mb = int(content_length) / (1024 * 1024)
+        if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
+            raise litellm.ImageFetchError(
+                f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
+            )
+
     image_bytes = response.content
+    
+    # Check actual size after download if Content-Length was not available
+    if content_length is None:
+        size_mb = len(image_bytes) / (1024 * 1024)
+        if size_mb > MAX_IMAGE_URL_DOWNLOAD_SIZE_MB:
+            raise litellm.ImageFetchError(
+                f"Error: Image size ({size_mb:.2f}MB) exceeds maximum allowed size ({MAX_IMAGE_URL_DOWNLOAD_SIZE_MB}MB). url={url}"
+            )
+    
     base64_image = base64.b64encode(image_bytes).decode("utf-8")
 
     image_type = response.headers.get("Content-Type")
@@ -48,6 +67,12 @@ def _process_image_response(response: Response, url: str) -> str:
 
 
 async def async_convert_url_to_base64(url: str) -> str:
+    # If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        raise litellm.ImageFetchError(
+            f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
+        )
+
     cached_result = in_memory_cache.get_cache(url)
     if cached_result:
         return cached_result
@@ -67,6 +92,12 @@ async def async_convert_url_to_base64(url: str) -> str:
 
 
 def convert_url_to_base64(url: str) -> str:
+    # If MAX_IMAGE_URL_DOWNLOAD_SIZE_MB is 0, block all image downloads
+    if MAX_IMAGE_URL_DOWNLOAD_SIZE_MB == 0:
+        raise litellm.ImageFetchError(
+            f"Error: Image URL download is disabled (MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0). url={url}"
+        )
+
     cached_result = in_memory_cache.get_cache(url)
     if cached_result:
         return cached_result

litellm/proxy/proxy_config.yaml

@@ -1,4 +1,7 @@
 model_list:
+  - model_name: gemini/*
+    litellm_params:
+      model: gemini/*
   - model_name: claude-sonnet-4-5-20250929
     litellm_params:
       model: bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0

tests/llm_translation/test_gemini.py

@@ -1401,3 +1401,37 @@ def test_anthropic_thinking_param_via_map_openai_params():
     assert "thinkingLevel" not in thinking_config_2, "Should NOT have thinkingLevel for Gemini 2"
     assert thinking_config_2["includeThoughts"] is True
     assert thinking_config_2["thinkingBudget"] == 10000
+
+
+def test_gemini_image_size_limit_exceeded():
+    """
+    Test that large images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected.
+    
+    This validates that the 50MB default limit prevents downloading very large images
+    that could cause memory issues and pod crashes.
+    """
+    messages = [
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": "What is in this image?"
+                },
+                {
+                    "type": "image_url",
+                    "image_url": "https://upload.wikimedia.org/wikipedia/commons/5/51/Blue_Marble_2002.jpg"
+                }
+            ]
+        }
+    ]
+    
+    with pytest.raises(litellm.ImageFetchError) as excinfo:
+        completion(
+            model="gemini/gemini-2.5-flash-lite",
+            messages=messages
+        )
+    
+    error_message = str(excinfo.value)
+    assert "Image size" in error_message
+    assert "exceeds maximum allowed size" in error_message

tests/test_litellm/litellm_core_utils/test_image_handling.py

@@ -1,7 +1,10 @@
+from unittest.mock import patch
+
 import pytest
 from httpx import Request, Response
 
 import litellm
+from litellm import constants
 from litellm.litellm_core_utils.prompt_templates.image_handling import (
     convert_url_to_base64,
 )
@@ -39,3 +42,99 @@ def test_completion_with_invalid_image_url(monkeypatch):
         )
     assert excinfo.value.status_code == 400
     assert "Unable to fetch image" in str(excinfo.value)
+
+
+class LargeImageClient:
+    """
+    Client that returns a large image exceeding size limit.
+    """
+
+    def __init__(self, size_mb=100, include_content_length=True):
+        self.size_mb = size_mb
+        self.include_content_length = include_content_length
+
+    def get(self, url, follow_redirects=True):
+        size_bytes = int(self.size_mb * 1024 * 1024)
+        headers = {"Content-Type": "image/jpeg"}
+        if self.include_content_length:
+            headers["Content-Length"] = str(size_bytes)
+        return Response(
+            status_code=200,
+            headers=headers,
+            content=b"x" * size_bytes,
+            request=Request("GET", url),
+        )
+
+
+def test_image_exceeds_size_limit_with_content_length(monkeypatch):
+    """
+    Test that images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected when Content-Length header is present.
+    """
+    monkeypatch.setattr(litellm, "module_level_client", LargeImageClient(size_mb=100))
+
+    with pytest.raises(litellm.ImageFetchError) as excinfo:
+        convert_url_to_base64("https://example.com/large-image.jpg")
+
+    assert "exceeds maximum allowed size" in str(excinfo.value)
+    assert "100.00MB" in str(excinfo.value)
+    assert "50.0MB" in str(excinfo.value)
+
+
+def test_image_exceeds_size_limit_without_content_length(monkeypatch):
+    """
+    Test that images exceeding MAX_IMAGE_URL_DOWNLOAD_SIZE_MB are rejected even without Content-Length header.
+    """
+    monkeypatch.setattr(
+        litellm, "module_level_client", LargeImageClient(size_mb=100, include_content_length=False)
+    )
+
+    with pytest.raises(litellm.ImageFetchError) as excinfo:
+        convert_url_to_base64("https://example.com/large-image.jpg")
+
+    assert "exceeds maximum allowed size" in str(excinfo.value)
+
+
+class SmallImageClient:
+    """
+    Client that returns a small valid image.
+    """
+
+    def get(self, url, follow_redirects=True):
+        size_bytes = 1024
+        headers = {
+            "Content-Type": "image/jpeg",
+            "Content-Length": str(size_bytes),
+        }
+        return Response(
+            status_code=200,
+            headers=headers,
+            content=b"x" * size_bytes,
+            request=Request("GET", url),
+        )
+
+
+def test_image_within_size_limit(monkeypatch):
+    """
+    Test that images within size limit are processed successfully.
+    """
+    monkeypatch.setattr(litellm, "module_level_client", SmallImageClient())
+
+    result = convert_url_to_base64("https://example.com/small-image.jpg")
+
+    assert result.startswith("data:image/jpeg;base64,")
+
+
+def test_image_size_limit_disabled(monkeypatch):
+    """
+    Test that setting MAX_IMAGE_URL_DOWNLOAD_SIZE_MB to 0 disables all image URL downloads.
+    """
+    import litellm.litellm_core_utils.prompt_templates.image_handling as image_handling
+
+    monkeypatch.setattr(litellm, "module_level_client", SmallImageClient())
+    monkeypatch.setattr(image_handling, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", 0)
+
+    with pytest.raises(litellm.ImageFetchError) as excinfo:
+        convert_url_to_base64("https://example.com/image.jpg")
+    
+    assert "Image URL download is disabled" in str(excinfo.value)
+    assert "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB=0" in str(excinfo.value)