From 3ca985451e5a416b33595a0031187cc7aa629fd0 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Mon, 27 Apr 2026 23:37:09 -0700
Subject: [PATCH 01/19] fix(vertex): preserve items on array branches inside
 anyOf with null

convert_anyof_null_to_nullable was stripping the items field from array
branches inside anyOf when a sibling null branch was present, leaving
{"type": "array"} without items. Vertex requires items whenever
type == "array" (even inside anyOf) and rejects the call with
INVALID_ARGUMENT.

Leave the (possibly empty) items in place so the downstream process_items
step can convert {} to {"type": "object"}, which is what Vertex wants.

Also:
- Update test_build_vertex_schema expected output, which was codifying
  the broken shape.
- Convert test_gemini_tool_calling_not_working to a hermetic mock test
  that asserts the request body sent to Vertex includes items inside
  the callbacks anyOf array branch. The previous form made a real
  network call and was flaky in CI.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 litellm/llms/vertex_ai/common_utils.py        | 10 +--
 .../test_amazing_vertex_completion.py         | 70 +++++++++++++++++--
 .../vertex_ai/test_vertex_ai_common_utils.py  |  6 +-
 3 files changed, 74 insertions(+), 12 deletions(-)

diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py
index ccd4d4f293..9b23520dcd 100644
--- a/litellm/llms/vertex_ai/common_utils.py
+++ b/litellm/llms/vertex_ai/common_utils.py
@@ -710,14 +710,10 @@ def convert_anyof_null_to_nullable(schema, depth=0):
 
         if contains_null:
             # set all types to nullable following guidance found here: https://cloud.google.com/vertex-ai/generative-ai/docs/samples/generativeaionvertexai-gemini-controlled-generation-response-schema-3#generativeaionvertexai_gemini_controlled_generation_response_schema_3-python
+            # Empty `items: {}` on array branches is left in place; downstream
+            # process_items() converts it to {"type": "object"}, which Vertex
+            # requires whenever type == "array" (even inside anyOf).
             for atype in anyof:
-                # Remove items field if type is array and items is empty
-                if (
-                    atype.get("type") == "array"
-                    and "items" in atype
-                    and not atype["items"]
-                ):
-                    atype.pop("items")
                 atype["nullable"] = True
 
     properties = schema.get("properties", None)
diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py
index 9070a9feab..3b4ecb82b1 100644
--- a/tests/local_testing/test_amazing_vertex_completion.py
+++ b/tests/local_testing/test_amazing_vertex_completion.py
@@ -3569,8 +3569,14 @@ def test_gemini_tool_calling_working_demo():
 
 
 def test_gemini_tool_calling_not_working():
-    load_vertex_ai_credentials()
-    litellm._turn_on_debug()
+    """
+    Regression test: tool params with anyOf containing both an empty-items
+    array branch and a null branch must serialize with items present on the
+    array branch (Vertex rejects array types missing `items`).
+    """
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+    from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+
     args = {
         "messages": [
             {
@@ -3637,8 +3643,64 @@ def test_gemini_tool_calling_not_working():
         ],
         "vertex_location": "global",
     }
-    response = completion(model="vertex_ai/gemini-3-flash-preview", **args)
-    print(response)
+
+    client = HTTPHandler()
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.headers = {"Content-Type": "application/json"}
+    mock_response.json.return_value = {
+        "candidates": [
+            {
+                "content": {
+                    "role": "model",
+                    "parts": [{"text": "Hello!"}],
+                },
+                "finishReason": "STOP",
+            }
+        ],
+        "usageMetadata": {
+            "promptTokenCount": 10,
+            "candidatesTokenCount": 5,
+            "totalTokenCount": 15,
+        },
+    }
+
+    with (
+        patch.object(client, "post", return_value=mock_response) as mock_post,
+        patch.object(
+            VertexBase,
+            "_ensure_access_token",
+            return_value=("fake-token", "fake-project"),
+        ),
+    ):
+        completion(
+            model="vertex_ai/gemini-3-flash-preview",
+            client=client,
+            **args,
+        )
+
+    sent_body = mock_post.call_args.kwargs.get(
+        "json"
+    ) or mock_post.call_args.kwargs.get("data")
+    assert sent_body is not None, "expected request body to be sent"
+    if isinstance(sent_body, str):
+        sent_body = json.loads(sent_body)
+
+    function_decl = sent_body["tools"][0]["function_declarations"][0]
+    callbacks_schema = function_decl["parameters"]["properties"]["config"][
+        "properties"
+    ]["callbacks"]
+    array_branches = [
+        branch
+        for branch in callbacks_schema["anyOf"]
+        if branch.get("type", "").lower() == "array"
+    ]
+    assert array_branches, "expected an array branch in callbacks anyOf"
+    for branch in array_branches:
+        assert "items" in branch and branch["items"], (
+            f"array branch in callbacks.anyOf must include non-empty items "
+            f"(Vertex rejects array types missing items). Got: {branch}"
+        )
 
 
 def test_vertex_ai_llama_tool_calling():
diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
index ef93375c3c..dc3be7114f 100644
--- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
+++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
@@ -225,7 +225,11 @@ def test_build_vertex_schema():
                     "metadata": {"type": "object"},
                     "callbacks": {
                         "anyOf": [
-                            {"type": "array", "nullable": True},
+                            {
+                                "type": "array",
+                                "items": {"type": "object"},
+                                "nullable": True,
+                            },
                             {"type": "object", "nullable": True},
                         ]
                     },

From 0dd64baa669aef52738f1d628982537707d29e95 Mon Sep 17 00:00:00 2001
From: michelligabriele <gabriele.michelli@icloud.com>
Date: Tue, 28 Apr 2026 17:25:11 +0200
Subject: [PATCH 02/19] fix(caching): preserve prompt_tokens_details through
 embedding cache round-trip (#26653)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix(caching): preserve prompt_tokens_details through embedding cache round-trip

The embedding caching layer was dropping prompt_tokens_details (including
image_count) because CachedEmbedding had no field for usage metadata and
the cache retrieval code reconstructed Usage without it. This caused
inconsistent responses where the first call returned image_count but
cached responses did not, breaking cost tracking for multimodal embeddings.

Add prompt_tokens_details to CachedEmbedding, persist per-item details
during cache storage, aggregate them on retrieval, and merge them in
combine_usage() for partial cache hits.

* style: apply Black formatting to caching files

* fix(caching): address Greptile review — cyclic import, guarded construction, nested dict merge

Move PromptTokensDetailsWrapper to inline import to resolve CodeQL cyclic
import warning. Guard PromptTokensDetailsWrapper construction with
try/except to handle unexpected cached keys. Add recursive dict merging
in _merge_prompt_tokens_details for nested fields like
cache_creation_token_details.
---
 litellm/caching/caching.py                    |  61 +++++-
 litellm/caching/caching_handler.py            |  88 +++++++++
 litellm/types/caching.py                      |   1 +
 .../caching/test_caching_handler.py           | 180 ++++++++++++++++++
 4 files changed, 328 insertions(+), 2 deletions(-)

diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py
index 6a68ba8c4d..ce1bc26c5e 100644
--- a/litellm/caching/caching.py
+++ b/litellm/caching/caching.py
@@ -650,7 +650,10 @@ class Cache:
             verbose_logger.exception(f"LiteLLM Cache: Excepton add_cache: {str(e)}")
 
     def _convert_to_cached_embedding(
-        self, embedding_response: Any, model: Optional[str]
+        self,
+        embedding_response: Any,
+        model: Optional[str],
+        prompt_tokens_details: Optional[dict] = None,
     ) -> CachedEmbedding:
         """
         Convert any embedding response into the standardized CachedEmbedding TypedDict format.
@@ -662,6 +665,7 @@ class Cache:
                     "index": embedding_response.get("index"),
                     "object": embedding_response.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
             elif hasattr(embedding_response, "model_dump"):
                 data = embedding_response.model_dump()
@@ -670,6 +674,7 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
             else:
                 data = vars(embedding_response)
@@ -678,10 +683,54 @@ class Cache:
                     "index": data.get("index"),
                     "object": data.get("object"),
                     "model": model,
+                    "prompt_tokens_details": prompt_tokens_details,
                 }
         except KeyError as e:
             raise ValueError(f"Missing expected key in embedding response: {e}")
 
+    def _get_per_item_prompt_tokens_details(
+        self,
+        result: EmbeddingResponse,
+        idx_in_result_data: int,
+    ) -> Optional[dict]:
+        """
+        Extract per-item prompt_tokens_details from a response for caching.
+
+        For single-item responses (common for multimodal providers like Bedrock Titan,
+        Nova, Vertex AI), returns the full prompt_tokens_details.
+        For multi-item responses, distributes integer fields evenly across items
+        so that summing all per-item details reconstructs the original totals.
+        """
+        if result.usage is None or result.usage.prompt_tokens_details is None:
+            return None
+
+        details = result.usage.prompt_tokens_details
+        if hasattr(details, "model_dump"):
+            details_dict = details.model_dump(exclude_none=True)
+        elif isinstance(details, dict):
+            details_dict = {k: v for k, v in details.items() if v is not None}
+        else:
+            return None
+
+        if not details_dict:
+            return None
+
+        num_items = len(result.data)
+        if num_items <= 1:
+            return details_dict
+
+        # Distribute integer/float fields evenly across items
+        per_item: dict = {}
+        for key, value in details_dict.items():
+            if isinstance(value, int):
+                quotient, remainder = divmod(value, num_items)
+                per_item[key] = quotient + (1 if idx_in_result_data < remainder else 0)
+            elif isinstance(value, float):
+                per_item[key] = value / num_items
+            else:
+                per_item[key] = value
+        return per_item if per_item else None
+
     def add_embedding_response_to_cache(
         self,
         result: EmbeddingResponse,
@@ -693,10 +742,18 @@ class Cache:
         kwargs["cache_key"] = preset_cache_key
         embedding_response = result.data[idx_in_result_data]
 
+        # Extract per-item prompt_tokens_details from response usage
+        prompt_tokens_details = self._get_per_item_prompt_tokens_details(
+            result=result,
+            idx_in_result_data=idx_in_result_data,
+        )
+
         # Always convert to properly typed CachedEmbedding
         model_name = result.model
         embedding_dict: CachedEmbedding = self._convert_to_cached_embedding(
-            embedding_response, model_name
+            embedding_response,
+            model_name,
+            prompt_tokens_details=prompt_tokens_details,
         )
 
         cache_key, cached_data, kwargs = self._add_cache_logic(
diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py
index 2bec705946..7d514e648f 100644
--- a/litellm/caching/caching_handler.py
+++ b/litellm/caching/caching_handler.py
@@ -59,6 +59,7 @@ from litellm.types.utils import (
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.utils import PromptTokensDetailsWrapper
 else:
     LiteLLMLoggingObj = Any
 
@@ -415,6 +416,7 @@ class LLMCachingHandler:
             final_embedding_cached_response._hidden_params["cache_hit"] = True
 
             prompt_tokens = 0
+            aggregated_details: Optional[dict] = None
             for val in non_null_list:
                 idx, cr = val  # (idx, cr) tuple
                 if cr is not None:
@@ -431,11 +433,35 @@ class LLMCachingHandler:
                         prompt_tokens += token_counter(
                             text=kwargs_input_as_list[idx], count_response_tokens=True
                         )
+                    # Aggregate prompt_tokens_details from cached items
+                    item_details = cr.get("prompt_tokens_details")
+                    if item_details:
+                        if aggregated_details is None:
+                            aggregated_details = {}
+                        for key, value in item_details.items():
+                            if isinstance(value, (int, float)):
+                                aggregated_details[key] = (
+                                    aggregated_details.get(key, 0) + value
+                                )
+                            else:
+                                aggregated_details[key] = value
+
             ## USAGE
+            prompt_tokens_details: Optional["PromptTokensDetailsWrapper"] = None
+            if aggregated_details:
+                from litellm.types.utils import PromptTokensDetailsWrapper
+
+                try:
+                    prompt_tokens_details = PromptTokensDetailsWrapper(
+                        **aggregated_details
+                    )
+                except Exception:
+                    prompt_tokens_details = None
             usage = Usage(
                 prompt_tokens=prompt_tokens,
                 completion_tokens=0,
                 total_tokens=prompt_tokens,
+                prompt_tokens_details=prompt_tokens_details,
             )
             final_embedding_cached_response.usage = usage
         if len(remaining_list) == 0:
@@ -478,8 +504,70 @@ class LLMCachingHandler:
             prompt_tokens=usage1.prompt_tokens + usage2.prompt_tokens,
             completion_tokens=usage1.completion_tokens + usage2.completion_tokens,
             total_tokens=usage1.total_tokens + usage2.total_tokens,
+            prompt_tokens_details=self._merge_prompt_tokens_details(
+                usage1.prompt_tokens_details,
+                usage2.prompt_tokens_details,
+            ),
         )
 
+    def _merge_prompt_tokens_details(
+        self,
+        details1: Optional["PromptTokensDetailsWrapper"],
+        details2: Optional["PromptTokensDetailsWrapper"],
+    ) -> Optional["PromptTokensDetailsWrapper"]:
+        """Merge two PromptTokensDetailsWrapper objects by summing numeric fields."""
+        if details1 is None and details2 is None:
+            return None
+        if details1 is None:
+            return details2
+        if details2 is None:
+            return details1
+
+        dict1 = (
+            details1.model_dump(exclude_none=True)
+            if hasattr(details1, "model_dump")
+            else {}
+        )
+        dict2 = (
+            details2.model_dump(exclude_none=True)
+            if hasattr(details2, "model_dump")
+            else {}
+        )
+
+        merged: dict = {}
+        for key in set(dict1.keys()) | set(dict2.keys()):
+            v1 = dict1.get(key, 0)
+            v2 = dict2.get(key, 0)
+            if isinstance(v1, (int, float)) and isinstance(v2, (int, float)):
+                merged[key] = v1 + v2
+            elif isinstance(v1, dict) and isinstance(v2, dict):
+                # Recursively merge nested dicts (e.g. cache_creation_token_details)
+                nested: dict = {}
+                for nk in set(v1.keys()) | set(v2.keys()):
+                    nv1 = v1.get(nk, 0)
+                    nv2 = v2.get(nk, 0)
+                    if isinstance(nv1, (int, float)) and isinstance(nv2, (int, float)):
+                        nested[nk] = nv1 + nv2
+                    elif nv1:
+                        nested[nk] = nv1
+                    else:
+                        nested[nk] = nv2
+                merged[key] = nested
+            elif v1:
+                merged[key] = v1
+            else:
+                merged[key] = v2
+
+        if not merged:
+            return None
+
+        from litellm.types.utils import PromptTokensDetailsWrapper
+
+        try:
+            return PromptTokensDetailsWrapper(**merged)
+        except Exception:
+            return None
+
     def _combine_cached_embedding_response_with_api_result(
         self,
         _caching_handler_response: CachingHandlerResponse,
diff --git a/litellm/types/caching.py b/litellm/types/caching.py
index c8194ce2e7..f8050b292c 100644
--- a/litellm/types/caching.py
+++ b/litellm/types/caching.py
@@ -118,3 +118,4 @@ class CachedEmbedding(TypedDict):
     index: Optional[int]
     object: Optional[str]
     model: Optional[str]
+    prompt_tokens_details: Optional[dict]
diff --git a/tests/test_litellm/caching/test_caching_handler.py b/tests/test_litellm/caching/test_caching_handler.py
index 837ce7d405..742a4f410d 100644
--- a/tests/test_litellm/caching/test_caching_handler.py
+++ b/tests/test_litellm/caching/test_caching_handler.py
@@ -52,3 +52,183 @@ async def test_process_async_embedding_cached_response():
 
     print(f"response: {response}")
     assert len(response.data) == 1
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_preserves_prompt_tokens_details():
+    """Test that prompt_tokens_details (including image_count) survives a full cache hit."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    cached_result = [
+        {
+            "embedding": [-0.025, -0.019],
+            "index": 0,
+            "object": "embedding",
+            "model": "amazon.titan-embed-image-v1",
+            "prompt_tokens_details": {"image_count": 1},
+        }
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={"model": "amazon.titan-embed-image-v1", "input": "base64imagedata"},
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="amazon.titan-embed-image-v1",
+    )
+
+    assert cache_hit
+    assert response.usage is not None
+    assert response.usage.prompt_tokens_details is not None
+    assert response.usage.prompt_tokens_details.image_count == 1
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_backward_compat_no_prompt_tokens_details():
+    """Test that old cached items without prompt_tokens_details still work."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    # Old-format cached item — no prompt_tokens_details field
+    cached_result = [
+        {
+            "embedding": [-0.025, -0.019],
+            "index": 0,
+            "object": "embedding",
+            "model": "text-embedding-ada-002",
+        }
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={"model": "text-embedding-ada-002", "input": "test"},
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="text-embedding-ada-002",
+    )
+
+    assert cache_hit
+    assert response.usage is not None
+    assert response.usage.prompt_tokens_details is None
+
+
+@pytest.mark.asyncio
+async def test_embedding_cache_aggregates_multiple_image_counts():
+    """Test that image_count is summed correctly across multiple cached items."""
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    cached_result = [
+        {
+            "embedding": [-0.025, -0.019],
+            "index": 0,
+            "object": "embedding",
+            "model": "amazon.titan-embed-image-v1",
+            "prompt_tokens_details": {"image_count": 1},
+        },
+        {
+            "embedding": [0.031, 0.042],
+            "index": 1,
+            "object": "embedding",
+            "model": "amazon.titan-embed-image-v1",
+            "prompt_tokens_details": {"image_count": 1},
+        },
+    ]
+
+    mock_logging_obj = MagicMock()
+    mock_logging_obj.async_success_handler = AsyncMock()
+    response, cache_hit = llm_caching_handler._process_async_embedding_cached_response(
+        final_embedding_cached_response=None,
+        cached_result=cached_result,
+        kwargs={
+            "model": "amazon.titan-embed-image-v1",
+            "input": ["img1", "img2"],
+        },
+        logging_obj=mock_logging_obj,
+        start_time=datetime.now(),
+        model="amazon.titan-embed-image-v1",
+    )
+
+    assert cache_hit
+    assert response.usage.prompt_tokens_details is not None
+    assert response.usage.prompt_tokens_details.image_count == 2
+
+
+def test_combine_usage_merges_prompt_tokens_details():
+    """Test that combine_usage merges prompt_tokens_details from both Usage objects."""
+    from litellm.types.utils import PromptTokensDetailsWrapper, Usage
+
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    usage1 = Usage(
+        prompt_tokens=10,
+        completion_tokens=0,
+        total_tokens=10,
+        prompt_tokens_details=PromptTokensDetailsWrapper(image_count=1),
+    )
+    usage2 = Usage(
+        prompt_tokens=20,
+        completion_tokens=0,
+        total_tokens=20,
+        prompt_tokens_details=PromptTokensDetailsWrapper(image_count=2),
+    )
+
+    combined = llm_caching_handler.combine_usage(usage1, usage2)
+
+    assert combined.prompt_tokens == 30
+    assert combined.total_tokens == 30
+    assert combined.prompt_tokens_details is not None
+    assert combined.prompt_tokens_details.image_count == 3
+
+
+def test_combine_usage_handles_none_details():
+    """Test that combine_usage works when one or both sides have null prompt_tokens_details."""
+    from litellm.types.utils import PromptTokensDetailsWrapper, Usage
+
+    llm_caching_handler = LLMCachingHandler(
+        original_function=MagicMock(),
+        request_kwargs={},
+        start_time=datetime.now(),
+    )
+
+    # Both null
+    usage_a = Usage(prompt_tokens=10, completion_tokens=0, total_tokens=10)
+    usage_b = Usage(prompt_tokens=20, completion_tokens=0, total_tokens=20)
+    combined = llm_caching_handler.combine_usage(usage_a, usage_b)
+    assert combined.prompt_tokens_details is None
+
+    # Only first has details
+    usage_c = Usage(
+        prompt_tokens=10,
+        completion_tokens=0,
+        total_tokens=10,
+        prompt_tokens_details=PromptTokensDetailsWrapper(image_count=1),
+    )
+    combined = llm_caching_handler.combine_usage(usage_c, usage_b)
+    assert combined.prompt_tokens_details is not None
+    assert combined.prompt_tokens_details.image_count == 1
+
+    # Only second has details
+    combined = llm_caching_handler.combine_usage(usage_a, usage_c)
+    assert combined.prompt_tokens_details is not None
+    assert combined.prompt_tokens_details.image_count == 1

From 10aed9e9816c61600765766428c1c167327e2c64 Mon Sep 17 00:00:00 2001
From: milan-berri <milan@berri.ai>
Date: Tue, 28 Apr 2026 18:38:17 +0300
Subject: [PATCH 03/19] feat(logging): add retry settings for generic API
 logger (#26645)

* Add retry settings for generic API logger

Made-with: Cursor

* Refine generic API retry behavior

Made-with: Cursor
---
 .../generic_api/generic_api_callback.py       | 72 +++++++++++---
 .../logging_callback_manager.py               | 13 +++
 .../test_logging_callback_manager.py          | 37 ++++++++
 .../test_generic_api_callback.py              | 94 +++++++++++++++++++
 4 files changed, 205 insertions(+), 11 deletions(-)

diff --git a/litellm/integrations/generic_api/generic_api_callback.py b/litellm/integrations/generic_api/generic_api_callback.py
index 9a8060520d..2982df8fda 100644
--- a/litellm/integrations/generic_api/generic_api_callback.py
+++ b/litellm/integrations/generic_api/generic_api_callback.py
@@ -11,8 +11,9 @@ import json
 import os
 import re
 import traceback
-from typing import Dict, List, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Union
 
+import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
@@ -103,6 +104,9 @@ class GenericAPILogger(CustomBatchLogger):
         event_types: Optional[List[API_EVENT_TYPES]] = None,
         callback_name: Optional[str] = None,
         log_format: Optional[LOG_FORMAT_TYPES] = None,
+        max_retries: int = 0,
+        retry_delay: float = 1.0,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
         **kwargs,
     ):
         """
@@ -114,6 +118,9 @@ class GenericAPILogger(CustomBatchLogger):
             event_types: Optional[List[API_EVENT_TYPES]] = None,
             callback_name: Optional[str] = None - If provided, loads config from generic_api_compatible_callbacks.json
             log_format: Optional[LOG_FORMAT_TYPES] = None - Format for log output: "json_array" (default), "ndjson", or "single"
+            max_retries: Number of retry attempts after the initial request fails. Defaults to 0.
+            retry_delay: Initial retry delay in seconds. Retries use exponential backoff.
+            timeout: Optional timeout to use for Generic API callback requests.
         """
         #########################################################
         # Check if callback_name is provided and load config
@@ -162,6 +169,10 @@ class GenericAPILogger(CustomBatchLogger):
         self.endpoint: str = endpoint
         self.event_types: Optional[List[API_EVENT_TYPES]] = event_types
         self.callback_name: Optional[str] = callback_name
+        self.max_retries = max(0, int(max_retries or 0))
+        retry_delay_value = 0.0 if retry_delay is None else retry_delay
+        self.retry_delay = max(0.0, float(retry_delay_value))
+        self.timeout = timeout
 
         # Validate and store log_format
         if log_format is not None and log_format not in [
@@ -226,6 +237,53 @@ class GenericAPILogger(CustomBatchLogger):
 
         return headers_dict
 
+    def _should_retry_exception(self, exception: Exception) -> bool:
+        if isinstance(exception, (litellm.Timeout, httpx.TransportError)):
+            return True
+
+        if isinstance(exception, httpx.HTTPStatusError):
+            return exception.response.status_code >= 500
+
+        return False
+
+    async def _sleep_before_retry(self, attempt: int) -> None:
+        if self.retry_delay <= 0:
+            return
+
+        delay = self.retry_delay * (2**attempt)
+        await asyncio.sleep(delay)
+
+    async def _post_with_retries(self, data: str) -> httpx.Response:
+        post_kwargs: Dict[str, Any] = {
+            "url": self.endpoint,
+            "headers": self.headers,
+            "data": data,
+        }
+        if self.timeout is not None:
+            post_kwargs["timeout"] = self.timeout
+
+        total_attempts = self.max_retries + 1
+        for attempt in range(total_attempts):
+            try:
+                return await self.async_httpx_client.post(**post_kwargs)
+            except Exception as e:
+                is_last_attempt = attempt == self.max_retries
+                should_retry = self._should_retry_exception(e)
+                if is_last_attempt or not should_retry:
+                    raise
+
+                verbose_logger.warning(
+                    "Generic API Logger - retrying request to %s after error: %s "
+                    "(attempt %s/%s)",
+                    self.endpoint,
+                    str(e),
+                    attempt + 1,
+                    total_attempts,
+                )
+                await self._sleep_before_retry(attempt)
+
+        raise RuntimeError("Generic API Logger retry loop exited unexpectedly")
+
     async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
         """
         Async Log success events to Generic API Endpoint
@@ -325,11 +383,7 @@ class GenericAPILogger(CustomBatchLogger):
                 # Send each log as individual HTTP request in parallel
                 tasks = []
                 for log_entry in self.log_queue:
-                    task = self.async_httpx_client.post(
-                        url=self.endpoint,
-                        headers=self.headers,
-                        data=safe_dumps(log_entry),
-                    )
+                    task = self._post_with_retries(data=safe_dumps(log_entry))
                     tasks.append(task)
 
                 # Execute all requests in parallel
@@ -356,11 +410,7 @@ class GenericAPILogger(CustomBatchLogger):
                     raise ValueError(f"Unknown log_format: {self.log_format}")
 
                 # Make POST request
-                response = await self.async_httpx_client.post(
-                    url=self.endpoint,
-                    headers=self.headers,
-                    data=data,
-                )
+                response = await self._post_with_retries(data=data)
 
                 verbose_logger.debug(
                     f"Generic API Logger - sent batch to {self.endpoint}, "
diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py
index c5c150274c..6c749118de 100644
--- a/litellm/litellm_core_utils/logging_callback_manager.py
+++ b/litellm/litellm_core_utils/logging_callback_manager.py
@@ -221,6 +221,13 @@ class LoggingCallbackManager:
             headers = callback_config.get("headers")
             event_types = callback_config.get("event_types")
             log_format = callback_config.get("log_format")
+            max_retries = max(0, int(callback_config.get("max_retries", 0) or 0))
+            retry_delay_value = callback_config.get("retry_delay")
+            retry_delay = max(
+                0.0,
+                float(0.0 if retry_delay_value is None else retry_delay_value),
+            )
+            timeout = callback_config.get("timeout")
 
             if endpoint is None or headers is None:
                 verbose_logger.warning(
@@ -236,6 +243,9 @@ class LoggingCallbackManager:
                 and cached_logger.headers == headers
                 and cached_logger.event_types == event_types
                 and cached_logger.log_format == log_format
+                and cached_logger.max_retries == max_retries
+                and cached_logger.retry_delay == retry_delay
+                and cached_logger.timeout == timeout
             ):
                 return cached_logger
 
@@ -244,6 +254,9 @@ class LoggingCallbackManager:
                 headers=headers,
                 event_types=event_types,
                 log_format=log_format,
+                max_retries=max_retries,
+                retry_delay=retry_delay,
+                timeout=timeout,
             )
             _generic_api_logger_cache[callback] = new_logger
             return new_logger
diff --git a/tests/litellm_utils_tests/test_logging_callback_manager.py b/tests/litellm_utils_tests/test_logging_callback_manager.py
index 88ae07fd81..d9540f8f85 100644
--- a/tests/litellm_utils_tests/test_logging_callback_manager.py
+++ b/tests/litellm_utils_tests/test_logging_callback_manager.py
@@ -366,3 +366,40 @@ def test_generic_api_compatible_callbacks_json_unknown_callback():
     # Should return the string unchanged
     assert result == "unknown_callback", "Unknown callback should be returned as-is"
     assert isinstance(result, str), "Unknown callback should remain a string"
+
+
+@pytest.mark.asyncio
+async def test_generic_api_callback_settings_retry_config():
+    """
+    Test that generic_api callback_settings are passed to GenericAPILogger.
+    """
+    from litellm.integrations.generic_api.generic_api_callback import GenericAPILogger
+    from litellm.litellm_core_utils.logging_callback_manager import (
+        _generic_api_logger_cache,
+    )
+
+    callback_name = "test_generic_api_retry_config"
+    _generic_api_logger_cache.pop(callback_name, None)
+    litellm.callback_settings[callback_name] = {
+        "callback_type": "generic_api",
+        "endpoint": "https://example.com/api/logs",
+        "headers": {"Content-Type": "application/json"},
+        "max_retries": 2,
+        "retry_delay": 0.5,
+        "timeout": 3,
+    }
+
+    try:
+        result = LoggingCallbackManager._add_custom_callback_generic_api_str(
+            callback_name
+        )
+
+        assert isinstance(result, GenericAPILogger)
+        assert result.endpoint == "https://example.com/api/logs"
+        assert result.headers == {"Content-Type": "application/json"}
+        assert result.max_retries == 2
+        assert result.retry_delay == 0.5
+        assert result.timeout == 3
+    finally:
+        litellm.callback_settings.pop(callback_name, None)
+        _generic_api_logger_cache.pop(callback_name, None)
diff --git a/tests/logging_callback_tests/test_generic_api_callback.py b/tests/logging_callback_tests/test_generic_api_callback.py
index 528a5101df..6984b6fa00 100644
--- a/tests/logging_callback_tests/test_generic_api_callback.py
+++ b/tests/logging_callback_tests/test_generic_api_callback.py
@@ -8,6 +8,7 @@ sys.path.insert(0, os.path.abspath("../.."))
 import asyncio
 import litellm
 import gzip
+import httpx
 import json
 import logging
 import time
@@ -470,3 +471,96 @@ async def test_generic_api_callback_invalid_log_format():
             endpoint=test_endpoint,
             log_format="invalid_format",  # type: ignore  # Intentionally invalid for testing
         )
+
+
+@pytest.mark.asyncio
+async def test_generic_api_callback_retries_timeout_then_succeeds():
+    """
+    Test that GenericAPILogger retries LiteLLM timeout errors when configured.
+    """
+    test_endpoint = "https://example.com/api/logs"
+    generic_logger = GenericAPILogger(
+        endpoint=test_endpoint,
+        max_retries=1,
+        retry_delay=0,
+        timeout=0.2,
+    )
+
+    mock_post = AsyncMock()
+    mock_post.side_effect = [
+        litellm.Timeout(
+            message="Connection timed out",
+            model="default-model-name",
+            llm_provider="litellm-httpx-handler",
+        ),
+        type("Response", (), {"status_code": 200})(),
+    ]
+    generic_logger.async_httpx_client.post = mock_post
+    generic_logger.log_queue = [{"event": "timeout-retry"}]
+
+    await generic_logger.async_send_batch()
+
+    assert mock_post.call_count == 2
+    first_call = mock_post.call_args_list[0][1]
+    assert first_call["url"] == test_endpoint
+    assert first_call["timeout"] == 0.2
+    assert json.loads(first_call["data"]) == [{"event": "timeout-retry"}]
+
+
+@pytest.mark.asyncio
+async def test_generic_api_callback_retries_5xx_then_succeeds():
+    """
+    Test that GenericAPILogger retries transient HTTP 5xx errors when configured.
+    """
+    test_endpoint = "https://example.com/api/logs"
+    generic_logger = GenericAPILogger(
+        endpoint=test_endpoint,
+        max_retries=1,
+        retry_delay=0,
+    )
+
+    request = httpx.Request("POST", test_endpoint)
+    response = httpx.Response(status_code=503, request=request)
+    mock_post = AsyncMock()
+    mock_post.side_effect = [
+        httpx.HTTPStatusError(
+            "Server error",
+            request=request,
+            response=response,
+        ),
+        type("Response", (), {"status_code": 200})(),
+    ]
+    generic_logger.async_httpx_client.post = mock_post
+    generic_logger.log_queue = [{"event": "5xx-retry"}]
+
+    await generic_logger.async_send_batch()
+
+    assert mock_post.call_count == 2
+
+
+@pytest.mark.asyncio
+async def test_generic_api_callback_does_not_retry_4xx():
+    """
+    Test that GenericAPILogger does not retry non-transient HTTP 4xx errors.
+    """
+    test_endpoint = "https://example.com/api/logs"
+    generic_logger = GenericAPILogger(
+        endpoint=test_endpoint,
+        max_retries=2,
+        retry_delay=0,
+    )
+
+    request = httpx.Request("POST", test_endpoint)
+    response = httpx.Response(status_code=401, request=request)
+    mock_post = AsyncMock()
+    mock_post.side_effect = httpx.HTTPStatusError(
+        "Unauthorized",
+        request=request,
+        response=response,
+    )
+    generic_logger.async_httpx_client.post = mock_post
+    generic_logger.log_queue = [{"event": "4xx-no-retry"}]
+
+    await generic_logger.async_send_batch()
+
+    mock_post.assert_called_once()

From 52fb23a512894cc283c1a94a88eebea3745b05b5 Mon Sep 17 00:00:00 2001
From: milan-berri <milan@berri.ai>
Date: Tue, 28 Apr 2026 18:41:20 +0300
Subject: [PATCH 04/19] fix(logging): backfill streaming hidden response cost
 (#26606)

* fix(logging): backfill streaming hidden response cost

Made-with: Cursor

* fix(logging): avoid mutating streaming hidden params

Backfill calculated streaming response cost into logging payload copies so OTEL spans expose hidden_params.response_cost without mutating the response object.

Made-with: Cursor

* fix black formatting

Apply the repo-pinned Black 24.10.0 formatting expected by CI.

Made-with: Cursor

* fix(types): allow numeric hidden response cost

Allow standard logging hidden params to carry numeric response_cost values, matching LiteLLM's calculated cost payloads.

Made-with: Cursor

* refactor(logging): simplify hidden response cost backfill

Clean up metadata initialization and reuse the raw response cost when deciding whether to backfill hidden params.

Made-with: Cursor
---
 litellm/litellm_core_utils/litellm_logging.py | 36 ++++---
 litellm/types/utils.py                        |  2 +-
 .../test_litellm_logging.py                   | 98 +++++++++++++++++++
 3 files changed, 123 insertions(+), 13 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index 57341472b4..fb103afea0 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -1725,12 +1725,18 @@ class Logging(LiteLLMLoggingBaseClass):
             return
         if self.model_call_details.get("litellm_params") is None:
             return
-        self.model_call_details["litellm_params"].setdefault("metadata", {})
-        if self.model_call_details["litellm_params"]["metadata"] is None:
-            self.model_call_details["litellm_params"]["metadata"] = {}
-        self.model_call_details["litellm_params"]["metadata"]["hidden_params"] = (
-            getattr(logging_result, "_hidden_params", {})
-        )
+        metadata_hidden_params = hidden_params.copy()
+        response_cost = self.model_call_details.get("response_cost")
+        if (
+            metadata_hidden_params.get("response_cost") is None
+            and response_cost is not None
+        ):
+            metadata_hidden_params["response_cost"] = response_cost
+
+        litellm_params = self.model_call_details["litellm_params"]
+        metadata = litellm_params.get("metadata") or {}
+        litellm_params["metadata"] = metadata
+        metadata["hidden_params"] = metadata_hidden_params
 
     def _process_hidden_params_and_response_cost(
         self,
@@ -5438,11 +5444,6 @@ def get_standard_logging_object_payload(
             completion_start_time_float=completion_start_time_float,
             stream=kwargs.get("stream", False),
         )
-        # clean up litellm hidden params
-        clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
-            hidden_params
-        )
-
         # clean up litellm metadata
         clean_metadata = StandardLoggingPayloadSetup.get_standard_logging_metadata(
             metadata=metadata,
@@ -5476,6 +5477,18 @@ def get_standard_logging_object_payload(
         ## Get model cost information ##
         base_model = _get_base_model_from_metadata(model_call_details=kwargs)
         custom_pricing = use_custom_pricing_for_model(litellm_params=litellm_params)
+        raw_response_cost = kwargs.get("response_cost")
+        response_cost: float = raw_response_cost or 0.0
+
+        # clean up litellm hidden params
+        clean_hidden_params = StandardLoggingPayloadSetup.get_hidden_params(
+            hidden_params
+        )
+        if (
+            clean_hidden_params["response_cost"] is None
+            and raw_response_cost is not None
+        ):
+            clean_hidden_params["response_cost"] = response_cost
 
         model_cost_information = StandardLoggingPayloadSetup.get_model_cost_information(
             base_model=base_model,
@@ -5484,7 +5497,6 @@ def get_standard_logging_object_payload(
             init_response_obj=init_response_obj,
             api_base=litellm_params.get("api_base"),
         )
-        response_cost: float = kwargs.get("response_cost", 0) or 0.0
 
         error_information = StandardLoggingPayloadSetup.get_error_information(
             original_exception=original_exception,
diff --git a/litellm/types/utils.py b/litellm/types/utils.py
index a212d56c1a..ed29d49fc2 100644
--- a/litellm/types/utils.py
+++ b/litellm/types/utils.py
@@ -2659,7 +2659,7 @@ class StandardLoggingHiddenParams(TypedDict):
     ]  # id of the model in the router, separates multiple models with the same name but different credentials
     cache_key: Optional[str]
     api_base: Optional[str]
-    response_cost: Optional[str]
+    response_cost: Optional[Union[str, float]]
     litellm_overhead_time_ms: Optional[float]
     additional_headers: Optional[StandardLoggingAdditionalHeaders]
     batch_models: Optional[List[str]]
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index cf7be6bf1c..3348118a02 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -2337,6 +2337,104 @@ def test_merge_hidden_params_from_response_into_metadata_populates_metadata():
     assert meta["hidden_params"]["model_id"] == "mid-test"
 
 
+def test_merge_hidden_params_from_response_into_metadata_backfills_response_cost():
+    """Streaming metadata should include the already-calculated response cost."""
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+    logging_obj = LiteLLMLoggingObj(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="acompletion",
+        start_time=time.time(),
+        litellm_call_id="merge-hp-cost-test",
+        function_id="merge-hp-cost-fn",
+    )
+    logging_obj.model_call_details = {
+        "litellm_params": {"metadata": {}},
+        "response_cost": 0.002,
+    }
+
+    class _Resp:
+        _hidden_params = {"response_cost": None, "model_id": "mid-test"}
+
+    response = _Resp()
+    logging_obj._merge_hidden_params_from_response_into_metadata(response)
+    meta = logging_obj.model_call_details["litellm_params"]["metadata"]
+    assert meta["hidden_params"]["response_cost"] == 0.002
+    assert meta["hidden_params"]["model_id"] == "mid-test"
+    assert response._hidden_params["response_cost"] is None
+
+
+def test_standard_logging_hidden_params_backfills_response_cost_without_mutating_response():
+    """Streaming standard logging payload should expose the calculated response cost."""
+    from datetime import datetime
+
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+    from litellm.types.utils import Usage
+
+    logging_obj = LiteLLMLoggingObj(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="acompletion",
+        start_time=time.time(),
+        litellm_call_id="standard-hp-cost-test",
+        function_id="standard-hp-cost-fn",
+    )
+    logging_obj.model_call_details = {
+        "litellm_params": {"metadata": {}, "proxy_server_request": {}},
+        "litellm_call_id": "standard-hp-cost-test",
+        "call_type": "acompletion",
+        "stream": True,
+        "model": "gpt-4o-mini",
+        "custom_llm_provider": "openai",
+        "optional_params": {"stream": True},
+        "response_cost": 0.002,
+    }
+    response = ModelResponse(
+        id="standard-hp-cost-response",
+        model="gpt-4o-mini",
+        usage=Usage(prompt_tokens=10, completion_tokens=5, total_tokens=15),
+    )
+    response._hidden_params = {"response_cost": None, "model_id": "mid-test"}
+
+    payload = logging_obj._build_standard_logging_payload(
+        response, datetime.now(), datetime.now()
+    )
+
+    assert payload is not None
+    assert payload["hidden_params"]["response_cost"] == 0.002
+    assert response._hidden_params["response_cost"] is None
+
+
+def test_merge_hidden_params_from_response_into_metadata_preserves_response_cost():
+    """Do not overwrite provider-supplied response cost when it already exists."""
+    from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+
+    logging_obj = LiteLLMLoggingObj(
+        model="gpt-4o-mini",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=True,
+        call_type="acompletion",
+        start_time=time.time(),
+        litellm_call_id="merge-hp-preserve-cost-test",
+        function_id="merge-hp-preserve-cost-fn",
+    )
+    logging_obj.model_call_details = {
+        "litellm_params": {"metadata": {}},
+        "response_cost": 0.002,
+    }
+
+    class _Resp:
+        _hidden_params = {"response_cost": 0.001, "model_id": "mid-test"}
+
+    logging_obj._merge_hidden_params_from_response_into_metadata(_Resp())
+    meta = logging_obj.model_call_details["litellm_params"]["metadata"]
+    assert meta["hidden_params"]["response_cost"] == 0.001
+    assert meta["hidden_params"]["model_id"] == "mid-test"
+
+
 def test_merge_hidden_params_from_response_into_metadata_no_op_when_empty():
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 

From 1d56e732e835e9ad12fa63e92400e7b61b6c4440 Mon Sep 17 00:00:00 2001
From: Sameer Kankute <sameer@berri.ai>
Date: Tue, 28 Apr 2026 21:14:40 +0530
Subject: [PATCH 05/19] fix(vertex-ai): reuse anthropic messages config
 instances (#26099)

Cache provider config lookups for Vertex Anthropic messages so repeated requests reuse the same config object and preserve credential cache state. Add a regression test to catch any future loss of config reuse.

Made-with: Cursor
---
 litellm/utils.py                              | 15 +++++++++--
 ...artner_models_anthropic_messages_config.py | 26 +++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/litellm/utils.py b/litellm/utils.py
index e1ad1db63e..e63bf402bf 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -8410,6 +8410,17 @@ class ProviderConfigManager:
         model: str,
         provider: LlmProviders,
     ) -> Optional[BaseAnthropicMessagesConfig]:
+        return ProviderConfigManager._get_provider_anthropic_messages_config_cached(
+            model=model, provider=provider
+        )
+
+    @staticmethod
+    @lru_cache(maxsize=DEFAULT_MAX_LRU_CACHE_SIZE)
+    def _get_provider_anthropic_messages_config_cached(
+        model: str,
+        provider: LlmProviders,
+    ) -> Optional[BaseAnthropicMessagesConfig]:
+        model_lower = model.lower()
         if litellm.LlmProviders.ANTHROPIC == provider:
             return litellm.AnthropicMessagesConfig()
         # The 'BEDROCK' provider corresponds to Amazon's implementation of Anthropic Claude v3.
@@ -8419,14 +8430,14 @@ class ProviderConfigManager:
 
             return BedrockModelInfo.get_bedrock_provider_config_for_messages_api(model)
         elif litellm.LlmProviders.VERTEX_AI == provider:
-            if "claude" in model.lower():
+            if "claude" in model_lower:
                 from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
                     VertexAIPartnerModelsAnthropicMessagesConfig,
                 )
 
                 return VertexAIPartnerModelsAnthropicMessagesConfig()
         elif litellm.LlmProviders.AZURE_AI == provider:
-            if "claude" in model.lower():
+            if "claude" in model_lower:
                 from litellm.llms.azure_ai.anthropic.messages_transformation import (
                     AzureAnthropicMessagesConfig,
                 )
diff --git a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
index 214e5f0797..b8cd65d3c9 100644
--- a/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
+++ b/tests/test_litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py
@@ -311,3 +311,29 @@ def test_transform_anthropic_messages_request_removes_scope_from_cache_control()
     # scope removed from message content
     assert "scope" not in result["messages"][0]["content"][0]["cache_control"]
     assert result["messages"][0]["content"][0]["cache_control"]["type"] == "ephemeral"
+
+
+def test_provider_config_manager_reuses_vertex_anthropic_messages_config_instance():
+    """
+    Regression test: repeated provider config lookups for the same Vertex Claude model
+    should return the same config instance (which preserves auth cache state).
+    """
+    import litellm
+    from litellm.utils import ProviderConfigManager
+
+    ProviderConfigManager._get_provider_anthropic_messages_config_cached.cache_clear()
+    try:
+        first_config = ProviderConfigManager.get_provider_anthropic_messages_config(
+            model="claude-opus-4-6",
+            provider=litellm.LlmProviders.VERTEX_AI,
+        )
+        second_config = ProviderConfigManager.get_provider_anthropic_messages_config(
+            model="claude-opus-4-6",
+            provider=litellm.LlmProviders.VERTEX_AI,
+        )
+
+        assert isinstance(first_config, VertexAIPartnerModelsAnthropicMessagesConfig)
+        assert isinstance(second_config, VertexAIPartnerModelsAnthropicMessagesConfig)
+        assert first_config is second_config
+    finally:
+        ProviderConfigManager._get_provider_anthropic_messages_config_cached.cache_clear()

From 1af11d4371ac5aed4c0263a2d34061c28d9e3ba3 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 28 Apr 2026 09:23:55 -0700
Subject: [PATCH 06/19] fix(vertex): synthesize items for array types missing
 items entirely
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Companion to the prior commit. process_items only converted empty
`items: {}` to `{"type": "object"}`. But anyOf branches like
`{"type": "array"}` (no items field at all) were untouched, so after
convert_anyof_null_to_nullable stripped the null branch and added
nullable, the array branch was sent to Vertex as
`{"type": "array", "nullable": true}` — which Vertex rejects with
INVALID_ARGUMENT (`any_of[0].items: missing field`).

Make process_items synthesize `items: {"type": "object"}` for any
`type == "array"` schema where items is missing or empty.

Also:
- Convert test_gemini_tool_calling_working_demo to a hermetic mock
  test asserting items is present on the array branch in the sent
  body. Was previously a real-network call to Vertex and was the
  test the user reported still failing in CI.
- Add unit test test_build_vertex_schema_array_branch_missing_items_in_anyof
  covering the missing-items shape directly.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
 litellm/llms/vertex_ai/common_utils.py        |  9 ++-
 .../test_amazing_vertex_completion.py         | 70 +++++++++++++++++--
 .../vertex_ai/test_vertex_ai_common_utils.py  | 37 ++++++++++
 3 files changed, 111 insertions(+), 5 deletions(-)

diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py
index 9b23520dcd..b4bfde5f54 100644
--- a/litellm/llms/vertex_ai/common_utils.py
+++ b/litellm/llms/vertex_ai/common_utils.py
@@ -597,7 +597,14 @@ def process_items(schema, depth=0):
             f"Max depth of {DEFAULT_MAX_RECURSE_DEPTH} exceeded while processing schema. Please check the schema for excessive nesting."
         )
     if isinstance(schema, dict):
-        if "items" in schema and schema["items"] == {}:
+        # Vertex requires `items` whenever `type == "array"` (even inside anyOf).
+        # Normalize: empty `items: {}` and missing-items both become {"type": "object"}.
+        type_val = schema.get("type")
+        if (
+            isinstance(type_val, str)
+            and type_val.lower() == "array"
+            and ("items" not in schema or schema.get("items") == {})
+        ):
             schema["items"] = {"type": "object"}
         for key, value in schema.items():
             if isinstance(value, dict):
diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py
index 3b4ecb82b1..9782bf3c2a 100644
--- a/tests/local_testing/test_amazing_vertex_completion.py
+++ b/tests/local_testing/test_amazing_vertex_completion.py
@@ -3493,8 +3493,14 @@ def test_litellm_api_base(monkeypatch, provider, route):
 
 
 def test_gemini_tool_calling_working_demo():
-    load_vertex_ai_credentials()
-    litellm._turn_on_debug()
+    """
+    Regression test: tool params with anyOf containing a `{"type": "array"}`
+    branch (no items field at all) must synthesize items before the request
+    is sent to Vertex (Vertex rejects array types missing items).
+    """
+    from litellm.llms.custom_httpx.http_handler import HTTPHandler
+    from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+
     args = {
         "messages": [
             {
@@ -3564,8 +3570,64 @@ def test_gemini_tool_calling_working_demo():
         ],
         "vertex_location": "global",
     }
-    response = completion(model="vertex_ai/gemini-3-flash-preview", **args)
-    print(response)
+
+    client = HTTPHandler()
+    mock_response = MagicMock()
+    mock_response.status_code = 200
+    mock_response.headers = {"Content-Type": "application/json"}
+    mock_response.json.return_value = {
+        "candidates": [
+            {
+                "content": {
+                    "role": "model",
+                    "parts": [{"text": "Hello!"}],
+                },
+                "finishReason": "STOP",
+            }
+        ],
+        "usageMetadata": {
+            "promptTokenCount": 10,
+            "candidatesTokenCount": 5,
+            "totalTokenCount": 15,
+        },
+    }
+
+    with (
+        patch.object(client, "post", return_value=mock_response) as mock_post,
+        patch.object(
+            VertexBase,
+            "_ensure_access_token",
+            return_value=("fake-token", "fake-project"),
+        ),
+    ):
+        completion(
+            model="vertex_ai/gemini-3-flash-preview",
+            client=client,
+            **args,
+        )
+
+    sent_body = mock_post.call_args.kwargs.get(
+        "json"
+    ) or mock_post.call_args.kwargs.get("data")
+    assert sent_body is not None, "expected request body to be sent"
+    if isinstance(sent_body, str):
+        sent_body = json.loads(sent_body)
+
+    function_decl = sent_body["tools"][0]["function_declarations"][0]
+    callbacks_schema = function_decl["parameters"]["properties"]["config"][
+        "properties"
+    ]["callbacks"]
+    array_branches = [
+        branch
+        for branch in callbacks_schema["anyOf"]
+        if branch.get("type", "").lower() == "array"
+    ]
+    assert array_branches, "expected an array branch in callbacks anyOf"
+    for branch in array_branches:
+        assert "items" in branch and branch["items"], (
+            f"array branch in callbacks.anyOf must include non-empty items "
+            f"(Vertex rejects array types missing items). Got: {branch}"
+        )
 
 
 def test_gemini_tool_calling_not_working():
diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
index dc3be7114f..95507390df 100644
--- a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
+++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_common_utils.py
@@ -292,6 +292,43 @@ def test_process_items_basic():
     process_items(schema)
     assert schema["properties"]["nested"]["items"] == {"type": "object"}
 
+    # Vertex rejects array types missing `items` entirely (not just empty).
+    # Synthesize {"type": "object"} so the request validates.
+    schema = {"type": "array"}
+    process_items(schema)
+    assert schema["items"] == {"type": "object"}
+
+
+def test_build_vertex_schema_array_branch_missing_items_in_anyof():
+    """
+    Regression: an `anyOf` branch with `{"type": "array"}` (no items) must
+    end up with synthesized `items: {"type": "object"}` after the schema
+    transform — Vertex returns INVALID_ARGUMENT otherwise.
+    """
+    from litellm.llms.vertex_ai.common_utils import _build_vertex_schema
+
+    parameters = {
+        "properties": {
+            "callbacks": {
+                "anyOf": [
+                    {"type": "array"},
+                    {"type": "object"},
+                    {"type": "null"},
+                ]
+            }
+        },
+        "type": "object",
+    }
+
+    result = _build_vertex_schema(parameters)
+    callbacks_anyof = result["properties"]["callbacks"]["anyOf"]
+    array_branches = [b for b in callbacks_anyof if b.get("type") == "array"]
+    assert array_branches, "expected an array branch to remain after transform"
+    for branch in array_branches:
+        assert branch.get("items") == {
+            "type": "object"
+        }, f"array branch must have items synthesized; got {branch}"
+
 
 def test_vertex_ai_complex_response_schema():
     import json

From dc46467235fa498d3d84482b9942604d5d694b4f Mon Sep 17 00:00:00 2001
From: Ryan Crabbe <ryan@berri.ai>
Date: Tue, 28 Apr 2026 14:24:19 -0700
Subject: [PATCH 07/19] fix(tests): replace deprecated Bedrock Claude 3.7
 Sonnet model ID

AWS Bedrock has reached end-of-life for `claude-3-7-sonnet-20250219-v1:0`,
returning 404s with "This model version has reached the end of its life."
Update test references to `claude-sonnet-4-5-20250929-v1:0` (same capability
surface: thinking, tools, prompt caching, PDF input, vision, computer use).

The bedrock/invoke pass-through tests stay on Sonnet 3.5 since Sonnet 4.5
is converse-only on Bedrock.
---
 .../litellm_utils_tests/test_health_check.py  |  4 +--
 tests/litellm_utils_tests/test_utils.py       |  6 ++--
 .../test_bedrock_anthropic_regression.py      | 12 ++++----
 .../test_bedrock_completion.py                | 10 +++----
 .../test_litellm_proxy_provider.py            |  2 +-
 tests/llm_translation/test_optional_params.py |  2 +-
 tests/local_testing/test_function_calling.py  |  2 +-
 ..._anthropic_messages_prompt_caching_test.py |  4 +--
 .../test_anthropic_messages_prompt_caching.py |  4 +--
 .../open_telemetry/data/captured_kwargs.json  |  2 +-
 .../data/captured_response.json               |  2 +-
 .../test_anthropic_cache_control_hook.py      | 22 +++++++--------
 .../integrations/test_opentelemetry.py        |  2 +-
 ...llm_core_utils_prompt_templates_factory.py |  2 +-
 .../chat/test_converse_transformation.py      | 12 ++++----
 tests/test_litellm/test_utils.py              | 28 +++++++++----------
 16 files changed, 58 insertions(+), 58 deletions(-)

diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py
index a41907722e..45c6a04ad5 100644
--- a/tests/litellm_utils_tests/test_health_check.py
+++ b/tests/litellm_utils_tests/test_health_check.py
@@ -314,11 +314,11 @@ def test_update_litellm_params_for_health_check():
     # Issue #15807: Fixes health checks sending "region/model" as model ID to AWS
     model_info = {}
     litellm_params = {
-        "model": "bedrock/us-gov-west-1/anthropic.claude-3-7-sonnet-20250219-v1:0",
+        "model": "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0",
         "api_key": "fake_key",
     }
     updated_params = _update_litellm_params_for_health_check(model_info, litellm_params)
-    assert updated_params["model"] == "anthropic.claude-3-7-sonnet-20250219-v1:0"
+    assert updated_params["model"] == "anthropic.claude-sonnet-4-5-20250929-v1:0"
 
     # Test with Bedrock cross-region inference profile - should preserve the inference profile prefix
     # AWS requires inference profile IDs like "us.anthropic.claude..." for cross-region routing
diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py
index d5df4ef75a..20af6e1023 100644
--- a/tests/litellm_utils_tests/test_utils.py
+++ b/tests/litellm_utils_tests/test_utils.py
@@ -2309,11 +2309,11 @@ def test_get_provider_audio_transcription_config():
 @pytest.mark.parametrize(
     "model, expected_bool",
     [
-        ("anthropic.claude-3-7-sonnet-20250219-v1:0", True),
-        ("us.anthropic.claude-3-7-sonnet-20250219-v1:0", True),
+        ("anthropic.claude-sonnet-4-5-20250929-v1:0", True),
+        ("us.anthropic.claude-sonnet-4-5-20250929-v1:0", True),
     ],
 )
-def test_claude_3_7_sonnet_supports_pdf_input(model, expected_bool):
+def test_claude_sonnet_4_5_supports_pdf_input(model, expected_bool):
     from litellm.utils import supports_pdf_input
 
     assert supports_pdf_input(model) == expected_bool
diff --git a/tests/llm_translation/test_bedrock_anthropic_regression.py b/tests/llm_translation/test_bedrock_anthropic_regression.py
index 5928ca0223..8b8ce0a6cc 100644
--- a/tests/llm_translation/test_bedrock_anthropic_regression.py
+++ b/tests/llm_translation/test_bedrock_anthropic_regression.py
@@ -134,7 +134,7 @@ class TestBedrockAnthropicPromptCachingRegression:
         if "converse" in model_prefix:
             config = AmazonConverseConfig()
             result = config.transform_request(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 optional_params={},
                 litellm_params={},
@@ -162,7 +162,7 @@ class TestBedrockAnthropicPromptCachingRegression:
         else:
             config = AmazonAnthropicClaudeConfig()
             result = config.transform_request(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 optional_params={},
                 litellm_params={},
@@ -227,7 +227,7 @@ class TestBedrockAnthropicPromptCachingRegression:
         if "converse" in model_prefix:
             config = AmazonConverseConfig()
             result = config._transform_request_helper(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 system_content_blocks=[],
                 optional_params={},
                 messages=messages,
@@ -236,7 +236,7 @@ class TestBedrockAnthropicPromptCachingRegression:
         else:
             config = AmazonAnthropicClaudeConfig()
             result = config.transform_request(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 optional_params={},
                 litellm_params={},
@@ -498,7 +498,7 @@ class TestBedrockAnthropicCombinedRegressions:
         if "converse" in model_prefix:
             config = AmazonConverseConfig()
             result = config._transform_request_helper(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 system_content_blocks=[],
                 optional_params={},
                 messages=messages,
@@ -518,7 +518,7 @@ class TestBedrockAnthropicCombinedRegressions:
         else:
             config = AmazonAnthropicClaudeConfig()
             result = config.transform_request(
-                model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 optional_params={},
                 litellm_params={},
diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py
index ddfe383f2a..15f950224d 100644
--- a/tests/llm_translation/test_bedrock_completion.py
+++ b/tests/llm_translation/test_bedrock_completion.py
@@ -1323,7 +1323,7 @@ def test_base_aws_llm_get_credentials():
 def test_bedrock_completion_test_2():
     litellm.set_verbose = True
     data = {
-        "model": "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0",
+        "model": "bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0",
         "messages": [
             {
                 "role": "system",
@@ -1630,7 +1630,7 @@ def test_bedrock_completion_test_4(modify_params):
     litellm.modify_params = modify_params
 
     data = {
-        "model": "anthropic.claude-3-7-sonnet-20250219-v1:0",
+        "model": "anthropic.claude-sonnet-4-5-20250929-v1:0",
         "messages": [
             {
                 "role": "user",
@@ -2115,7 +2115,7 @@ class TestBedrockConverseAnthropicUnitTests(BaseAnthropicChatTest):
 
     def get_base_completion_call_args_with_thinking(self) -> dict:
         return {
-            "model": "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+            "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
             "thinking": {"type": "enabled", "budget_tokens": 16000},
         }
 
@@ -2828,7 +2828,7 @@ async def test_bedrock_thinking_in_assistant_message(sync_mode):
         client = AsyncHTTPHandler()
 
     params = {
-        "model": "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         "messages": [
             {
                 "role": "assistant",
@@ -2887,7 +2887,7 @@ async def test_bedrock_stream_thinking_content_openwebui():
     ```
     """
     response = await litellm.acompletion(
-        model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         messages=[{"role": "user", "content": "Hello who is this?"}],
         stream=True,
         max_tokens=1080,
diff --git a/tests/llm_translation/test_litellm_proxy_provider.py b/tests/llm_translation/test_litellm_proxy_provider.py
index 8fc961d12d..8b6f37bfbc 100644
--- a/tests/llm_translation/test_litellm_proxy_provider.py
+++ b/tests/llm_translation/test_litellm_proxy_provider.py
@@ -580,7 +580,7 @@ def test_litellm_gateway_from_sdk_with_response_cost_in_additional_headers():
 def test_litellm_gateway_from_sdk_with_thinking_param():
     try:
         response = litellm.completion(
-            model="litellm_proxy/anthropic.claude-3-7-sonnet-20250219-v1:0",
+            model="litellm_proxy/anthropic.claude-sonnet-4-5-20250929-v1:0",
             messages=[{"role": "user", "content": "Hello world"}],
             api_base="http://0.0.0.0:4000",
             api_key="sk-PIp1h0RekR",
diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py
index 82a3d96b02..b40ce11bb9 100644
--- a/tests/llm_translation/test_optional_params.py
+++ b/tests/llm_translation/test_optional_params.py
@@ -1828,7 +1828,7 @@ def test_azure_response_format_param():
     "model, provider",
     [
         ("claude-3-7-sonnet-20240620-v1:0", "anthropic"),
-        ("anthropic.claude-3-7-sonnet-20250219-v1:0", "bedrock"),
+        ("anthropic.claude-sonnet-4-5-20250929-v1:0", "bedrock"),
         ("invoke/anthropic.claude-3-7-sonnet-20240620-v1:0", "bedrock"),
         ("claude-3-7-sonnet@20250219", "vertex_ai"),
     ],
diff --git a/tests/local_testing/test_function_calling.py b/tests/local_testing/test_function_calling.py
index b52805c066..02affa1d57 100644
--- a/tests/local_testing/test_function_calling.py
+++ b/tests/local_testing/test_function_calling.py
@@ -159,7 +159,7 @@ def test_aaparallel_function_call(model):
     "model",
     [
         "anthropic/claude-4-sonnet-20250514",
-        "bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
     ],
 )
 @pytest.mark.flaky(retries=3, delay=1)
diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
index 3c71af97c9..d6502afbe7 100644
--- a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
+++ b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
@@ -96,8 +96,8 @@ class BaseAnthropicMessagesPromptCachingTest(ABC):
         Returns the model string to use for tests.
 
         Examples:
-        - "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0"
-        - "bedrock/invoke/anthropic.claude-3-7-sonnet-20250219-v1:0"
+        - "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0"
+        - "bedrock/invoke/anthropic.claude-3-5-sonnet-20241022-v2:0"
         """
         pass
 
diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
index 83a47a0149..bfdbf75351 100644
--- a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
+++ b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
@@ -31,7 +31,7 @@ class TestBedrockConversePromptCaching(BaseAnthropicMessagesPromptCachingTest):
     """
 
     def get_model(self) -> str:
-        return "bedrock/converse/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+        return "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
 
 
 class TestBedrockInvokePromptCaching(BaseAnthropicMessagesPromptCachingTest):
@@ -43,4 +43,4 @@ class TestBedrockInvokePromptCaching(BaseAnthropicMessagesPromptCachingTest):
     """
 
     def get_model(self) -> str:
-        return "bedrock/invoke/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+        return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json
index 913e3bfeda..818e4fa3ea 100644
--- a/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json
+++ b/tests/test_litellm/integrations/open_telemetry/data/captured_kwargs.json
@@ -1 +1 @@
-{"litellm_trace_id": null, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "input": [{"role": "user", "content": "What is the capital of France?"}], "litellm_params": {"acompletion": true, "api_key": null, "force_timeout": 600, "logger_fn": null, "verbose": false, "custom_llm_provider": "bedrock", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "model_alias_map": {}, "completion_call_id": null, "aembedding": null, "metadata": {"requester_metadata": {}, "user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_user_id": null, "user_api_key_org_id": null, "user_api_key_team_alias": null, "user_api_key_end_user_id": null, "user_api_key_user_email": null, "user_api_key": "unused-for-aws-bedrock", "user_api_end_user_max_budget": null, "litellm_api_version": "1.72.3", "global_max_parallel_requests": null, "user_api_key_team_max_budget": null, "user_api_key_team_spend": null, "user_api_key_spend": 0.0, "user_api_key_max_budget": null, "user_api_key_model_max_budget": {}, "user_api_key_metadata": {}, "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "endpoint": "http://0.0.0.0:44444/chat/completions", "litellm_parent_otel_span": null, "requester_ip_address": "", "model_group": "claude-3-7-sonnet", "model_group_size": 1, "deployment": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "api_base": null, "caching_groups": null, "hidden_params": {"custom_llm_provider": "bedrock", "region_name": null, "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "api_base": null, "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "response_cost": 0.001047, "additional_headers": {"x-litellm-model-group": "claude-3-7-sonnet", "x-litellm-attempted-retries": 0, "x-litellm-attempted-fallbacks": 0}, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "litellm_overhead_time_ms": 231.156, "_response_ms": 236.798}}, "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "proxy_server_request": {"url": "http://0.0.0.0:44444/chat/completions", "method": "POST", "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "body": {"messages": [{"role": "user", "content": "What is the capital of France?"}], "model": "claude-3-7-sonnet", "stream": false}}, "preset_cache_key": null, "no-log": null, "stream_response": {}, "input_cost_per_token": null, "input_cost_per_second": null, "output_cost_per_token": null, "output_cost_per_second": null, "cooldown_time": null, "text_completion": null, "azure_ad_token_provider": null, "user_continue_message": null, "base_model": null, "litellm_trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "litellm_session_id": null, "hf_model_name": null, "custom_prompt_dict": {}, "litellm_metadata": null, "disable_add_transform_inline_image_block": null, "drop_params": null, "prompt_id": null, "prompt_variables": null, "async_call": null, "ssl_verify": null, "merge_reasoning_content_in_choices": false, "api_version": null, "azure_ad_token": null, "tenant_id": null, "client_id": null, "client_secret": null, "azure_username": null, "azure_password": null, "max_retries": 0, "timeout": 6000.0, "bucket_name": null, "vertex_credentials": null, "vertex_project": null, "use_litellm_proxy": false}, "applied_guardrails": [], "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "messages": [{"role": "user", "content": "What is the capital of France?"}], "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "start_time": "2025-06-22 10:59:08.159939", "stream": false, "user": null, "call_type": "acompletion", "completion_start_time": "2025-06-22 10:59:08.399523", "standard_callback_dynamic_params": {}, "stream_options": null, "max_retries": 0, "provider": "aws", "region": "us-west-2", "custom_llm_provider": "bedrock", "api_key": "", "additional_args": {"complete_input_dict": "{\"messages\": [{\"role\": \"user\", \"content\": [{\"text\": \"What is the capital of France?\"}]}], \"additionalModelRequestFields\": {\"provider\": \"aws\", \"region\": \"us-west-2\"}, \"system\": [], \"inferenceConfig\": {}}"}, "log_event_type": "post_api_call", "api_call_start_time": "2025-06-22 10:59:08.387641", "llm_api_duration_ms": 5.642, "original_response": "{\"metrics\":{\"latencyMs\":1513},\"output\":{\"message\":{\"content\":[{\"text\":\"The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.\"}],\"role\":\"assistant\"}},\"stopReason\":\"end_turn\",\"usage\":{\"cacheReadInputTokenCount\":0,\"cacheReadInputTokens\":0,\"cacheWriteInputTokenCount\":0,\"cacheWriteInputTokens\":0,\"inputTokens\":14,\"outputTokens\":67,\"totalTokens\":81}}", "end_time": "2025-06-22 10:59:08.399523", "cache_hit": null, "response_cost": 0.001047, "standard_logging_object": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "call_type": "acompletion", "cache_hit": null, "stream": true, "status": "success", "custom_llm_provider": "bedrock", "saved_cache_cost": 0.0, "startTime": 1750615148.162725, "endTime": 1750615148.399523, "completionStartTime": 1750615148.399523, "response_time": 0.23679804801940918, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "metadata": {"user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_org_id": null, "user_api_key_user_id": null, "user_api_key_team_alias": null, "user_api_key_user_email": null, "spend_logs_metadata": null, "requester_ip_address": "", "requester_metadata": {}, "user_api_key_end_user_id": null, "prompt_management_metadata": null, "applied_guardrails": [], "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "requester_custom_headers": {"x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600"}}, "cache_key": null, "response_cost": 0.001047, "total_tokens": 81, "prompt_tokens": 14, "completion_tokens": 67, "request_tags": [], "end_user": "", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-3-7-sonnet-20250219-v1%3A0/converse", "model_group": "claude-3-7-sonnet", "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "requester_ip_address": "", "messages": [{"role": "user", "content": "What is the capital of France?"}], "response": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}, "model_parameters": {"stream": false}, "hidden_params": {"model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "cache_key": null, "api_base": null, "response_cost": 0.001047, "additional_headers": {}, "litellm_overhead_time_ms": 231.156, "batch_models": null, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "usage_object": null}, "model_map_information": {"model_map_key": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "model_map_value": {"key": "anthropic.claude-3-7-sonnet-20250219-v1:0", "max_tokens": 8192, "max_input_tokens": 200000, "max_output_tokens": 8192, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_reasoning_token": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "bedrock_converse", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": null, "supports_audio_output": null, "supports_pdf_input": true, "supports_embedding_image_input": null, "supports_native_streaming": null, "supports_web_search": null, "supports_url_context": null, "supports_reasoning": true, "supports_computer_use": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["max_tokens", "max_completion_tokens", "stream", "stream_options", "stop", "temperature", "top_p", "extra_headers", "response_format", "tools", "tool_choice", "thinking", "reasoning_effort"]}}, "error_str": null, "error_information": {"error_code": "", "error_class": "", "llm_provider": "", "traceback": "", "error_message": ""}, "response_cost_failure_debug_info": null, "guardrail_information": null, "standard_built_in_tools_params": {"web_search_options": null, "file_search": null}}, "async_complete_streaming_response": "ModelResponse(id='chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1', created=1750615148, model='arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None))], usage=Usage(completion_tokens=67, prompt_tokens=14, total_tokens=81, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0))"}
\ No newline at end of file
+{"litellm_trace_id": null, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "input": [{"role": "user", "content": "What is the capital of France?"}], "litellm_params": {"acompletion": true, "api_key": null, "force_timeout": 600, "logger_fn": null, "verbose": false, "custom_llm_provider": "bedrock", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "model_alias_map": {}, "completion_call_id": null, "aembedding": null, "metadata": {"requester_metadata": {}, "user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_user_id": null, "user_api_key_org_id": null, "user_api_key_team_alias": null, "user_api_key_end_user_id": null, "user_api_key_user_email": null, "user_api_key": "unused-for-aws-bedrock", "user_api_end_user_max_budget": null, "litellm_api_version": "1.72.3", "global_max_parallel_requests": null, "user_api_key_team_max_budget": null, "user_api_key_team_spend": null, "user_api_key_spend": 0.0, "user_api_key_max_budget": null, "user_api_key_model_max_budget": {}, "user_api_key_metadata": {}, "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "endpoint": "http://0.0.0.0:44444/chat/completions", "litellm_parent_otel_span": null, "requester_ip_address": "", "model_group": "claude-3-7-sonnet", "model_group_size": 1, "deployment": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "api_base": null, "caching_groups": null, "hidden_params": {"custom_llm_provider": "bedrock", "region_name": null, "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "litellm_call_id": "dbecd23a-e71a-49cf-90d4-712a8a8e29c5", "api_base": null, "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "response_cost": 0.001047, "additional_headers": {"x-litellm-model-group": "claude-3-7-sonnet", "x-litellm-attempted-retries": 0, "x-litellm-attempted-fallbacks": 0}, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "litellm_overhead_time_ms": 231.156, "_response_ms": 236.798}}, "model_info": {"id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "db_model": false}, "proxy_server_request": {"url": "http://0.0.0.0:44444/chat/completions", "method": "POST", "headers": {"host": "0.0.0.0:44444", "accept-encoding": "gzip, deflate, zstd", "connection": "keep-alive", "accept": "application/json", "content-type": "application/json", "user-agent": "AsyncOpenAI/Python 1.84.0", "x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600", "content-length": "116"}, "body": {"messages": [{"role": "user", "content": "What is the capital of France?"}], "model": "claude-3-7-sonnet", "stream": false}}, "preset_cache_key": null, "no-log": null, "stream_response": {}, "input_cost_per_token": null, "input_cost_per_second": null, "output_cost_per_token": null, "output_cost_per_second": null, "cooldown_time": null, "text_completion": null, "azure_ad_token_provider": null, "user_continue_message": null, "base_model": null, "litellm_trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "litellm_session_id": null, "hf_model_name": null, "custom_prompt_dict": {}, "litellm_metadata": null, "disable_add_transform_inline_image_block": null, "drop_params": null, "prompt_id": null, "prompt_variables": null, "async_call": null, "ssl_verify": null, "merge_reasoning_content_in_choices": false, "api_version": null, "azure_ad_token": null, "tenant_id": null, "client_id": null, "client_secret": null, "azure_username": null, "azure_password": null, "max_retries": 0, "timeout": 6000.0, "bucket_name": null, "vertex_credentials": null, "vertex_project": null, "use_litellm_proxy": false}, "applied_guardrails": [], "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "messages": [{"role": "user", "content": "What is the capital of France?"}], "optional_params": {"stream": false, "max_retries": 0, "provider": "aws", "region": "us-west-2"}, "start_time": "2025-06-22 10:59:08.159939", "stream": false, "user": null, "call_type": "acompletion", "completion_start_time": "2025-06-22 10:59:08.399523", "standard_callback_dynamic_params": {}, "stream_options": null, "max_retries": 0, "provider": "aws", "region": "us-west-2", "custom_llm_provider": "bedrock", "api_key": "", "additional_args": {"complete_input_dict": "{\"messages\": [{\"role\": \"user\", \"content\": [{\"text\": \"What is the capital of France?\"}]}], \"additionalModelRequestFields\": {\"provider\": \"aws\", \"region\": \"us-west-2\"}, \"system\": [], \"inferenceConfig\": {}}"}, "log_event_type": "post_api_call", "api_call_start_time": "2025-06-22 10:59:08.387641", "llm_api_duration_ms": 5.642, "original_response": "{\"metrics\":{\"latencyMs\":1513},\"output\":{\"message\":{\"content\":[{\"text\":\"The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.\"}],\"role\":\"assistant\"}},\"stopReason\":\"end_turn\",\"usage\":{\"cacheReadInputTokenCount\":0,\"cacheReadInputTokens\":0,\"cacheWriteInputTokenCount\":0,\"cacheWriteInputTokens\":0,\"inputTokens\":14,\"outputTokens\":67,\"totalTokens\":81}}", "end_time": "2025-06-22 10:59:08.399523", "cache_hit": null, "response_cost": 0.001047, "standard_logging_object": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "trace_id": "4c97150b-b1a3-4dec-bd7a-734786b1b3bc", "call_type": "acompletion", "cache_hit": null, "stream": true, "status": "success", "custom_llm_provider": "bedrock", "saved_cache_cost": 0.0, "startTime": 1750615148.162725, "endTime": 1750615148.399523, "completionStartTime": 1750615148.399523, "response_time": 0.23679804801940918, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "metadata": {"user_api_key_hash": "unused-for-aws-bedrock", "user_api_key_alias": null, "user_api_key_team_id": null, "user_api_key_org_id": null, "user_api_key_user_id": null, "user_api_key_team_alias": null, "user_api_key_user_email": null, "spend_logs_metadata": null, "requester_ip_address": "", "requester_metadata": {}, "user_api_key_end_user_id": null, "prompt_management_metadata": null, "applied_guardrails": [], "mcp_tool_call_metadata": null, "vector_store_request_metadata": null, "usage_object": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}, "requester_custom_headers": {"x-stainless-lang": "python", "x-stainless-package-version": "1.84.0", "x-stainless-os": "MacOS", "x-stainless-arch": "arm64", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.10", "x-stainless-async": "async:asyncio", "x-stainless-retry-count": "0", "x-stainless-read-timeout": "600"}}, "cache_key": null, "response_cost": 0.001047, "total_tokens": 81, "prompt_tokens": 14, "completion_tokens": 67, "request_tags": [], "end_user": "", "api_base": "https://bedrock-runtime.us-west-2.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-west-2%3A1234567890123%3Ainference-profile%2Fus.anthropic.claude-sonnet-4-5-20250929-v1%3A0/converse", "model_group": "claude-3-7-sonnet", "model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "requester_ip_address": "", "messages": [{"role": "user", "content": "What is the capital of France?"}], "response": {"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}, "model_parameters": {"stream": false}, "hidden_params": {"model_id": "6bace4d6db0105943b3b0bfe7eb1a62c06e6f16f008cc4673fdf918eb3e9e62a", "cache_key": null, "api_base": null, "response_cost": 0.001047, "additional_headers": {}, "litellm_overhead_time_ms": 231.156, "batch_models": null, "litellm_model_name": "bedrock/arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "usage_object": null}, "model_map_information": {"model_map_key": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "model_map_value": {"key": "anthropic.claude-sonnet-4-5-20250929-v1:0", "max_tokens": 8192, "max_input_tokens": 200000, "max_output_tokens": 8192, "input_cost_per_token": 3e-06, "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_character": null, "input_cost_per_token_above_128k_tokens": null, "input_cost_per_token_above_200k_tokens": null, "input_cost_per_query": null, "input_cost_per_second": null, "input_cost_per_audio_token": null, "input_cost_per_token_batches": null, "output_cost_per_token_batches": null, "output_cost_per_token": 1.5e-05, "output_cost_per_audio_token": null, "output_cost_per_character": null, "output_cost_per_reasoning_token": null, "output_cost_per_token_above_128k_tokens": null, "output_cost_per_character_above_128k_tokens": null, "output_cost_per_token_above_200k_tokens": null, "output_cost_per_second": null, "output_cost_per_image": null, "output_vector_size": null, "litellm_provider": "bedrock_converse", "mode": "chat", "supports_system_messages": null, "supports_response_schema": true, "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, "supports_assistant_prefill": true, "supports_prompt_caching": true, "supports_audio_input": null, "supports_audio_output": null, "supports_pdf_input": true, "supports_embedding_image_input": null, "supports_native_streaming": null, "supports_web_search": null, "supports_url_context": null, "supports_reasoning": true, "supports_computer_use": true, "search_context_cost_per_query": null, "tpm": null, "rpm": null, "supported_openai_params": ["max_tokens", "max_completion_tokens", "stream", "stream_options", "stop", "temperature", "top_p", "extra_headers", "response_format", "tools", "tool_choice", "thinking", "reasoning_effort"]}}, "error_str": null, "error_information": {"error_code": "", "error_class": "", "llm_provider": "", "traceback": "", "error_message": ""}, "response_cost_failure_debug_info": null, "guardrail_information": null, "standard_built_in_tools_params": {"web_search_options": null, "file_search": null}}, "async_complete_streaming_response": "ModelResponse(id='chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1', created=1750615148, model='arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None))], usage=Usage(completion_tokens=67, prompt_tokens=14, total_tokens=81, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None), cache_creation_input_tokens=0, cache_read_input_tokens=0))"}
\ No newline at end of file
diff --git a/tests/test_litellm/integrations/open_telemetry/data/captured_response.json b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json
index 3cf77781cc..1fa1889909 100644
--- a/tests/test_litellm/integrations/open_telemetry/data/captured_response.json
+++ b/tests/test_litellm/integrations/open_telemetry/data/captured_response.json
@@ -1 +1 @@
-{"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}
\ No newline at end of file
+{"id": "chatcmpl-fa9be5b7-9487-46ab-86de-6462d578fea1", "created": 1750615148, "model": "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0", "object": "chat.completion", "system_fingerprint": null, "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "The capital of France is Paris. Paris has been the capital city of France since 987 CE when Hugh Capet, the first king of the Capetian dynasty, made the city his seat of government. Today, Paris is not only the political capital but also the cultural and economic center of France.", "role": "assistant", "tool_calls": null, "function_call": null}}], "usage": {"completion_tokens": 67, "prompt_tokens": 14, "total_tokens": 81, "completion_tokens_details": null, "prompt_tokens_details": {"audio_tokens": null, "cached_tokens": 0}, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}}
\ No newline at end of file
diff --git a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
index 271d58061a..1a4d03528e 100644
--- a/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
+++ b/tests/test_litellm/integrations/test_anthropic_cache_control_hook.py
@@ -220,7 +220,7 @@ async def test_anthropic_cache_control_hook_negative_indices():
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             # Test with multiple messages and negative indices
             response = await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=[
                     {
                         "role": "system",
@@ -352,7 +352,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds_logging():
                 ]
 
                 await litellm.acompletion(
-                    model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                    model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                     messages=messages,
                     cache_control_injection_points=[
                         {"location": "message", "index": 10}
@@ -420,7 +420,7 @@ async def test_anthropic_cache_control_hook_negative_out_of_bounds_logging():
                 ]
 
                 await litellm.acompletion(
-                    model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                    model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                     messages=messages,
                     cache_control_injection_points=[
                         {
@@ -486,7 +486,7 @@ async def test_anthropic_cache_control_hook_multiple_user_messages():
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             # Test with multiple user messages and negative indices
             response = await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=[
                     {
                         "role": "user",
@@ -586,7 +586,7 @@ async def test_anthropic_cache_control_hook_out_of_bounds(bad_index):
             ]
 
             await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 cache_control_injection_points=[
                     {"location": "message", "index": bad_index}
@@ -651,7 +651,7 @@ async def test_anthropic_cache_control_hook_single_message(message_list):
         client = AsyncHTTPHandler()
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=message_list,
                 cache_control_injection_points=[{"location": "message", "index": -1}],
                 client=client,
@@ -691,7 +691,7 @@ async def test_anthropic_cache_control_hook_empty_message_list():
                 match="bedrock requires at least one non-system message",
             ):
                 await litellm.acompletion(
-                    model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                    model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                     messages=[],
                     cache_control_injection_points=[
                         {"location": "message", "index": -1}
@@ -742,7 +742,7 @@ async def test_anthropic_cache_control_hook_no_op():
             ]
 
             await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=messages,
                 # No cache_control_injection_points parameter
                 client=client,
@@ -799,7 +799,7 @@ async def test_anthropic_cache_control_hook_multiple_content_items_last_only():
         client = AsyncHTTPHandler()
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             response = await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=[
                     {
                         "role": "user",
@@ -874,7 +874,7 @@ async def test_anthropic_cache_control_hook_document_analysis_multiple_pages():
         client = AsyncHTTPHandler()
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             response = await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=[
                     {
                         "role": "user",
@@ -1057,7 +1057,7 @@ async def test_anthropic_cache_control_hook_string_negative_index():
         client = AsyncHTTPHandler()
         with patch.object(client, "post", return_value=mock_response) as mock_post:
             await litellm.acompletion(
-                model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+                model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
                 messages=[
                     {"role": "user", "content": "First message"},
                     {"role": "assistant", "content": "First response"},
diff --git a/tests/test_litellm/integrations/test_opentelemetry.py b/tests/test_litellm/integrations/test_opentelemetry.py
index f710647189..b31bbca889 100644
--- a/tests/test_litellm/integrations/test_opentelemetry.py
+++ b/tests/test_litellm/integrations/test_opentelemetry.py
@@ -262,7 +262,7 @@ class TestOpenTelemetryProviderInitialization(unittest.TestCase):
 class TestOpenTelemetry(unittest.TestCase):
     POLL_INTERVAL = 0.05
     POLL_TIMEOUT = 2.0
-    MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-3-7-sonnet-20250219-v1:0"
+    MODEL = "arn:aws:bedrock:us-west-2:1234567890123:inference-profile/us.anthropic.claude-sonnet-4-5-20250929-v1:0"
     HERE = os.path.dirname(__file__)
 
     @patch.dict(os.environ, {}, clear=True)
diff --git a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
index 72cfd89408..f8708dd2f7 100644
--- a/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
+++ b/tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py
@@ -77,7 +77,7 @@ async def test_anthropic_bedrock_thinking_blocks_with_none_content():
     # test _bedrock_converse_messages_pt_async
     result = await BedrockConverseMessagesProcessor._bedrock_converse_messages_pt_async(
         messages=messages,
-        model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         llm_provider="bedrock",
     )
 
diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
index 38a59c694e..8e53e57f1e 100644
--- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
+++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
@@ -279,7 +279,7 @@ def test_reasoning_with_forced_tool_choice_switches_to_auto():
     }
 
     optional_params = config.map_openai_params(
-        model="bedrock/us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         non_default_params=non_default_params,
         optional_params={},
         drop_params=False,
@@ -2797,7 +2797,7 @@ def test_thinking_with_max_completion_tokens():
     result = config.map_openai_params(
         non_default_params=non_default_params_with_max_completion,
         optional_params=optional_params,
-        model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         drop_params=False,
     )
 
@@ -2819,7 +2819,7 @@ def test_thinking_with_max_completion_tokens():
     result = config.map_openai_params(
         non_default_params=non_default_params_with_max_tokens,
         optional_params=optional_params,
-        model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         drop_params=False,
     )
 
@@ -2842,7 +2842,7 @@ def test_thinking_with_max_completion_tokens():
     result = config.map_openai_params(
         non_default_params=non_default_params_without_max,
         optional_params=optional_params,
-        model="us.anthropic.claude-3-7-sonnet-20250219-v1:0",
+        model="us.anthropic.claude-sonnet-4-5-20250929-v1:0",
         drop_params=False,
     )
 
@@ -3617,7 +3617,7 @@ class TestBedrockMinThinkingBudgetTokens:
     """Test that thinking.budget_tokens is clamped to the Bedrock minimum (1024)."""
 
     def _map_params(
-        self, thinking_value, model="anthropic.claude-3-7-sonnet-20250219-v1:0"
+        self, thinking_value, model="anthropic.claude-sonnet-4-5-20250929-v1:0"
     ):
         """Helper to call map_openai_params with the given thinking value."""
         config = AmazonConverseConfig()
@@ -3651,7 +3651,7 @@ class TestBedrockMinThinkingBudgetTokens:
         result = config.map_openai_params(
             non_default_params={},
             optional_params={},
-            model="anthropic.claude-3-7-sonnet-20250219-v1:0",
+            model="anthropic.claude-sonnet-4-5-20250929-v1:0",
             drop_params=False,
         )
         assert "thinking" not in result or result.get("thinking") is None
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index 93c61e003d..b8a4220c67 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -1179,7 +1179,7 @@ def test_get_model_info_shows_supports_computer_use():
     "model, custom_llm_provider",
     [
         ("gpt-3.5-turbo", "openai"),
-        ("anthropic.claude-3-7-sonnet-20250219-v1:0", "bedrock"),
+        ("anthropic.claude-sonnet-4-5-20250929-v1:0", "bedrock"),
         ("gemini-2.5-pro", "vertex_ai"),
     ],
 )
@@ -1325,7 +1325,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/bedrock-claude-3-opus",
-                "bedrock/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
             ),
             (
@@ -1623,7 +1623,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/bedrock-claude-3-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Bedrock Claude 3 Opus via Converse API",
             ),
@@ -1710,7 +1710,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/staging-claude-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Staging Claude Opus",
             ),
@@ -1722,7 +1722,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/high-performance-claude",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "High-performance Claude deployment",
             ),
@@ -1860,7 +1860,7 @@ class TestProxyFunctionCalling:
         bedrock_models = [
             "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0",
             "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
-            "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+            "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
         ]
 
         for model in bedrock_models:
@@ -1892,7 +1892,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/bedrock-claude-3-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Bedrock Claude 3 Opus via Converse API",
             ),
@@ -1979,7 +1979,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/staging-claude-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Staging Claude Opus",
             ),
@@ -1991,7 +1991,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/high-performance-claude",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "High-performance Claude deployment",
             ),
@@ -2129,7 +2129,7 @@ class TestProxyFunctionCalling:
         bedrock_models = [
             "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0",
             "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
-            "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+            "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
         ]
 
         for model in bedrock_models:
@@ -2161,7 +2161,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/bedrock-claude-3-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Bedrock Claude 3 Opus via Converse API",
             ),
@@ -2248,7 +2248,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/staging-claude-opus",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "Staging Claude Opus",
             ),
@@ -2260,7 +2260,7 @@ class TestProxyFunctionCalling:
             ),
             (
                 "litellm_proxy/high-performance-claude",
-                "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+                "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
                 False,
                 "High-performance Claude deployment",
             ),
@@ -2398,7 +2398,7 @@ class TestProxyFunctionCalling:
         bedrock_models = [
             "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0",
             "bedrock/converse/anthropic.claude-3-sonnet-20240229-v1:0",
-            "bedrock/converse/anthropic.claude-3-7-sonnet-20250219-v1:0",
+            "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0",
         ]
 
         for model in bedrock_models:

From b1a0a3fc17d616ad2993a4c73f4eecbf31ef005c Mon Sep 17 00:00:00 2001
From: Ryan Crabbe <ryan@berri.ai>
Date: Tue, 28 Apr 2026 14:51:47 -0700
Subject: [PATCH 08/19] fix(tests): use Sonnet 4.5 for Bedrock invoke
 prompt-caching tests

Claude 3.5 Sonnet v2 reached EOL on Bedrock 2026-03-01, returning the same
404 EOL error as 3.7 Sonnet. Sonnet 4.5 supports both InvokeModel and
Converse APIs on Bedrock, so use the same model for both routes.
---
 .../base_anthropic_messages_prompt_caching_test.py              | 2 +-
 .../test_anthropic_messages_prompt_caching.py                   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
index d6502afbe7..5fc4ecefb3 100644
--- a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
+++ b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py
@@ -97,7 +97,7 @@ class BaseAnthropicMessagesPromptCachingTest(ABC):
 
         Examples:
         - "bedrock/converse/anthropic.claude-sonnet-4-5-20250929-v1:0"
-        - "bedrock/invoke/anthropic.claude-3-5-sonnet-20241022-v2:0"
+        - "bedrock/invoke/anthropic.claude-sonnet-4-5-20250929-v1:0"
         """
         pass
 
diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
index bfdbf75351..a194ded12f 100644
--- a/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
+++ b/tests/pass_through_unit_tests/test_anthropic_messages_prompt_caching.py
@@ -43,4 +43,4 @@ class TestBedrockInvokePromptCaching(BaseAnthropicMessagesPromptCachingTest):
     """
 
     def get_model(self) -> str:
-        return "bedrock/invoke/us.anthropic.claude-3-5-sonnet-20241022-v2:0"
+        return "bedrock/invoke/us.anthropic.claude-sonnet-4-5-20250929-v1:0"

From 6052ce1017aa27e7692da2d0664bfe91f659acfc Mon Sep 17 00:00:00 2001
From: Michael Riad Zaky <michaelr@Michaels-MacBook-Air.local>
Date: Fri, 24 Apr 2026 16:44:50 -0700
Subject: [PATCH 09/19] cache LiteLLM_Config param reads in DualCache + batch
 scheduler-tick fetch

---
 litellm/proxy/proxy_server.py                 | 55 +++++++++---
 litellm/proxy/utils.py                        | 89 +++++++++++++++++++
 tests/test_litellm/proxy/test_proxy_server.py | 20 +++++
 3 files changed, 150 insertions(+), 14 deletions(-)

diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 007dbe5fa7..8f676df04c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -497,14 +497,18 @@ from litellm.proxy.utils import (
     _get_redoc_url,
     _is_projected_spend_over_limit,
     _is_valid_team_configs,
+    get_config_param,
     get_custom_url,
     get_error_message_str,
     get_server_root_path,
     handle_exception_on_proxy,
     hash_password,
     hash_token,
+    invalidate_config_param,
+    litellm_config_cache,
     migrate_passwords_to_scrypt_async,
     model_dump_with_preserved_fields,
+    prefetch_config_params,
     update_spend,
 )
 from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router
@@ -2929,8 +2933,13 @@ class ProxyConfig:
             ## INIT PROXY REDIS USAGE CLIENT ##
             redis_usage_cache = litellm.cache.cache
             spend_counter_cache.redis_cache = redis_usage_cache
+            litellm_config_cache.redis_cache = redis_usage_cache
             # Note: PKCE verifier storage uses redis_usage_cache directly (not
             # user_api_key_cache) to avoid routing all API-key lookups through Redis.
+        elif litellm_config_cache.redis_cache is None:
+            verbose_proxy_logger.info(
+                "litellm_config_cache: no Redis configured; cluster-wide cache sharing disabled."
+            )
 
     def switch_on_llm_response_caching(self):
         """
@@ -4846,10 +4855,7 @@ class ProxyConfig:
             "environment_variables",
         ]
         for k in keys:
-            response = prisma_client.get_generic_data(
-                key="param_name", value=k, table_name="config"
-            )
-            _tasks.append(response)
+            _tasks.append(get_config_param(prisma_client, k))
 
         responses = await asyncio.gather(*_tasks)
         for response in responses:
@@ -4931,6 +4937,19 @@ class ProxyConfig:
         global llm_router, llm_model_list, master_key, general_settings
 
         try:
+            # warm the config cache so the per-param reads below all hit
+            await prefetch_config_params(
+                prisma_client,
+                [
+                    "general_settings",
+                    "router_settings",
+                    "litellm_settings",
+                    "environment_variables",
+                    "model_cost_map_reload_config",
+                    "anthropic_beta_headers_reload_config",
+                ],
+            )
+
             # Only load models from DB if "models" is in supported_db_objects (or if supported_db_objects is not set)
             if self._should_load_db_object(object_type="models"):
                 new_models = await self._get_models_from_db(prisma_client=prisma_client)
@@ -4940,8 +4959,8 @@ class ProxyConfig:
                     new_models=new_models, proxy_logging_obj=proxy_logging_obj
                 )
 
-            db_general_settings = await prisma_client.db.litellm_config.find_first(
-                where={"param_name": "general_settings"}
+            db_general_settings = await get_config_param(
+                prisma_client, "general_settings"
             )
 
             # update general settings
@@ -5034,10 +5053,7 @@ class ProxyConfig:
         from litellm.proxy.hooks.mcp_semantic_filter import SemanticToolFilterHook
 
         try:
-            # Load litellm_settings from DB
-            config_record = await prisma_client.db.litellm_config.find_unique(
-                where={"param_name": "litellm_settings"}
-            )
+            config_record = await get_config_param(prisma_client, "litellm_settings")
 
             if config_record is None or config_record.param_value is None:
                 return
@@ -5192,8 +5208,8 @@ class ProxyConfig:
         """
         try:
             # Get model cost map reload configuration from database
-            config_record = await prisma_client.db.litellm_config.find_unique(
-                where={"param_name": "model_cost_map_reload_config"}
+            config_record = await get_config_param(
+                prisma_client, "model_cost_map_reload_config"
             )
 
             if config_record is None or config_record.param_value is None:
@@ -5288,6 +5304,7 @@ class ProxyConfig:
                         },
                     },
                 )
+                await invalidate_config_param("model_cost_map_reload_config")
 
                 verbose_proxy_logger.info(
                     f"Model cost map reloaded successfully. Models count: {len(new_model_cost_map) if new_model_cost_map else 0}"
@@ -5307,8 +5324,8 @@ class ProxyConfig:
         """
         try:
             # Get anthropic beta headers reload configuration from database
-            config_record = await prisma_client.db.litellm_config.find_unique(
-                where={"param_name": "anthropic_beta_headers_reload_config"}
+            config_record = await get_config_param(
+                prisma_client, "anthropic_beta_headers_reload_config"
             )
 
             if config_record is None or config_record.param_value is None:
@@ -5396,6 +5413,7 @@ class ProxyConfig:
                         },
                     },
                 )
+                await invalidate_config_param("anthropic_beta_headers_reload_config")
 
                 # Count providers in config
                 provider_count = sum(
@@ -12674,6 +12692,7 @@ async def update_config(  # noqa: PLR0915
                         "update": {"param_value": v},
                     },
                 )
+                await invalidate_config_param(k)
 
         ### OLD LOGIC [TODO] MOVE TO DB ###
 
@@ -12861,6 +12880,7 @@ async def update_config_general_settings(
             "update": {"param_value": json.dumps(general_settings)},  # type: ignore
         },
     )
+    await invalidate_config_param("general_settings")
 
     return response
 
@@ -13144,6 +13164,7 @@ async def delete_config_general_settings(
             "update": {"param_value": json.dumps(general_settings)},  # type: ignore
         },
     )
+    await invalidate_config_param("general_settings")
 
     return response
 
@@ -13509,6 +13530,7 @@ async def reload_model_cost_map(
                 },
             },
         )
+        await invalidate_config_param("model_cost_map_reload_config")
 
         models_count = len(new_model_cost_map) if new_model_cost_map else 0
         verbose_proxy_logger.info(
@@ -13578,6 +13600,7 @@ async def schedule_model_cost_map_reload(
                 },
             },
         )
+        await invalidate_config_param("model_cost_map_reload_config")
 
         verbose_proxy_logger.info(
             f"Model cost map reload scheduled for every {hours} hours"
@@ -13631,6 +13654,7 @@ async def cancel_model_cost_map_reload(
         await prisma_client.db.litellm_config.delete(
             where={"param_name": "model_cost_map_reload_config"}
         )
+        await invalidate_config_param("model_cost_map_reload_config")
 
         verbose_proxy_logger.info("Model cost map reload schedule cancelled")
 
@@ -13861,6 +13885,7 @@ async def reload_anthropic_beta_headers(
                 },
             },
         )
+        await invalidate_config_param("anthropic_beta_headers_reload_config")
 
         provider_count = sum(
             1 for k in new_config.keys() if k not in ["provider_aliases", "description"]
@@ -13934,6 +13959,7 @@ async def schedule_anthropic_beta_headers_reload(
                 },
             },
         )
+        await invalidate_config_param("anthropic_beta_headers_reload_config")
 
         verbose_proxy_logger.info(
             f"Anthropic beta headers reload scheduled for every {hours} hours"
@@ -13987,6 +14013,7 @@ async def cancel_anthropic_beta_headers_reload(
         await prisma_client.db.litellm_config.delete(
             where={"param_name": "anthropic_beta_headers_reload_config"}
         )
+        await invalidate_config_param("anthropic_beta_headers_reload_config")
 
         verbose_proxy_logger.info("Anthropic beta headers reload schedule cancelled")
 
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
index 712853a33c..3a1184c434 100644
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -2442,6 +2442,92 @@ async def _lookup_deprecated_key(
     return None
 
 
+# DualCache for LiteLLM_Config param_name reads.
+# Redis layer is attached in proxy_server._init_cache.
+LITELLM_CONFIG_CACHE_TTL_SECONDS: int = int(
+    os.environ.get("LITELLM_CONFIG_PARAM_CACHE_TTL_SECONDS", "60")
+)
+_CONFIG_CACHE_MISS: str = "__litellm_config_param_miss__"
+
+litellm_config_cache: DualCache = DualCache(
+    default_in_memory_ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS,
+    default_redis_ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS,
+)
+
+
+class _ConfigRow:
+    """Mimics the Prisma litellm_config row shape for cached entries."""
+
+    __slots__ = ("param_name", "param_value")
+
+    def __init__(self, param_name: str, param_value: Any) -> None:
+        self.param_name = param_name
+        self.param_value = param_value
+
+
+def _config_cache_key(param_name: str) -> str:
+    return f"litellm_config:param:{param_name}"
+
+
+def _pack_config_row(row: Any) -> Dict[str, Any]:
+    return {"param_name": row.param_name, "param_value": row.param_value}
+
+
+def _unpack_config_row(cached: Any) -> Optional[_ConfigRow]:
+    if cached is None or cached == _CONFIG_CACHE_MISS:
+        return None
+    if isinstance(cached, dict):
+        return _ConfigRow(cached["param_name"], cached["param_value"])
+    return None
+
+
+async def get_config_param(prisma_client: Any, param_name: str) -> Optional[Any]:
+    """Cached read of a LiteLLM_Config row; returns row, _ConfigRow shim, or None."""
+    cache_key = _config_cache_key(param_name)
+    cached = await litellm_config_cache.async_get_cache(cache_key)
+    if cached is not None:
+        return _unpack_config_row(cached)
+
+    row = await prisma_client.get_generic_data(
+        key="param_name", value=param_name, table_name="config"
+    )
+    cache_value: Any = _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
+    await litellm_config_cache.async_set_cache(
+        cache_key, cache_value, ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS
+    )
+    return row
+
+
+async def invalidate_config_param(param_name: str) -> None:
+    """Evict from both cache layers; call after every LiteLLM_Config write."""
+    await litellm_config_cache.async_delete_cache(_config_cache_key(param_name))
+
+
+async def prefetch_config_params(prisma_client: Any, param_names: List[str]) -> None:
+    """Batch-load LiteLLM_Config rows into the cache with one find_many."""
+    if not param_names:
+        return
+    try:
+        rows = await prisma_client.db.litellm_config.find_many(
+            where={"param_name": {"in": param_names}}  # type: ignore
+        )
+    except Exception as e:
+        verbose_proxy_logger.debug(
+            "prefetch_config_params failed, falling through to per-param queries: %s",
+            e,
+        )
+        return
+    by_name = {row.param_name: row for row in rows}
+    for name in param_names:
+        row = by_name.get(name)
+        cache_value: Any = (
+            _pack_config_row(row) if row is not None else _CONFIG_CACHE_MISS
+        )
+        await litellm_config_cache.async_set_cache(
+            _config_cache_key(name), cache_value, ttl=LITELLM_CONFIG_CACHE_TTL_SECONDS
+        )
+
+
 class PrismaClient:
     spend_log_transactions: List = []
     _spend_log_transactions_lock = asyncio.Lock()
@@ -3310,6 +3396,9 @@ class PrismaClient:
 
                     tasks.append(updated_table_row)
                 await asyncio.gather(*tasks)
+                # invalidate cache so other pods see writes from save_config
+                for k in data.keys():
+                    await invalidate_config_param(k)
                 verbose_proxy_logger.info("Data Inserted into Config Table")
             elif table_name == "spend":
                 db_data = self.jsonify_object(data=data)
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 3349a138ee..1f4f82a64e 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -2544,6 +2544,14 @@ class TestPriceDataReloadAPI:
 class TestPriceDataReloadIntegration:
     """Integration tests for the complete price data reload feature"""
 
+    @pytest.fixture(autouse=True)
+    def _flush_litellm_config_cache(self):
+        from litellm.proxy.utils import litellm_config_cache
+
+        litellm_config_cache.flush_cache()
+        yield
+        litellm_config_cache.flush_cache()
+
     @pytest.fixture
     def client_with_auth(self):
         """Create a test client with authentication"""
@@ -2601,6 +2609,7 @@ class TestPriceDataReloadIntegration:
     def test_distributed_reload_check_function(self):
         """Test the _check_and_reload_model_cost_map function"""
         from litellm.proxy.proxy_server import ProxyConfig
+        from litellm.proxy.utils import litellm_config_cache
 
         proxy_config = ProxyConfig()
 
@@ -2609,14 +2618,19 @@ class TestPriceDataReloadIntegration:
 
         # Test case 1: No config in database
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=None)
+        # _check_and_reload_model_cost_map routes through get_config_param,
+        # which calls prisma.get_generic_data on a cache miss.
+        mock_prisma.get_generic_data = AsyncMock(return_value=None)
 
         # Should return early without reloading
         asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
 
         # Test case 2: Config with interval but not time to reload
+        litellm_config_cache.flush_cache()
         mock_config = MagicMock()
         mock_config.param_value = {"interval_hours": 6, "force_reload": False}
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config)
+        mock_prisma.get_generic_data = AsyncMock(return_value=mock_config)
 
         # Mock current time and last reload time
         with patch(
@@ -2632,8 +2646,10 @@ class TestPriceDataReloadIntegration:
                 asyncio.run(proxy_config._check_and_reload_model_cost_map(mock_prisma))
 
         # Test case 3: Config with force reload
+        litellm_config_cache.flush_cache()
         mock_config.param_value = {"interval_hours": 6, "force_reload": True}
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config)
+        mock_prisma.get_generic_data = AsyncMock(return_value=mock_config)
         mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None)
 
         original_model_cost = litellm.model_cost.copy()
@@ -2675,6 +2691,8 @@ class TestPriceDataReloadIntegration:
         mock_config = MagicMock()
         mock_config.param_value = {"interval_hours": 24, "force_reload": True}
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config)
+        # _check_and_reload_model_cost_map now reads through get_generic_data.
+        mock_prisma.get_generic_data = AsyncMock(return_value=mock_config)
         mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None)
 
         original_model_cost = litellm.model_cost.copy()
@@ -2770,6 +2788,8 @@ class TestPriceDataReloadIntegration:
         mock_config = MagicMock()
         mock_config.param_value = {"interval_hours": 12, "force_reload": True}
         mock_prisma.db.litellm_config.find_unique = AsyncMock(return_value=mock_config)
+        # _check_and_reload_anthropic_beta_headers now reads through get_generic_data.
+        mock_prisma.get_generic_data = AsyncMock(return_value=mock_config)
         mock_prisma.db.litellm_config.upsert = AsyncMock(return_value=None)
 
         with patch(

From 21ed38971d244c0a034604f6439c0584d55b4d20 Mon Sep 17 00:00:00 2001
From: Michael-RZ-Berri <michael@berri.ai>
Date: Tue, 28 Apr 2026 17:04:40 -0700
Subject: [PATCH 10/19] lazy-load optional feature routers on first request
 (#26534)

Co-authored-by: Michael Riad Zaky <michaelr@Mac.localdomain>
---
 litellm/proxy/_lazy_features.py               | 307 ++++++++++++++++++
 litellm/proxy/proxy_server.py                 | 126 ++-----
 tests/proxy_unit_tests/test_proxy_routes.py   |  14 +
 tests/test_litellm/proxy/test_proxy_server.py | 252 ++++++++++++++
 .../test_vector_store_endpoints.py            |  15 +
 5 files changed, 609 insertions(+), 105 deletions(-)
 create mode 100644 litellm/proxy/_lazy_features.py

diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py
new file mode 100644
index 0000000000..450c9483f3
--- /dev/null
+++ b/litellm/proxy/_lazy_features.py
@@ -0,0 +1,307 @@
+"""
+Lazy registration for optional feature routers. Each LAZY_FEATURES entry
+imports its module only on the first request matching its path prefix,
+saving ~700 MB at idle for deployments that don't use these features.
+First hit pays the import cost (1-3 s for heavy modules); /openapi.json
+omits each feature's routes until the feature is warmed.
+"""
+
+import asyncio
+import importlib
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Callable, Tuple
+
+from starlette.types import Receive, Scope, Send
+
+from litellm._logging import verbose_proxy_logger
+
+if TYPE_CHECKING:
+    from fastapi import FastAPI
+
+
+def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], None]:
+    def _register(app: "FastAPI", module: object) -> None:
+        app.include_router(getattr(module, attr_name))
+
+    return _register
+
+
+def _mount_app(
+    prefix: str, attr_name: str = "app"
+) -> Callable[["FastAPI", object], None]:
+    def _register(app: "FastAPI", module: object) -> None:
+        app.mount(path=prefix, app=getattr(module, attr_name))
+
+    return _register
+
+
+@dataclass(frozen=True)
+class LazyFeature:
+    name: str
+    module_path: str
+    path_prefixes: Tuple[str, ...]
+    register_fn: Callable[["FastAPI", object], None] = field(
+        default_factory=lambda: _include_router("router")
+    )
+    # For routes whose path has a leading parameter (e.g. /{server}/authorize)
+    # — startswith can't match those, so the matcher also checks endswith.
+    path_suffixes: Tuple[str, ...] = ()
+
+
+LAZY_FEATURES: Tuple[LazyFeature, ...] = (
+    LazyFeature(
+        name="guardrails",
+        module_path="litellm.proxy.guardrails.guardrail_endpoints",
+        path_prefixes=(
+            "/guardrails",
+            "/v2/guardrails",
+            "/apply_guardrail",
+            "/policies/usage",
+        ),
+    ),
+    LazyFeature(
+        name="policies",
+        module_path="litellm.proxy.management_endpoints.policy_endpoints",
+        # Trailing slash to avoid matching /policies/... (policy_engine).
+        path_prefixes=("/policy/", "/utils/test_policies_and_guardrails"),
+    ),
+    LazyFeature(
+        name="policy_engine",
+        module_path="litellm.proxy.policy_engine.policy_endpoints",
+        path_prefixes=("/policies",),
+    ),
+    LazyFeature(
+        name="policy_resolve",
+        module_path="litellm.proxy.policy_engine.policy_resolve_endpoints",
+        path_prefixes=("/policies/resolve", "/policies/attachments/estimate-impact"),
+    ),
+    LazyFeature(
+        name="agents",
+        module_path="litellm.proxy.agent_endpoints.endpoints",
+        path_prefixes=("/v1/agents", "/agents", "/agent/"),
+    ),
+    LazyFeature(
+        name="a2a",
+        module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
+        path_prefixes=("/a2a", "/v1/a2a"),
+    ),
+    LazyFeature(
+        name="vector_stores",
+        module_path="litellm.proxy.vector_store_endpoints.endpoints",
+        path_prefixes=("/v1/vector_stores", "/vector_stores", "/v1/indexes"),
+    ),
+    LazyFeature(
+        name="vector_store_management",
+        module_path="litellm.proxy.vector_store_endpoints.management_endpoints",
+        # Trailing slash to avoid matching /vector_stores/... (vector_stores).
+        path_prefixes=("/vector_store/", "/v1/vector_store/"),
+    ),
+    LazyFeature(
+        name="vector_store_files",
+        # Routes appear under both /v1/vector_stores/{id}/files and the
+        # un-versioned form, so both prefixes must trigger the load.
+        module_path="litellm.proxy.vector_store_files_endpoints.endpoints",
+        path_prefixes=("/v1/vector_stores", "/vector_stores"),
+    ),
+    LazyFeature(
+        name="tools",
+        module_path="litellm.proxy.management_endpoints.tool_management_endpoints",
+        path_prefixes=("/v1/tool", "/tool"),
+    ),
+    LazyFeature(
+        name="search_tools",
+        module_path="litellm.proxy.search_endpoints.search_tool_management",
+        path_prefixes=("/search_tools",),
+    ),
+    # mcp_management owns most /v1/mcp/* admin routes; mcp_app is the mounted
+    # streaming sub-app at /mcp.
+    LazyFeature(
+        name="mcp_management",
+        module_path="litellm.proxy.management_endpoints.mcp_management_endpoints",
+        path_prefixes=("/v1/mcp/",),
+    ),
+    LazyFeature(
+        # Also serves /.well-known/oauth-* (OAuth metadata discovery).
+        # No /mcp/oauth prefix here: the mounted /mcp sub-app would
+        # shadow it, and there are no actual routes there anyway.
+        name="mcp_byok_oauth",
+        module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints",
+        path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"),
+    ),
+    LazyFeature(
+        # Serves OAuth dance endpoints (/authorize, /token, /callback,
+        # /register) plus several /.well-known/ discovery URLs at the proxy
+        # root — needed for MCP-over-OAuth flows even before /mcp is hit.
+        name="mcp_discoverable",
+        module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints",
+        path_prefixes=(
+            "/.well-known/oauth-",
+            "/.well-known/openid-configuration",
+            "/.well-known/jwks.json",
+            "/authorize",
+            "/token",
+            "/callback",
+            "/register",
+        ),
+        # Catches the /{mcp_server_name}/authorize|token|register variants.
+        path_suffixes=("/authorize", "/token", "/register"),
+    ),
+    LazyFeature(
+        name="mcp_rest",
+        module_path="litellm.proxy._experimental.mcp_server.rest_endpoints",
+        path_prefixes=("/mcp-rest",),
+    ),
+    LazyFeature(
+        # Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant
+        # here would defeat lazy loading.
+        name="mcp_app",
+        module_path="litellm.proxy._experimental.mcp_server.server",
+        path_prefixes=("/mcp",),
+        register_fn=_mount_app("/mcp", attr_name="app"),
+    ),
+    LazyFeature(
+        name="config_overrides",
+        module_path="litellm.proxy.management_endpoints.config_override_endpoints",
+        path_prefixes=("/config_overrides",),
+    ),
+    LazyFeature(
+        name="realtime",
+        module_path="litellm.proxy.realtime_endpoints.endpoints",
+        path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"),
+    ),
+    LazyFeature(
+        name="anthropic_passthrough",
+        module_path="litellm.proxy.anthropic_endpoints.endpoints",
+        path_prefixes=("/v1/messages", "/anthropic", "/api/event_logging"),
+    ),
+    LazyFeature(
+        name="anthropic_skills",
+        module_path="litellm.proxy.anthropic_endpoints.skills_endpoints",
+        path_prefixes=("/v1/skills", "/skills"),
+    ),
+    LazyFeature(
+        name="langfuse_passthrough",
+        module_path="litellm.proxy.vertex_ai_endpoints.langfuse_endpoints",
+        path_prefixes=("/langfuse",),
+    ),
+    LazyFeature(
+        name="evals",
+        module_path="litellm.proxy.openai_evals_endpoints.endpoints",
+        path_prefixes=("/v1/evals", "/evals"),
+    ),
+    LazyFeature(
+        name="claude_code_marketplace",
+        module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints",
+        path_prefixes=("/claude-code",),
+        register_fn=_include_router("claude_code_marketplace_router"),
+    ),
+    LazyFeature(
+        name="scim",
+        module_path="litellm.proxy.management_endpoints.scim.scim_v2",
+        path_prefixes=("/scim",),
+        register_fn=_include_router("scim_router"),
+    ),
+    LazyFeature(
+        name="cloudzero",
+        module_path="litellm.proxy.spend_tracking.cloudzero_endpoints",
+        path_prefixes=("/cloudzero",),
+    ),
+    LazyFeature(
+        name="vantage",
+        module_path="litellm.proxy.spend_tracking.vantage_endpoints",
+        path_prefixes=("/vantage",),
+    ),
+    LazyFeature(
+        name="usage_ai",
+        module_path="litellm.proxy.management_endpoints.usage_endpoints",
+        path_prefixes=("/usage/ai",),
+    ),
+    LazyFeature(
+        name="prompts",
+        module_path="litellm.proxy.prompts.prompt_endpoints",
+        path_prefixes=("/prompts", "/utils/dotprompt_json_converter"),
+    ),
+    LazyFeature(
+        name="jwt_mappings",
+        module_path="litellm.proxy.management_endpoints.jwt_key_mapping_endpoints",
+        path_prefixes=("/jwt/key/mapping",),
+    ),
+    LazyFeature(
+        name="compliance",
+        module_path="litellm.proxy.management_endpoints.compliance_endpoints",
+        path_prefixes=("/compliance",),
+    ),
+    LazyFeature(
+        name="access_groups",
+        module_path="litellm.proxy.management_endpoints.access_group_endpoints",
+        path_prefixes=("/access_group", "/v1/access_group", "/v1/unified_access_group"),
+    ),
+)
+
+
+class LazyFeatureMiddleware:
+    """ASGI middleware that imports + registers a feature router on first
+    matching request. Idempotent; once loaded, subsequent requests skip."""
+
+    def __init__(
+        self,
+        app,
+        fastapi_app: "FastAPI",
+        features: Tuple[LazyFeature, ...] = LAZY_FEATURES,
+    ):
+        self.app = app
+        self._fastapi_app = fastapi_app
+        self._features = features
+        self._loaded: set = set()
+        # Per-feature locks so independent features can load in parallel.
+        self._locks: dict = {}
+
+    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
+        # Short-circuit once every feature has loaded.
+        if scope["type"] in ("http", "websocket") and len(self._loaded) < len(
+            self._features
+        ):
+            path = scope.get("path", "")
+            for feat in self._features:
+                if feat.module_path in self._loaded:
+                    continue
+                if any(path.startswith(p) for p in feat.path_prefixes) or any(
+                    path.endswith(s) for s in feat.path_suffixes
+                ):
+                    await self._load(feat)
+        await self.app(scope, receive, send)
+
+    async def _load(self, feat: LazyFeature) -> None:
+        lock = self._locks.setdefault(feat.module_path, asyncio.Lock())
+        async with lock:
+            if feat.module_path in self._loaded:
+                return
+            try:
+                # Import on a thread (heavy modules take 1-3 s). register_fn
+                # mutates app.router.routes, so it stays on the loop thread.
+                loop = asyncio.get_running_loop()
+                module = await loop.run_in_executor(
+                    None, importlib.import_module, feat.module_path
+                )
+                feat.register_fn(self._fastapi_app, module)
+                self._loaded.add(feat.module_path)
+                self._fastapi_app.openapi_schema = None
+                verbose_proxy_logger.info(
+                    "Lazy-loaded optional feature %r (module: %s)",
+                    feat.name,
+                    feat.module_path,
+                )
+            except Exception as exc:
+                # Mark loaded anyway so we don't retry on every request.
+                self._loaded.add(feat.module_path)
+                verbose_proxy_logger.warning(
+                    "Failed to lazy-load optional feature %r (module: %s): %s. "
+                    "This feature's endpoints will return 404 until restart.",
+                    feat.name,
+                    feat.module_path,
+                    exc,
+                )
+
+
+def attach_lazy_features(app: "FastAPI") -> None:
+    app.add_middleware(LazyFeatureMiddleware, fastapi_app=app)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index 8f676df04c..c03a63f211 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -235,37 +235,11 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
-from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
-    router as mcp_byok_oauth_router,
-)
-from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
-    router as mcp_discoverable_endpoints_router,
-)
-from litellm.proxy._experimental.mcp_server.rest_endpoints import (
-    router as mcp_rest_endpoints_router,
-)
-from litellm.proxy._experimental.mcp_server.server import app as mcp_app
-from litellm.proxy._experimental.mcp_server.tool_registry import (
-    global_mcp_tool_registry,
-)
 from litellm.proxy._types import *
-from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router
-from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
-from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router
-from litellm.proxy.agent_endpoints.model_list_helpers import (
-    append_agents_to_model_group,
-    append_agents_to_model_info,
-)
+from litellm.proxy._lazy_features import attach_lazy_features
 from litellm.proxy.analytics_endpoints.analytics_endpoints import (
     router as analytics_router,
 )
-from litellm.proxy.anthropic_endpoints.claude_code_endpoints import (
-    claude_code_marketplace_router,
-)
-from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router
-from litellm.proxy.anthropic_endpoints.skills_endpoints import (
-    router as anthropic_skills_router,
-)
 from litellm.proxy.auth.auth_checks import (
     ExperimentalUIJWTToken,
     get_team_object,
@@ -328,7 +302,6 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config
 from litellm.proxy.google_endpoints.endpoints import router as google_router
-from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router
 from litellm.proxy.guardrails.init_guardrails import (
     init_guardrails_v2,
     initialize_guardrails,
@@ -344,9 +317,6 @@ from litellm.proxy.hooks.prompt_injection_detection import (
 from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
 from litellm.proxy.image_endpoints.endpoints import router as image_router
 from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
-from litellm.proxy.management_endpoints.access_group_endpoints import (
-    router as access_group_router,
-)
 from litellm.proxy.management_endpoints.budget_management_endpoints import (
     router as budget_management_router,
 )
@@ -360,12 +330,6 @@ from litellm.proxy.management_endpoints.common_utils import (
     _user_has_admin_privileges,
     admin_can_invite_user,
 )
-from litellm.proxy.management_endpoints.compliance_endpoints import (
-    router as compliance_router,
-)
-from litellm.proxy.management_endpoints.config_override_endpoints import (
-    router as config_override_router,
-)
 from litellm.proxy.management_endpoints.cost_tracking_settings import (
     router as cost_tracking_settings_router,
 )
@@ -379,9 +343,6 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import (
     router as internal_user_router,
 )
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
-from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import (
-    router as jwt_key_mapping_router,
-)
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     delete_verification_tokens,
     duration_in_seconds,
@@ -390,9 +351,6 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     router as key_management_router,
 )
-from litellm.proxy.management_endpoints.mcp_management_endpoints import (
-    router as mcp_management_router,
-)
 from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
     router as model_access_group_management_router,
 )
@@ -407,11 +365,9 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
 from litellm.proxy.management_endpoints.organization_endpoints import (
     router as organization_router,
 )
-from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router
 from litellm.proxy.management_endpoints.router_settings_endpoints import (
     router as router_settings_router,
 )
-from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
 from litellm.proxy.management_endpoints.tag_management_endpoints import (
     router as tag_management_router,
 )
@@ -423,15 +379,11 @@ from litellm.proxy.management_endpoints.team_endpoints import (
     update_team,
     validate_membership,
 )
-from litellm.proxy.management_endpoints.tool_management_endpoints import (
-    router as tool_management_router,
-)
 from litellm.proxy.memory.memory_endpoints import router as memory_router
 from litellm.proxy.management_endpoints.ui_sso import (
     get_disabled_non_admin_personal_key_creation,
 )
 from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
-from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
 from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
     router as user_agent_analytics_router,
 )
@@ -441,7 +393,6 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
 )
 from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
 from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
-from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
 from litellm.proxy.openai_files_endpoints.files_endpoints import (
     router as openai_files_router,
 )
@@ -461,27 +412,16 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     router as pass_through_router,
 )
-from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router
-from litellm.proxy.policy_engine.policy_resolve_endpoints import (
-    router as policy_resolve_router,
-)
-from litellm.proxy.prompts.prompt_endpoints import router as prompts_router
 from litellm.proxy.public_endpoints import router as public_endpoints_router
 from litellm.proxy.rag_endpoints.endpoints import router as rag_router
-from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router
 from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
 from litellm.proxy.response_api_endpoints.endpoints import router as response_router
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.search_endpoints.endpoints import router as search_router
-from litellm.proxy.search_endpoints.search_tool_management import (
-    router as search_tool_management_router,
-)
-from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router
 from litellm.proxy.spend_tracking.spend_management_endpoints import (
     router as spend_management_router,
 )
 from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
-from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router
 from litellm.proxy.types_utils.utils import get_instance_fn
 from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
     router as ui_crud_endpoints_router,
@@ -511,16 +451,6 @@ from litellm.proxy.utils import (
     prefetch_config_params,
     update_spend,
 )
-from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router
-from litellm.proxy.vector_store_endpoints.management_endpoints import (
-    router as vector_store_management_router,
-)
-from litellm.proxy.vector_store_files_endpoints.endpoints import (
-    router as vector_store_files_router,
-)
-from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import (
-    router as langfuse_router,
-)
 from litellm.proxy.video_endpoints.endpoints import router as video_router
 from litellm.router import (
     AssistantsTypedDict,
@@ -3854,11 +3784,19 @@ class ProxyConfig:
         ## MCP TOOLS
         mcp_tools_config = config.get("mcp_tools", None)
         if mcp_tools_config:
+            from litellm.proxy._experimental.mcp_server.tool_registry import (
+                global_mcp_tool_registry,
+            )
+
             global_mcp_tool_registry.load_tools_from_config(mcp_tools_config)
 
         ## AGENTS
         agent_config = config.get("agent_list", None)
         if agent_config:
+            from litellm.proxy.agent_endpoints.agent_registry import (
+                global_agent_registry,
+            )
+
             global_agent_registry.load_agents_from_config(agent_config)  # type: ignore
 
         mcp_servers_config = config.get("mcp_servers", None)
@@ -10576,6 +10514,10 @@ async def model_info_v2(
     verbose_proxy_logger.debug("all_models: %s", all_models)
 
     # Append A2A agents to models list
+    from litellm.proxy.agent_endpoints.model_list_helpers import (
+        append_agents_to_model_info,
+    )
+
     all_models = await append_agents_to_model_info(
         models=all_models,
         user_api_key_dict=user_api_key_dict,
@@ -11425,6 +11367,10 @@ async def model_group_info(
     )
 
     # Append A2A agents to model groups
+    from litellm.proxy.agent_endpoints.model_list_helpers import (
+        append_agents_to_model_group,
+    )
+
     model_groups = await append_agents_to_model_group(
         model_groups=model_groups,
         user_api_key_dict=user_api_key_dict,
@@ -14230,65 +14176,40 @@ app.include_router(container_router)
 app.include_router(search_router)
 app.include_router(image_router)
 app.include_router(fine_tuning_router)
-app.include_router(vector_store_router)
-app.include_router(vector_store_management_router)
-app.include_router(vector_store_files_router)
 app.include_router(credential_router)
 app.include_router(llm_passthrough_router)
-app.include_router(webrtc_router)
-app.include_router(mcp_management_router)
-app.include_router(mcp_byok_oauth_router)
-app.include_router(anthropic_router)
-app.include_router(anthropic_skills_router)
-app.include_router(evals_router)
-app.include_router(claude_code_marketplace_router)
-app.include_router(google_router)
-app.include_router(langfuse_router)
 app.include_router(pass_through_router)
 app.include_router(health_router)
 app.include_router(key_management_router)
 app.include_router(internal_user_router)
 app.include_router(team_router)
 app.include_router(ui_sso_router)
-app.include_router(scim_router)
 app.include_router(organization_router)
 app.include_router(customer_router)
 app.include_router(spend_management_router)
-app.include_router(cloudzero_router)
-app.include_router(vantage_router)
 app.include_router(caching_router)
 app.include_router(analytics_router)
-app.include_router(guardrails_router)
-app.include_router(policy_router)
-app.include_router(usage_ai_router)
-app.include_router(policy_crud_router)
-app.include_router(policy_resolve_router)
-app.include_router(search_tool_management_router)
-app.include_router(prompts_router)
 app.include_router(callback_management_endpoints_router)
 app.include_router(debugging_endpoints_router)
 app.include_router(ui_crud_endpoints_router)
 app.include_router(openai_files_router)
 app.include_router(team_callback_router)
-app.include_router(jwt_key_mapping_router)
 app.include_router(budget_management_router)
 app.include_router(model_management_router)
 app.include_router(model_access_group_management_router)
 app.include_router(tag_management_router)
-app.include_router(tool_management_router)
 app.include_router(memory_router)
 app.include_router(cost_tracking_settings_router)
 app.include_router(router_settings_router)
 app.include_router(fallback_management_router)
 app.include_router(cache_settings_router)
-app.include_router(config_override_router)
 app.include_router(user_agent_analytics_router)
 app.include_router(enterprise_router)
 app.include_router(ui_discovery_endpoints_router)
-app.include_router(agent_endpoints_router)
-app.include_router(compliance_router)
-app.include_router(a2a_router)
-app.include_router(access_group_router)
+# Eager: /models/{name}:method overlaps with the OpenAI /models endpoint.
+app.include_router(google_router)
+
+attach_lazy_features(app)
 
 
 async def _stream_mcp_asgi_response(
@@ -14521,8 +14442,3 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
             f"Error handling dynamic MCP route for {mcp_server_name}: {str(e)}"
         )
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
-
-
-app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
-app.include_router(mcp_rest_endpoints_router)
-app.include_router(mcp_discoverable_endpoints_router)
diff --git a/tests/proxy_unit_tests/test_proxy_routes.py b/tests/proxy_unit_tests/test_proxy_routes.py
index 812e4e1ac4..67eca5206d 100644
--- a/tests/proxy_unit_tests/test_proxy_routes.py
+++ b/tests/proxy_unit_tests/test_proxy_routes.py
@@ -39,6 +39,20 @@ def test_routes_on_litellm_proxy():
 
     this prevents accidentelly deleting /threads, or /batches etc
     """
+    # Force-load lazy features so the test sees the full route set. Continue
+    # on per-feature import failure — the assertion below still catches
+    # missing-route regressions.
+    import importlib
+
+    from litellm.proxy._lazy_features import LAZY_FEATURES
+
+    for feat in LAZY_FEATURES:
+        try:
+            module = importlib.import_module(feat.module_path)
+            feat.register_fn(app, module)
+        except Exception as exc:
+            print(f"warning: failed to force-load {feat.name}: {exc}")
+
     _all_routes = []
     for route in app.routes:
 
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 1f4f82a64e..7a96f6cbd1 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -5471,3 +5471,255 @@ async def test_reseed_warms_cache_even_on_zero_db_spend():
     finally:
         ps.spend_counter_cache = orig_counter
         ps.prisma_client = orig_prisma
+
+
+# ---------------------------------------------------------------------------
+# Lazy feature loading (LazyFeatureMiddleware) — verifies that optional
+# routers are NOT imported at module load and ARE imported on first request
+# to a matching path prefix. The same module isn't re-imported on subsequent
+# requests.
+# ---------------------------------------------------------------------------
+
+
+import sys
+
+
+class TestLazyFeatureRegistry:
+    """Sanity checks on the registry shape — guards against accidental edits."""
+
+    def test_registry_entries_have_required_fields(self):
+        from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature
+
+        assert len(LAZY_FEATURES) > 0
+        for feat in LAZY_FEATURES:
+            assert isinstance(feat, LazyFeature)
+            assert feat.name
+            assert feat.module_path
+            assert feat.path_prefixes
+            assert all(p.startswith("/") for p in feat.path_prefixes)
+            assert callable(feat.register_fn)
+
+    def test_registry_names_unique(self):
+        from litellm.proxy._lazy_features import LAZY_FEATURES
+
+        names = [f.name for f in LAZY_FEATURES]
+        assert len(names) == len(set(names)), "duplicate feature names"
+
+
+class TestLazyFeaturesNotImportedAtStartup:
+    """
+    The whole point of the refactor: gated feature modules must NOT be
+    present in `sys.modules` immediately after `proxy_server` imports.
+    """
+
+    def test_heavy_modules_absent_at_startup(self):
+        # Force a fresh `proxy_server` import in a subprocess so other tests
+        # in this run (which may have triggered lazy loads via the TestClient)
+        # don't pollute the result.
+        import subprocess
+
+        check = (
+            "import sys; "
+            "from litellm.proxy.proxy_server import app; "  # noqa: F401
+            "heavy = ["
+            "'litellm.proxy._experimental.mcp_server.rest_endpoints',"
+            "'litellm.proxy._experimental.mcp_server.server',"
+            "'litellm.proxy.management_endpoints.config_override_endpoints',"
+            "'litellm.proxy.guardrails.guardrail_endpoints',"
+            "'litellm.proxy.openai_evals_endpoints.endpoints',"
+            "]; "
+            "still_present = [m for m in heavy if m in sys.modules]; "
+            "print('PRESENT_AT_STARTUP:', still_present)"
+        )
+        result = subprocess.run(
+            [sys.executable, "-c", check],
+            capture_output=True,
+            text=True,
+            timeout=120,
+        )
+        # Last non-empty line of stdout (skip warnings printed before)
+        out_lines = [
+            line for line in result.stdout.strip().splitlines() if line.strip()
+        ]
+        report = next((line for line in out_lines if "PRESENT_AT_STARTUP" in line), "")
+        assert report, f"no report emitted (stderr: {result.stderr[-500:]})"
+        assert (
+            "PRESENT_AT_STARTUP: []" in report
+        ), f"expected no heavy modules at startup, got: {report}"
+
+
+class TestLazyFeatureMiddleware:
+    """Behavior of the middleware itself, exercised in isolation."""
+
+    @pytest.mark.asyncio
+    async def test_first_request_triggers_load_subsequent_does_not(self):
+        from fastapi import FastAPI
+
+        from litellm.proxy._lazy_features import (
+            LazyFeature,
+            LazyFeatureMiddleware,
+        )
+
+        loads = []
+
+        def fake_register(app, module):
+            loads.append(getattr(module, "__name__", "?"))
+
+        feat = LazyFeature(
+            name="dummy",
+            module_path="json",  # any always-importable stdlib module
+            path_prefixes=("/dummy",),
+            register_fn=fake_register,
+        )
+
+        # Build a minimal ASGI receiver to satisfy the middleware contract
+        async def downstream(scope, receive, send):
+            # echo back; no-op handler
+            await send({"type": "http.response.start", "status": 200, "headers": []})
+            await send({"type": "http.response.body", "body": b""})
+
+        target_app = FastAPI()
+        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
+
+        async def receive():
+            return {"type": "http.request", "body": b"", "more_body": False}
+
+        sent: list = []
+
+        async def send(message):
+            sent.append(message)
+
+        # First request matching the prefix triggers register
+        await mw(
+            {"type": "http", "path": "/dummy/x", "method": "GET", "headers": []},
+            receive,
+            send,
+        )
+        assert loads == ["json"]
+
+        # Second matching request must NOT re-register
+        sent.clear()
+        await mw(
+            {"type": "http", "path": "/dummy/y", "method": "GET", "headers": []},
+            receive,
+            send,
+        )
+        assert loads == ["json"], "register_fn called twice for the same feature"
+
+        # Non-matching path must not trigger anything
+        await mw(
+            {"type": "http", "path": "/unrelated", "method": "GET", "headers": []},
+            receive,
+            send,
+        )
+        assert loads == ["json"]
+
+    @pytest.mark.asyncio
+    async def test_concurrent_first_requests_only_register_once(self):
+        """
+        Two requests to the same prefix arriving in parallel must result in
+        exactly one `register_fn` invocation — the lock prevents the import +
+        register from racing with itself.
+        """
+        from fastapi import FastAPI
+
+        from litellm.proxy._lazy_features import (
+            LazyFeature,
+            LazyFeatureMiddleware,
+        )
+
+        loads = []
+
+        def slow_register(app, module):
+            loads.append(getattr(module, "__name__", "?"))
+
+        feat = LazyFeature(
+            name="dummy_concurrent",
+            module_path="json",
+            path_prefixes=("/dummy_c",),
+            register_fn=slow_register,
+        )
+
+        async def downstream(scope, receive, send):
+            await send({"type": "http.response.start", "status": 200, "headers": []})
+            await send({"type": "http.response.body", "body": b""})
+
+        target_app = FastAPI()
+        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
+
+        async def receive():
+            return {"type": "http.request", "body": b"", "more_body": False}
+
+        sent: list = []
+
+        async def send(message):
+            sent.append(message)
+
+        async def hit():
+            await mw(
+                {
+                    "type": "http",
+                    "path": "/dummy_c/x",
+                    "method": "GET",
+                    "headers": [],
+                },
+                receive,
+                send,
+            )
+
+        await asyncio.gather(hit(), hit(), hit(), hit(), hit())
+        assert loads == [
+            "json"
+        ], f"expected one registration despite concurrent first hits, got {loads}"
+
+    @pytest.mark.asyncio
+    async def test_failing_import_does_not_loop(self):
+        """
+        If a feature's module can't be imported, the middleware should mark it
+        loaded anyway so subsequent requests don't repeatedly retry the failing
+        import (which would amplify the cost on every request).
+        """
+        from fastapi import FastAPI
+
+        from litellm.proxy._lazy_features import (
+            LazyFeature,
+            LazyFeatureMiddleware,
+        )
+
+        attempts = []
+
+        def fail_register(app, module):
+            attempts.append("called")
+            raise RuntimeError("boom")
+
+        feat = LazyFeature(
+            name="failing",
+            module_path="json",
+            path_prefixes=("/fail",),
+            register_fn=fail_register,
+        )
+
+        async def downstream(scope, receive, send):
+            await send({"type": "http.response.start", "status": 200, "headers": []})
+            await send({"type": "http.response.body", "body": b""})
+
+        target_app = FastAPI()
+        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
+
+        async def receive():
+            return {"type": "http.request", "body": b"", "more_body": False}
+
+        sent: list = []
+
+        async def send(message):
+            sent.append(message)
+
+        for _ in range(3):
+            await mw(
+                {"type": "http", "path": "/fail/x", "method": "GET", "headers": []},
+                receive,
+                send,
+            )
+        assert attempts == [
+            "called"
+        ], f"failing register_fn should be invoked once, not on every request; got {attempts}"
diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
index 44cc5cc445..1e596aa567 100644
--- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
+++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
@@ -786,8 +786,23 @@ class TestVectorStoreManagementEndpointsExist:
         - POST /vector_store/info
         - POST /vector_store/update
         """
+        import importlib
+
+        from litellm.proxy._lazy_features import LAZY_FEATURES
         from litellm.proxy.proxy_server import app
 
+        # Force-register the lazy vector_store_management routes so the
+        # assertions can find them.
+        already_registered = any(
+            getattr(r, "path", None) == "/vector_store/new" for r in app.routes
+        )
+        if not already_registered:
+            for feat in LAZY_FEATURES:
+                if feat.name == "vector_store_management":
+                    module = importlib.import_module(feat.module_path)
+                    feat.register_fn(app, module)
+                    break
+
         # Define expected endpoints
         expected_endpoints = [
             ("POST", "/vector_store/new"),

From 0520d5ce117a51994a862b8df6384fa6b1a52d74 Mon Sep 17 00:00:00 2001
From: Michael-RZ-Berri <michael@berri.ai>
Date: Tue, 28 Apr 2026 17:05:36 -0700
Subject: [PATCH 11/19] [Fix] Unify cost calc in success_handler dict and typed
 branches (#26629)

* Unify cost calc in success_handler dict and typed branches

* Trim verbose comments and docstrings

---------

Co-authored-by: Michael Riad Zaky <michaelr@Mac.localdomain>
Co-authored-by: Michael Riad Zaky <michaelr@Michaels-MacBook-Air.local>
---
 litellm/litellm_core_utils/litellm_logging.py |  17 +--
 .../test_litellm_logging.py                   | 138 ++++++++++++++++++
 2 files changed, 142 insertions(+), 13 deletions(-)

diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py
index fb103afea0..829c1c9ca0 100644
--- a/litellm/litellm_core_utils/litellm_logging.py
+++ b/litellm/litellm_core_utils/litellm_logging.py
@@ -1467,6 +1467,8 @@ class Logging(LiteLLMLoggingBaseClass):
             LiteLLMRealtimeStreamLoggingObject,
             OpenAIModerationResponse,
             "SearchResponse",
+            dict,
+            list,
         ],
         cache_hit: Optional[bool] = None,
         litellm_model_name: Optional[str] = None,
@@ -1744,6 +1746,7 @@ class Logging(LiteLLMLoggingBaseClass):
         start_time,
         end_time,
     ):
+        """Resolve hidden params, compute response cost, and emit the standard logging payload."""
         hidden_params = getattr(logging_result, "_hidden_params", {})
         if hidden_params:
             if self.model_call_details.get("litellm_params") is not None:
@@ -1877,24 +1880,12 @@ class Logging(LiteLLMLoggingBaseClass):
             ):
                 if self._is_recognized_call_type_for_logging(
                     logging_result=logging_result
-                ):
+                ) or isinstance(logging_result, (dict, list)):
                     self._process_hidden_params_and_response_cost(
                         logging_result=logging_result,
                         start_time=start_time,
                         end_time=end_time,
                     )
-                elif isinstance(result, dict) or isinstance(result, list):
-                    self.model_call_details["standard_logging_object"] = (
-                        self._build_standard_logging_payload(
-                            result, start_time, end_time
-                        )
-                    )
-                    if (
-                        standard_logging_payload := self.model_call_details.get(
-                            "standard_logging_object"
-                        )
-                    ) is not None:
-                        emit_standard_logging_payload(standard_logging_payload)
             elif standard_logging_object is not None:
                 self.model_call_details["standard_logging_object"] = (
                     standard_logging_object
diff --git a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
index 3348118a02..1764d9c609 100644
--- a/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
+++ b/tests/test_litellm/litellm_core_utils/test_litellm_logging.py
@@ -2534,3 +2534,141 @@ def test_get_standard_logging_object_payload_includes_litellm_call_id(logging_ob
 
     assert payload is not None
     assert payload["litellm_call_id"] == call_id
+
+
+def _make_dict_logging_obj():
+    """Build a Logging instance configured for a non-streaming dict result."""
+    obj = LitellmLogging(
+        model="claude-haiku-4-5@20251001",
+        messages=[{"role": "user", "content": "hi"}],
+        stream=False,
+        call_type="acompletion",
+        litellm_call_id="test-call-id",
+        start_time=time.time(),
+        function_id="test-fn",
+    )
+    obj.model_call_details = {
+        "model": "claude-haiku-4-5@20251001",
+        "custom_llm_provider": "vertex_ai",
+        "litellm_params": {"metadata": {}},
+        "response_cost": None,
+    }
+    return obj
+
+
+def test_success_handler_computes_cost_for_dict_response():
+    """Non-streaming dict responses run through the cost calculator."""
+    logging_obj = _make_dict_logging_obj()
+    expected_cost = 0.42
+    with (
+        patch.object(
+            logging_obj,
+            "_response_cost_calculator",
+            return_value=expected_cost,
+        ) as mock_calc,
+        patch.object(
+            logging_obj,
+            "_build_standard_logging_payload",
+            return_value={"response_cost": expected_cost},
+        ),
+        patch(
+            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
+        ),
+        patch.object(
+            logging_obj,
+            "_is_recognized_call_type_for_logging",
+            return_value=False,
+        ),
+        patch.object(
+            logging_obj,
+            "_transform_usage_objects",
+            side_effect=lambda result: result,
+        ),
+    ):
+        logging_obj.success_handler(
+            result={"id": "msg_1"},
+            start_time=time.time(),
+            end_time=time.time(),
+        )
+        mock_calc.assert_called_once()
+        assert logging_obj.model_call_details["response_cost"] == expected_cost
+
+
+def test_success_handler_preserves_precomputed_cost_for_dict_response():
+    """Precomputed response_cost on model_call_details must not be overwritten."""
+    logging_obj = _make_dict_logging_obj()
+    precomputed_cost = 1.23
+    logging_obj.model_call_details["response_cost"] = precomputed_cost
+    with (
+        patch.object(
+            logging_obj,
+            "_response_cost_calculator",
+            return_value=9.99,
+        ) as mock_calc,
+        patch.object(
+            logging_obj,
+            "_build_standard_logging_payload",
+            return_value={"response_cost": precomputed_cost},
+        ),
+        patch(
+            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
+        ),
+        patch.object(
+            logging_obj,
+            "_is_recognized_call_type_for_logging",
+            return_value=False,
+        ),
+        patch.object(
+            logging_obj,
+            "_transform_usage_objects",
+            side_effect=lambda result: result,
+        ),
+    ):
+        logging_obj.success_handler(
+            result={"id": "msg_2"},
+            start_time=time.time(),
+            end_time=time.time(),
+        )
+        mock_calc.assert_not_called()
+        assert logging_obj.model_call_details["response_cost"] == precomputed_cost
+
+
+def test_success_handler_unified_helper_runs_for_typed_results():
+    """Recognized typed responses still flow through the unified helper."""
+    logging_obj = _make_dict_logging_obj()
+    expected_cost = 0.10
+    typed_result = MagicMock()
+    typed_result._hidden_params = {}
+
+    with (
+        patch.object(
+            logging_obj,
+            "_response_cost_calculator",
+            return_value=expected_cost,
+        ) as mock_calc,
+        patch.object(
+            logging_obj,
+            "_build_standard_logging_payload",
+            return_value={"response_cost": expected_cost},
+        ),
+        patch(
+            "litellm.litellm_core_utils.litellm_logging.emit_standard_logging_payload"
+        ),
+        patch.object(
+            logging_obj,
+            "_is_recognized_call_type_for_logging",
+            return_value=True,
+        ),
+        patch.object(
+            logging_obj,
+            "_transform_usage_objects",
+            side_effect=lambda result: result,
+        ),
+    ):
+        logging_obj.success_handler(
+            result=typed_result,
+            start_time=time.time(),
+            end_time=time.time(),
+        )
+        mock_calc.assert_called_once()
+        assert logging_obj.model_call_details["response_cost"] == expected_cost

From fd32f29e39ad54aa058779dbb2c5f91f2946a39f Mon Sep 17 00:00:00 2001
From: Krrish Dholakia <krrish+github@berri.ai>
Date: Tue, 28 Apr 2026 17:21:41 -0700
Subject: [PATCH 12/19] Revert "lazy-load optional feature routers on first
 request (#26534)" (#26727)

This reverts commit 21ed38971d244c0a034604f6439c0584d55b4d20.
---
 litellm/proxy/_lazy_features.py               | 307 ------------------
 litellm/proxy/proxy_server.py                 | 126 +++++--
 tests/proxy_unit_tests/test_proxy_routes.py   |  14 -
 tests/test_litellm/proxy/test_proxy_server.py | 252 --------------
 .../test_vector_store_endpoints.py            |  15 -
 5 files changed, 105 insertions(+), 609 deletions(-)
 delete mode 100644 litellm/proxy/_lazy_features.py

diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py
deleted file mode 100644
index 450c9483f3..0000000000
--- a/litellm/proxy/_lazy_features.py
+++ /dev/null
@@ -1,307 +0,0 @@
-"""
-Lazy registration for optional feature routers. Each LAZY_FEATURES entry
-imports its module only on the first request matching its path prefix,
-saving ~700 MB at idle for deployments that don't use these features.
-First hit pays the import cost (1-3 s for heavy modules); /openapi.json
-omits each feature's routes until the feature is warmed.
-"""
-
-import asyncio
-import importlib
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Callable, Tuple
-
-from starlette.types import Receive, Scope, Send
-
-from litellm._logging import verbose_proxy_logger
-
-if TYPE_CHECKING:
-    from fastapi import FastAPI
-
-
-def _include_router(attr_name: str = "router") -> Callable[["FastAPI", object], None]:
-    def _register(app: "FastAPI", module: object) -> None:
-        app.include_router(getattr(module, attr_name))
-
-    return _register
-
-
-def _mount_app(
-    prefix: str, attr_name: str = "app"
-) -> Callable[["FastAPI", object], None]:
-    def _register(app: "FastAPI", module: object) -> None:
-        app.mount(path=prefix, app=getattr(module, attr_name))
-
-    return _register
-
-
-@dataclass(frozen=True)
-class LazyFeature:
-    name: str
-    module_path: str
-    path_prefixes: Tuple[str, ...]
-    register_fn: Callable[["FastAPI", object], None] = field(
-        default_factory=lambda: _include_router("router")
-    )
-    # For routes whose path has a leading parameter (e.g. /{server}/authorize)
-    # — startswith can't match those, so the matcher also checks endswith.
-    path_suffixes: Tuple[str, ...] = ()
-
-
-LAZY_FEATURES: Tuple[LazyFeature, ...] = (
-    LazyFeature(
-        name="guardrails",
-        module_path="litellm.proxy.guardrails.guardrail_endpoints",
-        path_prefixes=(
-            "/guardrails",
-            "/v2/guardrails",
-            "/apply_guardrail",
-            "/policies/usage",
-        ),
-    ),
-    LazyFeature(
-        name="policies",
-        module_path="litellm.proxy.management_endpoints.policy_endpoints",
-        # Trailing slash to avoid matching /policies/... (policy_engine).
-        path_prefixes=("/policy/", "/utils/test_policies_and_guardrails"),
-    ),
-    LazyFeature(
-        name="policy_engine",
-        module_path="litellm.proxy.policy_engine.policy_endpoints",
-        path_prefixes=("/policies",),
-    ),
-    LazyFeature(
-        name="policy_resolve",
-        module_path="litellm.proxy.policy_engine.policy_resolve_endpoints",
-        path_prefixes=("/policies/resolve", "/policies/attachments/estimate-impact"),
-    ),
-    LazyFeature(
-        name="agents",
-        module_path="litellm.proxy.agent_endpoints.endpoints",
-        path_prefixes=("/v1/agents", "/agents", "/agent/"),
-    ),
-    LazyFeature(
-        name="a2a",
-        module_path="litellm.proxy.agent_endpoints.a2a_endpoints",
-        path_prefixes=("/a2a", "/v1/a2a"),
-    ),
-    LazyFeature(
-        name="vector_stores",
-        module_path="litellm.proxy.vector_store_endpoints.endpoints",
-        path_prefixes=("/v1/vector_stores", "/vector_stores", "/v1/indexes"),
-    ),
-    LazyFeature(
-        name="vector_store_management",
-        module_path="litellm.proxy.vector_store_endpoints.management_endpoints",
-        # Trailing slash to avoid matching /vector_stores/... (vector_stores).
-        path_prefixes=("/vector_store/", "/v1/vector_store/"),
-    ),
-    LazyFeature(
-        name="vector_store_files",
-        # Routes appear under both /v1/vector_stores/{id}/files and the
-        # un-versioned form, so both prefixes must trigger the load.
-        module_path="litellm.proxy.vector_store_files_endpoints.endpoints",
-        path_prefixes=("/v1/vector_stores", "/vector_stores"),
-    ),
-    LazyFeature(
-        name="tools",
-        module_path="litellm.proxy.management_endpoints.tool_management_endpoints",
-        path_prefixes=("/v1/tool", "/tool"),
-    ),
-    LazyFeature(
-        name="search_tools",
-        module_path="litellm.proxy.search_endpoints.search_tool_management",
-        path_prefixes=("/search_tools",),
-    ),
-    # mcp_management owns most /v1/mcp/* admin routes; mcp_app is the mounted
-    # streaming sub-app at /mcp.
-    LazyFeature(
-        name="mcp_management",
-        module_path="litellm.proxy.management_endpoints.mcp_management_endpoints",
-        path_prefixes=("/v1/mcp/",),
-    ),
-    LazyFeature(
-        # Also serves /.well-known/oauth-* (OAuth metadata discovery).
-        # No /mcp/oauth prefix here: the mounted /mcp sub-app would
-        # shadow it, and there are no actual routes there anyway.
-        name="mcp_byok_oauth",
-        module_path="litellm.proxy._experimental.mcp_server.byok_oauth_endpoints",
-        path_prefixes=("/v1/mcp/oauth", "/.well-known/oauth-"),
-    ),
-    LazyFeature(
-        # Serves OAuth dance endpoints (/authorize, /token, /callback,
-        # /register) plus several /.well-known/ discovery URLs at the proxy
-        # root — needed for MCP-over-OAuth flows even before /mcp is hit.
-        name="mcp_discoverable",
-        module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints",
-        path_prefixes=(
-            "/.well-known/oauth-",
-            "/.well-known/openid-configuration",
-            "/.well-known/jwks.json",
-            "/authorize",
-            "/token",
-            "/callback",
-            "/register",
-        ),
-        # Catches the /{mcp_server_name}/authorize|token|register variants.
-        path_suffixes=("/authorize", "/token", "/register"),
-    ),
-    LazyFeature(
-        name="mcp_rest",
-        module_path="litellm.proxy._experimental.mcp_server.rest_endpoints",
-        path_prefixes=("/mcp-rest",),
-    ),
-    LazyFeature(
-        # Hardcoded /mcp matches BASE_MCP_ROUTE; importing the constant
-        # here would defeat lazy loading.
-        name="mcp_app",
-        module_path="litellm.proxy._experimental.mcp_server.server",
-        path_prefixes=("/mcp",),
-        register_fn=_mount_app("/mcp", attr_name="app"),
-    ),
-    LazyFeature(
-        name="config_overrides",
-        module_path="litellm.proxy.management_endpoints.config_override_endpoints",
-        path_prefixes=("/config_overrides",),
-    ),
-    LazyFeature(
-        name="realtime",
-        module_path="litellm.proxy.realtime_endpoints.endpoints",
-        path_prefixes=("/openai/v1/realtime", "/v1/realtime", "/realtime"),
-    ),
-    LazyFeature(
-        name="anthropic_passthrough",
-        module_path="litellm.proxy.anthropic_endpoints.endpoints",
-        path_prefixes=("/v1/messages", "/anthropic", "/api/event_logging"),
-    ),
-    LazyFeature(
-        name="anthropic_skills",
-        module_path="litellm.proxy.anthropic_endpoints.skills_endpoints",
-        path_prefixes=("/v1/skills", "/skills"),
-    ),
-    LazyFeature(
-        name="langfuse_passthrough",
-        module_path="litellm.proxy.vertex_ai_endpoints.langfuse_endpoints",
-        path_prefixes=("/langfuse",),
-    ),
-    LazyFeature(
-        name="evals",
-        module_path="litellm.proxy.openai_evals_endpoints.endpoints",
-        path_prefixes=("/v1/evals", "/evals"),
-    ),
-    LazyFeature(
-        name="claude_code_marketplace",
-        module_path="litellm.proxy.anthropic_endpoints.claude_code_endpoints",
-        path_prefixes=("/claude-code",),
-        register_fn=_include_router("claude_code_marketplace_router"),
-    ),
-    LazyFeature(
-        name="scim",
-        module_path="litellm.proxy.management_endpoints.scim.scim_v2",
-        path_prefixes=("/scim",),
-        register_fn=_include_router("scim_router"),
-    ),
-    LazyFeature(
-        name="cloudzero",
-        module_path="litellm.proxy.spend_tracking.cloudzero_endpoints",
-        path_prefixes=("/cloudzero",),
-    ),
-    LazyFeature(
-        name="vantage",
-        module_path="litellm.proxy.spend_tracking.vantage_endpoints",
-        path_prefixes=("/vantage",),
-    ),
-    LazyFeature(
-        name="usage_ai",
-        module_path="litellm.proxy.management_endpoints.usage_endpoints",
-        path_prefixes=("/usage/ai",),
-    ),
-    LazyFeature(
-        name="prompts",
-        module_path="litellm.proxy.prompts.prompt_endpoints",
-        path_prefixes=("/prompts", "/utils/dotprompt_json_converter"),
-    ),
-    LazyFeature(
-        name="jwt_mappings",
-        module_path="litellm.proxy.management_endpoints.jwt_key_mapping_endpoints",
-        path_prefixes=("/jwt/key/mapping",),
-    ),
-    LazyFeature(
-        name="compliance",
-        module_path="litellm.proxy.management_endpoints.compliance_endpoints",
-        path_prefixes=("/compliance",),
-    ),
-    LazyFeature(
-        name="access_groups",
-        module_path="litellm.proxy.management_endpoints.access_group_endpoints",
-        path_prefixes=("/access_group", "/v1/access_group", "/v1/unified_access_group"),
-    ),
-)
-
-
-class LazyFeatureMiddleware:
-    """ASGI middleware that imports + registers a feature router on first
-    matching request. Idempotent; once loaded, subsequent requests skip."""
-
-    def __init__(
-        self,
-        app,
-        fastapi_app: "FastAPI",
-        features: Tuple[LazyFeature, ...] = LAZY_FEATURES,
-    ):
-        self.app = app
-        self._fastapi_app = fastapi_app
-        self._features = features
-        self._loaded: set = set()
-        # Per-feature locks so independent features can load in parallel.
-        self._locks: dict = {}
-
-    async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
-        # Short-circuit once every feature has loaded.
-        if scope["type"] in ("http", "websocket") and len(self._loaded) < len(
-            self._features
-        ):
-            path = scope.get("path", "")
-            for feat in self._features:
-                if feat.module_path in self._loaded:
-                    continue
-                if any(path.startswith(p) for p in feat.path_prefixes) or any(
-                    path.endswith(s) for s in feat.path_suffixes
-                ):
-                    await self._load(feat)
-        await self.app(scope, receive, send)
-
-    async def _load(self, feat: LazyFeature) -> None:
-        lock = self._locks.setdefault(feat.module_path, asyncio.Lock())
-        async with lock:
-            if feat.module_path in self._loaded:
-                return
-            try:
-                # Import on a thread (heavy modules take 1-3 s). register_fn
-                # mutates app.router.routes, so it stays on the loop thread.
-                loop = asyncio.get_running_loop()
-                module = await loop.run_in_executor(
-                    None, importlib.import_module, feat.module_path
-                )
-                feat.register_fn(self._fastapi_app, module)
-                self._loaded.add(feat.module_path)
-                self._fastapi_app.openapi_schema = None
-                verbose_proxy_logger.info(
-                    "Lazy-loaded optional feature %r (module: %s)",
-                    feat.name,
-                    feat.module_path,
-                )
-            except Exception as exc:
-                # Mark loaded anyway so we don't retry on every request.
-                self._loaded.add(feat.module_path)
-                verbose_proxy_logger.warning(
-                    "Failed to lazy-load optional feature %r (module: %s): %s. "
-                    "This feature's endpoints will return 404 until restart.",
-                    feat.name,
-                    feat.module_path,
-                    exc,
-                )
-
-
-def attach_lazy_features(app: "FastAPI") -> None:
-    app.add_middleware(LazyFeatureMiddleware, fastapi_app=app)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
index c03a63f211..8f676df04c 100644
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -235,11 +235,37 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import (
+    router as mcp_byok_oauth_router,
+)
+from litellm.proxy._experimental.mcp_server.discoverable_endpoints import (
+    router as mcp_discoverable_endpoints_router,
+)
+from litellm.proxy._experimental.mcp_server.rest_endpoints import (
+    router as mcp_rest_endpoints_router,
+)
+from litellm.proxy._experimental.mcp_server.server import app as mcp_app
+from litellm.proxy._experimental.mcp_server.tool_registry import (
+    global_mcp_tool_registry,
+)
 from litellm.proxy._types import *
-from litellm.proxy._lazy_features import attach_lazy_features
+from litellm.proxy.agent_endpoints.a2a_endpoints import router as a2a_router
+from litellm.proxy.agent_endpoints.agent_registry import global_agent_registry
+from litellm.proxy.agent_endpoints.endpoints import router as agent_endpoints_router
+from litellm.proxy.agent_endpoints.model_list_helpers import (
+    append_agents_to_model_group,
+    append_agents_to_model_info,
+)
 from litellm.proxy.analytics_endpoints.analytics_endpoints import (
     router as analytics_router,
 )
+from litellm.proxy.anthropic_endpoints.claude_code_endpoints import (
+    claude_code_marketplace_router,
+)
+from litellm.proxy.anthropic_endpoints.endpoints import router as anthropic_router
+from litellm.proxy.anthropic_endpoints.skills_endpoints import (
+    router as anthropic_skills_router,
+)
 from litellm.proxy.auth.auth_checks import (
     ExperimentalUIJWTToken,
     get_team_object,
@@ -302,6 +328,7 @@ from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router
 from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config
 from litellm.proxy.google_endpoints.endpoints import router as google_router
+from litellm.proxy.guardrails.guardrail_endpoints import router as guardrails_router
 from litellm.proxy.guardrails.init_guardrails import (
     init_guardrails_v2,
     initialize_guardrails,
@@ -317,6 +344,9 @@ from litellm.proxy.hooks.prompt_injection_detection import (
 from litellm.proxy.hooks.proxy_track_cost_callback import _ProxyDBLogger
 from litellm.proxy.image_endpoints.endpoints import router as image_router
 from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request
+from litellm.proxy.management_endpoints.access_group_endpoints import (
+    router as access_group_router,
+)
 from litellm.proxy.management_endpoints.budget_management_endpoints import (
     router as budget_management_router,
 )
@@ -330,6 +360,12 @@ from litellm.proxy.management_endpoints.common_utils import (
     _user_has_admin_privileges,
     admin_can_invite_user,
 )
+from litellm.proxy.management_endpoints.compliance_endpoints import (
+    router as compliance_router,
+)
+from litellm.proxy.management_endpoints.config_override_endpoints import (
+    router as config_override_router,
+)
 from litellm.proxy.management_endpoints.cost_tracking_settings import (
     router as cost_tracking_settings_router,
 )
@@ -343,6 +379,9 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import (
     router as internal_user_router,
 )
 from litellm.proxy.management_endpoints.internal_user_endpoints import user_update
+from litellm.proxy.management_endpoints.jwt_key_mapping_endpoints import (
+    router as jwt_key_mapping_router,
+)
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     delete_verification_tokens,
     duration_in_seconds,
@@ -351,6 +390,9 @@ from litellm.proxy.management_endpoints.key_management_endpoints import (
 from litellm.proxy.management_endpoints.key_management_endpoints import (
     router as key_management_router,
 )
+from litellm.proxy.management_endpoints.mcp_management_endpoints import (
+    router as mcp_management_router,
+)
 from litellm.proxy.management_endpoints.model_access_group_management_endpoints import (
     router as model_access_group_management_router,
 )
@@ -365,9 +407,11 @@ from litellm.proxy.management_endpoints.model_management_endpoints import (
 from litellm.proxy.management_endpoints.organization_endpoints import (
     router as organization_router,
 )
+from litellm.proxy.management_endpoints.policy_endpoints import router as policy_router
 from litellm.proxy.management_endpoints.router_settings_endpoints import (
     router as router_settings_router,
 )
+from litellm.proxy.management_endpoints.scim.scim_v2 import scim_router
 from litellm.proxy.management_endpoints.tag_management_endpoints import (
     router as tag_management_router,
 )
@@ -379,11 +423,15 @@ from litellm.proxy.management_endpoints.team_endpoints import (
     update_team,
     validate_membership,
 )
+from litellm.proxy.management_endpoints.tool_management_endpoints import (
+    router as tool_management_router,
+)
 from litellm.proxy.memory.memory_endpoints import router as memory_router
 from litellm.proxy.management_endpoints.ui_sso import (
     get_disabled_non_admin_personal_key_creation,
 )
 from litellm.proxy.management_endpoints.ui_sso import router as ui_sso_router
+from litellm.proxy.management_endpoints.usage_endpoints import router as usage_ai_router
 from litellm.proxy.management_endpoints.user_agent_analytics_endpoints import (
     router as user_agent_analytics_router,
 )
@@ -393,6 +441,7 @@ from litellm.proxy.middleware.in_flight_requests_middleware import (
 )
 from litellm.proxy.middleware.prometheus_auth_middleware import PrometheusAuthMiddleware
 from litellm.proxy.ocr_endpoints.endpoints import router as ocr_router
+from litellm.proxy.openai_evals_endpoints.endpoints import router as evals_router
 from litellm.proxy.openai_files_endpoints.files_endpoints import (
     router as openai_files_router,
 )
@@ -412,16 +461,27 @@ from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
 from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
     router as pass_through_router,
 )
+from litellm.proxy.policy_engine.policy_endpoints import router as policy_crud_router
+from litellm.proxy.policy_engine.policy_resolve_endpoints import (
+    router as policy_resolve_router,
+)
+from litellm.proxy.prompts.prompt_endpoints import router as prompts_router
 from litellm.proxy.public_endpoints import router as public_endpoints_router
 from litellm.proxy.rag_endpoints.endpoints import router as rag_router
+from litellm.proxy.realtime_endpoints.endpoints import router as webrtc_router
 from litellm.proxy.rerank_endpoints.endpoints import router as rerank_router
 from litellm.proxy.response_api_endpoints.endpoints import router as response_router
 from litellm.proxy.route_llm_request import route_request
 from litellm.proxy.search_endpoints.endpoints import router as search_router
+from litellm.proxy.search_endpoints.search_tool_management import (
+    router as search_tool_management_router,
+)
+from litellm.proxy.spend_tracking.cloudzero_endpoints import router as cloudzero_router
 from litellm.proxy.spend_tracking.spend_management_endpoints import (
     router as spend_management_router,
 )
 from litellm.proxy.spend_tracking.spend_tracking_utils import get_logging_payload
+from litellm.proxy.spend_tracking.vantage_endpoints import router as vantage_router
 from litellm.proxy.types_utils.utils import get_instance_fn
 from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import (
     router as ui_crud_endpoints_router,
@@ -451,6 +511,16 @@ from litellm.proxy.utils import (
     prefetch_config_params,
     update_spend,
 )
+from litellm.proxy.vector_store_endpoints.endpoints import router as vector_store_router
+from litellm.proxy.vector_store_endpoints.management_endpoints import (
+    router as vector_store_management_router,
+)
+from litellm.proxy.vector_store_files_endpoints.endpoints import (
+    router as vector_store_files_router,
+)
+from litellm.proxy.vertex_ai_endpoints.langfuse_endpoints import (
+    router as langfuse_router,
+)
 from litellm.proxy.video_endpoints.endpoints import router as video_router
 from litellm.router import (
     AssistantsTypedDict,
@@ -3784,19 +3854,11 @@ class ProxyConfig:
         ## MCP TOOLS
         mcp_tools_config = config.get("mcp_tools", None)
         if mcp_tools_config:
-            from litellm.proxy._experimental.mcp_server.tool_registry import (
-                global_mcp_tool_registry,
-            )
-
             global_mcp_tool_registry.load_tools_from_config(mcp_tools_config)
 
         ## AGENTS
         agent_config = config.get("agent_list", None)
         if agent_config:
-            from litellm.proxy.agent_endpoints.agent_registry import (
-                global_agent_registry,
-            )
-
             global_agent_registry.load_agents_from_config(agent_config)  # type: ignore
 
         mcp_servers_config = config.get("mcp_servers", None)
@@ -10514,10 +10576,6 @@ async def model_info_v2(
     verbose_proxy_logger.debug("all_models: %s", all_models)
 
     # Append A2A agents to models list
-    from litellm.proxy.agent_endpoints.model_list_helpers import (
-        append_agents_to_model_info,
-    )
-
     all_models = await append_agents_to_model_info(
         models=all_models,
         user_api_key_dict=user_api_key_dict,
@@ -11367,10 +11425,6 @@ async def model_group_info(
     )
 
     # Append A2A agents to model groups
-    from litellm.proxy.agent_endpoints.model_list_helpers import (
-        append_agents_to_model_group,
-    )
-
     model_groups = await append_agents_to_model_group(
         model_groups=model_groups,
         user_api_key_dict=user_api_key_dict,
@@ -14176,40 +14230,65 @@ app.include_router(container_router)
 app.include_router(search_router)
 app.include_router(image_router)
 app.include_router(fine_tuning_router)
+app.include_router(vector_store_router)
+app.include_router(vector_store_management_router)
+app.include_router(vector_store_files_router)
 app.include_router(credential_router)
 app.include_router(llm_passthrough_router)
+app.include_router(webrtc_router)
+app.include_router(mcp_management_router)
+app.include_router(mcp_byok_oauth_router)
+app.include_router(anthropic_router)
+app.include_router(anthropic_skills_router)
+app.include_router(evals_router)
+app.include_router(claude_code_marketplace_router)
+app.include_router(google_router)
+app.include_router(langfuse_router)
 app.include_router(pass_through_router)
 app.include_router(health_router)
 app.include_router(key_management_router)
 app.include_router(internal_user_router)
 app.include_router(team_router)
 app.include_router(ui_sso_router)
+app.include_router(scim_router)
 app.include_router(organization_router)
 app.include_router(customer_router)
 app.include_router(spend_management_router)
+app.include_router(cloudzero_router)
+app.include_router(vantage_router)
 app.include_router(caching_router)
 app.include_router(analytics_router)
+app.include_router(guardrails_router)
+app.include_router(policy_router)
+app.include_router(usage_ai_router)
+app.include_router(policy_crud_router)
+app.include_router(policy_resolve_router)
+app.include_router(search_tool_management_router)
+app.include_router(prompts_router)
 app.include_router(callback_management_endpoints_router)
 app.include_router(debugging_endpoints_router)
 app.include_router(ui_crud_endpoints_router)
 app.include_router(openai_files_router)
 app.include_router(team_callback_router)
+app.include_router(jwt_key_mapping_router)
 app.include_router(budget_management_router)
 app.include_router(model_management_router)
 app.include_router(model_access_group_management_router)
 app.include_router(tag_management_router)
+app.include_router(tool_management_router)
 app.include_router(memory_router)
 app.include_router(cost_tracking_settings_router)
 app.include_router(router_settings_router)
 app.include_router(fallback_management_router)
 app.include_router(cache_settings_router)
+app.include_router(config_override_router)
 app.include_router(user_agent_analytics_router)
 app.include_router(enterprise_router)
 app.include_router(ui_discovery_endpoints_router)
-# Eager: /models/{name}:method overlaps with the OpenAI /models endpoint.
-app.include_router(google_router)
-
-attach_lazy_features(app)
+app.include_router(agent_endpoints_router)
+app.include_router(compliance_router)
+app.include_router(a2a_router)
+app.include_router(access_group_router)
 
 
 async def _stream_mcp_asgi_response(
@@ -14442,3 +14521,8 @@ async def dynamic_mcp_route(mcp_server_name: str, request: Request):
             f"Error handling dynamic MCP route for {mcp_server_name}: {str(e)}"
         )
         raise HTTPException(status_code=500, detail=f"Internal server error: {str(e)}")
+
+
+app.mount(path=BASE_MCP_ROUTE, app=mcp_app)
+app.include_router(mcp_rest_endpoints_router)
+app.include_router(mcp_discoverable_endpoints_router)
diff --git a/tests/proxy_unit_tests/test_proxy_routes.py b/tests/proxy_unit_tests/test_proxy_routes.py
index 67eca5206d..812e4e1ac4 100644
--- a/tests/proxy_unit_tests/test_proxy_routes.py
+++ b/tests/proxy_unit_tests/test_proxy_routes.py
@@ -39,20 +39,6 @@ def test_routes_on_litellm_proxy():
 
     this prevents accidentelly deleting /threads, or /batches etc
     """
-    # Force-load lazy features so the test sees the full route set. Continue
-    # on per-feature import failure — the assertion below still catches
-    # missing-route regressions.
-    import importlib
-
-    from litellm.proxy._lazy_features import LAZY_FEATURES
-
-    for feat in LAZY_FEATURES:
-        try:
-            module = importlib.import_module(feat.module_path)
-            feat.register_fn(app, module)
-        except Exception as exc:
-            print(f"warning: failed to force-load {feat.name}: {exc}")
-
     _all_routes = []
     for route in app.routes:
 
diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py
index 7a96f6cbd1..1f4f82a64e 100644
--- a/tests/test_litellm/proxy/test_proxy_server.py
+++ b/tests/test_litellm/proxy/test_proxy_server.py
@@ -5471,255 +5471,3 @@ async def test_reseed_warms_cache_even_on_zero_db_spend():
     finally:
         ps.spend_counter_cache = orig_counter
         ps.prisma_client = orig_prisma
-
-
-# ---------------------------------------------------------------------------
-# Lazy feature loading (LazyFeatureMiddleware) — verifies that optional
-# routers are NOT imported at module load and ARE imported on first request
-# to a matching path prefix. The same module isn't re-imported on subsequent
-# requests.
-# ---------------------------------------------------------------------------
-
-
-import sys
-
-
-class TestLazyFeatureRegistry:
-    """Sanity checks on the registry shape — guards against accidental edits."""
-
-    def test_registry_entries_have_required_fields(self):
-        from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature
-
-        assert len(LAZY_FEATURES) > 0
-        for feat in LAZY_FEATURES:
-            assert isinstance(feat, LazyFeature)
-            assert feat.name
-            assert feat.module_path
-            assert feat.path_prefixes
-            assert all(p.startswith("/") for p in feat.path_prefixes)
-            assert callable(feat.register_fn)
-
-    def test_registry_names_unique(self):
-        from litellm.proxy._lazy_features import LAZY_FEATURES
-
-        names = [f.name for f in LAZY_FEATURES]
-        assert len(names) == len(set(names)), "duplicate feature names"
-
-
-class TestLazyFeaturesNotImportedAtStartup:
-    """
-    The whole point of the refactor: gated feature modules must NOT be
-    present in `sys.modules` immediately after `proxy_server` imports.
-    """
-
-    def test_heavy_modules_absent_at_startup(self):
-        # Force a fresh `proxy_server` import in a subprocess so other tests
-        # in this run (which may have triggered lazy loads via the TestClient)
-        # don't pollute the result.
-        import subprocess
-
-        check = (
-            "import sys; "
-            "from litellm.proxy.proxy_server import app; "  # noqa: F401
-            "heavy = ["
-            "'litellm.proxy._experimental.mcp_server.rest_endpoints',"
-            "'litellm.proxy._experimental.mcp_server.server',"
-            "'litellm.proxy.management_endpoints.config_override_endpoints',"
-            "'litellm.proxy.guardrails.guardrail_endpoints',"
-            "'litellm.proxy.openai_evals_endpoints.endpoints',"
-            "]; "
-            "still_present = [m for m in heavy if m in sys.modules]; "
-            "print('PRESENT_AT_STARTUP:', still_present)"
-        )
-        result = subprocess.run(
-            [sys.executable, "-c", check],
-            capture_output=True,
-            text=True,
-            timeout=120,
-        )
-        # Last non-empty line of stdout (skip warnings printed before)
-        out_lines = [
-            line for line in result.stdout.strip().splitlines() if line.strip()
-        ]
-        report = next((line for line in out_lines if "PRESENT_AT_STARTUP" in line), "")
-        assert report, f"no report emitted (stderr: {result.stderr[-500:]})"
-        assert (
-            "PRESENT_AT_STARTUP: []" in report
-        ), f"expected no heavy modules at startup, got: {report}"
-
-
-class TestLazyFeatureMiddleware:
-    """Behavior of the middleware itself, exercised in isolation."""
-
-    @pytest.mark.asyncio
-    async def test_first_request_triggers_load_subsequent_does_not(self):
-        from fastapi import FastAPI
-
-        from litellm.proxy._lazy_features import (
-            LazyFeature,
-            LazyFeatureMiddleware,
-        )
-
-        loads = []
-
-        def fake_register(app, module):
-            loads.append(getattr(module, "__name__", "?"))
-
-        feat = LazyFeature(
-            name="dummy",
-            module_path="json",  # any always-importable stdlib module
-            path_prefixes=("/dummy",),
-            register_fn=fake_register,
-        )
-
-        # Build a minimal ASGI receiver to satisfy the middleware contract
-        async def downstream(scope, receive, send):
-            # echo back; no-op handler
-            await send({"type": "http.response.start", "status": 200, "headers": []})
-            await send({"type": "http.response.body", "body": b""})
-
-        target_app = FastAPI()
-        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
-
-        async def receive():
-            return {"type": "http.request", "body": b"", "more_body": False}
-
-        sent: list = []
-
-        async def send(message):
-            sent.append(message)
-
-        # First request matching the prefix triggers register
-        await mw(
-            {"type": "http", "path": "/dummy/x", "method": "GET", "headers": []},
-            receive,
-            send,
-        )
-        assert loads == ["json"]
-
-        # Second matching request must NOT re-register
-        sent.clear()
-        await mw(
-            {"type": "http", "path": "/dummy/y", "method": "GET", "headers": []},
-            receive,
-            send,
-        )
-        assert loads == ["json"], "register_fn called twice for the same feature"
-
-        # Non-matching path must not trigger anything
-        await mw(
-            {"type": "http", "path": "/unrelated", "method": "GET", "headers": []},
-            receive,
-            send,
-        )
-        assert loads == ["json"]
-
-    @pytest.mark.asyncio
-    async def test_concurrent_first_requests_only_register_once(self):
-        """
-        Two requests to the same prefix arriving in parallel must result in
-        exactly one `register_fn` invocation — the lock prevents the import +
-        register from racing with itself.
-        """
-        from fastapi import FastAPI
-
-        from litellm.proxy._lazy_features import (
-            LazyFeature,
-            LazyFeatureMiddleware,
-        )
-
-        loads = []
-
-        def slow_register(app, module):
-            loads.append(getattr(module, "__name__", "?"))
-
-        feat = LazyFeature(
-            name="dummy_concurrent",
-            module_path="json",
-            path_prefixes=("/dummy_c",),
-            register_fn=slow_register,
-        )
-
-        async def downstream(scope, receive, send):
-            await send({"type": "http.response.start", "status": 200, "headers": []})
-            await send({"type": "http.response.body", "body": b""})
-
-        target_app = FastAPI()
-        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
-
-        async def receive():
-            return {"type": "http.request", "body": b"", "more_body": False}
-
-        sent: list = []
-
-        async def send(message):
-            sent.append(message)
-
-        async def hit():
-            await mw(
-                {
-                    "type": "http",
-                    "path": "/dummy_c/x",
-                    "method": "GET",
-                    "headers": [],
-                },
-                receive,
-                send,
-            )
-
-        await asyncio.gather(hit(), hit(), hit(), hit(), hit())
-        assert loads == [
-            "json"
-        ], f"expected one registration despite concurrent first hits, got {loads}"
-
-    @pytest.mark.asyncio
-    async def test_failing_import_does_not_loop(self):
-        """
-        If a feature's module can't be imported, the middleware should mark it
-        loaded anyway so subsequent requests don't repeatedly retry the failing
-        import (which would amplify the cost on every request).
-        """
-        from fastapi import FastAPI
-
-        from litellm.proxy._lazy_features import (
-            LazyFeature,
-            LazyFeatureMiddleware,
-        )
-
-        attempts = []
-
-        def fail_register(app, module):
-            attempts.append("called")
-            raise RuntimeError("boom")
-
-        feat = LazyFeature(
-            name="failing",
-            module_path="json",
-            path_prefixes=("/fail",),
-            register_fn=fail_register,
-        )
-
-        async def downstream(scope, receive, send):
-            await send({"type": "http.response.start", "status": 200, "headers": []})
-            await send({"type": "http.response.body", "body": b""})
-
-        target_app = FastAPI()
-        mw = LazyFeatureMiddleware(downstream, fastapi_app=target_app, features=(feat,))
-
-        async def receive():
-            return {"type": "http.request", "body": b"", "more_body": False}
-
-        sent: list = []
-
-        async def send(message):
-            sent.append(message)
-
-        for _ in range(3):
-            await mw(
-                {"type": "http", "path": "/fail/x", "method": "GET", "headers": []},
-                receive,
-                send,
-            )
-        assert attempts == [
-            "called"
-        ], f"failing register_fn should be invoked once, not on every request; got {attempts}"
diff --git a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
index 1e596aa567..44cc5cc445 100644
--- a/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
+++ b/tests/test_litellm/proxy/vector_store_endpoints/test_vector_store_endpoints.py
@@ -786,23 +786,8 @@ class TestVectorStoreManagementEndpointsExist:
         - POST /vector_store/info
         - POST /vector_store/update
         """
-        import importlib
-
-        from litellm.proxy._lazy_features import LAZY_FEATURES
         from litellm.proxy.proxy_server import app
 
-        # Force-register the lazy vector_store_management routes so the
-        # assertions can find them.
-        already_registered = any(
-            getattr(r, "path", None) == "/vector_store/new" for r in app.routes
-        )
-        if not already_registered:
-            for feat in LAZY_FEATURES:
-                if feat.name == "vector_store_management":
-                    module = importlib.import_module(feat.module_path)
-                    feat.register_fn(app, module)
-                    break
-
         # Define expected endpoints
         expected_endpoints = [
             ("POST", "/vector_store/new"),

From f8bb29aebfb4530a66120f55157f5dc144e136b9 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 28 Apr 2026 17:43:17 -0700
Subject: [PATCH 13/19] =?UTF-8?q?bump:=20version=201.83.14=20=E2=86=92=201?=
 =?UTF-8?q?.84.0?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 pyproject.toml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/pyproject.toml b/pyproject.toml
index a15fa5a06a..657632d69e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.83.14"
+version = "1.84.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -236,7 +236,7 @@ source-exclude = [
 profile = "black"
 
 [tool.commitizen]
-version = "1.83.14"
+version = "1.84.0"
 version_files = [
     "pyproject.toml:^version",
 ]

From b4d9006f92c14b6fa7161b286b303561211ce04d Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 28 Apr 2026 17:43:36 -0700
Subject: [PATCH 14/19] uv lock

---
 uv.lock | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/uv.lock b/uv.lock
index 53f032cfba..f837e2b5ef 100644
--- a/uv.lock
+++ b/uv.lock
@@ -9,7 +9,7 @@ resolution-markers = [
 ]
 
 [options]
-exclude-newer = "2026-04-23T02:32:27.506663Z"
+exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
 exclude-newer-span = "P3D"
 
 [manifest]
@@ -3085,7 +3085,7 @@ wheels = [
 
 [[package]]
 name = "litellm"
-version = "1.83.14"
+version = "1.84.0"
 source = { editable = "." }
 dependencies = [
     { name = "aiohttp" },

From 1da1eb661b3aafd39d8705da66c915d330a258b8 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 28 Apr 2026 19:33:18 -0700
Subject: [PATCH 15/19] ci(release): accept PEP 440 tag forms in create-release
 workflow

The tag validator required a leading `v`, so dispatching create-release
with `1.84.0` (or `1.84.0rc1`, `1.84.0.dev42`, `1.84.0.post1`) failed
even though those are the new naming convention. Make the leading `v`
optional in both create-release.yml and create-release-branch.yml so
both legacy (`v1.83.10-stable`, `v1.83.14.rc.1`, `v1.82.3.dev.9`,
`v1.82.3-stable.patch.4`, `v1.83.13-nightly`) and new PEP 440 forms are
accepted during the transition. Refresh the input descriptions to show
the new examples.
---
 .github/workflows/create-release-branch.yml | 8 ++++----
 .github/workflows/create-release.yml        | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml
index 13b76c94df..ec2651306f 100644
--- a/.github/workflows/create-release-branch.yml
+++ b/.github/workflows/create-release-branch.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
         required: true
         type: string
       commit_hash:
@@ -14,7 +14,7 @@ on:
   workflow_call:
     inputs:
       tag:
-        description: "Release tag"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -40,8 +40,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
index 68ab397d82..c0aec1687e 100644
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 

From 3a5980804c2aef672ef1f324e101e2c6694285f7 Mon Sep 17 00:00:00 2001
From: Yuneng Jiang <yuneng@berri.ai>
Date: Tue, 28 Apr 2026 19:38:13 -0700
Subject: [PATCH 16/19] ci(release): mark rc / dev / nightly tags as GitHub
 pre-releases

`prerelease: false` was hardcoded, so dispatching create-release with
`1.84.0rc1`, `1.84.0.dev42`, or legacy `v1.83.13-nightly` would publish
them as stable releases on the GitHub Releases page. Derive the flag
from the tag instead.

The detector matches `rc`, `.dev`, `nightly`, `alpha`, `beta`. PEP 440
post-releases (`1.84.0.post1`) and legacy `-stable[.patch.N]` are
stable maintenance releases per PEP 440, so they intentionally do not
match.
---
 .github/workflows/create-release.yml | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml
index c0aec1687e..39d078267f 100644
--- a/.github/workflows/create-release.yml
+++ b/.github/workflows/create-release.yml
@@ -45,6 +45,11 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|\.dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +94,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });

From 4ae2996f08398bc4fd35c5e940fd94ec8fa0bbe6 Mon Sep 17 00:00:00 2001
From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com>
Date: Tue, 28 Apr 2026 20:10:42 -0700
Subject: [PATCH 17/19] Add gpt-image-2 support (#26644) (#26705)

* Add gpt-image-2 support

* Address gpt-image-2 PR feedback

Co-authored-by: Emerson Gomes <emerson.gomes@thalesgroup.com>
---
 .../get_llm_provider_logic.py                 |  1 +
 .../litellm_core_utils/llm_cost_calc/utils.py |  8 +-
 .../llms/azure/image_generation/__init__.py   |  2 +-
 .../image_generation/gpt_transformation.py    |  2 +-
 .../image_generation/cost_calculator.py       |  6 +-
 .../image_generation/gpt_transformation.py    |  2 +-
 ...odel_prices_and_context_window_backup.json | 64 +++++++++++++++
 litellm/utils.py                              |  1 +
 model_prices_and_context_window.json          | 64 +++++++++++++++
 .../test_gpt_image_cost_calculator.py         | 80 ++++++++++++++++++-
 tests/test_litellm/test_utils.py              | 79 ++++++++++++++++++
 11 files changed, 298 insertions(+), 11 deletions(-)

diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py
index 95bcd4d718..4ff077efe7 100644
--- a/litellm/litellm_core_utils/get_llm_provider_logic.py
+++ b/litellm/litellm_core_utils/get_llm_provider_logic.py
@@ -348,6 +348,7 @@ def get_llm_provider(  # noqa: PLR0915
             or "ft:gpt-3.5-turbo" in model
             or "ft:gpt-4" in model  # catches ft:gpt-4-0613, ft:gpt-4o
             or model in litellm.openai_image_generation_models
+            or model.startswith("gpt-image")
             or model in litellm.openai_video_generation_models
         ):
             custom_llm_provider = "openai"
diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py
index 888999504f..59d0465e6d 100644
--- a/litellm/litellm_core_utils/llm_cost_calc/utils.py
+++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py
@@ -982,9 +982,9 @@ class CostCalculatorUtils:
                 image_response=completion_response,
             )
         elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )
@@ -1004,9 +1004,9 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )
diff --git a/litellm/llms/azure/image_generation/__init__.py b/litellm/llms/azure/image_generation/__init__.py
index fcdf49f291..a9cf151464 100644
--- a/litellm/llms/azure/image_generation/__init__.py
+++ b/litellm/llms/azure/image_generation/__init__.py
@@ -24,6 +24,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
         return AzureDallE3ImageGenerationConfig()
     else:
         verbose_logger.debug(
-            f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format."
+            f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format."
         )
         return AzureGPTImageGenerationConfig()
diff --git a/litellm/llms/azure/image_generation/gpt_transformation.py b/litellm/llms/azure/image_generation/gpt_transformation.py
index 1f5f65f693..2d46592e3f 100644
--- a/litellm/llms/azure/image_generation/gpt_transformation.py
+++ b/litellm/llms/azure/image_generation/gpt_transformation.py
@@ -3,7 +3,7 @@ from litellm.llms.openai.image_generation import GPTImageGenerationConfig
 
 class AzureGPTImageGenerationConfig(GPTImageGenerationConfig):
     """
-    Azure gpt-image-1 image generation config
+    Azure gpt-image image generation config
     """
 
     pass
diff --git a/litellm/llms/openai/image_generation/cost_calculator.py b/litellm/llms/openai/image_generation/cost_calculator.py
index 8bca75172f..d009a085fa 100644
--- a/litellm/llms/openai/image_generation/cost_calculator.py
+++ b/litellm/llms/openai/image_generation/cost_calculator.py
@@ -1,5 +1,5 @@
 """
-Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini)
+Cost calculator for OpenAI image generation models (gpt-image family)
 
 These models use token-based pricing instead of pixel-based pricing like DALL-E.
 """
@@ -17,13 +17,13 @@ def cost_calculator(
     custom_llm_provider: Optional[str] = None,
 ) -> float:
     """
-    Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models.
+    Calculate cost for OpenAI gpt-image models.
 
     Uses the same usage format as Responses API, so we reuse the helper
     to transform to chat completion format and use generic_cost_per_token.
 
     Args:
-        model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini")
+        model: The model name (e.g., "gpt-image-1", "gpt-image-2")
         image_response: The ImageResponse containing usage data
         custom_llm_provider: Optional provider name
 
diff --git a/litellm/llms/openai/image_generation/gpt_transformation.py b/litellm/llms/openai/image_generation/gpt_transformation.py
index c106d7f17b..68f799e574 100644
--- a/litellm/llms/openai/image_generation/gpt_transformation.py
+++ b/litellm/llms/openai/image_generation/gpt_transformation.py
@@ -15,7 +15,7 @@ if TYPE_CHECKING:
 
 class GPTImageGenerationConfig(BaseImageGenerationConfig):
     """
-    OpenAI gpt-image-1 image generation config
+    OpenAI gpt-image image generation config
     """
 
     def get_supported_openai_params(
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 8511d785fb..e4268fac81 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -5103,6 +5103,38 @@
             "/v1/images/edits"
         ]
     },
+    "azure/gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "azure/gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "azure/low/1024-x-1024/gpt-image-1-mini": {
         "input_cost_per_pixel": 2.0751953125e-09,
         "litellm_provider": "azure",
@@ -19083,6 +19115,38 @@
         "supports_vision": true,
         "supports_pdf_input": true
     },
+    "gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "low/1024-x-1024/gpt-image-1.5": {
         "input_cost_per_image": 0.009,
         "litellm_provider": "openai",
diff --git a/litellm/utils.py b/litellm/utils.py
index e63bf402bf..027c9fedce 100644
--- a/litellm/utils.py
+++ b/litellm/utils.py
@@ -6526,6 +6526,7 @@ def validate_environment(  # noqa: PLR0915
             or model in litellm.open_ai_text_completion_models
             or model in litellm.open_ai_embedding_models
             or model in litellm.openai_image_generation_models
+            or model.startswith("gpt-image")
         ):
             if "OPENAI_API_KEY" in os.environ:
                 keys_in_environment = True
diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json
index 114883f530..ca7d323ad6 100644
--- a/model_prices_and_context_window.json
+++ b/model_prices_and_context_window.json
@@ -5117,6 +5117,38 @@
             "/v1/images/edits"
         ]
     },
+    "azure/gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "azure/gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "azure/low/1024-x-1024/gpt-image-1-mini": {
         "input_cost_per_pixel": 2.0751953125e-09,
         "litellm_provider": "azure",
@@ -19097,6 +19129,38 @@
         "supports_vision": true,
         "supports_pdf_input": true
     },
+    "gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "low/1024-x-1024/gpt-image-1.5": {
         "input_cost_per_image": 0.009,
         "litellm_provider": "openai",
diff --git a/tests/test_litellm/test_gpt_image_cost_calculator.py b/tests/test_litellm/test_gpt_image_cost_calculator.py
index 620c073498..6644b1389c 100644
--- a/tests/test_litellm/test_gpt_image_cost_calculator.py
+++ b/tests/test_litellm/test_gpt_image_cost_calculator.py
@@ -29,8 +29,21 @@ from litellm.types.utils import (
 )
 
 
+@pytest.fixture(autouse=True)
+def _use_local_model_cost_map(monkeypatch):
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
 class TestGPTImageCostCalculator:
-    """Test the OpenAI gpt-image-1 cost calculator"""
+    """Test the OpenAI gpt-image cost calculator"""
 
     def test_gpt_image_1_cost_with_text_only(self):
         """Test cost calculation with only text input tokens"""
@@ -149,6 +162,44 @@ class TestGPTImageCostCalculator:
 
         assert cost == 0.0
 
+    def test_gpt_image_2_cost_with_text_and_image_tokens(self):
+        """Test cost calculation for gpt-image-2 token pricing"""
+        from litellm.llms.openai.image_generation.cost_calculator import cost_calculator
+
+        usage = Usage(
+            prompt_tokens=600,
+            completion_tokens=5000,
+            total_tokens=5600,
+            prompt_tokens_details=PromptTokensDetailsWrapper(
+                text_tokens=100,
+                image_tokens=500,
+            ),
+            completion_tokens_details=CompletionTokensDetailsWrapper(
+                text_tokens=1000,
+                image_tokens=4000,
+            ),
+        )
+
+        image_response = ImageResponse(
+            created=1234567890,
+            data=[ImageObject(url="http://example.com/image.jpg")],
+        )
+        image_response.usage = usage
+
+        cost = cost_calculator(
+            model="gpt-image-2",
+            image_response=image_response,
+            custom_llm_provider="openai",
+        )
+
+        # GPT Image 2 pricing:
+        # Text input: 100 * $5/1M = 0.0005
+        # Image input: 500 * $8/1M = 0.004
+        # Text output: 1000 * $10/1M = 0.01
+        # Image output: 4000 * $30/1M = 0.12
+        expected_cost = 0.0005 + 0.004 + 0.01 + 0.12
+        assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
+
 
 class TestGPTImageCostRouting:
     """Test that gpt-image models are properly routed to the token-based calculator"""
@@ -182,6 +233,33 @@ class TestGPTImageCostRouting:
         expected_cost = 0.0005 + 0.2
         assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
 
+    def test_openai_gpt_image_2_routes_to_token_calculator(self):
+        """Test that OpenAI gpt-image-2 routes to token-based calculator"""
+        from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
+
+        usage = Usage(
+            prompt_tokens=100,
+            completion_tokens=5000,
+            total_tokens=5100,
+            prompt_tokens_details=PromptTokensDetailsWrapper(text_tokens=100),
+            completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=5000),
+        )
+
+        image_response = ImageResponse(
+            created=1234567890,
+            data=[ImageObject(url="http://example.com/image.jpg")],
+        )
+        image_response.usage = usage
+
+        cost = CostCalculatorUtils.route_image_generation_cost_calculator(
+            model="gpt-image-2",
+            completion_response=image_response,
+            custom_llm_provider="openai",
+        )
+
+        expected_cost = 0.0005 + 0.15
+        assert abs(cost - expected_cost) < 1e-6, f"Expected {expected_cost}, got {cost}"
+
     def test_openai_dalle_routes_to_pixel_calculator(self):
         """Test that OpenAI DALL-E still routes to pixel-based calculator"""
         from litellm.litellm_core_utils.llm_cost_calc.utils import CostCalculatorUtils
diff --git a/tests/test_litellm/test_utils.py b/tests/test_litellm/test_utils.py
index b8a4220c67..f28fe3ed25 100644
--- a/tests/test_litellm/test_utils.py
+++ b/tests/test_litellm/test_utils.py
@@ -32,6 +32,19 @@ from litellm.utils import (
 # Adds the parent directory to the system path
 
 
+@pytest.fixture
+def local_model_cost_map(monkeypatch):
+    original_model_cost = litellm.model_cost
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.get_model_info.cache_clear()
+    try:
+        yield
+    finally:
+        litellm.model_cost = original_model_cost
+        litellm.get_model_info.cache_clear()
+
+
 def test_check_provider_match_azure_ai_allows_openai_and_azure():
     """
     Test that azure_ai provider can match openai and azure models.
@@ -198,6 +211,72 @@ def test_get_optional_params_image_gen_filters_empty_values():
     assert optional_params == {}
 
 
+def test_gpt_image_provider_detection_covers_existing_family():
+    for image_model in ("gpt-image-1", "gpt-image-1-mini", "gpt-image-1.5"):
+        model, custom_llm_provider, _, _ = litellm.get_llm_provider(model=image_model)
+
+        assert model == image_model
+        assert custom_llm_provider == "openai"
+
+
+def test_gpt_image_2_provider_and_model_info(local_model_cost_map):
+
+    model, custom_llm_provider, _, _ = litellm.get_llm_provider(model="gpt-image-2")
+
+    assert model == "gpt-image-2"
+    assert custom_llm_provider == "openai"
+
+    model_info = litellm.get_model_info(model="gpt-image-2")
+    assert model_info["litellm_provider"] == "openai"
+    assert model_info["mode"] == "image_generation"
+    assert model_info["input_cost_per_token"] == 5e-06
+    assert model_info["input_cost_per_image_token"] == 8e-06
+    assert model_info["output_cost_per_token"] == 1e-05
+    assert model_info["output_cost_per_image_token"] == 3e-05
+    assert (
+        "/v1/images/generations"
+        in litellm.model_cost["gpt-image-2"]["supported_endpoints"]
+    )
+    assert (
+        "/v1/images/edits" in litellm.model_cost["gpt-image-2"]["supported_endpoints"]
+    )
+    assert model_info["supports_vision"] is True
+    assert model_info["supports_pdf_input"] is True
+
+
+def test_gpt_image_2_snapshot_model_info(local_model_cost_map):
+    model, custom_llm_provider, _, _ = litellm.get_llm_provider(
+        model="gpt-image-2-2026-04-21"
+    )
+
+    assert model == "gpt-image-2-2026-04-21"
+    assert custom_llm_provider == "openai"
+
+    model_info = litellm.get_model_info(model="gpt-image-2-2026-04-21")
+    assert model_info["litellm_provider"] == "openai"
+    assert model_info["mode"] == "image_generation"
+    assert model_info["output_cost_per_image_token"] == 3e-05
+
+
+def test_azure_gpt_image_2_model_info(local_model_cost_map):
+    model, custom_llm_provider, _, _ = litellm.get_llm_provider(
+        model="azure/gpt-image-2"
+    )
+
+    assert model == "gpt-image-2"
+    assert custom_llm_provider == "azure"
+
+    model_info = litellm.get_model_info(
+        model="gpt-image-2", custom_llm_provider="azure"
+    )
+    assert model_info["litellm_provider"] == "azure"
+    assert model_info["mode"] == "image_generation"
+    assert model_info["input_cost_per_token"] == 5e-06
+    assert model_info["input_cost_per_image_token"] == 8e-06
+    assert model_info["output_cost_per_token"] == 1e-05
+    assert model_info["output_cost_per_image_token"] == 3e-05
+
+
 def test_all_model_configs():
     from litellm.llms.vertex_ai.vertex_ai_partner_models.ai21.transformation import (
         VertexAIAi21Config,

From 44ab016743c9b59f2dcc4c17f4f6b6431d1108d2 Mon Sep 17 00:00:00 2001
From: xinrui <94846330+xinrui-z@users.noreply.github.com>
Date: Wed, 29 Apr 2026 11:18:30 +0800
Subject: [PATCH 18/19] feat(provider): add AIHubMix as an OpenAI-compatible
 provider (#24294)

* feat: add AIHubMix provider to providers.json

* fix: add aihubmix to provider_endpoints_support.json for CI check

---------

Co-authored-by: yuneng-jiang <yuneng@berri.ai>
---
 litellm/llms/openai_like/providers.json |  5 +++++
 provider_endpoints_support.json         | 17 +++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json
index 275c352b39..5dd1247001 100644
--- a/litellm/llms/openai_like/providers.json
+++ b/litellm/llms/openai_like/providers.json
@@ -101,5 +101,10 @@
     "param_mappings": {
       "max_completion_tokens": "max_tokens"
     }
+  },
+  "aihubmix": {
+    "base_url": "https://aihubmix.com/v1",
+    "api_key_env": "AIHUBMIX_API_KEY",
+    "api_base_env": "AIHUBMIX_API_BASE"
   }
 }
diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json
index 6f23c87f91..ed49c14621 100644
--- a/provider_endpoints_support.json
+++ b/provider_endpoints_support.json
@@ -193,6 +193,23 @@
         "a2a": false
       }
     },
+    "aihubmix": {
+      "display_name": "AIHubMix (`aihubmix`)",
+      "url": "https://docs.litellm.ai/docs/providers/aihubmix",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": true,
+        "image_generations": true,
+        "audio_transcriptions": true,
+        "audio_speech": true,
+        "moderations": true,
+        "batches": false,
+        "rerank": true,
+        "a2a": false
+      }
+    },
     "assemblyai": {
       "display_name": "AssemblyAI (`assemblyai`)",
       "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai",

From 3215874e400de2989db7b15ef85c27dd703381af Mon Sep 17 00:00:00 2001
From: Cursor Agent <cursoragent@cursor.com>
Date: Wed, 29 Apr 2026 03:48:41 +0000
Subject: [PATCH 19/19] fix(test): scope ERROR log assertion to LiteLLM logger
 in test_model_alias_map

The test was flaking on unrelated asyncio ERROR records (e.g. "Unclosed
client session" from background tasks in other tests). Restrict the
assertion to records emitted by LiteLLM loggers so the test only fails
on errors actually produced by the code under test.

Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
---
 tests/local_testing/test_model_alias_map.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/tests/local_testing/test_model_alias_map.py b/tests/local_testing/test_model_alias_map.py
index cf731d6628..14c1de2f6a 100644
--- a/tests/local_testing/test_model_alias_map.py
+++ b/tests/local_testing/test_model_alias_map.py
@@ -30,10 +30,9 @@ def test_model_alias_map(caplog):
         )
         print(response.model)
 
-        captured_logs = [rec.levelname for rec in caplog.records]
-
-        for log in captured_logs:
-            assert "ERROR" not in log
+        for rec in caplog.records:
+            if rec.levelname == "ERROR" and rec.name.startswith("LiteLLM"):
+                pytest.fail(f"Unexpected litellm ERROR log: {rec.getMessage()}")
 
         assert "llama-3.1-8b-instant" in response.model
     except litellm.ServiceUnavailableError: