Support dropping non-openai params when specified in additional_drop_params + Add VertexAI Anthropic support on /v1/messages (#11246)

* feat(utils.py): support dropping non-openai params when specified via additional drop params

Closes https://github.com/BerriAI/litellm/issues/11205

* fix(utils.py): fix linting error

* refactor(handler.py): add custom llm provider to anthropic messages provider config exception

* feat: initial commit adding vertex ai anthropic support on `/v1/messages`

* test: add working unit test

* test(vertex_ai_partner_models/anthropic): add /v1/messages support for anthropic api

Adds vertex ai auth

* feat(vertex_ai/anthropic): return correct url when calling via `/v1/messages`

* fix: more alignment to expected anthropic request format

* fix: fix ruff linting check

* Removed syntax error from docs (#11242)

* [Feat]: Add Bedrock InvokeAgents as a /chat/completions route on LiteLLM (#11239)

* feat: init structure for bedrock AGENTs

* feat: add basic  routing for bedrock AGENTs

* feat: add basic transforms for bedrock AGENTs

* fix: url for bedrock agent runtime

* fix: working agents request

* feat: working agents non-streaming request

* feat: bedrock agents

* feat: add streaming for bedrock agents

* feat: add cost tracking for bedrock agents

* docs litellm with bedrock agents

* fix: linting errors

* test: invoke agents tests

* fix: import session handling

* Revert "fix: import session handling"

This reverts commit deb257dc10.

* fix: linting pin mypy

* [Feat]: Guardrails - Add streaming for bedrock post guard (#11247)

* feat: add streaming for bedrock post guard

* fix: bedrock guardrails

* fix: add clear comments

* Update litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Update litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* fix: clean up bedrock guardrails

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* [Fix] Responses API - Session management  (#11254)

* fix: import session handling

* fix: imports for session handler

* tests: tests for session handler

* Update enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* bump: bump litellm enterprise

* fixes: test_create_user_default_budget

* fix: fix linting error

* fix: fix linting error

---------

Co-authored-by: Fadil Rahman <87557055+fadil4u@users.noreply.github.com>
Co-authored-by: Ishaan Jaff <ishaanjaffer0324@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

litellm/litellm_core_utils/llm_response_utils/get_api_base.py

@@ -72,13 +72,11 @@ def get_api_base(
         _optional_params.vertex_location is not None
         and _optional_params.vertex_project is not None
     ):
-        from litellm.llms.vertex_ai.vertex_ai_partner_models.main import (
-            VertexPartnerProvider,
-            create_vertex_url,
-        )
+        from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
+        from litellm.types.llms.vertex_ai import VertexPartnerProvider
 
         if "claude" in model:
-            _api_base = create_vertex_url(
+            _api_base = VertexBase.create_vertex_url(
                 vertex_location=_optional_params.vertex_location,
                 vertex_project=_optional_params.vertex_project,
                 model=model,

litellm/llms/anthropic/experimental_pass_through/messages/handler.py

@@ -140,7 +140,7 @@ def anthropic_messages_handler(
     )
     if anthropic_messages_provider_config is None:
         raise ValueError(
-            f"Anthropic messages provider config not found for model: {model}"
+            f"Anthropic messages provider config not found for model: {model}, custom_llm_provider: {custom_llm_provider}"
         )
     if custom_llm_provider is None:
         raise ValueError(

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -1,4 +1,4 @@
-from typing import Any, AsyncIterator, Dict, List, Optional
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple
 
 import httpx
 
@@ -50,7 +50,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
             api_base = f"{api_base}/v1/messages"
         return api_base
 
-    def validate_environment(
+    def validate_anthropic_messages_environment(
         self,
         headers: dict,
         model: str,
@@ -59,14 +59,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
         litellm_params: dict,
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
-    ) -> dict:
-        if "x-api-key" not in headers:
+    ) -> Tuple[dict, Optional[str]]:
+        if "x-api-key" not in headers and api_key:
             headers["x-api-key"] = api_key
         if "anthropic-version" not in headers:
             headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION
         if "content-type" not in headers:
             headers["content-type"] = "application/json"
-        return headers
+        return headers, api_base
 
     def transform_anthropic_messages_request(
         self,

litellm/llms/base_llm/anthropic_messages/transformation.py

@@ -18,7 +18,7 @@ else:
 
 class BaseAnthropicMessagesConfig(ABC):
     @abstractmethod
-    def validate_environment(
+    def validate_anthropic_messages_environment(  # use different name because return type is different from base config's validate_environment
         self,
         headers: dict,
         model: str,
@@ -27,13 +27,17 @@ class BaseAnthropicMessagesConfig(ABC):
         litellm_params: dict,
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
-    ) -> dict:
+    ) -> Tuple[dict, Optional[str]]:
         """
         OPTIONAL
 
         Validate the environment for the request
+
+        Returns:
+        - headers: dict
+        - api_base: Optional[str] - If the provider needs to update the api_base, return it here. Otherwise, return None.
         """
-        return headers
+        return headers, api_base
 
     @abstractmethod
     def get_complete_url(

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -38,6 +38,18 @@ class AmazonAnthropicClaude3MessagesConfig(
         BaseAnthropicMessagesConfig.__init__(self, **kwargs)
         AmazonInvokeConfig.__init__(self, **kwargs)
 
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> Tuple[dict, Optional[str]]:
+        return headers, api_base
+
     def sign_request(
         self,
         headers: dict,
@@ -59,18 +71,6 @@ class AmazonAnthropicClaude3MessagesConfig(
             fake_stream=fake_stream,
         )
 
-    def validate_environment(
-        self,
-        headers: dict,
-        model: str,
-        messages: List[Any],
-        optional_params: dict,
-        litellm_params: dict,
-        api_key: Optional[str] = None,
-        api_base: Optional[str] = None,
-    ) -> dict:
-        return headers
-
     def get_complete_url(
         self,
         api_base: Optional[str],

litellm/llms/custom_httpx/llm_http_handler.py

@@ -1097,7 +1097,10 @@ class BaseLLMHTTPHandler:
             if provider_specific_header
             else {}
         )
-        headers = anthropic_messages_provider_config.validate_environment(
+        (
+            headers,
+            api_base,
+        ) = anthropic_messages_provider_config.validate_anthropic_messages_environment(
             headers=extra_headers or {},
             model=model,
             messages=messages,

litellm/llms/vertex_ai/batches/handler.py

@@ -43,7 +43,7 @@ class VertexAIBatchPrediction(VertexLLM):
             custom_llm_provider="vertex_ai",
         )
 
-        default_api_base = self.create_vertex_url(
+        default_api_base = self.create_vertex_batch_url(
             vertex_location=vertex_location or "us-central1",
             vertex_project=vertex_project or project_id,
         )
@@ -117,7 +117,7 @@ class VertexAIBatchPrediction(VertexLLM):
         )
         return vertex_batch_response
 
-    def create_vertex_url(
+    def create_vertex_batch_url(
         self,
         vertex_location: str,
         vertex_project: str,
@@ -145,7 +145,7 @@ class VertexAIBatchPrediction(VertexLLM):
             custom_llm_provider="vertex_ai",
         )
 
-        default_api_base = self.create_vertex_url(
+        default_api_base = self.create_vertex_batch_url(
             vertex_location=vertex_location or "us-central1",
             vertex_project=vertex_project or project_id,
         )

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py

@@ -0,0 +1,96 @@
+from typing import Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm.llms.anthropic.experimental_pass_through.messages.transformation import (
+    AnthropicMessagesConfig,
+)
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.vertex_ai import VertexPartnerProvider
+from litellm.types.router import GenericLiteLLMParams
+
+from ....vertex_llm_base import VertexBase
+
+
+class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
+    def validate_anthropic_messages_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[Any],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> Tuple[dict, Optional[str]]:
+        """
+        OPTIONAL
+
+        Validate the environment for the request
+        """
+        if "Authorization" not in headers:
+            vertex_ai_project = (
+                optional_params.pop("vertex_project", None)
+                or optional_params.pop("vertex_ai_project", None)
+                or litellm.vertex_project
+                or get_secret_str("VERTEXAI_PROJECT")
+            )
+            vertex_credentials = (
+                optional_params.pop("vertex_credentials", None)
+                or optional_params.pop("vertex_ai_credentials", None)
+                or get_secret_str("VERTEXAI_CREDENTIALS")
+            )
+
+            access_token, project_id = self._ensure_access_token(
+                credentials=vertex_credentials,
+                project_id=vertex_ai_project,
+                custom_llm_provider="vertex_ai",
+            )
+
+            headers["Authorization"] = f"Bearer {access_token}"
+
+            api_base = self.get_complete_vertex_url(
+                custom_api_base=api_base,
+                vertex_location=optional_params.pop("vertex_location", None),
+                vertex_project=vertex_ai_project,
+                project_id=project_id,
+                partner=VertexPartnerProvider.claude,
+                stream=optional_params.get("stream", False),
+                model=model,
+            )
+
+        headers["content-type"] = "application/json"
+        return headers, api_base
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        if api_base is None:
+            raise ValueError(
+                "api_base is required. Unable to determine the correct api_base for the request."
+            )
+        return api_base  # no transformation is needed - handled in validate_environment
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        anthropic_messages_request = super().transform_anthropic_messages_request(
+            model=model,
+            messages=messages,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
+        anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16"
+        return anthropic_messages_request

litellm/llms/vertex_ai/vertex_ai_partner_models/main.py

@@ -1,12 +1,12 @@
 # What is this?
 ## API Handler for calling Vertex AI Partner Models
-from enum import Enum
 from typing import Callable, Optional, Union
 
 import httpx  # type: ignore
 
 import litellm
 from litellm import LlmProviders
+from litellm.types.llms.vertex_ai import VertexPartnerProvider
 from litellm.utils import ModelResponse
 
 from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler
@@ -15,13 +15,6 @@ from ..vertex_llm_base import VertexBase
 base_llm_http_handler = BaseLLMHTTPHandler()
 
 
-class VertexPartnerProvider(str, Enum):
-    mistralai = "mistralai"
-    llama = "llama"
-    ai21 = "ai21"
-    claude = "claude"
-
-
 class VertexAIError(Exception):
     def __init__(self, status_code, message):
         self.status_code = status_code
@@ -35,78 +28,10 @@ class VertexAIError(Exception):
         )  # Call the base class constructor with the parameters it needs
 
 
-def create_vertex_url(
-    vertex_location: str,
-    vertex_project: str,
-    partner: VertexPartnerProvider,
-    stream: Optional[bool],
-    model: str,
-    api_base: Optional[str] = None,
-) -> str:
-    """Return the base url for the vertex partner models"""
-
-    api_base = api_base or f"https://{vertex_location}-aiplatform.googleapis.com"
-    if partner == VertexPartnerProvider.llama:
-        return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions"
-    elif partner == VertexPartnerProvider.mistralai:
-        if stream:
-            return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:streamRawPredict"
-        else:
-            return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict"
-    elif partner == VertexPartnerProvider.ai21:
-        if stream:
-            return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:streamRawPredict"
-        else:
-            return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:rawPredict"
-    elif partner == VertexPartnerProvider.claude:
-        if stream:
-            return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:streamRawPredict"
-        else:
-            return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:rawPredict"
-
-
 class VertexAIPartnerModels(VertexBase):
     def __init__(self) -> None:
         pass
 
-    def get_complete_url(
-        self,
-        custom_api_base: Optional[str],
-        vertex_location: Optional[str],
-        vertex_project: Optional[str],
-        project_id: str,
-        partner: VertexPartnerProvider,
-        stream: Optional[bool],
-        model: str,
-    ) -> str:
-        api_base = self.get_api_base(
-            api_base=custom_api_base, vertex_location=vertex_location
-        )
-        default_api_base = create_vertex_url(
-            vertex_location=vertex_location or "us-central1",
-            vertex_project=vertex_project or project_id,
-            partner=partner,  # type: ignore
-            stream=stream,
-            model=model,
-            api_base=api_base,
-        )
-
-        if len(default_api_base.split(":")) > 1:
-            endpoint = default_api_base.split(":")[-1]
-        else:
-            endpoint = ""
-
-        _, api_base = self._check_custom_proxy(
-            api_base=custom_api_base,
-            custom_llm_provider="vertex_ai",
-            gemini_api_key=None,
-            endpoint=endpoint,
-            stream=stream,
-            auth_header=None,
-            url=default_api_base,
-        )
-        return api_base
-
     def completion(
         self,
         model: str,
@@ -181,7 +106,7 @@ class VertexAIPartnerModels(VertexBase):
             else:
                 raise ValueError(f"Unknown partner model: {model}")
 
-            api_base = self.get_complete_url(
+            api_base = self.get_complete_vertex_url(
                 custom_api_base=api_base,
                 vertex_location=vertex_location,
                 vertex_project=vertex_project,

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.asyncify import asyncify
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
-from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
+from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES, VertexPartnerProvider
 
 from .common_utils import _get_gemini_url, _get_vertex_url, all_gemini_url_modes
 
@@ -150,6 +150,74 @@ class VertexBase:
         else:
             return f"https://{self.get_default_vertex_location()}-aiplatform.googleapis.com"
 
+    @staticmethod
+    def create_vertex_url(
+        vertex_location: str,
+        vertex_project: str,
+        partner: VertexPartnerProvider,
+        stream: Optional[bool],
+        model: str,
+        api_base: Optional[str] = None,
+    ) -> str:
+        """Return the base url for the vertex partner models"""
+
+        api_base = api_base or f"https://{vertex_location}-aiplatform.googleapis.com"
+        if partner == VertexPartnerProvider.llama:
+            return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions"
+        elif partner == VertexPartnerProvider.mistralai:
+            if stream:
+                return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:streamRawPredict"
+            else:
+                return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict"
+        elif partner == VertexPartnerProvider.ai21:
+            if stream:
+                return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:streamRawPredict"
+            else:
+                return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:rawPredict"
+        elif partner == VertexPartnerProvider.claude:
+            if stream:
+                return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:streamRawPredict"
+            else:
+                return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:rawPredict"
+
+    def get_complete_vertex_url(
+        self,
+        custom_api_base: Optional[str],
+        vertex_location: Optional[str],
+        vertex_project: Optional[str],
+        project_id: str,
+        partner: VertexPartnerProvider,
+        stream: Optional[bool],
+        model: str,
+    ) -> str:
+        api_base = self.get_api_base(
+            api_base=custom_api_base, vertex_location=vertex_location
+        )
+        default_api_base = VertexBase.create_vertex_url(
+            vertex_location=vertex_location or "us-central1",
+            vertex_project=vertex_project or project_id,
+            partner=partner,
+            stream=stream,
+            model=model,
+            api_base=api_base,
+        )
+
+        if len(default_api_base.split(":")) > 1:
+            endpoint = default_api_base.split(":")[-1]
+        else:
+            endpoint = ""
+
+        _, api_base = self._check_custom_proxy(
+            api_base=custom_api_base,
+            custom_llm_provider="vertex_ai",
+            gemini_api_key=None,
+            endpoint=endpoint,
+            stream=stream,
+            auth_header=None,
+            url=default_api_base,
+        )
+        return api_base
+
     def refresh_auth(self, credentials: Any) -> None:
         from google.auth.transport.requests import (
             Request,  # type: ignore[import-untyped]

litellm/proxy/_new_secret_config.yaml

@@ -8,6 +8,12 @@ model_list:
     litellm_params:
       model: text-embedding-3-small
       api_key: os.environ/OPENAI_API_KEY
-  
+  - model_name: openai:gpt-4o
+    litellm_params:
+      model: openai/gpt-4o
+      api_key: os.environ/OPENAI_API_KEY
+      drop_params: true
+      additional_drop_params:
+        - red
 litellm_settings:
   cache: true

litellm/types/llms/vertex_ai.py

@@ -577,3 +577,10 @@ class VertexBatchPredictionResponse(TypedDict, total=False):
 
 
 VERTEX_CREDENTIALS_TYPES = Union[str, Dict[str, str]]
+
+
+class VertexPartnerProvider(str, Enum):
+    mistralai = "mistralai"
+    llama = "llama"
+    ai21 = "ai21"
+    claude = "claude"

litellm/utils.py

@@ -3688,13 +3688,22 @@ def add_provider_specific_params_to_optional_params(
                 if k not in openai_params:
                     extra_body[k] = passed_params[k]
             optional_params.setdefault("extra_body", {})
-            optional_params["extra_body"] = {
+            initial_extra_body = {
                 **optional_params["extra_body"],
                 **extra_body,
             }
 
+            if additional_drop_params is not None:
+                processed_extra_body = {
+                    k: v
+                    for k, v in initial_extra_body.items()
+                    if k not in additional_drop_params
+                }
+            else:
+                processed_extra_body = initial_extra_body
+
             optional_params["extra_body"] = _ensure_extra_body_is_safe(
-                extra_body=optional_params["extra_body"]
+                extra_body=processed_extra_body
             )
     else:
         for k in passed_params.keys():
@@ -6608,6 +6617,13 @@ class ProviderConfigManager:
         # This mapping ensures that the correct configuration is returned for BEDROCK.
         elif litellm.LlmProviders.BEDROCK == provider:
             return litellm.AmazonAnthropicClaude3MessagesConfig()
+        elif litellm.LlmProviders.VERTEX_AI == provider:
+            if "claude" in model:
+                from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import (
+                    VertexAIPartnerModelsAnthropicMessagesConfig,
+                )
+
+                return VertexAIPartnerModelsAnthropicMessagesConfig()
         return None
 
     @staticmethod

tests/llm_translation/test_optional_params.py

@@ -1487,4 +1487,16 @@ def test_cohere_embed_dimensions_param():
         custom_llm_provider="cohere",
         encoding_format="float",
     )
-    assert optional_params["embedding_types"] == ["float"]
+    assert optional_params["embedding_types"] == ["float"]
+
+def test_optional_params_with_additional_drop_params():
+    optional_params = get_optional_params(
+        model="gpt-4o",
+        custom_llm_provider="openai",
+        additional_drop_params=["red"],
+        drop_params=True,
+        red="blue"
+    )
+    print(f"optional_params: {optional_params}")
+    assert "red" not in optional_params
+    assert "red" not in optional_params["extra_body"]

tests/local_testing/test_amazing_vertex_completion.py

@@ -1530,6 +1530,61 @@ async def test_gemini_pro_json_schema_args_sent_httpx(
 
             assert resp.model == model.split("/")[1]
 
+@pytest.mark.asyncio
+async def test_anthropic_message_via_anthropic_messages():
+    from litellm.llms.custom_httpx.llm_http_handler import AsyncHTTPHandler
+    from unittest.mock import MagicMock, AsyncMock
+
+    load_vertex_ai_credentials()
+    os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True"
+    litellm.model_cost = litellm.get_model_cost_map(url="")
+    litellm.set_verbose = True
+    client = AsyncHTTPHandler()
+
+    httpx_response = AsyncMock()
+    httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic
+
+    call_1_kwargs = {}
+    call_2_kwargs = {}
+    with patch.object(client, "post", new=httpx_response) as mock_call:
+        messages = [{"role": "user", "content": "List 5 cookie recipes"}]
+        response = await litellm.anthropic_messages(model="vertex_ai/claude-3-5-sonnet@20240620", messages=messages, max_tokens=100, client=client)
+
+        print(f"response: {response}")
+        assert mock_call.call_count == 1
+        call_1_kwargs = mock_call.call_args.kwargs
+
+    with patch.object(client, "post", new=httpx_response) as mock_call:
+        response_2 = await litellm.acompletion(model="vertex_ai/claude-3-5-sonnet@20240620", messages=messages, max_tokens=100, client=client)
+        print(f"response_2: {response_2}")
+        call_args = mock_call.call_args
+        print(f"call_args: {call_args}")
+        call_2_kwargs = mock_call.call_args.kwargs
+        call_2_kwargs["url"] = call_args[0][0]
+    
+    """
+    Compare Call 1 and Call 2
+
+    Expect:
+        - url 
+        - headers
+        - data / json
+
+        to be the same, except for the Authorization header.
+    """
+    print(f"call_1_kwargs: {call_1_kwargs}")
+    print(f"call_2_kwargs: {call_2_kwargs}")
+    assert call_1_kwargs["url"] == call_2_kwargs["url"], f"Expected url to be the same, but got {call_1_kwargs['url']} and Expected {call_2_kwargs['url']}"
+    assert "Authorization".lower() in [k.lower() for k in call_1_kwargs["headers"].keys()], f"Expected Authorization header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}"
+    assert "content-type".lower() in [k.lower() for k in call_1_kwargs["headers"].keys()], f"Expected Content-Type header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}"
+
+    ## validate request body
+    print(f"call 1 kwargs keys: {call_1_kwargs.keys()}")
+    print(f"call_2_kwargs['json']: {type(call_2_kwargs['json'])}")
+    print(f"call_1_kwargs['data']: {type(call_1_kwargs['data'])}")
+    call_1_kwargs_data = json.loads(call_1_kwargs["data"])
+    for k, v in call_2_kwargs["json"].items():
+        assert k in call_1_kwargs_data, f"Expected {k} to be present in call_1_kwargs['data'], but got {call_1_kwargs_data.keys()}"
 
 @pytest.mark.parametrize(
     "model, vertex_location, supports_response_schema",
@@ -3781,3 +3836,5 @@ def test_vertex_schema_test():
     )
 
     print(response)
+
+