From 5d4ae9aa4dae85eae3d02deb9c56441bb2717228 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 29 May 2025 23:42:48 -0700 Subject: [PATCH] Support dropping non-openai params when specified in `additional_drop_params` + Add VertexAI Anthropic support on `/v1/messages` (#11246) * feat(utils.py): support dropping non-openai params when specified via additional drop params Closes https://github.com/BerriAI/litellm/issues/11205 * fix(utils.py): fix linting error * refactor(handler.py): add custom llm provider to anthropic messages provider config exception * feat: initial commit adding vertex ai anthropic support on `/v1/messages` * test: add working unit test * test(vertex_ai_partner_models/anthropic): add /v1/messages support for anthropic api Adds vertex ai auth * feat(vertex_ai/anthropic): return correct url when calling via `/v1/messages` * fix: more alignment to expected anthropic request format * fix: fix ruff linting check * Removed syntax error from docs (#11242) * [Feat]: Add Bedrock InvokeAgents as a /chat/completions route on LiteLLM (#11239) * feat: init structure for bedrock AGENTs * feat: add basic routing for bedrock AGENTs * feat: add basic transforms for bedrock AGENTs * fix: url for bedrock agent runtime * fix: working agents request * feat: working agents non-streaming request * feat: bedrock agents * feat: add streaming for bedrock agents * feat: add cost tracking for bedrock agents * docs litellm with bedrock agents * fix: linting errors * test: invoke agents tests * fix: import session handling * Revert "fix: import session handling" This reverts commit deb257dc107fc72bdee932953ee803023b73c838. * fix: linting pin mypy * [Feat]: Guardrails - Add streaming for bedrock post guard (#11247) * feat: add streaming for bedrock post guard * fix: bedrock guardrails * fix: add clear comments * Update litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * fix: clean up bedrock guardrails --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * [Fix] Responses API - Session management (#11254) * fix: import session handling * fix: imports for session handler * tests: tests for session handler * Update enterprise/litellm_enterprise/enterprise_callbacks/session_handler.py Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * bump: bump litellm enterprise * fixes: test_create_user_default_budget * fix: fix linting error * fix: fix linting error --------- Co-authored-by: Fadil Rahman <87557055+fadil4u@users.noreply.github.com> Co-authored-by: Ishaan Jaff Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- .../llm_response_utils/get_api_base.py | 8 +- .../messages/handler.py | 2 +- .../messages/transformation.py | 10 +- .../anthropic_messages/transformation.py | 10 +- .../anthropic_claude3_transformation.py | 24 ++--- litellm/llms/custom_httpx/llm_http_handler.py | 5 +- litellm/llms/vertex_ai/batches/handler.py | 6 +- .../transformation.py | 96 +++++++++++++++++++ .../vertex_ai_partner_models/main.py | 79 +-------------- litellm/llms/vertex_ai/vertex_llm_base.py | 70 +++++++++++++- .../proxy/_experimental/out/onboarding.html | 1 - litellm/proxy/_new_secret_config.yaml | 8 +- litellm/types/llms/vertex_ai.py | 7 ++ litellm/utils.py | 20 +++- tests/llm_translation/test_optional_params.py | 14 ++- .../test_amazing_vertex_completion.py | 57 +++++++++++ 16 files changed, 304 insertions(+), 113 deletions(-) create mode 100644 litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py delete mode 100644 litellm/proxy/_experimental/out/onboarding.html diff --git a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py index 6f9fa36591..c23bbb936b 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_api_base.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_api_base.py @@ -72,13 +72,11 @@ def get_api_base( _optional_params.vertex_location is not None and _optional_params.vertex_project is not None ): - from litellm.llms.vertex_ai.vertex_ai_partner_models.main import ( - VertexPartnerProvider, - create_vertex_url, - ) + from litellm.llms.vertex_ai.vertex_llm_base import VertexBase + from litellm.types.llms.vertex_ai import VertexPartnerProvider if "claude" in model: - _api_base = create_vertex_url( + _api_base = VertexBase.create_vertex_url( vertex_location=_optional_params.vertex_location, vertex_project=_optional_params.vertex_project, model=model, diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index b7c8fb5650..daec92a3bd 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -140,7 +140,7 @@ def anthropic_messages_handler( ) if anthropic_messages_provider_config is None: raise ValueError( - f"Anthropic messages provider config not found for model: {model}" + f"Anthropic messages provider config not found for model: {model}, custom_llm_provider: {custom_llm_provider}" ) if custom_llm_provider is None: raise ValueError( diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py index 5b5e2e6f36..aee56dc6f9 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/transformation.py @@ -1,4 +1,4 @@ -from typing import Any, AsyncIterator, Dict, List, Optional +from typing import Any, AsyncIterator, Dict, List, Optional, Tuple import httpx @@ -50,7 +50,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): api_base = f"{api_base}/v1/messages" return api_base - def validate_environment( + def validate_anthropic_messages_environment( self, headers: dict, model: str, @@ -59,14 +59,14 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig): litellm_params: dict, api_key: Optional[str] = None, api_base: Optional[str] = None, - ) -> dict: - if "x-api-key" not in headers: + ) -> Tuple[dict, Optional[str]]: + if "x-api-key" not in headers and api_key: headers["x-api-key"] = api_key if "anthropic-version" not in headers: headers["anthropic-version"] = DEFAULT_ANTHROPIC_API_VERSION if "content-type" not in headers: headers["content-type"] = "application/json" - return headers + return headers, api_base def transform_anthropic_messages_request( self, diff --git a/litellm/llms/base_llm/anthropic_messages/transformation.py b/litellm/llms/base_llm/anthropic_messages/transformation.py index 710a107688..5bf16eb3cf 100644 --- a/litellm/llms/base_llm/anthropic_messages/transformation.py +++ b/litellm/llms/base_llm/anthropic_messages/transformation.py @@ -18,7 +18,7 @@ else: class BaseAnthropicMessagesConfig(ABC): @abstractmethod - def validate_environment( + def validate_anthropic_messages_environment( # use different name because return type is different from base config's validate_environment self, headers: dict, model: str, @@ -27,13 +27,17 @@ class BaseAnthropicMessagesConfig(ABC): litellm_params: dict, api_key: Optional[str] = None, api_base: Optional[str] = None, - ) -> dict: + ) -> Tuple[dict, Optional[str]]: """ OPTIONAL Validate the environment for the request + + Returns: + - headers: dict + - api_base: Optional[str] - If the provider needs to update the api_base, return it here. Otherwise, return None. """ - return headers + return headers, api_base @abstractmethod def get_complete_url( diff --git a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py index ff475a95db..03623bf86d 100644 --- a/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py +++ b/litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py @@ -38,6 +38,18 @@ class AmazonAnthropicClaude3MessagesConfig( BaseAnthropicMessagesConfig.__init__(self, **kwargs) AmazonInvokeConfig.__init__(self, **kwargs) + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + return headers, api_base + def sign_request( self, headers: dict, @@ -59,18 +71,6 @@ class AmazonAnthropicClaude3MessagesConfig( fake_stream=fake_stream, ) - def validate_environment( - self, - headers: dict, - model: str, - messages: List[Any], - optional_params: dict, - litellm_params: dict, - api_key: Optional[str] = None, - api_base: Optional[str] = None, - ) -> dict: - return headers - def get_complete_url( self, api_base: Optional[str], diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 6523184c08..2d337c5c9e 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1097,7 +1097,10 @@ class BaseLLMHTTPHandler: if provider_specific_header else {} ) - headers = anthropic_messages_provider_config.validate_environment( + ( + headers, + api_base, + ) = anthropic_messages_provider_config.validate_anthropic_messages_environment( headers=extra_headers or {}, model=model, messages=messages, diff --git a/litellm/llms/vertex_ai/batches/handler.py b/litellm/llms/vertex_ai/batches/handler.py index dc3f93857a..7932881f48 100644 --- a/litellm/llms/vertex_ai/batches/handler.py +++ b/litellm/llms/vertex_ai/batches/handler.py @@ -43,7 +43,7 @@ class VertexAIBatchPrediction(VertexLLM): custom_llm_provider="vertex_ai", ) - default_api_base = self.create_vertex_url( + default_api_base = self.create_vertex_batch_url( vertex_location=vertex_location or "us-central1", vertex_project=vertex_project or project_id, ) @@ -117,7 +117,7 @@ class VertexAIBatchPrediction(VertexLLM): ) return vertex_batch_response - def create_vertex_url( + def create_vertex_batch_url( self, vertex_location: str, vertex_project: str, @@ -145,7 +145,7 @@ class VertexAIBatchPrediction(VertexLLM): custom_llm_provider="vertex_ai", ) - default_api_base = self.create_vertex_url( + default_api_base = self.create_vertex_batch_url( vertex_location=vertex_location or "us-central1", vertex_project=vertex_project or project_id, ) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py new file mode 100644 index 0000000000..2545fe0ed7 --- /dev/null +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -0,0 +1,96 @@ +from typing import Any, Dict, List, Optional, Tuple + +import litellm +from litellm.llms.anthropic.experimental_pass_through.messages.transformation import ( + AnthropicMessagesConfig, +) +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.vertex_ai import VertexPartnerProvider +from litellm.types.router import GenericLiteLLMParams + +from ....vertex_llm_base import VertexBase + + +class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase): + def validate_anthropic_messages_environment( + self, + headers: dict, + model: str, + messages: List[Any], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> Tuple[dict, Optional[str]]: + """ + OPTIONAL + + Validate the environment for the request + """ + if "Authorization" not in headers: + vertex_ai_project = ( + optional_params.pop("vertex_project", None) + or optional_params.pop("vertex_ai_project", None) + or litellm.vertex_project + or get_secret_str("VERTEXAI_PROJECT") + ) + vertex_credentials = ( + optional_params.pop("vertex_credentials", None) + or optional_params.pop("vertex_ai_credentials", None) + or get_secret_str("VERTEXAI_CREDENTIALS") + ) + + access_token, project_id = self._ensure_access_token( + credentials=vertex_credentials, + project_id=vertex_ai_project, + custom_llm_provider="vertex_ai", + ) + + headers["Authorization"] = f"Bearer {access_token}" + + api_base = self.get_complete_vertex_url( + custom_api_base=api_base, + vertex_location=optional_params.pop("vertex_location", None), + vertex_project=vertex_ai_project, + project_id=project_id, + partner=VertexPartnerProvider.claude, + stream=optional_params.get("stream", False), + model=model, + ) + + headers["content-type"] = "application/json" + return headers, api_base + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base is None: + raise ValueError( + "api_base is required. Unable to determine the correct api_base for the request." + ) + return api_base # no transformation is needed - handled in validate_environment + + def transform_anthropic_messages_request( + self, + model: str, + messages: List[Dict], + anthropic_messages_optional_request_params: Dict, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Dict: + anthropic_messages_request = super().transform_anthropic_messages_request( + model=model, + messages=messages, + anthropic_messages_optional_request_params=anthropic_messages_optional_request_params, + litellm_params=litellm_params, + headers=headers, + ) + + anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" + return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index bcfdcb69ca..36c1704439 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -1,12 +1,12 @@ # What is this? ## API Handler for calling Vertex AI Partner Models -from enum import Enum from typing import Callable, Optional, Union import httpx # type: ignore import litellm from litellm import LlmProviders +from litellm.types.llms.vertex_ai import VertexPartnerProvider from litellm.utils import ModelResponse from ...custom_httpx.llm_http_handler import BaseLLMHTTPHandler @@ -15,13 +15,6 @@ from ..vertex_llm_base import VertexBase base_llm_http_handler = BaseLLMHTTPHandler() -class VertexPartnerProvider(str, Enum): - mistralai = "mistralai" - llama = "llama" - ai21 = "ai21" - claude = "claude" - - class VertexAIError(Exception): def __init__(self, status_code, message): self.status_code = status_code @@ -35,78 +28,10 @@ class VertexAIError(Exception): ) # Call the base class constructor with the parameters it needs -def create_vertex_url( - vertex_location: str, - vertex_project: str, - partner: VertexPartnerProvider, - stream: Optional[bool], - model: str, - api_base: Optional[str] = None, -) -> str: - """Return the base url for the vertex partner models""" - - api_base = api_base or f"https://{vertex_location}-aiplatform.googleapis.com" - if partner == VertexPartnerProvider.llama: - return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions" - elif partner == VertexPartnerProvider.mistralai: - if stream: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict" - elif partner == VertexPartnerProvider.ai21: - if stream: - return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:rawPredict" - elif partner == VertexPartnerProvider.claude: - if stream: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:streamRawPredict" - else: - return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:rawPredict" - - class VertexAIPartnerModels(VertexBase): def __init__(self) -> None: pass - def get_complete_url( - self, - custom_api_base: Optional[str], - vertex_location: Optional[str], - vertex_project: Optional[str], - project_id: str, - partner: VertexPartnerProvider, - stream: Optional[bool], - model: str, - ) -> str: - api_base = self.get_api_base( - api_base=custom_api_base, vertex_location=vertex_location - ) - default_api_base = create_vertex_url( - vertex_location=vertex_location or "us-central1", - vertex_project=vertex_project or project_id, - partner=partner, # type: ignore - stream=stream, - model=model, - api_base=api_base, - ) - - if len(default_api_base.split(":")) > 1: - endpoint = default_api_base.split(":")[-1] - else: - endpoint = "" - - _, api_base = self._check_custom_proxy( - api_base=custom_api_base, - custom_llm_provider="vertex_ai", - gemini_api_key=None, - endpoint=endpoint, - stream=stream, - auth_header=None, - url=default_api_base, - ) - return api_base - def completion( self, model: str, @@ -181,7 +106,7 @@ class VertexAIPartnerModels(VertexBase): else: raise ValueError(f"Unknown partner model: {model}") - api_base = self.get_complete_url( + api_base = self.get_complete_vertex_url( custom_api_base=api_base, vertex_location=vertex_location, vertex_project=vertex_project, diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 6d118cc658..261b0876ff 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -11,7 +11,7 @@ from typing import TYPE_CHECKING, Any, Dict, Literal, Optional, Tuple from litellm._logging import verbose_logger from litellm.litellm_core_utils.asyncify import asyncify from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES +from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES, VertexPartnerProvider from .common_utils import _get_gemini_url, _get_vertex_url, all_gemini_url_modes @@ -150,6 +150,74 @@ class VertexBase: else: return f"https://{self.get_default_vertex_location()}-aiplatform.googleapis.com" + @staticmethod + def create_vertex_url( + vertex_location: str, + vertex_project: str, + partner: VertexPartnerProvider, + stream: Optional[bool], + model: str, + api_base: Optional[str] = None, + ) -> str: + """Return the base url for the vertex partner models""" + + api_base = api_base or f"https://{vertex_location}-aiplatform.googleapis.com" + if partner == VertexPartnerProvider.llama: + return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/openapi/chat/completions" + elif partner == VertexPartnerProvider.mistralai: + if stream: + return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:streamRawPredict" + else: + return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/mistralai/models/{model}:rawPredict" + elif partner == VertexPartnerProvider.ai21: + if stream: + return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:streamRawPredict" + else: + return f"{api_base}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/publishers/ai21/models/{model}:rawPredict" + elif partner == VertexPartnerProvider.claude: + if stream: + return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:streamRawPredict" + else: + return f"{api_base}/v1/projects/{vertex_project}/locations/{vertex_location}/publishers/anthropic/models/{model}:rawPredict" + + def get_complete_vertex_url( + self, + custom_api_base: Optional[str], + vertex_location: Optional[str], + vertex_project: Optional[str], + project_id: str, + partner: VertexPartnerProvider, + stream: Optional[bool], + model: str, + ) -> str: + api_base = self.get_api_base( + api_base=custom_api_base, vertex_location=vertex_location + ) + default_api_base = VertexBase.create_vertex_url( + vertex_location=vertex_location or "us-central1", + vertex_project=vertex_project or project_id, + partner=partner, + stream=stream, + model=model, + api_base=api_base, + ) + + if len(default_api_base.split(":")) > 1: + endpoint = default_api_base.split(":")[-1] + else: + endpoint = "" + + _, api_base = self._check_custom_proxy( + api_base=custom_api_base, + custom_llm_provider="vertex_ai", + gemini_api_key=None, + endpoint=endpoint, + stream=stream, + auth_header=None, + url=default_api_base, + ) + return api_base + def refresh_auth(self, credentials: Any) -> None: from google.auth.transport.requests import ( Request, # type: ignore[import-untyped] diff --git a/litellm/proxy/_experimental/out/onboarding.html b/litellm/proxy/_experimental/out/onboarding.html deleted file mode 100644 index 11a85a5cf6..0000000000 --- a/litellm/proxy/_experimental/out/onboarding.html +++ /dev/null @@ -1 +0,0 @@ -LiteLLM Dashboard \ No newline at end of file diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 1878442800..6a12bc98a7 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -8,6 +8,12 @@ model_list: litellm_params: model: text-embedding-3-small api_key: os.environ/OPENAI_API_KEY - + - model_name: openai:gpt-4o + litellm_params: + model: openai/gpt-4o + api_key: os.environ/OPENAI_API_KEY + drop_params: true + additional_drop_params: + - red litellm_settings: cache: true \ No newline at end of file diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index be43a7969e..3cca1c099b 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -577,3 +577,10 @@ class VertexBatchPredictionResponse(TypedDict, total=False): VERTEX_CREDENTIALS_TYPES = Union[str, Dict[str, str]] + + +class VertexPartnerProvider(str, Enum): + mistralai = "mistralai" + llama = "llama" + ai21 = "ai21" + claude = "claude" diff --git a/litellm/utils.py b/litellm/utils.py index 47638afd11..4d6356cea0 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3688,13 +3688,22 @@ def add_provider_specific_params_to_optional_params( if k not in openai_params: extra_body[k] = passed_params[k] optional_params.setdefault("extra_body", {}) - optional_params["extra_body"] = { + initial_extra_body = { **optional_params["extra_body"], **extra_body, } + if additional_drop_params is not None: + processed_extra_body = { + k: v + for k, v in initial_extra_body.items() + if k not in additional_drop_params + } + else: + processed_extra_body = initial_extra_body + optional_params["extra_body"] = _ensure_extra_body_is_safe( - extra_body=optional_params["extra_body"] + extra_body=processed_extra_body ) else: for k in passed_params.keys(): @@ -6608,6 +6617,13 @@ class ProviderConfigManager: # This mapping ensures that the correct configuration is returned for BEDROCK. elif litellm.LlmProviders.BEDROCK == provider: return litellm.AmazonAnthropicClaude3MessagesConfig() + elif litellm.LlmProviders.VERTEX_AI == provider: + if "claude" in model: + from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.experimental_pass_through.transformation import ( + VertexAIPartnerModelsAnthropicMessagesConfig, + ) + + return VertexAIPartnerModelsAnthropicMessagesConfig() return None @staticmethod diff --git a/tests/llm_translation/test_optional_params.py b/tests/llm_translation/test_optional_params.py index 4ce18be8ad..6e5a7f0ac2 100644 --- a/tests/llm_translation/test_optional_params.py +++ b/tests/llm_translation/test_optional_params.py @@ -1487,4 +1487,16 @@ def test_cohere_embed_dimensions_param(): custom_llm_provider="cohere", encoding_format="float", ) - assert optional_params["embedding_types"] == ["float"] \ No newline at end of file + assert optional_params["embedding_types"] == ["float"] + +def test_optional_params_with_additional_drop_params(): + optional_params = get_optional_params( + model="gpt-4o", + custom_llm_provider="openai", + additional_drop_params=["red"], + drop_params=True, + red="blue" + ) + print(f"optional_params: {optional_params}") + assert "red" not in optional_params + assert "red" not in optional_params["extra_body"] \ No newline at end of file diff --git a/tests/local_testing/test_amazing_vertex_completion.py b/tests/local_testing/test_amazing_vertex_completion.py index e36b57e557..109e9b5ce5 100644 --- a/tests/local_testing/test_amazing_vertex_completion.py +++ b/tests/local_testing/test_amazing_vertex_completion.py @@ -1530,6 +1530,61 @@ async def test_gemini_pro_json_schema_args_sent_httpx( assert resp.model == model.split("/")[1] +@pytest.mark.asyncio +async def test_anthropic_message_via_anthropic_messages(): + from litellm.llms.custom_httpx.llm_http_handler import AsyncHTTPHandler + from unittest.mock import MagicMock, AsyncMock + + load_vertex_ai_credentials() + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + litellm.set_verbose = True + client = AsyncHTTPHandler() + + httpx_response = AsyncMock() + httpx_response.side_effect = vertex_httpx_mock_post_valid_response_anthropic + + call_1_kwargs = {} + call_2_kwargs = {} + with patch.object(client, "post", new=httpx_response) as mock_call: + messages = [{"role": "user", "content": "List 5 cookie recipes"}] + response = await litellm.anthropic_messages(model="vertex_ai/claude-3-5-sonnet@20240620", messages=messages, max_tokens=100, client=client) + + print(f"response: {response}") + assert mock_call.call_count == 1 + call_1_kwargs = mock_call.call_args.kwargs + + with patch.object(client, "post", new=httpx_response) as mock_call: + response_2 = await litellm.acompletion(model="vertex_ai/claude-3-5-sonnet@20240620", messages=messages, max_tokens=100, client=client) + print(f"response_2: {response_2}") + call_args = mock_call.call_args + print(f"call_args: {call_args}") + call_2_kwargs = mock_call.call_args.kwargs + call_2_kwargs["url"] = call_args[0][0] + + """ + Compare Call 1 and Call 2 + + Expect: + - url + - headers + - data / json + + to be the same, except for the Authorization header. + """ + print(f"call_1_kwargs: {call_1_kwargs}") + print(f"call_2_kwargs: {call_2_kwargs}") + assert call_1_kwargs["url"] == call_2_kwargs["url"], f"Expected url to be the same, but got {call_1_kwargs['url']} and Expected {call_2_kwargs['url']}" + assert "Authorization".lower() in [k.lower() for k in call_1_kwargs["headers"].keys()], f"Expected Authorization header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}" + assert "content-type".lower() in [k.lower() for k in call_1_kwargs["headers"].keys()], f"Expected Content-Type header to be present in call_1_kwargs, but got {call_1_kwargs['headers'].keys()}" + + ## validate request body + print(f"call 1 kwargs keys: {call_1_kwargs.keys()}") + print(f"call_2_kwargs['json']: {type(call_2_kwargs['json'])}") + print(f"call_1_kwargs['data']: {type(call_1_kwargs['data'])}") + call_1_kwargs_data = json.loads(call_1_kwargs["data"]) + for k, v in call_2_kwargs["json"].items(): + assert k in call_1_kwargs_data, f"Expected {k} to be present in call_1_kwargs['data'], but got {call_1_kwargs_data.keys()}" @pytest.mark.parametrize( "model, vertex_location, supports_response_schema", @@ -3781,3 +3836,5 @@ def test_vertex_schema_test(): ) print(response) + +