diff --git a/litellm/_logging.py b/litellm/_logging.py index 7824fcfa67..d072cc549d 100644 --- a/litellm/_logging.py +++ b/litellm/_logging.py @@ -86,6 +86,8 @@ _SECRET_RE = _build_secret_patterns() def _redact_string(value: str) -> str: + if not _ENABLE_SECRET_REDACTION: + return value return _SECRET_RE.sub(_REDACTED, value) diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index bc54786420..ef062ff47a 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -6,7 +6,7 @@ from typing import Any, Optional import httpx import litellm -from litellm._logging import verbose_logger +from litellm._logging import _redact_string, verbose_logger from litellm.types.utils import LlmProviders from ..exceptions import ( @@ -2304,7 +2304,7 @@ def exception_type( # type: ignore # noqa: PLR0915 else: # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors raise APIConnectionError( - message=f"{exception_provider} APIConnectionError - {message}\n{traceback.format_exc()}", + message=f"{exception_provider} APIConnectionError - {message}\n{_redact_string(traceback.format_exc())}", llm_provider="azure", model=model, litellm_debug_info=extra_information, @@ -2431,7 +2431,7 @@ def exception_type( # type: ignore # noqa: PLR0915 else: raise APIConnectionError( message="{}\n{}".format( - str(original_exception), traceback.format_exc() + str(original_exception), _redact_string(traceback.format_exc()) ), llm_provider=custom_llm_provider, model=model, @@ -2460,7 +2460,7 @@ def exception_type( # type: ignore # noqa: PLR0915 setattr(e, "litellm_response_headers", litellm_response_headers) raise e # it's already mapped raised_exc = APIConnectionError( - message="{}\n{}".format(original_exception, traceback.format_exc()), + message="{}\n{}".format(original_exception, _redact_string(traceback.format_exc())), llm_provider="", model="", ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index e84c1e13a8..455e651643 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -36,7 +36,7 @@ from litellm import ( log_raw_request_response, turn_off_message_logging, ) -from litellm._logging import _is_debugging_on, verbose_logger +from litellm._logging import _is_debugging_on, _redact_string, verbose_logger from litellm._uuid import uuid from litellm.batches.batch_utils import _handle_completed_batch from litellm.caching.caching import DualCache, InMemoryCache @@ -2848,7 +2848,11 @@ class Logging(LiteLLMLoggingBaseClass): self.model_call_details["log_event_type"] = "failed_api_call" self.model_call_details["exception"] = exception - self.model_call_details["traceback_exception"] = traceback_exception + self.model_call_details["traceback_exception"] = ( + _redact_string(traceback_exception) + if isinstance(traceback_exception, str) + else traceback_exception + ) self.model_call_details["end_time"] = end_time self.model_call_details.setdefault("original_response", None) self.model_call_details["response_cost"] = 0 @@ -2871,7 +2875,7 @@ class Logging(LiteLLMLoggingBaseClass): end_time=end_time, logging_obj=self, status="failure", - error_str=str(exception), + error_str=_redact_string(str(exception)), original_exception=exception, standard_built_in_tools_params=self.standard_built_in_tools_params, ) diff --git a/litellm/llms/azure/realtime/handler.py b/litellm/llms/azure/realtime/handler.py index 6d00ecd51c..1f3428f2ca 100644 --- a/litellm/llms/azure/realtime/handler.py +++ b/litellm/llms/azure/realtime/handler.py @@ -6,7 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast -from litellm._logging import verbose_proxy_logger +from litellm._logging import _redact_string, verbose_proxy_logger from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from ....litellm_core_utils.litellm_logging import Logging as LiteLLMLogging @@ -118,7 +118,7 @@ class AzureOpenAIRealtime(AzureChatCompletion): await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - await websocket.close(code=e.status_code, reason=str(e)) + await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception: verbose_proxy_logger.exception( "Error in AzureOpenAIRealtime.async_realtime" diff --git a/litellm/llms/bedrock/realtime/handler.py b/litellm/llms/bedrock/realtime/handler.py index cde9f3e6fc..8405ff500d 100644 --- a/litellm/llms/bedrock/realtime/handler.py +++ b/litellm/llms/bedrock/realtime/handler.py @@ -8,7 +8,7 @@ import asyncio import json from typing import Any, Optional -from litellm._logging import verbose_proxy_logger +from litellm._logging import _redact_string, verbose_proxy_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from ..base_aws_llm import BaseAWSLLM @@ -152,7 +152,7 @@ class BedrockRealtime(BaseAWSLLM): f"Error in BedrockRealtime.async_realtime: {e}" ) try: - await websocket.close(code=1011, reason=f"Internal error: {str(e)}") + await websocket.close(code=1011, reason=_redact_string(f"Internal error: {str(e)}")) except Exception: pass raise diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index 001547557d..fdb05d1a91 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -316,16 +316,65 @@ def mask_sensitive_info(error_message): return error_message +def _safe_get_response_text(response: httpx.Response) -> str: + """Safely read response text, falling back to empty string on decoding errors.""" + try: + return response.text + except Exception: + return "" + + +async def _safe_aread_response(response: httpx.Response) -> bytes: + """Safely read async response body, falling back to empty bytes on errors.""" + try: + return await response.aread() + except Exception: + return b"" + + +def _safe_read_response(response: httpx.Response) -> bytes: + """Safely read sync response body, falling back to empty bytes on errors.""" + try: + return response.read() + except Exception: + return b"" + + +def _raise_masked_sync_error(e: httpx.HTTPStatusError, stream: bool) -> None: + """Raise a MaskedHTTPStatusError for sync HTTP handlers.""" + if stream: + _body = mask_sensitive_info(_safe_read_response(e.response)) + raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + _text = mask_sensitive_info(_safe_get_response_text(e.response)) + raise MaskedHTTPStatusError(e, message=_text, text=_text) from None + + +async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> None: + """Raise a MaskedHTTPStatusError for async HTTP handlers.""" + if stream: + _body = await _safe_aread_response(e.response) + raise MaskedHTTPStatusError(e, message=_body, text=_body) from None + _text = mask_sensitive_info(_safe_get_response_text(e.response)) + raise MaskedHTTPStatusError(e, message=_text, text=_text) from None + + class MaskedHTTPStatusError(httpx.HTTPStatusError): def __init__( self, original_error, message: Optional[str] = None, text: Optional[str] = None ): # Create a new error with the masked URL masked_url = mask_sensitive_info(str(original_error.request.url)) - # Create a new error that looks like the original, but with a masked URL + # Mask the original exception message too (it contains the full URL) + masked_original_message = mask_sensitive_info(str(original_error)) + + # Safely access response content — decompression can fail (e.g. zlib error) + try: + response_content = original_error.response.content + except Exception: + response_content = b"" super().__init__( - message=original_error.message, + message=masked_original_message, request=httpx.Request( method=original_error.request.method, url=masked_url, @@ -334,12 +383,13 @@ class MaskedHTTPStatusError(httpx.HTTPStatusError): ), response=httpx.Response( status_code=original_error.response.status_code, - content=original_error.response.content, + content=response_content, headers=original_error.response.headers, ), ) self.message = message self.text = text + self.status_code = original_error.response.status_code class AsyncHTTPHandler: @@ -501,16 +551,7 @@ class AsyncHTTPHandler: headers=headers, ) except httpx.HTTPStatusError as e: - if stream is True: - setattr(e, "message", await e.response.aread()) - setattr(e, "text", await e.response.aread()) - else: - setattr(e, "message", mask_sensitive_info(e.response.text)) - setattr(e, "text", mask_sensitive_info(e.response.text)) - - setattr(e, "status_code", e.response.status_code) - - raise e + await _raise_masked_async_error(e, stream) except Exception as e: raise e @@ -571,12 +612,7 @@ class AsyncHTTPHandler: headers=headers, ) except httpx.HTTPStatusError as e: - setattr(e, "status_code", e.response.status_code) - if stream is True: - setattr(e, "message", await e.response.aread()) - else: - setattr(e, "message", e.response.text) - raise e + await _raise_masked_async_error(e, stream) except Exception as e: raise e @@ -637,12 +673,7 @@ class AsyncHTTPHandler: headers=headers, ) except httpx.HTTPStatusError as e: - setattr(e, "status_code", e.response.status_code) - if stream is True: - setattr(e, "message", await e.response.aread()) - else: - setattr(e, "message", e.response.text) - raise e + await _raise_masked_async_error(e, stream) except Exception as e: raise e @@ -690,12 +721,7 @@ class AsyncHTTPHandler: finally: await new_client.aclose() except httpx.HTTPStatusError as e: - setattr(e, "status_code", e.response.status_code) - if stream is True: - setattr(e, "message", await e.response.aread()) - else: - setattr(e, "message", e.response.text) - raise e + await _raise_masked_async_error(e, stream) except Exception as e: raise e @@ -1035,16 +1061,7 @@ class HTTPHandler: llm_provider="litellm-httpx-handler", ) except httpx.HTTPStatusError as e: - if stream is True: - setattr(e, "message", mask_sensitive_info(e.response.read())) - setattr(e, "text", mask_sensitive_info(e.response.read())) - else: - error_text = mask_sensitive_info(e.response.text) - setattr(e, "message", error_text) - setattr(e, "text", error_text) - - setattr(e, "status_code", e.response.status_code) - raise e + _raise_masked_sync_error(e, stream) except Exception as e: raise e @@ -1083,17 +1100,7 @@ class HTTPHandler: llm_provider="litellm-httpx-handler", ) except httpx.HTTPStatusError as e: - if stream is True: - setattr(e, "message", mask_sensitive_info(e.response.read())) - setattr(e, "text", mask_sensitive_info(e.response.read())) - else: - error_text = mask_sensitive_info(e.response.text) - setattr(e, "message", error_text) - setattr(e, "text", error_text) - - setattr(e, "status_code", e.response.status_code) - - raise e + _raise_masked_sync_error(e, stream) except Exception as e: raise e @@ -1130,6 +1137,8 @@ class HTTPHandler: model="default-model-name", llm_provider="litellm-httpx-handler", ) + except httpx.HTTPStatusError as e: + _raise_masked_sync_error(e, stream) except Exception as e: raise e @@ -1168,17 +1177,7 @@ class HTTPHandler: llm_provider="litellm-httpx-handler", ) except httpx.HTTPStatusError as e: - if stream is True: - setattr(e, "message", mask_sensitive_info(e.response.read())) - setattr(e, "text", mask_sensitive_info(e.response.read())) - else: - error_text = mask_sensitive_info(e.response.text) - setattr(e, "message", error_text) - setattr(e, "text", error_text) - - setattr(e, "status_code", e.response.status_code) - - raise e + _raise_masked_sync_error(e, stream) except Exception as e: raise e diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 7a8820a878..d8ef7e7440 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -22,7 +22,7 @@ import litellm import litellm.litellm_core_utils import litellm.types import litellm.types.utils -from litellm._logging import verbose_logger +from litellm._logging import _redact_string, verbose_logger from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming @@ -4789,12 +4789,12 @@ class BaseLLMHTTPHandler: except websockets.exceptions.InvalidStatusCode as e: # type: ignore verbose_logger.exception(f"Error connecting to backend: {e}") - await websocket.close(code=e.status_code, reason=str(e)) + await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: verbose_logger.exception(f"Error connecting to backend: {e}") try: await websocket.close( - code=1011, reason=f"Internal server error: {str(e)}" + code=1011, reason=_redact_string(f"Internal server error: {str(e)}") ) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str( @@ -5076,12 +5076,12 @@ class BaseLLMHTTPHandler: except websockets.exceptions.InvalidStatusCode as e: # type: ignore verbose_logger.exception(f"Error connecting to responses WS backend: {e}") - await websocket.close(code=e.status_code, reason=str(e)) + await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: verbose_logger.exception(f"Error in responses WS: {e}") try: await websocket.close( - code=1011, reason=f"Internal server error: {str(e)}" + code=1011, reason=_redact_string(f"Internal server error: {str(e)}") ) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str( diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index 87c107fab3..bc963d62b5 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -28,7 +28,7 @@ class GeminiModelInfo(BaseLLMModelInfo): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - """Google AI Studio sends api key in query params""" + """Google AI Studio sends api key via x-goog-api-key header""" return headers @property @@ -75,7 +75,8 @@ class GeminiModelInfo(BaseLLMModelInfo): ) response = litellm.module_level_client.get( - url=f"{api_base}{endpoint}?key={api_key}", + url=f"{api_base}{endpoint}", + headers={"x-goog-api-key": api_key}, ) if response.status_code != 200: diff --git a/litellm/llms/gemini/files/transformation.py b/litellm/llms/gemini/files/transformation.py index a29ed66e63..c30fba6326 100644 --- a/litellm/llms/gemini/files/transformation.py +++ b/litellm/llms/gemini/files/transformation.py @@ -86,7 +86,7 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): if not final_api_key: raise ValueError("api_key is required") - url = "{}/{}?key={}".format(api_base, endpoint, final_api_key) + url = "{}/{}".format(api_base, endpoint) return url def get_supported_openai_params( @@ -231,9 +231,9 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig): ) api_base = api_base.rstrip("/") - url = f"{api_base}/v1beta/{file_part}?key={api_key}" + url = f"{api_base}/v1beta/{file_part}" - # Return empty params dict - API key is already in URL, no query params needed + # API key is passed via x-goog-api-key header (set in validate_environment) return url, {} def _normalize_gemini_file_id(self, file_id: str) -> str: diff --git a/litellm/llms/gemini/interactions/transformation.py b/litellm/llms/gemini/interactions/transformation.py index 772530342e..c34da83cb8 100644 --- a/litellm/llms/gemini/interactions/transformation.py +++ b/litellm/llms/gemini/interactions/transformation.py @@ -75,9 +75,13 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): model: str, litellm_params: Optional[GenericLiteLLMParams], ) -> dict: - """Google AI Studio uses API key in query params, not headers.""" + """Google AI Studio uses x-goog-api-key header for authentication.""" headers = headers or {} headers["Content-Type"] = "application/json" + if litellm_params: + api_key = GeminiModelInfo.get_api_key(litellm_params.get("api_key")) + if api_key: + headers["x-goog-api-key"] = api_key return headers def get_complete_url( @@ -98,11 +102,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): "Google API key is required. Set GOOGLE_API_KEY or GEMINI_API_KEY environment variable." ) - query_params = f"key={api_key}" if stream: - query_params += "&alt=sse" + return f"{api_base}/{self.api_version}/interactions?alt=sse" - return f"{api_base}/{self.api_version}/interactions?{query_params}" + return f"{api_base}/{self.api_version}/interactions" def transform_request( self, @@ -200,11 +203,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): ) -> Tuple[str, Dict]: """GET /{api_version}/interactions/{interaction_id}""" resolved_api_base = GeminiModelInfo.get_api_base(api_base) - api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) - if not api_key: + if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}", + f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}", {}, ) @@ -234,11 +236,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): ) -> Tuple[str, Dict]: """DELETE /{api_version}/interactions/{interaction_id}""" resolved_api_base = GeminiModelInfo.get_api_base(api_base) - api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) - if not api_key: + if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}?key={api_key}", + f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}", {}, ) @@ -265,11 +266,10 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig): ) -> Tuple[str, Dict]: """POST /{api_version}/interactions/{interaction_id}:cancel (if supported)""" resolved_api_base = GeminiModelInfo.get_api_base(api_base) - api_key = GeminiModelInfo.get_api_key(litellm_params.api_key) - if not api_key: + if not GeminiModelInfo.get_api_key(litellm_params.api_key): raise ValueError("Google API key is required") return ( - f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel?key={api_key}", + f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel", {}, ) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 2bb7bcd8b4..4fac5aceb5 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -85,6 +85,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): raise ValueError("api_key is required for Gemini API calls") api_base = api_base.replace("https://", "wss://") api_base = api_base.replace("http://", "ws://") + # WebSocket connections do not support custom HTTP headers in all clients, + # so the API key must remain as a query parameter here. This is an accepted + # limitation; httpx is not used for WebSocket so MaskedHTTPStatusError + # already covers the main leak vector. return f"{api_base}/ws/google.ai.generativelanguage.v1beta.GenerativeService.BidiGenerateContent?key={api_key}" def map_model_turn_event( diff --git a/litellm/llms/gemini/vector_stores/transformation.py b/litellm/llms/gemini/vector_stores/transformation.py index 11fd77aeca..e6e8369643 100644 --- a/litellm/llms/gemini/vector_stores/transformation.py +++ b/litellm/llms/gemini/vector_stores/transformation.py @@ -48,7 +48,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): def get_auth_credentials( self, litellm_params: dict ) -> BaseVectorStoreAuthCredentials: - """Gemini uses API key in query params, not headers.""" + """Gemini uses x-goog-api-key header for authentication.""" return {} def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: @@ -79,6 +79,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): api_key = litellm_params.get("api_key") or get_api_key_from_env() if api_key: self._cached_api_key = api_key + headers["x-goog-api-key"] = api_key return headers @@ -133,13 +134,10 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): if model and model.startswith("gemini/"): model = model.replace("gemini/", "") - # Get API key - Gemini requires it as a query parameter api_key = litellm_params.get("api_key") or GeminiModelInfo.get_api_key() if not api_key: raise ValueError("GEMINI_API_KEY or GOOGLE_API_KEY is required") - - # Build the URL for generateContent with API key - url = f"{api_base}/models/{model}:generateContent?key={api_key}" + url = f"{api_base}/models/{model}:generateContent" # Build file_search tool configuration (using snake_case as per Gemini docs) file_search_config: Dict[str, Any] = { @@ -286,10 +284,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig): """ url = f"{api_base}/fileSearchStores" - # Append API key as query parameter (required by Gemini) - api_key = self._cached_api_key or get_api_key_from_env() - if api_key: - url = f"{url}?key={api_key}" + # API key is passed via x-goog-api-key header (set in validate_environment) request_body: Dict[str, Any] = {} diff --git a/litellm/llms/openai/realtime/handler.py b/litellm/llms/openai/realtime/handler.py index c04857fc25..5ca0a3186f 100644 --- a/litellm/llms/openai/realtime/handler.py +++ b/litellm/llms/openai/realtime/handler.py @@ -6,6 +6,7 @@ This requires websockets, and is currently only supported on LiteLLM Proxy. from typing import Any, Optional, cast +from litellm._logging import _redact_string from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES from litellm.types.realtime import RealtimeQueryParams @@ -148,11 +149,11 @@ class OpenAIRealtime(OpenAIChatCompletion): await realtime_streaming.bidirectional_forward() except websockets.exceptions.InvalidStatusCode as e: # type: ignore - await websocket.close(code=e.status_code, reason=str(e)) + await websocket.close(code=e.status_code, reason=_redact_string(str(e))) except Exception as e: try: await websocket.close( - code=1011, reason=f"Internal server error: {str(e)}" + code=1011, reason=_redact_string(f"Internal server error: {str(e)}") ) except RuntimeError as close_error: if "already completed" in str(close_error) or "websocket.close" in str( diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 5895a91f3a..43e77f4fb7 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -337,8 +337,13 @@ def _get_gemini_url( mode: all_gemini_url_modes, model: str, stream: Optional[bool], - gemini_api_key: Optional[str], ) -> Tuple[str, str]: + """Build the Gemini API URL for the given mode. + + The API key is NOT included in the URL. Callers must pass it via the + ``x-goog-api-key`` header instead to avoid leaking credentials in + error tracebacks. + """ from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -352,27 +357,27 @@ def _get_gemini_url( endpoint = "generateContent" if stream is True: endpoint = "streamGenerateContent" - url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}&alt=sse".format( - api_version, _gemini_model_name, endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/{}/{}:{}?alt=sse".format( + api_version, _gemini_model_name, endpoint ) else: - url = "https://generativelanguage.googleapis.com/{}/{}:{}?key={}".format( - api_version, _gemini_model_name, endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/{}/{}:{}".format( + api_version, _gemini_model_name, endpoint ) elif mode == "embedding": endpoint = "embedContent" - url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format( - _gemini_model_name, endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format( + _gemini_model_name, endpoint ) elif mode == "batch_embedding": endpoint = "batchEmbedContents" - url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format( - _gemini_model_name, endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format( + _gemini_model_name, endpoint ) elif mode == "count_tokens": endpoint = "countTokens" - url = "https://generativelanguage.googleapis.com/v1beta/{}:{}?key={}".format( - _gemini_model_name, endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/v1beta/{}:{}".format( + _gemini_model_name, endpoint ) elif mode == "image_generation": raise ValueError( diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index b677cf3b1e..0b872ddd8f 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -62,10 +62,10 @@ class ContextCachingEndpoints(VertexBase): token, url """ if custom_llm_provider == "gemini": - auth_header = None + auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment] endpoint = "cachedContents" - url = "https://generativelanguage.googleapis.com/v1beta/{}?key={}".format( - endpoint, gemini_api_key + url = "https://generativelanguage.googleapis.com/v1beta/{}".format( + endpoint ) elif custom_llm_provider == "vertex_ai": auth_header = vertex_auth_header @@ -353,7 +353,9 @@ class ContextCachingEndpoints(VertexBase): headers = { "Content-Type": "application/json", } - if token is not None: + if isinstance(token, dict): + headers.update(token) + elif token is not None: headers["Authorization"] = f"Bearer {token}" if extra_headers is not None: headers.update(extra_headers) @@ -501,7 +503,9 @@ class ContextCachingEndpoints(VertexBase): headers = { "Content-Type": "application/json", } - if token is not None: + if isinstance(token, dict): + headers.update(token) + elif token is not None: headers["Authorization"] = f"Bearer {token}" if extra_headers is not None: headers.update(extra_headers) diff --git a/litellm/llms/vertex_ai/vertex_llm_base.py b/litellm/llms/vertex_ai/vertex_llm_base.py index 68d8f0d046..430cc27adc 100644 --- a/litellm/llms/vertex_ai/vertex_llm_base.py +++ b/litellm/llms/vertex_ai/vertex_llm_base.py @@ -473,9 +473,8 @@ class VertexBase: mode=mode, model=model, stream=stream, - gemini_api_key=gemini_api_key, ) - auth_header = None # this field is not used for gemin + auth_header = {"x-goog-api-key": gemini_api_key} # type: ignore[assignment] else: vertex_location = self.get_vertex_region( vertex_region=vertex_location, diff --git a/litellm/main.py b/litellm/main.py index ddd37b4753..22dffc0bbe 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -40,6 +40,7 @@ from typing import ( get_args, ) +from litellm._logging import _redact_string from litellm._uuid import uuid if TYPE_CHECKING: @@ -7244,7 +7245,7 @@ async def ahealth_check( f"Mode {mode} not supported. See modes here: https://docs.litellm.ai/docs/proxy/health" ) except Exception as e: - stack_trace = traceback.format_exc() + stack_trace = _redact_string(traceback.format_exc()) if isinstance(stack_trace, str): stack_trace = stack_trace[:1000] diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 037f913ad0..c4717ad9cf 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -22,7 +22,7 @@ from fastapi import HTTPException, Request, status from fastapi.responses import JSONResponse, Response, StreamingResponse import litellm -from litellm._logging import verbose_proxy_logger +from litellm._logging import _redact_string, verbose_proxy_logger from litellm._uuid import uuid from litellm.constants import ( DD_TRACER_STREAMING_CHUNK_YIELD_RESOURCE, @@ -1785,7 +1785,7 @@ class ProxyBaseLLMRequestProcessing: if isinstance(e, HTTPException): raise e - error_traceback = traceback.format_exc() + error_traceback = _redact_string(traceback.format_exc()) error_msg = f"{str(e)}\n\n{error_traceback}" proxy_exception = ProxyException( message=getattr(e, "message", error_msg), diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index a6f81986a6..ac3bf6d498 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -77,7 +77,7 @@ from litellm import ( ModelResponseStream, Router, ) -from litellm._logging import verbose_proxy_logger +from litellm._logging import _redact_string, verbose_proxy_logger from litellm._service_logger import ServiceLogging, ServiceTypes from litellm.caching.caching import DualCache, RedisCache from litellm.caching.dual_cache import LimitedSizeOrderedDict @@ -155,7 +155,7 @@ def print_verbose(print_statement): verbose_proxy_logger.debug("{}\n{}".format(print_statement, traceback.format_exc())) if litellm.set_verbose: - print(f"LiteLLM Proxy: {print_statement}") # noqa + print(f"LiteLLM Proxy: {_redact_string(str(print_statement))}") # noqa def _get_email_logger_class(): @@ -1721,6 +1721,7 @@ class ProxyLogging: error_message = str(original_exception) if isinstance(traceback_str, str): error_message += traceback_str[:1000] + error_message = _redact_string(error_message) asyncio.create_task( self.alerting_handler( message=f"DB read/write call failed: {error_message}", @@ -1791,7 +1792,7 @@ class ProxyLogging: asyncio.create_task( self.alerting_handler( - message=f"LLM API call failed: `{exception_str}`", + message=_redact_string(f"LLM API call failed: `{exception_str}`"), level="High", alert_type=AlertType.llm_exceptions, request_data=request_data, diff --git a/litellm/rag/ingestion/gemini_ingestion.py b/litellm/rag/ingestion/gemini_ingestion.py index 96495b9f3f..af6eb928e2 100644 --- a/litellm/rag/ingestion/gemini_ingestion.py +++ b/litellm/rag/ingestion/gemini_ingestion.py @@ -143,7 +143,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): Returns: Store name (format: fileSearchStores/xxxxxxx) """ - url = f"{base_url}/fileSearchStores?key={api_key}" + url = f"{base_url}/fileSearchStores" request_body = {"displayName": display_name} @@ -154,7 +154,10 @@ class GeminiRAGIngestion(BaseRAGIngestion): response = await client.post( url, json=request_body, - headers={"Content-Type": "application/json"}, + headers={ + "Content-Type": "application/json", + "x-goog-api-key": api_key, + }, ) if response.status_code != 200: @@ -228,7 +231,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): # base_url is like: https://generativelanguage.googleapis.com/v1beta # We need: https://generativelanguage.googleapis.com/upload/v1beta/{store_id}:uploadToFileSearchStore api_base = base_url.replace("/v1beta", "") # Get base without version - url = f"{api_base}/upload/v1beta/{vector_store_id}:uploadToFileSearchStore?key={api_key}" + url = f"{api_base}/upload/v1beta/{vector_store_id}:uploadToFileSearchStore" # Build request body with chunking config and metadata if provided request_body: Dict[str, Any] = {"displayName": filename} @@ -263,6 +266,7 @@ class GeminiRAGIngestion(BaseRAGIngestion): "X-Goog-Upload-Header-Content-Length": str(file_size), "X-Goog-Upload-Header-Content-Type": content_type, "Content-Type": "application/json", + "x-goog-api-key": api_key, } verbose_logger.debug(f"Initiating resumable upload: {url}") diff --git a/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py index d814f8ec97..8ca3a4d149 100644 --- a/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py +++ b/tests/litellm/llms/vertex_ai/test_gemini_batch_embeddings.py @@ -310,7 +310,7 @@ def test_gemini_multimodal_embedding_e2e(): ) as mock_get_token: mock_get_token.return_value = ( {"x-goog-api-key": "test-key"}, - "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2-preview:embedContent?key=test-key" + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2-preview:embedContent" ) mock_response = MagicMock() diff --git a/tests/test_litellm/google_genai/test_google_genai_adapter.py b/tests/test_litellm/google_genai/test_google_genai_adapter.py index 05b2209837..2af6867f09 100644 --- a/tests/test_litellm/google_genai/test_google_genai_adapter.py +++ b/tests/test_litellm/google_genai/test_google_genai_adapter.py @@ -1127,80 +1127,58 @@ async def test_google_generate_content_with_openai(): passed_fields = passed_fields - set(GenericLiteLLMParams.model_fields.keys()) # extra_headers is now explicitly passed through for providers that need custom headers assert passed_fields == set(["model", "messages", "extra_headers"]), f"Expected model, messages, and extra_headers to be passed through, got {passed_fields}" -@pytest.mark.asyncio -async def test_agenerate_content_x_goog_api_key_header(): +def test_validate_environment_sets_x_goog_api_key(): """ - Test that agenerate_content passes x-goog-api-key header correctly. - - This test verifies that when calling agenerate_content with a Google GenAI model, - the HTTP request includes the x-goog-api-key header with the correct API key value. + Test that VertexGeminiConfig.validate_environment correctly merges an + x-goog-api-key dict into the request headers. + + This is the mechanism by which Google AI Studio (Gemini) requests get + authenticated via header instead of a query-string ?key= parameter. """ - import os - import unittest.mock + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) - import httpx - test_api_key = "test-gemini-api-key-123" - - # Mock environment to ensure we use our test API key - with unittest.mock.patch.dict(os.environ, {"GEMINI_API_KEY": test_api_key}, clear=False): - # Mock the AsyncHTTPHandler's post method to capture headers - with unittest.mock.patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", new_callable=unittest.mock.AsyncMock) as mock_post: - # Mock a successful response - mock_response = unittest.mock.MagicMock() - mock_response.json.return_value = { - "candidates": [ - { - "content": { - "parts": [{"text": "Hello! How can I help you today?"}], - "role": "model" - }, - "finishReason": "STOP", - "index": 0 - } - ], - "usageMetadata": { - "promptTokenCount": 5, - "candidatesTokenCount": 10, - "totalTokenCount": 15 - } - } - mock_response.status_code = 200 - mock_response.headers = {} - mock_post.return_value = mock_response - - # Call agenerate_content with Google AI Studio model - try: - response = await agenerate_content( - model="gemini/gemini-1.5-flash", - contents=[ - {"role": "user", "parts": [{"text": "Hello, world!"}]} - ], - api_key=test_api_key - ) - except Exception: - # Ignore any response processing errors, we just want to check the headers - pass - - # Verify that AsyncHTTPHandler.post was called - mock_post.assert_called_once() - - # Get the arguments passed to the post call - call_args, call_kwargs = mock_post.call_args - - # Verify that headers contain x-goog-api-key - headers = call_kwargs.get("headers", {}) - assert "x-goog-api-key" in headers, f"x-goog-api-key header not found in headers: {list(headers.keys())}" - - # Verify the API key is set (could be our test key or from api_key parameter) - api_key_value = headers["x-goog-api-key"] - assert api_key_value == test_api_key, f"Expected x-goog-api-key to be {test_api_key}, got {api_key_value}" - - # Verify other expected headers - assert headers.get("Content-Type") == "application/json", f"Expected Content-Type application/json, got {headers.get('Content-Type')}" - print(f"✓ Test passed: x-goog-api-key header correctly set to {api_key_value}") - print(f"✓ All headers: {list(headers.keys())}") + # Simulate what _get_token_and_url returns for Gemini: a dict auth_header + auth_header_dict = {"x-goog-api-key": test_api_key} + + headers = VertexGeminiConfig().validate_environment( + api_key=auth_header_dict, + headers=None, + model="gemini-2.5-flash", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert "x-goog-api-key" in headers, f"x-goog-api-key not in headers: {headers}" + assert headers["x-goog-api-key"] == test_api_key + assert headers["Content-Type"] == "application/json" + + +def test_get_gemini_url_excludes_api_key(): + """ + Verify that _get_gemini_url never embeds the API key in the URL. + + API keys in URLs leak through httpx error tracebacks. The key must be + sent via the x-goog-api-key header instead. + """ + from litellm.llms.vertex_ai.common_utils import _get_gemini_url + + for mode in ("chat", "embedding", "batch_embedding", "count_tokens"): + url, _ = _get_gemini_url( + mode=mode, + model="gemini-2.5-flash", + stream=False, + ) + assert "key=" not in url, f"API key found in URL for mode={mode}: {url}" + + # Streaming chat should only have ?alt=sse + url, _ = _get_gemini_url(mode="chat", model="gemini-2.5-flash", stream=True) + assert "key=" not in url, f"API key found in streaming URL: {url}" + assert "alt=sse" in url, f"Missing alt=sse in streaming URL: {url}" def test_inline_data_base64_image_transformation(): diff --git a/tests/test_litellm/llms/gemini/files/test_gemini_files_transformation.py b/tests/test_litellm/llms/gemini/files/test_gemini_files_transformation.py index 6cc97cd95e..21c036254e 100644 --- a/tests/test_litellm/llms/gemini/files/test_gemini_files_transformation.py +++ b/tests/test_litellm/llms/gemini/files/test_gemini_files_transformation.py @@ -37,12 +37,12 @@ class TestGoogleAIStudioFilesTransformation: litellm_params=litellm_params, ) - # Verify URL is constructed exactly as required: - # https://generativelanguage.googleapis.com/v1beta/files/{file_id}?key=API_KEY + # API key is passed via x-goog-api-key header, not in URL assert ( url - == "https://generativelanguage.googleapis.com/v1beta/files/test123?key=test-api-key" + == "https://generativelanguage.googleapis.com/v1beta/files/test123" ) + assert "key=" not in url # CRITICAL: params should be empty dict, not contain Content-Type or any other params # These would be incorrectly interpreted as query parameters @@ -64,12 +64,12 @@ class TestGoogleAIStudioFilesTransformation: litellm_params=litellm_params, ) - # Verify URL is constructed exactly as required: - # https://generativelanguage.googleapis.com/v1beta/files/{file_id}?key=API_KEY + # API key is passed via x-goog-api-key header, not in URL assert ( url - == "https://generativelanguage.googleapis.com/v1beta/files/test123?key=test-api-key" + == "https://generativelanguage.googleapis.com/v1beta/files/test123" ) + assert "key=" not in url # CRITICAL: params should be empty dict assert params == {}, f"Expected empty params dict, got: {params}" @@ -79,11 +79,10 @@ class TestGoogleAIStudioFilesTransformation: def test_transform_retrieve_file_request_with_raw_id_only(self): """ - Regression guard for the exact retrieval URL format. + Regression guard: API key must NOT appear in the URL. - If someone changes the method and stops producing: - https://generativelanguage.googleapis.com/v1beta/files/{file_id}?key=API_KEY - this test should fail. + The key is sent via x-goog-api-key header to prevent leaking + credentials in httpx error tracebacks. """ file_id = "cctqueckiggb" litellm_params = {"api_key": "test-api-key"} @@ -96,8 +95,9 @@ class TestGoogleAIStudioFilesTransformation: assert ( url - == "https://generativelanguage.googleapis.com/v1beta/files/cctqueckiggb?key=test-api-key" + == "https://generativelanguage.googleapis.com/v1beta/files/cctqueckiggb" ) + assert "key=" not in url assert params == {} @patch.dict("os.environ", {}, clear=True) @@ -285,10 +285,10 @@ class TestGoogleAIStudioFilesTransformation: litellm_params={}, ) - # Verify URL structure + # Verify URL structure - API key must NOT be in URL assert api_base in url assert "upload/v1beta/files" in url - assert f"key={api_key}" in url + assert "key=" not in url def test_transform_delete_file_request_with_full_uri(self): """Test delete file request transformation with full URI"""