Merge branch 'litellm_internal_staging' into litellm_token_verification_query_opt

.github/workflows/create-release-branch.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable) — branch will be named release/<tag>"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted) — branch will be named release/<tag>"
         required: true
         type: string
       commit_hash:
@@ -14,7 +14,7 @@ on:
   workflow_call:
     inputs:
       tag:
-        description: "Release tag"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -40,8 +40,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. v1.83.0-stable)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -30,8 +30,8 @@ jobs:
             echo "::error::commit_hash must be a full 40-character commit SHA"
             exit 1
           fi
-          if ! echo "${TAG}" | grep -qE '^v[0-9]+\.[0-9]+\.[0-9]+'; then
-            echo "::error::tag must start with vX.Y.Z"
+          if ! echo "${TAG}" | grep -qE '^v?[0-9]+\.[0-9]+\.[0-9]+'; then
+            echo "::error::tag must start with X.Y.Z (optional leading v), e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, or v1.83.10-stable"
             exit 1
           fi
 
@@ -45,6 +45,11 @@ jobs:
             const tag = process.env.TAG;
             const commitHash = process.env.COMMIT_HASH;
 
+            // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
+            // are stable maintenance releases, not pre-releases.
+            const isPrerelease = /(?:rc|nightly|alpha|beta|\.dev)/i.test(tag);
+
             const cosignSection = [
               `## Verify Docker Image Signature`,
               ``,
@@ -89,7 +94,7 @@ jobs:
                 target_commitish: commitHash,
                 name: tag,
                 owner: context.repo.owner,
-                prerelease: false,
+                prerelease: isPrerelease,
                 repo: context.repo.repo,
                 tag_name: tag,
               });

.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

litellm-js/proxy/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

litellm-js/spend-logs/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429120000_search_tools_on_object_permission/migration.sql

@@ -0,0 +1,2 @@
+-- Search tool allowlists live on LiteLLM_ObjectPermissionTable (with agents, MCP, vector stores).
+ALTER TABLE "LiteLLM_ObjectPermissionTable" ADD COLUMN IF NOT EXISTS "search_tools" TEXT[] DEFAULT ARRAY[]::TEXT[];

litellm-proxy-extras/litellm_proxy_extras/migrations/20260429161855_workflow_runs_tables/migration.sql

@@ -0,0 +1,75 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowRun" (
+    "run_id" TEXT NOT NULL,
+    "session_id" TEXT NOT NULL,
+    "workflow_type" TEXT NOT NULL,
+    "status" TEXT NOT NULL DEFAULT 'pending',
+    "created_by" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "input" JSONB,
+    "output" JSONB,
+    "metadata" JSONB,
+
+    CONSTRAINT "LiteLLM_WorkflowRun_pkey" PRIMARY KEY ("run_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowEvent" (
+    "event_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "event_type" TEXT NOT NULL,
+    "step_name" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "data" JSONB,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowEvent_pkey" PRIMARY KEY ("event_id")
+);
+
+-- CreateTable
+CREATE TABLE "LiteLLM_WorkflowMessage" (
+    "message_id" TEXT NOT NULL,
+    "run_id" TEXT NOT NULL,
+    "role" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "sequence_number" INTEGER NOT NULL,
+    "session_id" TEXT,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+
+    CONSTRAINT "LiteLLM_WorkflowMessage_pkey" PRIMARY KEY ("message_id")
+);
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowRun_session_id_key" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_workflow_type_status_idx" ON "LiteLLM_WorkflowRun"("workflow_type", "status");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_session_id_idx" ON "LiteLLM_WorkflowRun"("session_id");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_at_idx" ON "LiteLLM_WorkflowRun"("created_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowRun_created_by_idx" ON "LiteLLM_WorkflowRun"("created_by");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowEvent_run_id_idx" ON "LiteLLM_WorkflowEvent"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowEvent_run_id_sequence_number_key" ON "LiteLLM_WorkflowEvent"("run_id", "sequence_number");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_WorkflowMessage_run_id_idx" ON "LiteLLM_WorkflowMessage"("run_id");
+
+-- CreateIndex
+CREATE UNIQUE INDEX "LiteLLM_WorkflowMessage_run_id_sequence_number_key" ON "LiteLLM_WorkflowMessage"("run_id", "sequence_number");
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowEvent" ADD CONSTRAINT "LiteLLM_WorkflowEvent_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+
+-- AddForeignKey
+ALTER TABLE "LiteLLM_WorkflowMessage" ADD CONSTRAINT "LiteLLM_WorkflowMessage_run_id_fkey" FOREIGN KEY ("run_id") REFERENCES "LiteLLM_WorkflowRun"("run_id") ON DELETE RESTRICT ON UPDATE CASCADE;
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
   models                String[]       @default([])
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
+  search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -1290,3 +1291,80 @@ model LiteLLM_AdaptiveRouterSession {
   @@id([session_id, router_name, model_name])
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
+
+// ---------------------------------------------------------------------------
+// Workflow Run Tracking
+//
+// Generic durable state tracking for any agent or automated workflow.
+// Design: three tables — run (header + materialized status), event (append-only
+// source of truth for state transitions), message (conversation inbox/outbox).
+//
+// Usage:
+//   - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
+//   - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
+//   - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
+//     the proxy — all spend logs for this run are automatically tagged.
+// ---------------------------------------------------------------------------
+
+// One instance of work being done. `status` is a materialized cache of the
+// latest event; the event log is the authoritative source of truth.
+model LiteLLM_WorkflowRun {
+  run_id        String   @id @default(uuid())
+  session_id    String   @unique @default(uuid())
+  workflow_type String
+  status        String   @default("pending")
+  created_by    String?  // user_id of the key that created this run; null = created by master key
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+  input         Json?
+  output        Json?
+  metadata      Json?
+
+  events   LiteLLM_WorkflowEvent[]
+  messages LiteLLM_WorkflowMessage[]
+
+  @@index([workflow_type, status])
+  @@index([session_id])
+  @@index([created_at])
+  @@index([created_by])
+}
+
+// Append-only log of state transitions. Never mutate rows here.
+// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
+// Status auto-update rules (applied by the append endpoint):
+//   step.started   → run.status = running
+//   step.failed    → run.status = failed
+//   hook.waiting   → run.status = paused
+//   hook.received  → run.status = running
+model LiteLLM_WorkflowEvent {
+  event_id        String   @id @default(uuid())
+  run_id          String
+  event_type      String
+  step_name       String
+  sequence_number Int
+  data            Json?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}
+
+// Conversation inbox/outbox — full message content, separate from the durable
+// event log. Spend logs truncate messages; this table stores them in full.
+// `session_id` here is the Claude --resume session ID (or similar).
+model LiteLLM_WorkflowMessage {
+  message_id      String   @id @default(uuid())
+  run_id          String
+  role            String
+  content         String
+  sequence_number Int
+  session_id      String?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}

litellm/batches/main.py

@@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config(
         )
     else:
         raise litellm.exceptions.BadRequestError(
-            message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
-                custom_llm_provider
-            ),
+            message=(
+                "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. "
+                "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
+                "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
+            ).format(custom_llm_provider),
             model="n/a",
             llm_provider=custom_llm_provider,
             response=httpx.Response(
                 status_code=400,
                 content="Unsupported provider",
-                request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+                request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),  # type: ignore
             ),
         )
     return response

litellm/constants.py

@@ -224,6 +224,16 @@ AIOHTTP_CONNECTOR_LIMIT_PER_HOST = int(
 )
 AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120))
 AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300))
+# TCP keep-alive (SO_KEEPALIVE) — opt-in. Required when running behind NAT/LBs
+# whose idle timeout is shorter than provider response timeouts (e.g. AWS NAT
+# Gateway: 350s vs OpenAI/Azure: 600s). Without this, the kernel sends nothing
+# during a long provider call and the NAT reaps the flow before the response
+# arrives. Enabling SO_KEEPALIVE makes the kernel emit TCP probes that reset
+# the NAT idle timer.
+AIOHTTP_SO_KEEPALIVE = os.getenv("AIOHTTP_SO_KEEPALIVE", "False").lower() == "true"
+AIOHTTP_TCP_KEEPIDLE = int(os.getenv("AIOHTTP_TCP_KEEPIDLE", 60))
+AIOHTTP_TCP_KEEPINTVL = int(os.getenv("AIOHTTP_TCP_KEEPINTVL", 30))
+AIOHTTP_TCP_KEEPCNT = int(os.getenv("AIOHTTP_TCP_KEEPCNT", 5))
 # enable_cleanup_closed is only needed for Python versions with the SSL leak bug
 # Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960)
 # Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78
@@ -1383,6 +1393,10 @@ except (ValueError, TypeError):
 LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
 LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
 LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
+# Stable identifier substituted in place of the master key on UserAPIKeyAuth
+# objects so the master key (or its hash) never propagates to spend logs,
+# Prometheus metrics, audit trails, or any other downstream consumer.
+LITELLM_PROXY_MASTER_KEY_ALIAS = "litellm_proxy_master_key"
 
 # Key Rotation Constants
 LITELLM_KEY_ROTATION_ENABLED = os.getenv("LITELLM_KEY_ROTATION_ENABLED", "false")

litellm/google_genai/streaming_iterator.py

@@ -1,6 +1,6 @@
 import asyncio
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, List, Optional
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
 
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.proxy.pass_through_endpoints.success_handler import (
@@ -29,12 +29,14 @@ class BaseGoogleGenAIGenerateContentStreamingIterator:
         litellm_logging_obj: LiteLLMLoggingObj,
         request_body: dict,
         model: str,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         self.litellm_logging_obj = litellm_logging_obj
         self.request_body = request_body
         self.start_time = datetime.now()
         self.collected_chunks: List[bytes] = []
         self.model = model
+        self._hidden_params: Dict[str, Any] = hidden_params or {}
 
     async def _handle_async_streaming_logging(
         self,
@@ -76,11 +78,13 @@ class GoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model
@@ -130,11 +134,13 @@ class AsyncGoogleGenAIGenerateContentStreamingIterator(
         litellm_metadata: dict,
         custom_llm_provider: str,
         request_body: Optional[dict] = None,
+        hidden_params: Optional[Dict[str, Any]] = None,
     ):
         super().__init__(
             litellm_logging_obj=logging_obj,
             request_body=request_body or {},
             model=model,
+            hidden_params=hidden_params,
         )
         self.response = response
         self.model = model

litellm/integrations/prompt_management_base.py

@@ -87,9 +87,7 @@ class PromptManagementBase(ABC):
         try:
             messages = compiled_prompt_client["prompt_template"] + client_messages
         except Exception as e:
-            raise ValueError(
-                f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
-            )
+            raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
 
         compiled_prompt_client["completed_messages"] = messages
         return compiled_prompt_client
@@ -116,9 +114,7 @@ class PromptManagementBase(ABC):
         try:
             messages = compiled_prompt_client["prompt_template"] + client_messages
         except Exception as e:
-            raise ValueError(
-                f"Error compiling prompt: {e}. Prompt id={prompt_id}, prompt_variables={prompt_variables}, client_messages={client_messages}, dynamic_callback_params={dynamic_callback_params}"
-            )
+            raise ValueError(f"Error compiling prompt: {e}. Prompt id={prompt_id}")
 
         compiled_prompt_client["completed_messages"] = messages
         return compiled_prompt_client

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -1,4 +1,5 @@
 from typing import Optional, Tuple
+from urllib.parse import urlparse
 
 import litellm
 from litellm.constants import REPLICATE_MODEL_NAME_WITH_ID_LENGTH
@@ -8,6 +9,43 @@ from litellm.secret_managers.main import get_secret, get_secret_str
 from ..types.router import LiteLLM_Params
 
 
+def _endpoint_matches_api_base(endpoint: str, api_base: str) -> bool:
+    """
+    Match a registered openai-compatible endpoint against a caller-supplied
+    ``api_base`` using parsed-URL semantics, not unanchored substring search.
+
+    Both inputs may be a bare hostname (``api.perplexity.ai``), host+path
+    (``api.deepinfra.com/v1/openai``), or a full URL
+    (``https://api.cerebras.ai/v1``). Hostnames must match exactly
+    (case-insensitive); if the registered endpoint has a non-trivial path,
+    the api_base path must start with it on a segment boundary.
+
+    The naive ``endpoint in api_base`` shape lets a caller pass
+    ``https://attacker.com/api.groq.com/openai/v1`` to coerce the proxy
+    into reading the server's GROQ_API_KEY from the environment and
+    forwarding it to the attacker's host as a Bearer credential.
+    """
+
+    def _parse(value: str):
+        # Ensure urlparse sees a scheme so it populates hostname / path.
+        normalized = value if "://" in value else f"https://{value}"
+        return urlparse(normalized)
+
+    parsed_endpoint = _parse(endpoint)
+    parsed_url = _parse(api_base)
+
+    endpoint_host = (parsed_endpoint.hostname or "").lower()
+    url_host = (parsed_url.hostname or "").lower()
+    if not endpoint_host or endpoint_host != url_host:
+        return False
+
+    endpoint_path = parsed_endpoint.path.rstrip("/")
+    if not endpoint_path:
+        return True
+    url_path = parsed_url.path.rstrip("/")
+    return url_path == endpoint_path or url_path.startswith(endpoint_path + "/")
+
+
 def _is_non_openai_azure_model(model: str) -> bool:
     try:
         model_name = model.split("/", 1)[1]
@@ -210,7 +248,7 @@ def get_llm_provider(  # noqa: PLR0915
         # check if api base is a known openai compatible endpoint
         if api_base:
             for endpoint in litellm.openai_compatible_endpoints:
-                if endpoint in api_base:
+                if _endpoint_matches_api_base(endpoint, api_base):
                     if endpoint == "api.perplexity.ai":
                         custom_llm_provider = "perplexity"
                         dynamic_api_key = get_secret_str("PERPLEXITYAI_API_KEY")
@@ -348,6 +386,7 @@ def get_llm_provider(  # noqa: PLR0915
             or "ft:gpt-3.5-turbo" in model
             or "ft:gpt-4" in model  # catches ft:gpt-4-0613, ft:gpt-4o
             or model in litellm.openai_image_generation_models
+            or model.startswith("gpt-image")
             or model in litellm.openai_video_generation_models
         ):
             custom_llm_provider = "openai"

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -982,9 +982,9 @@ class CostCalculatorUtils:
                 image_response=completion_response,
             )
         elif custom_llm_provider == litellm.LlmProviders.OPENAI.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )
@@ -1004,9 +1004,9 @@ class CostCalculatorUtils:
                 optional_params=optional_params,
             )
         elif custom_llm_provider == litellm.LlmProviders.AZURE.value:
-            # Check if this is a gpt-image model (token-based pricing)
+            # gpt-image models use token-based pricing.
             model_lower = model.lower()
-            if "gpt-image-1" in model_lower:
+            if "gpt-image" in model_lower:
                 from litellm.llms.openai.image_generation.cost_calculator import (
                     cost_calculator as openai_gpt_image_cost_calculator,
                 )

litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py

@@ -824,8 +824,6 @@ def convert_to_model_response_object(  # noqa: PLR0915
             stream=stream,
             start_time=start_time,
             end_time=end_time,
-            hidden_params=hidden_params,
-            _response_headers=_response_headers,
             convert_tool_call_to_json_mode=convert_tool_call_to_json_mode,
         )
         raise Exception(

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1661,6 +1661,20 @@ def _sanitize_anthropic_tool_use_id(tool_use_id: str) -> str:
     return sanitized
 
 
+_ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES = {"application/pdf", "text/plain"}
+
+
+def _is_anthropic_document_data_uri(url: str) -> bool:
+    # Anthropic's base64 document source accepts only application/pdf and
+    # text/plain (see select_anthropic_content_block_type_for_file). Routing
+    # other mimes here would produce a document block the API rejects, so we
+    # leave them on the image code path.
+    match = re.match(r"data:([^;,]+)", url)
+    if not match:
+        return False
+    return match.group(1) in _ANTHROPIC_DOCUMENT_BASE64_MEDIA_TYPES
+
+
 def convert_to_anthropic_tool_result(
     message: Union[ChatCompletionToolMessage, ChatCompletionFunctionMessage],
     force_base64: bool = False,
@@ -1698,14 +1712,24 @@ def convert_to_anthropic_tool_result(
     """
     anthropic_content: Union[
         str,
-        List[Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]],
+        List[
+            Union[
+                AnthropicMessagesToolResultContent,
+                AnthropicMessagesImageParam,
+                AnthropicMessagesDocumentParam,
+            ]
+        ],
     ] = ""
     if isinstance(message["content"], str):
         anthropic_content = message["content"]
     elif isinstance(message["content"], List):
         content_list = message["content"]
         anthropic_content_list: List[
-            Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
+            Union[
+                AnthropicMessagesToolResultContent,
+                AnthropicMessagesImageParam,
+                AnthropicMessagesDocumentParam,
+            ]
         ] = []
         for content in content_list:
             if content["type"] == "text":
@@ -1720,21 +1744,62 @@ def convert_to_anthropic_tool_result(
                     text_content["cache_control"] = cache_control_value
                 anthropic_content_list.append(text_content)
             elif content["type"] == "image_url":
+                image_url_value = content["image_url"]
                 format = (
-                    content["image_url"].get("format")
-                    if isinstance(content["image_url"], dict)
+                    image_url_value.get("format")
+                    if isinstance(image_url_value, dict)
                     else None
                 )
-                _anthropic_image_param = create_anthropic_image_param(
-                    content["image_url"], format=format, is_bedrock_invoke=force_base64
+                url_str = (
+                    image_url_value.get("url")
+                    if isinstance(image_url_value, dict)
+                    else image_url_value
                 )
-                _anthropic_image_param = add_cache_control_to_content(
-                    anthropic_content_element=_anthropic_image_param,
+                # Data URIs with non-image mime types (e.g. application/pdf) must
+                # translate to Anthropic document blocks, not image blocks —
+                # wrapping a PDF in `type: "image"` is rejected by the API.
+                if isinstance(url_str, str) and _is_anthropic_document_data_uri(
+                    url_str
+                ):
+                    synth_file_message: ChatCompletionFileObject = {
+                        "type": "file",
+                        "file": {"file_data": url_str},
+                    }
+                    _document_block = anthropic_process_openai_file_message(
+                        synth_file_message
+                    )
+                    _document_block = add_cache_control_to_content(
+                        anthropic_content_element=cast(
+                            AnthropicMessagesDocumentParam, _document_block
+                        ),
+                        original_content_element=content,
+                    )
+                    anthropic_content_list.append(
+                        cast(AnthropicMessagesDocumentParam, _document_block)
+                    )
+                else:
+                    _anthropic_image_param = create_anthropic_image_param(
+                        image_url_value,
+                        format=format,
+                        is_bedrock_invoke=force_base64,
+                    )
+                    _anthropic_image_param = add_cache_control_to_content(
+                        anthropic_content_element=_anthropic_image_param,
+                        original_content_element=content,
+                    )
+                    anthropic_content_list.append(
+                        cast(AnthropicMessagesImageParam, _anthropic_image_param)
+                    )
+            elif content["type"] == "file":
+                file_content = cast(ChatCompletionFileObject, content)
+                _file_block = anthropic_process_openai_file_message(file_content)
+                _file_block = add_cache_control_to_content(
+                    anthropic_content_element=cast(
+                        AnthropicMessagesDocumentParam, _file_block
+                    ),
                     original_content_element=content,
                 )
-                anthropic_content_list.append(
-                    cast(AnthropicMessagesImageParam, _anthropic_image_param)
-                )
+                anthropic_content_list.append(_file_block)
 
         anthropic_content = anthropic_content_list
     anthropic_tool_result: Optional[AnthropicMessagesToolResultParam] = None
@@ -3977,6 +4042,55 @@ def _convert_to_bedrock_tool_call_result(
                     tool_result_content_blocks.append(
                         BedrockToolResultContentBlock(image=_block["image"])
                     )
+                elif "document" in _block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(document=_block["document"])
+                    )
+                else:
+                    verbose_logger.warning(
+                        "Bedrock Converse: unrecognized BedrockContentBlock keys "
+                        "%s for image_url tool-result block %s; dropping.",
+                        list(_block.keys()),
+                        content,
+                    )
+            elif content["type"] == "file":
+                # Match the user-message path (_process_file_message): accept
+                # either file_data (base64 data URI) or file_id (server-side
+                # reference / URL) and hand off to BedrockImageProcessor. Raise
+                # BadRequestError on both-None rather than silently dropping.
+                file_obj = content.get("file") or {}
+                file_data = file_obj.get("file_data")
+                file_id = file_obj.get("file_id")
+                if file_data is None and file_id is None:
+                    raise litellm.BadRequestError(
+                        message="file_data and file_id cannot both be None. Got={}".format(
+                            content
+                        ),
+                        model="",
+                        llm_provider="bedrock",
+                    )
+                file_format = file_obj.get("format")
+                _file_block: BedrockContentBlock = (
+                    BedrockImageProcessor.process_image_sync(
+                        image_url=cast(str, file_id or file_data),
+                        format=file_format,
+                    )
+                )
+                if "document" in _file_block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(document=_file_block["document"])
+                    )
+                elif "image" in _file_block:
+                    tool_result_content_blocks.append(
+                        BedrockToolResultContentBlock(image=_file_block["image"])
+                    )
+                else:
+                    verbose_logger.warning(
+                        "Bedrock Converse: unrecognized BedrockContentBlock keys "
+                        "%s for file tool-result block %s; dropping.",
+                        list(_file_block.keys()),
+                        content,
+                    )
 
     message.get("name", "")
     id = str(message.get("tool_call_id", str(uuid.uuid4())))

litellm/litellm_core_utils/sensitive_data_masker.py

@@ -21,6 +21,8 @@ class SensitiveDataMasker:
             "auth",
             "authorization",
             "credential",
+            # Plural form: Vertex uses ``vertex_credentials``; segment-exact
+            # matching otherwise misses it because "credential" != "credentials".
             "credentials",
             "access",
             "private",

litellm/llms/azure/image_generation/__init__.py

@@ -24,6 +24,6 @@ def get_azure_image_generation_config(model: str) -> BaseImageGenerationConfig:
         return AzureDallE3ImageGenerationConfig()
     else:
         verbose_logger.debug(
-            f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image-1 model format."
+            f"Using AzureGPTImageGenerationConfig for model: {model}. This follows the gpt-image model format."
         )
         return AzureGPTImageGenerationConfig()

litellm/llms/azure/image_generation/gpt_transformation.py

@@ -3,7 +3,7 @@ from litellm.llms.openai.image_generation import GPTImageGenerationConfig
 
 class AzureGPTImageGenerationConfig(GPTImageGenerationConfig):
     """
-    Azure gpt-image-1 image generation config
+    Azure gpt-image image generation config
     """
 
     pass

litellm/llms/azure_ai/vector_stores/transformation.py

@@ -95,6 +95,7 @@ class AzureAIVectorStoreConfig(BaseVectorStoreConfig, BaseAzureLLM):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict[str, Any]]:
         """
         Transform search request for Azure AI Search API

litellm/llms/base_llm/vector_store/transformation.py

@@ -59,6 +59,7 @@ class BaseVectorStoreConfig:
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         pass
 
@@ -70,6 +71,7 @@ class BaseVectorStoreConfig:
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         """
         Optional async version of transform_search_vector_store_request.
@@ -84,6 +86,7 @@ class BaseVectorStoreConfig:
             api_base=api_base,
             litellm_logging_obj=litellm_logging_obj,
             litellm_params=litellm_params,
+            extra_body=extra_body,
         )
 
     @abstractmethod

litellm/llms/bedrock/chat/converse_transformation.py

@@ -1942,8 +1942,8 @@ class AmazonConverseConfig(BaseConfig):
             completion_response = ConverseResponseBlock(**response.json())  # type: ignore
         except Exception as e:
             raise BedrockError(
-                message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
-                    response.text, str(e)
+                message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
+                    str(e)
                 ),
                 status_code=422,
             )

litellm/llms/bedrock/vector_stores/transformation.py

@@ -1,14 +1,16 @@
-from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+from copy import deepcopy
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 from urllib.parse import urlparse
 
 import httpx
 
+from litellm._logging import verbose_logger
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.types.integrations.rag.bedrock_knowledgebase import (
     BedrockKBContent,
-    BedrockKBResponse,
     BedrockKBRetrievalConfiguration,
+    BedrockKBResponse,
     BedrockKBRetrievalQuery,
 )
 from litellm.types.router import GenericLiteLLMParams
@@ -202,6 +204,7 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         if isinstance(query, list):
             query = " ".join(query)
@@ -213,24 +216,46 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
         }
 
         retrieval_config: Dict[str, Any] = {}
+
+        if isinstance(extra_body, dict):
+            retrieval_config = deepcopy(
+                extra_body.get("retrievalConfiguration")
+                or extra_body.get("retrieval_configuration")
+                or {}
+            )
         max_results = vector_store_search_optional_params.get("max_num_results")
         if max_results is not None:
+            existing_number_of_results = retrieval_config.get(
+                "vectorSearchConfiguration", {}
+            ).get("numberOfResults")
+            if (
+                existing_number_of_results is not None
+                and existing_number_of_results != max_results
+            ):
+                verbose_logger.debug(
+                    "Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.numberOfResults (%s) with max_num_results=%s",
+                    existing_number_of_results,
+                    max_results,
+                )
             retrieval_config.setdefault("vectorSearchConfiguration", {})[
                 "numberOfResults"
             ] = max_results
         filters = vector_store_search_optional_params.get("filters")
         if filters is not None:
+            existing_filter = retrieval_config.get("vectorSearchConfiguration", {}).get(
+                "filter"
+            )
+            if existing_filter is not None and existing_filter != filters:
+                verbose_logger.debug(
+                    "Overriding extra_body retrievalConfiguration.vectorSearchConfiguration.filter with filters from vector_store_search_optional_params"
+                )
             retrieval_config.setdefault("vectorSearchConfiguration", {})[
                 "filter"
             ] = filters
         if retrieval_config:
-            # Create a properly typed retrieval configuration
-            typed_retrieval_config: BedrockKBRetrievalConfiguration = {}
-            if "vectorSearchConfiguration" in retrieval_config:
-                typed_retrieval_config["vectorSearchConfiguration"] = retrieval_config[
-                    "vectorSearchConfiguration"
-                ]
-            request_body["retrievalConfiguration"] = typed_retrieval_config
+            request_body["retrievalConfiguration"] = cast(
+                BedrockKBRetrievalConfiguration, retrieval_config
+            )
 
         litellm_logging_obj.model_call_details["query"] = query
         return url, request_body

litellm/llms/custom_httpx/http_handler.py

@@ -1,5 +1,7 @@
 import asyncio
+import inspect
 import os
+import socket
 import ssl
 import sys
 import time
@@ -29,6 +31,10 @@ from litellm.constants import (
     AIOHTTP_CONNECTOR_LIMIT_PER_HOST,
     AIOHTTP_KEEPALIVE_TIMEOUT,
     AIOHTTP_NEEDS_CLEANUP_CLOSED,
+    AIOHTTP_SO_KEEPALIVE,
+    AIOHTTP_TCP_KEEPCNT,
+    AIOHTTP_TCP_KEEPIDLE,
+    AIOHTTP_TCP_KEEPINTVL,
     AIOHTTP_TTL_DNS_CACHE,
     COMPLETION_HTTP_FALLBACK_SECONDS,
     DEFAULT_SSL_CIPHERS,
@@ -54,6 +60,57 @@ except Exception:
     version = "0.0.0"
 
 
+# aiohttp 3.10+ exposes a `socket_factory` kwarg on TCPConnector. Older
+# versions don't — detect once and skip the keep-alive wiring there.
+# https://docs.aiohttp.org/en/stable/client_reference.html#aiohttp.TCPConnector
+_AIOHTTP_SUPPORTS_SOCKET_FACTORY = (
+    "socket_factory" in inspect.signature(TCPConnector.__init__).parameters
+)
+
+
+def _build_aiohttp_keepalive_socket_factory() -> (
+    Optional[Callable[[Tuple[Any, ...]], socket.socket]]
+):
+    """
+    Build a socket_factory that enables SO_KEEPALIVE on aiohttp TCP sockets.
+
+    Why: by default, aiohttp creates sockets without SO_KEEPALIVE, so the kernel
+    sends nothing during a long idle TCP connection. NAT/LB hops (e.g. AWS NAT
+    Gateway, 350s idle timeout) reap the flow well before slow provider
+    responses (OpenAI/Azure: up to 600s) arrive. Enabling SO_KEEPALIVE makes
+    the kernel emit TCP probes that reset the NAT idle timer.
+
+    Returns None when AIOHTTP_SO_KEEPALIVE is disabled or aiohttp is too old.
+    """
+    if not AIOHTTP_SO_KEEPALIVE or not _AIOHTTP_SUPPORTS_SOCKET_FACTORY:
+        return None
+
+    def factory(addr_info: Tuple[Any, ...]) -> socket.socket:
+        family, type_, proto = addr_info[0], addr_info[1], addr_info[2]
+        sock = socket.socket(family=family, type=type_, proto=proto)
+        sock.setblocking(False)
+        sock.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)
+        # Linux: TCP_KEEPIDLE is idle-before-first-probe.
+        # macOS/Darwin: TCP_KEEPALIVE is the equivalent.
+        if hasattr(socket, "TCP_KEEPIDLE"):
+            sock.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPIDLE, AIOHTTP_TCP_KEEPIDLE
+            )
+        elif hasattr(socket, "TCP_KEEPALIVE"):
+            sock.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPALIVE, AIOHTTP_TCP_KEEPIDLE
+            )
+        if hasattr(socket, "TCP_KEEPINTVL"):
+            sock.setsockopt(
+                socket.IPPROTO_TCP, socket.TCP_KEEPINTVL, AIOHTTP_TCP_KEEPINTVL
+            )
+        if hasattr(socket, "TCP_KEEPCNT"):
+            sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_KEEPCNT, AIOHTTP_TCP_KEEPCNT)
+        return sock
+
+    return factory
+
+
 def get_default_headers() -> dict:
     """
     Get default headers for HTTP requests.
@@ -935,6 +992,11 @@ class AsyncHTTPHandler:
             transport_connector_kwargs["limit_per_host"] = (
                 AIOHTTP_CONNECTOR_LIMIT_PER_HOST
             )
+        # Returns None when SO_KEEPALIVE is disabled or aiohttp is too old to
+        # accept socket_factory — version detection lives inside the builder.
+        socket_factory = _build_aiohttp_keepalive_socket_factory()
+        if socket_factory is not None:
+            transport_connector_kwargs["socket_factory"] = socket_factory
 
         return LiteLLMAiohttpTransport(
             client=lambda: ClientSession(

litellm/llms/custom_httpx/llm_http_handler.py

@@ -155,6 +155,30 @@ else:
     LiteLLMLoggingObj = Any
 
 
+def _google_genai_streaming_hidden_params(
+    *,
+    api_base: str,
+    litellm_params: GenericLiteLLMParams,
+    logging_obj: LiteLLMLoggingObj,
+    response_headers: httpx.Headers,
+) -> Dict[str, Any]:
+    """Pre-stream metadata for proxy response headers (mirrors CustomStreamWrapper._hidden_params)."""
+    from litellm.litellm_core_utils.core_helpers import process_response_headers
+
+    _model_info: Dict[str, Any] = dict(
+        getattr(litellm_params, "model_info", None) or {}
+    )
+    _raw_id = _model_info.get("id") or logging_obj.get_router_model_id() or ""
+    _model_id = _raw_id if isinstance(_raw_id, str) else str(_raw_id)
+    return {
+        "model_id": _model_id,
+        "api_base": api_base,
+        "cache_key": "",
+        "response_cost": "",
+        "additional_headers": process_response_headers(response_headers),
+    }
+
+
 class BaseLLMHTTPHandler:
     async def _make_common_async_call(
         self,
@@ -8585,6 +8609,7 @@ class BaseLLMHTTPHandler:
                 api_base=api_base,
                 litellm_logging_obj=logging_obj,
                 litellm_params=dict(litellm_params),
+                extra_body=extra_body,
             )
         else:
             (
@@ -8597,6 +8622,7 @@ class BaseLLMHTTPHandler:
                 api_base=api_base,
                 litellm_logging_obj=logging_obj,
                 litellm_params=dict(litellm_params),
+                extra_body=extra_body,
             )
         all_optional_params: Dict[str, Any] = dict(litellm_params)
         all_optional_params.update(vector_store_search_optional_params or {})
@@ -8697,6 +8723,7 @@ class BaseLLMHTTPHandler:
             api_base=api_base,
             litellm_logging_obj=logging_obj,
             litellm_params=dict(litellm_params),
+            extra_body=extra_body,
         )
 
         all_optional_params: Dict[str, Any] = dict(litellm_params)
@@ -10425,6 +10452,12 @@ class BaseLLMHTTPHandler:
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
                     request_body=data,
+                    hidden_params=_google_genai_streaming_hidden_params(
+                        api_base=api_base,
+                        litellm_params=litellm_params,
+                        logging_obj=logging_obj,
+                        response_headers=response.headers,
+                    ),
                 )
             else:
                 response = sync_httpx_client.post(
@@ -10534,6 +10567,12 @@ class BaseLLMHTTPHandler:
                     litellm_metadata=litellm_metadata or {},
                     custom_llm_provider=custom_llm_provider,
                     request_body=data,
+                    hidden_params=_google_genai_streaming_hidden_params(
+                        api_base=api_base,
+                        litellm_params=litellm_params,
+                        logging_obj=logging_obj,
+                        response_headers=response.headers,
+                    ),
                 )
             else:
                 response = await async_httpx_client.post(

litellm/llms/gemini/vector_stores/transformation.py

@@ -118,6 +118,7 @@ class GeminiVectorStoreConfig(BaseVectorStoreConfig):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         """
         Transform search request to Gemini's generateContent format.

litellm/llms/milvus/vector_stores/transformation.py

@@ -130,6 +130,7 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict[str, Any]]:
         """
         Transform search request for Azure AI Search API

litellm/llms/openai/image_generation/cost_calculator.py

@@ -1,5 +1,5 @@
 """
-Cost calculator for OpenAI image generation models (gpt-image-1, gpt-image-1-mini)
+Cost calculator for OpenAI image generation models (gpt-image family)
 
 These models use token-based pricing instead of pixel-based pricing like DALL-E.
 """
@@ -17,13 +17,13 @@ def cost_calculator(
     custom_llm_provider: Optional[str] = None,
 ) -> float:
     """
-    Calculate cost for OpenAI gpt-image-1 and gpt-image-1-mini models.
+    Calculate cost for OpenAI gpt-image models.
 
     Uses the same usage format as Responses API, so we reuse the helper
     to transform to chat completion format and use generic_cost_per_token.
 
     Args:
-        model: The model name (e.g., "gpt-image-1", "gpt-image-1-mini")
+        model: The model name (e.g., "gpt-image-1", "gpt-image-2")
         image_response: The ImageResponse containing usage data
         custom_llm_provider: Optional provider name
 

litellm/llms/openai/image_generation/gpt_transformation.py

@@ -15,7 +15,7 @@ if TYPE_CHECKING:
 
 class GPTImageGenerationConfig(BaseImageGenerationConfig):
     """
-    OpenAI gpt-image-1 image generation config
+    OpenAI gpt-image image generation config
     """
 
     def get_supported_openai_params(

litellm/llms/openai/vector_stores/transformation.py

@@ -106,6 +106,7 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         url = f"{api_base}/{vector_store_id}/search"
         typed_request_body = VectorStoreSearchRequest(

litellm/llms/openai_like/providers.json

@@ -101,5 +101,10 @@
     "param_mappings": {
       "max_completion_tokens": "max_tokens"
     }
+  },
+  "aihubmix": {
+    "base_url": "https://aihubmix.com/v1",
+    "api_key_env": "AIHUBMIX_API_KEY",
+    "api_base_env": "AIHUBMIX_API_BASE"
   }
 }

litellm/llms/pg_vector/vector_stores/transformation.py

@@ -80,6 +80,7 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         url = f"{api_base}/{vector_store_id}/search"
         _, request_body = super().transform_search_vector_store_request(
@@ -89,5 +90,6 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
             api_base=api_base,
             litellm_logging_obj=litellm_logging_obj,
             litellm_params=litellm_params,
+            extra_body=extra_body,
         )
         return url, request_body

litellm/llms/ragflow/vector_stores/transformation.py

@@ -102,6 +102,7 @@ class RAGFlowVectorStoreConfig(BaseVectorStoreConfig):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         """RAGFlow vector stores are management-only, search is not supported."""
         raise NotImplementedError(

litellm/llms/s3_vectors/vector_stores/transformation.py

@@ -79,6 +79,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         """Sync version - generates embedding synchronously."""
         # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name
@@ -140,6 +141,7 @@ class S3VectorsVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
         """Async version - generates embedding asynchronously."""
         # For S3 Vectors, vector_store_id should be in format: bucket_name:index_name

litellm/llms/vertex_ai/batches/handler.py

@@ -4,6 +4,7 @@ from typing import Any, Coroutine, Dict, Optional, Union
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
@@ -224,8 +225,14 @@ class VertexAIBatchPrediction(VertexLLM):
                     },
                 )
 
-        response = sync_handler.get(
-            url=api_base,
+        # ``api_base`` here can come from caller-supplied request kwargs
+        # (clientside override). Wrap the fetch in ``safe_get`` so DNS
+        # rebind / private / cloud-metadata targets are rejected; the
+        # proxy auth gate already blocks malicious clientside ``api_base``
+        # at the boundary — this is defense-in-depth for SDK callers.
+        response = safe_get(
+            sync_handler,
+            api_base,
             headers=headers,
         )
 
@@ -270,8 +277,13 @@ class VertexAIBatchPrediction(VertexLLM):
                     },
                 )
 
-        response = await client.get(
-            url=api_base,
+        # Mirror the sync path: ``api_base`` may come from caller-supplied
+        # request kwargs, so wrap the fetch in ``async_safe_get`` to reject
+        # DNS-rebind / private / cloud-metadata targets. Defense-in-depth
+        # behind the proxy auth gate's clientside ``api_base`` check.
+        response = await async_safe_get(
+            client,
+            api_base,
             headers=headers,
         )
         if response.status_code != 200:

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -2395,8 +2395,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             completion_response = GenerateContentResponseBody(**raw_response.json())  # type: ignore
         except Exception as e:
             raise VertexAIError(
-                message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
-                    raw_response.text, str(e)
+                message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
+                    str(e)
                 ),
                 status_code=422,
                 headers=raw_response.headers,
@@ -2530,8 +2530,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
 
         except Exception as e:
             raise VertexAIError(
-                message="Received={}, Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
-                    completion_response, str(e)
+                message="Error converting to valid response block={}. File an issue if litellm error - https://github.com/BerriAI/litellm/issues".format(
+                    str(e)
                 ),
                 status_code=422,
                 headers=raw_response.headers,

litellm/llms/vertex_ai/vector_stores/rag_api/transformation.py

@@ -100,6 +100,7 @@ class VertexVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict[str, Any]]:
         """
         Transform search request for Vertex AI RAG API

litellm/llms/vertex_ai/vector_stores/search_api/transformation.py

@@ -107,6 +107,7 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
         api_base: str,
         litellm_logging_obj: LiteLLMLoggingObj,
         litellm_params: dict,
+        extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict[str, Any]]:
         """
         Transform search request for Vertex AI RAG API

litellm/model_prices_and_context_window_backup.json

@@ -712,6 +712,7 @@
     },
     "anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -735,6 +736,7 @@
     },
     "anthropic.claude-haiku-4-5@20251001": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -955,6 +957,7 @@
     },
     "anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -982,6 +985,7 @@
     },
     "anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1011,6 +1015,7 @@
     },
     "global.anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1040,6 +1045,7 @@
     },
     "us.anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1127,6 +1133,7 @@
     },
     "anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1157,6 +1164,7 @@
     },
     "global.anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1187,6 +1195,7 @@
     },
     "us.anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1277,6 +1286,7 @@
     },
     "anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1305,6 +1315,7 @@
     },
     "global.anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1333,6 +1344,7 @@
     },
     "us.anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
         "cache_read_input_token_cost": 3.3e-07,
         "input_cost_per_token": 3.3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1447,11 +1459,13 @@
     },
     "anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "input_cost_per_token_above_200k_tokens": 6e-06,
         "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -5103,6 +5117,38 @@
             "/v1/images/edits"
         ]
     },
+    "azure/gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "azure/gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "azure/low/1024-x-1024/gpt-image-1-mini": {
         "input_cost_per_pixel": 2.0751953125e-09,
         "litellm_provider": "azure",
@@ -17889,11 +17935,13 @@
     },
     "global.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "input_cost_per_token_above_200k_tokens": 6e-06,
         "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -17950,6 +17998,7 @@
     },
     "global.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -19083,6 +19132,38 @@
         "supports_vision": true,
         "supports_pdf_input": true
     },
+    "gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "low/1024-x-1024/gpt-image-1.5": {
         "input_cost_per_image": 0.009,
         "litellm_provider": "openai",
@@ -30052,6 +30133,7 @@
     },
     "us.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.375e-06,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-06,
         "cache_read_input_token_cost": 1.1e-07,
         "input_cost_per_token": 1.1e-06,
         "litellm_provider": "bedrock_converse",
@@ -30203,11 +30285,13 @@
     },
     "us.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
         "cache_read_input_token_cost": 3.3e-07,
         "input_cost_per_token": 3.3e-06,
         "input_cost_per_token_above_200k_tokens": 6.6e-06,
         "output_cost_per_token_above_200k_tokens": 2.475e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6.6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -30308,6 +30392,7 @@
     },
     "us.anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -30335,6 +30420,7 @@
     },
     "global.anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",

litellm/passthrough/main.py

@@ -21,6 +21,7 @@ import httpx
 from httpx._types import CookieTypes, QueryParamTypes, RequestFiles
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
 from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
@@ -390,19 +391,28 @@ def _sync_streaming(
 ):
     from litellm.utils import executor
 
+    raw_bytes: List[bytes] = []
+    flush_scheduled = False
     try:
-        raw_bytes: List[bytes] = []
         for chunk in response.iter_bytes():  # type: ignore
             raw_bytes.append(chunk)
             yield chunk
-
-        executor.submit(
-            litellm_logging_obj.flush_passthrough_collected_chunks,
-            raw_bytes=raw_bytes,
-            provider_config=provider_config,
-        )
-    except Exception as e:
-        raise e
+    finally:
+        if not flush_scheduled and raw_bytes:
+            flush_scheduled = True
+            try:
+                executor.submit(
+                    litellm_logging_obj.flush_passthrough_collected_chunks,
+                    raw_bytes=raw_bytes,
+                    provider_config=provider_config,
+                )
+            except Exception as e:
+                verbose_logger.exception(
+                    "Failed to schedule passthrough spend-tracking flush "
+                    "in _sync_streaming; %d buffered chunks dropped: %s",
+                    len(raw_bytes),
+                    e,
+                )
 
 
 async def _async_streaming(
@@ -411,23 +421,45 @@ async def _async_streaming(
     provider_config: "BasePassthroughConfig",
 ):
     iter_response = await response
+
     try:
         iter_response.raise_for_status()
-        raw_bytes: List[bytes] = []
-
-        async for chunk in iter_response.aiter_bytes():  # type: ignore
-            raw_bytes.append(chunk)
-            yield chunk
-
-        asyncio.create_task(
-            litellm_logging_obj.async_flush_passthrough_collected_chunks(
-                raw_bytes=raw_bytes,
-                provider_config=provider_config,
-            )
-        )
     except Exception:
         try:
             await iter_response.aclose()
         except Exception:
             pass
         raise
+
+    raw_bytes: List[bytes] = []
+    flush_scheduled = False
+    try:
+        async for chunk in iter_response.aiter_bytes():  # type: ignore
+            raw_bytes.append(chunk)
+            yield chunk
+    except Exception:
+        try:
+            await iter_response.aclose()
+        except Exception:
+            pass
+        raise
+    finally:
+        # GeneratorExit (raised on client disconnect) is not caught by
+        # `except Exception`; the finally block ensures partial usage
+        # still gets flushed for spend tracking. See LIT-2642.
+        if not flush_scheduled and raw_bytes:
+            flush_scheduled = True
+            try:
+                asyncio.create_task(
+                    litellm_logging_obj.async_flush_passthrough_collected_chunks(
+                        raw_bytes=raw_bytes,
+                        provider_config=provider_config,
+                    )
+                )
+            except Exception as e:
+                verbose_logger.exception(
+                    "Failed to schedule passthrough spend-tracking flush "
+                    "in _async_streaming; %d buffered chunks dropped: %s",
+                    len(raw_bytes),
+                    e,
+                )

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -117,7 +117,10 @@ class MCPRequestHandler:
             return b"{}"
 
         request.body = mock_body  # type: ignore
-        if ".well-known" in str(request.url):  # public routes
+        # Only OAuth metadata routes registered under /.well-known/ are public.
+        # Match on request.url.path (path-only, exact prefix) so the substring
+        # cannot be smuggled via query string, hostname, or a deeper URL segment.
+        if request.url.path.startswith("/.well-known/"):
             validated_user_api_key_auth = UserAPIKeyAuth()
         elif has_explicit_litellm_key:
             # Explicit x-litellm-api-key provided - always validate normally
@@ -126,27 +129,37 @@ class MCPRequestHandler:
             )
         elif oauth2_headers:
             # No x-litellm-api-key, but Authorization header present.
-            # Could be a LiteLLM key (backward compat) OR an OAuth2 token
-            # from an upstream MCP provider (e.g. Atlassian).
-            # Try LiteLLM auth first; on auth failure, treat as OAuth2 passthrough.
+            # Could be a LiteLLM key (backward compat) OR an opaque OAuth2 token
+            # the operator wants forwarded to an upstream OAuth2-mode MCP server.
+            # Try LiteLLM auth first; on auth failure, only fall back to anonymous
+            # passthrough when the request actually targets a server whose operator
+            # configured ``auth_type=oauth2``. For any other server (api_key,
+            # bearer_token, basic, etc.), a failed LiteLLM auth is a real failure
+            # and must propagate — otherwise an attacker can exchange any garbage
+            # bearer for an anonymous session.
             try:
                 validated_user_api_key_auth = await user_api_key_auth(
                     api_key=litellm_api_key, request=request
                 )
-            except HTTPException as e:
-                if e.status_code in (401, 403):
+            except (HTTPException, ProxyException) as e:
+                # HTTPException.status_code is int; ProxyException.code is
+                # normalized to str in its __init__ but can be ``"None"`` or any
+                # non-numeric string when the caller didn't supply a numeric
+                # code, so we compare against both int and str forms rather
+                # than coercing (``int("None")`` would raise ValueError and
+                # rewrite the auth error as a 500).
+                status = e.status_code if isinstance(e, HTTPException) else e.code
+                if status in (
+                    401,
+                    403,
+                    "401",
+                    "403",
+                ) and MCPRequestHandler._target_servers_use_oauth2(
+                    path=request.url.path, mcp_servers=mcp_servers
+                ):
                     verbose_logger.debug(
-                        "MCP OAuth2: Authorization header is not a valid LiteLLM key, "
-                        "treating as OAuth2 token passthrough"
-                    )
-                    validated_user_api_key_auth = UserAPIKeyAuth()
-                else:
-                    raise
-            except ProxyException as e:
-                if str(e.code) in ("401", "403"):
-                    verbose_logger.debug(
-                        "MCP OAuth2: Authorization header is not a valid LiteLLM key, "
-                        "treating as OAuth2 token passthrough"
+                        "MCP OAuth2: target server is OAuth2-mode, treating "
+                        "Authorization as upstream OAuth2 token passthrough"
                     )
                     validated_user_api_key_auth = UserAPIKeyAuth()
                 else:
@@ -165,6 +178,62 @@ class MCPRequestHandler:
             dict(headers),
         )
 
+    @staticmethod
+    def _extract_target_server_names_from_path(path: str) -> List[str]:
+        """
+        Extract the target MCP server name from the standard MCP transport
+        URL patterns: ``/mcp/{server_name}[/...]`` and
+        ``/{server_name}/mcp[/...]``. Returns ``[]`` for any other path so
+        callers fail closed when the target cannot be resolved.
+
+        REST/admin endpoints, OAuth2 server endpoints
+        (``/{server_name}/authorize``, ``/token`` etc.), and ``.well-known``
+        discovery routes intentionally fall through — those flows do not need
+        OAuth2 token passthrough. Clients aggregating multiple servers should
+        use ``x-mcp-servers``, which takes precedence over path parsing.
+        """
+        segments = [s for s in path.split("/") if s]
+        if len(segments) >= 2 and segments[0] == "mcp":
+            return [segments[1]]
+        if len(segments) >= 2 and segments[1] == "mcp":
+            return [segments[0]]
+        return []
+
+    @staticmethod
+    def _target_servers_use_oauth2(path: str, mcp_servers: Optional[List[str]]) -> bool:
+        """
+        True only when EVERY MCP server the request targets is configured for
+        ``auth_type == oauth2``. If any target is non-OAuth2 — or if the target
+        cannot be resolved at all — return False so the caller fails closed.
+
+        Used to gate the "treat Authorization as opaque OAuth2 token" fallback
+        in :meth:`process_mcp_request` so a failed LiteLLM-auth cannot be
+        exchanged for an anonymous session against a non-OAuth2 server.
+        """
+        # Inline imports avoid a circular dependency: mcp_server_manager imports
+        # from this module.
+        from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
+            global_mcp_server_manager,
+        )
+        from litellm.types.mcp import MCPAuth
+
+        # Use the x-mcp-servers header verbatim when present (including the
+        # explicitly-empty list, which means "no targets" → fail closed).
+        # Only fall back to path parsing when the header was absent entirely.
+        target_names = (
+            mcp_servers
+            if mcp_servers is not None
+            else MCPRequestHandler._extract_target_server_names_from_path(path)
+        )
+        if not target_names:
+            return False
+
+        for name in target_names:
+            server = global_mcp_server_manager.get_mcp_server_by_name(name)
+            if server is None or server.auth_type != MCPAuth.oauth2:
+                return False
+        return True
+
     @staticmethod
     def _get_mcp_auth_header_from_headers(headers: Headers) -> Optional[str]:
         """

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -131,6 +131,22 @@ def decode_state_hash(encrypted_state: str) -> dict:
     return state_data
 
 
+def _get_validated_client_redirect_uri(state_data: Dict[str, Any]) -> str:
+    """Return a loopback client redirect URI from OAuth state."""
+    redirect_uri = state_data.get("client_redirect_uri") or state_data.get("base_url")
+    if not redirect_uri or not isinstance(redirect_uri, str):
+        raise HTTPException(status_code=400, detail="Invalid redirect URI")
+    validate_loopback_redirect_uri(redirect_uri)
+    return redirect_uri
+
+
+def _append_query_params(url: str, params: Dict[str, str]) -> str:
+    parsed = urlparse(url)
+    query_params = parse_qsl(parsed.query, keep_blank_values=True)
+    query_params.extend(params.items())
+    return urlunparse(parsed._replace(query=urlencode(query_params)))
+
+
 def _resolve_oauth2_server_for_root_endpoints(
     client_ip: Optional[str] = None,
 ) -> Optional[MCPServer]:
@@ -568,7 +584,7 @@ async def authorize(
         else None
     )
     if mcp_server is None and mcp_server_name is None:
-        mcp_server = _resolve_oauth2_server_for_root_endpoints()
+        mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
     if mcp_server is None:
         raise HTTPException(status_code=404, detail="MCP server not found")
     # Use server's stored client_id when caller doesn't supply one.
@@ -630,7 +646,7 @@ async def token_endpoint(
         lookup_name, client_ip=client_ip
     )
     if mcp_server is None and mcp_server_name is None:
-        mcp_server = _resolve_oauth2_server_for_root_endpoints()
+        mcp_server = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
     if mcp_server is None:
         raise HTTPException(status_code=404, detail="MCP server not found")
     return await exchange_token_with_server(
@@ -651,7 +667,6 @@ async def token_endpoint(
 async def callback(code: str, state: str):
     try:
         state_data = decode_state_hash(state)
-        base_url = state_data["base_url"]
         original_state = state_data["original_state"]
 
         # Re-validate loopback at the sink. /authorize rejects non-loopback
@@ -659,10 +674,10 @@ async def callback(code: str, state: str):
         # minted before that check was added have no expiry and remain
         # valid indefinitely. Validating here blocks the open-redirect +
         # code-theft primitive even for pre-fix states.
-        validate_loopback_redirect_uri(base_url)
+        redirect_uri = _get_validated_client_redirect_uri(state_data)
 
         params = {"code": code, "state": original_state}
-        complete_returned_url = f"{base_url}?{urlencode(params)}"
+        complete_returned_url = _append_query_params(redirect_uri, params)
         return RedirectResponse(url=complete_returned_url, status_code=302)
 
     except HTTPException:
@@ -719,16 +734,16 @@ def _build_oauth_protected_resource_response(
     )
 
     request_base_url = get_request_base_url(request)
+    client_ip = IPAddressUtils.get_mcp_client_ip(request)
 
     # When no server name provided, try to resolve the single OAuth2 server
     if mcp_server_name is None:
-        resolved = _resolve_oauth2_server_for_root_endpoints()
+        resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
         if resolved:
             mcp_server_name = resolved.server_name or resolved.name
 
     mcp_server: Optional[MCPServer] = None
     if mcp_server_name:
-        client_ip = IPAddressUtils.get_mcp_client_ip(request)
         mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
             mcp_server_name, client_ip=client_ip
         )
@@ -835,10 +850,11 @@ def _build_oauth_authorization_server_response(
     )
 
     request_base_url = get_request_base_url(request)
+    client_ip = IPAddressUtils.get_mcp_client_ip(request)
 
     # When no server name provided, try to resolve the single OAuth2 server
     if mcp_server_name is None:
-        resolved = _resolve_oauth2_server_for_root_endpoints()
+        resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
         if resolved:
             mcp_server_name = resolved.server_name or resolved.name
 
@@ -855,7 +871,6 @@ def _build_oauth_authorization_server_response(
 
     mcp_server: Optional[MCPServer] = None
     if mcp_server_name:
-        client_ip = IPAddressUtils.get_mcp_client_ip(request)
         mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
             mcp_server_name, client_ip=client_ip
         )
@@ -1007,8 +1022,9 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
         "client_secret": "dummy",
         "redirect_uris": [f"{request_base_url}/callback"],
     }
+    client_ip = IPAddressUtils.get_mcp_client_ip(request)
     if not mcp_server_name:
-        resolved = _resolve_oauth2_server_for_root_endpoints()
+        resolved = _resolve_oauth2_server_for_root_endpoints(client_ip=client_ip)
         if resolved:
             return await register_client_with_server(
                 request=request,
@@ -1021,7 +1037,6 @@ async def register_client(request: Request, mcp_server_name: Optional[str] = Non
             )
         return dummy_return
 
-    client_ip = IPAddressUtils.get_mcp_client_ip(request)
     mcp_server = global_mcp_server_manager.get_mcp_server_by_name(
         mcp_server_name, client_ip=client_ip
     )

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -50,8 +50,11 @@ from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mc
 from litellm.proxy._experimental.mcp_server.utils import (
     MCP_TOOL_PREFIX_SEPARATOR,
     add_server_prefix_to_name,
+    compute_short_server_prefix,
     get_server_prefix,
+    is_short_mcp_tool_prefix_enabled,
     is_tool_name_prefixed,
+    iter_known_server_prefixes,
     merge_mcp_headers,
     normalize_server_name,
     split_server_prefix_from_name,
@@ -106,6 +109,12 @@ if not _separator_probe.is_valid:
         SEP_986_URL,
     )
 
+_AZURE_ENTRA_HOSTS = {
+    "login.microsoftonline.com",  # Global
+    "login.microsoftonline.us",  # US Government
+    "login.chinacloudapi.cn",  # China
+}
+
 
 def _warn_on_server_name_fields(
     *,
@@ -364,6 +373,7 @@ class MCPServerManager:
                 aws_session_name=server_config.get("aws_session_name", None),
                 instructions=server_config.get("instructions", None),
             )
+            self._assign_unique_short_prefix(new_server)
             self.config_mcp_servers[server_id] = new_server
 
             # Check if this is an OpenAPI-based server
@@ -726,6 +736,7 @@ class MCPServerManager:
         try:
             if mcp_server.server_id not in self.registry:
                 new_server = await self.build_mcp_server_from_table(mcp_server)
+                self._assign_unique_short_prefix(new_server)
                 self.registry[mcp_server.server_id] = new_server
                 await self._maybe_register_openapi_tools(new_server)
                 verbose_logger.debug(f"Added MCP Server: {new_server.name}")
@@ -738,6 +749,12 @@ class MCPServerManager:
         try:
             if mcp_server.server_id in self.registry:
                 new_server = await self.build_mcp_server_from_table(mcp_server)
+                # Carry the previously-resolved short prefix across so the
+                # tool names stay stable for clients holding cached lists.
+                existing_prefix = self.registry[mcp_server.server_id].short_prefix
+                if existing_prefix and not new_server.short_prefix:
+                    new_server.short_prefix = existing_prefix
+                self._assign_unique_short_prefix(new_server)
                 self.registry[mcp_server.server_id] = new_server
                 await self._maybe_register_openapi_tools(new_server)
                 verbose_logger.debug(f"Updated MCP Server: {new_server.name}")
@@ -1236,7 +1253,11 @@ class MCPServerManager:
 
             ## HANDLE OPENAPI TOOLS
             if server.spec_path:
-                _tools = global_mcp_tool_registry.list_tools(tool_prefix=server.name)
+                # OpenAPI tools were stored in the registry under the prefix
+                # active at registration time — fetch by that same prefix.
+                _tools = global_mcp_tool_registry.list_tools(
+                    tool_prefix=get_server_prefix(server)
+                )
                 tools = global_mcp_tool_registry.convert_tools_to_mcp_sdk_tool_type(
                     _tools
                 )
@@ -1488,11 +1509,28 @@ class MCPServerManager:
             client = get_async_httpx_client(llm_provider=httpxSpecialProvider.MCP)
             response = await client.get(server_url)
             response.raise_for_status()
-            verbose_logger.warning(
-                "MCP OAuth discovery unexpectedly succeeded for %s; server did not challenge",
-                server_url,
+            (
+                authorization_servers,
+                resource_scopes,
+            ) = await self._attempt_well_known_discovery(server_url)
+            metadata = await self._fetch_authorization_server_metadata(
+                authorization_servers
             )
-            raise RuntimeError("OAuth discovery must not succeed without a challenge")
+            if (
+                metadata is None
+                and not resource_scopes
+                and authorization_servers
+                and response.status_code == 200
+            ):
+                verbose_logger.warning(
+                    "MCP OAuth discovery for %s received 200 OK without RFC 9728 challenge and no discoverable authorization metadata.",
+                    server_url,
+                )
+            if metadata is None and resource_scopes:
+                return MCPOAuthMetadata(scopes=resource_scopes)
+            if metadata is not None and resource_scopes:
+                metadata.scopes = resource_scopes
+            return metadata
         except HTTPStatusError as exc:
             verbose_logger.debug(
                 "MCP OAuth discovery for %s received status error: %s",
@@ -1510,8 +1548,8 @@ class MCPServerManager:
                 header_value
             )
 
-            authorization_servers: List[str] = []
-            resource_scopes: Optional[List[str]] = None
+            authorization_servers = []
+            resource_scopes = None
             if resource_metadata_url:
                 (
                     authorization_servers,
@@ -1674,6 +1712,9 @@ class MCPServerManager:
                 f"{base}/.well-known/oauth-authorization-server/{path}"
             )
             candidate_urls.append(f"{base}/.well-known/openid-configuration/{path}")
+            candidate_urls.append(
+                f"{issuer_url.rstrip('/')}/.well-known/openid-configuration"
+            )
         candidate_urls.append(f"{base}/.well-known/oauth-authorization-server")
         candidate_urls.append(f"{base}/.well-known/openid-configuration")
         candidate_urls.append(issuer_url.rstrip("/"))
@@ -1713,7 +1754,28 @@ class MCPServerManager:
             ):
                 return metadata
 
-        return None
+        return self._build_azure_authorization_server_metadata(parsed)
+
+    @staticmethod
+    def _build_azure_authorization_server_metadata(
+        parsed_issuer_url: Any,
+    ) -> Optional[MCPOAuthMetadata]:
+        path_parts = [
+            part for part in (parsed_issuer_url.path or "").split("/") if part
+        ]
+        if (
+            parsed_issuer_url.netloc not in _AZURE_ENTRA_HOSTS
+            or len(path_parts) != 2
+            or path_parts[1] != "v2.0"
+        ):
+            return None
+
+        tenant = path_parts[0]
+        base = f"{parsed_issuer_url.scheme}://{parsed_issuer_url.netloc}/{tenant}"
+        return MCPOAuthMetadata(
+            authorization_url=f"{base}/oauth2/v2.0/authorize",
+            token_url=f"{base}/oauth2/v2.0/token",
+        )
 
     @staticmethod
     def _decrypt_credential_field(
@@ -1810,6 +1872,63 @@ class MCPServerManager:
             verbose_logger.warning(f"Error listing tools from {server_name}: {str(e)}")
             return []
 
+    _SHORT_PREFIX_MAX_REHASH_ATTEMPTS = 1024
+
+    def _assign_unique_short_prefix(self, server: MCPServer) -> None:
+        """Resolve and cache a collision-free short tool prefix on ``server``.
+
+        Called at registration time for every MCP server entering the
+        registry.  Mutates ``server.short_prefix`` in place.  No-ops when
+        ``LITELLM_USE_SHORT_MCP_TOOL_PREFIX`` is disabled, when the server
+        has no ``server_id`` (synthetic temp-server objects), or when a
+        prefix is already cached.
+
+        Collision strategy: take the natural hash; if it's already used by
+        a *different* server in the combined registry, rehash with an
+        incrementing attempt counter until we find an unused slot.  The
+        attempt counter is folded into the hash so the resulting prefix is
+        still deterministic for a given (server_id, set-of-other-server-ids)
+        pair within one process.
+        """
+        if not is_short_mcp_tool_prefix_enabled():
+            return
+        if server.short_prefix:
+            return
+        if not server.server_id:
+            return
+
+        used: Dict[str, str] = {}
+        for other in self.get_registry().values():
+            if other.server_id == server.server_id:
+                continue
+            if other.short_prefix:
+                used[other.short_prefix] = other.server_id
+
+        for attempt in range(self._SHORT_PREFIX_MAX_REHASH_ATTEMPTS):
+            candidate = compute_short_server_prefix(server.server_id, attempt=attempt)
+            if candidate not in used:
+                server.short_prefix = candidate
+                if attempt > 0:
+                    verbose_logger.info(
+                        "MCP short-prefix collision resolved for server %s: "
+                        "natural hash collided with %s, using rehashed prefix "
+                        "%s (attempt=%d).",
+                        server.server_id,
+                        used.get(
+                            compute_short_server_prefix(server.server_id, attempt=0),
+                            "<unknown>",
+                        ),
+                        candidate,
+                        attempt,
+                    )
+                return
+
+        raise RuntimeError(
+            f"Unable to assign a unique short MCP tool prefix for server "
+            f"{server.server_id} after {self._SHORT_PREFIX_MAX_REHASH_ATTEMPTS} "
+            "attempts; the 3-character prefix space is too crowded."
+        )
+
     def _create_prefixed_tools(
         self, tools: List[MCPTool], server: MCPServer, add_prefix: bool = True
     ) -> List[MCPTool]:
@@ -1838,9 +1957,13 @@ class MCPServerManager:
             tool_copy.name = name_to_use
             prefixed_tools.append(tool_copy)
 
-            # Update tool to server mapping for resolution (support both forms)
+            # Register every known prefix form (alias, server_name, server_id,
+            # short ID) so call_tool can resolve regardless of which form a
+            # caller / cached client is using.
             self.tool_name_to_mcp_server_name_mapping[original_name] = prefix
-            self.tool_name_to_mcp_server_name_mapping[prefixed_name] = prefix
+            for known_prefix in iter_known_server_prefixes(server):
+                qualified = add_server_prefix_to_name(original_name, known_prefix)
+                self.tool_name_to_mcp_server_name_mapping[qualified] = prefix
 
         verbose_logger.info(
             f"Successfully fetched {len(prefixed_tools)} tools from server {server.name}"
@@ -2601,37 +2724,43 @@ class MCPServerManager:
         Returns:
             MCPServer if found, None otherwise
         """
+        registry_servers = list(self.get_registry().values())
+
+        # Build prefix → server lookup covering every known form a tool name
+        # may take (alias / server_name / server_id / short ID).  This is what
+        # makes the short-prefix mode work without breaking historical names.
+        prefix_to_server: Dict[str, MCPServer] = {}
+        for server in registry_servers:
+            for known_prefix in iter_known_server_prefixes(server):
+                normalised = normalize_server_name(known_prefix)
+                prefix_to_server.setdefault(normalised, server)
+
         # First try with the original tool name
         if tool_name in self.tool_name_to_mcp_server_name_mapping:
             server_name = self.tool_name_to_mcp_server_name_mapping[tool_name]
-            for server in self.get_registry().values():
-                if normalize_server_name(server.name) == normalize_server_name(
-                    server_name
-                ):
+            normalised_lookup = normalize_server_name(server_name)
+            if normalised_lookup in prefix_to_server:
+                return prefix_to_server[normalised_lookup]
+            for server in registry_servers:
+                if normalize_server_name(server.name) == normalised_lookup:
                     return server
 
-        # If not found and tool name is prefixed, try extracting server name from prefix
-        known_prefixes = {
-            normalize_server_name(get_server_prefix(s))
-            for s in self.get_registry().values()
-            if get_server_prefix(s)
-        }
-        if is_tool_name_prefixed(tool_name, known_server_prefixes=known_prefixes):
+        # If not found and tool name is prefixed, extract the prefix and
+        # match against any known form.
+        if is_tool_name_prefixed(
+            tool_name, known_server_prefixes=set(prefix_to_server.keys())
+        ):
             (
                 original_tool_name,
                 server_name_from_prefix,
             ) = split_server_prefix_from_name(tool_name)
-            if original_tool_name in self.tool_name_to_mcp_server_name_mapping:
-                for server in self.get_registry().values():
-                    if server.server_name is None:
-                        if normalize_server_name(server.name) == normalize_server_name(
-                            server_name_from_prefix
-                        ):
-                            return server
-                    elif normalize_server_name(
-                        server.server_name
-                    ) == normalize_server_name(server_name_from_prefix):
-                        return server
+            normalised_prefix = normalize_server_name(server_name_from_prefix)
+            matched_server = prefix_to_server.get(normalised_prefix)
+            if matched_server is not None and (
+                original_tool_name in self.tool_name_to_mcp_server_name_mapping
+                or tool_name in self.tool_name_to_mcp_server_name_mapping
+            ):
+                return matched_server
 
         return None
 
@@ -2666,6 +2795,9 @@ class MCPServerManager:
         previous_registry = self.registry
         new_registry: Dict[str, MCPServer] = {}
 
+        # Stage one: build every server.  Stage two assigns short prefixes
+        # against the *full* set so dedup is deterministic regardless of
+        # iteration order.
         for server in db_mcp_servers:
             existing_server = previous_registry.get(server.server_id)
 
@@ -2689,10 +2821,21 @@ class MCPServerManager:
                 f"Building server from DB: {server.server_id} ({server.server_name})"
             )
             new_server = await self.build_mcp_server_from_table(server)
+            # Carry the cached short_prefix from the previous registry entry
+            # (if any) so the prefix is stable across reloads.
+            if existing_server is not None and existing_server.short_prefix:
+                new_server.short_prefix = existing_server.short_prefix
             new_registry[server.server_id] = new_server
-            await self._maybe_register_openapi_tools(new_server)
 
+        # Swap in the new registry first so _assign_unique_short_prefix
+        # sees the complete set when checking for collisions.
         self.registry = new_registry
+        for new_server in new_registry.values():
+            self._assign_unique_short_prefix(new_server)
+            # Register OpenAPI tools *after* the final short prefix is assigned
+            # so the tools are stored in the global registry under the same
+            # prefix that lookups will use.
+            await self._maybe_register_openapi_tools(new_server)
 
         verbose_logger.debug(
             "MCP registry refreshed (%s servers in registry)", len(new_registry)

litellm/proxy/_experimental/mcp_server/server.py

@@ -49,6 +49,7 @@ from litellm.proxy._experimental.mcp_server.utils import (
     LITELLM_MCP_SERVER_VERSION,
     add_server_prefix_to_name,
     get_server_prefix,
+    iter_known_server_prefixes,
 )
 from litellm.proxy._types import UserAPIKeyAuth
 from litellm.proxy.auth.ip_address_utils import IPAddressUtils
@@ -711,13 +712,7 @@ if MCP_AVAILABLE:
                 for server in allowed_mcp_servers:
                     if server:
                         match_list = [
-                            s.lower()
-                            for s in [
-                                server.alias,
-                                server.server_name,
-                                server.server_id,
-                            ]
-                            if s is not None
+                            s.lower() for s in iter_known_server_prefixes(server) if s
                         ]
 
                         if server_or_group.lower() in match_list:
@@ -2031,11 +2026,13 @@ if MCP_AVAILABLE:
         # Remove prefix from tool name for logging and processing
         original_tool_name, server_name = split_server_prefix_from_name(name)
 
-        # If tool name is unprefixed, resolve its server so we can enforce permissions
-        if not server_name:
-            mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
-            if mcp_server:
-                server_name = mcp_server.name
+        # Resolve the actual MCP server up-front so the permission check uses
+        # the canonical server.name even when the tool name is prefixed with a
+        # short ID (LITELLM_USE_SHORT_MCP_TOOL_PREFIX) that doesn't match the
+        # server's display name directly.
+        mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name)
+        if mcp_server is not None:
+            server_name = mcp_server.name
 
         # Only enforce server-level permissions when we can resolve a server
         if server_name:

litellm/proxy/_experimental/mcp_server/utils.py

@@ -2,10 +2,11 @@
 MCP Server Utilities
 """
 
-from typing import Any, Dict, Mapping, Optional, Tuple
+from typing import Any, Dict, Iterator, Mapping, Optional, Tuple
 
-import os
+import hashlib
 import importlib
+import os
 
 # Constants
 LITELLM_MCP_SERVER_NAME = "litellm-mcp-server"
@@ -14,6 +15,89 @@ LITELLM_MCP_SERVER_DESCRIPTION = "MCP Server for LiteLLM"
 MCP_TOOL_PREFIX_SEPARATOR = os.environ.get("MCP_TOOL_PREFIX_SEPARATOR", "-")
 MCP_TOOL_PREFIX_FORMAT = "{server_name}{separator}{tool_name}"
 
+# ---------------------------------------------------------------------------
+# Short-ID tool prefix (opt-in)
+# ---------------------------------------------------------------------------
+# When LITELLM_USE_SHORT_MCP_TOOL_PREFIX is truthy the prefix attached to MCP
+# tool / prompt / resource / resource-template names switches from the
+# (potentially long) human-readable server name to a deterministic three
+# character ID derived from the server's ``server_id``.
+#
+# Why three characters?
+#   * The first character is restricted to 52 alphabetic characters
+#     ([A-Za-z]) and the remaining two characters use the full base62
+#     alphabet ([0-9A-Za-z]).  That guarantees the prefix never starts
+#     with a digit so it remains a valid identifier for every model API
+#     (some providers historically required a leading alphabetic char).
+#   * 52 * 62 * 62 = 199_888 distinct IDs.  The chance of a real local
+#     tool name happening to begin with the exact prefix LiteLLM assigned
+#     to a given MCP server is negligible in practice.
+#   * The IDs are short enough that prefixed tool names stay well under
+#     the 60-character upper bound enforced by some model APIs (Anthropic
+#     etc.) even for long upstream tool names.
+#   * The mapping is deterministic (SHA-256 of ``server_id`` → three
+#     characters drawn from the alphabets above), so the prefix is stable
+#     across processes, workers and restarts without any persistence
+#     layer.  Two servers with different ``server_id`` values can in
+#     principle hash to the same three chars; that natural-hash collision
+#     IS a routing-correctness issue (the second registrant would otherwise
+#     have its tools misrouted to the first), so registration goes through
+#     ``MCPServerManager._assign_unique_short_prefix`` which rehashes with
+#     a deterministic attempt counter until it finds an unused prefix and
+#     caches the result on ``MCPServer.short_prefix``.  A collision is
+#     logged at INFO when it happens.
+#
+# This flag is intentionally opt-in for the first release so customers can
+# migrate.  It will become the default in a future release.
+SHORT_MCP_TOOL_PREFIX_LENGTH = 3
+_BASE62_ALPHABET = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+# Subset of _BASE62_ALPHABET used for the *first* character only, to
+# guarantee the prefix never starts with a digit.
+_BASE52_ALPHA_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
+
+
+def is_short_mcp_tool_prefix_enabled() -> bool:
+    """Return True when the short-ID tool prefix mode is enabled.
+
+    Read at call time (not import time) so tests and runtime config changes
+    take effect without reimporting the module.
+    """
+    raw = os.environ.get("LITELLM_USE_SHORT_MCP_TOOL_PREFIX", "")
+    return raw.strip().lower() in ("1", "true", "yes", "on")
+
+
+def compute_short_server_prefix(server_id: str, attempt: int = 0) -> str:
+    """Derive the deterministic three-character prefix for a server.
+
+    Uses SHA-256 of ``f"{server_id}#{attempt}"`` and folds the first eight
+    bytes into a fixed-length string whose first character is drawn from
+    ``_BASE52_ALPHA_ALPHABET`` (so the prefix never starts with a digit)
+    and whose remaining characters are drawn from the full base62
+    alphabet.  Pass ``attempt > 0`` to rehash to a different prefix when
+    the natural hash collides with a prefix already assigned to another
+    server (see ``MCPServerManager._assign_unique_short_prefix``).  An
+    empty ``server_id`` raises ``ValueError`` — short prefixes require a
+    stable identifier to be deterministic.
+    """
+    if not server_id:
+        raise ValueError("compute_short_server_prefix requires a non-empty server_id")
+
+    seed = server_id if attempt == 0 else f"{server_id}#{attempt}"
+    digest = hashlib.sha256(seed.encode("utf-8")).digest()
+    value = int.from_bytes(digest[:8], "big")
+
+    # Build chars from least-significant to most-significant; we reverse
+    # at the end so the first emitted char comes from the high-order
+    # bits of the digest (which is the position we constrain to be
+    # alphabetic).
+    chars = []
+    for position in range(SHORT_MCP_TOOL_PREFIX_LENGTH):
+        is_first_char = position == SHORT_MCP_TOOL_PREFIX_LENGTH - 1
+        alphabet = _BASE52_ALPHA_ALPHABET if is_first_char else _BASE62_ALPHABET
+        value, idx = divmod(value, len(alphabet))
+        chars.append(alphabet[idx])
+    return "".join(reversed(chars))
+
 
 def is_mcp_available() -> bool:
     """
@@ -82,7 +166,25 @@ def add_server_prefix_to_name(name: str, server_name: str) -> str:
 
 
 def get_server_prefix(server: Any) -> str:
-    """Return the prefix for a server: alias if present, else server_name, else server_id"""
+    """Return the prefix for a server.
+
+    When the short-prefix mode is enabled (``LITELLM_USE_SHORT_MCP_TOOL_PREFIX``)
+    a three-character base62 ID is returned.  We prefer the cached
+    ``server.short_prefix`` value when set — that field is populated at
+    registration time by ``MCPServerManager._assign_unique_short_prefix``
+    and resolves natural-hash collisions deterministically — and only fall
+    back to the natural hash for ad-hoc / temp-server objects without a
+    cached value.  In default mode the historical behaviour is preserved:
+    alias if present, else server_name, else server_id.
+    """
+    if is_short_mcp_tool_prefix_enabled():
+        cached = getattr(server, "short_prefix", None)
+        if cached:
+            return cached
+        server_id = getattr(server, "server_id", None)
+        if server_id:
+            return compute_short_server_prefix(server_id)
+
     if hasattr(server, "alias") and server.alias:
         return server.alias
     if hasattr(server, "server_name") and server.server_name:
@@ -92,6 +194,36 @@ def get_server_prefix(server: Any) -> str:
     return ""
 
 
+def iter_known_server_prefixes(server: Any) -> Iterator[str]:
+    """Yield every prefix form that may appear in tool names for ``server``.
+
+    Always includes the *current* prefix returned by ``get_server_prefix``.
+    Additionally yields the historical (alias / server_name / server_id) and
+    short-ID forms so the routing layer can resolve tool names regardless of
+    which prefix mode was active when the client first observed them.
+    """
+    seen = set()
+
+    def _emit(value: Optional[str]) -> Iterator[str]:
+        if value and value not in seen:
+            seen.add(value)
+            yield value
+
+    yield from _emit(get_server_prefix(server))
+    yield from _emit(getattr(server, "short_prefix", None))
+
+    server_id = getattr(server, "server_id", None)
+    if server_id:
+        try:
+            yield from _emit(compute_short_server_prefix(server_id))
+        except ValueError:
+            pass
+
+    yield from _emit(getattr(server, "alias", None))
+    yield from _emit(getattr(server, "server_name", None))
+    yield from _emit(server_id)
+
+
 def split_server_prefix_from_name(prefixed_name: str) -> Tuple[str, str]:
     """Return the unprefixed name plus the server name used as prefix."""
     if MCP_TOOL_PREFIX_SEPARATOR in prefixed_name:

litellm/proxy/_experimental/out/chat.txt

@@ -1,22 +0,0 @@
-1:"$Sreact.fragment"
-2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
-7:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"]
-a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
-b:"$Sreact.suspense"
-d:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"]
-f:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
-11:I[168027,[],"default"]
-:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
-:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"P":null,"b":"zxkD4-EPlgfKHDTw8O869","c":["","chat"],"q":"","i":false,"f":[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true,"nonce":"$undefined"}]],["$","$La",null,{"children":["$","$b",null,{"name":"Next.MetadataOutlet","children":"$@c"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$Ld",null,{"children":"$Le"}],["$","div",null,{"hidden":true,"children":["$","$Lf",null,{"children":["$","$b",null,{"name":"Next.Metadata","children":"$L10"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$11",[]],"S":true}
-8:{}
-9:"$0:f:0:1:1:children:1:children:0:props:children:0:props:serverProvidedParams:params"
-e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
-12:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
-c:null
-10:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L12","4",{}]]

litellm/proxy/_experimental/out/chat/__next._full.txt

@@ -1,22 +0,0 @@
-1:"$Sreact.fragment"
-2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-6:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
-7:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"]
-a:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
-b:"$Sreact.suspense"
-d:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"]
-f:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
-11:I[168027,[],"default"]
-:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
-:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"P":null,"b":"zxkD4-EPlgfKHDTw8O869","c":["","chat"],"q":"","i":false,"f":[[["",{"children":["chat",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[null,["$","$L4",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","forbidden":"$undefined","unauthorized":"$undefined"}]]}],{"children":[["$","$1","c",{"children":[["$","$L6",null,{"Component":"$7","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@8","$@9"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true,"nonce":"$undefined"}]],["$","$La",null,{"children":["$","$b",null,{"name":"Next.MetadataOutlet","children":"$@c"}]}]]}],{},null,false,false]},null,false,false]},null,false,false],["$","$1","h",{"children":[null,["$","$Ld",null,{"children":"$Le"}],["$","div",null,{"hidden":true,"children":["$","$Lf",null,{"children":["$","$b",null,{"name":"Next.Metadata","children":"$L10"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],false]],"m":"$undefined","G":["$11",[]],"S":true}
-8:{}
-9:"$0:f:0:1:1:children:1:children:0:props:children:0:props:serverProvidedParams:params"
-e:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]
-12:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
-c:null
-10:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L12","4",{}]]

litellm/proxy/_experimental/out/chat/__next._head.txt

@@ -1,6 +0,0 @@
-1:"$Sreact.fragment"
-2:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"]
-3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"]
-4:"$Sreact.suspense"
-5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"]
-0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false}

litellm/proxy/_experimental/out/chat/__next._index.txt

@@ -1,8 +0,0 @@
-1:"$Sreact.fragment"
-2:I[867271,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-3:I[71195,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js"],"default"]
-4:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-5:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
-0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]]}],"loading":null,"isPartial":false}

litellm/proxy/_experimental/out/chat/__next._tree.txt

@@ -1,4 +0,0 @@
-:HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"]
-:HL["/litellm-asset-prefix/_next/static/chunks/e0cb6755699177c1.css","style"]
-:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
-0:{"buildId":"zxkD4-EPlgfKHDTw8O869","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"chat","paramType":null,"paramKey":"chat","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":false}},"isRootLayout":true},"staleTime":300}

litellm/proxy/_experimental/out/chat/__next.chat.__PAGE__.txt

@@ -1,9 +0,0 @@
-1:"$Sreact.fragment"
-2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"]
-3:I[321443,["/litellm-asset-prefix/_next/static/chunks/9e09de50158b3159.js","/litellm-asset-prefix/_next/static/chunks/7e5fe5584502da06.js","/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js"],"default"]
-6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"]
-7:"$Sreact.suspense"
-0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0a6c418370a8c183.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/cf9c81fc7166f4d4.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/00ff280cdb7d7ee5.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/a1abfc2f35c701cc.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/4980372eaa37b78b.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/119ecee91f911bc8.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/ac3cf77acb5bf234.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/fcdf7322b0aa3e2e.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/89aa55578de861b7.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false}
-4:{}
-5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
-8:null

litellm/proxy/_experimental/out/chat/__next.chat.txt

@@ -1,4 +0,0 @@
-1:"$Sreact.fragment"
-2:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-3:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"]
-0:{"buildId":"zxkD4-EPlgfKHDTw8O869","rsc":["$","$1","c",{"children":[null,["$","$L2",null,{"parallelRouterKey":"children","template":["$","$L3",null,{}]}]]}],"loading":null,"isPartial":false}

litellm/proxy/_types.py

@@ -904,6 +904,7 @@ class LiteLLM_ObjectPermissionBase(LiteLLMPydanticObjectBase):
     agents: Optional[List[str]] = None
     agent_access_groups: Optional[List[str]] = None
     models: Optional[List[str]] = None
+    search_tools: Optional[List[str]] = None
 
 
 class BudgetLimitEntry(LiteLLMPydanticObjectBase):
@@ -1934,6 +1935,7 @@ class LiteLLM_ObjectPermissionTable(LiteLLMPydanticObjectBase):
     agent_access_groups: Optional[List[str]] = []
     mcp_toolsets: Optional[List[str]] = None
     blocked_tools: Optional[List[str]] = []
+    search_tools: Optional[List[str]] = []
 
 
 class LiteLLM_TeamTable(TeamBase):

litellm/proxy/auth/auth_checks.py

@@ -376,7 +376,7 @@ def _guardrail_modification_check(
         coerced = _coerce_to_dict(container)
         if coerced is None:
             return False
-        return any(coerced.get(key) for key in _GUARDRAIL_MODIFICATION_KEYS)
+        return any(key in coerced for key in _GUARDRAIL_MODIFICATION_KEYS)
 
     # Check both metadata keys — callers can populate either depending on the
     # endpoint. Cover the top-level too so root-level injection is rejected.
@@ -922,7 +922,8 @@ async def get_team_member_default_budget(
     Fetches the team-level default per-member budget referenced by team.metadata["team_member_budget_id"].
 
     This budget is applied to team members whose TeamMembership row has no
-    linked budget. Results are cached for performance.
+    linked budget, or whose linked budget has max_budget=NULL. Results are
+    cached for performance.
 
     Args:
         budget_id: The budget_id pulled from team.metadata["team_member_budget_id"]
@@ -2995,6 +2996,116 @@ async def can_user_call_model(
     )
 
 
+def _search_tool_names_from_object_permission(
+    object_permission: Optional[LiteLLM_ObjectPermissionTable],
+) -> List[str]:
+    """Return allowlisted search tool names from object_permission (empty = unrestricted)."""
+    if object_permission is None:
+        return []
+    raw = object_permission.search_tools
+    if not raw:
+        return []
+    return list(raw)
+
+
+def _can_object_call_search_tool(
+    search_tool_name: str,
+    allowed_search_tools: List[str],
+    object_type: Literal["key", "team", "project"],
+) -> Literal[True]:
+    """
+    Check if an object (key/team/project) can access a specific search tool.
+
+    Similar to _can_object_call_model but for search tools.
+
+    Args:
+        search_tool_name: The search tool being requested
+        allowed_search_tools: List of allowed search tool names for this object
+        object_type: Type of object for error messaging
+
+    Returns:
+        True if access is allowed
+
+    Raises:
+        ProxyException if access is denied
+    """
+    # Empty list means all search tools are allowed
+    if not allowed_search_tools:
+        return True
+
+    # Check if the search tool is in the allowlist
+    if search_tool_name in allowed_search_tools:
+        return True
+
+    # Access denied
+    raise ProxyException(
+        message=f"{object_type.capitalize()} not allowed to access search tool: {search_tool_name}. "
+        f"Allowed search tools: {allowed_search_tools}",
+        type=ProxyErrorTypes.key_model_access_denied,
+        param="search_tool_name",
+        code=status.HTTP_403_FORBIDDEN,
+    )
+
+
+async def can_key_call_search_tool(
+    search_tool_name: str,
+    valid_token: UserAPIKeyAuth,
+) -> Literal[True]:
+    """
+    Check if a key can access a specific search tool.
+
+    Similar to can_key_call_model but for search tools.
+
+    Args:
+        search_tool_name: The search tool being requested
+        valid_token: The authenticated key
+
+    Returns:
+        True if access is allowed
+
+    Raises:
+        ProxyException if access is denied
+    """
+    return _can_object_call_search_tool(
+        search_tool_name=search_tool_name,
+        allowed_search_tools=_search_tool_names_from_object_permission(
+            valid_token.object_permission
+        ),
+        object_type="key",
+    )
+
+
+async def can_team_call_search_tool(
+    search_tool_name: str,
+    team_object: Optional[LiteLLM_TeamTable],
+) -> Literal[True]:
+    """
+    Check if a team can access a specific search tool.
+
+    Similar to can_team_access_model but for search tools.
+
+    Args:
+        search_tool_name: The search tool being requested
+        team_object: The team object
+
+    Returns:
+        True if access is allowed
+
+    Raises:
+        ProxyException if access is denied
+    """
+    if team_object is None:
+        return True
+
+    return _can_object_call_search_tool(
+        search_tool_name=search_tool_name,
+        allowed_search_tools=_search_tool_names_from_object_permission(
+            team_object.object_permission
+        ),
+        object_type="team",
+    )
+
+
 async def is_valid_fallback_model(
     model: str,
     llm_router: Optional[Router],
@@ -3326,6 +3437,7 @@ async def _check_team_member_budget(
         if (
             team_membership is not None
             and team_membership.litellm_budget_table is not None
+            and team_membership.litellm_budget_table.max_budget is not None
         ):
             team_member_budget = team_membership.litellm_budget_table.max_budget
         else:

litellm/proxy/auth/auth_utils.py

@@ -6,9 +6,11 @@ from typing import Any, List, Optional, Tuple
 
 from fastapi import HTTPException, Request, status
 
+import litellm
 from litellm import Router, provider_list
 from litellm._logging import verbose_proxy_logger
 from litellm.constants import STANDARD_CUSTOMER_ID_HEADERS
+from litellm.litellm_core_utils.url_utils import SSRFError, validate_url
 from litellm.proxy._types import *
 from litellm.types.router import CONFIGURABLE_CLIENTSIDE_AUTH_PARAMS
 
@@ -53,6 +55,12 @@ def _check_valid_ip(
 def check_complete_credentials(request_body: dict) -> bool:
     """
     if 'api_base' in request body. Check if complete credentials given. Prevent malicious attacks.
+
+    Supplying an ``api_key`` is necessary but not sufficient: even with
+    credentials supplied, an ``api_base`` / ``base_url`` that resolves to a
+    private/internal/cloud-metadata address would still allow the proxy to
+    be used as an SSRF pivot. Validate any URL fields here so the gate
+    can't be bypassed with ``api_key=anything`` plus a malicious target.
     """
     given_model: Optional[str] = None
 
@@ -70,10 +78,27 @@ def check_complete_credentials(request_body: dict) -> bool:
         return False
 
     api_key_value = request_body.get("api_key")
-    if api_key_value and isinstance(api_key_value, str) and api_key_value.strip():
-        return True
+    if not (api_key_value and isinstance(api_key_value, str) and api_key_value.strip()):
+        return False
 
-    return False
+    # ``validate_url`` itself doesn't consult the toggle; ``safe_get`` /
+    # ``async_safe_get`` do. Mirror that here so admins who explicitly
+    # disabled URL validation (e.g. for an internal Ollama endpoint they
+    # accept the SSRF risk for) aren't blocked at the proxy boundary.
+    if getattr(litellm, "user_url_validation", False):
+        for url_field in ("api_base", "base_url"):
+            url_value = request_body.get(url_field)
+            if not url_value or not isinstance(url_value, str):
+                continue
+            try:
+                validate_url(url_value)
+            except SSRFError as e:
+                raise ValueError(
+                    f"Rejected request: client-side {url_field}={url_value!r} "
+                    f"is rejected by the SSRF guard ({e})."
+                )
+
+    return True
 
 
 def check_regex_or_str_match(request_body_value: Any, regex_str: str) -> bool:
@@ -159,15 +184,42 @@ def is_request_body_safe(
         "aws_web_identity_token",
         "aws_role_name",
         "vertex_credentials",
+        # Endpoint-targeting fields that retarget the outbound request or
+        # an observability callback. An attacker-controlled value either
+        # exfiltrates the request payload (incl. messages + admin-set
+        # tokens) to the attacker's host, or coerces the proxy into
+        # authenticating against the attacker's host with admin secrets.
+        "aws_bedrock_runtime_endpoint",
+        "langsmith_base_url",
+        "langfuse_host",
+        "posthog_host",
+        "braintrust_host",
+        "slack_webhook_url",
+        # Provider-specific endpoint overrides that flow into the outbound
+        # request via ``optional_params``. Same threat as ``api_base``:
+        # ``s3_endpoint_url`` redirects Bedrock file uploads to attacker
+        # S3; ``sagemaker_base_url`` redirects all SageMaker traffic;
+        # ``deployment_url`` redirects SAP deployments.
+        "s3_endpoint_url",
+        "sagemaker_base_url",
+        "deployment_url",
     ]
 
+    # The blocklist is enforced unconditionally. Legitimate clientside
+    # credential / endpoint passthrough goes through one of the two
+    # explicit admin opt-ins (``general_settings.allow_client_side_credentials``
+    # proxy-wide or ``configurable_clientside_auth_params`` per deployment).
+    # Historically there was a third, *implicit*, *caller-controlled* path:
+    # ``check_complete_credentials`` returned True when the caller supplied
+    # any non-empty ``api_key``, which made the entire blocklist a no-op.
+    # That bypass turned every missing entry on the blocklist into an
+    # exploitable SSRF / credential-exfil hole — see GHSA-jh89-88fc-qrfp,
+    # GHSA-3frq-6r6h-7j64, and the chain of veria-admin findings (Dv_m860l,
+    # b_yRJeQ5, stN90yjP, LBlyOAc8, U2TD78kg). Removed: the blocklist now
+    # has a single, predictable failure mode for missing entries (a 400),
+    # not a credential leak.
     for param in banned_params:
-        if (
-            param in request_body
-            and not check_complete_credentials(  # allow client-credentials to be passed to proxy
-                request_body=request_body
-            )
-        ):
+        if param in request_body:
             if general_settings.get("allow_client_side_credentials") is True:
                 return True
             elif (
@@ -182,7 +234,10 @@ def is_request_body_safe(
                 return True
             raise ValueError(
                 f"Rejected Request: {param} is not allowed in request body. "
-                "Enable with `general_settings::allow_client_side_credentials` on proxy config.yaml. "
+                "Clientside passthrough requires explicit admin opt-in via "
+                "either `general_settings.allow_client_side_credentials = true` "
+                "(proxy-wide) or `configurable_clientside_auth_params` on the "
+                "deployment in your proxy config.yaml. "
                 "Relevant Issue: https://huntr.com/bounties/4001e1a2-7b7a-4776-a3ae-e6692ec3d997",
             )
 

litellm/proxy/auth/user_api_key_auth.py

@@ -20,6 +20,8 @@ from fastapi.security.api_key import APIKeyHeader
 import litellm
 from litellm._logging import verbose_logger, verbose_proxy_logger
 from litellm._service_logger import ServiceLogging
+from litellm.caching import DualCache
+from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
 from litellm.litellm_core_utils.dd_tracing import tracer
 from litellm.litellm_core_utils.dot_notation_indexing import get_nested_value
 from litellm.proxy._types import *
@@ -1117,10 +1119,14 @@ async def _user_api_key_auth_builder(  # noqa: PLR0915
             )
 
         if is_master_key_valid:
+            # Substitute a stable alias for the raw master key so neither the
+            # master key nor its hash propagates into spend logs, Prometheus
+            # /metrics labels, audit trails, rate-limit buckets, or any other
+            # downstream consumer of UserAPIKeyAuth.api_key.
             _user_api_key_obj = await _return_user_api_key_auth_obj(
                 user_obj=None,
                 user_role=LitellmUserRoles.PROXY_ADMIN,
-                api_key=master_key,
+                api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
                 parent_otel_span=parent_otel_span,
                 valid_token_dict={
                     **end_user_params,

litellm/proxy/batches_endpoints/endpoints.py

@@ -474,6 +474,10 @@ async def retrieve_batch(  # noqa: PLR0915
             )
             # Fix: The helper sets "file_id" but we need "batch_id"
             data["batch_id"] = data.pop("file_id", original_batch_id)
+            # Provider-config providers (e.g. bedrock) require `model` in kwargs
+            # so litellm.aretrieve_batch can load BedrockBatchesConfig. Without
+            # it the call falls into the legacy provider switch and 400s.
+            data["model"] = model_from_id
 
             # Retrieve batch using model credentials
             response = await litellm.aretrieve_batch(

litellm/proxy/common_request_processing.py

@@ -619,6 +619,67 @@ class ProxyBaseLLMRequestProcessing:
             verbose_proxy_logger.error(f"Error setting custom headers: {e}")
             return {}
 
+    @staticmethod
+    async def build_litellm_proxy_success_headers_from_llm_response(
+        *,
+        response: Any,
+        request_data: dict,
+        request: Request,
+        user_api_key_dict: UserAPIKeyAuth,
+        logging_obj: LiteLLMLoggingObj,
+        version: Optional[str],
+        proxy_logging_obj: ProxyLogging,
+    ) -> Dict[str, str]:
+        """
+        Build LiteLLM proxy response headers for routes that call the LLM directly
+        (e.g. Google native :generateContent) instead of base_process_llm_request.
+        """
+        if isinstance(response, dict):
+            hidden_params = response.get("_hidden_params") or {}
+        else:
+            hidden_params = getattr(response, "_hidden_params", None) or {}
+        if not isinstance(hidden_params, dict):
+            hidden_params = {}
+
+        model_id = ProxyBaseLLMRequestProcessing._get_model_id_from_response(
+            hidden_params, request_data
+        )
+
+        cache_key = hidden_params.get("cache_key", None) or ""
+        api_base = hidden_params.get("api_base", None) or ""
+        response_cost = hidden_params.get("response_cost", None) or ""
+        fastest_response_batch_completion = hidden_params.get(
+            "fastest_response_batch_completion", None
+        )
+        additional_headers = hidden_params.get("additional_headers", {}) or {}
+
+        custom_headers = ProxyBaseLLMRequestProcessing.get_custom_headers(
+            user_api_key_dict=user_api_key_dict,
+            call_id=logging_obj.litellm_call_id,
+            model_id=model_id,
+            cache_key=cache_key,
+            api_base=api_base,
+            version=version,
+            response_cost=response_cost,
+            model_region=getattr(user_api_key_dict, "allowed_model_region", ""),
+            fastest_response_batch_completion=fastest_response_batch_completion,
+            request_data=request_data,
+            hidden_params=hidden_params,
+            litellm_logging_obj=logging_obj,
+            **additional_headers,
+        )
+
+        callback_headers = await proxy_logging_obj.post_call_response_headers_hook(
+            data=request_data,
+            user_api_key_dict=user_api_key_dict,
+            response=response,
+            request_headers=dict(request.headers),
+        )
+        if callback_headers:
+            custom_headers.update(callback_headers)
+
+        return custom_headers
+
     async def common_processing_pre_call_logic(
         self,
         request: Request,
@@ -875,7 +936,7 @@ class ProxyBaseLLMRequestProcessing:
         else:
             verbose_proxy_logger.debug(
                 "Request received by LiteLLM:\n%s",
-                json.dumps(self.data, indent=4, default=str),
+                _payload_str,
             )
 
     async def base_process_llm_request(  # noqa: PLR0915
@@ -1511,9 +1572,7 @@ class ProxyBaseLLMRequestProcessing:
         _response = assembled_response
         try:
             from litellm.proxy.proxy_server import llm_router as _global_llm_router
-            from litellm.proxy.utils import (
-                _check_and_merge_model_level_guardrails,
-            )
+            from litellm.proxy.utils import _check_and_merge_model_level_guardrails
 
             guardrail_data = _check_and_merge_model_level_guardrails(
                 data=captured_data, llm_router=_global_llm_router
@@ -1690,11 +1749,12 @@ class ProxyBaseLLMRequestProcessing:
         elif isinstance(e, httpx.HTTPStatusError):
             # Handle httpx.HTTPStatusError - extract actual error from response
             # This matches the original behavior before the refactor in commit 511d435f6f
-            error_body = await e.response.aread()
+            http_status_error: httpx.HTTPStatusError = e
+            error_body = await http_status_error.response.aread()
             error_text = error_body.decode("utf-8")
 
             raise HTTPException(
-                status_code=e.response.status_code,
+                status_code=http_status_error.response.status_code,
                 detail={"error": error_text},
             )
         error_msg = f"{str(e)}"

litellm/proxy/db/exception_handler.py

@@ -1,5 +1,6 @@
-from typing import Union
+from typing import Any, Awaitable, Callable, Optional, Union
 
+from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import (
     DB_CONNECTION_ERROR_TYPES,
     ProxyErrorTypes,
@@ -123,3 +124,138 @@ class PrismaDBExceptionHandler:
         ):
             return None
         raise e
+
+
+# Default fallback timeouts when neither the caller nor the prisma_client
+# expose `_db_auth_reconnect_timeout_seconds` / `_db_auth_reconnect_lock_timeout_seconds`.
+# Match the auth path's existing defaults so behavior is uniform across read paths.
+_DEFAULT_RECONNECT_TIMEOUT_SECONDS = 2.0
+_DEFAULT_RECONNECT_LOCK_TIMEOUT_SECONDS = 0.1
+
+
+def _coerce_timeout(value: Any, fallback: float) -> float:
+    """Return `value` if it is a real int/float, else `fallback`. Guards
+    against tests that mock `prisma_client` and leave the timeout slots as
+    MagicMock instances."""
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return float(value)
+    return fallback
+
+
+async def call_with_db_reconnect_retry(
+    prisma_client: Any,
+    coro_factory: Callable[[], Awaitable[Any]],
+    *,
+    reason: str,
+    timeout_seconds: Optional[float] = None,
+    lock_timeout_seconds: Optional[float] = None,
+) -> Any:
+    """Run a Prisma read coroutine with one transport-reconnect-and-retry.
+
+    The canonical "self-heal a transient DB transport blip" wrapper used by
+    `PrismaClient.get_generic_data` and other read paths. Mirrors the inline
+    pattern in `auth_checks._fetch_key_object_from_db_with_reconnect` so we
+    have a single implementation rather than three drifting copies.
+
+    Behavior:
+      1. Await `coro_factory()`. On success, return its value.
+      2. On exception, if it is NOT a transport error (per
+         `is_database_transport_error`), re-raise — data-layer errors like
+         `UniqueViolationError` mean the DB is reachable, reconnect would be
+         pointless.
+      3. If `prisma_client` does not expose `attempt_db_reconnect`, re-raise.
+         This guards against partial stand-ins / older clients in tests.
+      4. Call `prisma_client.attempt_db_reconnect(reason=...)`. If it returns
+         False (cooldown / lock contention / reconnect failure), re-raise.
+      5. Otherwise await `coro_factory()` a second time and return / propagate
+         its result. At-most-one retry by construction — no infinite loop.
+
+    `coro_factory` MUST be a zero-arg callable that returns a fresh awaitable
+    on each call. Passing an already-awaited coroutine would fail on retry
+    with `RuntimeError: cannot reuse already awaited coroutine`.
+
+    `reason` should follow `<subsystem>_<operation>_<table>_failure` so
+    telemetry distinguishes between fan-out callers (e.g.
+    `_update_config_from_db` issues four concurrent reads).
+
+    Args:
+        prisma_client: The `PrismaClient` (or stand-in) that owns
+            `attempt_db_reconnect` and the `_db_auth_reconnect_*` defaults.
+        coro_factory: Zero-arg callable returning the read awaitable.
+        reason: Telemetry tag forwarded to `attempt_db_reconnect`.
+        timeout_seconds: Optional override for the reconnect cycle timeout.
+            Defaults to `prisma_client._db_auth_reconnect_timeout_seconds`,
+            then to 2.0s.
+        lock_timeout_seconds: Optional override for how long the helper will
+            wait to acquire the reconnect lock. Defaults to
+            `prisma_client._db_auth_reconnect_lock_timeout_seconds`, then to
+            0.1s.
+
+    Returns:
+        Whatever `coro_factory()` returns (on first or second attempt).
+
+    Raises:
+        Whatever `coro_factory()` raises if the failure is not a transport
+        error, or if the reconnect attempt does not succeed, or if the retry
+        also fails.
+    """
+    try:
+        return await coro_factory()
+    except Exception as first_exc:
+        if not PrismaDBExceptionHandler.is_database_transport_error(first_exc):
+            raise
+        if not hasattr(prisma_client, "attempt_db_reconnect"):
+            raise
+
+        resolved_timeout = _coerce_timeout(
+            (
+                timeout_seconds
+                if timeout_seconds is not None
+                else getattr(prisma_client, "_db_auth_reconnect_timeout_seconds", None)
+            ),
+            _DEFAULT_RECONNECT_TIMEOUT_SECONDS,
+        )
+        resolved_lock_timeout = _coerce_timeout(
+            (
+                lock_timeout_seconds
+                if lock_timeout_seconds is not None
+                else getattr(
+                    prisma_client, "_db_auth_reconnect_lock_timeout_seconds", None
+                )
+            ),
+            _DEFAULT_RECONNECT_LOCK_TIMEOUT_SECONDS,
+        )
+
+        verbose_proxy_logger.warning(
+            "DB transport error on read; attempting reconnect-and-retry. reason=%s error=%s",
+            reason,
+            first_exc,
+        )
+
+        # Preserve the original transport error in telemetry. If
+        # `attempt_db_reconnect` itself raises (e.g. lock cancellation, timer
+        # error, unexpected internal failure), surfacing that exception
+        # instead of `first_exc` would mask the actual DB transport problem
+        # in `failure_handler` / `db_exceptions` alerts. Chain the reconnect
+        # error as the cause for debuggability without losing the original.
+        try:
+            did_reconnect = await prisma_client.attempt_db_reconnect(
+                reason=reason,
+                timeout_seconds=resolved_timeout,
+                lock_timeout_seconds=resolved_lock_timeout,
+            )
+        except Exception as reconnect_exc:
+            verbose_proxy_logger.warning(
+                "DB reconnect attempt raised; preserving original transport error. "
+                "reason=%s reconnect_error=%s",
+                reason,
+                reconnect_exc,
+            )
+            raise first_exc from reconnect_exc
+        if not did_reconnect:
+            raise
+
+        # At most one retry. If the retry also raises a transport error, we
+        # propagate — repeated reconnect-loops are the watchdog's job, not
+        # this helper's.
+        return await coro_factory()

litellm/proxy/db/prisma_client.py

@@ -52,18 +52,25 @@ class PrismaWrapper:
             engine = self._original_prisma._engine
             process = getattr(engine, "process", None) if engine is not None else None
             if process is not None:
-                return process.pid
+                pid = process.pid
+                if isinstance(pid, int):
+                    return pid
         except (AttributeError, TypeError):
             pass
         return 0
 
     @staticmethod
     async def _kill_engine_process(pid: int) -> None:
-        """Force-kill an orphaned engine subprocess to prevent DB connection pool leaks.
+        """Force-kill the engine subprocess to prevent DB connection pool leaks.
 
-        Called when disconnect() fails and the old engine process may still be
-        holding open connections. Sends SIGTERM for graceful shutdown, waits
-        briefly, then SIGKILL as a backstop.
+        Called on every reconnect (in `recreate_prisma_client`) to retire the
+        old query-engine subprocess without invoking prisma-client-py's
+        synchronous `disconnect()` — which blocks the asyncio event loop on
+        `subprocess.Popen.wait()` for 30-120+ seconds when the engine is
+        stuck on TCP close.
+
+        Sends SIGTERM for graceful shutdown, waits briefly, then SIGKILL as
+        a backstop.
         """
         if pid <= 0:
             return
@@ -72,7 +79,7 @@ class PrismaWrapper:
         except (ProcessLookupError, PermissionError, OSError):
             return  # Already dead or inaccessible
         verbose_proxy_logger.warning(
-            "Sent SIGTERM to orphaned prisma-query-engine PID %s after failed disconnect.",
+            "Sent SIGTERM to prisma-query-engine PID %s during reconnect.",
             pid,
         )
         # Brief wait for graceful shutdown, then force-kill
@@ -217,15 +224,18 @@ class PrismaWrapper:
     async def recreate_prisma_client(
         self, new_db_url: str, http_client: Optional[Any] = None
     ):
-        """Disconnect and reconnect the Prisma client with a new database URL."""
+        """Disconnect and reconnect the Prisma client with a new database URL.
+
+        Kills the old engine subprocess directly (SIGTERM → SIGKILL) rather than
+        calling `disconnect()`. prisma-client-py's `disconnect()` calls a
+        synchronous `subprocess.Popen.wait()` that can freeze the asyncio event
+        loop for 30-120+ seconds when the engine is stuck on TCP close,
+        breaking `/health/liveliness` and causing Kubernetes pod restarts.
+        """
         from prisma import Prisma  # type: ignore
 
         old_engine_pid = self._get_engine_pid()
-
-        try:
-            await self._original_prisma.disconnect()
-        except Exception as e:
-            verbose_proxy_logger.warning(f"Failed to disconnect Prisma client: {e}")
+        if old_engine_pid > 0:
             await self._kill_engine_process(old_engine_pid)
 
         if http_client is not None:

litellm/proxy/google_endpoints/endpoints.py

@@ -35,6 +35,7 @@ async def google_generate_content(
         general_settings,
         llm_router,
         proxy_config,
+        proxy_logging_obj,
         version,
     )
 
@@ -73,6 +74,16 @@ async def google_generate_content(
     if llm_router is None:
         raise HTTPException(status_code=500, detail="Router not initialized")
     response = await llm_router.agenerate_content(**data)
+    success_headers = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response(
+        response=response,
+        request_data=data,
+        request=request,
+        user_api_key_dict=user_api_key_dict,
+        logging_obj=logging_obj,
+        version=version,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+    fastapi_response.headers.update(success_headers)
     return response
 
 
@@ -95,6 +106,7 @@ async def google_stream_generate_content(
         general_settings,
         llm_router,
         proxy_config,
+        proxy_logging_obj,
         version,
     )
 
@@ -137,9 +149,24 @@ async def google_stream_generate_content(
         raise HTTPException(status_code=500, detail="Router not initialized")
     response = await llm_router.agenerate_content_stream(**data)
 
+    success_headers = await ProxyBaseLLMRequestProcessing.build_litellm_proxy_success_headers_from_llm_response(
+        response=response,
+        request_data=data,
+        request=request,
+        user_api_key_dict=user_api_key_dict,
+        logging_obj=logging_obj,
+        version=version,
+        proxy_logging_obj=proxy_logging_obj,
+    )
+
     # Check if response is an async iterator (streaming response)
     if response is not None and hasattr(response, "__aiter__"):
-        return StreamingResponse(content=response, media_type="text/event-stream")
+        return StreamingResponse(
+            content=response,
+            media_type="text/event-stream",
+            headers=success_headers,
+        )
+    fastapi_response.headers.update(success_headers)
     return response
 
 

litellm/proxy/management_endpoints/key_management_endpoints.py

@@ -65,6 +65,7 @@ from litellm.proxy.management_helpers.object_permission_utils import (
     attach_object_permission_to_dict,
     handle_update_object_permission_common,
     validate_key_mcp_servers_against_team,
+    validate_key_search_tools_against_team,
 )
 from litellm.proxy.management_helpers.team_member_permission_checks import (
     TeamMemberPermissionChecks,
@@ -768,6 +769,10 @@ async def _common_key_generation_helper(  # noqa: PLR0915
         object_permission=data_json.get("object_permission"),
         team_obj=team_table,
     )
+    await validate_key_search_tools_against_team(
+        object_permission=data_json.get("object_permission"),
+        team_obj=team_table,
+    )
 
     data_json = await _set_object_permission(
         data_json=data_json,
@@ -2010,6 +2015,10 @@ async def _validate_mcp_servers_for_key_update(
         object_permission=object_permission_dict,
         team_obj=effective_team_obj,
     )
+    await validate_key_search_tools_against_team(
+        object_permission=object_permission_dict,
+        team_obj=effective_team_obj,
+    )
 
 
 async def _validate_update_key_data(

litellm/proxy/management_endpoints/team_callback_endpoints.py

@@ -4,15 +4,22 @@ Endpoints to control callbacks per team
 Use this when each team should control its own callbacks
 """
 
+import asyncio
+import copy
 import json
 import traceback
-from typing import List, Optional
+from datetime import datetime, timezone
+from typing import Any, List, Optional
 
 from fastapi import APIRouter, Depends, Header, HTTPException, Request, status
 
+import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm._uuid import uuid
 from litellm.proxy._types import (
     AddTeamCallback,
+    LiteLLM_AuditLogs,
+    LitellmTableNames,
     ProxyErrorTypes,
     ProxyException,
     TeamCallbackMetadata,
@@ -24,6 +31,106 @@ from litellm.proxy.management_helpers.utils import management_endpoint_wrapper
 router = APIRouter()
 
 
+_CALLBACK_VARS_REDACTED = "***REDACTED***"
+
+
+def _redact_callback_secrets(metadata: Any) -> Any:
+    """Strip secret values out of a team-metadata snapshot before audit logging.
+
+    Both ``team_metadata["logging"]`` (list of ``AddTeamCallback`` dicts) and
+    ``team_metadata["callback_settings"]["callback_vars"]`` carry provider
+    credentials such as ``langfuse_secret_key``, ``langsmith_api_key``, and
+    ``gcs_path_service_account``.  Persisting them verbatim into
+    ``LiteLLM_AuditLogs`` would let anyone with read access to the audit
+    table harvest team callback credentials, so we replace each value with
+    a fixed marker.  The keys themselves are kept so the audit reader can
+    still see *which* fields changed.
+    """
+    if not isinstance(metadata, dict):
+        return metadata
+    redacted = copy.deepcopy(metadata)
+    logging_entries = redacted.get("logging")
+    if isinstance(logging_entries, list):
+        for entry in logging_entries:
+            if isinstance(entry, dict) and isinstance(entry.get("callback_vars"), dict):
+                entry["callback_vars"] = {
+                    k: _CALLBACK_VARS_REDACTED for k in entry["callback_vars"]
+                }
+    callback_settings = redacted.get("callback_settings")
+    if isinstance(callback_settings, dict) and isinstance(
+        callback_settings.get("callback_vars"), dict
+    ):
+        callback_settings["callback_vars"] = {
+            k: _CALLBACK_VARS_REDACTED for k in callback_settings["callback_vars"]
+        }
+    return redacted
+
+
+def _log_audit_task_exception(task: "asyncio.Task[None]") -> None:
+    """Surface a fire-and-forget audit-log task failure.
+
+    ``asyncio.create_task`` swallows exceptions silently — if the audit
+    write fails (transient DB error etc.) we'd otherwise lose the row
+    without any signal.  Log at warning level so the operator sees there's
+    a gap in the audit trail.
+    """
+    if task.cancelled():
+        return
+    exc = task.exception()
+    if exc is not None:
+        verbose_proxy_logger.warning("Failed to write team-callback audit log: %s", exc)
+
+
+async def _emit_team_callback_audit_log(
+    *,
+    team_id: str,
+    before_metadata: Any,
+    after_metadata: Any,
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_changed_by: Optional[str],
+) -> None:
+    """Emit an audit-log row for a team-callback mutation.
+
+    Mirrors the ``store_audit_logs``-gated pattern used in
+    ``team_endpoints.py``: the call is async-fire-and-forget and is a no-op
+    when audit logging is not enabled on the proxy.  Captured under
+    ``LitellmTableNames.TEAM_TABLE_NAME`` so the row co-locates with other
+    team mutations in the audit table.
+
+    Callback secrets are redacted before serialization so the audit table
+    cannot itself become a credential-harvest sink.
+    """
+    if litellm.store_audit_logs is not True:
+        return
+
+    from litellm.proxy.management_helpers.audit_logs import (
+        create_audit_log_for_update,
+    )
+    from litellm.proxy.proxy_server import litellm_proxy_admin_name
+
+    redacted_before = _redact_callback_secrets(before_metadata)
+    redacted_after = _redact_callback_secrets(after_metadata)
+
+    task = asyncio.create_task(
+        create_audit_log_for_update(
+            request_data=LiteLLM_AuditLogs(
+                id=str(uuid.uuid4()),
+                updated_at=datetime.now(timezone.utc),
+                changed_by=litellm_changed_by
+                or user_api_key_dict.user_id
+                or litellm_proxy_admin_name,
+                changed_by_api_key=user_api_key_dict.api_key,
+                table_name=LitellmTableNames.TEAM_TABLE_NAME,
+                object_id=team_id,
+                action="updated",
+                updated_values=json.dumps({"metadata": redacted_after}, default=str),
+                before_value=json.dumps({"metadata": redacted_before}, default=str),
+            )
+        )
+    )
+    task.add_done_callback(_log_audit_task_exception)
+
+
 @router.post(
     "/team/{team_id:path}/callback",
     tags=["team management"],
@@ -123,6 +230,7 @@ async def add_team_callbacks(
                     param="callback_name",
                 )
 
+        before_metadata = copy.deepcopy(team_metadata)
         team_callback_settings.append(data.model_dump())
 
         team_metadata["logging"] = team_callback_settings
@@ -132,6 +240,14 @@ async def add_team_callbacks(
             where={"team_id": team_id}, data={"metadata": team_metadata_json}  # type: ignore
         )
 
+        await _emit_team_callback_audit_log(
+            team_id=team_id,
+            before_metadata=before_metadata,
+            after_metadata=team_metadata,
+            user_api_key_dict=user_api_key_dict,
+            litellm_changed_by=litellm_changed_by,
+        )
+
         return {
             "status": "success",
             "data": new_team_row,
@@ -165,6 +281,10 @@ async def disable_team_logging(
     http_request: Request,
     team_id: str,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    litellm_changed_by: Optional[str] = Header(
+        None,
+        description="The litellm-changed-by header enables tracking of actions performed by authorized users on behalf of other users, providing an audit trail for accountability",
+    ),
 ):
     """
     Disable all logging callbacks for a team
@@ -198,6 +318,7 @@ async def disable_team_logging(
 
         # Update team metadata to disable logging
         team_metadata = _existing_team.metadata
+        before_metadata = copy.deepcopy(team_metadata)
         team_callback_settings = team_metadata.get("callback_settings", {})
         team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings)
 
@@ -222,6 +343,17 @@ async def disable_team_logging(
                 },
             )
 
+        # Disabling a team's logging callbacks is itself a logging-control
+        # action — emit an audit-log row so the action remains traceable
+        # even though the team's own observability is now off.
+        await _emit_team_callback_audit_log(
+            team_id=team_id,
+            before_metadata=before_metadata,
+            after_metadata=team_metadata,
+            user_api_key_dict=user_api_key_dict,
+            litellm_changed_by=litellm_changed_by,
+        )
+
         return {
             "status": "success",
             "message": f"Logging disabled for team {team_id}",

litellm/proxy/management_endpoints/team_endpoints.py

@@ -2003,21 +2003,34 @@ def team_member_add_duplication_check(
 async def _validate_team_member_add_permissions(
     user_api_key_dict: UserAPIKeyAuth,
     complete_team_data: LiteLLM_TeamTable,
+    data: TeamMemberAddRequest,
 ) -> None:
-    """Validate if user has permission to add members to the team."""
+    """Validate if user has permission to add members to the team.
+
+    Standard users can self-join an *available team*, but the bypass
+    must not be allowed to escalate them to ``role=admin`` or to add
+    other users into the team.  When access is granted via the
+    available-team bypass we therefore enforce that every member in
+    the request matches the caller's own ``user_id`` and is being
+    added with ``role="user"``.
+    """
     if (
-        hasattr(user_api_key_dict, "user_role")
-        and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
-        and not _is_user_team_admin(
-            user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
-        )
-        and not await _is_user_org_admin_for_team(
-            user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
-        )
-        and not _is_available_team(
-            team_id=complete_team_data.team_id,
-            user_api_key_dict=user_api_key_dict,
-        )
+        getattr(user_api_key_dict, "user_role", None)
+        == LitellmUserRoles.PROXY_ADMIN.value
+    ):
+        return
+    if _is_user_team_admin(
+        user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
+    ):
+        return
+    if await _is_user_org_admin_for_team(
+        user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
+    ):
+        return
+
+    if not _is_available_team(
+        team_id=complete_team_data.team_id,
+        user_api_key_dict=user_api_key_dict,
     ):
         raise HTTPException(
             status_code=403,
@@ -2029,6 +2042,34 @@ async def _validate_team_member_add_permissions(
             },
         )
 
+    # Available-team self-join: caller may add only themselves, only as a
+    # standard user.  Enforce that here so the bypass cannot be used as a
+    # privilege-escalation or cross-user-injection primitive.
+    members = data.member if isinstance(data.member, list) else [data.member]
+    caller_user_id = getattr(user_api_key_dict, "user_id", None)
+    for member in members:
+        if getattr(member, "role", "user") != "user":
+            raise HTTPException(
+                status_code=403,
+                detail={
+                    "error": (
+                        "Available-team self-join cannot assign 'admin' role. "
+                        "Only proxy/team/org admins can add admins to a team."
+                    )
+                },
+            )
+        member_user_id = getattr(member, "user_id", None)
+        if not caller_user_id or not member_user_id or member_user_id != caller_user_id:
+            raise HTTPException(
+                status_code=403,
+                detail={
+                    "error": (
+                        "Available-team self-join can only add the caller "
+                        "(user_id must match the authenticated user's user_id)."
+                    )
+                },
+            )
+
 
 async def _process_team_members(
     data: TeamMemberAddRequest,
@@ -2049,8 +2090,11 @@ async def _process_team_members(
 
     # Resolve allowed_models: explicit request value, or fall back to team's default_team_member_models
     member_allowed_models = data.allowed_models
-    if member_allowed_models is None and complete_team_data.default_team_member_models:
-        member_allowed_models = complete_team_data.default_team_member_models
+    team_default_member_models = getattr(
+        complete_team_data, "default_team_member_models", None
+    )
+    if member_allowed_models is None and team_default_member_models:
+        member_allowed_models = team_default_member_models
 
     if isinstance(data.member, Member):
         try:
@@ -2381,6 +2425,7 @@ async def team_member_add(
     await _validate_team_member_add_permissions(
         user_api_key_dict=user_api_key_dict,
         complete_team_data=complete_team_data,
+        data=data,
     )
 
     # Validate and populate user_email/user_id for members before processing
@@ -4694,6 +4739,8 @@ async def update_team_member_permissions(
 
     complete_team_data = LiteLLM_TeamTable(**existing_team_row.model_dump())
 
+    # Available-team self-join must NOT grant write access to team-wide
+    # permission policies; only proxy/team/org admins can update them.
     if (
         hasattr(user_api_key_dict, "user_role")
         and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
@@ -4703,16 +4750,12 @@ async def update_team_member_permissions(
         and not await _is_user_org_admin_for_team(
             user_api_key_dict=user_api_key_dict, team_obj=complete_team_data
         )
-        and not _is_available_team(
-            team_id=complete_team_data.team_id,
-            user_api_key_dict=user_api_key_dict,
-        )
     ):
         raise HTTPException(
             status_code=403,
             detail={
                 "error": "Call not allowed. User not proxy admin OR team admin. route={}, team_id={}".format(
-                    "/team/member_add",
+                    "/team/permissions_update",
                     complete_team_data.team_id,
                 )
             },

litellm/proxy/management_endpoints/workflow_management_endpoints.py

@@ -0,0 +1,492 @@
+"""
+WORKFLOW RUN MANAGEMENT
+
+Generic durable state tracking for agents and automated workflows.
+
+POST   /v1/workflows/runs                       - Create a workflow run
+GET    /v1/workflows/runs                       - List runs (filter by type, status)
+GET    /v1/workflows/runs/{run_id}              - Get run with latest event
+PATCH  /v1/workflows/runs/{run_id}              - Update status, metadata, output
+POST   /v1/workflows/runs/{run_id}/events       - Append event (updates run status)
+GET    /v1/workflows/runs/{run_id}/events       - Full event log
+POST   /v1/workflows/runs/{run_id}/messages     - Append conversation message
+GET    /v1/workflows/runs/{run_id}/messages     - Fetch conversation history
+"""
+
+import json
+from typing import Any, Dict, Literal, Optional
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+
+try:
+    from prisma.errors import UniqueViolationError
+except ImportError:
+    UniqueViolationError = None  # type: ignore
+from pydantic import BaseModel
+
+from litellm._logging import verbose_proxy_logger
+from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+
+router = APIRouter()
+
+_MAX_SEQUENCE_RETRIES = 5
+
+
+def _json(value: Any) -> str:
+    """Serialize a Python value for prisma-client-py Json fields (must be a string)."""
+    return json.dumps(value)
+
+
+def _is_admin(user_api_key_dict: UserAPIKeyAuth) -> bool:
+    return user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value
+
+
+def _caller_key(user_api_key_dict: UserAPIKeyAuth) -> Optional[str]:
+    """Return the hashed key token that identifies this caller, or None for master key."""
+    return user_api_key_dict.token
+
+
+# Status transitions driven by event_type
+_EVENT_STATUS_MAP: Dict[str, str] = {
+    "step.started": "running",
+    "step.failed": "failed",
+    "hook.waiting": "paused",
+    "hook.received": "running",
+}
+
+
+# ---------------------------------------------------------------------------
+# Request / Response models
+# ---------------------------------------------------------------------------
+
+
+class WorkflowRunCreateRequest(BaseModel):
+    workflow_type: str
+    input: Optional[Dict[str, Any]] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
+WorkflowRunStatus = Literal["pending", "running", "paused", "completed", "failed"]
+
+
+class WorkflowRunUpdateRequest(BaseModel):
+    status: Optional[WorkflowRunStatus] = None
+    output: Optional[Dict[str, Any]] = None
+    metadata: Optional[Dict[str, Any]] = None
+
+
+class WorkflowEventCreateRequest(BaseModel):
+    event_type: str
+    step_name: str
+    data: Optional[Dict[str, Any]] = None
+
+
+class WorkflowMessageCreateRequest(BaseModel):
+    role: str
+    content: str
+    session_id: Optional[str] = None
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+async def _get_next_sequence_number(prisma_client: Any, run_id: str, table: str) -> int:
+    """Return MAX(sequence_number) + 1 for the given run, for either events or messages."""
+    if table == "events":
+        rows = await prisma_client.db.litellm_workflowevent.find_many(
+            where={"run_id": run_id},
+            order={"sequence_number": "desc"},
+            take=1,
+        )
+    else:
+        rows = await prisma_client.db.litellm_workflowmessage.find_many(
+            where={"run_id": run_id},
+            order={"sequence_number": "desc"},
+            take=1,
+        )
+    return (rows[0].sequence_number + 1) if rows else 0
+
+
+async def _require_run(
+    prisma_client: Any,
+    run_id: str,
+    user_api_key_dict: Optional[UserAPIKeyAuth] = None,
+) -> Any:
+    """Return the run or raise 404. For non-admin callers, also enforce key ownership."""
+    run = await prisma_client.db.litellm_workflowrun.find_unique(
+        where={"run_id": run_id}
+    )
+    if run is None:
+        raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+    if user_api_key_dict is not None and not _is_admin(user_api_key_dict):
+        caller = _caller_key(user_api_key_dict)
+        if not caller or run.created_by != caller:
+            raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+    return run
+
+
+# ---------------------------------------------------------------------------
+# Endpoints
+# ---------------------------------------------------------------------------
+
+
+@router.post(
+    "/v1/workflows/runs",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def create_workflow_run(
+    data: WorkflowRunCreateRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Create a new workflow run. Returns run_id and session_id.
+
+    The caller's API key token is stored as created_by so that non-admin keys
+    can only see and modify their own runs.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    try:
+        create_data: Dict[str, Any] = {
+            "workflow_type": data.workflow_type,
+            "created_by": _caller_key(user_api_key_dict),
+        }
+        if data.input is not None:
+            create_data["input"] = _json(data.input)
+        if data.metadata is not None:
+            create_data["metadata"] = _json(data.metadata)
+        run = await prisma_client.db.litellm_workflowrun.create(data=create_data)
+        return run
+    except Exception as e:
+        verbose_proxy_logger.exception("Error creating workflow run: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get(
+    "/v1/workflows/runs",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def list_workflow_runs(
+    workflow_type: Optional[str] = Query(None),
+    status: Optional[str] = Query(None),
+    limit: int = Query(50, ge=1, le=250),
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """List workflow runs. Filter by workflow_type and/or status.
+
+    Non-admin callers only see runs created by their own API key.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    where: Dict[str, Any] = {}
+    if workflow_type:
+        where["workflow_type"] = workflow_type
+    if status:
+        statuses = [s.strip() for s in status.split(",")]
+        where["status"] = {"in": statuses} if len(statuses) > 1 else statuses[0]
+
+    # Non-admin callers are scoped to their own key.
+    if not _is_admin(user_api_key_dict):
+        caller = _caller_key(user_api_key_dict)
+        if caller:
+            where["created_by"] = caller
+
+    try:
+        runs = await prisma_client.db.litellm_workflowrun.find_many(
+            where=where,
+            order={"created_at": "desc"},
+            take=limit,
+        )
+        return {"runs": runs, "count": len(runs)}
+    except Exception as e:
+        verbose_proxy_logger.exception("Error listing workflow runs: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.get(
+    "/v1/workflows/runs/{run_id}",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def get_workflow_run(
+    run_id: str,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Get a workflow run with its most recent event."""
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    try:
+        run = await prisma_client.db.litellm_workflowrun.find_unique(
+            where={"run_id": run_id},
+            include={"events": {"order_by": {"sequence_number": "desc"}, "take": 1}},
+        )
+        if run is None:
+            raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+        if not _is_admin(user_api_key_dict):
+            caller = _caller_key(user_api_key_dict)
+            if not caller or run.created_by != caller:
+                raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+        return run
+    except HTTPException:
+        raise
+    except Exception as e:
+        verbose_proxy_logger.exception("Error getting workflow run: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.patch(
+    "/v1/workflows/runs/{run_id}",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def update_workflow_run(
+    run_id: str,
+    data: WorkflowRunUpdateRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Update status, metadata, or output on a workflow run."""
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    update: Dict[str, Any] = {}
+    if data.status is not None:
+        update["status"] = data.status
+    if data.output is not None:
+        update["output"] = _json(data.output)
+    if data.metadata is not None:
+        update["metadata"] = _json(data.metadata)
+
+    if not update:
+        raise HTTPException(status_code=400, detail="No fields to update")
+
+    # Enforce ownership before writing.
+    await _require_run(prisma_client, run_id, user_api_key_dict)
+
+    try:
+        run = await prisma_client.db.litellm_workflowrun.update(
+            where={"run_id": run_id},
+            data=update,
+        )
+        if run is None:
+            raise HTTPException(status_code=404, detail=f"Run '{run_id}' not found")
+        return run
+    except HTTPException:
+        raise
+    except Exception as e:
+        verbose_proxy_logger.exception("Error updating workflow run: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post(
+    "/v1/workflows/runs/{run_id}/events",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def append_workflow_event(
+    run_id: str,
+    data: WorkflowEventCreateRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Append an event to the run's event log. Also updates run.status if event_type maps to a status.
+
+    Sequence numbers use optimistic concurrency: on a unique-constraint collision
+    (concurrent append), retries up to _MAX_SEQUENCE_RETRIES times with a fresh MAX+1.
+    The event+status update is atomic in a single DB transaction.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    await _require_run(prisma_client, run_id, user_api_key_dict)
+
+    new_status = _EVENT_STATUS_MAP.get(data.event_type)
+
+    for attempt in range(_MAX_SEQUENCE_RETRIES):
+        try:
+            seq = await _get_next_sequence_number(prisma_client, run_id, "events")
+            event_data: Dict[str, Any] = {
+                "run_id": run_id,
+                "event_type": data.event_type,
+                "step_name": data.step_name,
+                "sequence_number": seq,
+            }
+            if data.data is not None:
+                event_data["data"] = _json(data.data)
+
+            async with prisma_client.db.tx() as tx:
+                event = await tx.litellm_workflowevent.create(data=event_data)
+                if new_status:
+                    await tx.litellm_workflowrun.update(
+                        where={"run_id": run_id},
+                        data={"status": new_status},
+                    )
+
+            return event
+
+        except Exception as e:
+            if UniqueViolationError is not None and isinstance(e, UniqueViolationError):
+                if attempt == _MAX_SEQUENCE_RETRIES - 1:
+                    verbose_proxy_logger.exception(
+                        "Sequence number collision after %d retries for run %s",
+                        _MAX_SEQUENCE_RETRIES,
+                        run_id,
+                    )
+                    raise HTTPException(
+                        status_code=409,
+                        detail="Concurrent write conflict — please retry",
+                    )
+                continue
+            verbose_proxy_logger.exception("Error appending workflow event: %s", e)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    raise HTTPException(
+        status_code=500, detail="Failed to append event"
+    )  # pragma: no cover
+
+
+@router.get(
+    "/v1/workflows/runs/{run_id}/events",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def list_workflow_events(
+    run_id: str,
+    limit: int = Query(100, ge=1, le=500),
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Fetch event log for a run, ordered by sequence_number. Default limit 100, max 500."""
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    await _require_run(prisma_client, run_id, user_api_key_dict)
+
+    try:
+        events = await prisma_client.db.litellm_workflowevent.find_many(
+            where={"run_id": run_id},
+            order={"sequence_number": "asc"},
+            take=limit,
+        )
+        return {"events": events, "count": len(events)}
+    except Exception as e:
+        verbose_proxy_logger.exception("Error listing workflow events: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@router.post(
+    "/v1/workflows/runs/{run_id}/messages",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def append_workflow_message(
+    run_id: str,
+    data: WorkflowMessageCreateRequest,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Append a conversation message. Stores full content (not truncated).
+
+    Uses optimistic concurrency for sequence numbers.
+    """
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    await _require_run(prisma_client, run_id, user_api_key_dict)
+
+    for attempt in range(_MAX_SEQUENCE_RETRIES):
+        try:
+            seq = await _get_next_sequence_number(prisma_client, run_id, "messages")
+            msg_data: Dict[str, Any] = {
+                "run_id": run_id,
+                "role": data.role,
+                "content": data.content,
+                "sequence_number": seq,
+            }
+            if data.session_id is not None:
+                msg_data["session_id"] = data.session_id
+            msg = await prisma_client.db.litellm_workflowmessage.create(data=msg_data)
+            return msg
+
+        except Exception as e:
+            if UniqueViolationError is not None and isinstance(e, UniqueViolationError):
+                if attempt == _MAX_SEQUENCE_RETRIES - 1:
+                    verbose_proxy_logger.exception(
+                        "Sequence number collision after %d retries for run %s",
+                        _MAX_SEQUENCE_RETRIES,
+                        run_id,
+                    )
+                    raise HTTPException(
+                        status_code=409,
+                        detail="Concurrent write conflict — please retry",
+                    )
+                continue
+            verbose_proxy_logger.exception("Error appending workflow message: %s", e)
+            raise HTTPException(status_code=500, detail=str(e))
+
+    raise HTTPException(
+        status_code=500, detail="Failed to append message"
+    )  # pragma: no cover
+
+
+@router.get(
+    "/v1/workflows/runs/{run_id}/messages",
+    tags=["workflow management"],
+    dependencies=[Depends(user_api_key_auth)],
+)
+async def list_workflow_messages(
+    run_id: str,
+    limit: int = Query(100, ge=1, le=500),
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """Fetch conversation history for a run, ordered by sequence_number. Default limit 100, max 500."""
+    from litellm.proxy.proxy_server import prisma_client
+
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+
+    await _require_run(prisma_client, run_id, user_api_key_dict)
+
+    try:
+        messages = await prisma_client.db.litellm_workflowmessage.find_many(
+            where={"run_id": run_id},
+            order={"sequence_number": "asc"},
+            take=limit,
+        )
+        return {"messages": messages, "count": len(messages)}
+    except Exception as e:
+        verbose_proxy_logger.exception("Error listing workflow messages: %s", e)
+        raise HTTPException(status_code=500, detail=str(e))

litellm/proxy/management_helpers/object_permission_utils.py

@@ -335,8 +335,9 @@ async def validate_key_mcp_servers_against_team(
         disallowed_servers = requested_servers - all_allowed_servers
         if disallowed_servers:
             if team_obj is not None:
+                team_id = team_obj.team_id
                 detail = (
-                    f"Key requests MCP servers not allowed by team '{team_obj.team_id}': "
+                    f"Key requests MCP servers not allowed by team '{team_id}': "
                     f"{sorted(disallowed_servers)}. "
                     f"Team allows: {sorted(team_allowed_servers)}. "
                     f"Global (allow_all_keys) servers: {sorted(allow_all_keys_servers)}."
@@ -365,8 +366,9 @@ async def validate_key_mcp_servers_against_team(
         disallowed_groups = requested_access_groups - team_access_groups
         if disallowed_groups:
             if team_obj is not None:
+                team_id = team_obj.team_id
                 detail = (
-                    f"Key requests MCP access groups not allowed by team '{team_obj.team_id}': "
+                    f"Key requests MCP access groups not allowed by team '{team_id}': "
                     f"{sorted(disallowed_groups)}. "
                     f"Team allows: {sorted(team_access_groups)}."
                 )
@@ -390,13 +392,60 @@ async def validate_key_mcp_servers_against_team(
         if team_mcp_toolsets:
             disallowed_toolsets = requested_toolsets - set(team_mcp_toolsets)
             if disallowed_toolsets:
+                team_id = team_obj.team_id
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN,
                     detail={
                         "error": (
-                            f"Key requests MCP toolsets not allowed by team '{team_obj.team_id}': "
+                            f"Key requests MCP toolsets not allowed by team '{team_id}': "
                             f"{sorted(disallowed_toolsets)}. "
                             f"Team allows: {sorted(team_mcp_toolsets)}."
                         )
                     },
                 )
+
+
+def _extract_requested_search_tools(object_permission: Optional[dict]) -> List[str]:
+    """Return search_tool_name values from a key's object_permission dict."""
+    if not object_permission or not isinstance(object_permission, dict):
+        return []
+    raw = object_permission.get("search_tools")
+    if not isinstance(raw, list):
+        return []
+    return [str(x) for x in raw if x]
+
+
+async def validate_key_search_tools_against_team(
+    object_permission: Optional[dict],
+    team_obj: Optional["LiteLLM_TeamTableCachedObj"],
+) -> None:
+    """
+    Validate key object_permission.search_tools is a subset of the team's allowlist.
+
+    Empty team allowlist means no restriction at team layer (skip).
+    """
+    requested = _extract_requested_search_tools(object_permission)
+    if not requested:
+        return
+
+    team_tools: List[str] = []
+    if team_obj is not None and team_obj.object_permission is not None:
+        st = team_obj.object_permission.search_tools
+        if st:
+            team_tools = list(st)
+
+    if not team_tools:
+        return
+
+    disallowed = set(requested) - set(team_tools)
+    if disallowed:
+        team_id = team_obj.team_id if team_obj is not None else "unknown"
+        raise HTTPException(
+            status_code=status.HTTP_403_FORBIDDEN,
+            detail={
+                "error": (
+                    f"Key requests search tools not allowed by team '{team_id}': "
+                    f"{sorted(disallowed)}. Team allows: {sorted(team_tools)}."
+                )
+            },
+        )

litellm/proxy/pass_through_endpoints/llm_provider_handlers/anthropic_passthrough_logging_handler.py

@@ -549,10 +549,16 @@ class AnthropicPassthroughLoggingHandler:
                 # Create a mock user API key dict for the managed object storage
                 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 
+                _request_metadata = (kwargs.get("litellm_params", {}) or {}).get(
+                    "metadata", {}
+                ) or {}
+
                 user_api_key_dict = UserAPIKeyAuth(
-                    user_id=kwargs.get("user_id", "default-user"),
+                    user_id=_request_metadata.get(
+                        "user_api_key_user_id", "default-user"
+                    ),
                     api_key="",
-                    team_id=None,
+                    team_id=_request_metadata.get("user_api_key_team_id"),
                     team_alias=None,
                     user_role=LitellmUserRoles.CUSTOMER,  # Use proper enum value
                     user_email=None,

litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py

@@ -849,10 +849,16 @@ class VertexPassthroughLoggingHandler:
                 # Create a mock user API key dict for the managed object storage
                 from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
 
+                _request_metadata = (kwargs.get("litellm_params", {}) or {}).get(
+                    "metadata", {}
+                ) or {}
+
                 user_api_key_dict = UserAPIKeyAuth(
-                    user_id=kwargs.get("user_id", "default-user"),
+                    user_id=_request_metadata.get(
+                        "user_api_key_user_id", "default-user"
+                    ),
                     api_key="",
-                    team_id=None,
+                    team_id=_request_metadata.get("user_api_key_team_id"),
                     team_alias=None,
                     user_role=LitellmUserRoles.CUSTOMER,  # Use proper enum value
                     user_email=None,

litellm/proxy/pass_through_endpoints/streaming_handler.py

@@ -36,21 +36,16 @@ class PassThroughStreamingHandler:
         passthrough_success_handler_obj: PassThroughEndpointLogging,
         url_route: str,
     ):
-        """
-        - Yields chunks from the response
-        - Collect non-empty chunks for post-processing (logging)
-        - Inject cost into chunks if include_cost_in_streaming_usage is enabled
-        """
-        try:
-            raw_bytes: List[bytes] = []
-            # Extract model name for cost injection
-            model_name = PassThroughStreamingHandler._extract_model_for_cost_injection(
-                request_body=request_body,
-                url_route=url_route,
-                endpoint_type=endpoint_type,
-                litellm_logging_obj=litellm_logging_obj,
-            )
+        raw_bytes: List[bytes] = []
+        logging_scheduled = False
+        model_name = PassThroughStreamingHandler._extract_model_for_cost_injection(
+            request_body=request_body,
+            url_route=url_route,
+            endpoint_type=endpoint_type,
+            litellm_logging_obj=litellm_logging_obj,
+        )
 
+        try:
             async for chunk in response.aiter_bytes():
                 raw_bytes.append(chunk)
                 if (
@@ -58,7 +53,6 @@ class PassThroughStreamingHandler:
                     and model_name
                 ):
                     if endpoint_type == EndpointType.VERTEX_AI:
-                        # Only handle streamRawPredict (uses Anthropic format)
                         if "streamRawPredict" in url_route or "rawPredict" in url_route:
                             modified_chunk = ProxyBaseLLMRequestProcessing._process_chunk_with_cost_injection(
                                 chunk, model_name
@@ -73,25 +67,32 @@ class PassThroughStreamingHandler:
                             chunk = modified_chunk
 
                 yield chunk
-
-            # After all chunks are processed, handle post-processing
-            end_time = datetime.now()
-
-            asyncio.create_task(
-                PassThroughStreamingHandler._route_streaming_logging_to_handler(
-                    litellm_logging_obj=litellm_logging_obj,
-                    passthrough_success_handler_obj=passthrough_success_handler_obj,
-                    url_route=url_route,
-                    request_body=request_body or {},
-                    endpoint_type=endpoint_type,
-                    start_time=start_time,
-                    raw_bytes=raw_bytes,
-                    end_time=end_time,
-                )
-            )
         except Exception as e:
             verbose_proxy_logger.error(f"Error in chunk_processor: {str(e)}")
             raise
+        finally:
+            # GeneratorExit (raised on client disconnect) is not caught by
+            # `except Exception`; the finally block ensures partial usage
+            # still gets logged for spend tracking. See LIT-2642.
+            if not logging_scheduled and raw_bytes:
+                logging_scheduled = True
+                try:
+                    asyncio.create_task(
+                        PassThroughStreamingHandler._route_streaming_logging_to_handler(
+                            litellm_logging_obj=litellm_logging_obj,
+                            passthrough_success_handler_obj=passthrough_success_handler_obj,
+                            url_route=url_route,
+                            request_body=request_body or {},
+                            endpoint_type=endpoint_type,
+                            start_time=start_time,
+                            raw_bytes=raw_bytes,
+                            end_time=datetime.now(),
+                        )
+                    )
+                except Exception as e:
+                    verbose_proxy_logger.error(
+                        f"Error scheduling chunk_processor logging: {str(e)}"
+                    )
 
     @staticmethod
     async def _route_streaming_logging_to_handler(

litellm/proxy/proxy_server.py

@@ -94,6 +94,7 @@ from litellm.proxy._types import (
     TeamDefaultSettings,
     TokenCountRequest,
     TransformRequestBody,
+    UI_TEAM_ID,
     UserAPIKeyAuth,
 )
 from litellm.proxy.common_utils.cache_pydantic_utils import CacheCodec
@@ -431,6 +432,9 @@ from litellm.proxy.management_endpoints.team_endpoints import (
 from litellm.proxy.management_endpoints.tool_management_endpoints import (
     router as tool_management_router,
 )
+from litellm.proxy.management_endpoints.workflow_management_endpoints import (
+    router as workflow_management_router,
+)
 from litellm.proxy.memory.memory_endpoints import router as memory_router
 from litellm.proxy.management_endpoints.ui_sso import (
     get_disabled_non_admin_personal_key_creation,
@@ -747,6 +751,10 @@ async def _initialize_shared_aiohttp_session():
     try:
         from aiohttp import ClientSession, TCPConnector
 
+        from litellm.llms.custom_httpx.http_handler import (
+            _build_aiohttp_keepalive_socket_factory,
+        )
+
         connector_kwargs: Dict[str, Any] = {
             "keepalive_timeout": AIOHTTP_KEEPALIVE_TIMEOUT,
             "ttl_dns_cache": AIOHTTP_TTL_DNS_CACHE,
@@ -757,6 +765,9 @@ async def _initialize_shared_aiohttp_session():
             connector_kwargs["limit"] = AIOHTTP_CONNECTOR_LIMIT
         if AIOHTTP_CONNECTOR_LIMIT_PER_HOST > 0:
             connector_kwargs["limit_per_host"] = AIOHTTP_CONNECTOR_LIMIT_PER_HOST
+        socket_factory = _build_aiohttp_keepalive_socket_factory()
+        if socket_factory is not None:
+            connector_kwargs["socket_factory"] = socket_factory
 
         connector = TCPConnector(**connector_kwargs)
         session = ClientSession(connector=connector)
@@ -12065,7 +12076,7 @@ async def onboarding(invite_link: str, request: Request):
     """
     - Get the invite link
     - Validate it's still 'valid'
-    - Invalidate the link (prevents abuse)
+    - Return a short-lived onboarding token
     - Get user from db
     - Pass in user_email if set
     """
@@ -12103,7 +12114,7 @@ async def onboarding(invite_link: str, request: Request):
         )
 
     #### CHECK IF ALREADY USED
-    if invite_obj.is_accepted is True:
+    if invite_obj.is_accepted is True or invite_obj.accepted_at is not None:
         raise HTTPException(
             status_code=401,
             detail={"error": "Invitation link has already been used."},
@@ -12119,24 +12130,6 @@ async def onboarding(invite_link: str, request: Request):
             status_code=401, detail={"error": "User does not exist in db."}
         )
 
-    user_email = user_obj.user_email
-
-    response = await generate_key_helper_fn(
-        request_type="key",
-        **{
-            "user_role": user_obj.user_role,
-            "duration": LITELLM_UI_SESSION_DURATION,
-            "key_max_budget": litellm.max_ui_session_budget,
-            "models": [],
-            "aliases": {},
-            "config": {},
-            "spend": 0,
-            "user_id": user_obj.user_id,
-            "team_id": "litellm-dashboard",
-        },  # type: ignore
-    )
-    key = response["token"]  # type: ignore
-
     litellm_dashboard_ui = get_custom_url(str(request.base_url))
     if litellm_dashboard_ui.endswith("/"):
         litellm_dashboard_ui += "ui/onboarding"
@@ -12144,13 +12137,24 @@ async def onboarding(invite_link: str, request: Request):
         litellm_dashboard_ui += "/ui/onboarding"
     import jwt
 
+    user_email = user_obj.user_email
+    onboarding_token = jwt.encode(  # type: ignore
+        {
+            "token_type": "litellm_onboarding",
+            "invitation_link": invite_link,
+            "user_id": user_obj.user_id,
+            "exp": litellm.utils.get_utc_datetime() + timedelta(minutes=15),
+        },
+        master_key,
+        algorithm="HS256",
+    )
     disabled_non_admin_personal_key_creation = (
         get_disabled_non_admin_personal_key_creation()
     )
 
     returned_ui_token_object = ReturnedUITokenObject(
         user_id=user_obj.user_id,
-        key=key,
+        key=onboarding_token,
         user_email=user_obj.user_email,
         user_role=user_obj.user_role,
         login_method="username_password",
@@ -12175,8 +12179,117 @@ async def onboarding(invite_link: str, request: Request):
     }
 
 
+def _get_onboarding_claims_from_request(request: Request) -> dict:
+    global master_key, general_settings
+
+    if master_key is None:
+        raise ProxyException(
+            message="Master Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.",
+            type=ProxyErrorTypes.auth_error,
+            param="master_key",
+            code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+        )
+
+    auth_header_name = general_settings.get("litellm_key_header_name", "Authorization")
+    onboarding_auth_header = request.headers.get(auth_header_name)
+    if onboarding_auth_header is None:
+        raise HTTPException(
+            status_code=401,
+            detail={"error": "Missing onboarding session for invitation link."},
+        )
+    onboarding_token = onboarding_auth_header
+    if onboarding_token.lower().startswith("bearer "):
+        onboarding_token = onboarding_token.split(" ", 1)[1]
+
+    import jwt
+
+    try:
+        return jwt.decode(
+            onboarding_token,
+            master_key,
+            algorithms=["HS256"],
+        )
+    except Exception:
+        raise HTTPException(
+            status_code=401,
+            detail={"error": "Invalid onboarding session for invitation link."},
+        )
+
+
+async def _rollback_onboarding_invite_claim(
+    invitation_link: str,
+    user_id: str,
+) -> None:
+    global prisma_client
+
+    if prisma_client is None:
+        return
+
+    try:
+        await prisma_client.db.litellm_invitationlink.update_many(
+            where={"id": invitation_link, "is_accepted": True},
+            data={
+                "accepted_at": None,
+                "is_accepted": False,
+                "updated_at": litellm.utils.get_utc_datetime(),
+                "updated_by": user_id,
+            },
+        )
+    except Exception:
+        verbose_proxy_logger.exception(
+            "Failed to roll back onboarding invitation after session key mint failed."
+        )
+
+
+async def _generate_onboarding_ui_session_token(user_obj: Any) -> str:
+    global master_key, general_settings
+
+    response = await generate_key_helper_fn(
+        request_type="key",
+        **{
+            "user_role": user_obj.user_role,
+            "duration": LITELLM_UI_SESSION_DURATION,
+            "key_max_budget": litellm.max_ui_session_budget,
+            "models": [],
+            "aliases": {},
+            "config": {},
+            "spend": 0,
+            "user_id": user_obj.user_id,
+            "team_id": UI_TEAM_ID,
+        },  # type: ignore
+    )
+    key = response["token"]  # type: ignore
+
+    from litellm.types.proxy.ui_sso import ReturnedUITokenObject
+
+    import jwt
+
+    disabled_non_admin_personal_key_creation = (
+        get_disabled_non_admin_personal_key_creation()
+    )
+    returned_ui_token_object = ReturnedUITokenObject(
+        user_id=user_obj.user_id,
+        key=key,
+        user_email=user_obj.user_email,
+        user_role=user_obj.user_role,
+        login_method="username_password",
+        premium_user=premium_user,
+        auth_header_name=general_settings.get(
+            "litellm_key_header_name", "Authorization"
+        ),
+        disabled_non_admin_personal_key_creation=disabled_non_admin_personal_key_creation,
+        server_root_path=get_server_root_path(),
+    )
+    assert master_key is not None
+    return jwt.encode(  # type: ignore
+        cast(dict, returned_ui_token_object),
+        master_key,
+        algorithm="HS256",
+    )
+
+
 @app.post("/onboarding/claim_token", include_in_schema=False)
-async def claim_onboarding_link(data: InvitationClaim):
+async def claim_onboarding_link(data: InvitationClaim, request: Request):
     """
     Special route. Allows UI link share user to update their password.
 
@@ -12188,7 +12301,7 @@ async def claim_onboarding_link(data: InvitationClaim):
 
     This route can only update user password.
     """
-    global prisma_client
+    global prisma_client, master_key, general_settings
     ### VALIDATE INVITE LINK ###
     if prisma_client is None:
         raise HTTPException(
@@ -12213,7 +12326,7 @@ async def claim_onboarding_link(data: InvitationClaim):
         )
 
     #### CHECK IF ALREADY USED
-    if invite_obj.is_accepted is True:
+    if invite_obj.is_accepted is True or invite_obj.accepted_at is not None:
         raise HTTPException(
             status_code=401,
             detail={"error": "Invitation link has already been used."},
@@ -12229,32 +12342,87 @@ async def claim_onboarding_link(data: InvitationClaim):
                 )
             },
         )
-    ### UPDATE USER OBJECT ###
-    hashed_pw = hash_password(data.password)
-    user_obj = await prisma_client.db.litellm_usertable.update(
-        where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
-    )
 
-    if user_obj is None:
+    onboarding_claims = _get_onboarding_claims_from_request(request=request)
+    if (
+        onboarding_claims.get("token_type") != "litellm_onboarding"
+        or onboarding_claims.get("invitation_link") != data.invitation_link
+        or onboarding_claims.get("user_id") != data.user_id
+    ):
         raise HTTPException(
-            status_code=401, detail={"error": "User does not exist in db."}
+            status_code=401,
+            detail={"error": "Invalid onboarding session for invitation link."},
         )
 
-    #### MARK LINK AS USED
+    hashed_pw = hash_password(data.password)
     current_time = litellm.utils.get_utc_datetime()
-    await prisma_client.db.litellm_invitationlink.update(
-        where={"id": data.invitation_link},
-        data={
-            "accepted_at": current_time,
-            "updated_at": current_time,
-            "is_accepted": True,
-            "updated_by": invite_obj.user_id,  # type: ignore
-        },
-    )
+    async with prisma_client.db.tx() as tx:
+        updated_count = await tx.litellm_invitationlink.update_many(
+            where={"id": data.invitation_link, "is_accepted": False},
+            data={
+                "is_accepted": True,
+                "updated_at": current_time,
+                "updated_by": invite_obj.user_id,  # type: ignore
+            },
+        )
+        if updated_count == 0:
+            raise HTTPException(
+                status_code=401,
+                detail={"error": "Invitation link has already been used."},
+            )
+
+        ### UPDATE USER OBJECT ###
+        user_obj = await tx.litellm_usertable.update(
+            where={"user_id": invite_obj.user_id}, data={"password": hashed_pw}
+        )
+
+        if user_obj is None:
+            raise HTTPException(
+                status_code=401, detail={"error": "User does not exist in db."}
+            )
+
+        #### MARK LINK AS USED
+        current_time = litellm.utils.get_utc_datetime()
+        await tx.litellm_invitationlink.update(
+            where={"id": data.invitation_link},
+            data={
+                "accepted_at": current_time,
+                "updated_at": current_time,
+                "updated_by": invite_obj.user_id,  # type: ignore
+            },
+        )
 
     if user_obj and hasattr(user_obj, "__dict__"):
         user_obj.__dict__.pop("password", None)
-    return user_obj
+
+    try:
+        jwt_token = await _generate_onboarding_ui_session_token(user_obj=user_obj)
+    except Exception as e:
+        await _rollback_onboarding_invite_claim(
+            invitation_link=data.invitation_link,
+            user_id=data.user_id,
+        )
+        if isinstance(e, HTTPException):
+            raise e
+        raise HTTPException(
+            status_code=500,
+            detail={
+                "error": "Failed to create onboarding session. Please retry the invitation link."
+            },
+        ) from e
+
+    litellm_dashboard_ui = get_custom_url(str(request.base_url))
+    if litellm_dashboard_ui.endswith("/"):
+        litellm_dashboard_ui += "ui/"
+    else:
+        litellm_dashboard_ui += "/ui/"
+    litellm_dashboard_ui += "?login=success"
+    return {
+        "login_url": litellm_dashboard_ui,
+        "token": jwt_token,
+        "user_email": user_obj.user_email,
+        "user": user_obj,
+    }
 
 
 @app.get("/get_logo_url", include_in_schema=False)
@@ -14319,6 +14487,7 @@ app.include_router(model_management_router)
 app.include_router(model_access_group_management_router)
 app.include_router(tag_management_router)
 app.include_router(tool_management_router)
+app.include_router(workflow_management_router)
 app.include_router(memory_router)
 app.include_router(cost_tracking_settings_router)
 app.include_router(router_settings_router)

litellm/proxy/schema.prisma

@@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
   models                String[]       @default([])
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
+  search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -1290,3 +1291,80 @@ model LiteLLM_AdaptiveRouterSession {
   @@id([session_id, router_name, model_name])
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
+
+// ---------------------------------------------------------------------------
+// Workflow Run Tracking
+//
+// Generic durable state tracking for any agent or automated workflow.
+// Design: three tables — run (header + materialized status), event (append-only
+// source of truth for state transitions), message (conversation inbox/outbox).
+//
+// Usage:
+//   - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
+//   - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
+//   - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
+//     the proxy — all spend logs for this run are automatically tagged.
+// ---------------------------------------------------------------------------
+
+// One instance of work being done. `status` is a materialized cache of the
+// latest event; the event log is the authoritative source of truth.
+model LiteLLM_WorkflowRun {
+  run_id        String   @id @default(uuid())
+  session_id    String   @unique @default(uuid())
+  workflow_type String
+  status        String   @default("pending")
+  created_by    String?  // user_id of the key that created this run; null = created by master key
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+  input         Json?
+  output        Json?
+  metadata      Json?
+
+  events   LiteLLM_WorkflowEvent[]
+  messages LiteLLM_WorkflowMessage[]
+
+  @@index([workflow_type, status])
+  @@index([session_id])
+  @@index([created_at])
+  @@index([created_by])
+}
+
+// Append-only log of state transitions. Never mutate rows here.
+// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
+// Status auto-update rules (applied by the append endpoint):
+//   step.started   → run.status = running
+//   step.failed    → run.status = failed
+//   hook.waiting   → run.status = paused
+//   hook.received  → run.status = running
+model LiteLLM_WorkflowEvent {
+  event_id        String   @id @default(uuid())
+  run_id          String
+  event_type      String
+  step_name       String
+  sequence_number Int
+  data            Json?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}
+
+// Conversation inbox/outbox — full message content, separate from the durable
+// event log. Spend logs truncate messages; this table stores them in full.
+// `session_id` here is the Claude --resume session ID (or similar).
+model LiteLLM_WorkflowMessage {
+  message_id      String   @id @default(uuid())
+  run_id          String
+  role            String
+  content         String
+  sequence_number Int
+  session_id      String?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}

litellm/proxy/search_endpoints/endpoints.py

@@ -134,10 +134,48 @@ async def search(
 
     if "search_tool_name" in data and data["search_tool_name"]:
         data["model"] = data["search_tool_name"]
+        search_tool_name_value = data["search_tool_name"]
+
+        # Authorization check: verify key can access this search tool
+        from litellm.proxy.auth.auth_checks import (
+            can_key_call_search_tool,
+            can_team_call_search_tool,
+            get_team_object,
+        )
+
+        try:
+            # Check key-level access
+            await can_key_call_search_tool(
+                search_tool_name=search_tool_name_value,
+                valid_token=user_api_key_dict,
+            )
+
+            # Check team-level access if key is associated with a team
+            if user_api_key_dict.team_id:
+                from litellm.proxy.proxy_server import (
+                    prisma_client,
+                    proxy_logging_obj,
+                    user_api_key_cache,
+                )
+
+                team_object = await get_team_object(
+                    team_id=user_api_key_dict.team_id,
+                    prisma_client=prisma_client,
+                    user_api_key_cache=user_api_key_cache,
+                    parent_otel_span=user_api_key_dict.parent_otel_span,
+                    proxy_logging_obj=proxy_logging_obj,
+                )
+                await can_team_call_search_tool(
+                    search_tool_name=search_tool_name_value,
+                    team_object=team_object,
+                )
+        except Exception as e:
+            verbose_proxy_logger.error(
+                f"Search tool authorization failed for {search_tool_name_value}: {str(e)}"
+            )
+            raise
 
         if llm_router is not None and hasattr(llm_router, "search_tools"):
-            search_tool_name_value = data["search_tool_name"]
-
             verbose_proxy_logger.debug(
                 f"Search endpoint - Looking for search_tool_name: {search_tool_name_value}. "
                 f"Available search tools in router: {[tool.get('search_tool_name') for tool in llm_router.search_tools]}. "
@@ -163,6 +201,16 @@ async def search(
                     data["metadata"] = {}
                 data["metadata"]["model_group"] = search_tool_name_value
 
+    # Ensure team context is available to search router credential resolution.
+    # add_litellm_data_to_request() also injects these values, but this keeps
+    # search endpoint behavior explicit and resilient for direct router paths.
+    if "metadata" not in data or not isinstance(data.get("metadata"), dict):
+        data["metadata"] = {}
+    if getattr(user_api_key_dict, "team_metadata", None) is not None:
+        data["metadata"]["user_api_key_team_metadata"] = user_api_key_dict.team_metadata
+    if getattr(user_api_key_dict, "team_id", None) is not None:
+        data["metadata"]["user_api_key_team_id"] = user_api_key_dict.team_id
+
     # Process request using ProxyBaseLLMRequestProcessing
     processor = ProxyBaseLLMRequestProcessing(data=data)
     try:

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -53,20 +53,13 @@ def _get_max_string_length_prompt_in_db() -> int:
 
 
 def _is_master_key(api_key: Optional[str], _master_key: Optional[str]) -> bool:
+    """
+    Raw-only constant-time master-key comparison. The hashed form is never
+    considered equivalent — only the raw master-key string matches.
+    """
     if _master_key is None or api_key is None:
         return False
-
-    ## string comparison
-    is_master_key = secrets.compare_digest(api_key, _master_key)
-    if is_master_key:
-        return True
-
-    ## hash comparison
-    is_master_key = secrets.compare_digest(api_key, hash_token(_master_key))
-    if is_master_key:
-        return True
-
-    return False
+    return secrets.compare_digest(api_key, _master_key)
 
 
 def _get_spend_logs_metadata(
@@ -235,8 +228,6 @@ def _extract_usage_for_ocr_call(response_obj: Any, response_obj_dict: dict) -> d
 def get_logging_payload(  # noqa: PLR0915
     kwargs, response_obj, start_time, end_time
 ) -> SpendLogsPayload:
-    from litellm.proxy.proxy_server import general_settings, master_key
-
     if kwargs is None:
         kwargs = {}
 
@@ -295,11 +286,6 @@ def get_logging_payload(  # noqa: PLR0915
         if api_key.startswith("sk-"):
             # hash the api_key
             api_key = hash_token(api_key)
-        if (
-            _is_master_key(api_key=api_key, _master_key=master_key)
-            and general_settings.get("disable_adding_master_key_hash_to_db") is True
-        ):
-            api_key = "litellm_proxy_master_key"  # use a known alias, if the user disabled storing master key in db
 
     if (
         standard_logging_payload is not None
@@ -324,11 +310,6 @@ def get_logging_payload(  # noqa: PLR0915
         and standard_logging_payload.get("request_tags") is not None
     ):  # use 'tags' from standard logging payload instead
         request_tags = json.dumps(standard_logging_payload["request_tags"])
-    if (
-        _is_master_key(api_key=api_key, _master_key=master_key)
-        and general_settings.get("disable_adding_master_key_hash_to_db") is True
-    ):
-        api_key = "litellm_proxy_master_key"  # use a known alias, if the user disabled storing master key in db
 
     _model_id = metadata.get("model_info", {}).get("id", "")
     _model_group = metadata.get("model_group", "")

litellm/proxy/utils.py

@@ -106,7 +106,10 @@ from litellm.proxy.db.create_views import (
     should_create_missing_views,
 )
 from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
-from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
+from litellm.proxy.db.exception_handler import (
+    PrismaDBExceptionHandler,
+    call_with_db_reconnect_retry,
+)
 from litellm.proxy.db.log_db_metrics import log_db_metrics
 from litellm.proxy.db.prisma_client import PrismaWrapper
 from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import (
@@ -2779,30 +2782,42 @@ class PrismaClient:
         table_name: Literal["users", "keys", "config", "spend"],
     ):
         """
-        Generic implementation of get data
+        Generic implementation of get data.
+
+        Self-heals across a single transient transport blip via
+        `call_with_db_reconnect_retry`: on `httpx.ReadError` /
+        `ClientNotConnectedError` / similar, attempt one DB reconnect and
+        retry once before surfacing the failure. Restores the 1.82.6 behavior
+        that was lost in 1.83.x — see issue #25143.
         """
         start_time = time.time()
-        try:
+
+        async def _do_query():
             if table_name == "users":
-                response = await self.db.litellm_usertable.find_first(
+                return await self.db.litellm_usertable.find_first(
                     where={key: value}  # type: ignore
                 )
             elif table_name == "keys":
-                response = await self.db.litellm_verificationtoken.find_first(  # type: ignore
+                return await self.db.litellm_verificationtoken.find_first(  # type: ignore
                     where={key: value}  # type: ignore
                 )
             elif table_name == "config":
-                response = await self.db.litellm_config.find_first(  # type: ignore
+                return await self.db.litellm_config.find_first(  # type: ignore
                     where={key: value}  # type: ignore
                 )
             elif table_name == "spend":
-                response = await self.db.l.find_first(  # type: ignore
+                return await self.db.l.find_first(  # type: ignore
                     where={key: value}  # type: ignore
                 )
-            return response
-        except Exception as e:
-            import traceback
+            return None
 
+        try:
+            return await call_with_db_reconnect_retry(
+                self,
+                _do_query,
+                reason=f"prisma_get_generic_data_{table_name}_lookup_failure",
+            )
+        except Exception as e:
             error_msg = f"LiteLLM Prisma Client Exception get_generic_data: {str(e)}"
             verbose_proxy_logger.error(error_msg)
             error_msg = error_msg + "\nException Type: {}".format(type(e))
@@ -4183,8 +4198,11 @@ class PrismaClient:
 
         Uses the _engine_confirmed_dead flag (set by waitpid thread / pidfd / poll
         handlers) to choose between heavy reconnect (engine dead -- recreate
-        Prisma client, re-arm watcher) and lightweight reconnect (network
-        blip -- disconnect, connect, SELECT 1).
+        Prisma client, re-arm watcher) and direct reconnect (network blip --
+        recreate Prisma client, re-arm watcher, SELECT 1). Both paths recreate
+        the client via the non-blocking kill-then-construct flow rather than
+        calling disconnect(), which blocks the event loop on the synchronous
+        subprocess.Popen.wait() inside prisma-client-py (see issue #26191).
         """
         effective_timeout = (
             timeout_seconds
@@ -4204,7 +4222,6 @@ class PrismaClient:
             )
             self._reap_all_zombies()
             self._cleanup_engine_watcher()
-            self._engine_confirmed_dead = False
 
             async def _do_heavy_reconnect() -> None:
                 db_url = os.getenv("DATABASE_URL", "")
@@ -4217,23 +4234,32 @@ class PrismaClient:
                 await self._start_engine_watcher()
 
             await asyncio.wait_for(_do_heavy_reconnect(), timeout=effective_timeout)
+            # Only clear the "dead engine" flag after the heavy reconnect
+            # actually completed. If `_do_heavy_reconnect()` raises (timeout,
+            # missing DATABASE_URL, recreate failure), the flag stays True so
+            # the next attempt re-enters the heavy branch instead of silently
+            # demoting to the lightweight path.
+            self._engine_confirmed_dead = False
         else:
             verbose_proxy_logger.debug(
                 "Performing Prisma DB reconnect (engine alive or unknown)."
             )
 
             async def _do_direct_reconnect() -> None:
-                old_pid = self._get_engine_pid()
-                try:
-                    await self.db.disconnect()
-                except Exception as disconnect_err:
-                    verbose_proxy_logger.warning(
-                        "Prisma DB disconnect before reconnect failed: %s",
-                        disconnect_err,
+                db_url = os.getenv("DATABASE_URL", "")
+                if not db_url:
+                    verbose_proxy_logger.error(
+                        "DATABASE_URL not set; cannot reconnect Prisma client."
                     )
-                    await PrismaWrapper._kill_engine_process(old_pid)
-
-                await self.db.connect()
+                    raise RuntimeError("DATABASE_URL not set")
+                # Fresh Prisma client + new engine subprocess. The previous
+                # "lightweight" path called `disconnect()` which blocks the
+                # event loop on `subprocess.Popen.wait()`; since that call
+                # ends up killing the engine anyway, we do it non-blockingly
+                # via `_kill_engine_process` inside `recreate_prisma_client`.
+                self._cleanup_engine_watcher()
+                await self.db.recreate_prisma_client(db_url)
+                await self._start_engine_watcher()
                 await self.db.query_raw("SELECT 1")
 
             await asyncio.wait_for(_do_direct_reconnect(), timeout=effective_timeout)

litellm/proxy/vector_store_endpoints/management_endpoints.py

@@ -16,7 +16,9 @@ from fastapi import APIRouter, Depends, HTTPException
 
 import litellm
 from litellm._logging import verbose_proxy_logger
+from litellm.constants import REDACTED_BY_LITELM_STRING
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.sensitive_data_masker import SensitiveDataMasker
 from litellm.proxy._types import (
     LiteLLM_ManagedVectorStoresTable,
     ResponseLiteLLM_ManagedVectorStore,
@@ -38,6 +40,81 @@ from litellm.vector_stores.vector_store_registry import VectorStoreRegistry
 
 router = APIRouter()
 
+_LITELLM_PARAMS_MASKER = SensitiveDataMasker()
+
+
+_REDACT_LITELLM_PARAMS_MAX_DEPTH = 10
+
+
+def _redact_sensitive_litellm_params(litellm_params: Any, _depth: int = 0) -> Any:
+    """
+    Replace credential-bearing values in ``litellm_params`` with
+    ``REDACTED_BY_LITELM`` while preserving non-secret keys (``api_base``,
+    ``region``, ``model``, ``api_version``).
+
+    Handles three input shapes:
+
+    * ``dict`` — recurse into nested dicts (e.g. ``litellm_embedding_config``
+      which itself carries ``api_key`` / ``aws_*`` / ``vertex_credentials``).
+    * ``str`` — the in-memory registry occasionally holds the params as a
+      JSON-serialized string. Parse, redact, re-serialize. If parsing
+      fails, return the redaction sentinel rather than echo the value
+      back verbatim.
+    * Anything else, or ``None`` — passed through.
+
+    Recursion depth is bounded by ``_REDACT_LITELLM_PARAMS_MAX_DEPTH`` —
+    matching the convention of other allowlisted recursive helpers in the
+    repo (see ``tests/code_coverage_tests/recursive_detector.py``).
+    """
+    if _depth >= _REDACT_LITELLM_PARAMS_MAX_DEPTH:
+        return REDACTED_BY_LITELM_STRING
+    if litellm_params is None:
+        return None
+    if isinstance(litellm_params, str):
+        try:
+            parsed = json.loads(litellm_params)
+        except (TypeError, ValueError):
+            return REDACTED_BY_LITELM_STRING
+        return json.dumps(_redact_sensitive_litellm_params(parsed, _depth + 1))
+    if not isinstance(litellm_params, dict):
+        return litellm_params
+    out: Dict[str, Any] = {}
+    for k, v in litellm_params.items():
+        if _LITELLM_PARAMS_MASKER.is_sensitive_key(k):
+            out[k] = REDACTED_BY_LITELM_STRING
+        elif isinstance(v, dict):
+            out[k] = _redact_sensitive_litellm_params(v, _depth + 1)
+        else:
+            out[k] = v
+    return out
+
+
+async def _fetch_and_authorize_vector_store(
+    vector_store_id: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    prisma_client: Any,
+) -> "LiteLLM_ManagedVectorStore":
+    """
+    Look up a vector store by id and confirm the caller can access it.
+    Raises HTTPException(404) on miss and HTTPException(403) on access
+    denial.
+    """
+    row = await prisma_client.db.litellm_managedvectorstorestable.find_unique(
+        where={"vector_store_id": vector_store_id}
+    )
+    if row is None:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Vector store with ID {vector_store_id} not found",
+        )
+    typed = LiteLLM_ManagedVectorStore(**row.model_dump())
+    if not await _check_vector_store_access(typed, user_api_key_dict):
+        raise HTTPException(
+            status_code=403,
+            detail="Access denied: You do not have permission to access this vector store",
+        )
+    return typed
+
 
 def _resolve_embedding_config_from_router(
     embedding_model: str, llm_router
@@ -555,7 +632,11 @@ async def list_vector_stores(
         accessible_vector_stores = []
         for vs in vector_store_map.values():
             if await _check_vector_store_access(vs, user_api_key_dict):
-                accessible_vector_stores.append(vs)
+                redacted = LiteLLM_ManagedVectorStore(**vs)
+                redacted["litellm_params"] = _redact_sensitive_litellm_params(
+                    vs.get("litellm_params")
+                )
+                accessible_vector_stores.append(redacted)
 
         total_count = len(accessible_vector_stores)
         total_pages = (total_count + page_size - 1) // page_size
@@ -716,33 +797,29 @@ async def get_vector_store_info(
                     created_at=vector_store.get("created_at") or None,
                     updated_at=vector_store.get("updated_at") or None,
                     litellm_credential_name=vector_store.get("litellm_credential_name"),
-                    litellm_params=vector_store.get("litellm_params") or None,
+                    litellm_params=_redact_sensitive_litellm_params(
+                        vector_store.get("litellm_params")
+                    ),
                     team_id=vector_store.get("team_id") or None,
                     user_id=vector_store.get("user_id") or None,
                 )
                 return {"vector_store": vector_store_pydantic_obj}
 
-        vector_store = (
-            await prisma_client.db.litellm_managedvectorstorestable.find_unique(
-                where={"vector_store_id": data.vector_store_id}
-            )
+        vector_store_typed = await _fetch_and_authorize_vector_store(
+            vector_store_id=data.vector_store_id,
+            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
         )
-        if vector_store is None:
-            raise HTTPException(
-                status_code=404,
-                detail=f"Vector store with ID {data.vector_store_id} not found",
+        vector_store_dict = dict(vector_store_typed)
+        if "litellm_params" in vector_store_dict:
+            vector_store_dict["litellm_params"] = _redact_sensitive_litellm_params(
+                vector_store_dict["litellm_params"]
             )
-
-        # Check access control for DB vector store
-        vector_store_dict = vector_store.model_dump()  # type: ignore[attr-defined]
-        vector_store_typed = LiteLLM_ManagedVectorStore(**vector_store_dict)
-        if not await _check_vector_store_access(vector_store_typed, user_api_key_dict):
-            raise HTTPException(
-                status_code=403,
-                detail="Access denied: You do not have permission to access this vector store",
-            )
-
         return {"vector_store": vector_store_dict}
+    except HTTPException:
+        # Preserve 403/404 from the access-control / not-found checks above;
+        # the catch-all below would otherwise rewrite them as 500.
+        raise
     except Exception as e:
         verbose_proxy_logger.exception(f"Error getting vector store info: {str(e)}")
         raise HTTPException(status_code=500, detail=str(e))
@@ -773,6 +850,15 @@ async def update_vector_store(
         update_data = data.model_dump(exclude_unset=True)
         vector_store_id = update_data.pop("vector_store_id")
 
+        # Per-store access control: anyone authenticated who passes the
+        # premium-feature gate could otherwise update *any* vector store —
+        # including stores belonging to other teams.
+        await _fetch_and_authorize_vector_store(
+            vector_store_id=vector_store_id,
+            user_api_key_dict=user_api_key_dict,
+            prisma_client=prisma_client,
+        )
+
         # Handle metadata serialization
         if update_data.get("vector_store_metadata") is not None:
             update_data["vector_store_metadata"] = safe_dumps(
@@ -820,11 +906,24 @@ async def update_vector_store(
                 f"Updated vector store {vector_store_id} in both database and in-memory registry"
             )
 
+        # The DB row is returned in full, so the response would otherwise
+        # echo the persisted ``litellm_params`` (including provider
+        # credentials) back to the caller — even when the caller only
+        # changed unrelated fields like ``vector_store_description``.
+        response_vs = LiteLLM_ManagedVectorStore(**updated_vs)
+        response_vs["litellm_params"] = _redact_sensitive_litellm_params(
+            updated_vs.get("litellm_params")
+        )
         return {
             "status": "success",
             "message": f"Vector store {vector_store_id} updated successfully",
-            "vector_store": updated_vs,
+            "vector_store": response_vs,
         }
+    except HTTPException:
+        # Preserve 403/404 responses from the access-control / not-found
+        # checks above; the catch-all below would otherwise rewrite them
+        # as 500 with the original status code embedded in the detail.
+        raise
     except Exception as e:
         verbose_proxy_logger.exception(f"Error updating vector store: {str(e)}")
         raise HTTPException(status_code=500, detail=str(e))

litellm/proxy/workflows/README.md

@@ -0,0 +1,150 @@
+# Workflow Run Tracking
+
+Generic durable state tracking for agents and automated workflows built on the LiteLLM proxy.
+
+## The Problem
+
+Agents like [shin-builder](https://github.com/BerriAI/shin-builder) run multi-stage pipelines (triage → plan → implement → PR). Their task state and conversation history lived in memory — a process restart lost everything.
+
+## Three-Table Design
+
+```
+WorkflowRun      one instance of work (header + materialized status)
+WorkflowEvent    append-only state transitions (source of truth for replay)
+WorkflowMessage  conversation inbox/outbox (full content, not truncated)
+```
+
+**WorkflowEvent is the source of truth.** `WorkflowRun.status` is a materialized cache updated automatically when events are appended. If you need to debug a run, replay its events.
+
+## API
+
+All endpoints require a valid LiteLLM API key (`Authorization: Bearer sk-...`).
+
+### Runs
+
+```
+POST   /v1/workflows/runs                  Create a run
+GET    /v1/workflows/runs                  List runs (?workflow_type=&status=)
+GET    /v1/workflows/runs/{run_id}         Get run + latest event
+PATCH  /v1/workflows/runs/{run_id}         Update status / metadata / output
+```
+
+### Events
+
+```
+POST   /v1/workflows/runs/{run_id}/events  Append event (auto-updates run status)
+GET    /v1/workflows/runs/{run_id}/events  Full event log (ordered by sequence)
+```
+
+### Messages
+
+```
+POST   /v1/workflows/runs/{run_id}/messages  Append message
+GET    /v1/workflows/runs/{run_id}/messages  Conversation history (ordered by sequence)
+```
+
+## Quick Start
+
+```bash
+# Create a run
+curl -X POST http://localhost:4000/v1/workflows/runs \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{"workflow_type": "shin-builder", "metadata": {"title": "Fix login bug"}}'
+
+# {"run_id": "abc-123", "session_id": "xyz-456", "status": "pending", ...}
+
+# Mark step started (sets status → running)
+curl -X POST http://localhost:4000/v1/workflows/runs/abc-123/events \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{"event_type": "step.started", "step_name": "grill", "data": {"claude_session_id": "sess-789"}}'
+
+# Store a conversation message
+curl -X POST http://localhost:4000/v1/workflows/runs/abc-123/messages \
+  -H "Authorization: Bearer sk-1234" \
+  -H "Content-Type: application/json" \
+  -d '{"role": "user", "content": "What is the expected behavior?", "session_id": "sess-789"}'
+
+# Restart recovery: fetch active runs and resume from last event's data.claude_session_id
+curl "http://localhost:4000/v1/workflows/runs?status=running,paused&workflow_type=shin-builder" \
+  -H "Authorization: Bearer sk-1234"
+```
+
+## Status Auto-Update Rules
+
+When you append an event, the run's status is updated automatically:
+
+| event_type      | run.status |
+|-----------------|------------|
+| `step.started`  | `running`  |
+| `step.failed`   | `failed`   |
+| `hook.waiting`  | `paused`   |
+| `hook.received` | `running`  |
+
+Set `status = completed` explicitly via PATCH when the workflow finishes.
+
+## Linking to Spend Logs
+
+`WorkflowRun.session_id` is generated automatically (UUID). Pass it as the `x-litellm-session-id` header when making completions through the proxy:
+
+```python
+headers = {"x-litellm-session-id": run.session_id}
+```
+
+All spend log entries for this run are then tagged automatically. Query cost per run:
+
+```
+POST /ui/spend_logs/view_session_spend_logs?session_id={run.session_id}
+```
+
+## Sequence Numbers
+
+Sequence numbers on events and messages are assigned server-side (`MAX + 1` per run). Callers never supply them. This guarantees ordering even under concurrent writes.
+
+## Using from shin-builder
+
+Replace the in-memory `tasks.py` dict with calls to these endpoints:
+
+```python
+import httpx
+
+class WorkflowRunClient:
+    def __init__(self, base_url: str, api_key: str):
+        self._client = httpx.AsyncClient(
+            base_url=base_url,
+            headers={"Authorization": f"Bearer {api_key}"},
+        )
+
+    async def create_task(self, title: str, **metadata) -> dict:
+        r = await self._client.post("/v1/workflows/runs", json={
+            "workflow_type": "shin-builder",
+            "metadata": {"title": title, **metadata},
+        })
+        r.raise_for_status()
+        return r.json()
+
+    async def list_active_tasks(self) -> list:
+        r = await self._client.get(
+            "/v1/workflows/runs",
+            params={"workflow_type": "shin-builder", "status": "running,paused"},
+        )
+        r.raise_for_status()
+        return r.json()["runs"]
+
+    async def transition(self, run_id: str, step_name: str, event_type: str, data: dict = None):
+        r = await self._client.post(f"/v1/workflows/runs/{run_id}/events", json={
+            "event_type": event_type,
+            "step_name": step_name,
+            "data": data or {},
+        })
+        r.raise_for_status()
+
+    async def append_message(self, run_id: str, role: str, content: str, session_id: str = None):
+        r = await self._client.post(f"/v1/workflows/runs/{run_id}/messages", json={
+            "role": role, "content": content, "session_id": session_id,
+        })
+        r.raise_for_status()
+```
+
+On startup, call `list_active_tasks()` to restore in-flight runs. The last `step.started` event's `data.claude_session_id` gives you the `--resume` ID.

litellm/router_utils/clientside_credential_handler.py

@@ -11,9 +11,60 @@ If given, generate a unique model_id for the deployment.
 Ensures cooldowns are applied correctly.
 """
 
+from typing import List
+
 clientside_credential_keys = ["api_key", "api_base", "base_url"]
 
 
+def _admin_config_fields_to_clear_on_base_override() -> List[str]:
+    """
+    Provider-specific credential / endpoint-targeting fields that must NOT
+    flow through to a client-redirected upstream.
+
+    Built dynamically from ``CredentialLiteLLMParams.model_fields`` so any
+    new provider field added there (Bedrock endpoint, Watsonx region, etc.)
+    is gated automatically — plus a fixed list of kwargs-only fields that
+    aren't declared on the typed model.
+    """
+    from litellm.types.router import CredentialLiteLLMParams
+
+    typed_fields = [
+        f
+        for f in CredentialLiteLLMParams.model_fields
+        if f not in clientside_credential_keys
+    ]
+    kwargs_only_fields = [
+        # Caller-supplied via **kwargs, not declared on CredentialLiteLLMParams.
+        "organization",
+        "extra_body",
+        "extra_headers",
+        "default_headers",
+        "api_type",
+        "azure_ad_token",
+        "azure_ad_token_provider",
+        "aws_session_token",
+        "aws_sts_endpoint",
+        "aws_web_identity_token",
+        "aws_role_name",
+        # OCI provider — consumed by litellm/llms/oci/* via optional_params
+        # and not declared on CredentialLiteLLMParams. Without these here,
+        # an admin's OCI signing key / tenancy / fingerprint would flow
+        # through to an attacker-redirected upstream.
+        "oci_signer",
+        "oci_user",
+        "oci_fingerprint",
+        "oci_tenancy",
+        "oci_key",
+        "oci_key_file",
+    ]
+    return typed_fields + kwargs_only_fields
+
+
+_ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE = (
+    _admin_config_fields_to_clear_on_base_override()
+)
+
+
 def is_clientside_credential(request_kwargs: dict) -> bool:
     """
     Check if the credential is a clientside credential.
@@ -34,4 +85,20 @@ def get_dynamic_litellm_params(litellm_params: dict, request_kwargs: dict) -> di
     for key in clientside_credential_keys:
         if key in request_kwargs:
             litellm_params[key] = request_kwargs[key]
+
+    # If the caller redirected api_base/base_url to a client-controlled value,
+    # don't forward the admin's organization / extra_body / region / token /
+    # vertex / aws fields — those were meant for the original upstream.
+    # Always drop the admin's value first, then write the caller's value back
+    # if they resupplied the field. The naive
+    # ``if field not in request_kwargs: pop`` shape lets a caller *echo* a
+    # field name (with any value, including an empty string) to keep the
+    # admin's value in ``litellm_params`` and have it forwarded to the
+    # redirected upstream.
+    if "api_base" in request_kwargs or "base_url" in request_kwargs:
+        for field in _ADMIN_CONFIG_FIELDS_TO_CLEAR_ON_BASE_OVERRIDE:
+            litellm_params.pop(field, None)
+            if field in request_kwargs:
+                litellm_params[field] = request_kwargs[field]
+
     return litellm_params

litellm/router_utils/search_api_router.py

@@ -8,7 +8,7 @@ import asyncio
 import random
 import traceback
 from functools import partial
-from typing import Any, Callable
+from typing import Any, Callable, Dict, Optional, Tuple
 
 from litellm._logging import verbose_router_logger
 
@@ -20,6 +20,28 @@ class SearchAPIRouter:
     Provides methods for search tool selection, load balancing, and fallback handling.
     """
 
+    @staticmethod
+    def _resolve_search_provider_credentials(
+        *,
+        tool_litellm_params: Dict[str, Any],
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """
+        Resolve search provider credentials from tool configuration ONLY.
+
+        Credentials are stored only in search_tool.litellm_params, never in team/key metadata.
+        This ensures secrets are not exposed in team/key API responses.
+
+        Args:
+            tool_litellm_params: Search tool litellm_params with credentials
+
+        Returns:
+            Tuple of (api_key, api_base) from tool configuration
+        """
+        resolved_api_key: Optional[str] = tool_litellm_params.get("api_key")
+        resolved_api_base: Optional[str] = tool_litellm_params.get("api_base")
+
+        return resolved_api_key, resolved_api_base
+
     @staticmethod
     async def update_router_search_tools(router_instance: Any, search_tools: list):
         """
@@ -198,14 +220,15 @@ class SearchAPIRouter:
             # Extract search provider and other params from litellm_params
             litellm_params = selected_tool.get("litellm_params", {})
             search_provider = litellm_params.get("search_provider")
-            api_key = litellm_params.get("api_key")
-            api_base = litellm_params.get("api_base")
-
             if not search_provider:
                 raise ValueError(
                     f"search_provider not found in litellm_params for search tool '{search_tool_name}'"
                 )
 
+            api_key, api_base = SearchAPIRouter._resolve_search_provider_credentials(
+                tool_litellm_params=litellm_params,
+            )
+
             verbose_router_logger.debug(
                 f"Selected search tool with provider: {search_provider}"
             )

litellm/types/llms/anthropic.py

@@ -330,7 +330,11 @@ class AnthropicMessagesToolResultParam(TypedDict, total=False):
     content: Union[
         str,
         Iterable[
-            Union[AnthropicMessagesToolResultContent, AnthropicMessagesImageParam]
+            Union[
+                AnthropicMessagesToolResultContent,
+                AnthropicMessagesImageParam,
+                AnthropicMessagesDocumentParam,
+            ]
         ],
     ]
     cache_control: Optional[Union[dict, ChatCompletionCachedContent]]

litellm/types/mcp_server/mcp_server_manager.py

@@ -81,6 +81,12 @@ class MCPServer(BaseModel):
     # Defaults to the token's expires_in minus the expiry buffer, or
     # MCP_PER_USER_TOKEN_DEFAULT_TTL when expires_in is absent.
     token_storage_ttl_seconds: Optional[int] = None
+    # Resolved short-ID tool prefix when LITELLM_USE_SHORT_MCP_TOOL_PREFIX is
+    # enabled.  Set by ``MCPServerManager._assign_unique_short_prefix`` at
+    # registration time so that natural-hash collisions between two
+    # different ``server_id`` values are bumped deterministically.  Left
+    # ``None`` in default-prefix mode.
+    short_prefix: Optional[str] = None
     model_config = ConfigDict(arbitrary_types_allowed=True)
 
     @property

litellm/utils.py

@@ -6526,6 +6526,7 @@ def validate_environment(  # noqa: PLR0915
             or model in litellm.open_ai_text_completion_models
             or model in litellm.open_ai_embedding_models
             or model in litellm.openai_image_generation_models
+            or model.startswith("gpt-image")
         ):
             if "OPENAI_API_KEY" in os.environ:
                 keys_in_environment = True

model_prices_and_context_window.json

@@ -712,6 +712,7 @@
     },
     "anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -735,6 +736,7 @@
     },
     "anthropic.claude-haiku-4-5@20251001": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -955,6 +957,7 @@
     },
     "anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -982,6 +985,7 @@
     },
     "anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1011,6 +1015,7 @@
     },
     "global.anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1040,6 +1045,7 @@
     },
     "us.anthropic.claude-opus-4-6-v1": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1127,6 +1133,7 @@
     },
     "anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1171,6 +1178,7 @@
     },
     "global.anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1201,6 +1209,7 @@
     },
     "us.anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -1291,6 +1300,7 @@
     },
     "anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1319,6 +1329,7 @@
     },
     "global.anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1347,6 +1358,7 @@
     },
     "us.anthropic.claude-sonnet-4-6": {
         "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
         "cache_read_input_token_cost": 3.3e-07,
         "input_cost_per_token": 3.3e-06,
         "litellm_provider": "bedrock_converse",
@@ -1461,11 +1473,13 @@
     },
     "anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "input_cost_per_token_above_200k_tokens": 6e-06,
         "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -5117,6 +5131,38 @@
             "/v1/images/edits"
         ]
     },
+    "azure/gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "azure/gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "input_cost_per_image_token": 8e-06,
+        "litellm_provider": "azure",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "azure/low/1024-x-1024/gpt-image-1-mini": {
         "input_cost_per_pixel": 2.0751953125e-09,
         "litellm_provider": "azure",
@@ -17903,11 +17949,13 @@
     },
     "global.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 3.75e-06,
+        "cache_creation_input_token_cost_above_1hr": 6e-06,
         "cache_read_input_token_cost": 3e-07,
         "input_cost_per_token": 3e-06,
         "input_cost_per_token_above_200k_tokens": 6e-06,
         "output_cost_per_token_above_200k_tokens": 2.25e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 7.5e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.2e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -17964,6 +18012,7 @@
     },
     "global.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 2e-06,
         "cache_read_input_token_cost": 1e-07,
         "input_cost_per_token": 1e-06,
         "litellm_provider": "bedrock_converse",
@@ -19097,6 +19146,38 @@
         "supports_vision": true,
         "supports_pdf_input": true
     },
+    "gpt-image-2": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
+    "gpt-image-2-2026-04-21": {
+        "cache_read_input_image_token_cost": 2e-06,
+        "cache_read_input_token_cost": 1.25e-06,
+        "input_cost_per_token": 5e-06,
+        "litellm_provider": "openai",
+        "mode": "image_generation",
+        "output_cost_per_token": 1e-05,
+        "input_cost_per_image_token": 8e-06,
+        "output_cost_per_image_token": 3e-05,
+        "supported_endpoints": [
+            "/v1/images/generations",
+            "/v1/images/edits"
+        ],
+        "supports_vision": true,
+        "supports_pdf_input": true
+    },
     "low/1024-x-1024/gpt-image-1.5": {
         "input_cost_per_image": 0.009,
         "litellm_provider": "openai",
@@ -30106,6 +30187,7 @@
     },
     "us.anthropic.claude-haiku-4-5-20251001-v1:0": {
         "cache_creation_input_token_cost": 1.375e-06,
+        "cache_creation_input_token_cost_above_1hr": 2.2e-06,
         "cache_read_input_token_cost": 1.1e-07,
         "input_cost_per_token": 1.1e-06,
         "litellm_provider": "bedrock_converse",
@@ -30257,11 +30339,13 @@
     },
     "us.anthropic.claude-sonnet-4-5-20250929-v1:0": {
         "cache_creation_input_token_cost": 4.125e-06,
+        "cache_creation_input_token_cost_above_1hr": 6.6e-06,
         "cache_read_input_token_cost": 3.3e-07,
         "input_cost_per_token": 3.3e-06,
         "input_cost_per_token_above_200k_tokens": 6.6e-06,
         "output_cost_per_token_above_200k_tokens": 2.475e-05,
         "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06,
+        "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05,
         "cache_read_input_token_cost_above_200k_tokens": 6.6e-07,
         "litellm_provider": "bedrock_converse",
         "max_input_tokens": 200000,
@@ -30362,6 +30446,7 @@
     },
     "us.anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.875e-06,
+        "cache_creation_input_token_cost_above_1hr": 1.1e-05,
         "cache_read_input_token_cost": 5.5e-07,
         "input_cost_per_token": 5.5e-06,
         "litellm_provider": "bedrock_converse",
@@ -30389,6 +30474,7 @@
     },
     "global.anthropic.claude-opus-4-5-20251101-v1:0": {
         "cache_creation_input_token_cost": 6.25e-06,
+        "cache_creation_input_token_cost_above_1hr": 1e-05,
         "cache_read_input_token_cost": 5e-07,
         "input_cost_per_token": 5e-06,
         "litellm_provider": "bedrock_converse",

provider_endpoints_support.json

@@ -193,6 +193,23 @@
         "a2a": false
       }
     },
+    "aihubmix": {
+      "display_name": "AIHubMix (`aihubmix`)",
+      "url": "https://docs.litellm.ai/docs/providers/aihubmix",
+      "endpoints": {
+        "chat_completions": true,
+        "messages": true,
+        "responses": true,
+        "embeddings": true,
+        "image_generations": true,
+        "audio_transcriptions": true,
+        "audio_speech": true,
+        "moderations": true,
+        "batches": false,
+        "rerank": true,
+        "a2a": false
+      }
+    },
     "assemblyai": {
       "display_name": "AssemblyAI (`assemblyai`)",
       "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai",

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.83.14"
+version = "1.84.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -236,7 +236,7 @@ source-exclude = [
 profile = "black"
 
 [tool.commitizen]
-version = "1.83.14"
+version = "1.84.0"
 version_files = [
     "pyproject.toml:^version",
 ]

schema.prisma

@@ -277,6 +277,7 @@ model LiteLLM_ObjectPermissionTable {
   models                String[]       @default([])
   blocked_tools         String[]       @default([]) // Tool names blocked for any key/team/user with this permission
   mcp_toolsets          String[]       @default([]) // Toolset IDs granted to this key/team/user
+  search_tools          String[]       @default([]) // search_tool_name values this key/team/user may call
   teams                 LiteLLM_TeamTable[]
   projects              LiteLLM_ProjectTable[]
   verification_tokens   LiteLLM_VerificationToken[]
@@ -1290,3 +1291,80 @@ model LiteLLM_AdaptiveRouterSession {
   @@id([session_id, router_name, model_name])
   @@index([last_activity_at], map: "idx_adaptive_router_session_activity")
 }
+
+// ---------------------------------------------------------------------------
+// Workflow Run Tracking
+//
+// Generic durable state tracking for any agent or automated workflow.
+// Design: three tables — run (header + materialized status), event (append-only
+// source of truth for state transitions), message (conversation inbox/outbox).
+//
+// Usage:
+//   - Set `workflow_type` to identify the owning system (e.g. "shin-builder").
+//   - Store domain-specific fields in `metadata` (worktree_path, pr_url, etc.).
+//   - `session_id` on WorkflowRun matches `x-litellm-session-id` header sent to
+//     the proxy — all spend logs for this run are automatically tagged.
+// ---------------------------------------------------------------------------
+
+// One instance of work being done. `status` is a materialized cache of the
+// latest event; the event log is the authoritative source of truth.
+model LiteLLM_WorkflowRun {
+  run_id        String   @id @default(uuid())
+  session_id    String   @unique @default(uuid())
+  workflow_type String
+  status        String   @default("pending")
+  created_by    String?  // user_id of the key that created this run; null = created by master key
+  created_at    DateTime @default(now())
+  updated_at    DateTime @updatedAt
+  input         Json?
+  output        Json?
+  metadata      Json?
+
+  events   LiteLLM_WorkflowEvent[]
+  messages LiteLLM_WorkflowMessage[]
+
+  @@index([workflow_type, status])
+  @@index([session_id])
+  @@index([created_at])
+  @@index([created_by])
+}
+
+// Append-only log of state transitions. Never mutate rows here.
+// `step_name` and `event_type` are caller-defined strings — no hardcoded enums.
+// Status auto-update rules (applied by the append endpoint):
+//   step.started   → run.status = running
+//   step.failed    → run.status = failed
+//   hook.waiting   → run.status = paused
+//   hook.received  → run.status = running
+model LiteLLM_WorkflowEvent {
+  event_id        String   @id @default(uuid())
+  run_id          String
+  event_type      String
+  step_name       String
+  sequence_number Int
+  data            Json?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}
+
+// Conversation inbox/outbox — full message content, separate from the durable
+// event log. Spend logs truncate messages; this table stores them in full.
+// `session_id` here is the Claude --resume session ID (or similar).
+model LiteLLM_WorkflowMessage {
+  message_id      String   @id @default(uuid())
+  run_id          String
+  role            String
+  content         String
+  sequence_number Int
+  session_id      String?
+  created_at      DateTime @default(now())
+
+  run LiteLLM_WorkflowRun @relation(fields: [run_id], references: [run_id])
+
+  @@unique([run_id, sequence_number])
+  @@index([run_id])
+}

tests/batches_tests/test_openai_batches_and_files.py

@@ -557,6 +557,7 @@ async def test_avertex_batch_prediction(monkeypatch):
             mock_get_response = MagicMock()
             mock_get_response.json.return_value = mock_vertex_batch_response
             mock_get_response.status_code = 200
+            mock_get_response.is_redirect = False
             mock_get_response.raise_for_status.return_value = None
             mock_get.return_value = mock_get_response
 

tests/code_coverage_tests/recursive_detector.py

@@ -46,6 +46,7 @@ IGNORE_FUNCTIONS = [
     "dict",  # max depth set. _LiteLLMParamsDictView.dict() calls builtin dict(), not itself.
     "_read_image_bytes",  # max depth set.
     "_get_masked_values",  # max depth set (default 20) to prevent infinite recursion while masking nested sensitive config dicts.
+    "_redact_sensitive_litellm_params",  # max depth set (default 10).
 ]
 
 

tests/litellm/proxy/test_prisma_engine_watchdog.py

@@ -254,32 +254,48 @@ async def test_run_reconnect_cycle_uses_heavy_path_when_confirmed_dead(
 
 
 @pytest.mark.asyncio
-async def test_run_reconnect_cycle_uses_lightweight_path_when_engine_alive(
+async def test_run_reconnect_cycle_uses_direct_path_when_engine_alive(
     engine_client,
 ) -> None:
-    """_run_reconnect_cycle uses disconnect/connect when engine is alive."""
-    engine_client._engine_pid = 1234
+    """Direct reconnect (engine alive) calls recreate_prisma_client + SELECT 1.
 
-    with patch.object(engine_client, "_is_engine_alive", return_value=True):
+    The old "lightweight" path called `disconnect()` + `connect()`, which
+    blocks the event loop on the sync `process.wait()` inside aclose().
+    The fix routes both engine-alive and engine-dead paths through
+    `recreate_prisma_client`, which non-blockingly kills the old engine.
+    """
+    engine_client._engine_pid = 1234
+    engine_client._start_engine_watcher = AsyncMock()
+
+    with (
+        patch.object(engine_client, "_is_engine_alive", return_value=True),
+        patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}),
+    ):
         await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
 
-    engine_client.db.connect.assert_awaited_once()
+    engine_client.db.recreate_prisma_client.assert_awaited_once_with(
+        "postgresql://test"
+    )
     engine_client.db.query_raw.assert_awaited_once_with("SELECT 1")
-    engine_client.db.recreate_prisma_client.assert_not_awaited()
+    engine_client.db.disconnect.assert_not_awaited()
 
 
 @pytest.mark.asyncio
-async def test_run_reconnect_cycle_uses_lightweight_path_when_pid_unknown(
+async def test_run_reconnect_cycle_uses_direct_path_when_pid_unknown(
     engine_client,
 ) -> None:
-    """_run_reconnect_cycle uses lightweight path when engine PID is not tracked."""
+    """When the engine PID is not tracked, direct reconnect still runs."""
     engine_client._engine_pid = 0
+    engine_client._start_engine_watcher = AsyncMock()
 
-    await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
+    with patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}):
+        await engine_client._run_reconnect_cycle(timeout_seconds=5.0)
 
-    engine_client.db.connect.assert_awaited_once()
+    engine_client.db.recreate_prisma_client.assert_awaited_once_with(
+        "postgresql://test"
+    )
     engine_client.db.query_raw.assert_awaited_once_with("SELECT 1")
-    engine_client.db.recreate_prisma_client.assert_not_awaited()
+    engine_client.db.disconnect.assert_not_awaited()
 
 
 @pytest.mark.asyncio
@@ -473,36 +489,38 @@ def test_on_engine_death_from_thread_ignores_stale_pid(engine_client):
 
 
 @pytest.mark.asyncio
-async def test_escalation_after_consecutive_lightweight_failures(engine_client):
-    """After N consecutive lightweight reconnect failures, _engine_confirmed_dead
+async def test_escalation_after_consecutive_direct_reconnect_failures(engine_client):
+    """After N consecutive direct reconnect failures, _engine_confirmed_dead
     is set to True so _run_reconnect_cycle takes the heavy reconnect path."""
     engine_client._reconnect_escalation_threshold = 3
     engine_client._consecutive_reconnect_failures = 0
     engine_client._db_reconnect_cooldown_seconds = 0  # disable cooldown for test
+    engine_client._start_engine_watcher = AsyncMock(return_value=None)
 
-    # Make lightweight reconnect fail every time
-    engine_client.db.disconnect = AsyncMock(return_value=None)
-    engine_client.db.connect = AsyncMock(side_effect=Exception("connect failed"))
+    # Make direct reconnect fail every time
+    engine_client.db.recreate_prisma_client = AsyncMock(
+        side_effect=Exception("recreate failed")
+    )
 
     # Run 3 failed reconnect attempts
-    for i in range(3):
-        result = await engine_client._attempt_reconnect_inside_lock(
-            force=True, reason="test", timeout_seconds=5.0
-        )
-        assert result is False
+    with patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}):
+        for _ in range(3):
+            result = await engine_client._attempt_reconnect_inside_lock(
+                force=True, reason="test", timeout_seconds=5.0
+            )
+            assert result is False
 
     assert engine_client._consecutive_reconnect_failures == 3
 
-    # Next attempt should escalate: _engine_confirmed_dead set to True before _run_reconnect_cycle
+    # Next attempt should escalate to the heavy path (recreate_prisma_client still
+    # the call, but via the _engine_confirmed_dead branch that also re-arms the watcher).
     engine_client.db.recreate_prisma_client = AsyncMock(return_value=None)
-    engine_client._start_engine_watcher = AsyncMock(return_value=None)
 
     with patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}):
         result = await engine_client._attempt_reconnect_inside_lock(
             force=True, reason="test_escalation", timeout_seconds=5.0
         )
 
-    # Heavy reconnect should have been attempted (recreate_prisma_client called)
     engine_client.db.recreate_prisma_client.assert_awaited_once()
 
 
@@ -511,15 +529,16 @@ async def test_successful_reconnect_resets_failure_counter(engine_client):
     """A successful reconnect resets _consecutive_reconnect_failures to 0."""
     engine_client._consecutive_reconnect_failures = 2
     engine_client._db_reconnect_cooldown_seconds = 0
+    engine_client._start_engine_watcher = AsyncMock()
 
     # Make reconnect succeed
-    engine_client.db.disconnect = AsyncMock(return_value=None)
-    engine_client.db.connect = AsyncMock(return_value=None)
+    engine_client.db.recreate_prisma_client = AsyncMock(return_value=None)
     engine_client.db.query_raw = AsyncMock(return_value=[{"result": 1}])
 
-    result = await engine_client._attempt_reconnect_inside_lock(
-        force=True, reason="test", timeout_seconds=5.0
-    )
+    with patch.dict(os.environ, {"DATABASE_URL": "postgresql://test"}):
+        result = await engine_client._attempt_reconnect_inside_lock(
+            force=True, reason="test", timeout_seconds=5.0
+        )
 
     assert result is True
     assert engine_client._consecutive_reconnect_failures == 0

tests/local_testing/test_model_alias_map.py

@@ -30,10 +30,9 @@ def test_model_alias_map(caplog):
         )
         print(response.model)
 
-        captured_logs = [rec.levelname for rec in caplog.records]
-
-        for log in captured_logs:
-            assert "ERROR" not in log
+        for rec in caplog.records:
+            if rec.levelname == "ERROR" and rec.name.startswith("LiteLLM"):
+                pytest.fail(f"Unexpected litellm ERROR log: {rec.getMessage()}")
 
         assert "llama-3.1-8b-instant" in response.model
     except litellm.ServiceUnavailableError:

tests/logging_callback_tests/test_bedrock_knowledgebase_hook.py

@@ -354,6 +354,7 @@ async def test_bedrock_kb_request_body_has_transformed_filters(
                 api_base=api_base,
                 litellm_logging_obj=logging_obj,
                 litellm_params=litellm_params_dict,
+                extra_body=None,
             )
         )
         captured_request_body["url"] = url

tests/otel_tests/test_prometheus.py

@@ -115,6 +115,13 @@ async def test_proxy_failure_metrics():
             "litellm_llm_api_failed_requests_metric_total{",  # Deprecated but may still be used
         ]
 
+        # Master-key auth substitutes LITELLM_PROXY_MASTER_KEY_ALIAS for
+        # hash_token(master_key) so the master key (or its hash) never
+        # propagates into metrics. See PR #26484.
+        from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+        expected_hashed_api_key = LITELLM_PROXY_MASTER_KEY_ALIAS
+
         # Check if either pattern is in metrics and contains required fields
         found_metric = False
         for pattern in expected_patterns:
@@ -125,8 +132,7 @@ async def test_proxy_failure_metrics():
                         'api_key_alias="None"' in line
                         and 'exception_class="Openai.RateLimitError"' in line
                         and 'exception_status="429"' in line
-                        and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                        in line
+                        and f'hashed_api_key="{expected_hashed_api_key}"' in line
                         and 'requested_model="fake-azure-endpoint"' in line
                         and 'route="/chat/completions"' in line
                     ):
@@ -135,8 +141,7 @@ async def test_proxy_failure_metrics():
                 # For deprecated llm_api metric, check llm-specific fields
                 elif "litellm_llm_api_failed_requests_metric_total{" in line:
                     if (
-                        'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                        in line
+                        f'hashed_api_key="{expected_hashed_api_key}"' in line
                         and 'model="429"' in line
                     ):  # The deprecated metric uses the actual model from the request
                         found_metric = True
@@ -156,8 +161,7 @@ async def test_proxy_failure_metrics():
         for line in metrics.split("\n"):
             if (
                 total_requests_pattern in line
-                and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                in line
+                and f'hashed_api_key="{expected_hashed_api_key}"' in line
                 and 'requested_model="fake-azure-endpoint"' in line
                 and 'status_code="429"' in line
             ):
@@ -195,6 +199,12 @@ async def test_proxy_success_metrics():
 
         assert END_USER_ID not in metrics
 
+        # Master-key auth substitutes LITELLM_PROXY_MASTER_KEY_ALIAS for
+        # hash_token(master_key) (PR #26484).
+        from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+        expected_hashed_api_key = LITELLM_PROXY_MASTER_KEY_ALIAS
+
         # Check if the success metric is present and correct - use flexible matching
         # Check for request_total_latency_metric with required fields
         # Note: The model can be "gpt-3.5-turbo-0301" or similar depending on what's returned
@@ -203,8 +213,7 @@ async def test_proxy_success_metrics():
             if (
                 "litellm_request_total_latency_metric_bucket{" in line
                 and 'api_key_alias="None"' in line
-                and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                in line
+                and f'hashed_api_key="{expected_hashed_api_key}"' in line
                 and 'requested_model="fake-openai-endpoint"' in line
                 and 'le="0.005"' in line
             ):
@@ -221,8 +230,7 @@ async def test_proxy_success_metrics():
             if (
                 "litellm_llm_api_latency_metric_bucket{" in line
                 and 'api_key_alias="None"' in line
-                and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                in line
+                and f'hashed_api_key="{expected_hashed_api_key}"' in line
                 and 'requested_model="fake-openai-endpoint"' in line
                 and 'le="0.005"' in line
             ):
@@ -298,6 +306,12 @@ async def test_proxy_fallback_metrics():
 
         print("/metrics", metrics)
 
+        # Master-key auth substitutes LITELLM_PROXY_MASTER_KEY_ALIAS for
+        # hash_token(master_key) (PR #26484).
+        from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+        expected_hashed_api_key = LITELLM_PROXY_MASTER_KEY_ALIAS
+
         # Check if successful fallback metric is incremented - use flexible matching
         found_successful_fallback = False
         for line in metrics.split("\n"):
@@ -307,8 +321,7 @@ async def test_proxy_fallback_metrics():
                 and 'exception_class="Openai.RateLimitError"' in line
                 and 'exception_status="429"' in line
                 and 'fallback_model="fake-openai-endpoint"' in line
-                and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                in line
+                and f'hashed_api_key="{expected_hashed_api_key}"' in line
                 and 'requested_model="fake-azure-endpoint"' in line
                 and "1.0" in line
             ):
@@ -328,8 +341,7 @@ async def test_proxy_fallback_metrics():
                 and 'exception_class="Openai.RateLimitError"' in line
                 and 'exception_status="429"' in line
                 and 'fallback_model="unknown-model"' in line
-                and 'hashed_api_key="88dc28d0f030c55ed4ab77ed8faf098196cb1c05df778539800c9f1243fe6b4b"'
-                in line
+                and f'hashed_api_key="{expected_hashed_api_key}"' in line
                 and 'requested_model="fake-azure-endpoint"' in line
                 and "1.0" in line
             ):

tests/proxy_admin_ui_tests/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

tests/proxy_admin_ui_tests/ui_unit_tests/.npmrc

@@ -2,4 +2,4 @@
 # Packages needing lifecycle scripts: npm rebuild <pkg>
 ignore-scripts=true
 # Protects local npm install only — npm ci (used in CI) ignores this
-min-release-age=3d
+min-release-age=3

tests/proxy_unit_tests/test_key_generate_prisma.py

@@ -2715,7 +2715,12 @@ async def test_master_key_hashing(prisma_client):
             request=request, api_key=bearer_token
         )
 
-        assert result.api_key == hash_token(master_key)
+        # Master-key auth substitutes a stable alias so the master key (or
+        # its hash) never propagates into spend logs / metrics / audit trails.
+        from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+
+        assert result.api_key == LITELLM_PROXY_MASTER_KEY_ALIAS
+        assert result.api_key != hash_token(master_key)
 
     except Exception as e:
         print("Got Exception", e)

tests/proxy_unit_tests/test_user_api_key_auth.py

@@ -1123,11 +1123,17 @@ async def test_jwt_non_admin_team_route_access(monkeypatch):
 @pytest.mark.asyncio
 async def test_x_litellm_api_key():
     """
-    Check if auth can pick up x-litellm-api-key header, even if Bearer token is provided
+    Check if auth can pick up x-litellm-api-key header, even if Bearer token is provided.
+
+    On a master-key match, ``UserAPIKeyAuth.api_key`` (and the derived
+    ``token``) are now the stable alias ``LITELLM_PROXY_MASTER_KEY_ALIAS``
+    rather than ``hash_token(master_key)`` — the master key (or its hash)
+    must not propagate into spend logs / metrics / audit trails.
     """
     from fastapi import Request
     from starlette.datastructures import URL
 
+    from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
     from litellm.proxy._types import (
         LiteLLM_TeamTable,
         LiteLLM_TeamTableCachedObj,
@@ -1153,7 +1159,8 @@ async def test_x_litellm_api_key():
         api_key="Bearer " + ignored_key,
         custom_litellm_key_header=master_key,
     )
-    assert valid_token.token == hash_token(master_key)
+    assert valid_token.token == LITELLM_PROXY_MASTER_KEY_ALIAS
+    assert valid_token.token != hash_token(master_key)
 
 
 @pytest.mark.asyncio

tests/test_litellm/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py

@@ -2476,3 +2476,186 @@ def test_bedrock_tools_pt_passes_ttl_for_claude_4_5():
     cache_blocks_old = [b for b in result_old if "cachePoint" in b]
     assert len(cache_blocks_old) == 1
     assert "ttl" not in cache_blocks_old[0]["cachePoint"]
+
+
+def test_convert_to_anthropic_tool_result_openai_file_pdf_becomes_document():
+    """
+    OpenAI `{type: "file", file: {file_data: "data:application/pdf;..."}}` inside
+    a tool-message content list should translate to an Anthropic document block
+    inside the tool_result content. Reuses anthropic_process_openai_file_message,
+    which already handles this for user messages.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_anthropic_tool_result,
+    )
+
+    pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
+    message = {
+        "tool_call_id": "toolu_pdf_1",
+        "role": "tool",
+        "name": "fetch_document",
+        "content": [
+            {
+                "type": "file",
+                "file": {
+                    "file_data": f"data:application/pdf;base64,{pdf_b64}",
+                    "filename": "summary.pdf",
+                },
+            },
+        ],
+    }
+
+    result = convert_to_anthropic_tool_result(message)
+
+    assert result["type"] == "tool_result"
+    assert result["tool_use_id"] == "toolu_pdf_1"
+    content = result["content"]
+    assert isinstance(content, list) and len(content) == 1
+    block = content[0]
+    assert block["type"] == "document"
+    assert block["source"]["type"] == "base64"
+    assert block["source"]["media_type"] == "application/pdf"
+    assert block["source"]["data"] == pdf_b64
+
+
+def test_convert_to_anthropic_tool_result_image_url_pdf_data_uri_becomes_document():
+    """
+    Regression: a PDF sent as an `image_url` data URI on the tool-result path
+    must translate to an Anthropic document block (not an image block — Anthropic
+    rejects image blocks whose media_type is a non-image like application/pdf).
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_anthropic_tool_result,
+    )
+
+    pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
+    message = {
+        "tool_call_id": "toolu_pdf_img_1",
+        "role": "tool",
+        "name": "fetch_document",
+        "content": [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:application/pdf;base64,{pdf_b64}",
+                },
+            },
+        ],
+    }
+
+    result = convert_to_anthropic_tool_result(message)
+
+    content = result["content"]
+    assert isinstance(content, list) and len(content) == 1
+    block = content[0]
+    assert block["type"] == "document"
+    assert block["source"]["media_type"] == "application/pdf"
+    assert block["source"]["data"] == pdf_b64
+
+
+def test_convert_to_anthropic_tool_result_image_url_unsupported_mime_stays_image_path():
+    """
+    An `image_url` data URI whose mime is neither application/pdf nor text/plain
+    (e.g. application/json) must NOT be routed through the document path. Anthropic
+    only accepts application/pdf and text/plain as base64 document media_types —
+    anything else would produce a document block the API rejects. The old
+    (pre-fix) behavior was to wrap such data as an image block, which also
+    fails but stays on the image code path; preserve that failure mode rather
+    than switching to a document path that is equally broken.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_anthropic_tool_result,
+    )
+
+    message = {
+        "tool_call_id": "toolu_json_1",
+        "role": "tool",
+        "name": "fetch_json",
+        "content": [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "data:application/json;base64,eyJrIjoidiJ9",
+                },
+            },
+        ],
+    }
+
+    result = convert_to_anthropic_tool_result(message)
+
+    content = result["content"]
+    assert isinstance(content, list) and len(content) == 1
+    block = content[0]
+    assert block["type"] == "image", (
+        f"unsupported mime {block.get('source', {}).get('media_type')!r} "
+        f"should not be routed to document path; got {block}"
+    )
+
+
+def test_convert_to_anthropic_tool_result_image_url_text_plain_data_uri_becomes_document():
+    """
+    text/plain is one of the two mimes Anthropic accepts as a base64 document
+    media_type. Confirm it routes through the document path so tightening the
+    gate to {application/pdf, text/plain} (not "application/*") covers both.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_anthropic_tool_result,
+    )
+
+    txt_b64 = "aGVsbG8="  # "hello"
+    message = {
+        "tool_call_id": "toolu_txt_1",
+        "role": "tool",
+        "name": "fetch_text",
+        "content": [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:text/plain;base64,{txt_b64}",
+                },
+            },
+        ],
+    }
+
+    result = convert_to_anthropic_tool_result(message)
+
+    content = result["content"]
+    assert isinstance(content, list) and len(content) == 1
+    block = content[0]
+    assert block["type"] == "document"
+    assert block["source"]["media_type"] == "text/plain"
+    assert block["source"]["data"] == txt_b64
+
+
+def test_convert_to_anthropic_tool_result_image_url_png_still_becomes_image():
+    """
+    Regression: image_url with a real image mime type must continue to translate
+    to an Anthropic image block. Locks in existing behavior after the
+    data-URI-mime-type branching for PDFs.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        convert_to_anthropic_tool_result,
+    )
+
+    png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
+    message = {
+        "tool_call_id": "toolu_png_1",
+        "role": "tool",
+        "name": "fetch_image",
+        "content": [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:image/png;base64,{png_b64}",
+                },
+            },
+        ],
+    }
+
+    result = convert_to_anthropic_tool_result(message)
+
+    content = result["content"]
+    assert isinstance(content, list) and len(content) == 1
+    block = content[0]
+    assert block["type"] == "image"
+    assert block["source"]["media_type"] == "image/png"

tests/test_litellm/litellm_core_utils/test_get_llm_provider_endpoint_match.py

@@ -0,0 +1,138 @@
+"""
+Regression tests for the parsed-URL hostname match used to identify a
+caller-supplied ``api_base`` as a known openai-compatible provider.
+
+The previous shape (``if endpoint in api_base:``) used unanchored
+substring search, which let a caller pass
+``https://attacker.com/api.groq.com/openai/v1`` and have the proxy
+return ``GROQ_API_KEY`` as the dynamic credential — exfiltrating the
+server's real provider key to an attacker-controlled host on the
+outbound request.
+"""
+
+import os
+import sys
+from unittest.mock import patch
+
+import pytest
+
+sys.path.insert(0, os.path.abspath("../../.."))
+
+from litellm.litellm_core_utils.get_llm_provider_logic import (
+    _endpoint_matches_api_base,
+    get_llm_provider,
+)
+
+
+class TestEndpointMatchesApiBase:
+    """Direct unit tests on the parsed-URL matcher."""
+
+    @pytest.mark.parametrize(
+        "endpoint, api_base",
+        [
+            # Bare hostname endpoint, exact host match.
+            ("api.perplexity.ai", "https://api.perplexity.ai/v1"),
+            # Endpoint includes a path; api_base path starts with it.
+            ("api.groq.com/openai/v1", "https://api.groq.com/openai/v1"),
+            # Endpoint with full URL scheme.
+            ("https://api.cerebras.ai/v1", "https://api.cerebras.ai/v1/chat"),
+            # Trailing-slash on registered endpoint must not break match.
+            ("https://llm.chutes.ai/v1/", "https://llm.chutes.ai/v1/chat"),
+            # Case-insensitive on hostname.
+            ("api.groq.com/openai/v1", "https://API.GROQ.COM/openai/v1"),
+        ],
+    )
+    def test_legitimate_provider_urls_match(self, endpoint, api_base):
+        assert _endpoint_matches_api_base(endpoint, api_base) is True
+
+    @pytest.mark.parametrize(
+        "endpoint, api_base",
+        [
+            # Attacker host, registered endpoint smuggled into path.
+            (
+                "api.groq.com/openai/v1",
+                "https://attacker.com/api.groq.com/openai/v1",
+            ),
+            # Attacker host, registered endpoint smuggled into a path segment.
+            (
+                "api.groq.com/openai/v1",
+                "https://attacker.com/foo/api.groq.com/openai/v1",
+            ),
+            # Lookalike host that contains the registered host as a suffix label.
+            (
+                "api.groq.com/openai/v1",
+                "https://api.groq.com.attacker.com/openai/v1",
+            ),
+            # Lookalike host with the registered host as a prefix.
+            (
+                "api.groq.com/openai/v1",
+                "https://api.groq.com.evil.example/openai/v1",
+            ),
+            # Right host, wrong path — endpoint requires ``/openai/v1`` prefix.
+            ("api.groq.com/openai/v1", "https://api.groq.com/v1"),
+            # Path-segment lookalike: ``/openai/v10`` must not match ``/openai/v1``.
+            ("api.groq.com/openai/v1", "https://api.groq.com/openai/v10"),
+            # Userinfo / @-injection trick — the ``hostname`` after ``@`` is
+            # what httpx connects to.
+            (
+                "api.groq.com/openai/v1",
+                "https://api.groq.com@attacker.com/openai/v1",
+            ),
+        ],
+    )
+    def test_attacker_smuggling_does_not_match(self, endpoint, api_base):
+        assert _endpoint_matches_api_base(endpoint, api_base) is False
+
+
+class TestGetLlmProviderRejectsAttackerSmuggledApiBase:
+    """
+    End-to-end: ``get_llm_provider`` must NOT return the server's stored
+    secret (e.g. ``GROQ_API_KEY``) for an api_base whose hostname is
+    attacker-controlled, even when the registered endpoint string appears
+    elsewhere in the URL.
+    """
+
+    def test_attacker_host_does_not_yield_groq_secret(self):
+        # The function may either fall through (different provider) or
+        # raise BadRequestError because the model can't be identified.
+        # The invariant under test is that ``GROQ_API_KEY`` is never
+        # looked up against an attacker-controlled hostname.
+        import litellm
+
+        with patch(
+            "litellm.litellm_core_utils.get_llm_provider_logic.get_secret_str",
+            return_value="server-real-groq-key",
+        ) as mocked_secret:
+            try:
+                _, _, dynamic_api_key, _ = get_llm_provider(
+                    model="some-model",
+                    api_base="https://attacker.com/api.groq.com/openai/v1",
+                )
+                # If it returned, the dynamic key must not be the secret.
+                assert dynamic_api_key != "server-real-groq-key"
+            except litellm.exceptions.BadRequestError:
+                # Acceptable outcome: provider unidentifiable, no secret
+                # was returned.
+                pass
+
+        # Regardless of return / raise, the secret must never have been
+        # read against this attacker-controlled api_base.
+        groq_lookups = [
+            call
+            for call in mocked_secret.call_args_list
+            if call.args and call.args[0] == "GROQ_API_KEY"
+        ]
+        assert groq_lookups == []
+
+    def test_legitimate_groq_api_base_still_resolves(self):
+        with patch(
+            "litellm.litellm_core_utils.get_llm_provider_logic.get_secret_str",
+            return_value="server-real-groq-key",
+        ):
+            _, provider, dynamic_api_key, _ = get_llm_provider(
+                model="some-model",
+                api_base="https://api.groq.com/openai/v1",
+            )
+
+        assert provider == "groq"
+        assert dynamic_api_key == "server-real-groq-key"

tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py

@@ -4146,3 +4146,257 @@ def test_transform_response_finish_reason_stop_when_json_mode_filters_all_tools(
 
     # finish_reason must be "stop", not "tool_calls"
     assert result.choices[0].finish_reason == "stop"
+
+
+def test_bedrock_tool_message_openai_file_pdf_becomes_document():
+    """
+    OpenAI Chat Completions `{type: "file", file: {file_data: "data:application/pdf;...", filename}}`
+    inside a tool message content list should translate to a Bedrock
+    toolResult.content[].document block.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    pdf_b64 = "JVBERi0xLjQKJeLjz9MK"  # tiny "%PDF-1.4\n" header
+    messages = [
+        {"role": "user", "content": "Summarize the attached PDF."},
+        {
+            "tool_call_id": "tooluse_pdf_1",
+            "role": "tool",
+            "name": "fetch_document",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {
+                        "file_data": f"data:application/pdf;base64,{pdf_b64}",
+                        "filename": "summary.pdf",
+                    },
+                },
+            ],
+        },
+    ]
+
+    translated_msg = _bedrock_converse_messages_pt(
+        messages=messages, model="", llm_provider=""
+    )
+
+    tool_result = translated_msg[-1]["content"][-1]["toolResult"]
+    assert tool_result["toolUseId"] == "tooluse_pdf_1"
+    assert len(tool_result["content"]) == 1
+    block = tool_result["content"][0]
+    assert "document" in block, f"expected document block, got {block}"
+    assert block["document"]["format"] == "pdf"
+    assert block["document"]["source"]["bytes"] == pdf_b64
+    assert block["document"]["name"].startswith("DocumentPDFmessages_")
+    assert block["document"]["name"].endswith("_pdf")
+
+
+def test_bedrock_tool_message_image_url_pdf_data_uri_becomes_document():
+    """
+    Regression for the processor-returns-document-but-wrapper-drops-it bug:
+    when a caller sends a PDF as an `image_url` data URI on the tool-result path,
+    BedrockImageProcessor correctly returns a {"document": ...} block, but the
+    tool-result wrapper only appended the "image" case, silently dropping documents.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    pdf_b64 = "JVBERi0xLjQKJeLjz9MK"
+    messages = [
+        {"role": "user", "content": "Summarize the attached PDF."},
+        {
+            "tool_call_id": "tooluse_pdf_img_1",
+            "role": "tool",
+            "name": "fetch_document",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:application/pdf;base64,{pdf_b64}",
+                    },
+                },
+            ],
+        },
+    ]
+
+    translated_msg = _bedrock_converse_messages_pt(
+        messages=messages, model="", llm_provider=""
+    )
+
+    tool_result = translated_msg[-1]["content"][-1]["toolResult"]
+    assert tool_result["toolUseId"] == "tooluse_pdf_img_1"
+    assert len(tool_result["content"]) == 1
+    block = tool_result["content"][0]
+    assert "document" in block, f"expected document block, got {block}"
+    assert block["document"]["format"] == "pdf"
+    assert block["document"]["source"]["bytes"] == pdf_b64
+
+
+def test_bedrock_tool_message_file_id_http_url_becomes_document():
+    """
+    OpenAI `file.file_id` is a server-side file reference. The Bedrock
+    user-message path (_process_file_message at factory.py:4796) accepts either
+    `file_data` or `file_id` and forwards to BedrockImageProcessor. The
+    tool-result path must match: when `file_id` is an http(s) PDF URL, it
+    should resolve to a Bedrock document block, not be silently dropped.
+    """
+    from unittest.mock import patch
+
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        BedrockImageProcessor,
+        _bedrock_converse_messages_pt,
+    )
+
+    pdf_url = "https://example.com/whitepaper.pdf"
+    fake_document_block = {
+        "document": {
+            "format": "pdf",
+            "name": "fake_doc",
+            "source": {"bytes": "ZmFrZQ=="},
+        }
+    }
+    messages = [
+        {"role": "user", "content": "Summarize the attached PDF."},
+        {
+            "tool_call_id": "tooluse_fid_1",
+            "role": "tool",
+            "name": "fetch_document",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": pdf_url,
+                        "filename": "whitepaper.pdf",
+                    },
+                },
+            ],
+        },
+    ]
+
+    with patch.object(
+        BedrockImageProcessor,
+        "process_image_sync",
+        return_value=fake_document_block,
+    ) as mock_proc:
+        translated_msg = _bedrock_converse_messages_pt(
+            messages=messages, model="", llm_provider=""
+        )
+
+    mock_proc.assert_called_once()
+    assert mock_proc.call_args.kwargs["image_url"] == pdf_url
+
+    tool_result = translated_msg[-1]["content"][-1]["toolResult"]
+    assert len(tool_result["content"]) == 1
+    block = tool_result["content"][0]
+    assert "document" in block, f"expected document block, got {block}"
+    assert block["document"]["source"]["bytes"] == "ZmFrZQ=="
+
+
+def test_bedrock_tool_message_file_without_data_or_id_raises():
+    """
+    The user-message path raises BadRequestError when a `type: "file"` block
+    has neither `file_data` nor `file_id` (factory.py:4802-4809). The
+    tool-result path must match — silently dropping the block makes the model
+    see an empty tool result and obscures the caller bug.
+    """
+    import litellm
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    messages = [
+        {"role": "user", "content": "Summarize."},
+        {
+            "tool_call_id": "tooluse_bad_1",
+            "role": "tool",
+            "name": "fetch_document",
+            "content": [
+                {
+                    "type": "file",
+                    "file": {"filename": "nothing.pdf"},
+                },
+            ],
+        },
+    ]
+
+    with pytest.raises(litellm.BadRequestError):
+        _bedrock_converse_messages_pt(messages=messages, model="", llm_provider="")
+
+
+def test_bedrock_tool_message_image_url_png_still_becomes_image():
+    """
+    Regression: image_url with an image mime type must continue to translate
+    to a Bedrock image block (not document). Locks in existing behavior after
+    the document-passthrough fix.
+    """
+    from litellm.litellm_core_utils.prompt_templates.factory import (
+        _bedrock_converse_messages_pt,
+    )
+
+    png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGBgAAAABQABXvMqOgAAAABJRU5ErkJggg=="
+    messages = [
+        {"role": "user", "content": "Describe the attached image."},
+        {
+            "tool_call_id": "tooluse_png_1",
+            "role": "tool",
+            "name": "fetch_image",
+            "content": [
+                {
+                    "type": "image_url",
+                    "image_url": {
+                        "url": f"data:image/png;base64,{png_b64}",
+                    },
+                },
+            ],
+        },
+    ]
+
+    translated_msg = _bedrock_converse_messages_pt(
+        messages=messages, model="", llm_provider=""
+    )
+
+    tool_result = translated_msg[-1]["content"][-1]["toolResult"]
+    assert len(tool_result["content"]) == 1
+    block = tool_result["content"][0]
+    assert "image" in block, f"expected image block, got {block}"
+    assert "document" not in block
+    assert block["image"]["format"] == "png"
+    assert block["image"]["source"]["bytes"] == png_b64
+
+
+def test_transform_response_does_not_leak_body_on_parse_failure():
+    from litellm.llms.bedrock.common_utils import BedrockError
+
+    leaky_body = {"output": {"message": {"content": [{"text": "secret content"}]}}}
+
+    class MockResponse:
+        def json(self):
+            return leaky_body
+
+        @property
+        def text(self):
+            return json.dumps(leaky_body)
+
+    with patch(
+        "litellm.llms.bedrock.chat.converse_transformation.ConverseResponseBlock",
+        side_effect=KeyError("missing required field"),
+    ):
+        with pytest.raises(BedrockError) as exc_info:
+            AmazonConverseConfig()._transform_response(
+                model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
+                response=MockResponse(),
+                model_response=ModelResponse(),
+                stream=False,
+                logging_obj=None,
+                optional_params={},
+                api_key=None,
+                data=None,
+                messages=[],
+                encoding=None,
+            )
+
+    msg = str(exc_info.value)
+    assert "secret content" not in msg
+    assert "Error converting to valid response block" in msg

tests/test_litellm/llms/bedrock/vector_stores/test_bedrock_vector_store_transformation.py

@@ -21,7 +21,112 @@ def test_transform_search_request():
         api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases",
         litellm_logging_obj=mock_log,
         litellm_params={},
+        extra_body=None,
     )
 
     assert url.endswith("/kb123/retrieve")
     assert body["retrievalQuery"].get("text") == "hello"
+
+
+def test_transform_search_request_uses_only_retrieval_config_from_extra_body():
+    config = BedrockVectorStoreConfig()
+    mock_log = MagicMock()
+    mock_log.model_call_details = {}
+
+    url, body = config.transform_search_vector_store_request(
+        vector_store_id="kb123",
+        query="hello",
+        vector_store_search_optional_params={},
+        api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases",
+        litellm_logging_obj=mock_log,
+        litellm_params={},
+        extra_body={
+            "retrievalConfiguration": {
+                "vectorSearchConfiguration": {
+                    "overrideSearchType": "HYBRID",
+                    "numberOfResults": 8,
+                }
+            },
+            "unrelatedField": {"should_not": "be_forwarded"},
+        },
+    )
+
+    assert url.endswith("/kb123/retrieve")
+    assert body["retrievalQuery"].get("text") == "hello"
+    assert (
+        body["retrievalConfiguration"]["vectorSearchConfiguration"][
+            "overrideSearchType"
+        ]
+        == "HYBRID"
+    )
+    assert "unrelatedField" not in body
+
+
+def test_transform_search_request_does_not_mutate_extra_body_and_overrides_number_of_results():
+    config = BedrockVectorStoreConfig()
+    mock_log = MagicMock()
+    mock_log.model_call_details = {}
+    extra_body = {
+        "retrievalConfiguration": {
+            "vectorSearchConfiguration": {
+                "overrideSearchType": "HYBRID",
+                "numberOfResults": 8,
+            }
+        }
+    }
+
+    _, body = config.transform_search_vector_store_request(
+        vector_store_id="kb123",
+        query="hello",
+        vector_store_search_optional_params={"max_num_results": 10},
+        api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases",
+        litellm_logging_obj=mock_log,
+        litellm_params={},
+        extra_body=extra_body,
+    )
+
+    assert (
+        body["retrievalConfiguration"]["vectorSearchConfiguration"]["numberOfResults"]
+        == 10
+    )
+    assert (
+        extra_body["retrievalConfiguration"]["vectorSearchConfiguration"][
+            "numberOfResults"
+        ]
+        == 8
+    )
+
+
+def test_transform_search_request_overrides_filter_without_mutating_extra_body():
+    config = BedrockVectorStoreConfig()
+    mock_log = MagicMock()
+    mock_log.model_call_details = {}
+    extra_body = {
+        "retrievalConfiguration": {
+            "vectorSearchConfiguration": {
+                "filter": {"equals": {"key": "tenant", "value": "a"}}
+            }
+        }
+    }
+    new_filter = {"equals": {"key": "tenant", "value": "b"}}
+
+    _, body = config.transform_search_vector_store_request(
+        vector_store_id="kb123",
+        query="hello",
+        vector_store_search_optional_params={"filters": new_filter},
+        api_base="https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases",
+        litellm_logging_obj=mock_log,
+        litellm_params={},
+        extra_body=extra_body,
+    )
+
+    assert (
+        body["retrievalConfiguration"]["vectorSearchConfiguration"]["filter"]
+        == new_filter
+    )
+    assert (
+        extra_body["retrievalConfiguration"]["vectorSearchConfiguration"]["filter"][
+            "equals"
+        ]["value"]
+        == "a"
+    )