Merge branch 'BerriAI:main' into docs/elasticsearch-logging-tutorial

.pre-commit-config.yaml

@@ -14,12 +14,12 @@ repos:
         types: [python]
         files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
         exclude: ^litellm/__init__.py$
-    -   id: black
-        name: black
-        entry: poetry run black
-        language: system
-        types: [python]
-        files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
+    # -   id: black
+    #     name: black
+    #     entry: poetry run black
+    #     language: system
+    #     types: [python]
+    #     files: (litellm/|litellm_proxy_extras/|enterprise/).*\.py
 -   repo: https://github.com/pycqa/flake8
     rev: 7.0.0  # The version of flake8 to use
     hooks:

deploy/charts/litellm-helm/templates/deployment.yaml

@@ -1,6 +1,8 @@
 apiVersion: apps/v1
 kind: Deployment
 metadata:
+  annotations:
+  {{- toYaml .Values.deploymentAnnotations | nindent 4 }}
   name: {{ include "litellm.fullname" . }}
   labels:
     {{- include "litellm.labels" . | nindent 4 }}

deploy/charts/litellm-helm/values.yaml

@@ -27,6 +27,9 @@ serviceAccount:
   # If not set and create is true, a name is generated using the fullname template
   name: ""
 
+# annotations for litellm deployment
+deploymentAnnotations: {}
+# annotations for litellm pods
 podAnnotations: {}
 podLabels: {}
 

docs/my-website/docs/completion/document_understanding.md

@@ -9,6 +9,7 @@ Works for:
 - Vertex AI models (Gemini + Anthropic)
 - Bedrock Models
 - Anthropic API Models
+- OpenAI API Models
 
 ## Quick Start
 

docs/my-website/docs/mcp.md

@@ -106,7 +106,7 @@ curl --location 'https://api.openai.com/v1/responses' \
             "server_url": "<your-litellm-proxy-base-url>/mcp",
             "require_approval": "never",
             "headers": {
-                "x-litellm-api-key": "YOUR_LITELLM_API_KEY"
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
             }
         }
     ],
@@ -136,7 +136,7 @@ curl --location '<your-litellm-proxy-base-url>/v1/responses' \
             "server_url": "<your-litellm-proxy-base-url>/mcp",
             "require_approval": "never",
             "headers": {
-                "x-litellm-api-key": "YOUR_LITELLM_API_KEY"
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
             }
         }
     ],
@@ -165,7 +165,7 @@ Use tools directly from Cursor IDE with LiteLLM MCP:
     "LiteLLM": {
       "url": "<your-litellm-proxy-base-url>/mcp",
       "headers": {
-        "x-litellm-api-key": "$LITELLM_API_KEY"
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY"
       }
     }
   }
@@ -187,7 +187,7 @@ Connect to LiteLLM MCP using HTTP transport. Compatible with any MCP client that
 
 **Headers:**
 ```text showLineNumbers
-x-litellm-api-key: YOUR_LITELLM_API_KEY
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
 ```
 
 This URL can be used with any MCP client that supports HTTP transport. Refer to your client documentation to determine the appropriate transport method.
@@ -226,7 +226,7 @@ server_url = "<your-litellm-proxy-base-url>/mcp"
 transport = StreamableHttpTransport(
     server_url,
     headers={
-        "x-litellm-api-key": "YOUR_LITELLM_API_KEY"
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY"
     }
 )
 
@@ -265,6 +265,182 @@ if __name__ == "__main__":
 </Tabs>
 
 
+## Using your MCP with client side credentials
+
+Use this if you want to pass a client side authentication token to LiteLLM to then pass to your MCP to auth to your MCP.
+
+You can specify your MCP auth token using the header `x-mcp-auth`. LiteLLM will forward this token to your MCP server for authentication.
+
+<Tabs>
+<TabItem value="openai" label="OpenAI API">
+
+#### Connect via OpenAI Responses API with MCP Auth
+
+Use the OpenAI Responses API and include the `x-mcp-auth` header for your MCP server authentication:
+
+```bash title="cURL Example with MCP Auth" showLineNumbers
+curl --location 'https://api.openai.com/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $OPENAI_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+                "x-mcp-auth": YOUR_MCP_AUTH_TOKEN
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+
+<TabItem value="litellm" label="LiteLLM Proxy">
+
+#### Connect via LiteLLM Proxy Responses API with MCP Auth
+
+Use this when calling LiteLLM Proxy for LLM API requests to `/v1/responses` endpoint with MCP authentication:
+
+```bash title="cURL Example with MCP Auth" showLineNumbers
+curl --location '<your-litellm-proxy-base-url>/v1/responses' \
+--header 'Content-Type: application/json' \
+--header "Authorization: Bearer $LITELLM_API_KEY" \
+--data '{
+    "model": "gpt-4o",
+    "tools": [
+        {
+            "type": "mcp",
+            "server_label": "litellm",
+            "server_url": "<your-litellm-proxy-base-url>/mcp",
+            "require_approval": "never",
+            "headers": {
+                "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+                "x-mcp-auth": "YOUR_MCP_AUTH_TOKEN"
+            }
+        }
+    ],
+    "input": "Run available tools",
+    "tool_choice": "required"
+}'
+```
+
+</TabItem>
+
+<TabItem value="cursor" label="Cursor IDE">
+
+#### Connect via Cursor IDE with MCP Auth
+
+Use tools directly from Cursor IDE with LiteLLM MCP and include your MCP authentication token:
+
+**Setup Instructions:**
+
+1. **Open Cursor Settings**: Use `⇧+⌘+J` (Mac) or `Ctrl+Shift+J` (Windows/Linux)
+2. **Navigate to MCP Tools**: Go to the "MCP Tools" tab and click "New MCP Server"
+3. **Add Configuration**: Copy and paste the JSON configuration below, then save with `Cmd+S` or `Ctrl+S`
+
+```json title="Cursor MCP Configuration with Auth" showLineNumbers
+{
+  "mcpServers": {
+    "LiteLLM": {
+      "url": "<your-litellm-proxy-base-url>/mcp",
+      "headers": {
+        "x-litellm-api-key": "Bearer $LITELLM_API_KEY",
+        "x-mcp-auth": "$MCP_AUTH_TOKEN"
+      }
+    }
+  }
+}
+```
+
+</TabItem>
+
+<TabItem value="http" label="Streamable HTTP">
+
+#### Connect via Streamable HTTP Transport with MCP Auth
+
+Connect to LiteLLM MCP using HTTP transport with MCP authentication:
+
+**Server URL:**
+```text showLineNumbers
+<your-litellm-proxy-base-url>/mcp
+```
+
+**Headers:**
+```text showLineNumbers
+x-litellm-api-key: Bearer YOUR_LITELLM_API_KEY
+x-mcp-auth: Bearer YOUR_MCP_AUTH_TOKEN
+```
+
+This URL can be used with any MCP client that supports HTTP transport. The `x-mcp-auth` header will be forwarded to your MCP server for authentication.
+
+</TabItem>
+
+<TabItem value="fastmcp" label="Python FastMCP">
+
+#### Connect via Python FastMCP Client with MCP Auth
+
+Use the Python FastMCP client to connect to your LiteLLM MCP server with MCP authentication:
+
+```python title="Python FastMCP Example with MCP Auth" showLineNumbers
+import asyncio
+import json
+
+from fastmcp import Client
+from fastmcp.client.transports import StreamableHttpTransport
+
+# Create the transport with your LiteLLM MCP server URL and auth headers
+server_url = "<your-litellm-proxy-base-url>/mcp"
+transport = StreamableHttpTransport(
+    server_url,
+    headers={
+        "x-litellm-api-key": "Bearer YOUR_LITELLM_API_KEY",
+        "x-mcp-auth": "Bearer YOUR_MCP_AUTH_TOKEN"
+    }
+)
+
+# Initialize the client with the transport
+client = Client(transport=transport)
+
+
+async def main():
+    # Connection is established here
+    print("Connecting to LiteLLM MCP server with authentication...")
+    async with client:
+        print(f"Client connected: {client.is_connected()}")
+
+        # Make MCP calls within the context
+        print("Fetching available tools...")
+        tools = await client.list_tools()
+
+        print(f"Available tools: {json.dumps([t.name for t in tools], indent=2)}")
+        
+        # Example: Call a tool (replace 'tool_name' with an actual tool name)
+        if tools:
+            tool_name = tools[0].name
+            print(f"Calling tool: {tool_name}")
+            
+            # Call the tool with appropriate arguments
+            result = await client.call_tool(tool_name, arguments={})
+            print(f"Tool result: {result}")
+
+
+# Run the example
+if __name__ == "__main__":
+    asyncio.run(main())
+```
+
+</TabItem>
+</Tabs>
+
+
 ## ✨ MCP Permission Management
 
 LiteLLM supports managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access.

docs/my-website/docs/providers/meta_llama.md

@@ -45,7 +45,7 @@ os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
 messages = [{"content": "Hello, how are you?", "role": "user"}]
 
 # Meta Llama call
-response = completion(model="meta_llama/Llama-3.3-70B-Instruct", messages=messages)
+response = completion(model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8", messages=messages)
 ```
 
 ### Streaming
@@ -61,7 +61,7 @@ messages = [{"content": "Hello, how are you?", "role": "user"}]
 
 # Meta Llama call with streaming
 response = completion(
-    model="meta_llama/Llama-3.3-70B-Instruct",
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
     messages=messages,
     stream=True
 )
@@ -70,6 +70,104 @@ for chunk in response:
     print(chunk)
 ```
 
+### Function Calling
+
+```python showLineNumbers title="Meta Llama Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "What's the weather like in San Francisco?", "role": "user"}]
+
+# Define the function
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "get_weather",
+            "description": "Get the current weather in a given location",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "location": {
+                        "type": "string",
+                        "description": "The city and state, e.g. San Francisco, CA"
+                    },
+                    "unit": {
+                        "type": "string",
+                        "enum": ["celsius", "fahrenheit"]
+                    }
+                },
+                "required": ["location"]
+            }
+        }
+    }
+]
+
+# Meta Llama call with function calling
+response = completion(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response.choices[0].message.tool_calls)
+```
+
+### Tool Use
+
+```python showLineNumbers title="Meta Llama Tool Use"
+import os
+import litellm
+from litellm import completion
+
+os.environ["LLAMA_API_KEY"] = ""  # your Meta Llama API key
+
+messages = [{"content": "Create a chart showing the population growth of New York City from 2010 to 2020", "role": "user"}]
+
+# Define the tools
+tools = [
+    {
+        "type": "function",
+        "function": {
+            "name": "create_chart",
+            "description": "Create a chart with the provided data",
+            "parameters": {
+                "type": "object",
+                "properties": {
+                    "chart_type": {
+                        "type": "string",
+                        "enum": ["bar", "line", "pie", "scatter"],
+                        "description": "The type of chart to create"
+                    },
+                    "title": {
+                        "type": "string",
+                        "description": "The title of the chart"
+                    },
+                    "data": {
+                        "type": "object",
+                        "description": "The data to plot in the chart"
+                    }
+                },
+                "required": ["chart_type", "title", "data"]
+            }
+        }
+    }
+]
+
+# Meta Llama call with tool use
+response = completion(
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response.choices[0].message.content)
+```
 
 ## Usage - LiteLLM Proxy
 
@@ -111,7 +209,7 @@ client = OpenAI(
 
 # Non-streaming response
 response = client.chat.completions.create(
-    model="meta_llama/Llama-3.3-70B-Instruct",
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
     messages=[{"role": "user", "content": "Write a short poem about AI."}]
 )
 
@@ -129,7 +227,7 @@ client = OpenAI(
 
 # Streaming response
 response = client.chat.completions.create(
-    model="meta_llama/Llama-3.3-70B-Instruct",
+    model="meta_llama/Llama-4-Maverick-17B-128E-Instruct-FP8",
     messages=[{"role": "user", "content": "Write a short poem about AI."}],
     stream=True
 )

docs/my-website/docs/proxy/alerting.md

@@ -148,7 +148,7 @@ client = openai.OpenAI(
 
 # request sent to model set on litellm proxy, `litellm --model`
 response = client.chat.completions.create(
-    model="gpt-3.5-turbo",
+    model="gpt-4o",
     messages = [], 
     extra_body={
         "metadata": {

docs/my-website/docs/proxy/billing.md

@@ -101,7 +101,7 @@ client = openai.OpenAI(
 )
 
 # request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+response = client.chat.completions.create(model="gpt-4o", messages = [
     {
         "role": "user",
         "content": "this is a test request, write a short poem"
@@ -127,7 +127,7 @@ os.environ["OPENAI_API_KEY"] = "sk-tXL0wt5-lOOVK9sfY2UacA" # 👈 Team's Key
 
 chat = ChatOpenAI(
     openai_api_base="http://0.0.0.0:4000",
-    model = "gpt-3.5-turbo",
+    model = "gpt-4o",
     temperature=0.1,
 )
 
@@ -198,7 +198,7 @@ For:
   curl --location 'http://0.0.0.0:4000/chat/completions' \
   --header 'Content-Type: application/json' \
   --data ' {
-        "model": "gpt-3.5-turbo",
+        "model": "gpt-4o",
         "messages": [
           {
             "role": "user",
@@ -220,7 +220,7 @@ For:
   )
 
   # request sent to model set on litellm proxy, `litellm --model`
-  response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+  response = client.chat.completions.create(model="gpt-4o", messages = [
       {
           "role": "user",
           "content": "this is a test request, write a short poem"
@@ -247,7 +247,7 @@ For:
 
   chat = ChatOpenAI(
       openai_api_base="http://0.0.0.0:4000",
-      model = "gpt-3.5-turbo",
+      model = "gpt-4o",
       temperature=0.1,
       extra_body={
           "user": "my_customer_id"  # 👈 whatever your customer id is
@@ -306,7 +306,7 @@ client = openai.OpenAI(
 )
 
 # request sent to model set on litellm proxy, `litellm --model`
-response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [
+response = client.chat.completions.create(model="gpt-4o", messages = [
     {
         "role": "user",
         "content": "this is a test request, write a short poem"

docs/my-website/docs/proxy/cost_tracking.md

@@ -577,6 +577,35 @@ curl -X GET 'http://localhost:4000/global/spend/report?start_date=2024-04-01&end
 </Tabs>
 
 
+## 📊 Spend Logs API - Individual Transaction Logs
+
+The `/spend/logs` endpoint now supports a `summarize` parameter to control data format when using date filters.
+
+### Key Parameters
+
+| Parameter | Description |
+|-----------|-------------|
+| `summarize` | **New parameter**: `true` (default) = aggregated data, `false` = individual transaction logs |
+
+### Examples
+
+**Get individual transaction logs:**
+```bash
+curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02&summarize=false" \
+-H "Authorization: Bearer sk-1234"
+```
+
+**Get summarized data (default):**
+```bash
+curl -X GET "http://localhost:4000/spend/logs?start_date=2024-01-01&end_date=2024-01-02" \
+-H "Authorization: Bearer sk-1234"
+```
+
+**Use Cases:**
+- `summarize=false`: Analytics dashboards, ETL processes, detailed audit trails
+- `summarize=true`: Daily spending reports, high-level cost tracking (legacy behavior)
+
+
 ## ✨ Custom Spend Log metadata
 
 Log specific key,value pairs as part of the metadata for a spend log

docs/my-website/docs/proxy/guardrails/bedrock.md

@@ -22,8 +22,10 @@ guardrails:
     litellm_params:
       guardrail: bedrock  # supported values: "aporia", "bedrock", "lakera"
       mode: "during_call"
-      guardrailIdentifier: ff6ujrregl1q # your guardrail ID on bedrock
-      guardrailVersion: "DRAFT"         # your guardrail version on bedrock
+      guardrailIdentifier: ff6ujrregl1q      # your guardrail ID on bedrock
+      guardrailVersion: "DRAFT"              # your guardrail version on bedrock
+      aws_region_name: os.environ/AWS_REGION # region guardrail is defined
+      aws_role_name: os.environ/AWS_ROLE_ARN # your role with permissions to use the guardrail
   
 ```
 
@@ -158,6 +160,8 @@ guardrails:
       mode: "pre_call"  # Important: must use pre_call mode for masking
       guardrailIdentifier: wf0hkdb5x07f
       guardrailVersion: "DRAFT"
+      aws_region_name: os.environ/AWS_REGION
+      aws_role_name: os.environ/AWS_ROLE_ARN
       mask_request_content: true    # Enable masking in user requests
       mask_response_content: true   # Enable masking in model responses
 ```

docs/my-website/docs/proxy/prometheus.md

@@ -23,9 +23,9 @@ If you're using the LiteLLM CLI with `litellm --config proxy_config.yaml` then y
 Add this to your proxy config.yaml 
 ```yaml
 model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
     litellm_params:
-      model: gpt-3.5-turbo
+      model: gpt-4o
 litellm_settings:
   callbacks: ["prometheus"]
 ```
@@ -40,7 +40,7 @@ Test Request
 curl --location 'http://0.0.0.0:4000/chat/completions' \
     --header 'Content-Type: application/json' \
     --data '{
-    "model": "gpt-3.5-turbo",
+    "model": "gpt-4o",
     "messages": [
         {
         "role": "user",
@@ -201,9 +201,9 @@ Track custom metrics on prometheus on all events mentioned above.
 
 ```yaml
 model_list:
-  - model_name: openai/gpt-3.5-turbo
+  - model_name: openai/gpt-4o
     litellm_params:
-      model: openai/gpt-3.5-turbo
+      model: openai/gpt-4o
       api_key: os.environ/OPENAI_API_KEY
 
 litellm_settings:
@@ -218,7 +218,7 @@ curl -L -X POST 'http://0.0.0.0:4000/v1/chat/completions' \
 -H 'Content-Type: application/json' \
 -H 'Authorization: Bearer <LITELLM_API_KEY>' \
 -d '{
-    "model": "openai/gpt-3.5-turbo",
+    "model": "openai/gpt-4o",
     "messages": [
       {
         "role": "user",
@@ -254,9 +254,9 @@ Configure which metrics to emit by specifying them in `prometheus_metrics_config
 
 ```yaml
 model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
     litellm_params:
-      model: gpt-3.5-turbo
+      model: gpt-4o
 
 litellm_settings:
   callbacks: ["prometheus"]
@@ -358,9 +358,9 @@ To monitor the health of litellm adjacent services (redis / postgres), do:
 
 ```yaml
 model_list:
- - model_name: gpt-3.5-turbo
+ - model_name: gpt-4o
     litellm_params:
-      model: gpt-3.5-turbo
+      model: gpt-4o
 litellm_settings:
   service_callback: ["prometheus_system"]
 ```

docs/my-website/docs/proxy/self_serve.md

@@ -314,6 +314,10 @@ litellm_settings:
     max_budget: 100                # Optional[float], optional): $100 budget for a new SSO sign in user
     budget_duration: 30d           # Optional[str], optional): 30 days budget_duration for a new SSO sign in user
     models: ["gpt-3.5-turbo"]      # Optional[List[str]], optional): models to be used by a new SSO sign in user
+    teams: # Optional[List[NewUserRequestTeam]], optional): teams to be used by the user
+      - team_id: "team_id_1" # Required[str]: team_id to be used by the user
+        max_budget_in_team: 100 # Optional[float], optional): $100 budget for the team. Defaults to None.
+        user_role: "user" # Optional[str], optional): "user" or "admin". Defaults to "user"
   
   default_team_params:             # Default Params to apply when litellm auto creates a team from SSO IDP provider
     max_budget: 100                # Optional[float], optional): $100 budget for the team

docs/my-website/release_notes/v1.72.6-stable/index.md

@@ -1,5 +1,5 @@
 ---
-title: "[PRE-RELEASE] v1.72.6-stable"
+title: "v1.72.6-stable - MCP Gateway Permission Management"
 slug: "v1-72-6-stable"
 date: 2025-06-14T10:00:00
 authors:
@@ -36,15 +36,15 @@ The production version will be released on Wednesday.
 docker run
 -e STORE_MODEL_IN_DB=True
 -p 4000:4000
-ghcr.io/berriai/litellm:main-v1.72.6.rc
+ghcr.io/berriai/litellm:main-v1.72.6-stable
 ```
 </TabItem>
 
 <TabItem value="pip" label="Pip">
 
-:::info
-This version is not out yet. 
-:::
+``` showLineNumbers title="pip install litellm"
+pip install litellm==1.72.6.post2
+```
 
 </TabItem>
 </Tabs>

docs/my-website/sidebars.js

@@ -63,6 +63,15 @@ const sidebars = {
         "proxy/custom_prompt_management"
       ].sort()
     },
+    {
+      type: "category",
+      label: "AI Tools (OpenWebUI, Claude Code, etc.)",
+      items: [
+        "tutorials/openweb_ui",
+        "tutorials/openai_codex",
+        "tutorials/claude_responses_api",
+      ]
+    },
     
   ],
   // But you can create a sidebar manually

litellm-proxy-extras/litellm_proxy_extras/migrations/20250618225828_add_health_check_table/migration.sql

@@ -0,0 +1,28 @@
+-- CreateTable
+CREATE TABLE "LiteLLM_HealthCheckTable" (
+    "health_check_id" TEXT NOT NULL,
+    "model_name" TEXT NOT NULL,
+    "model_id" TEXT,
+    "status" TEXT NOT NULL,
+    "healthy_count" INTEGER NOT NULL DEFAULT 0,
+    "unhealthy_count" INTEGER NOT NULL DEFAULT 0,
+    "error_message" TEXT,
+    "response_time_ms" DOUBLE PRECISION,
+    "details" JSONB,
+    "checked_by" TEXT,
+    "checked_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+
+    CONSTRAINT "LiteLLM_HealthCheckTable_pkey" PRIMARY KEY ("health_check_id")
+);
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_HealthCheckTable_model_name_idx" ON "LiteLLM_HealthCheckTable"("model_name");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_HealthCheckTable_checked_at_idx" ON "LiteLLM_HealthCheckTable"("checked_at");
+
+-- CreateIndex
+CREATE INDEX "LiteLLM_HealthCheckTable_status_idx" ON "LiteLLM_HealthCheckTable"("status");
+

litellm-proxy-extras/litellm_proxy_extras/schema.prisma

@@ -498,4 +498,24 @@ model LiteLLM_GuardrailsTable {
   guardrail_info Json?
   created_at DateTime @default(now())
   updated_at DateTime @updatedAt
+}
+
+model LiteLLM_HealthCheckTable {
+  health_check_id String @id @default(uuid())
+  model_name String
+  model_id String?
+  status String
+  healthy_count Int @default(0)
+  unhealthy_count Int @default(0)
+  error_message String?
+  response_time_ms Float?
+  details Json?
+  checked_by String?
+  checked_at DateTime @default(now())
+  created_at DateTime @default(now())
+  updated_at DateTime @updatedAt
+
+  @@index([model_name])
+  @@index([checked_at])
+  @@index([status])
 }

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.2.3"
+version = "0.2.5"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.2.3"
+version = "0.2.5"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/__init__.py

@@ -2,7 +2,7 @@
 import warnings
 
 warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
-### INIT VARIABLES ###########
+### INIT VARIABLES ############
 import threading
 import os
 from typing import Callable, List, Optional, Dict, Union, Any, Literal, get_args
@@ -220,6 +220,7 @@ ssl_certificate: Optional[str] = None
 disable_streaming_logging: bool = False
 disable_token_counter: bool = False
 disable_add_transform_inline_image_block: bool = False
+disable_add_user_agent_to_request_tags: bool = False
 in_memory_llm_clients_cache: LLMClientCache = LLMClientCache()
 safe_memory_mode: bool = False
 enable_azure_ad_token_refresh: Optional[bool] = False

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -11,6 +11,7 @@ from typing import (
     Iterable,
     Iterator,
     List,
+    Literal,
     Optional,
     Tuple,
     Union,
@@ -25,6 +26,7 @@ from litellm.llms.base_llm.bridges.completion_transformation import (
 )
 
 if TYPE_CHECKING:
+    from openai.types.responses import ResponseInputImageParam
     from pydantic import BaseModel
 
     from litellm import LiteLLMLoggingObj, ModelResponse
@@ -32,6 +34,7 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import (
         ALL_RESPONSES_API_TOOL_PARAMS,
         AllMessageValues,
+        ChatCompletionImageObject,
         ChatCompletionThinkingBlock,
         OpenAIMessageContentListBlock,
     )
@@ -141,10 +144,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                 responses_api_request["max_output_tokens"] = value
             elif key == "tools" and value is not None:
                 # Convert chat completion tools to responses API tools format
-                responses_api_request[
-                    "tools"
-                ] = self._convert_tools_to_responses_format(
-                    cast(List[Dict[str, Any]], value)
+                responses_api_request["tools"] = (
+                    self._convert_tools_to_responses_format(
+                        cast(List[Dict[str, Any]], value)
+                    )
                 )
             elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys():
                 responses_api_request[key] = value  # type: ignore
@@ -320,6 +323,36 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         else:
             return {"type": "output_text", "text": content}
 
+    def _convert_content_to_responses_format_image(
+        self, content: "ChatCompletionImageObject", role: str
+    ) -> "ResponseInputImageParam":
+        from openai.types.responses import ResponseInputImageParam
+
+        content_image_url = content.get("image_url")
+        actual_image_url: Optional[str] = None
+        detail: Optional[Literal["low", "high", "auto"]] = None
+
+        if isinstance(content_image_url, str):
+            actual_image_url = content_image_url
+        elif isinstance(content_image_url, dict):
+            actual_image_url = content_image_url.get("url")
+            detail = cast(
+                Optional[Literal["low", "high", "auto"]],
+                content_image_url.get("detail"),
+            )
+
+        if actual_image_url is None:
+            raise ValueError(f"Invalid image URL: {content_image_url}")
+
+        image_param = ResponseInputImageParam(
+            image_url=actual_image_url, detail="auto", type="input_image"
+        )
+
+        if detail:
+            image_param["detail"] = detail
+
+        return image_param
+
     def _convert_content_to_responses_format(
         self,
         content: Union[
@@ -331,6 +364,8 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
         role: str,
     ) -> List[Dict[str, Any]]:
         """Convert chat completion content to responses API format"""
+        from litellm.types.llms.openai import ChatCompletionImageObject
+
         verbose_logger.debug(
             f"Chat provider: Converting content to responses format - input type: {type(content)}"
         )
@@ -360,10 +395,12 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
                         verbose_logger.debug(f"Chat provider:   text -> {converted}")
                     elif original_type == "image_url":
                         # Map to responses API image format
-                        converted = {
-                            "type": "input_image",
-                            "image_url": item.get("image_url", {}),
-                        }
+                        converted = cast(
+                            dict,
+                            self._convert_content_to_responses_format_image(
+                                cast(ChatCompletionImageObject, item), role
+                            ),
+                        )
                         result.append(converted)
                         verbose_logger.debug(
                             f"Chat provider:   image_url -> {converted}"

litellm/endpoints/speech/speech_to_completion_bridge/handler.py

@@ -0,0 +1,126 @@
+"""
+Handler for transforming /chat/completions api requests to litellm.responses requests
+"""
+
+from typing import TYPE_CHECKING, Optional, TypedDict, Union
+
+if TYPE_CHECKING:
+    from litellm import LiteLLMLoggingObj
+    from litellm.types.llms.openai import HttpxBinaryResponseContent
+
+
+class SpeechToCompletionBridgeHandlerInputKwargs(TypedDict):
+    model: str
+    input: str
+    voice: Optional[Union[str, dict]]
+    optional_params: dict
+    litellm_params: dict
+    logging_obj: "LiteLLMLoggingObj"
+    headers: dict
+    custom_llm_provider: str
+
+
+class SpeechToCompletionBridgeHandler:
+    def __init__(self):
+        from .transformation import SpeechToCompletionBridgeTransformationHandler
+
+        super().__init__()
+        self.transformation_handler = SpeechToCompletionBridgeTransformationHandler()
+
+    def validate_input_kwargs(
+        self, kwargs: dict
+    ) -> SpeechToCompletionBridgeHandlerInputKwargs:
+        from litellm import LiteLLMLoggingObj
+
+        model = kwargs.get("model")
+        if model is None or not isinstance(model, str):
+            raise ValueError("model is required")
+
+        custom_llm_provider = kwargs.get("custom_llm_provider")
+        if custom_llm_provider is None or not isinstance(custom_llm_provider, str):
+            raise ValueError("custom_llm_provider is required")
+
+        input = kwargs.get("input")
+        if input is None or not isinstance(input, str):
+            raise ValueError("input is required")
+
+        optional_params = kwargs.get("optional_params")
+        if optional_params is None or not isinstance(optional_params, dict):
+            raise ValueError("optional_params is required")
+
+        litellm_params = kwargs.get("litellm_params")
+        if litellm_params is None or not isinstance(litellm_params, dict):
+            raise ValueError("litellm_params is required")
+
+        headers = kwargs.get("headers")
+        if headers is None or not isinstance(headers, dict):
+            raise ValueError("headers is required")
+
+        headers = kwargs.get("headers")
+        if headers is None or not isinstance(headers, dict):
+            raise ValueError("headers is required")
+
+        logging_obj = kwargs.get("logging_obj")
+        if logging_obj is None or not isinstance(logging_obj, LiteLLMLoggingObj):
+            raise ValueError("logging_obj is required")
+
+        return SpeechToCompletionBridgeHandlerInputKwargs(
+            model=model,
+            input=input,
+            voice=kwargs.get("voice"),
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            logging_obj=logging_obj,
+            custom_llm_provider=custom_llm_provider,
+            headers=headers,
+        )
+
+    def speech(
+        self,
+        model: str,
+        input: str,
+        voice: Optional[Union[str, dict]],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+        logging_obj: "LiteLLMLoggingObj",
+        custom_llm_provider: str,
+    ) -> "HttpxBinaryResponseContent":
+        received_args = locals()
+        from litellm import completion
+        from litellm.types.utils import ModelResponse
+
+        validated_kwargs = self.validate_input_kwargs(received_args)
+        model = validated_kwargs["model"]
+        input = validated_kwargs["input"]
+        optional_params = validated_kwargs["optional_params"]
+        litellm_params = validated_kwargs["litellm_params"]
+        headers = validated_kwargs["headers"]
+        logging_obj = validated_kwargs["logging_obj"]
+        custom_llm_provider = validated_kwargs["custom_llm_provider"]
+        voice = validated_kwargs["voice"]
+
+        request_data = self.transformation_handler.transform_request(
+            model=model,
+            input=input,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+            litellm_logging_obj=logging_obj,
+            custom_llm_provider=custom_llm_provider,
+            voice=voice,
+        )
+
+        result = completion(
+            **request_data,
+        )
+
+        if isinstance(result, ModelResponse):
+            return self.transformation_handler.transform_response(
+                model_response=result,
+            )
+        else:
+            raise Exception("Unmapped response type. Got type: {}".format(type(result)))
+
+
+speech_to_completion_bridge_handler = SpeechToCompletionBridgeHandler()

litellm/endpoints/speech/speech_to_completion_bridge/transformation.py

@@ -0,0 +1,134 @@
+from typing import TYPE_CHECKING, Optional, Union, cast
+
+from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
+
+if TYPE_CHECKING:
+    from litellm import Logging as LiteLLMLoggingObj
+    from litellm.types.llms.openai import HttpxBinaryResponseContent
+    from litellm.types.utils import ModelResponse
+
+
+class SpeechToCompletionBridgeTransformationHandler:
+    def transform_request(
+        self,
+        model: str,
+        input: str,
+        voice: Optional[Union[str, dict]],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+        litellm_logging_obj: "LiteLLMLoggingObj",
+        custom_llm_provider: str,
+    ) -> dict:
+        passed_optional_params = {}
+        for op in optional_params:
+            if op in OPENAI_CHAT_COMPLETION_PARAMS:
+                passed_optional_params[op] = optional_params[op]
+
+        if voice is not None:
+            if isinstance(voice, str):
+                passed_optional_params["audio"] = {"voice": voice}
+                if "response_format" in optional_params:
+                    passed_optional_params["audio"]["format"] = optional_params[
+                        "response_format"
+                    ]
+
+        return_kwargs = {
+            "model": model,
+            "messages": [
+                {
+                    "role": "user",
+                    "content": input,
+                }
+            ],
+            "modalities": ["audio"],
+            **passed_optional_params,
+            **litellm_params,
+            "headers": headers,
+            "litellm_logging_obj": litellm_logging_obj,
+            "custom_llm_provider": custom_llm_provider,
+        }
+
+        # filter out None values
+        return_kwargs = {k: v for k, v in return_kwargs.items() if v is not None}
+        return return_kwargs
+
+    def _convert_pcm16_to_wav(
+        self, pcm_data: bytes, sample_rate: int = 24000, channels: int = 1
+    ) -> bytes:
+        """
+        Convert raw PCM16 data to WAV format.
+
+        Args:
+            pcm_data: Raw PCM16 audio data
+            sample_rate: Sample rate in Hz (Gemini TTS typically uses 24000)
+            channels: Number of audio channels (1 for mono)
+
+        Returns:
+            bytes: WAV formatted audio data
+        """
+        import struct
+
+        # WAV header parameters
+        byte_rate = sample_rate * channels * 2  # 2 bytes per sample (16-bit)
+        block_align = channels * 2
+        data_size = len(pcm_data)
+        file_size = 36 + data_size
+
+        # Create WAV header
+        wav_header = struct.pack(
+            "<4sI4s4sIHHIIHH4sI",
+            b"RIFF",  # Chunk ID
+            file_size,  # Chunk Size
+            b"WAVE",  # Format
+            b"fmt ",  # Subchunk1 ID
+            16,  # Subchunk1 Size (PCM)
+            1,  # Audio Format (PCM)
+            channels,  # Number of Channels
+            sample_rate,  # Sample Rate
+            byte_rate,  # Byte Rate
+            block_align,  # Block Align
+            16,  # Bits per Sample
+            b"data",  # Subchunk2 ID
+            data_size,  # Subchunk2 Size
+        )
+
+        return wav_header + pcm_data
+
+    def _is_gemini_tts_model(self, model: str) -> bool:
+        """Check if the model is a Gemini TTS model that returns PCM16 data."""
+        return "gemini" in model.lower() and (
+            "tts" in model.lower() or "preview-tts" in model.lower()
+        )
+
+    def transform_response(
+        self, model_response: "ModelResponse"
+    ) -> "HttpxBinaryResponseContent":
+        import base64
+
+        import httpx
+
+        from litellm.types.llms.openai import HttpxBinaryResponseContent
+        from litellm.types.utils import Choices
+
+        audio_part = cast(Choices, model_response.choices[0]).message.audio
+        if audio_part is None:
+            raise ValueError("No audio part found in the response")
+        audio_content = audio_part.data
+
+        # Decode base64 to get binary content
+        binary_data = base64.b64decode(audio_content)
+
+        # Check if this is a Gemini TTS model that returns raw PCM16 data
+        model = getattr(model_response, "model", "")
+        headers = {}
+        if self._is_gemini_tts_model(model):
+            # Convert PCM16 to WAV format for proper audio file playback
+            binary_data = self._convert_pcm16_to_wav(binary_data)
+            headers["Content-Type"] = "audio/wav"
+        else:
+            headers["Content-Type"] = "audio/mpeg"
+
+        # Create an httpx.Response object
+        response = httpx.Response(status_code=200, content=binary_data, headers=headers)
+        return HttpxBinaryResponseContent(response)

litellm/experimental_mcp_client/client.py

@@ -0,0 +1,164 @@
+"""
+LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
+"""
+import base64
+from datetime import timedelta
+from typing import List, Optional
+
+from mcp import ClientSession
+from mcp.client.sse import sse_client
+from mcp.client.streamable_http import streamablehttp_client
+from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
+from mcp.types import CallToolResult as MCPCallToolResult
+from mcp.types import Tool as MCPTool
+
+from litellm.types.mcp import MCPAuth, MCPAuthType, MCPTransport, MCPTransportType
+
+
+def to_basic_auth(auth_value: str) -> str:
+    """Convert auth value to Basic Auth format."""
+    return base64.b64encode(auth_value.encode("utf-8")).decode()
+
+
+class MCPClient:
+    """
+    MCP Client supporting:
+      SSE and HTTP transports
+      Authentication via Bearer token, Basic Auth, or API Key
+      Tool calling with error handling and result parsing
+    """
+
+    def __init__(
+        self,
+        server_url: str,
+        transport_type: MCPTransportType = MCPTransport.http,
+        auth_type: MCPAuthType = None,
+        auth_value: Optional[str] = None,
+        timeout: float = 60.0,
+    ):
+        self.server_url: str = server_url
+        self.transport_type: MCPTransport = transport_type
+        self.auth_type: MCPAuthType = auth_type
+        self.timeout: float = timeout
+        self._mcp_auth_value: Optional[str] = None
+        self._session: Optional[ClientSession] = None
+        self._context = None
+        self._transport_ctx = None
+        self._transport = None
+        self._session_ctx = None
+
+        # handle the basic auth value if provided
+        if auth_value:
+            self.update_auth_value(auth_value)
+
+    async def __aenter__(self):
+        """
+        Enable async context manager support.
+          Initializes the transport and session.
+        """
+        await self.connect()
+        return self
+
+    async def connect(self):
+        """Initialize the transport and session."""
+        if self._session:
+            return  # Already connected
+            
+        headers = self._get_auth_headers()
+
+        if self.transport_type == MCPTransport.sse:
+            self._transport_ctx = sse_client(
+                url=self.server_url,
+                timeout=self.timeout,
+                headers=headers,
+            )
+            self._transport = await self._transport_ctx.__aenter__()
+            self._session_ctx = ClientSession(self._transport[0], self._transport[1])
+            self._session = await self._session_ctx.__aenter__()
+            await self._session.initialize()
+        else:
+            self._transport_ctx = streamablehttp_client(
+                url=self.server_url,
+                timeout=timedelta(seconds=self.timeout),
+                headers=headers,
+            )
+            self._transport = await self._transport_ctx.__aenter__()
+            self._session_ctx = ClientSession(self._transport[0], self._transport[1])
+            self._session = await self._session_ctx.__aenter__()
+            await self._session.initialize()
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb):
+        """Cleanup when exiting context manager."""
+        if self._session:
+            await self._session_ctx.__aexit__(exc_type, exc_val, exc_tb) # type: ignore
+        if self._transport_ctx:
+            await self._transport_ctx.__aexit__(exc_type, exc_val, exc_tb)
+
+    async def disconnect(self):
+        """Clean up session and connections."""
+        if self._session:
+            try:
+                # Ensure session is properly closed
+                await self._session.close()  # type: ignore 
+            except Exception:
+                pass
+            self._session = None
+
+        if self._context:
+            try:
+                await self._context.__aexit__(None, None, None) # type: ignore
+            except Exception:
+                pass
+            self._context = None
+
+    def update_auth_value(self, mcp_auth_value: str):
+        """
+        Set the authentication header for the MCP client.
+        """
+        if self.auth_type == MCPAuth.basic:
+            # Assuming mcp_auth_value is in format "username:password", convert it when updating
+            mcp_auth_value = to_basic_auth(mcp_auth_value)
+        self._mcp_auth_value = mcp_auth_value
+
+    def _get_auth_headers(self) -> dict:
+        """Generate authentication headers based on auth type."""
+        if not self._mcp_auth_value:
+            return {}
+
+        if self.auth_type == MCPAuth.bearer_token:
+            return {"Authorization": f"Bearer {self._mcp_auth_value}"}
+        elif self.auth_type == MCPAuth.basic:
+            return {"Authorization": f"Basic {self._mcp_auth_value}"}
+        elif self.auth_type == MCPAuth.api_key:
+            return {"X-API-Key": self._mcp_auth_value}
+        return {}
+
+    async def list_tools(self) -> List[MCPTool]:
+        """List available tools from the server."""
+        if not self._session:
+            await self.connect()
+        if self._session is None:
+            raise ValueError("Session is not initialized")
+
+        result = await self._session.list_tools()
+        return result.tools
+
+    async def call_tool(
+        self, call_tool_request_params: MCPCallToolRequestParams
+    ) -> MCPCallToolResult:
+        """
+        Call an MCP Tool.
+        """
+        if not self._session:
+            await self.connect()
+
+        if self._session is None:
+            raise ValueError("Session is not initialized")
+        
+        tool_result = await self._session.call_tool(
+            name=call_tool_request_params.name,
+            arguments=call_tool_request_params.arguments,
+        )
+        return tool_result
+        
+

litellm/integrations/prometheus.py

@@ -40,7 +40,7 @@ class PrometheusLogger(CustomLogger):
             from prometheus_client import Counter, Gauge, Histogram
 
             from litellm.proxy.proxy_server import CommonProxyErrors, premium_user
-            
+
             # Always initialize label_filters, even for non-premium users
             self.label_filters = self._parse_prometheus_config()
 
@@ -215,40 +215,25 @@ class PrometheusLogger(CustomLogger):
             self.litellm_remaining_requests_metric = self._gauge_factory(
                 "litellm_remaining_requests",
                 "LLM Deployment Analytics - remaining requests for model, returned from LLM API Provider",
-                labelnames=[
-                    "model_group",
-                    "api_provider",
-                    "api_base",
-                    "litellm_model_name",
-                    "hashed_api_key",
-                    "api_key_alias",
-                ],
+                labelnames=self.get_labels_for_metric(
+                    "litellm_remaining_requests_metric"
+                ),
             )
 
             self.litellm_remaining_tokens_metric = self._gauge_factory(
                 "litellm_remaining_tokens",
                 "remaining tokens for model, returned from LLM API Provider",
-                labelnames=[
-                    "model_group",
-                    "api_provider",
-                    "api_base",
-                    "litellm_model_name",
-                    "hashed_api_key",
-                    "api_key_alias",
-                ],
+                labelnames=self.get_labels_for_metric(
+                    "litellm_remaining_tokens_metric"
+                ),
             )
 
             self.litellm_overhead_latency_metric = self._histogram_factory(
                 "litellm_overhead_latency_metric",
                 "Latency overhead (milliseconds) added by LiteLLM processing",
-                labelnames=[
-                    "model_group",
-                    "api_provider",
-                    "api_base",
-                    "litellm_model_name",
-                    "hashed_api_key",
-                    "api_key_alias",
-                ],
+                labelnames=self.get_labels_for_metric(
+                    "litellm_overhead_latency_metric"
+                ),
                 buckets=LATENCY_BUCKETS,
             )
             # llm api provider budget metrics
@@ -566,6 +551,7 @@ class PrometheusLogger(CustomLogger):
             hashed_api_key=user_api_key,
             api_key_alias=user_api_key_alias,
             requested_model=standard_logging_payload["model_group"],
+            model_group=standard_logging_payload["model_group"],
             team=user_api_team,
             team_alias=user_api_team_alias,
             user=user_id,
@@ -1070,6 +1056,31 @@ class PrometheusLogger(CustomLogger):
 
             llm_provider = _litellm_params.get("custom_llm_provider", None)
 
+            # Create enum_values for the label factory (always create for use in different metrics)
+            enum_values = UserAPIKeyLabelValues(
+                litellm_model_name=litellm_model_name,
+                model_id=model_id,
+                api_base=api_base,
+                api_provider=llm_provider,
+                exception_status=(
+                    str(getattr(exception, "status_code", None)) if exception else None
+                ),
+                exception_class=(
+                    self._get_exception_class_name(exception) if exception else None
+                ),
+                requested_model=model_group,
+                hashed_api_key=standard_logging_payload["metadata"][
+                    "user_api_key_hash"
+                ],
+                api_key_alias=standard_logging_payload["metadata"][
+                    "user_api_key_alias"
+                ],
+                team=standard_logging_payload["metadata"]["user_api_key_team_id"],
+                team_alias=standard_logging_payload["metadata"][
+                    "user_api_key_team_alias"
+                ],
+            )
+
             """
             log these labels
             ["litellm_model_name", "model_id", "api_base", "api_provider"]
@@ -1081,25 +1092,14 @@ class PrometheusLogger(CustomLogger):
                 api_provider=llm_provider or "",
             )
             if exception is not None:
-                self.litellm_deployment_failure_responses.labels(
-                    litellm_model_name=litellm_model_name,
-                    model_id=model_id,
-                    api_base=api_base,
-                    api_provider=llm_provider,
-                    exception_status=str(getattr(exception, "status_code", None)),
-                    exception_class=self._get_exception_class_name(exception),
-                    requested_model=model_group,
-                    hashed_api_key=standard_logging_payload["metadata"][
-                        "user_api_key_hash"
-                    ],
-                    api_key_alias=standard_logging_payload["metadata"][
-                        "user_api_key_alias"
-                    ],
-                    team=standard_logging_payload["metadata"]["user_api_key_team_id"],
-                    team_alias=standard_logging_payload["metadata"][
-                        "user_api_key_team_alias"
-                    ],
-                ).inc()
+
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_deployment_failure_responses"
+                    ),
+                    enum_values=enum_values,
+                )
+                self.litellm_deployment_failure_responses.labels(**_labels).inc()
 
                 # tag based tracking
                 if standard_logging_payload is not None and isinstance(
@@ -1122,23 +1122,13 @@ class PrometheusLogger(CustomLogger):
                             }
                         ).inc()
 
-            self.litellm_deployment_total_requests.labels(
-                litellm_model_name=litellm_model_name,
-                model_id=model_id,
-                api_base=api_base,
-                api_provider=llm_provider,
-                requested_model=model_group,
-                hashed_api_key=standard_logging_payload["metadata"][
-                    "user_api_key_hash"
-                ],
-                api_key_alias=standard_logging_payload["metadata"][
-                    "user_api_key_alias"
-                ],
-                team=standard_logging_payload["metadata"]["user_api_key_team_id"],
-                team_alias=standard_logging_payload["metadata"][
-                    "user_api_key_team_alias"
-                ],
-            ).inc()
+            _labels = prometheus_label_factory(
+                supported_enum_labels=self.get_labels_for_metric(
+                    metric_name="litellm_deployment_total_requests"
+                ),
+                enum_values=enum_values,
+            )
+            self.litellm_deployment_total_requests.labels(**_labels).inc()
 
             pass
         except Exception as e:
@@ -1156,6 +1146,7 @@ class PrometheusLogger(CustomLogger):
         enum_values: UserAPIKeyLabelValues,
         output_tokens: float = 1.0,
     ):
+
         try:
             verbose_logger.debug("setting remaining tokens requests metric")
             standard_logging_payload: Optional[StandardLoggingPayload] = (
@@ -1165,9 +1156,7 @@ class PrometheusLogger(CustomLogger):
             if standard_logging_payload is None:
                 return
 
-            model_group = standard_logging_payload["model_group"]
             api_base = standard_logging_payload["api_base"]
-            _response_headers = request_kwargs.get("response_headers")
             _litellm_params = request_kwargs.get("litellm_params", {}) or {}
             _metadata = _litellm_params.get("metadata", {})
             litellm_model_name = request_kwargs.get("model", None)
@@ -1191,14 +1180,13 @@ class PrometheusLogger(CustomLogger):
             if litellm_overhead_time_ms := standard_logging_payload[
                 "hidden_params"
             ].get("litellm_overhead_time_ms"):
-                self.litellm_overhead_latency_metric.labels(
-                    model_group,
-                    llm_provider,
-                    api_base,
-                    litellm_model_name,
-                    standard_logging_payload["metadata"]["user_api_key_hash"],
-                    standard_logging_payload["metadata"]["user_api_key_alias"],
-                ).observe(
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_overhead_latency_metric"
+                    ),
+                    enum_values=enum_values,
+                )
+                self.litellm_overhead_latency_metric.labels(**_labels).observe(
                     litellm_overhead_time_ms / 1000
                 )  # set as seconds
 
@@ -1209,24 +1197,26 @@ class PrometheusLogger(CustomLogger):
                 "api_base",
                 "litellm_model_name"
                 """
-                self.litellm_remaining_requests_metric.labels(
-                    model_group,
-                    llm_provider,
-                    api_base,
-                    litellm_model_name,
-                    standard_logging_payload["metadata"]["user_api_key_hash"],
-                    standard_logging_payload["metadata"]["user_api_key_alias"],
-                ).set(remaining_requests)
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_remaining_requests_metric"
+                    ),
+                    enum_values=enum_values,
+                )
+                self.litellm_remaining_requests_metric.labels(**_labels).set(
+                    remaining_requests
+                )
 
             if remaining_tokens:
-                self.litellm_remaining_tokens_metric.labels(
-                    model_group,
-                    llm_provider,
-                    api_base,
-                    litellm_model_name,
-                    standard_logging_payload["metadata"]["user_api_key_hash"],
-                    standard_logging_payload["metadata"]["user_api_key_alias"],
-                ).set(remaining_tokens)
+                _labels = prometheus_label_factory(
+                    supported_enum_labels=self.get_labels_for_metric(
+                        metric_name="litellm_remaining_tokens_metric"
+                    ),
+                    enum_values=enum_values,
+                )
+                self.litellm_remaining_tokens_metric.labels(**_labels).set(
+                    remaining_tokens
+                )
 
             """
             log these labels
@@ -1239,41 +1229,21 @@ class PrometheusLogger(CustomLogger):
                 api_provider=llm_provider or "",
             )
 
-            self.litellm_deployment_success_responses.labels(
-                litellm_model_name=litellm_model_name,
-                model_id=model_id,
-                api_base=api_base,
-                api_provider=llm_provider,
-                requested_model=model_group,
-                hashed_api_key=standard_logging_payload["metadata"][
-                    "user_api_key_hash"
-                ],
-                api_key_alias=standard_logging_payload["metadata"][
-                    "user_api_key_alias"
-                ],
-                team=standard_logging_payload["metadata"]["user_api_key_team_id"],
-                team_alias=standard_logging_payload["metadata"][
-                    "user_api_key_team_alias"
-                ],
-            ).inc()
+            _labels = prometheus_label_factory(
+                supported_enum_labels=self.get_labels_for_metric(
+                    metric_name="litellm_deployment_success_responses"
+                ),
+                enum_values=enum_values,
+            )
+            self.litellm_deployment_success_responses.labels(**_labels).inc()
 
-            self.litellm_deployment_total_requests.labels(
-                litellm_model_name=litellm_model_name,
-                model_id=model_id,
-                api_base=api_base,
-                api_provider=llm_provider,
-                requested_model=model_group,
-                hashed_api_key=standard_logging_payload["metadata"][
-                    "user_api_key_hash"
-                ],
-                api_key_alias=standard_logging_payload["metadata"][
-                    "user_api_key_alias"
-                ],
-                team=standard_logging_payload["metadata"]["user_api_key_team_id"],
-                team_alias=standard_logging_payload["metadata"][
-                    "user_api_key_team_alias"
-                ],
-            ).inc()
+            _labels = prometheus_label_factory(
+                supported_enum_labels=self.get_labels_for_metric(
+                    metric_name="litellm_deployment_total_requests"
+                ),
+                enum_values=enum_values,
+            )
+            self.litellm_deployment_total_requests.labels(**_labels).inc()
 
             # Track deployment Latency
             response_ms: timedelta = end_time - start_time
@@ -1309,7 +1279,7 @@ class PrometheusLogger(CustomLogger):
                 ).observe(latency_per_token)
 
         except Exception as e:
-            verbose_logger.error(
+            verbose_logger.exception(
                 "Prometheus Error: set_llm_deployment_success_metrics. Exception occured - {}".format(
                     str(e)
                 )

litellm/litellm_core_utils/litellm_logging.py

@@ -288,9 +288,9 @@ class Logging(LiteLLMLoggingBaseClass):
         self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
         self.function_id = function_id
         self.streaming_chunks: List[Any] = []  # for generating complete stream response
-        self.sync_streaming_chunks: List[
-            Any
-        ] = []  # for generating complete stream response
+        self.sync_streaming_chunks: List[Any] = (
+            []
+        )  # for generating complete stream response
         self.log_raw_request_response = log_raw_request_response
 
         # Initialize dynamic callbacks
@@ -645,9 +645,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
             non_default_params
         ):
-            self.model_call_details[
-                "prompt_integration"
-            ] = anthropic_cache_control_logger.__class__.__name__
+            self.model_call_details["prompt_integration"] = (
+                anthropic_cache_control_logger.__class__.__name__
+            )
             return anthropic_cache_control_logger
 
         #########################################################
@@ -665,9 +665,9 @@ class Logging(LiteLLMLoggingBaseClass):
                         ),
                     )
                 )
-                self.model_call_details[
-                    "prompt_integration"
-                ] = vector_store_custom_logger.__class__.__name__
+                self.model_call_details["prompt_integration"] = (
+                    vector_store_custom_logger.__class__.__name__
+                )
                 return vector_store_custom_logger
 
         return None
@@ -719,9 +719,9 @@ class Logging(LiteLLMLoggingBaseClass):
             model
         ):  # if model name was changes pre-call, overwrite the initial model call name with the new one
             self.model_call_details["model"] = model
-        self.model_call_details["litellm_params"][
-            "api_base"
-        ] = self._get_masked_api_base(additional_args.get("api_base", ""))
+        self.model_call_details["litellm_params"]["api_base"] = (
+            self._get_masked_api_base(additional_args.get("api_base", ""))
+        )
 
     def pre_call(self, input, api_key, model=None, additional_args={}):  # noqa: PLR0915
         # Log the exact input to the LLM API
@@ -750,10 +750,10 @@ class Logging(LiteLLMLoggingBaseClass):
                 try:
                     # [Non-blocking Extra Debug Information in metadata]
                     if turn_off_message_logging is True:
-                        _metadata[
-                            "raw_request"
-                        ] = "redacted by litellm. \
+                        _metadata["raw_request"] = (
+                            "redacted by litellm. \
                             'litellm.turn_off_message_logging=True'"
+                        )
                     else:
                         curl_command = self._get_request_curl_command(
                             api_base=additional_args.get("api_base", ""),
@@ -764,32 +764,32 @@ class Logging(LiteLLMLoggingBaseClass):
 
                         _metadata["raw_request"] = str(curl_command)
                         # split up, so it's easier to parse in the UI
-                        self.model_call_details[
-                            "raw_request_typed_dict"
-                        ] = RawRequestTypedDict(
-                            raw_request_api_base=str(
-                                additional_args.get("api_base") or ""
-                            ),
-                            raw_request_body=self._get_raw_request_body(
-                                additional_args.get("complete_input_dict", {})
-                            ),
-                            raw_request_headers=self._get_masked_headers(
-                                additional_args.get("headers", {}) or {},
-                                ignore_sensitive_headers=True,
-                            ),
-                            error=None,
+                        self.model_call_details["raw_request_typed_dict"] = (
+                            RawRequestTypedDict(
+                                raw_request_api_base=str(
+                                    additional_args.get("api_base") or ""
+                                ),
+                                raw_request_body=self._get_raw_request_body(
+                                    additional_args.get("complete_input_dict", {})
+                                ),
+                                raw_request_headers=self._get_masked_headers(
+                                    additional_args.get("headers", {}) or {},
+                                    ignore_sensitive_headers=True,
+                                ),
+                                error=None,
+                            )
                         )
                 except Exception as e:
-                    self.model_call_details[
-                        "raw_request_typed_dict"
-                    ] = RawRequestTypedDict(
-                        error=str(e),
+                    self.model_call_details["raw_request_typed_dict"] = (
+                        RawRequestTypedDict(
+                            error=str(e),
+                        )
                     )
-                    _metadata[
-                        "raw_request"
-                    ] = "Unable to Log \
+                    _metadata["raw_request"] = (
+                        "Unable to Log \
                         raw request: {}".format(
-                        str(e)
+                            str(e)
+                        )
                     )
             if self.logger_fn and callable(self.logger_fn):
                 try:
@@ -1120,9 +1120,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details[
-                "response_cost_failure_debug_information"
-            ] = debug_info
+            self.model_call_details["response_cost_failure_debug_information"] = (
+                debug_info
+            )
             return None
 
         try:
@@ -1147,9 +1147,9 @@ class Logging(LiteLLMLoggingBaseClass):
             verbose_logger.debug(
                 f"response_cost_failure_debug_information: {debug_info}"
             )
-            self.model_call_details[
-                "response_cost_failure_debug_information"
-            ] = debug_info
+            self.model_call_details["response_cost_failure_debug_information"] = (
+                debug_info
+            )
 
         return None
 
@@ -1238,9 +1238,9 @@ class Logging(LiteLLMLoggingBaseClass):
                 end_time = datetime.datetime.now()
             if self.completion_start_time is None:
                 self.completion_start_time = end_time
-                self.model_call_details[
-                    "completion_start_time"
-                ] = self.completion_start_time
+                self.model_call_details["completion_start_time"] = (
+                    self.completion_start_time
+                )
             self.model_call_details["log_event_type"] = "successful_api_call"
             self.model_call_details["end_time"] = end_time
             self.model_call_details["cache_hit"] = cache_hit
@@ -1320,39 +1320,39 @@ class Logging(LiteLLMLoggingBaseClass):
                             "response_cost"
                         ]
                     else:
-                        self.model_call_details[
-                            "response_cost"
-                        ] = self._response_cost_calculator(result=logging_result)
+                        self.model_call_details["response_cost"] = (
+                            self._response_cost_calculator(result=logging_result)
+                        )
                     ## STANDARDIZED LOGGING PAYLOAD
 
-                    self.model_call_details[
-                        "standard_logging_object"
-                    ] = get_standard_logging_object_payload(
-                        kwargs=self.model_call_details,
-                        init_response_obj=logging_result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        logging_obj=self,
-                        status="success",
-                        standard_built_in_tools_params=self.standard_built_in_tools_params,
+                    self.model_call_details["standard_logging_object"] = (
+                        get_standard_logging_object_payload(
+                            kwargs=self.model_call_details,
+                            init_response_obj=logging_result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            logging_obj=self,
+                            status="success",
+                            standard_built_in_tools_params=self.standard_built_in_tools_params,
+                        )
                     )
                 elif isinstance(result, dict) or isinstance(result, list):
                     ## STANDARDIZED LOGGING PAYLOAD
-                    self.model_call_details[
-                        "standard_logging_object"
-                    ] = get_standard_logging_object_payload(
-                        kwargs=self.model_call_details,
-                        init_response_obj=result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        logging_obj=self,
-                        status="success",
-                        standard_built_in_tools_params=self.standard_built_in_tools_params,
+                    self.model_call_details["standard_logging_object"] = (
+                        get_standard_logging_object_payload(
+                            kwargs=self.model_call_details,
+                            init_response_obj=result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            logging_obj=self,
+                            status="success",
+                            standard_built_in_tools_params=self.standard_built_in_tools_params,
+                        )
                     )
             elif standard_logging_object is not None:
-                self.model_call_details[
-                    "standard_logging_object"
-                ] = standard_logging_object
+                self.model_call_details["standard_logging_object"] = (
+                    standard_logging_object
+                )
             else:  # streaming chunks + image gen.
                 self.model_call_details["response_cost"] = None
 
@@ -1424,23 +1424,23 @@ class Logging(LiteLLMLoggingBaseClass):
                 verbose_logger.debug(
                     "Logging Details LiteLLM-Success Call streaming complete"
                 )
-                self.model_call_details[
-                    "complete_streaming_response"
-                ] = complete_streaming_response
-                self.model_call_details[
-                    "response_cost"
-                ] = self._response_cost_calculator(result=complete_streaming_response)
+                self.model_call_details["complete_streaming_response"] = (
+                    complete_streaming_response
+                )
+                self.model_call_details["response_cost"] = (
+                    self._response_cost_calculator(result=complete_streaming_response)
+                )
                 ## STANDARDIZED LOGGING PAYLOAD
-                self.model_call_details[
-                    "standard_logging_object"
-                ] = get_standard_logging_object_payload(
-                    kwargs=self.model_call_details,
-                    init_response_obj=complete_streaming_response,
-                    start_time=start_time,
-                    end_time=end_time,
-                    logging_obj=self,
-                    status="success",
-                    standard_built_in_tools_params=self.standard_built_in_tools_params,
+                self.model_call_details["standard_logging_object"] = (
+                    get_standard_logging_object_payload(
+                        kwargs=self.model_call_details,
+                        init_response_obj=complete_streaming_response,
+                        start_time=start_time,
+                        end_time=end_time,
+                        logging_obj=self,
+                        status="success",
+                        standard_built_in_tools_params=self.standard_built_in_tools_params,
+                    )
                 )
             callbacks = self.get_combined_callback_list(
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
@@ -1761,10 +1761,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details[
-                                    "complete_response"
-                                ] = self.model_call_details.get(
-                                    "complete_streaming_response", {}
+                                self.model_call_details["complete_response"] = (
+                                    self.model_call_details.get(
+                                        "complete_streaming_response", {}
+                                    )
                                 )
                                 result = self.model_call_details["complete_response"]
                             openMeterLogger.log_success_event(
@@ -1803,10 +1803,10 @@ class Logging(LiteLLMLoggingBaseClass):
                             )
                         else:
                             if self.stream and complete_streaming_response:
-                                self.model_call_details[
-                                    "complete_response"
-                                ] = self.model_call_details.get(
-                                    "complete_streaming_response", {}
+                                self.model_call_details["complete_response"] = (
+                                    self.model_call_details.get(
+                                        "complete_streaming_response", {}
+                                    )
                                 )
                                 result = self.model_call_details["complete_response"]
 
@@ -1917,9 +1917,9 @@ class Logging(LiteLLMLoggingBaseClass):
         if complete_streaming_response is not None:
             print_verbose("Async success callbacks: Got a complete streaming response")
 
-            self.model_call_details[
-                "async_complete_streaming_response"
-            ] = complete_streaming_response
+            self.model_call_details["async_complete_streaming_response"] = (
+                complete_streaming_response
+            )
             try:
                 if self.model_call_details.get("cache_hit", False) is True:
                     self.model_call_details["response_cost"] = 0.0
@@ -1929,10 +1929,10 @@ class Logging(LiteLLMLoggingBaseClass):
                         model_call_details=self.model_call_details
                     )
                     # base_model defaults to None if not set on model_info
-                    self.model_call_details[
-                        "response_cost"
-                    ] = self._response_cost_calculator(
-                        result=complete_streaming_response
+                    self.model_call_details["response_cost"] = (
+                        self._response_cost_calculator(
+                            result=complete_streaming_response
+                        )
                     )
 
                 verbose_logger.debug(
@@ -1945,16 +1945,16 @@ class Logging(LiteLLMLoggingBaseClass):
                 self.model_call_details["response_cost"] = None
 
             ## STANDARDIZED LOGGING PAYLOAD
-            self.model_call_details[
-                "standard_logging_object"
-            ] = get_standard_logging_object_payload(
-                kwargs=self.model_call_details,
-                init_response_obj=complete_streaming_response,
-                start_time=start_time,
-                end_time=end_time,
-                logging_obj=self,
-                status="success",
-                standard_built_in_tools_params=self.standard_built_in_tools_params,
+            self.model_call_details["standard_logging_object"] = (
+                get_standard_logging_object_payload(
+                    kwargs=self.model_call_details,
+                    init_response_obj=complete_streaming_response,
+                    start_time=start_time,
+                    end_time=end_time,
+                    logging_obj=self,
+                    status="success",
+                    standard_built_in_tools_params=self.standard_built_in_tools_params,
+                )
             )
         callbacks = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@@ -2161,18 +2161,18 @@ class Logging(LiteLLMLoggingBaseClass):
 
         ## STANDARDIZED LOGGING PAYLOAD
 
-        self.model_call_details[
-            "standard_logging_object"
-        ] = get_standard_logging_object_payload(
-            kwargs=self.model_call_details,
-            init_response_obj={},
-            start_time=start_time,
-            end_time=end_time,
-            logging_obj=self,
-            status="failure",
-            error_str=str(exception),
-            original_exception=exception,
-            standard_built_in_tools_params=self.standard_built_in_tools_params,
+        self.model_call_details["standard_logging_object"] = (
+            get_standard_logging_object_payload(
+                kwargs=self.model_call_details,
+                init_response_obj={},
+                start_time=start_time,
+                end_time=end_time,
+                logging_obj=self,
+                status="failure",
+                error_str=str(exception),
+                original_exception=exception,
+                standard_built_in_tools_params=self.standard_built_in_tools_params,
+            )
         )
         return start_time, end_time
 
@@ -2990,9 +2990,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 endpoint=arize_config.endpoint,
             )
 
-            os.environ[
-                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-            ] = f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
+            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
+            )
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, ArizeLogger)
@@ -3016,9 +3016,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
 
             # auth can be disabled on local deployments of arize phoenix
             if arize_phoenix_config.otlp_auth_headers is not None:
-                os.environ[
-                    "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-                ] = arize_phoenix_config.otlp_auth_headers
+                os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                    arize_phoenix_config.otlp_auth_headers
+                )
 
             for callback in _in_memory_loggers:
                 if (
@@ -3118,9 +3118,9 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 exporter="otlp_http",
                 endpoint="https://langtrace.ai/api/trace",
             )
-            os.environ[
-                "OTEL_EXPORTER_OTLP_TRACES_HEADERS"
-            ] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
+            os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
+                f"api_key={os.getenv('LANGTRACE_API_KEY')}"
+            )
             for callback in _in_memory_loggers:
                 if (
                     isinstance(callback, OpenTelemetry)
@@ -3154,7 +3154,7 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
             )
 
             langfuse_otel_config = LangfuseOtelLogger.get_langfuse_otel_config()
-            
+
             # The endpoint and headers are now set as environment variables by get_langfuse_otel_config()
             otel_config = OpenTelemetryConfig(
                 exporter=langfuse_otel_config.protocol,
@@ -3720,10 +3720,10 @@ class StandardLoggingPayloadSetup:
             for key in StandardLoggingHiddenParams.__annotations__.keys():
                 if key in hidden_params:
                     if key == "additional_headers":
-                        clean_hidden_params[
-                            "additional_headers"
-                        ] = StandardLoggingPayloadSetup.get_additional_headers(
-                            hidden_params[key]
+                        clean_hidden_params["additional_headers"] = (
+                            StandardLoggingPayloadSetup.get_additional_headers(
+                                hidden_params[key]
+                            )
                         )
                     else:
                         clean_hidden_params[key] = hidden_params[key]  # type: ignore
@@ -3815,6 +3815,44 @@ class StandardLoggingPayloadSetup:
         else:
             return logging_obj.litellm_trace_id
 
+    @staticmethod
+    def _get_user_agent_tags(proxy_server_request: dict) -> Optional[List[str]]:
+        """
+        Return the user agent tags from the proxy server request for spend tracking
+        """
+        if litellm.disable_add_user_agent_to_request_tags is True:
+            return None
+        user_agent_tags: Optional[List[str]] = None
+        headers = proxy_server_request.get("headers", {})
+        if headers is not None and isinstance(headers, dict):
+            if "user-agent" in headers:
+                user_agent = headers["user-agent"]
+                if user_agent is not None:
+                    if user_agent_tags is None:
+                        user_agent_tags = []
+                    user_agent_part: Optional[str] = None
+                    if "/" in user_agent:
+                        user_agent_part = user_agent.split("/")[0]
+                    if user_agent_part is not None:
+                        user_agent_tags.append("User-Agent: " + user_agent_part)
+                    if user_agent is not None:
+                        user_agent_tags.append("User-Agent: " + user_agent)
+        return user_agent_tags
+
+    @staticmethod
+    def _get_request_tags(metadata: dict, proxy_server_request: dict) -> List[str]:
+        request_tags = (
+            metadata.get("tags", [])
+            if isinstance(metadata.get("tags", []), list)
+            else []
+        )
+        user_agent_tags = StandardLoggingPayloadSetup._get_user_agent_tags(
+            proxy_server_request
+        )
+        if user_agent_tags is not None:
+            request_tags.extend(user_agent_tags)
+        return request_tags
+
 
 def get_standard_logging_object_payload(
     kwargs: Optional[dict],
@@ -3885,10 +3923,8 @@ def get_standard_logging_object_payload(
         _model_id = metadata.get("model_info", {}).get("id", "")
         _model_group = metadata.get("model_group", "")
 
-        request_tags = (
-            metadata.get("tags", [])
-            if isinstance(metadata.get("tags", []), list)
-            else []
+        request_tags = StandardLoggingPayloadSetup._get_request_tags(
+            metadata=metadata, proxy_server_request=proxy_server_request
         )
 
         # cleanup timestamps
@@ -4103,9 +4139,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
     ):
         for k, v in metadata["user_api_key_metadata"].items():
             if k == "logging":  # prevent logging user logging keys
-                cleaned_user_api_key_metadata[
-                    k
-                ] = "scrubbed_by_litellm_for_sensitive_keys"
+                cleaned_user_api_key_metadata[k] = (
+                    "scrubbed_by_litellm_for_sensitive_keys"
+                )
             else:
                 cleaned_user_api_key_metadata[k] = v
 

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -1053,10 +1053,10 @@ def convert_to_gemini_tool_call_invoke(
         if tool_calls is not None:
             for tool in tool_calls:
                 if "function" in tool:
-                    gemini_function_call: Optional[
-                        VertexFunctionCall
-                    ] = _gemini_tool_call_invoke_helper(
-                        function_call_params=tool["function"]
+                    gemini_function_call: Optional[VertexFunctionCall] = (
+                        _gemini_tool_call_invoke_helper(
+                            function_call_params=tool["function"]
+                        )
                     )
                     if gemini_function_call is not None:
                         _parts_list.append(
@@ -1573,9 +1573,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
                             )
 
                             if "cache_control" in _content_element:
-                                _anthropic_content_element[
-                                    "cache_control"
-                                ] = _content_element["cache_control"]
+                                _anthropic_content_element["cache_control"] = (
+                                    _content_element["cache_control"]
+                                )
                             user_content.append(_anthropic_content_element)
                         elif m.get("type", "") == "text":
                             m = cast(ChatCompletionTextObject, m)
@@ -1613,9 +1613,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
                     )
 
                     if "cache_control" in _content_element:
-                        _anthropic_content_text_element[
-                            "cache_control"
-                        ] = _content_element["cache_control"]
+                        _anthropic_content_text_element["cache_control"] = (
+                            _content_element["cache_control"]
+                        )
 
                     user_content.append(_anthropic_content_text_element)
 
@@ -2433,8 +2433,10 @@ class BedrockImageProcessor:
 
         # Extract MIME type using regular expression
         mime_type_match = re.match(r"data:(.*?);base64", image_metadata)
+
         if mime_type_match:
             mime_type = mime_type_match.group(1)
+            mime_type = mime_type.split(";")[0]
             image_format = mime_type.split("/")[1]
         else:
             mime_type = "image/jpeg"
@@ -2458,6 +2460,7 @@ class BedrockImageProcessor:
 
         document_types = ["application", "text"]
         is_document = any(mime_type.startswith(doc_type) for doc_type in document_types)
+
         supported_image_and_video_formats: List[str] = (
             supported_video_formats + supported_image_formats
         )

litellm/llms/anthropic/chat/transformation.py

@@ -77,9 +77,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
     to pass metadata to anthropic, it's {"user_id": "any-relevant-information"}
     """
 
-    max_tokens: Optional[
-        int
-    ] = DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS  # anthropic requires a default value (Opus, Sonnet, and Haiku have the same default)
+    max_tokens: Optional[int] = (
+        DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS  # anthropic requires a default value (Opus, Sonnet, and Haiku have the same default)
+    )
     stop_sequences: Optional[list] = None
     temperature: Optional[int] = None
     top_p: Optional[int] = None
@@ -104,11 +104,16 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             if key != "self" and value is not None:
                 setattr(self.__class__, key, value)
 
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "anthropic"
+
     @classmethod
     def get_config(cls):
         return super().get_config()
 
     def get_supported_openai_params(self, model: str):
+
         params = [
             "stream",
             "stop",
@@ -447,11 +452,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 if mcp_servers:
                     optional_params["mcp_servers"] = mcp_servers
             if param == "tool_choice" or param == "parallel_tool_calls":
-                _tool_choice: Optional[
-                    AnthropicMessagesToolChoice
-                ] = self._map_tool_choice(
-                    tool_choice=non_default_params.get("tool_choice"),
-                    parallel_tool_use=non_default_params.get("parallel_tool_calls"),
+                _tool_choice: Optional[AnthropicMessagesToolChoice] = (
+                    self._map_tool_choice(
+                        tool_choice=non_default_params.get("tool_choice"),
+                        parallel_tool_use=non_default_params.get("parallel_tool_calls"),
+                    )
                 )
 
                 if _tool_choice is not None:
@@ -557,9 +562,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                         text=system_message_block["content"],
                     )
                     if "cache_control" in system_message_block:
-                        anthropic_system_message_content[
-                            "cache_control"
-                        ] = system_message_block["cache_control"]
+                        anthropic_system_message_content["cache_control"] = (
+                            system_message_block["cache_control"]
+                        )
                     anthropic_system_message_list.append(
                         anthropic_system_message_content
                     )
@@ -573,9 +578,9 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                             )
                         )
                         if "cache_control" in _content:
-                            anthropic_system_message_content[
-                                "cache_control"
-                            ] = _content["cache_control"]
+                            anthropic_system_message_content["cache_control"] = (
+                                _content["cache_control"]
+                            )
 
                         anthropic_system_message_list.append(
                             anthropic_system_message_content
@@ -735,9 +740,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
                 )
         return _message
 
-    def extract_response_content(
-        self, completion_response: dict
-    ) -> Tuple[
+    def extract_response_content(self, completion_response: dict) -> Tuple[
         str,
         Optional[List[Any]],
         Optional[

litellm/llms/azure/azure.py

@@ -771,10 +771,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             status_code = getattr(e, "status_code", 500)
             error_headers = getattr(e, "headers", None)
             error_response = getattr(e, "response", None)
+            error_text = str(e)
             if error_headers is None and error_response:
                 error_headers = getattr(error_response, "headers", None)
+                error_text = error_response.text
             raise AzureOpenAIError(
-                status_code=status_code, message=str(e), headers=error_headers
+                status_code=status_code, message=error_text, headers=error_headers
             )
 
     async def make_async_azure_httpx_request(

litellm/llms/azure/common_utils.py

@@ -162,7 +162,7 @@ def get_azure_ad_token_from_oidc(
     azure_ad_token: str,
     azure_client_id: Optional[str],
     azure_tenant_id: Optional[str],
-    scope: str = "https://cognitiveservices.azure.com/.default",
+    scope: Optional[str] = None,
 ) -> str:
     """
     Get Azure AD token from OIDC token
@@ -176,6 +176,8 @@ def get_azure_ad_token_from_oidc(
     Returns:
         `azure_ad_token_access_token` - str
     """
+    if scope is None:
+        scope = "https://cognitiveservices.azure.com/.default"
     azure_authority_host = os.getenv(
         "AZURE_AUTHORITY_HOST", "https://login.microsoftonline.com"
     )
@@ -209,6 +211,7 @@ def get_azure_ad_token_from_oidc(
         return azure_ad_token_access_token
 
     client = litellm.module_level_client
+
     req_token = client.post(
         f"{azure_authority_host}/{azure_tenant_id}/oauth2/v2.0/token",
         data={
@@ -338,13 +341,19 @@ class BaseAzureLLM(BaseOpenAILLM):
             "azure_password", os.getenv("AZURE_PASSWORD")
         )
         scope = litellm_params.get(
-            "azure_scope", os.getenv("AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"))
+            "azure_scope",
+            os.getenv("AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"),
+        )
+        if scope is None:
+            scope = "https://cognitiveservices.azure.com/.default"
         max_retries = litellm_params.get("max_retries")
         timeout = litellm_params.get("timeout")
         if (
             not api_key
             and azure_ad_token_provider is None
-            and tenant_id and client_id and client_secret
+            and tenant_id
+            and client_id
+            and client_secret
         ):
             verbose_logger.debug(
                 "Using Azure AD Token Provider from Entra ID for Azure Auth"
@@ -355,7 +364,12 @@ class BaseAzureLLM(BaseOpenAILLM):
                 client_secret=client_secret,
                 scope=scope,
             )
-        if azure_ad_token_provider is None and azure_username and azure_password and client_id:
+        if (
+            azure_ad_token_provider is None
+            and azure_username
+            and azure_password
+            and client_id
+        ):
             verbose_logger.debug("Using Azure Username and Password for Azure Auth")
             azure_ad_token_provider = get_azure_ad_token_from_username_password(
                 azure_username=azure_username,
@@ -381,7 +395,7 @@ class BaseAzureLLM(BaseOpenAILLM):
                 "Using Azure AD token provider based on Service Principal with Secret workflow for Azure Auth"
             )
             try:
-                azure_ad_token_provider = get_azure_ad_token_provider()
+                azure_ad_token_provider = get_azure_ad_token_provider(azure_scope=scope)
             except ValueError:
                 verbose_logger.debug("Azure AD Token Provider could not be used.")
         if api_version is None:
@@ -442,8 +456,10 @@ class BaseAzureLLM(BaseOpenAILLM):
         ## build base url - assume api base includes resource name
         tenant_id = litellm_params.get("tenant_id", os.getenv("AZURE_TENANT_ID"))
         client_id = litellm_params.get("client_id", os.getenv("AZURE_CLIENT_ID"))
-        scope = litellm_params.get("azure_scope", os.getenv(
-            "AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"))
+        scope = litellm_params.get(
+            "azure_scope",
+            os.getenv("AZURE_SCOPE", "https://cognitiveservices.azure.com/.default"),
+        )
         if client is None:
             if not api_base.endswith("/"):
                 api_base += "/"

litellm/llms/azure_ai/chat/transformation.py

@@ -53,6 +53,10 @@ class AzureAIStudioConfig(OpenAIConfig):
         else:
             headers["Authorization"] = f"Bearer {api_key}"
 
+        headers["Content-Type"] = (
+            "application/json"  # tell Azure AI Studio to expect JSON
+        )
+
         return headers
 
     def _should_use_api_key_header(self, api_base: str) -> bool:

litellm/llms/base_llm/chat/transformation.py

@@ -97,6 +97,7 @@ class BaseConfig(ABC):
                     types.BuiltinFunctionType,
                     classmethod,
                     staticmethod,
+                    property,
                 ),
             )
             and v is not None

litellm/llms/bedrock/base_aws_llm.py

@@ -330,9 +330,19 @@ class BaseAWSLLM:
                 and isinstance(standard_aws_region_name, str)
             ):
                 aws_region_name = standard_aws_region_name
-
         if aws_region_name is None:
-            aws_region_name = "us-west-2"
+            try:
+                import boto3
+
+                with tracer.trace("boto3.Session()"):
+                    session = boto3.Session()
+                configured_region = session.region_name
+                if configured_region:
+                    aws_region_name = configured_region
+                else:
+                    aws_region_name = "us-west-2"
+            except Exception:
+                aws_region_name = "us-west-2"
 
         return aws_region_name
 

litellm/llms/bedrock/chat/invoke_transformations/anthropic_claude3_transformation.py

@@ -28,6 +28,10 @@ class AmazonAnthropicClaude3Config(AmazonInvokeConfig, AnthropicConfig):
 
     anthropic_version: str = "bedrock-2023-05-31"
 
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "bedrock"
+
     def get_supported_openai_params(self, model: str) -> List[str]:
         return AnthropicConfig.get_supported_openai_params(self, model)
 

litellm/llms/custom_httpx/http_handler.py

@@ -533,6 +533,39 @@ class AsyncHTTPHandler:
         verbose_logger.debug("Using AiohttpTransport...")
         return True
 
+    @staticmethod
+    def _get_ssl_connector_kwargs(
+        ssl_verify: Optional[bool] = None,
+        ssl_context: Optional[ssl.SSLContext] = None,
+    ) -> Dict[str, Any]:
+        """
+        Helper method to get SSL connector initialization arguments for aiohttp TCPConnector.
+        
+        SSL Configuration Priority:
+        1. If ssl_context is provided -> use the custom SSL context
+        2. If ssl_verify is False -> disable SSL verification (ssl=False)
+        3. If ssl_verify is True/None -> use default SSL context with certifi CA bundle
+
+        Returns:
+            Dict with appropriate SSL configuration for TCPConnector
+        """
+        connector_kwargs: Dict[str, Any] = {
+            "local_addr": ("0.0.0.0", 0) if litellm.force_ipv4 else None,
+        }
+        
+        if ssl_context is not None:
+            # Priority 1: Use the provided custom SSL context
+            connector_kwargs["ssl"] = ssl_context
+        elif ssl_verify is False:
+            # Priority 2: Explicitly disable SSL verification
+            connector_kwargs["verify_ssl"] = False
+        else:
+            # Priority 3: Use our default SSL context with certifi CA bundle
+            # This covers ssl_verify=True and ssl_verify=None cases
+            connector_kwargs["ssl"] = AsyncHTTPHandler._get_ssl_context()
+        
+        return connector_kwargs
+
     @staticmethod
     def _create_aiohttp_transport(
         ssl_verify: Optional[bool] = None,
@@ -541,29 +574,34 @@ class AsyncHTTPHandler:
         """
         Creates an AiohttpTransport with RequestNotRead error handling
 
-        - If force_ipv4 is True, it will create an AiohttpTransport with local_addr set to "0.0.0.0"
-        - [Default] If force_ipv4 is False, it will create an AiohttpTransport with default settings
+        Note: aiohttp TCPConnector ssl parameter accepts:
+        - SSLContext: custom SSL context
+        - False: disable SSL verification
+        - True: use default SSL verification (equivalent to ssl.create_default_context())
         """
         from litellm.llms.custom_httpx.aiohttp_transport import LiteLLMAiohttpTransport
 
-        #########################################################
-        # If ssl_verify is None, set it to True
-        # TCP Connector does not allow ssl_verify to be None
-        # by default aiohttp sets ssl_verify to True
-        #########################################################
-        if ssl_verify is None:
-            ssl_verify = True
+        connector_kwargs = AsyncHTTPHandler._get_ssl_connector_kwargs(
+            ssl_verify=ssl_verify, ssl_context=ssl_context
+        )
 
         verbose_logger.debug("Creating AiohttpTransport...")
         return LiteLLMAiohttpTransport(
             client=lambda: ClientSession(
-                connector=TCPConnector(
-                    verify_ssl=ssl_verify,
-                    ssl_context=ssl_context,
-                    local_addr=("0.0.0.0", 0) if litellm.force_ipv4 else None,
-                )
+                connector=TCPConnector(**connector_kwargs)
             ),
         )
+    
+
+    @staticmethod
+    def _get_ssl_context() -> ssl.SSLContext:
+        """
+        Get the SSL context for the AiohttpTransport
+        """
+        import certifi
+        return ssl.create_default_context(
+            cafile=certifi.where()
+        )
 
     @staticmethod
     def _create_httpx_transport() -> Optional[AsyncHTTPTransport]:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2447,7 +2447,10 @@ class BaseLLMHTTPHandler:
         _is_async: bool = False,
         fake_stream: bool = False,
         litellm_metadata: Optional[Dict[str, Any]] = None,
-    ) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
+    ) -> Union[
+        ImageResponse,
+        Coroutine[Any, Any, ImageResponse],
+    ]:
         """
 
         Handles image edit requests.

litellm/llms/gemini/chat/transformation.py

@@ -1,12 +1,11 @@
-from typing import Dict, List, Optional
+from typing import List, Optional
 
-import litellm
 from litellm.litellm_core_utils.prompt_templates.factory import (
     convert_generic_image_chunk_to_openai_image_obj,
     convert_to_anthropic_image_obj,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.llms.vertex_ai import ContentType, PartType, SpeechConfig, VoiceConfig, PrebuiltVoiceConfig
+from litellm.types.llms.vertex_ai import ContentType, PartType
 from litellm.utils import supports_reasoning
 
 from ...vertex_ai.gemini.transformation import _gemini_convert_messages_with_history
@@ -96,56 +95,6 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig):
             supported_params.append("audio")
         return supported_params
 
-    def map_openai_params(
-        self,
-        non_default_params: Dict,
-        optional_params: Dict,
-        model: str,
-        drop_params: bool,
-    ) -> Dict:
-        # Handle audio parameter for TTS models
-        if self.is_model_gemini_audio_model(model):
-            for param, value in non_default_params.items():
-                if param == "audio" and isinstance(value, dict):
-                    # Validate audio format - Gemini TTS only supports pcm16
-                    audio_format = value.get("format")
-                    if audio_format is not None and audio_format != "pcm16":
-                        raise ValueError(
-                            f"Unsupported audio format for Gemini TTS models: {audio_format}. "
-                            f"Gemini TTS models only support 'pcm16' format as they return audio data in L16 PCM format. "
-                            f"Please set audio format to 'pcm16'."
-                        )
-
-                    # Map OpenAI audio parameter to Gemini speech config
-                    speech_config: SpeechConfig = {}
-
-                    if "voice" in value:
-                        prebuilt_voice_config: PrebuiltVoiceConfig = {
-                            "voiceName": value["voice"]
-                        }
-                        voice_config: VoiceConfig = {
-                            "prebuiltVoiceConfig": prebuilt_voice_config
-                        }
-                        speech_config["voiceConfig"] = voice_config
-
-                    if speech_config:
-                        optional_params["speechConfig"] = speech_config
-
-                    # Ensure audio modality is set
-                    if "responseModalities" not in optional_params:
-                        optional_params["responseModalities"] = ["AUDIO"]
-                    elif "AUDIO" not in optional_params["responseModalities"]:
-                        optional_params["responseModalities"].append("AUDIO")
-
-        if litellm.vertex_ai_safety_settings is not None:
-            optional_params["safety_settings"] = litellm.vertex_ai_safety_settings
-        return super().map_openai_params(
-            model=model,
-            non_default_params=non_default_params,
-            optional_params=optional_params,
-            drop_params=drop_params,
-        )
-
     def _transform_messages(
         self, messages: List[AllMessageValues]
     ) -> List[ContentType]:

litellm/llms/litellm_proxy/chat/transformation.py

@@ -2,13 +2,16 @@
 Translate from OpenAI's `/v1/chat/completions` to VLLM's `/v1/chat/completions`
 """
 
-from typing import List, Optional, Tuple
+from typing import TYPE_CHECKING, List, Optional, Tuple
 
 from litellm.secret_managers.main import get_secret_bool, get_secret_str
 from litellm.types.router import LiteLLM_Params
 
 from ...openai.chat.gpt_transformation import OpenAIGPTConfig
 
+if TYPE_CHECKING:
+    from litellm.types.llms.openai import AllMessageValues
+
 
 class LiteLLMProxyChatConfig(OpenAIGPTConfig):
     def get_supported_openai_params(self, model: str) -> List:
@@ -113,3 +116,33 @@ class LiteLLMProxyChatConfig(OpenAIGPTConfig):
         )
 
         return model, custom_llm_provider, api_key, api_base
+
+    def transform_request(
+        self,
+        model: str,
+        messages: List["AllMessageValues"],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        # don't transform the request
+        return {
+            "model": model,
+            "messages": messages,
+            **optional_params,
+        }
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: List["AllMessageValues"],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        # don't transform the request
+        return {
+            "model": model,
+            "messages": messages,
+            **optional_params,
+        }

litellm/llms/meta_llama/chat/transformation.py

@@ -6,9 +6,11 @@ Calls done in OpenAI/openai.py as Llama API is openai-compatible.
 Docs: https://llama.developer.meta.com/docs/features/compatibility/
 """
 
-from typing import Optional
+import warnings
+
+# Suppress Pydantic serialization warnings for Meta Llama responses
+warnings.filterwarnings("ignore", message="Pydantic serializer warnings")
 
-from litellm import get_model_info, verbose_logger
 from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig
 
 
@@ -17,27 +19,11 @@ class LlamaAPIConfig(OpenAIGPTConfig):
         """
         Llama API has limited support for OpenAI parameters
 
-        Tool calling, Functional Calling, tool choice are not working right now
+        function_call, tools, and tool_choice are working
         response_format: only json_schema is working
         """
-        supports_function_calling: Optional[bool] = None
-        supports_tool_choice: Optional[bool] = None
-        try:
-            model_info = get_model_info(model, custom_llm_provider="meta_llama")
-            supports_function_calling = model_info.get(
-                "supports_function_calling", False
-            )
-            supports_tool_choice = model_info.get("supports_tool_choice", False)
-        except Exception as e:
-            verbose_logger.debug(f"Error getting supported openai params: {e}")
-            pass
-
+        # Function calling and tool choice are now supported on Llama API
         optional_params = super().get_supported_openai_params(model)
-        if not supports_function_calling:
-            optional_params.remove("function_call")
-        if not supports_tool_choice:
-            optional_params.remove("tools")
-            optional_params.remove("tool_choice")
         return optional_params
 
     def map_openai_params(

litellm/llms/mistral/mistral_chat_transformation.py

@@ -86,8 +86,9 @@ class MistralConfig(OpenAIGPTConfig):
             "seed",
             "stop",
             "response_format",
+            "parallel_tool_calls",
         ]
-        
+
         # Add reasoning support for magistral models
         if "magistral" in model.lower():
             supported_params.extend(["thinking", "reasoning_effort"])
@@ -154,6 +155,8 @@ Then provide a clear, concise answer based on your reasoning."""
             if param == "thinking" and "magistral" in model.lower():
                 # Flag that we need to add reasoning system prompt
                 optional_params["_add_reasoning_prompt"] = True
+            if param == "parallel_tool_calls":
+                optional_params["parallel_tool_calls"] = value
         return optional_params
 
     def _get_openai_compatible_provider_info(
@@ -287,12 +290,18 @@ Then provide a clear, concise answer based on your reasoning."""
         """
         Mistral API only supports `name` in tool messages
 
-        If role == tool, then we keep `name`
+        If role == tool, then we keep `name` if it's not an empty string
         Otherwise, we drop `name`
         """
         _name = message.get("name")  # type: ignore
-        if _name is not None and message["role"] != "tool":
-            message.pop("name", None)  # type: ignore
+        
+        if _name is not None:
+            # Remove name if not a tool message
+            if message["role"] != "tool":
+                message.pop("name", None)  # type: ignore
+            # For tool messages, remove name if it's an empty string
+            elif isinstance(_name, str) and len(_name.strip()) == 0:
+                message.pop("name", None)  # type: ignore
 
         return message
 

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -304,9 +304,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 return None
 
         for tool in value:
-            openai_function_object: Optional[
-                ChatCompletionToolParamFunctionChunk
-            ] = None
+            openai_function_object: Optional[ChatCompletionToolParamFunctionChunk] = (
+                None
+            )
             if "function" in tool:  # tools list
                 _openai_function_object = ChatCompletionToolParamFunctionChunk(  # type: ignore
                     **tool["function"]
@@ -489,7 +489,49 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 status_code=400,
             )
 
-    def map_openai_params(
+    def _map_audio_params(self, value: dict) -> dict:
+        """
+        Expected input:
+        {
+            "voice": "alloy",
+            "format": "mp3",
+        }
+
+        Expected output:
+        speechConfig = {
+            voiceConfig: {
+                prebuiltVoiceConfig: {
+                    voiceName: "alloy",
+                }
+            }
+        }
+        """
+        from litellm.types.llms.vertex_ai import (
+            PrebuiltVoiceConfig,
+            SpeechConfig,
+            VoiceConfig,
+        )
+
+        # Validate audio format - Gemini TTS only supports pcm16
+        audio_format = value.get("format")
+        if audio_format is not None and audio_format != "pcm16":
+            raise ValueError(
+                f"Unsupported audio format for Gemini TTS models: {audio_format}. "
+                f"Gemini TTS models only support 'pcm16' format as they return audio data in L16 PCM format. "
+                f"Please set audio format to 'pcm16'."
+            )
+
+        # Map OpenAI audio parameter to Gemini speech config
+        speech_config: SpeechConfig = {}
+
+        if "voice" in value:
+            prebuilt_voice_config: PrebuiltVoiceConfig = {"voiceName": value["voice"]}
+            voice_config: VoiceConfig = {"prebuiltVoiceConfig": prebuilt_voice_config}
+            speech_config["voiceConfig"] = voice_config
+
+        return cast(dict, speech_config)
+
+    def map_openai_params(  # noqa: PLR0915
         self,
         non_default_params: Dict,
         optional_params: Dict,
@@ -507,6 +549,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 optional_params["stream"] = value
             elif param == "n":
                 optional_params["candidate_count"] = value
+            elif param == "audio" and isinstance(value, dict):
+                optional_params["speechConfig"] = self._map_audio_params(value)
             elif param == "stop":
                 if isinstance(value, str):
                     optional_params["stop_sequences"] = [value]
@@ -552,14 +596,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             elif param == "seed":
                 optional_params["seed"] = value
             elif param == "reasoning_effort" and isinstance(value, str):
-                optional_params[
-                    "thinkingConfig"
-                ] = VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
+                optional_params["thinkingConfig"] = (
+                    VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(value)
+                )
             elif param == "thinking":
-                optional_params[
-                    "thinkingConfig"
-                ] = VertexGeminiConfig._map_thinking_param(
-                    cast(AnthropicThinkingParam, value)
+                optional_params["thinkingConfig"] = (
+                    VertexGeminiConfig._map_thinking_param(
+                        cast(AnthropicThinkingParam, value)
+                    )
                 )
             elif param == "modalities" and isinstance(value, list):
                 response_modalities = self.map_response_modalities(value)
@@ -571,6 +615,15 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 )
         if litellm.vertex_ai_safety_settings is not None:
             optional_params["safety_settings"] = litellm.vertex_ai_safety_settings
+
+        # if audio param is set, ensure responseModalities is set to AUDIO
+        audio_param = optional_params.get("speechConfig")
+        if audio_param is not None:
+            if "responseModalities" not in optional_params:
+                optional_params["responseModalities"] = ["AUDIO"]
+            elif "AUDIO" not in optional_params["responseModalities"]:
+                optional_params["responseModalities"].append("AUDIO")
+
         return optional_params
 
     def get_mapped_special_auth_params(self) -> dict:
@@ -972,9 +1025,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             response_tokens_details = CompletionTokensDetailsWrapper()
             for detail in usage_metadata["responseTokensDetails"]:
                 if detail["modality"] == "TEXT":
-                    response_tokens_details.text_tokens = detail["tokenCount"]
+                    response_tokens_details.text_tokens = detail.get("tokenCount", 0)
                 elif detail["modality"] == "AUDIO":
-                    response_tokens_details.audio_tokens = detail["tokenCount"]
+                    response_tokens_details.audio_tokens = detail.get("tokenCount", 0)
         #########################################################
 
         if "promptTokensDetails" in usage_metadata:
@@ -1263,28 +1316,28 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             ## ADD METADATA TO RESPONSE ##
 
             setattr(model_response, "vertex_ai_grounding_metadata", grounding_metadata)
-            model_response._hidden_params[
-                "vertex_ai_grounding_metadata"
-            ] = grounding_metadata
+            model_response._hidden_params["vertex_ai_grounding_metadata"] = (
+                grounding_metadata
+            )
 
             setattr(
                 model_response, "vertex_ai_url_context_metadata", url_context_metadata
             )
 
-            model_response._hidden_params[
-                "vertex_ai_url_context_metadata"
-            ] = url_context_metadata
+            model_response._hidden_params["vertex_ai_url_context_metadata"] = (
+                url_context_metadata
+            )
 
             setattr(model_response, "vertex_ai_safety_results", safety_ratings)
-            model_response._hidden_params[
-                "vertex_ai_safety_results"
-            ] = safety_ratings  # older approach - maintaining to prevent regressions
+            model_response._hidden_params["vertex_ai_safety_results"] = (
+                safety_ratings  # older approach - maintaining to prevent regressions
+            )
 
             ## ADD CITATION METADATA ##
             setattr(model_response, "vertex_ai_citation_metadata", citation_metadata)
-            model_response._hidden_params[
-                "vertex_ai_citation_metadata"
-            ] = citation_metadata  # older approach - maintaining to prevent regressions
+            model_response._hidden_params["vertex_ai_citation_metadata"] = (
+                citation_metadata  # older approach - maintaining to prevent regressions
+            )
 
         except Exception as e:
             raise VertexAIError(

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py

@@ -40,6 +40,14 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
                 or get_secret_str("VERTEXAI_CREDENTIALS")
             )
 
+            vertex_ai_location = (
+                litellm_params.pop("vertex_location", None)
+                or litellm_params.pop("vertex_ai_location", None)
+                or litellm.vertex_location
+                or get_secret_str("VERTEXAI_LOCATION")
+                or get_secret_str("VERTEX_LOCATION")
+            )
+
             access_token, project_id = self._ensure_access_token(
                 credentials=vertex_credentials,
                 project_id=vertex_ai_project,
@@ -50,7 +58,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
 
             api_base = self.get_complete_vertex_url(
                 custom_api_base=api_base,
-                vertex_location=litellm_params.pop("vertex_location", None),
+                vertex_location=vertex_ai_location,
                 vertex_project=vertex_ai_project,
                 project_id=project_id,
                 partner=VertexPartnerProvider.claude,

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py

@@ -47,6 +47,10 @@ class VertexAIAnthropicConfig(AnthropicConfig):
     Note: Please make sure to modify the default parameters as required for your use case.
     """
 
+    @property
+    def custom_llm_provider(self) -> Optional[str]:
+        return "vertex_ai"
+
     def transform_request(
         self,
         model: str,

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -79,7 +79,15 @@ class VertexBase:
 
             # Check if the JSON object contains Workload Identity Federation configuration
             if "type" in json_obj and json_obj["type"] == "external_account":
-                creds = self._credentials_from_identity_pool(json_obj)
+                # If environment_id key contains "aws" value it corresponds to an AWS config file
+                if (
+                    "credential_source" in json_obj
+                    and "environment_id" in json_obj["credential_source"]
+                    and "aws" in json_obj["credential_source"]["environment_id"]
+                ):
+                    creds = self._credentials_from_identity_pool_with_aws(json_obj)
+                else:
+                    creds = self._credentials_from_identity_pool(json_obj)
             # Check if the JSON object contains Authorized User configuration (via gcloud auth application-default login)
             elif "type" in json_obj and json_obj["type"] == "authorized_user":
                 creds = self._credentials_from_authorized_user(
@@ -122,6 +130,11 @@ class VertexBase:
         from google.auth import identity_pool
 
         return identity_pool.Credentials.from_info(json_obj)
+    
+    def _credentials_from_identity_pool_with_aws(self, json_obj):
+        from google.auth import aws
+
+        return aws.Credentials.from_info(json_obj)
 
     def _credentials_from_authorized_user(self, json_obj, scopes):
         import google.oauth2.credentials

litellm/main.py

@@ -2810,9 +2810,9 @@ def completion(  # type: ignore # noqa: PLR0915
                     "aws_region_name" not in optional_params
                     or optional_params["aws_region_name"] is None
                 ):
-                    optional_params[
-                        "aws_region_name"
-                    ] = aws_bedrock_client.meta.region_name
+                    optional_params["aws_region_name"] = (
+                        aws_bedrock_client.meta.region_name
+                    )
 
             bedrock_route = BedrockModelInfo.get_bedrock_route(model)
             if bedrock_route == "converse":
@@ -4589,9 +4589,9 @@ def adapter_completion(
     new_kwargs = translation_obj.translate_completion_input_params(kwargs=kwargs)
 
     response: Union[ModelResponse, CustomStreamWrapper] = completion(**new_kwargs)  # type: ignore
-    translated_response: Optional[
-        Union[BaseModel, AdapterCompletionStreamWrapper]
-    ] = None
+    translated_response: Optional[Union[BaseModel, AdapterCompletionStreamWrapper]] = (
+        None
+    )
     if isinstance(response, ModelResponse):
         translated_response = translation_obj.translate_completion_output_params(
             response=response
@@ -5192,6 +5192,21 @@ def speech(  # noqa: PLR0915
                 model=model,
                 llm_provider=custom_llm_provider,
             )
+        if "gemini" in model:
+            from .endpoints.speech.speech_to_completion_bridge.handler import (
+                speech_to_completion_bridge_handler,
+            )
+
+            return speech_to_completion_bridge_handler.speech(
+                model=model,
+                input=input,
+                voice=voice,
+                optional_params=optional_params,
+                litellm_params=litellm_params_dict,
+                headers=headers or {},
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+            )
         response = vertex_text_to_speech.audio_speech(
             _is_async=aspeech,
             vertex_credentials=vertex_credentials,
@@ -5206,6 +5221,21 @@ def speech(  # noqa: PLR0915
             kwargs=kwargs,
             logging_obj=logging_obj,
         )
+    elif custom_llm_provider == "gemini":
+        from .endpoints.speech.speech_to_completion_bridge.handler import (
+            speech_to_completion_bridge_handler,
+        )
+
+        return speech_to_completion_bridge_handler.speech(
+            model=model,
+            input=input,
+            voice=voice,
+            optional_params=optional_params,
+            litellm_params=litellm_params_dict,
+            headers=headers or {},
+            logging_obj=logging_obj,
+            custom_llm_provider=custom_llm_provider,
+        )
 
     if response is None:
         raise Exception(
@@ -5549,9 +5579,9 @@ def stream_chunk_builder(  # noqa: PLR0915
         ]
 
         if len(content_chunks) > 0:
-            response["choices"][0]["message"][
-                "content"
-            ] = processor.get_combined_content(content_chunks)
+            response["choices"][0]["message"]["content"] = (
+                processor.get_combined_content(content_chunks)
+            )
 
         reasoning_chunks = [
             chunk
@@ -5562,9 +5592,9 @@ def stream_chunk_builder(  # noqa: PLR0915
         ]
 
         if len(reasoning_chunks) > 0:
-            response["choices"][0]["message"][
-                "reasoning_content"
-            ] = processor.get_combined_reasoning_content(reasoning_chunks)
+            response["choices"][0]["message"]["reasoning_content"] = (
+                processor.get_combined_reasoning_content(reasoning_chunks)
+            )
 
         audio_chunks = [
             chunk

litellm/model_prices_and_context_window_backup.json

@@ -451,9 +451,9 @@
         "max_input_tokens": 128000,
         "max_output_tokens": 16384,
         "input_cost_per_token": 2.5e-06,
-        "input_cost_per_audio_token": 4.0e-5,
+        "input_cost_per_audio_token": 4e-05,
         "output_cost_per_token": 1e-05,
-        "output_cost_per_audio_token": 8.0e-5,
+        "output_cost_per_audio_token": 8e-05,
         "litellm_provider": "openai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -594,7 +594,7 @@
         "max_output_tokens": 100000,
         "input_cost_per_token": 1.5e-06,
         "output_cost_per_token": 6e-06,
-        "cache_read_input_token_cost": 0.375e-06,
+        "cache_read_input_token_cost": 3.75e-07,
         "litellm_provider": "openai",
         "mode": "responses",
         "supports_pdf_input": true,
@@ -744,10 +744,10 @@
         "max_tokens": 100000,
         "max_input_tokens": 200000,
         "max_output_tokens": 100000,
-        "input_cost_per_token": 20e-06,
-        "input_cost_per_token_batches": 10e-06,
-        "output_cost_per_token_batches": 40e-06,
-        "output_cost_per_token": 80e-06,
+        "input_cost_per_token": 2e-05,
+        "input_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_batches": 4e-05,
+        "output_cost_per_token": 8e-05,
         "litellm_provider": "openai",
         "mode": "responses",
         "supports_function_calling": true,
@@ -774,10 +774,10 @@
         "max_tokens": 100000,
         "max_input_tokens": 200000,
         "max_output_tokens": 100000,
-        "input_cost_per_token": 20e-06,
-        "input_cost_per_token_batches": 10e-06,
-        "output_cost_per_token_batches": 40e-06,
-        "output_cost_per_token": 80e-06,
+        "input_cost_per_token": 2e-05,
+        "input_cost_per_token_batches": 1e-05,
+        "output_cost_per_token_batches": 4e-05,
+        "output_cost_per_token": 8e-05,
         "litellm_provider": "openai",
         "mode": "responses",
         "supports_function_calling": true,
@@ -806,7 +806,7 @@
         "max_output_tokens": 100000,
         "input_cost_per_token": 2e-06,
         "output_cost_per_token": 8e-06,
-        "cache_read_input_token_cost": 0.5e-06,
+        "cache_read_input_token_cost": 5e-07,
         "litellm_provider": "openai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -837,7 +837,7 @@
         "max_output_tokens": 100000,
         "input_cost_per_token": 2e-06,
         "output_cost_per_token": 8e-06,
-        "cache_read_input_token_cost": 0.5e-06,
+        "cache_read_input_token_cost": 5e-07,
         "litellm_provider": "openai",
         "mode": "chat",
         "supports_function_calling": true,
@@ -2685,7 +2685,7 @@
         "max_output_tokens": 100000,
         "input_cost_per_token": 1.5e-06,
         "output_cost_per_token": 6e-06,
-        "cache_read_input_token_cost": 0.375e-06,
+        "cache_read_input_token_cost": 3.75e-07,
         "litellm_provider": "azure",
         "mode": "responses",
         "supports_pdf_input": true,
@@ -4295,8 +4295,8 @@
         "max_tokens": 40000,
         "max_input_tokens": 40000,
         "max_output_tokens": 40000,
-        "input_cost_per_token": 0.5e-6,
-        "output_cost_per_token": 1.5e-6,
+        "input_cost_per_token": 5e-07,
+        "output_cost_per_token": 1.5e-06,
         "litellm_provider": "mistral",
         "mode": "chat",
         "source": "https://mistral.ai/pricing#api-pricing",
@@ -4309,7 +4309,7 @@
         "max_tokens": 40000,
         "max_input_tokens": 40000,
         "max_output_tokens": 40000,
-        "input_cost_per_token": 0.5e-06,
+        "input_cost_per_token": 5e-07,
         "output_cost_per_token": 1.5e-06,
         "litellm_provider": "mistral",
         "mode": "chat",
@@ -4579,9 +4579,9 @@
         "output_cost_per_token": 4e-06,
         "litellm_provider": "xai",
         "mode": "chat",
+        "supports_reasoning": true,
         "supports_function_calling": true,
         "supports_tool_choice": true,
-        "supports_reasoning": true,
         "supports_response_schema": false,
         "source": "https://x.ai/api#pricing",
         "supports_web_search": true
@@ -4616,21 +4616,6 @@
         "source": "https://x.ai/api#pricing",
         "supports_web_search": true
     },
-    "xai/grok-3-mini-fast-latest": {
-        "max_tokens": 131072,
-        "max_input_tokens": 131072,
-        "max_output_tokens": 131072,
-        "input_cost_per_token": 6e-07,
-        "output_cost_per_token": 4e-06,
-        "litellm_provider": "xai",
-        "mode": "chat",
-        "supports_reasoning": true,
-        "supports_function_calling": true,
-        "supports_tool_choice": true,
-        "supports_response_schema": false,
-        "source": "https://x.ai/api#pricing",
-        "supports_web_search": true
-    },
     "xai/grok-vision-beta": {
         "max_tokens": 8192,
         "max_input_tokens": 8192,
@@ -5812,9 +5797,9 @@
         "max_output_tokens": 4028,
         "litellm_provider": "meta_llama",
         "mode": "chat",
-        "supports_function_calling": false,
+        "supports_function_calling": true,
         "source": "https://llama.developer.meta.com/docs/models",
-        "supports_tool_choice": false,
+        "supports_tool_choice": true,
         "supported_modalities": [
             "text",
             "image"
@@ -5829,9 +5814,9 @@
         "max_output_tokens": 4028,
         "litellm_provider": "meta_llama",
         "mode": "chat",
-        "supports_function_calling": false,
+        "supports_function_calling": true,
         "source": "https://llama.developer.meta.com/docs/models",
-        "supports_tool_choice": false,
+        "supports_tool_choice": true,
         "supported_modalities": [
             "text",
             "image"
@@ -5846,9 +5831,9 @@
         "max_output_tokens": 4028,
         "litellm_provider": "meta_llama",
         "mode": "chat",
-        "supports_function_calling": false,
+        "supports_function_calling": true,
         "source": "https://llama.developer.meta.com/docs/models",
-        "supports_tool_choice": false,
+        "supports_tool_choice": true,
         "supported_modalities": [
             "text"
         ],
@@ -5862,9 +5847,9 @@
         "max_output_tokens": 4028,
         "litellm_provider": "meta_llama",
         "mode": "chat",
-        "supports_function_calling": false,
+        "supports_function_calling": true,
         "source": "https://llama.developer.meta.com/docs/models",
-        "supports_tool_choice": false,
+        "supports_tool_choice": true,
         "supported_modalities": [
             "text"
         ],
@@ -6719,6 +6704,136 @@
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
         "supports_web_search": true
     },
+    "gemini/gemini-2.5-pro": {
+        "max_tokens": 65535,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token_above_200k_tokens": 2.5e-06,
+        "output_cost_per_token": 1e-05,
+        "output_cost_per_token_above_200k_tokens": 1.5e-05,
+        "litellm_provider": "gemini",
+        "mode": "chat",
+        "rpm": 2000,
+        "tpm": 800000,
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_audio_input": true,
+        "supports_video_input": true,
+        "supports_pdf_input": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing",
+        "supports_web_search": true
+    },
+    "gemini/gemini-2.5-flash": {
+        "max_tokens": 65535,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 1e-06,
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "litellm_provider": "gemini",
+        "mode": "chat",
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": false,
+        "supports_tool_choice": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
+        "supports_parallel_function_calling": true,
+        "supports_web_search": true,
+        "supports_url_context": true,
+        "tpm": 8000000,
+        "rpm": 100000,
+        "supports_pdf_input": true
+    },
+    "gemini-2.5-flash": {
+        "max_tokens": 65535,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 1e-06,
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_reasoning_token": 2.5e-06,
+        "litellm_provider": "vertex_ai-language-models",
+        "mode": "chat",
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": false,
+        "supports_tool_choice": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
+        "supports_parallel_function_calling": true,
+        "supports_web_search": true,
+        "supports_url_context": true,
+        "supports_pdf_input": true
+    },
     "gemini/gemini-2.5-flash-preview-tts": {
         "max_tokens": 65535,
         "max_input_tokens": 1048576,
@@ -6768,9 +6883,9 @@
         "max_audio_per_prompt": 1,
         "max_pdf_size_mb": 30,
         "input_cost_per_audio_token": 1e-06,
-        "input_cost_per_token": 1.5e-07,
-        "output_cost_per_token": 6e-07,
-        "output_cost_per_reasoning_token": 3.5e-06,
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_reasoning_token": 2.5e-06,
         "litellm_provider": "gemini",
         "mode": "chat",
         "rpm": 10,
@@ -6797,7 +6912,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_web_search": true,
-        "supports_url_context": true
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini/gemini-2.5-flash-preview-04-17": {
         "max_tokens": 65535,
@@ -6838,7 +6954,53 @@
             "text"
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
+    },
+    "gemini/gemini-2.5-flash-lite-preview-06-17": {
+        "max_tokens": 65535,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 5e-07,
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 4e-07,
+        "output_cost_per_reasoning_token": 4e-07,
+        "litellm_provider": "gemini",
+        "mode": "chat",
+        "rpm": 15,
+        "tpm": 250000,
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": false,
+        "supports_tool_choice": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-lite",
+        "supports_parallel_function_calling": true,
+        "supports_web_search": true,
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini-2.5-flash-preview-05-20": {
         "max_tokens": 65535,
@@ -6851,9 +7013,9 @@
         "max_audio_per_prompt": 1,
         "max_pdf_size_mb": 30,
         "input_cost_per_audio_token": 1e-06,
-        "input_cost_per_token": 1.5e-07,
-        "output_cost_per_token": 6e-07,
-        "output_cost_per_reasoning_token": 3.5e-06,
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 2.5e-06,
+        "output_cost_per_reasoning_token": 2.5e-06,
         "litellm_provider": "vertex_ai-language-models",
         "mode": "chat",
         "supports_reasoning": true,
@@ -6880,7 +7042,8 @@
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_parallel_function_calling": true,
         "supports_web_search": true,
-        "supports_url_context": true
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini-2.5-flash-preview-04-17": {
         "max_tokens": 65535,
@@ -6921,7 +7084,51 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_parallel_function_calling": true,
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
+    },
+    "gemini-2.5-flash-lite-preview-06-17": {
+        "max_tokens": 65535,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 65535,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 5e-07,
+        "input_cost_per_token": 1e-07,
+        "output_cost_per_token": 4e-07,
+        "output_cost_per_reasoning_token": 4e-07,
+        "litellm_provider": "vertex_ai-language-models",
+        "mode": "chat",
+        "supports_reasoning": true,
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": false,
+        "supports_tool_choice": true,
+        "supported_endpoints": [
+            "/v1/chat/completions",
+            "/v1/completions",
+            "/v1/batch"
+        ],
+        "supported_modalities": [
+            "text",
+            "image",
+            "audio",
+            "video"
+        ],
+        "supported_output_modalities": [
+            "text"
+        ],
+        "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
+        "supports_parallel_function_calling": true,
+        "supports_web_search": true,
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini-2.0-flash": {
         "max_tokens": 8192,
@@ -7067,7 +7274,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_parallel_function_calling": true,
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
     },
     "gemini-2.5-pro-preview-05-06": {
         "max_tokens": 65535,
@@ -7112,7 +7320,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_parallel_function_calling": true,
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
     },
     "gemini-2.5-pro-preview-03-25": {
         "max_tokens": 65535,
@@ -7154,7 +7363,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/models#gemini-2.5-flash-preview",
         "supports_parallel_function_calling": true,
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
     },
     "gemini-2.0-flash-preview-image-generation": {
         "max_tokens": 8192,
@@ -7479,7 +7689,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview",
         "supports_web_search": true,
-        "supports_url_context": true
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini/gemini-2.5-pro-preview-05-06": {
         "max_tokens": 65535,
@@ -7517,7 +7728,8 @@
         ],
         "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview",
         "supports_web_search": true,
-        "supports_url_context": true
+        "supports_url_context": true,
+        "supports_pdf_input": true
     },
     "gemini/gemini-2.5-pro-preview-03-25": {
         "max_tokens": 65535,
@@ -7554,7 +7766,8 @@
             "text"
         ],
         "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-2.5-pro-preview",
-        "supports_web_search": true
+        "supports_web_search": true,
+        "supports_pdf_input": true
     },
     "gemini/gemini-2.0-flash-exp": {
         "max_tokens": 8192,
@@ -8392,13 +8605,13 @@
         "litellm_provider": "vertex_ai-image-models",
         "mode": "image_generation",
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
-    },    
+    },
     "vertex_ai/imagen-4.0-fast-generate-preview-06-06": {
         "output_cost_per_image": 0.02,
         "litellm_provider": "vertex_ai-image-models",
         "mode": "image_generation",
         "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
-    },    
+    },
     "vertex_ai/imagen-3.0-generate-002": {
         "output_cost_per_image": 0.04,
         "litellm_provider": "vertex_ai-image-models",
@@ -9434,6 +9647,21 @@
         "mode": "chat",
         "supports_tool_choice": true
     },
+    "openrouter/deepseek/deepseek-r1-0528": {
+        "max_tokens": 8192,
+        "max_input_tokens": 65336,
+        "max_output_tokens": 8192,
+        "input_cost_per_token": 5e-07,
+        "input_cost_per_token_cache_hit": 1.4e-07,
+        "output_cost_per_token": 2.15e-06,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_assistant_prefill": true,
+        "supports_reasoning": true,
+        "supports_tool_choice": true,
+        "supports_prompt_caching": true
+    },
     "openrouter/deepseek/deepseek-r1": {
         "max_tokens": 8192,
         "max_input_tokens": 65336,
@@ -9479,6 +9707,28 @@
         "mode": "chat",
         "supports_tool_choice": true
     },
+    "openrouter/google/gemini-2.5-pro": {
+        "max_tokens": 8192,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 8192,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 7e-07,
+        "input_cost_per_token": 1.25e-06,
+        "output_cost_per_token": 1e-05,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": true,
+        "supports_tool_choice": true
+    },
     "openrouter/google/gemini-pro-1.5": {
         "max_tokens": 8192,
         "max_input_tokens": 1000000,
@@ -9514,6 +9764,28 @@
         "supports_audio_output": true,
         "supports_tool_choice": true
     },
+    "openrouter/google/gemini-2.5-flash": {
+        "max_tokens": 8192,
+        "max_input_tokens": 1048576,
+        "max_output_tokens": 8192,
+        "max_images_per_prompt": 3000,
+        "max_videos_per_prompt": 10,
+        "max_video_length": 1,
+        "max_audio_length_hours": 8.4,
+        "max_audio_per_prompt": 1,
+        "max_pdf_size_mb": 30,
+        "input_cost_per_audio_token": 7e-07,
+        "input_cost_per_token": 3e-07,
+        "output_cost_per_token": 2.5e-06,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_system_messages": true,
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_response_schema": true,
+        "supports_audio_output": true,
+        "supports_tool_choice": true
+    },
     "openrouter/mistralai/mixtral-8x22b-instruct": {
         "max_tokens": 65536,
         "input_cost_per_token": 6.5e-07,
@@ -9656,6 +9928,23 @@
         "supports_vision": true,
         "supports_tool_choice": true
     },
+    "openrouter/anthropic/claude-sonnet-4": {
+        "supports_computer_use": true,
+        "max_tokens": 8192,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 8192,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "input_cost_per_image": 0.0048,
+        "litellm_provider": "openrouter",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_reasoning": true,
+        "tool_use_system_prompt_tokens": 159,
+        "supports_assistant_prefill": true,
+        "supports_tool_choice": true
+    },
     "openrouter/mistralai/mistral-large": {
         "max_tokens": 32000,
         "input_cost_per_token": 8e-06,
@@ -10584,6 +10873,46 @@
         "supports_response_schema": true,
         "source": "https://aws.amazon.com/bedrock/pricing/"
     },
+    "apac.amazon.nova-micro-v1:0": {
+        "max_tokens": 10000,
+        "max_input_tokens": 300000,
+        "max_output_tokens": 10000,
+        "input_cost_per_token": 3.7e-08,
+        "output_cost_per_token": 1.48e-07,
+        "litellm_provider": "bedrock_converse",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true
+    },
+    "apac.amazon.nova-lite-v1:0": {
+        "max_tokens": 10000,
+        "max_input_tokens": 128000,
+        "max_output_tokens": 10000,
+        "input_cost_per_token": 6.3e-08,
+        "output_cost_per_token": 2.52e-07,
+        "litellm_provider": "bedrock_converse",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true
+    },
+    "apac.amazon.nova-pro-v1:0": {
+        "max_tokens": 10000,
+        "max_input_tokens": 300000,
+        "max_output_tokens": 10000,
+        "input_cost_per_token": 8.4e-07,
+        "output_cost_per_token": 3.36e-06,
+        "litellm_provider": "bedrock_converse",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true
+    },
     "us.amazon.nova-premier-v1:0": {
         "max_tokens": 10000,
         "max_input_tokens": 1000000,
@@ -11069,6 +11398,93 @@
         "supports_reasoning": true,
         "supports_computer_use": true
     },
+    "apac.anthropic.claude-3-haiku-20240307-v1:0": {
+        "max_tokens": 4096,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "input_cost_per_token": 2.5e-07,
+        "output_cost_per_token": 1.25e-06,
+        "litellm_provider": "bedrock",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_pdf_input": true,
+        "supports_tool_choice": true
+    },
+    "apac.anthropic.claude-3-sonnet-20240229-v1:0": {
+        "max_tokens": 4096,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "litellm_provider": "bedrock",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_pdf_input": true,
+        "supports_tool_choice": true
+    },
+    "apac.anthropic.claude-3-5-sonnet-20240620-v1:0": {
+        "max_tokens": 4096,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 4096,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "litellm_provider": "bedrock",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_response_schema": true,
+        "supports_vision": true,
+        "supports_pdf_input": true,
+        "supports_tool_choice": true
+    },
+    "apac.anthropic.claude-3-5-sonnet-20241022-v2:0": {
+        "max_tokens": 8192,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 8192,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "litellm_provider": "bedrock",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_assistant_prefill": true,
+        "supports_computer_use": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true
+    },
+    "apac.anthropic.claude-sonnet-4-20250514-v1:0": {
+        "max_tokens": 64000,
+        "max_input_tokens": 200000,
+        "max_output_tokens": 64000,
+        "input_cost_per_token": 3e-06,
+        "output_cost_per_token": 1.5e-05,
+        "search_context_cost_per_query": {
+            "search_context_size_low": 0.01,
+            "search_context_size_medium": 0.01,
+            "search_context_size_high": 0.01
+        },
+        "cache_creation_input_token_cost": 3.75e-06,
+        "cache_read_input_token_cost": 3e-07,
+        "litellm_provider": "bedrock_converse",
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "tool_use_system_prompt_tokens": 159,
+        "supports_assistant_prefill": true,
+        "supports_pdf_input": true,
+        "supports_prompt_caching": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_reasoning": true,
+        "supports_computer_use": true
+    },
     "eu.anthropic.claude-3-5-haiku-20241022-v1:0": {
         "max_tokens": 8192,
         "max_input_tokens": 200000,
@@ -14524,7 +14940,7 @@
     },
     "deepgram/nova-3": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14538,7 +14954,7 @@
     },
     "deepgram/nova-3-general": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14552,7 +14968,7 @@
     },
     "deepgram/nova-3-medical": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00008667,
+        "input_cost_per_second": 8.667e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14566,7 +14982,7 @@
     },
     "deepgram/nova-2": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14580,7 +14996,7 @@
     },
     "deepgram/nova-2-general": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14594,7 +15010,7 @@
     },
     "deepgram/nova-2-meeting": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14608,7 +15024,7 @@
     },
     "deepgram/nova-2-phonecall": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14622,7 +15038,7 @@
     },
     "deepgram/nova-2-voicemail": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14636,7 +15052,7 @@
     },
     "deepgram/nova-2-finance": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14650,7 +15066,7 @@
     },
     "deepgram/nova-2-conversationalai": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14664,7 +15080,7 @@
     },
     "deepgram/nova-2-video": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14678,7 +15094,7 @@
     },
     "deepgram/nova-2-drivethru": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14692,7 +15108,7 @@
     },
     "deepgram/nova-2-automotive": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14706,7 +15122,7 @@
     },
     "deepgram/nova-2-atc": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14720,7 +15136,7 @@
     },
     "deepgram/nova": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14734,7 +15150,7 @@
     },
     "deepgram/nova-general": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -14748,7 +15164,7 @@
     },
     "deepgram/nova-phonecall": {
         "mode": "audio_transcription",
-        "input_cost_per_second": 0.00007167,
+        "input_cost_per_second": 7.167e-05,
         "output_cost_per_second": 0.0,
         "litellm_provider": "deepgram",
         "supported_endpoints": [
@@ -15020,4 +15436,4 @@
             "notes": "Deepgram's hosted OpenAI Whisper models - pricing may differ from native Deepgram models"
         }
     }
-}
+}

litellm/proxy/_experimental/mcp_server/auth/litellm_auth_handler.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from mcp.server.auth.middleware.bearer_auth import AuthenticatedUser
 
 from litellm.proxy._types import UserAPIKeyAuth
@@ -8,5 +10,6 @@ class LiteLLMAuthenticatedUser(AuthenticatedUser):
     Wrapper class to make UserAPIKeyAuth compatible with MCP's AuthenticatedUser
     """
 
-    def __init__(self, user_api_key_auth: UserAPIKeyAuth):
+    def __init__(self, user_api_key_auth: UserAPIKeyAuth, mcp_auth_header: Optional[str] = None):
         self.user_api_key_auth = user_api_key_auth
+        self.mcp_auth_header = mcp_auth_header

litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py

@@ -1,11 +1,11 @@
-from typing import List, Optional
+from typing import List, Optional, Tuple
 
 from starlette.datastructures import Headers
 from starlette.requests import Request
 from starlette.types import Scope
 
 from litellm._logging import verbose_logger
-from litellm.proxy._types import LiteLLM_TeamTableCachedObj, UserAPIKeyAuth
+from litellm.proxy._types import LiteLLM_TeamTable, SpecialHeaders, UserAPIKeyAuth
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
 
 
@@ -16,11 +16,14 @@ class UserAPIKeyAuthMCP:
     Utilizes the main `user_api_key_auth` function to validate the request
     """
 
-    LITELLM_API_KEY_HEADER_NAME_PRIMARY = "x-litellm-api-key"
-    LITELLM_API_KEY_HEADER_NAME_SECONDARY = "Authorization"
+    LITELLM_API_KEY_HEADER_NAME_PRIMARY = SpecialHeaders.custom_litellm_api_key.value
+    LITELLM_API_KEY_HEADER_NAME_SECONDARY = SpecialHeaders.openai_authorization.value
+
+    # This is the header to use if you want LiteLLM to use this header for authenticating to the MCP server
+    LITELLM_MCP_AUTH_HEADER_NAME = SpecialHeaders.mcp_auth.value
 
     @staticmethod
-    async def user_api_key_auth_mcp(scope: Scope) -> UserAPIKeyAuth:
+    async def user_api_key_auth_mcp(scope: Scope) -> Tuple[UserAPIKeyAuth, Optional[str]]:
         """
         Validate and extract headers from the ASGI scope for MCP requests.
 
@@ -29,6 +32,7 @@ class UserAPIKeyAuthMCP:
 
         Returns:
             UserAPIKeyAuth containing validated authentication information
+            mcp_auth_header: Optional[str] MCP auth header to be passed to the MCP server
 
         Raises:
             HTTPException: If headers are invalid or missing required headers
@@ -37,6 +41,7 @@ class UserAPIKeyAuthMCP:
         litellm_api_key = (
             UserAPIKeyAuthMCP.get_litellm_api_key_from_headers(headers) or ""
         )
+        mcp_auth_header = headers.get(UserAPIKeyAuthMCP.LITELLM_MCP_AUTH_HEADER_NAME)
 
         # Create a proper Request object with mock body method to avoid ASGI receive channel issues
         request = Request(scope=scope)
@@ -52,7 +57,7 @@ class UserAPIKeyAuthMCP:
             api_key=litellm_api_key, request=request
         )
 
-        return validated_user_api_key_auth
+        return validated_user_api_key_auth, mcp_auth_header
 
     @staticmethod
     def get_litellm_api_key_from_headers(headers: Headers) -> Optional[str]:
@@ -166,12 +171,7 @@ class UserAPIKeyAuthMCP:
         first we check if the team has a object_permission_id attached
             - if it does then we look up the object_permission for the team
         """
-        from litellm.proxy.auth.auth_checks import get_team_object
-        from litellm.proxy.proxy_server import (
-            prisma_client,
-            proxy_logging_obj,
-            user_api_key_cache,
-        )
+        from litellm.proxy.proxy_server import prisma_client
 
         if user_api_key_auth is None:
             return []
@@ -179,18 +179,19 @@ class UserAPIKeyAuthMCP:
         if user_api_key_auth.team_id is None:
             return []
 
-        team_obj: LiteLLM_TeamTableCachedObj = await get_team_object(
-            team_id=user_api_key_auth.team_id,
-            prisma_client=prisma_client,
-            user_api_key_cache=user_api_key_cache,
-            parent_otel_span=None,
-            proxy_logging_obj=proxy_logging_obj,
-            check_cache_only=True,
-        )
         if prisma_client is None:
             verbose_logger.debug("prisma_client is None")
             return []
 
+        team_obj: Optional[LiteLLM_TeamTable] = (
+            await prisma_client.db.litellm_teamtable.find_unique(
+                where={"team_id": user_api_key_auth.team_id},
+            )
+        )
+        if team_obj is None:
+            verbose_logger.debug("team_obj is None")
+            return []
+
         object_permissions = team_obj.object_permission
         if object_permissions is None:
             return []

litellm/proxy/_experimental/mcp_server/mcp_server_manager.py

@@ -7,16 +7,16 @@ This is a Proxy
 """
 
 import asyncio
+import hashlib
 import json
-import uuid
 from typing import Any, Dict, List, Optional, cast
 
-from mcp import ClientSession
-from mcp.client.sse import sse_client
+from mcp.types import CallToolRequestParams as MCPCallToolRequestParams
 from mcp.types import CallToolResult
 from mcp.types import Tool as MCPTool
 
 from litellm._logging import verbose_logger
+from litellm.experimental_mcp_client.client import MCPClient
 from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import (
     UserAPIKeyAuthMCP,
 )
@@ -29,12 +29,6 @@ from litellm.proxy._types import (
     MCPTransportType,
     UserAPIKeyAuth,
 )
-
-try:
-    from mcp.client.streamable_http import streamablehttp_client
-except ImportError:
-    streamablehttp_client = None  # type: ignore
-
 from litellm.types.mcp_server.mcp_server_manager import MCPInfo, MCPServer
 
 
@@ -82,13 +76,22 @@ class MCPServerManager:
             mcp_info = MCPInfo(**_mcp_info)
             mcp_info["server_name"] = server_name
             mcp_info["description"] = server_config.get("description", None)
-            server_id = str(uuid.uuid4())
+
+            # Generate stable server ID based on parameters
+            server_id = self._generate_stable_server_id(
+                server_name=server_name,
+                url=server_config["url"],
+                transport=server_config.get("transport", MCPTransport.http),
+                spec_version=server_config.get("spec_version", MCPSpecVersion.mar_2025),
+                auth_type=server_config.get("auth_type", None),
+            )
+
             new_server = MCPServer(
                 server_id=server_id,
                 name=server_name,
                 url=server_config["url"],
                 # TODO: utility fn the default values
-                transport=server_config.get("transport", MCPTransport.sse),
+                transport=server_config.get("transport", MCPTransport.http),
                 spec_version=server_config.get("spec_version", MCPSpecVersion.mar_2025),
                 auth_type=server_config.get("auth_type", None),
                 mcp_info=mcp_info,
@@ -155,7 +158,9 @@ class MCPServerManager:
             return list(self.get_registry().keys())
 
     async def list_tools(
-        self, user_api_key_auth: Optional[UserAPIKeyAuth] = None
+        self, 
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        mcp_auth_header: Optional[str] = None,
     ) -> List[MCPTool]:
         """
         List all tools available across all MCP Servers.
@@ -174,7 +179,10 @@ class MCPServerManager:
                 verbose_logger.warning(f"MCP Server {server_id} not found")
                 continue
             try:
-                tools = await self._get_tools_from_server(server)
+                tools = await self._get_tools_from_server(
+                    server=server,
+                    mcp_auth_header=mcp_auth_header,
+                )
                 list_tools_result.extend(tools)
             except Exception as e:
                 verbose_logger.exception(
@@ -183,7 +191,30 @@ class MCPServerManager:
 
         return list_tools_result
 
-    async def _get_tools_from_server(self, server: MCPServer) -> List[MCPTool]:
+    #########################################################
+    # Methods that call the upstream MCP servers
+    #########################################################
+    def _create_mcp_client(self, server: MCPServer, mcp_auth_header: Optional[str] = None) -> MCPClient:
+        """
+        Create an MCPClient instance for the given server.
+
+        Args:
+            server (MCPServer): The server configuration
+            mcp_auth_header: MCP auth header to be passed to the MCP server. This is optional and will be used if provided.
+
+        Returns:
+            MCPClient: Configured MCP client instance
+        """
+        transport = server.transport or MCPTransport.sse
+        return MCPClient(
+            server_url=server.url,
+            transport_type=transport,
+            auth_type=server.auth_type,
+            auth_value=mcp_auth_header or server.authentication_token,
+            timeout=60.0,
+        )
+
+    async def _get_tools_from_server(self, server: MCPServer, mcp_auth_header: Optional[str] = None) -> List[MCPTool]:
         """
         Helper method to get tools from a single MCP server.
 
@@ -194,57 +225,51 @@ class MCPServerManager:
             List[MCPTool]: List of tools available on the server
         """
         verbose_logger.debug(f"Connecting to url: {server.url}")
-
         verbose_logger.info("_get_tools_from_server...")
-        # send transport to connect to the server
-        if server.transport is None or server.transport == MCPTransport.sse:
-            async with sse_client(url=server.url) as (read, write):
-                async with ClientSession(read, write) as session:
-                    await session.initialize()
 
-                    tools_result = await session.list_tools()
-                    verbose_logger.debug(f"Tools from {server.name}: {tools_result}")
+        client = self._create_mcp_client(
+            server=server,
+            mcp_auth_header=mcp_auth_header,
+        )
+        async with client:
+            tools = await client.list_tools()
+            verbose_logger.debug(f"Tools from {server.name}: {tools}")
 
-                    # Update tool to server mapping
-                    for tool in tools_result.tools:
-                        self.tool_name_to_mcp_server_name_mapping[tool.name] = (
-                            server.name
-                        )
+            # Update tool to server mapping
+            for tool in tools:
+                self.tool_name_to_mcp_server_name_mapping[tool.name] = server.name
 
-                    return tools_result.tools
-        elif server.transport == MCPTransport.http:
-            if streamablehttp_client is None:
-                verbose_logger.error(
-                    "streamablehttp_client not available - install mcp with HTTP support"
-                )
-                raise ValueError(
-                    "streamablehttp_client not available - please run `pip install mcp -U`"
-                )
-            verbose_logger.debug(f"Using HTTP streamable transport for {server.url}")
-            async with streamablehttp_client(
-                url=server.url,
-            ) as (read_stream, write_stream, get_session_id):
-                async with ClientSession(read_stream, write_stream) as session:
-                    await session.initialize()
+            return tools
+    
+    async def call_tool(
+        self, 
+        name: str, 
+        arguments: Dict[str, Any],
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        mcp_auth_header: Optional[str] = None,
+    ) -> CallToolResult:
+        """
+        Call a tool with the given name and arguments
+        """
+        mcp_server = self._get_mcp_server_from_tool_name(name)
+        if mcp_server is None:
+            raise ValueError(f"Tool {name} not found")
 
-                    if get_session_id is not None:
-                        session_id = get_session_id()
-                        if session_id:
-                            verbose_logger.debug(f"HTTP session ID: {session_id}")
+        client = self._create_mcp_client(
+            server=mcp_server,
+            mcp_auth_header=mcp_auth_header,
+        )
+        async with client:
+            call_tool_params = MCPCallToolRequestParams(
+                name=name,
+                arguments=arguments,
+            )
+            return await client.call_tool(call_tool_params)
+    
+    #########################################################
+    # End of Methods that call the upstream MCP servers
+    #########################################################
 
-                    tools_result = await session.list_tools()
-                    verbose_logger.debug(f"Tools from {server.name}: {tools_result}")
-
-                    # Update tool to server mapping
-                    for tool in tools_result.tools:
-                        self.tool_name_to_mcp_server_name_mapping[tool.name] = (
-                            server.name
-                        )
-
-                    return tools_result.tools
-        else:
-            verbose_logger.warning(f"Unsupported transport type: {server.transport}")
-            return []
 
     def initialize_tool_name_to_mcp_server_name_mapping(self):
         """
@@ -269,46 +294,6 @@ class MCPServerManager:
             for tool in tools:
                 self.tool_name_to_mcp_server_name_mapping[tool.name] = server.name
 
-    async def call_tool(self, name: str, arguments: Dict[str, Any]):
-        """
-        Call a tool with the given name and arguments
-        """
-        mcp_server = self._get_mcp_server_from_tool_name(name)
-        if mcp_server is None:
-            raise ValueError(f"Tool {name} not found")
-        elif mcp_server.transport is None or mcp_server.transport == MCPTransport.sse:
-            async with sse_client(url=mcp_server.url) as (read, write):
-                async with ClientSession(read, write) as session:
-                    await session.initialize()
-                    return await session.call_tool(name, arguments)
-        elif mcp_server.transport == MCPTransport.http:
-            if streamablehttp_client is None:
-                verbose_logger.error(
-                    "streamablehttp_client not available - install mcp with HTTP support"
-                )
-                raise ValueError(
-                    "streamablehttp_client not available - please run `pip install mcp -U`"
-                )
-            verbose_logger.debug(
-                f"Using HTTP streamable transport for tool call: {name}"
-            )
-            async with streamablehttp_client(
-                url=mcp_server.url,
-            ) as (read_stream, write_stream, get_session_id):
-                async with ClientSession(read_stream, write_stream) as session:
-                    await session.initialize()
-
-                    if get_session_id is not None:
-                        session_id = get_session_id()
-                        if session_id:
-                            verbose_logger.debug(
-                                f"HTTP session ID for tool call: {session_id}"
-                            )
-
-                    return await session.call_tool(name, arguments)
-        else:
-            return CallToolResult(content=[], isError=True)
-
     def _get_mcp_server_from_tool_name(self, tool_name: str) -> Optional[MCPServer]:
         """
         Get the MCP Server from the tool name
@@ -343,5 +328,43 @@ class MCPServerManager:
                 return server
         return None
 
+    def _generate_stable_server_id(
+        self,
+        server_name: str,
+        url: str,
+        transport: str,
+        spec_version: str,
+        auth_type: Optional[str] = None,
+    ) -> str:
+        """
+        Generate a stable server ID based on server parameters using a hash function.
+
+        This is critical to ensure the server_id is stable across server restarts.
+        Some users store MCPs on the config.yaml and permission management is based on server_ids.
+
+        Eg a key might have mcp_servers = ["1234"], if the server_id changes across restarts, the key will no longer have access to the MCP.
+
+        Args:
+            server_name: Name of the server
+            url: Server URL
+            transport: Transport type (sse, http, etc.)
+            spec_version: MCP spec version
+            auth_type: Authentication type (optional)
+
+        Returns:
+            A deterministic server ID string
+        """
+        # Create a string from all the identifying parameters
+        params_string = (
+            f"{server_name}|{url}|{transport}|{spec_version}|{auth_type or ''}"
+        )
+
+        # Generate SHA-256 hash
+        hash_object = hashlib.sha256(params_string.encode("utf-8"))
+        hash_hex = hash_object.hexdigest()
+
+        # Take first 32 characters and format as UUID-like string
+        return hash_hex[:32]
+
 
 global_mcp_server_manager: MCPServerManager = MCPServerManager()

litellm/proxy/_experimental/mcp_server/rest_endpoints.py

@@ -70,7 +70,9 @@ if MCP_AVAILABLE:
             if server_id and server.server_id != server_id:
                 continue
             try:
-                tools = await global_mcp_server_manager._get_tools_from_server(server)
+                tools = await global_mcp_server_manager._get_tools_from_server(
+                    server=server,
+                )
                 for tool in tools:
                     list_tools_result.append(
                         ListMCPToolsRestAPIResponseObject(

litellm/proxy/_experimental/mcp_server/server.py

@@ -4,7 +4,7 @@ LiteLLM MCP Server Routes
 
 import asyncio
 import contextlib
-from typing import Any, AsyncIterator, Dict, List, Optional, Union
+from typing import Any, AsyncIterator, Dict, List, Optional, Tuple, Union
 
 from fastapi import FastAPI, HTTPException
 from pydantic import ConfigDict
@@ -166,11 +166,14 @@ if MCP_AVAILABLE:
         List all available tools
         """
         # Get user authentication from context variable
-        user_api_key_auth = get_auth_context()
+        user_api_key_auth, mcp_auth_header = get_auth_context()
         verbose_logger.debug(
             f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}"
         )
-        return await _list_mcp_tools(user_api_key_auth)
+        return await _list_mcp_tools(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+        )
 
     @server.call_tool()
     async def mcp_server_tool_call(
@@ -190,9 +193,15 @@ if MCP_AVAILABLE:
             HTTPException: If tool not found or arguments missing
         """
         # Validate arguments
+        user_api_key_auth, mcp_auth_header = get_auth_context()
+        verbose_logger.debug(
+            f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}"
+        )
         response = await call_mcp_tool(
             name=name,
             arguments=arguments,
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
         )
         return response
 
@@ -206,6 +215,7 @@ if MCP_AVAILABLE:
 
     async def _list_mcp_tools(
         user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        mcp_auth_header: Optional[str] = None,
     ) -> List[MCPTool]:
         """
         List all available tools
@@ -229,6 +239,7 @@ if MCP_AVAILABLE:
         tools_from_mcp_servers: List[MCPTool] = (
             await global_mcp_server_manager.list_tools(
                 user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=mcp_auth_header,
             )
         )
         verbose_logger.debug("TOOLS FROM MCP SERVERS: %s", tools_from_mcp_servers)
@@ -238,7 +249,11 @@ if MCP_AVAILABLE:
 
     @client
     async def call_mcp_tool(
-        name: str, arguments: Optional[Dict[str, Any]] = None, **kwargs: Any
+        name: str, 
+        arguments: Optional[Dict[str, Any]] = None, 
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        mcp_auth_header: Optional[str] = None, 
+        **kwargs: Any
     ) -> List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]:
         """
         Call a specific tool with the provided arguments
@@ -270,7 +285,12 @@ if MCP_AVAILABLE:
 
         # Try managed server tool first
         if name in global_mcp_server_manager.tool_name_to_mcp_server_name_mapping:
-            return await _handle_managed_mcp_tool(name, arguments)
+            return await _handle_managed_mcp_tool(
+                name=name,
+                arguments=arguments,
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=mcp_auth_header,
+            )
 
         # Fall back to local tool registry
         return await _handle_local_mcp_tool(name, arguments)
@@ -295,12 +315,17 @@ if MCP_AVAILABLE:
             )
 
     async def _handle_managed_mcp_tool(
-        name: str, arguments: Dict[str, Any]
+        name: str, 
+        arguments: Dict[str, Any],
+        user_api_key_auth: Optional[UserAPIKeyAuth] = None,
+        mcp_auth_header: Optional[str] = None,
     ) -> List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]:
         """Handle tool execution for managed server tools"""
         call_tool_result = await global_mcp_server_manager.call_tool(
             name=name,
             arguments=arguments,
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
         )
         verbose_logger.debug("CALL TOOL RESULT: %s", call_tool_result)
         return call_tool_result.content
@@ -325,11 +350,14 @@ if MCP_AVAILABLE:
         """Handle MCP requests through StreamableHTTP."""
         try:
             # Validate headers and log request info
-            user_api_key_auth: UserAPIKeyAuth = (
+            user_api_key_auth, mcp_auth_header = (
                 await UserAPIKeyAuthMCP.user_api_key_auth_mcp(scope)
             )
             # Set the auth context variable for easy access in MCP functions
-            set_auth_context(user_api_key_auth)
+            set_auth_context(
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=mcp_auth_header,
+            )
 
             # Ensure session managers are initialized
             if not _SESSION_MANAGERS_INITIALIZED:
@@ -346,11 +374,14 @@ if MCP_AVAILABLE:
         """Handle MCP requests through SSE."""
         try:
             # Validate headers and log request info
-            user_api_key_auth: UserAPIKeyAuth = (
+            user_api_key_auth, mcp_auth_header = (
                 await UserAPIKeyAuthMCP.user_api_key_auth_mcp(scope)
             )
             # Set the auth context variable for easy access in MCP functions
-            set_auth_context(user_api_key_auth)
+            set_auth_context(
+                user_api_key_auth=user_api_key_auth,
+                mcp_auth_header=mcp_auth_header,
+            )
 
             # Ensure session managers are initialized
             if not _SESSION_MANAGERS_INITIALIZED:
@@ -390,17 +421,31 @@ if MCP_AVAILABLE:
     ############ Auth Context Functions ####################
     ########################################################
 
-    def set_auth_context(user_api_key_auth: UserAPIKeyAuth) -> None:
-        """Set the UserAPIKeyAuth in the auth context variable."""
-        auth_user = LiteLLMAuthenticatedUser(user_api_key_auth)
+    def set_auth_context(user_api_key_auth: UserAPIKeyAuth, mcp_auth_header: Optional[str] = None) -> None:
+        """
+        Set the UserAPIKeyAuth in the auth context variable.
+
+        Args:
+            user_api_key_auth: UserAPIKeyAuth object
+            mcp_auth_header: MCP auth header to be passed to the MCP server
+        """
+        auth_user = LiteLLMAuthenticatedUser(
+            user_api_key_auth=user_api_key_auth,
+            mcp_auth_header=mcp_auth_header,
+        )
         auth_context_var.set(auth_user)
 
-    def get_auth_context() -> Optional[UserAPIKeyAuth]:
-        """Get the UserAPIKeyAuth from the auth context variable."""
+    def get_auth_context() -> Tuple[Optional[UserAPIKeyAuth], Optional[str]]:
+        """
+        Get the UserAPIKeyAuth from the auth context variable.
+
+        Returns:
+            Tuple[Optional[UserAPIKeyAuth], Optional[str]]: UserAPIKeyAuth object and MCP auth header
+        """
         auth_user = auth_context_var.get()
         if auth_user and isinstance(auth_user, LiteLLMAuthenticatedUser):
-            return auth_user.user_api_key_auth
-        return None
+            return auth_user.user_api_key_auth, auth_user.mcp_auth_header
+        return None, None
 
     ########################################################
     ############ End of Auth Context Functions #############

litellm/proxy/_experimental/out/_next/static/chunks/webpack-a426aae3231a8df1.js

@@ -1 +1 @@
-!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e].call(n.exports,n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/_next/",i={272:0,919:0,986:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(/^(272|919|986)$/.test(e))i[e]=0;else{var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();
+!function(){"use strict";var e,t,n,r,o,u,i,c,f,a={},l={};function d(e){var t=l[e];if(void 0!==t)return t.exports;var n=l[e]={id:e,loaded:!1,exports:{}},r=!0;try{a[e].call(n.exports,n,n.exports,d),r=!1}finally{r&&delete l[e]}return n.loaded=!0,n.exports}d.m=a,e=[],d.O=function(t,n,r,o){if(n){o=o||0;for(var u=e.length;u>0&&e[u-1][2]>o;u--)e[u]=e[u-1];e[u]=[n,r,o];return}for(var i=1/0,u=0;u<e.length;u++){for(var n=e[u][0],r=e[u][1],o=e[u][2],c=!0,f=0;f<n.length;f++)i>=o&&Object.keys(d.O).every(function(e){return d.O[e](n[f])})?n.splice(f--,1):(c=!1,o<i&&(i=o));if(c){e.splice(u--,1);var a=r();void 0!==a&&(t=a)}}return t},d.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return d.d(t,{a:t}),t},n=Object.getPrototypeOf?function(e){return Object.getPrototypeOf(e)}:function(e){return e.__proto__},d.t=function(e,r){if(1&r&&(e=this(e)),8&r||"object"==typeof e&&e&&(4&r&&e.__esModule||16&r&&"function"==typeof e.then))return e;var o=Object.create(null);d.r(o);var u={};t=t||[null,n({}),n([]),n(n)];for(var i=2&r&&e;"object"==typeof i&&!~t.indexOf(i);i=n(i))Object.getOwnPropertyNames(i).forEach(function(t){u[t]=function(){return e[t]}});return u.default=function(){return e},d.d(o,u),o},d.d=function(e,t){for(var n in t)d.o(t,n)&&!d.o(e,n)&&Object.defineProperty(e,n,{enumerable:!0,get:t[n]})},d.f={},d.e=function(e){return Promise.all(Object.keys(d.f).reduce(function(t,n){return d.f[n](e,t),t},[]))},d.u=function(e){},d.miniCssF=function(e){},d.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||Function("return this")()}catch(e){if("object"==typeof window)return window}}(),d.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)},r={},o="_N_E:",d.l=function(e,t,n,u){if(r[e]){r[e].push(t);return}if(void 0!==n)for(var i,c,f=document.getElementsByTagName("script"),a=0;a<f.length;a++){var l=f[a];if(l.getAttribute("src")==e||l.getAttribute("data-webpack")==o+n){i=l;break}}i||(c=!0,(i=document.createElement("script")).charset="utf-8",i.timeout=120,d.nc&&i.setAttribute("nonce",d.nc),i.setAttribute("data-webpack",o+n),i.src=d.tu(e)),r[e]=[t];var s=function(t,n){i.onerror=i.onload=null,clearTimeout(p);var o=r[e];if(delete r[e],i.parentNode&&i.parentNode.removeChild(i),o&&o.forEach(function(e){return e(n)}),t)return t(n)},p=setTimeout(s.bind(null,void 0,{type:"timeout",target:i}),12e4);i.onerror=s.bind(null,i.onerror),i.onload=s.bind(null,i.onload),c&&document.head.appendChild(i)},d.r=function(e){"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},d.nmd=function(e){return e.paths=[],e.children||(e.children=[]),e},d.tt=function(){return void 0===u&&(u={createScriptURL:function(e){return e}},"undefined"!=typeof trustedTypes&&trustedTypes.createPolicy&&(u=trustedTypes.createPolicy("nextjs#bundler",u))),u},d.tu=function(e){return d.tt().createScriptURL(e)},d.p="/litellm-asset-prefix/_next/",i={272:0,919:0,986:0},d.f.j=function(e,t){var n=d.o(i,e)?i[e]:void 0;if(0!==n){if(n)t.push(n[2]);else if(/^(272|919|986)$/.test(e))i[e]=0;else{var r=new Promise(function(t,r){n=i[e]=[t,r]});t.push(n[2]=r);var o=d.p+d.u(e),u=Error();d.l(o,function(t){if(d.o(i,e)&&(0!==(n=i[e])&&(i[e]=void 0),n)){var r=t&&("load"===t.type?"missing":t.type),o=t&&t.target&&t.target.src;u.message="Loading chunk "+e+" failed.\n("+r+": "+o+")",u.name="ChunkLoadError",u.type=r,u.request=o,n[1](u)}},"chunk-"+e,e)}}},d.O.j=function(e){return 0===i[e]},c=function(e,t){var n,r,o=t[0],u=t[1],c=t[2],f=0;if(o.some(function(e){return 0!==i[e]})){for(n in u)d.o(u,n)&&(d.m[n]=u[n]);if(c)var a=c(d)}for(e&&e(t);f<o.length;f++)r=o[f],d.o(i,r)&&i[r]&&i[r][0](),i[r]=0;return d.O(a)},(f=self.webpackChunk_N_E=self.webpackChunk_N_E||[]).forEach(c.bind(null,0)),f.push=c.bind(null,f.push.bind(f))}();

litellm/proxy/_experimental/out/_next/static/css/31b7f215e119031e.css

@@ -1 +1 @@
-@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/55c55f0601d81cf3-s.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/26a46d62cd723877-s.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/97e0cb1ae144a2a9-s.woff2) format("woff2");unicode-range:u+1f??}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/581909926a08bbc8-s.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/df0a9ae256c0569c-s.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/8e9860b6e62d6359-s.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c0,u+2113,u+2c60-2c7f,u+a720-a7ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/_next/static/media/e4af272ccee01ff0-s.p.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd}@font-face{font-family:__Inter_Fallback_b0dd8a;src:local("Arial");ascent-override:90.49%;descent-override:22.56%;line-gap-override:0.00%;size-adjust:107.06%}.__className_b0dd8a{font-family:__Inter_b0dd8a,__Inter_Fallback_b0dd8a;font-style:normal}
+@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/55c55f0601d81cf3-s.woff2) format("woff2");unicode-range:u+0460-052f,u+1c80-1c8a,u+20b4,u+2de0-2dff,u+a640-a69f,u+fe2e-fe2f}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/26a46d62cd723877-s.woff2) format("woff2");unicode-range:u+0301,u+0400-045f,u+0490-0491,u+04b0-04b1,u+2116}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/97e0cb1ae144a2a9-s.woff2) format("woff2");unicode-range:u+1f??}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/581909926a08bbc8-s.woff2) format("woff2");unicode-range:u+0370-0377,u+037a-037f,u+0384-038a,u+038c,u+038e-03a1,u+03a3-03ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/df0a9ae256c0569c-s.woff2) format("woff2");unicode-range:u+0102-0103,u+0110-0111,u+0128-0129,u+0168-0169,u+01a0-01a1,u+01af-01b0,u+0300-0301,u+0303-0304,u+0308-0309,u+0323,u+0329,u+1ea0-1ef9,u+20ab}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/8e9860b6e62d6359-s.woff2) format("woff2");unicode-range:u+0100-02ba,u+02bd-02c5,u+02c7-02cc,u+02ce-02d7,u+02dd-02ff,u+0304,u+0308,u+0329,u+1d00-1dbf,u+1e00-1e9f,u+1ef2-1eff,u+2020,u+20a0-20ab,u+20ad-20c0,u+2113,u+2c60-2c7f,u+a720-a7ff}@font-face{font-family:__Inter_b0dd8a;font-style:normal;font-weight:100 900;font-display:swap;src:url(/litellm-asset-prefix/_next/static/media/e4af272ccee01ff0-s.p.woff2) format("woff2");unicode-range:u+00??,u+0131,u+0152-0153,u+02bb-02bc,u+02c6,u+02da,u+02dc,u+0304,u+0308,u+0329,u+2000-206f,u+20ac,u+2122,u+2191,u+2193,u+2212,u+2215,u+feff,u+fffd}@font-face{font-family:__Inter_Fallback_b0dd8a;src:local("Arial");ascent-override:90.49%;descent-override:22.56%;line-gap-override:0.00%;size-adjust:107.06%}.__className_b0dd8a{font-family:__Inter_b0dd8a,__Inter_Fallback_b0dd8a;font-style:normal}

litellm/proxy/_experimental/out/assets/logos/aws.svg

@@ -1,34 +1,34 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 600 450" style="enable-background:new 0 0 600 450;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:none;}
-	.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#343B45;}
-	.st2{fill-rule:evenodd;clip-rule:evenodd;fill:#F4981A;}
-</style>
-<g id="_x31__stroke">
-	<g id="Amazon_1_">
-		<rect x="161.2" y="86.5" class="st0" width="277.8" height="277.8"/>
-		<g id="Amazon">
-			<path class="st1" d="M315,163.7c-8,0.6-17.2,1.2-26.4,2.4c-14.1,1.9-28.2,4.3-39.8,9.8c-22.7,9.2-38,28.8-38,57.6
-				c0,36.2,23.3,54.6,52.7,54.6c9.8,0,17.8-1.2,25.1-3.1c11.7-3.7,21.5-10.4,33.1-22.7c6.7,9.2,8.6,13.5,20.2,23.3
-				c3.1,1.2,6.1,1.2,8.6-0.6c7.4-6.1,20.3-17.2,27-23.3c3.1-2.5,2.5-6.1,0.6-9.2c-6.7-8.6-13.5-16-13.5-32.5V165
-				c0-23.3,1.9-44.8-15.3-60.7c-14.1-12.9-36.2-17.8-53.4-17.8h-7.4c-31.2,1.8-64.3,15.3-71.7,54c-1.2,4.9,2.5,6.8,4.9,7.4l34.3,4.3
-				c3.7-0.6,5.5-3.7,6.1-6.7c3.1-13.5,14.1-20.2,26.3-21.5h2.5c7.4,0,15.3,3.1,19.6,9.2c4.9,7.4,4.3,17.2,4.3,25.8L315,163.7
-				L315,163.7z M308.2,236.7c-4.3,8.6-11.7,14.1-19.6,16c-1.2,0-3.1,0.6-4.9,0.6c-13.5,0-21.4-10.4-21.4-25.8
-				c0-19.6,11.6-28.8,26.3-33.1c8-1.8,17.2-2.5,26.4-2.5v7.4C315,213.4,315.6,224.4,308.2,236.7z"/>
-			<path class="st2" d="M398.8,311.4c-1.4,0-2.8,0.3-4.1,0.9c-1.5,0.6-3,1.3-4.4,1.9l-2.1,0.9l-2.7,1.1v0
-				c-29.8,12.1-61.1,19.2-90.1,19.8c-1.1,0-2.1,0-3.2,0c-45.6,0-82.8-21.1-120.3-42c-1.3-0.7-2.7-1-4-1c-1.7,0-3.4,0.6-4.7,1.8
-				c-1.3,1.2-2,2.9-2,4.7c0,2.3,1.2,4.4,2.9,5.7c35.2,30.6,73.8,59,125.7,59c1,0,2,0,3.1,0c33-0.7,70.3-11.9,99.3-30.1l0.2-0.1
-				c3.8-2.3,7.6-4.9,11.2-7.7c2.2-1.6,3.8-4.2,3.8-6.9C407.2,314.6,403.2,311.4,398.8,311.4z M439,294.5L439,294.5
-				c-0.1-2.9-0.7-5.1-1.9-6.9l-0.1-0.2l-0.1-0.2c-1.2-1.3-2.4-1.8-3.7-2.4c-3.8-1.5-9.3-2.3-16-2.3c-4.8,0-10.1,0.5-15.4,1.6l0-0.4
-				l-5.3,1.8l-0.1,0l-3,1v0.1c-3.5,1.5-6.8,3.3-9.8,5.5c-1.9,1.4-3.4,3.2-3.5,6.1c0,1.5,0.7,3.3,2,4.3c1.3,1,2.8,1.4,4.1,1.4
-				c0.3,0,0.6,0,0.9-0.1l0.3,0l0.2,0c2.6-0.6,6.4-0.9,10.9-1.6c3.8-0.4,7.9-0.7,11.4-0.7c2.5,0,4.7,0.2,6.3,0.5
-				c0.8,0.2,1.3,0.4,1.6,0.5c0.1,0,0.2,0.1,0.2,0.1c0.1,0.2,0.2,0.8,0.1,1.5c0,2.9-1.2,8.4-2.9,13.7c-1.7,5.3-3.7,10.7-5,14.2
-				c-0.3,0.8-0.5,1.7-0.5,2.7c0,1.4,0.6,3.2,1.8,4.3c1.2,1.1,2.8,1.6,4.1,1.6h0.1c2,0,3.6-0.8,5.1-1.9
-				c13.6-12.2,18.3-31.7,18.5-42.6L439,294.5z"/>
-		</g>
-	</g>
-</g>
-</svg>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 600 450" style="enable-background:new 0 0 600 450;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:none;}
+	.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#343B45;}
+	.st2{fill-rule:evenodd;clip-rule:evenodd;fill:#F4981A;}
+</style>
+<g id="_x31__stroke">
+	<g id="Amazon_1_">
+		<rect x="161.2" y="86.5" class="st0" width="277.8" height="277.8"/>
+		<g id="Amazon">
+			<path class="st1" d="M315,163.7c-8,0.6-17.2,1.2-26.4,2.4c-14.1,1.9-28.2,4.3-39.8,9.8c-22.7,9.2-38,28.8-38,57.6
+				c0,36.2,23.3,54.6,52.7,54.6c9.8,0,17.8-1.2,25.1-3.1c11.7-3.7,21.5-10.4,33.1-22.7c6.7,9.2,8.6,13.5,20.2,23.3
+				c3.1,1.2,6.1,1.2,8.6-0.6c7.4-6.1,20.3-17.2,27-23.3c3.1-2.5,2.5-6.1,0.6-9.2c-6.7-8.6-13.5-16-13.5-32.5V165
+				c0-23.3,1.9-44.8-15.3-60.7c-14.1-12.9-36.2-17.8-53.4-17.8h-7.4c-31.2,1.8-64.3,15.3-71.7,54c-1.2,4.9,2.5,6.8,4.9,7.4l34.3,4.3
+				c3.7-0.6,5.5-3.7,6.1-6.7c3.1-13.5,14.1-20.2,26.3-21.5h2.5c7.4,0,15.3,3.1,19.6,9.2c4.9,7.4,4.3,17.2,4.3,25.8L315,163.7
+				L315,163.7z M308.2,236.7c-4.3,8.6-11.7,14.1-19.6,16c-1.2,0-3.1,0.6-4.9,0.6c-13.5,0-21.4-10.4-21.4-25.8
+				c0-19.6,11.6-28.8,26.3-33.1c8-1.8,17.2-2.5,26.4-2.5v7.4C315,213.4,315.6,224.4,308.2,236.7z"/>
+			<path class="st2" d="M398.8,311.4c-1.4,0-2.8,0.3-4.1,0.9c-1.5,0.6-3,1.3-4.4,1.9l-2.1,0.9l-2.7,1.1v0
+				c-29.8,12.1-61.1,19.2-90.1,19.8c-1.1,0-2.1,0-3.2,0c-45.6,0-82.8-21.1-120.3-42c-1.3-0.7-2.7-1-4-1c-1.7,0-3.4,0.6-4.7,1.8
+				c-1.3,1.2-2,2.9-2,4.7c0,2.3,1.2,4.4,2.9,5.7c35.2,30.6,73.8,59,125.7,59c1,0,2,0,3.1,0c33-0.7,70.3-11.9,99.3-30.1l0.2-0.1
+				c3.8-2.3,7.6-4.9,11.2-7.7c2.2-1.6,3.8-4.2,3.8-6.9C407.2,314.6,403.2,311.4,398.8,311.4z M439,294.5L439,294.5
+				c-0.1-2.9-0.7-5.1-1.9-6.9l-0.1-0.2l-0.1-0.2c-1.2-1.3-2.4-1.8-3.7-2.4c-3.8-1.5-9.3-2.3-16-2.3c-4.8,0-10.1,0.5-15.4,1.6l0-0.4
+				l-5.3,1.8l-0.1,0l-3,1v0.1c-3.5,1.5-6.8,3.3-9.8,5.5c-1.9,1.4-3.4,3.2-3.5,6.1c0,1.5,0.7,3.3,2,4.3c1.3,1,2.8,1.4,4.1,1.4
+				c0.3,0,0.6,0,0.9-0.1l0.3,0l0.2,0c2.6-0.6,6.4-0.9,10.9-1.6c3.8-0.4,7.9-0.7,11.4-0.7c2.5,0,4.7,0.2,6.3,0.5
+				c0.8,0.2,1.3,0.4,1.6,0.5c0.1,0,0.2,0.1,0.2,0.1c0.1,0.2,0.2,0.8,0.1,1.5c0,2.9-1.2,8.4-2.9,13.7c-1.7,5.3-3.7,10.7-5,14.2
+				c-0.3,0.8-0.5,1.7-0.5,2.7c0,1.4,0.6,3.2,1.8,4.3c1.2,1.1,2.8,1.6,4.1,1.6h0.1c2,0,3.6-0.8,5.1-1.9
+				c13.6-12.2,18.3-31.7,18.5-42.6L439,294.5z"/>
+		</g>
+	</g>
+</g>
+</svg>

litellm/proxy/_experimental/out/assets/logos/cerebras.svg

@@ -1,89 +1,89 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 800 600" style="enable-background:new 0 0 800 600;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#F05A28;}
-	.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#231F20;}
-</style>
-<g id="Contact">
-	<g id="Contact-us" transform="translate(-234.000000, -1114.000000)">
-		<g id="map" transform="translate(-6.000000, 1027.000000)">
-			<g id="Contact-box" transform="translate(190.000000, 36.000000)">
-				<g id="Group-26" transform="translate(50.000000, 51.000000)">
-					<g id="Group-3">
-						<path id="Fill-1" class="st0" d="M220.9,421c-17,0-33.1-3.4-47.8-9.5c-22-9.2-40.8-24.6-54.1-44c-13.3-19.4-21-42.7-21-67.9
-							c0-16.8,3.4-32.7,9.7-47.3c9.3-21.8,24.9-40.3,44.5-53.4c19.6-13.1,43.2-20.7,68.7-20.7v-18.3c-19.5,0-38.1,3.9-55.1,11
-							c-25.4,10.6-47,28.3-62.2,50.6c-15.3,22.3-24.2,49.2-24.2,78.1c0,19.3,4,37.7,11.1,54.4c10.7,25.1,28.7,46.4,51.2,61.5
-							c22.6,15.1,49.8,23.9,79.1,23.9V421z"/>
-						<path id="Fill-4" class="st0" d="M157.9,374.1c-11.5-9.6-20.1-21.2-25.9-33.9c-5.8-12.7-8.8-26.4-8.8-40.2
-							c0-11,1.9-22,5.6-32.5c3.8-10.5,9.4-20.5,17.1-29.6c9.6-11.4,21.3-20,34-25.8c12.7-5.8,26.6-8.7,40.4-8.7
-							c11,0,22.1,1.9,32.6,5.6c10.6,3.8,20.6,9.4,29.7,17l11.9-14.1c-10.8-9-22.8-15.8-35.4-20.2c-12.6-4.5-25.7-6.7-38.8-6.7
-							c-16.5,0-32.9,3.5-48.1,10.4c-15.2,6.9-29.1,17.2-40.5,30.7c-9.1,10.8-15.8,22.7-20.3,35.2c-4.5,12.5-6.7,25.6-6.7,38.7
-							c0,16.4,3.5,32.8,10.4,47.9c6.9,15.1,17.3,29,30.9,40.3L157.9,374.1z"/>
-						<path id="Fill-6" class="st0" d="M186.4,362.2c-12.1-6.4-21.6-15.7-28.1-26.6c-6.5-10.9-9.9-23.5-9.9-36.2
-							c0-11.2,2.6-22.5,8.3-33c6.4-12.1,15.8-21.5,26.8-27.9c11-6.5,23.6-9.9,36.4-9.9c11.2,0,22.6,2.6,33.2,8.2l8.6-16.3
-							c-13.3-7-27.7-10.4-41.9-10.3c-16.1,0-32,4.3-45.8,12.4c-13.8,8.1-25.7,20.1-33.7,35.2c-7,13.3-10.4,27.6-10.4,41.6
-							c0,16,4.3,31.8,12.5,45.5c8.2,13.8,20.2,25.5,35.4,33.5L186.4,362.2z"/>
-						<path id="Fill-8" class="st0" d="M221,344.6c-6.3,0-12.3-1.3-17.7-3.6c-8.2-3.4-15.1-9.2-20-16.5c-4.9-7.3-7.8-16-7.8-25.4
-							c0-6.3,1.3-12.3,3.6-17.7c3.4-8.1,9.2-15.1,16.5-20c7.3-4.9,16-7.8,25.4-7.8v-18.4c-8.8,0-17.2,1.8-24.9,5
-							c-11.5,4.9-21.2,12.9-28.1,23.1C161,273.6,157,286,157,299.2c0,8.8,1.8,17.2,5,24.9c4.9,11.5,13,21.2,23.2,28.1
-							C195.4,359,207.7,363,221,363V344.6z"/>
-					</g>
-					<g id="Group" transform="translate(22.000000, 13.000000)">
-						<path id="Fill-10" class="st1" d="M214,271.6c-2.1-2.2-4.4-4-6.7-5.3c-2.3-1.3-4.7-2-7.2-2c-3.4,0-6.3,0.6-9,1.8
-							c-2.6,1.2-4.9,2.8-6.8,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.4,5.4-1.4,8.2c0,2.8,0.5,5.6,1.4,8.2c1,2.6,2.4,5,4.3,7
-							c1.9,2,4.1,3.7,6.8,4.9c2.6,1.2,5.6,1.8,9,1.8c2.8,0,5.5-0.6,7.9-1.7c2.4-1.2,4.5-2.9,6.2-5.1l12.2,13.1
-							c-1.8,1.8-3.9,3.4-6.3,4.7c-2.4,1.3-4.8,2.4-7.2,3.2s-4.8,1.4-7,1.7c-2.2,0.4-4.2,0.5-5.8,0.5c-5.5,0-10.7-0.9-15.5-2.7
-							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
-							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c4.7,0,9.4,0.9,14.1,2.7
-							c4.7,1.8,8.9,4.6,12.4,8.4L214,271.6z"/>
-						<path id="Fill-12" class="st1" d="M280.4,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
-							c-3.2,3.1-5.2,7.4-5.9,12.6H280.4z M243,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
-							c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
-							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
-							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
-							c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H243z"/>
-						<path id="Fill-14" class="st1" d="M306.5,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
-							c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
-							c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
-						<path id="Fill-16" class="st1" d="M409,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
-							c-3.2,3.1-5.2,7.4-5.9,12.6H409z M371.6,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
-							c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
-							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
-							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
-							c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H371.6z"/>
-						<path id="Fill-18" class="st1" d="M494.6,286.2c0-2.8-0.5-5.6-1.5-8.2c-1-2.6-2.4-5-4.3-7c-1.9-2-4.2-3.7-6.9-4.9
-							c-2.7-1.2-5.7-1.8-9.1-1.8c-3.4,0-6.4,0.6-9.1,1.8c-2.7,1.2-5,2.8-6.9,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.5,5.4-1.5,8.2
-							c0,2.8,0.5,5.6,1.5,8.2c1,2.6,2.4,5,4.3,7c1.9,2,4.2,3.7,6.9,4.9c2.7,1.2,5.7,1.8,9.1,1.8c3.4,0,6.4-0.6,9.1-1.8
-							c2.7-1.2,5-2.8,6.9-4.9c1.9-2,3.3-4.4,4.3-7C494.1,291.8,494.6,289,494.6,286.2L494.6,286.2z M433.2,207.6h18.5v51.3h0.5
-							c0.9-1.2,2.1-2.5,3.5-3.7c1.4-1.3,3.2-2.5,5.2-3.6c2.1-1.1,4.4-2,7.1-2.7c2.7-0.7,5.8-1.1,9.3-1.1c5.2,0,10.1,1,14.5,3
-							c4.4,2,8.2,4.7,11.3,8.1c3.1,3.5,5.6,7.5,7.3,12.2c1.7,4.7,2.6,9.7,2.6,15.1c0,5.4-0.8,10.4-2.5,15.1
-							c-1.6,4.7-4.1,8.7-7.2,12.2c-3.2,3.5-7,6.2-11.6,8.1c-4.5,2-9.6,3-15.3,3c-5.2,0-10.1-1-14.7-3c-4.5-2-8.1-5.3-10.8-9.7h-0.3
-							v11h-17.6V207.6z"/>
-						<path id="Fill-20" class="st1" d="M520.9,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
-							c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
-							c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
-						<path id="Fill-22" class="st1" d="M616,290h-3.9c-2.6,0-5.5,0.1-8.7,0.3c-3.2,0.2-6.2,0.7-9.1,1.4c-2.8,0.8-5.2,1.9-7.2,3.3
-							c-2,1.5-2.9,3.5-2.9,6.2c0,1.7,0.4,3.2,1.2,4.3c0.8,1.2,1.8,2.2,3,3c1.2,0.8,2.6,1.4,4.2,1.8c1.5,0.4,3.1,0.5,4.6,0.5
-							c6.4,0,11.1-1.5,14.2-4.5c3-3,4.6-7.1,4.6-12.2V290z M617.1,312.7h-0.5c-2.7,4.2-6.1,7.2-10.2,9.1c-4.1,1.9-8.7,2.8-13.6,2.8
-							c-3.4,0-6.7-0.5-10-1.4s-6.1-2.3-8.7-4.1c-2.5-1.8-4.6-4.1-6.1-6.8s-2.3-5.9-2.3-9.6c0-4,0.7-7.3,2.2-10.1
-							c1.4-2.8,3.4-5.1,5.8-7c2.4-1.9,5.2-3.4,8.4-4.5c3.2-1.1,6.5-2,10-2.5c3.5-0.6,6.9-0.9,10.5-1.1c3.5-0.2,6.8-0.2,9.9-0.2h4.6
-							v-2c0-4.6-1.6-8-4.8-10.3c-3.2-2.3-7.3-3.4-12.2-3.4c-3.9,0-7.6,0.7-11,2.1c-3.4,1.4-6.4,3.2-8.8,5.6l-9.8-9.6
-							c4.1-4.2,9-7.1,14.5-9c5.5-1.8,11.2-2.7,17.1-2.7c5.3,0,9.7,0.6,13.3,1.7c3.6,1.2,6.6,2.7,9,4.5c2.4,1.8,4.2,3.9,5.5,6.3
-							c1.3,2.4,2.2,4.8,2.8,7.2c0.6,2.4,0.9,4.8,1,7.1c0.1,2.3,0.2,4.3,0.2,6v42h-16.7V312.7z"/>
-						<path id="Fill-24" class="st1" d="M683.6,269.9c-3.6-5-8.4-7.5-14.4-7.5c-2.5,0-4.9,0.6-7.2,1.8c-2.4,1.2-3.5,3.2-3.5,5.9
-							c0,2.2,1,3.9,2.9,4.9c1.9,1,4.4,1.9,7.4,2.6c3,0.7,6.2,1.4,9.6,2.2c3.4,0.8,6.6,1.9,9.6,3.5c3,1.6,5.4,3.7,7.4,6.5
-							c1.9,2.7,2.9,6.5,2.9,11.3c0,4.4-0.9,8-2.8,11c-1.9,3-4.3,5.4-7.4,7.2c-3,1.8-6.4,3.1-10.2,4c-3.8,0.8-7.6,1.2-11.3,1.2
-							c-5.7,0-11-0.8-15.8-2.4c-4.8-1.6-9.1-4.6-12.9-8.8l12.3-11.4c2.4,2.6,4.9,4.8,7.6,6.5c2.7,1.7,6,2.5,9.9,2.5
-							c1.3,0,2.7-0.2,4.1-0.5c1.4-0.3,2.8-0.8,4-1.5c1.2-0.7,2.2-1.6,3-2.7c0.8-1.1,1.1-2.3,1.1-3.7c0-2.5-1-4.4-2.9-5.6
-							c-1.9-1.2-4.4-2.2-7.4-3c-3-0.8-6.2-1.5-9.6-2.1c-3.4-0.7-6.6-1.7-9.6-3.2c-3-1.5-5.4-3.5-7.4-6.2c-1.9-2.6-2.9-6.3-2.9-11
-							c0-4.1,0.8-7.6,2.5-10.6c1.7-3,3.9-5.4,6.7-7.4c2.8-1.9,5.9-3.3,9.5-4.3c3.6-0.9,7.2-1.4,10.9-1.4c4.9,0,9.8,0.8,14.6,2.5
-							c4.8,1.7,8.7,4.5,11.7,8.6L683.6,269.9z"/>
-					</g>
-				</g>
-			</g>
-		</g>
-	</g>
-</g>
-</svg>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 26.0.3, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.0" id="katman_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 800 600" style="enable-background:new 0 0 800 600;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill-rule:evenodd;clip-rule:evenodd;fill:#F05A28;}
+	.st1{fill-rule:evenodd;clip-rule:evenodd;fill:#231F20;}
+</style>
+<g id="Contact">
+	<g id="Contact-us" transform="translate(-234.000000, -1114.000000)">
+		<g id="map" transform="translate(-6.000000, 1027.000000)">
+			<g id="Contact-box" transform="translate(190.000000, 36.000000)">
+				<g id="Group-26" transform="translate(50.000000, 51.000000)">
+					<g id="Group-3">
+						<path id="Fill-1" class="st0" d="M220.9,421c-17,0-33.1-3.4-47.8-9.5c-22-9.2-40.8-24.6-54.1-44c-13.3-19.4-21-42.7-21-67.9
+							c0-16.8,3.4-32.7,9.7-47.3c9.3-21.8,24.9-40.3,44.5-53.4c19.6-13.1,43.2-20.7,68.7-20.7v-18.3c-19.5,0-38.1,3.9-55.1,11
+							c-25.4,10.6-47,28.3-62.2,50.6c-15.3,22.3-24.2,49.2-24.2,78.1c0,19.3,4,37.7,11.1,54.4c10.7,25.1,28.7,46.4,51.2,61.5
+							c22.6,15.1,49.8,23.9,79.1,23.9V421z"/>
+						<path id="Fill-4" class="st0" d="M157.9,374.1c-11.5-9.6-20.1-21.2-25.9-33.9c-5.8-12.7-8.8-26.4-8.8-40.2
+							c0-11,1.9-22,5.6-32.5c3.8-10.5,9.4-20.5,17.1-29.6c9.6-11.4,21.3-20,34-25.8c12.7-5.8,26.6-8.7,40.4-8.7
+							c11,0,22.1,1.9,32.6,5.6c10.6,3.8,20.6,9.4,29.7,17l11.9-14.1c-10.8-9-22.8-15.8-35.4-20.2c-12.6-4.5-25.7-6.7-38.8-6.7
+							c-16.5,0-32.9,3.5-48.1,10.4c-15.2,6.9-29.1,17.2-40.5,30.7c-9.1,10.8-15.8,22.7-20.3,35.2c-4.5,12.5-6.7,25.6-6.7,38.7
+							c0,16.4,3.5,32.8,10.4,47.9c6.9,15.1,17.3,29,30.9,40.3L157.9,374.1z"/>
+						<path id="Fill-6" class="st0" d="M186.4,362.2c-12.1-6.4-21.6-15.7-28.1-26.6c-6.5-10.9-9.9-23.5-9.9-36.2
+							c0-11.2,2.6-22.5,8.3-33c6.4-12.1,15.8-21.5,26.8-27.9c11-6.5,23.6-9.9,36.4-9.9c11.2,0,22.6,2.6,33.2,8.2l8.6-16.3
+							c-13.3-7-27.7-10.4-41.9-10.3c-16.1,0-32,4.3-45.8,12.4c-13.8,8.1-25.7,20.1-33.7,35.2c-7,13.3-10.4,27.6-10.4,41.6
+							c0,16,4.3,31.8,12.5,45.5c8.2,13.8,20.2,25.5,35.4,33.5L186.4,362.2z"/>
+						<path id="Fill-8" class="st0" d="M221,344.6c-6.3,0-12.3-1.3-17.7-3.6c-8.2-3.4-15.1-9.2-20-16.5c-4.9-7.3-7.8-16-7.8-25.4
+							c0-6.3,1.3-12.3,3.6-17.7c3.4-8.1,9.2-15.1,16.5-20c7.3-4.9,16-7.8,25.4-7.8v-18.4c-8.8,0-17.2,1.8-24.9,5
+							c-11.5,4.9-21.2,12.9-28.1,23.1C161,273.6,157,286,157,299.2c0,8.8,1.8,17.2,5,24.9c4.9,11.5,13,21.2,23.2,28.1
+							C195.4,359,207.7,363,221,363V344.6z"/>
+					</g>
+					<g id="Group" transform="translate(22.000000, 13.000000)">
+						<path id="Fill-10" class="st1" d="M214,271.6c-2.1-2.2-4.4-4-6.7-5.3c-2.3-1.3-4.7-2-7.2-2c-3.4,0-6.3,0.6-9,1.8
+							c-2.6,1.2-4.9,2.8-6.8,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.4,5.4-1.4,8.2c0,2.8,0.5,5.6,1.4,8.2c1,2.6,2.4,5,4.3,7
+							c1.9,2,4.1,3.7,6.8,4.9c2.6,1.2,5.6,1.8,9,1.8c2.8,0,5.5-0.6,7.9-1.7c2.4-1.2,4.5-2.9,6.2-5.1l12.2,13.1
+							c-1.8,1.8-3.9,3.4-6.3,4.7c-2.4,1.3-4.8,2.4-7.2,3.2s-4.8,1.4-7,1.7c-2.2,0.4-4.2,0.5-5.8,0.5c-5.5,0-10.7-0.9-15.5-2.7
+							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
+							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c4.7,0,9.4,0.9,14.1,2.7
+							c4.7,1.8,8.9,4.6,12.4,8.4L214,271.6z"/>
+						<path id="Fill-12" class="st1" d="M280.4,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
+							c-3.2,3.1-5.2,7.4-5.9,12.6H280.4z M243,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
+							c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
+							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
+							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
+							c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H243z"/>
+						<path id="Fill-14" class="st1" d="M306.5,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
+							c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
+							c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
+						<path id="Fill-16" class="st1" d="M409,278.9c-0.1-5.4-1.8-9.6-5-12.7c-3.3-3.1-7.8-4.6-13.6-4.6c-5.5,0-9.8,1.6-13,4.7
+							c-3.2,3.1-5.2,7.4-5.9,12.6H409z M371.6,292.6c0.6,5.5,2.7,9.7,6.4,12.8c3.7,3,8.1,4.6,13.3,4.6c4.6,0,8.4-0.9,11.5-2.8
+							c3.1-1.9,5.8-4.2,8.2-7.1l13.1,9.9c-4.3,5.3-9,9-14.3,11.3c-5.3,2.2-10.8,3.3-16.6,3.3c-5.5,0-10.7-0.9-15.5-2.7
+							c-4.9-1.8-9.1-4.4-12.6-7.8c-3.6-3.3-6.4-7.4-8.5-12.1c-2.1-4.7-3.1-10-3.1-15.7c0-5.8,1-11,3.1-15.7
+							c2.1-4.7,4.9-8.7,8.5-12.1c3.6-3.3,7.8-5.9,12.6-7.8c4.9-1.8,10.1-2.7,15.5-2.7c5.1,0,9.7,0.9,13.9,2.7
+							c4.2,1.8,7.8,4.3,10.8,7.7c3,3.3,5.3,7.5,7,12.4c1.7,4.9,2.5,10.6,2.5,17v5H371.6z"/>
+						<path id="Fill-18" class="st1" d="M494.6,286.2c0-2.8-0.5-5.6-1.5-8.2c-1-2.6-2.4-5-4.3-7c-1.9-2-4.2-3.7-6.9-4.9
+							c-2.7-1.2-5.7-1.8-9.1-1.8c-3.4,0-6.4,0.6-9.1,1.8c-2.7,1.2-5,2.8-6.9,4.9c-1.9,2-3.3,4.4-4.3,7c-1,2.6-1.5,5.4-1.5,8.2
+							c0,2.8,0.5,5.6,1.5,8.2c1,2.6,2.4,5,4.3,7c1.9,2,4.2,3.7,6.9,4.9c2.7,1.2,5.7,1.8,9.1,1.8c3.4,0,6.4-0.6,9.1-1.8
+							c2.7-1.2,5-2.8,6.9-4.9c1.9-2,3.3-4.4,4.3-7C494.1,291.8,494.6,289,494.6,286.2L494.6,286.2z M433.2,207.6h18.5v51.3h0.5
+							c0.9-1.2,2.1-2.5,3.5-3.7c1.4-1.3,3.2-2.5,5.2-3.6c2.1-1.1,4.4-2,7.1-2.7c2.7-0.7,5.8-1.1,9.3-1.1c5.2,0,10.1,1,14.5,3
+							c4.4,2,8.2,4.7,11.3,8.1c3.1,3.5,5.6,7.5,7.3,12.2c1.7,4.7,2.6,9.7,2.6,15.1c0,5.4-0.8,10.4-2.5,15.1
+							c-1.6,4.7-4.1,8.7-7.2,12.2c-3.2,3.5-7,6.2-11.6,8.1c-4.5,2-9.6,3-15.3,3c-5.2,0-10.1-1-14.7-3c-4.5-2-8.1-5.3-10.8-9.7h-0.3
+							v11h-17.6V207.6z"/>
+						<path id="Fill-20" class="st1" d="M520.9,249.7h18.3v11.5h0.3c2-4.3,4.9-7.5,8.7-9.9c3.8-2.3,8.1-3.5,12.9-3.5
+							c1.1,0,2.2,0.1,3.3,0.3c1.1,0.2,2.2,0.5,3.3,0.8v17.6c-1.5-0.4-3-0.7-4.5-1c-1.5-0.3-2.9-0.4-4.3-0.4c-4.3,0-7.7,0.8-10.3,2.4
+							c-2.6,1.6-4.6,3.4-5.9,5.4c-1.4,2-2.3,4.1-2.7,6.1c-0.5,2-0.7,3.5-0.7,4.6v39h-18.3V249.7z"/>
+						<path id="Fill-22" class="st1" d="M616,290h-3.9c-2.6,0-5.5,0.1-8.7,0.3c-3.2,0.2-6.2,0.7-9.1,1.4c-2.8,0.8-5.2,1.9-7.2,3.3
+							c-2,1.5-2.9,3.5-2.9,6.2c0,1.7,0.4,3.2,1.2,4.3c0.8,1.2,1.8,2.2,3,3c1.2,0.8,2.6,1.4,4.2,1.8c1.5,0.4,3.1,0.5,4.6,0.5
+							c6.4,0,11.1-1.5,14.2-4.5c3-3,4.6-7.1,4.6-12.2V290z M617.1,312.7h-0.5c-2.7,4.2-6.1,7.2-10.2,9.1c-4.1,1.9-8.7,2.8-13.6,2.8
+							c-3.4,0-6.7-0.5-10-1.4s-6.1-2.3-8.7-4.1c-2.5-1.8-4.6-4.1-6.1-6.8s-2.3-5.9-2.3-9.6c0-4,0.7-7.3,2.2-10.1
+							c1.4-2.8,3.4-5.1,5.8-7c2.4-1.9,5.2-3.4,8.4-4.5c3.2-1.1,6.5-2,10-2.5c3.5-0.6,6.9-0.9,10.5-1.1c3.5-0.2,6.8-0.2,9.9-0.2h4.6
+							v-2c0-4.6-1.6-8-4.8-10.3c-3.2-2.3-7.3-3.4-12.2-3.4c-3.9,0-7.6,0.7-11,2.1c-3.4,1.4-6.4,3.2-8.8,5.6l-9.8-9.6
+							c4.1-4.2,9-7.1,14.5-9c5.5-1.8,11.2-2.7,17.1-2.7c5.3,0,9.7,0.6,13.3,1.7c3.6,1.2,6.6,2.7,9,4.5c2.4,1.8,4.2,3.9,5.5,6.3
+							c1.3,2.4,2.2,4.8,2.8,7.2c0.6,2.4,0.9,4.8,1,7.1c0.1,2.3,0.2,4.3,0.2,6v42h-16.7V312.7z"/>
+						<path id="Fill-24" class="st1" d="M683.6,269.9c-3.6-5-8.4-7.5-14.4-7.5c-2.5,0-4.9,0.6-7.2,1.8c-2.4,1.2-3.5,3.2-3.5,5.9
+							c0,2.2,1,3.9,2.9,4.9c1.9,1,4.4,1.9,7.4,2.6c3,0.7,6.2,1.4,9.6,2.2c3.4,0.8,6.6,1.9,9.6,3.5c3,1.6,5.4,3.7,7.4,6.5
+							c1.9,2.7,2.9,6.5,2.9,11.3c0,4.4-0.9,8-2.8,11c-1.9,3-4.3,5.4-7.4,7.2c-3,1.8-6.4,3.1-10.2,4c-3.8,0.8-7.6,1.2-11.3,1.2
+							c-5.7,0-11-0.8-15.8-2.4c-4.8-1.6-9.1-4.6-12.9-8.8l12.3-11.4c2.4,2.6,4.9,4.8,7.6,6.5c2.7,1.7,6,2.5,9.9,2.5
+							c1.3,0,2.7-0.2,4.1-0.5c1.4-0.3,2.8-0.8,4-1.5c1.2-0.7,2.2-1.6,3-2.7c0.8-1.1,1.1-2.3,1.1-3.7c0-2.5-1-4.4-2.9-5.6
+							c-1.9-1.2-4.4-2.2-7.4-3c-3-0.8-6.2-1.5-9.6-2.1c-3.4-0.7-6.6-1.7-9.6-3.2c-3-1.5-5.4-3.5-7.4-6.2c-1.9-2.6-2.9-6.3-2.9-11
+							c0-4.1,0.8-7.6,2.5-10.6c1.7-3,3.9-5.4,6.7-7.4c2.8-1.9,5.9-3.3,9.5-4.3c3.6-0.9,7.2-1.4,10.9-1.4c4.9,0,9.8,0.8,14.6,2.5
+							c4.8,1.7,8.7,4.5,11.7,8.6L683.6,269.9z"/>
+					</g>
+				</g>
+			</g>
+		</g>
+	</g>
+</g>
+</svg>

litellm/proxy/_experimental/out/assets/logos/deepseek.svg

@@ -1,25 +1,25 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!-- Generator: Adobe Illustrator 25.4.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 292.6 215.3" style="enable-background:new 0 0 292.6 215.3;" xml:space="preserve">
-<style type="text/css">
-	.st0{fill:#566AB2;}
-</style>
-<path class="st0" d="M191.3,123.7c-2.4,1-4.9,1.8-7.2,1.9c-3.6,0.2-7.6-1.3-9.7-3.1c-3.3-2.8-5.7-4.4-6.7-9.2
-	c-0.4-2.1-0.2-5.3,0.2-7.2c0.9-4-0.1-6.5-2.9-8.9c-2.3-1.9-5.2-2.4-8.4-2.4s-2.3-0.5-3.1-1c-1.3-0.7-2.4-2.3-1.4-4.4
-	c0.3-0.7,2-2.3,2.3-2.5c4.3-2.5,9.4-1.7,14,0.2c4.3,1.7,7.5,5,12.2,9.5c4.8,5.5,5.6,7,8.4,11.1c2.1,3.2,4.1,6.6,5.4,10.4
-	C195.2,120.5,194.2,122.4,191.3,123.7L191.3,123.7z M153.4,104.3c0-2.1,1.7-3.7,3.8-3.7s0.9,0.1,1.3,0.2c0.5,0.2,1,0.5,1.4,0.9
-	c0.7,0.7,1.1,1.6,1.1,2.6c0,2.1-1.7,3.8-3.8,3.8s-3.7-1.7-3.7-3.8H153.4z M141.2,182.8c-25.5-20-37.8-26.6-42.9-26.3
-	c-4.8,0.3-3.9,5.7-2.8,9.3c1.1,3.5,2.5,5.9,4.5,9c1.4,2,2.3,5.1-1.4,7.3c-8.2,5.1-22.5-1.7-23.1-2c-16.6-9.8-30.5-22.7-40.2-40.3
-	c-9.5-17-14.9-35.2-15.8-54.6c-0.2-4.7,1.1-6.4,5.8-7.2c6.2-1.1,12.5-1.4,18.7-0.5c26,3.8,48.1,15.4,66.7,33.8
-	c10.6,10.5,18.6,23,26.8,35.2c8.8,13,18.2,25.4,30.2,35.5c4.3,3.6,7.6,6.3,10.9,8.2c-9.8,1.1-26.1,1.3-37.2-7.5L141.2,182.8z
-	 M289.5,18c-3.1-1.5-4.4,1.4-6.3,2.8c-0.6,0.5-1.1,1.1-1.7,1.7c-4.5,4.8-9.8,8-16.8,7.6c-10.1-0.6-18.7,2.6-26.4,10.4
-	c-1.6-9.5-7-15.2-15.2-18.9c-4.3-1.9-8.6-3.8-11.6-7.9c-2.1-2.9-2.7-6.2-3.7-9.4c-0.7-2-1.3-3.9-3.6-4.3c-2.4-0.4-3.4,1.7-4.3,3.4
-	c-3.8,7-5.3,14.6-5.2,22.4c0.3,17.5,7.7,31.5,22.4,41.4c1.7,1.1,2.1,2.3,1.6,3.9c-1,3.4-2.2,6.7-3.3,10.1c-0.7,2.2-1.7,2.7-4,1.7
-	c-8.1-3.4-15-8.4-21.2-14.4c-10.4-10.1-19.9-21.2-31.6-30c-2.8-2.1-5.5-4-8.4-5.7c-12-11.7,1.6-21.3,4.7-22.4
-	c3.3-1.2,1.2-5.3-9.5-5.2c-10.6,0-20.3,3.6-32.8,8.4c-1.8,0.7-3.7,1.2-5.7,1.7c-11.3-2.1-22.9-2.6-35.1-1.2
-	c-23,2.5-41.4,13.4-54.8,32C1,68.3-2.8,93.6,1.9,120c4.9,27.8,19.1,50.9,41,68.9c22.6,18.7,48.7,27.8,78.5,26.1
-	c18.1-1,38.2-3.5,60.9-22.7c5.7,2.8,11.7,4,21.7,4.8c7.7,0.7,15.1-0.4,20.8-1.5c9-1.9,8.4-10.2,5.1-11.7
-	c-26.3-12.3-20.5-7.3-25.7-11.3c13.3-15.8,33.5-32.2,41.3-85.4c0.6-4.2,0.1-6.9,0-10.3c0-2.1,0.4-2.9,2.8-3.1
-	c6.6-0.8,13-2.6,18.8-5.8c17-9.3,23.9-24.6,25.5-42.9c0.2-2.8,0-5.7-3-7.2L289.5,18z"/>
-</svg>
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Generator: Adobe Illustrator 25.4.1, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 292.6 215.3" style="enable-background:new 0 0 292.6 215.3;" xml:space="preserve">
+<style type="text/css">
+	.st0{fill:#566AB2;}
+</style>
+<path class="st0" d="M191.3,123.7c-2.4,1-4.9,1.8-7.2,1.9c-3.6,0.2-7.6-1.3-9.7-3.1c-3.3-2.8-5.7-4.4-6.7-9.2
+	c-0.4-2.1-0.2-5.3,0.2-7.2c0.9-4-0.1-6.5-2.9-8.9c-2.3-1.9-5.2-2.4-8.4-2.4s-2.3-0.5-3.1-1c-1.3-0.7-2.4-2.3-1.4-4.4
+	c0.3-0.7,2-2.3,2.3-2.5c4.3-2.5,9.4-1.7,14,0.2c4.3,1.7,7.5,5,12.2,9.5c4.8,5.5,5.6,7,8.4,11.1c2.1,3.2,4.1,6.6,5.4,10.4
+	C195.2,120.5,194.2,122.4,191.3,123.7L191.3,123.7z M153.4,104.3c0-2.1,1.7-3.7,3.8-3.7s0.9,0.1,1.3,0.2c0.5,0.2,1,0.5,1.4,0.9
+	c0.7,0.7,1.1,1.6,1.1,2.6c0,2.1-1.7,3.8-3.8,3.8s-3.7-1.7-3.7-3.8H153.4z M141.2,182.8c-25.5-20-37.8-26.6-42.9-26.3
+	c-4.8,0.3-3.9,5.7-2.8,9.3c1.1,3.5,2.5,5.9,4.5,9c1.4,2,2.3,5.1-1.4,7.3c-8.2,5.1-22.5-1.7-23.1-2c-16.6-9.8-30.5-22.7-40.2-40.3
+	c-9.5-17-14.9-35.2-15.8-54.6c-0.2-4.7,1.1-6.4,5.8-7.2c6.2-1.1,12.5-1.4,18.7-0.5c26,3.8,48.1,15.4,66.7,33.8
+	c10.6,10.5,18.6,23,26.8,35.2c8.8,13,18.2,25.4,30.2,35.5c4.3,3.6,7.6,6.3,10.9,8.2c-9.8,1.1-26.1,1.3-37.2-7.5L141.2,182.8z
+	 M289.5,18c-3.1-1.5-4.4,1.4-6.3,2.8c-0.6,0.5-1.1,1.1-1.7,1.7c-4.5,4.8-9.8,8-16.8,7.6c-10.1-0.6-18.7,2.6-26.4,10.4
+	c-1.6-9.5-7-15.2-15.2-18.9c-4.3-1.9-8.6-3.8-11.6-7.9c-2.1-2.9-2.7-6.2-3.7-9.4c-0.7-2-1.3-3.9-3.6-4.3c-2.4-0.4-3.4,1.7-4.3,3.4
+	c-3.8,7-5.3,14.6-5.2,22.4c0.3,17.5,7.7,31.5,22.4,41.4c1.7,1.1,2.1,2.3,1.6,3.9c-1,3.4-2.2,6.7-3.3,10.1c-0.7,2.2-1.7,2.7-4,1.7
+	c-8.1-3.4-15-8.4-21.2-14.4c-10.4-10.1-19.9-21.2-31.6-30c-2.8-2.1-5.5-4-8.4-5.7c-12-11.7,1.6-21.3,4.7-22.4
+	c3.3-1.2,1.2-5.3-9.5-5.2c-10.6,0-20.3,3.6-32.8,8.4c-1.8,0.7-3.7,1.2-5.7,1.7c-11.3-2.1-22.9-2.6-35.1-1.2
+	c-23,2.5-41.4,13.4-54.8,32C1,68.3-2.8,93.6,1.9,120c4.9,27.8,19.1,50.9,41,68.9c22.6,18.7,48.7,27.8,78.5,26.1
+	c18.1-1,38.2-3.5,60.9-22.7c5.7,2.8,11.7,4,21.7,4.8c7.7,0.7,15.1-0.4,20.8-1.5c9-1.9,8.4-10.2,5.1-11.7
+	c-26.3-12.3-20.5-7.3-25.7-11.3c13.3-15.8,33.5-32.2,41.3-85.4c0.6-4.2,0.1-6.9,0-10.3c0-2.1,0.4-2.9,2.8-3.1
+	c6.6-0.8,13-2.6,18.8-5.8c17-9.3,23.9-24.6,25.5-42.9c0.2-2.8,0-5.7-3-7.2L289.5,18z"/>
+</svg>

litellm/proxy/_experimental/out/assets/logos/perplexity-ai.svg

@@ -1,16 +1,16 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!-- Generator: Adobe Illustrator 26.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
-<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
-	 viewBox="0 0 48 48" style="enable-background:new 0 0 48 48;" xml:space="preserve">
-<linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="10.5862" y1="1.61" x2="36.0543" y2="44.1206">
-	<stop  offset="0.002" style="stop-color:#9C55D4"/>
-	<stop  offset="0.003" style="stop-color:#20808D"/>
-	<stop  offset="0.3731" style="stop-color:#218F9B"/>
-	<stop  offset="1" style="stop-color:#22B1BC"/>
-</linearGradient>
-<path style="fill-rule:evenodd;clip-rule:evenodd;fill:url(#SVGID_1_);" d="M11.469,4l11.39,10.494v-0.002V4.024h2.217v10.517
-	L36.518,4v11.965h4.697v17.258h-4.683v10.654L25.077,33.813v10.18h-2.217V33.979L11.482,44V33.224H6.785V15.965h4.685V4z
-	 M21.188,18.155H9.002v12.878h2.477v-4.062L21.188,18.155z M13.699,27.943v11.17l9.16-8.068V19.623L13.699,27.943z M25.141,30.938
-	V19.612l9.163,8.321v5.291h0.012v5.775L25.141,30.938z M36.532,31.033h2.466V18.155H26.903l9.629,8.725V31.033z M34.301,15.965
-	V9.038l-7.519,6.927H34.301z M21.205,15.965h-7.519V9.038L21.205,15.965z"/>
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!-- Generator: Adobe Illustrator 26.1.0, SVG Export Plug-In . SVG Version: 6.00 Build 0)  -->
+<svg version="1.1" id="Layer_1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" x="0px" y="0px"
+	 viewBox="0 0 48 48" style="enable-background:new 0 0 48 48;" xml:space="preserve">
+<linearGradient id="SVGID_1_" gradientUnits="userSpaceOnUse" x1="10.5862" y1="1.61" x2="36.0543" y2="44.1206">
+	<stop  offset="0.002" style="stop-color:#9C55D4"/>
+	<stop  offset="0.003" style="stop-color:#20808D"/>
+	<stop  offset="0.3731" style="stop-color:#218F9B"/>
+	<stop  offset="1" style="stop-color:#22B1BC"/>
+</linearGradient>
+<path style="fill-rule:evenodd;clip-rule:evenodd;fill:url(#SVGID_1_);" d="M11.469,4l11.39,10.494v-0.002V4.024h2.217v10.517
+	L36.518,4v11.965h4.697v17.258h-4.683v10.654L25.077,33.813v10.18h-2.217V33.979L11.482,44V33.224H6.785V15.965h4.685V4z
+	 M21.188,18.155H9.002v12.878h2.477v-4.062L21.188,18.155z M13.699,27.943v11.17l9.16-8.068V19.623L13.699,27.943z M25.141,30.938
+	V19.612l9.163,8.321v5.291h0.012v5.775L25.141,30.938z M36.532,31.033h2.466V18.155H26.903l9.629,8.725V31.033z M34.301,15.965
+	V9.038l-7.519,6.927H34.301z M21.205,15.965h-7.519V9.038L21.205,15.965z"/>
 </svg>

litellm/proxy/_experimental/out/index.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[38488,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","402","static/chunks/402-cba8184a1bf36c3c.js","313","static/chunks/313-bba577664545e320.js","899","static/chunks/899-768e8bb08fd1e694.js","603","static/chunks/603-0fae74d4aee0db69.js","250","static/chunks/250-c07a7322d3091570.js","699","static/chunks/699-7660d0629950118b.js","931","static/chunks/app/page-e9d82ee77396a2eb.js"],"default",1]
+3:I[34125,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","402","static/chunks/402-239a4ed70dc393da.js","313","static/chunks/313-cf4a28394ee560d6.js","899","static/chunks/899-0459bccb48a6666d.js","966","static/chunks/966-d23842bbd48c2bc2.js","250","static/chunks/250-8d759a787d622b48.js","699","static/chunks/699-96a60f79614507d5.js","931","static/chunks/app/page-86bbb3edba7ec5f0.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["uuodwb5RNsYUyfdsOMmbB",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/20729c6c90dc7f33.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zyqLTLglamGh14G70gBJG",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/27aa365198bdc7c8.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/model_hub.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[52829,["402","static/chunks/402-cba8184a1bf36c3c.js","313","static/chunks/313-bba577664545e320.js","250","static/chunks/250-c07a7322d3091570.js","699","static/chunks/699-7660d0629950118b.js","418","static/chunks/app/model_hub/page-ce40c5a05f3174ca.js"],"default",1]
+3:I[52829,["402","static/chunks/402-239a4ed70dc393da.js","313","static/chunks/313-cf4a28394ee560d6.js","250","static/chunks/250-8d759a787d622b48.js","699","static/chunks/699-96a60f79614507d5.js","418","static/chunks/app/model_hub/page-ce40c5a05f3174ca.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["uuodwb5RNsYUyfdsOMmbB",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/20729c6c90dc7f33.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zyqLTLglamGh14G70gBJG",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/27aa365198bdc7c8.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_experimental/out/onboarding.txt

@@ -1,7 +1,7 @@
 2:I[19107,[],"ClientPageRoot"]
-3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","402","static/chunks/402-cba8184a1bf36c3c.js","899","static/chunks/899-768e8bb08fd1e694.js","250","static/chunks/250-c07a7322d3091570.js","461","static/chunks/app/onboarding/page-74706b15590d94d7.js"],"default",1]
+3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","402","static/chunks/402-239a4ed70dc393da.js","899","static/chunks/899-0459bccb48a6666d.js","250","static/chunks/250-8d759a787d622b48.js","461","static/chunks/app/onboarding/page-be52e5770ada890a.js"],"default",1]
 4:I[4707,[],""]
 5:I[36423,[],""]
-0:["uuodwb5RNsYUyfdsOMmbB",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/_next/static/css/20729c6c90dc7f33.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
+0:["zyqLTLglamGh14G70gBJG",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/31b7f215e119031e.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/27aa365198bdc7c8.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_b0dd8a","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
 6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
 1:null

litellm/proxy/_new_secret_config.yaml

@@ -3,12 +3,23 @@ model_list:
     litellm_params:
       model: codex-mini-latest
       api_key: os.environ/OPENAI_API_KEY
+  - model_name: bedrock/*
+    litellm_params:
+      model: bedrock/*
+  - model_name: eu.anthropic.claude-3-5-sonnet-20240620-v1:0
+    litellm_params:
+      model: eu.anthropic.claude-3-5-sonnet-20240620-v1:0
   - model_name: "gpt-4o-mini-openai"
     litellm_params:
       model: gpt-4o-mini
       api_key: os.environ/OPENAI_API_KEY
     model_info:
       access_groups: ["beta-models"] # 👈 Model Access Group
+  - model_name: azure_ai/Phi-3-medium
+    litellm_params:
+      model: azure_ai/Phi-3-medium
+      api_key: os.environ/AZURE_AI_PHI_3_MEDIUM_API_KEY
+      api_base: os.environ/AZURE_AI_PHI_3_MEDIUM_API_BASE
   - model_name: "bedrock-nova"
     litellm_params:
       model: us.amazon.nova-pro-v1:0
@@ -106,7 +117,3 @@ litellm_settings:
   #   supported_call_types: ["acompletion", "completion"]
   callbacks: ["prometheus", "langfuse"]
     
-
-
-
-

litellm/proxy/_types.py

@@ -17,6 +17,13 @@ from typing_extensions import Required, TypedDict
 
 from litellm.types.integrations.slack_alerting import AlertType
 from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject
+from litellm.types.mcp import (
+    MCPAuthType,
+    MCPSpecVersion,
+    MCPSpecVersionType,
+    MCPTransport,
+    MCPTransportType,
+)
 from litellm.types.router import RouterErrors, UpdateRouterConfig
 from litellm.types.utils import (
     CallTypes,
@@ -830,32 +837,6 @@ class SpecialMCPServerName(str, enum.Enum):
     all_team_servers = "all-team-mcpservers"
     all_proxy_servers = "all-proxy-mcpservers"
 
-
-class MCPTransport(str, enum.Enum):
-    sse = "sse"
-    http = "http"
-
-
-class MCPSpecVersion(str, enum.Enum):
-    nov_2024 = "2024-11-05"
-    mar_2025 = "2025-03-26"
-
-
-class MCPAuth(str, enum.Enum):
-    none = "none"
-    api_key = "api_key"
-    bearer_token = "bearer_token"
-    basic = "basic"
-
-
-# MCP Literals
-MCPTransportType = Literal[MCPTransport.sse, MCPTransport.http]
-MCPSpecVersionType = Literal[MCPSpecVersion.nov_2024, MCPSpecVersion.mar_2025]
-MCPAuthType = Optional[
-    Literal[MCPAuth.none, MCPAuth.api_key, MCPAuth.bearer_token, MCPAuth.basic]
-]
-
-
 # MCP Proxy Request Types
 class NewMCPServerRequest(LiteLLMPydanticObjectBase):
     server_id: Optional[str] = None
@@ -893,6 +874,12 @@ class LiteLLM_MCPServerTable(LiteLLMPydanticObjectBase):
     updated_by: Optional[str] = None
 
 
+class NewUserRequestTeam(LiteLLMPydanticObjectBase):
+    team_id: str
+    max_budget_in_team: Optional[float] = None
+    user_role: Literal["user", "admin"] = "user"
+
+
 class NewUserRequest(GenerateRequestBase):
     max_budget: Optional[float] = None
     user_email: Optional[str] = None
@@ -905,7 +892,7 @@ class NewUserRequest(GenerateRequestBase):
             LitellmUserRoles.INTERNAL_USER_VIEW_ONLY,
         ]
     ] = None
-    teams: Optional[list] = None
+    teams: Optional[Union[List[str], List[NewUserRequestTeam]]] = None
     auto_create_key: bool = (
         True  # flag used for returning a key as part of the /user/new response
     )
@@ -1449,7 +1436,16 @@ class PassThroughGenericEndpoint(LiteLLMPydanticObjectBase):
         description="The URL to which requests for this path should be forwarded."
     )
     headers: dict = Field(
-        description="Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint"
+        default={},
+        description="Key-value pairs of headers to be forwarded with the request. You can set any key value pair here and it will be forwarded to your target endpoint",
+    )
+    include_subpath: bool = Field(
+        default=False,
+        description="If True, requests to subpaths of the path will be forwarded to the target endpoint. For example, if the path is /bria and include_subpath is True, requests to /bria/v1/text-to-image/base/2.3 will be forwarded to the target endpoint.",
+    )
+    cost_per_request: float = Field(
+        default=0.0,
+        description="The USD cost per request to the target endpoint. This is used to calculate the cost of the request to the target endpoint.",
     )
 
 
@@ -2688,6 +2684,7 @@ class SpecialHeaders(enum.Enum):
     google_ai_studio_authorization = "x-goog-api-key"
     azure_apim_authorization = "Ocp-Apim-Subscription-Key"
     custom_litellm_api_key = "x-litellm-api-key"
+    mcp_auth = "x-mcp-auth"
 
 
 class LitellmDataForBackendLLMCall(TypedDict, total=False):
@@ -3022,6 +3019,11 @@ class DefaultInternalUserParams(LiteLLMPydanticObjectBase):
         default=None, description="Default list of models that new users can access"
     )
 
+    teams: Optional[Union[List[str], List[NewUserRequestTeam]]] = Field(
+        default=None,
+        description="Default teams for new users created",
+    )
+
 
 class BaseDailySpendTransaction(TypedDict):
     date: str

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -1,6 +1,7 @@
 import asyncio
 import copy
 import os
+import time
 import traceback
 from datetime import datetime, timedelta
 from typing import Dict, Literal, Optional, Union
@@ -302,12 +303,120 @@ async def health_services_endpoint(  # noqa: PLR0915
         )
 
 
+def _convert_health_check_to_dict(check) -> dict:
+    """Convert health check database record to dictionary format"""
+    return {
+        "health_check_id": check.health_check_id,
+        "model_name": check.model_name,
+        "model_id": check.model_id,
+        "status": check.status,
+        "healthy_count": check.healthy_count,
+        "unhealthy_count": check.unhealthy_count,
+        "error_message": check.error_message,
+        "response_time_ms": check.response_time_ms,
+        "details": check.details,
+        "checked_by": check.checked_by,
+        "checked_at": check.checked_at.isoformat() if check.checked_at else None,
+        "created_at": check.created_at.isoformat() if check.created_at else None,
+    }
+
+
+def _check_prisma_client():
+    """Helper to check if prisma_client is available and raise appropriate error"""
+    from litellm.proxy.proxy_server import prisma_client
+    
+    if prisma_client is None:
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail={"error": "Database not initialized"},
+        )
+    return prisma_client
+
+
+async def _save_health_check_to_db(
+    prisma_client, 
+    model_name: str, 
+    healthy_endpoints: list, 
+    unhealthy_endpoints: list, 
+    start_time: float, 
+    user_id: Optional[str],
+    model_id: Optional[str] = None
+):
+    """Helper function to save health check results to database"""
+    try:
+        # Extract error message from first unhealthy endpoint if available
+        error_message = (
+            str(unhealthy_endpoints[0]["error"])[:500] 
+            if unhealthy_endpoints and unhealthy_endpoints[0].get("error") 
+            else None
+        )
+        
+        await prisma_client.save_health_check_result(
+            model_name=model_name,
+            model_id=model_id,
+            status="healthy" if healthy_endpoints else "unhealthy",
+            healthy_count=len(healthy_endpoints),
+            unhealthy_count=len(unhealthy_endpoints),
+            error_message=error_message,
+            response_time_ms=(time.time() - start_time) * 1000,
+            details=None,  # Skip details for now to avoid JSON serialization issues
+            checked_by=user_id,
+        )
+    except Exception as db_error:
+        verbose_proxy_logger.warning(f"Failed to save health check to database for model {model_name}: {db_error}")
+        # Continue execution - don't let database save failure break health checks
+
+
+async def _perform_health_check_and_save(
+    model_list,
+    target_model,
+    cli_model,
+    details,
+    prisma_client,
+    start_time,
+    user_id,
+    model_id=None
+):
+    """Helper function to perform health check and save results to database"""
+    healthy_endpoints, unhealthy_endpoints = await perform_health_check(
+        model_list=model_list, 
+        cli_model=cli_model, 
+        model=target_model,
+        details=details
+    )
+    
+    # Optionally save health check result to database (non-blocking)
+    if prisma_client is not None:
+        # For CLI model, use cli_model name; for router models, use target_model
+        model_name_for_db = cli_model if cli_model is not None else target_model
+        if model_name_for_db is not None:
+            asyncio.create_task(_save_health_check_to_db(
+                prisma_client,
+                model_name_for_db,
+                healthy_endpoints,
+                unhealthy_endpoints,
+                start_time,
+                user_id,
+                model_id=model_id
+            ))
+    
+    return {
+        "healthy_endpoints": healthy_endpoints,
+        "unhealthy_endpoints": unhealthy_endpoints,
+        "healthy_count": len(healthy_endpoints),
+        "unhealthy_count": len(unhealthy_endpoints),
+    }
+
+
 @router.get("/health", tags=["health"], dependencies=[Depends(user_api_key_auth)])
 async def health_endpoint(
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
     model: Optional[str] = fastapi.Query(
         None, description="Specify the model name (optional)"
     ),
+    model_id: Optional[str] = fastapi.Query(
+        None, description="Specify the model ID (optional)"
+    ),
 ):
     """
     🚨 USE `/health/liveliness` to health check the proxy 🚨
@@ -329,23 +438,55 @@ async def health_endpoint(
         health_check_details,
         health_check_results,
         llm_model_list,
+        llm_router,
         use_background_health_checks,
         user_model,
+        prisma_client,
     )
+    import time
 
+    start_time = time.time()
+    
+    # Handle model_id parameter - convert to model name for health check
+    target_model = model
+    if model_id and not model:
+        # Use get_deployment from router to find the model name
+        if llm_router is not None:
+            try:
+                deployment = llm_router.get_deployment(model_id=model_id)
+                if deployment is not None:
+                    target_model = deployment.model_name
+                else:
+                    raise HTTPException(
+                        status_code=status.HTTP_404_NOT_FOUND,
+                        detail={"error": f"Model with ID {model_id} not found"},
+                    )
+            except Exception as e:
+                verbose_proxy_logger.error(f"Error getting deployment for model_id {model_id}: {e}")
+                raise HTTPException(
+                    status_code=status.HTTP_404_NOT_FOUND,
+                    detail={"error": f"Model with ID {model_id} not found"},
+                )
+        else:
+            raise HTTPException(
+                status_code=status.HTTP_404_NOT_FOUND,
+                detail={"error": f"Model with ID {model_id} not found"},
+            )
+    
     try:
         if llm_model_list is None:
             # if no router set, check if user set a model using litellm --model ollama/llama2
             if user_model is not None:
-                healthy_endpoints, unhealthy_endpoints = await perform_health_check(
-                    model_list=[], cli_model=user_model, details=health_check_details
+                return await _perform_health_check_and_save(
+                    model_list=[],
+                    target_model=None,
+                    cli_model=user_model,
+                    details=health_check_details,
+                    prisma_client=prisma_client,
+                    start_time=start_time,
+                    user_id=user_api_key_dict.user_id,
+                    model_id=None  # CLI model doesn't have model_id
                 )
-                return {
-                    "healthy_endpoints": healthy_endpoints,
-                    "unhealthy_endpoints": unhealthy_endpoints,
-                    "healthy_count": len(healthy_endpoints),
-                    "unhealthy_count": len(unhealthy_endpoints),
-                }
             raise HTTPException(
                 status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
                 detail={"error": "Model list not initialized"},
@@ -359,16 +500,16 @@ async def health_endpoint(
         if use_background_health_checks:
             return health_check_results
         else:
-            healthy_endpoints, unhealthy_endpoints = await perform_health_check(
-                _llm_model_list, model, details=health_check_details
+            return await _perform_health_check_and_save(
+                model_list=_llm_model_list,
+                target_model=target_model,
+                cli_model=None,
+                details=health_check_details,
+                prisma_client=prisma_client,
+                start_time=start_time,
+                user_id=user_api_key_dict.user_id,
+                model_id=model_id
             )
-
-            return {
-                "healthy_endpoints": healthy_endpoints,
-                "unhealthy_endpoints": unhealthy_endpoints,
-                "healthy_count": len(healthy_endpoints),
-                "unhealthy_count": len(unhealthy_endpoints),
-            }
     except Exception as e:
         verbose_proxy_logger.error(
             "litellm.proxy.proxy_server.py::health_endpoint(): Exception occured - {}".format(
@@ -379,6 +520,86 @@ async def health_endpoint(
         raise e
 
 
+@router.get("/health/history", tags=["health"], dependencies=[Depends(user_api_key_auth)])
+async def health_check_history_endpoint(
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    model: Optional[str] = fastapi.Query(
+        None, description="Filter by specific model name"
+    ),
+    status_filter: Optional[str] = fastapi.Query(
+        None, description="Filter by status (healthy/unhealthy)"
+    ),
+    limit: int = fastapi.Query(
+        100, description="Number of records to return", ge=1, le=1000
+    ),
+    offset: int = fastapi.Query(
+        0, description="Number of records to skip", ge=0
+    ),
+):
+    """
+    Get health check history for models
+    
+    Returns historical health check data with optional filtering.
+    """
+    prisma_client = _check_prisma_client()
+    
+    try:
+        history = await prisma_client.get_health_check_history(
+            model_name=model,
+            limit=limit,
+            offset=offset,
+            status_filter=status_filter,
+        )
+        
+        # Convert to dict format for JSON response using helper function
+        history_data = [_convert_health_check_to_dict(check) for check in history]
+        
+        return {
+            "health_checks": history_data,
+            "total_records": len(history_data),
+            "limit": limit,
+            "offset": offset,
+        }
+    except Exception as e:
+        verbose_proxy_logger.error(f"Error getting health check history: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail={"error": f"Failed to retrieve health check history: {str(e)}"},
+        )
+
+
+@router.get("/health/latest", tags=["health"], dependencies=[Depends(user_api_key_auth)])
+async def latest_health_checks_endpoint(
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    Get the latest health check status for all models
+    
+    Returns the most recent health check result for each model.
+    """
+    prisma_client = _check_prisma_client()
+    
+    try:
+        latest_checks = await prisma_client.get_all_latest_health_checks()
+        
+        # Convert to dict format for JSON response using helper function
+        checks_data = {
+            (check.model_id if check.model_id else check.model_name): _convert_health_check_to_dict(check)
+            for check in latest_checks
+        }
+        
+        return {
+            "latest_health_checks": checks_data,
+            "total_models": len(checks_data),
+        }
+    except Exception as e:
+        verbose_proxy_logger.error(f"Error getting latest health checks: {e}")
+        raise HTTPException(
+            status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
+            detail={"error": f"Failed to retrieve latest health checks: {str(e)}"},
+        )
+
+
 db_health_cache = {"status": "unknown", "last_updated": datetime.now()}
 
 

litellm/proxy/litellm_pre_call_utils.py

@@ -390,11 +390,11 @@ class LiteLLMProxyRequestSetup:
 
         ## KEY-LEVEL SPEND LOGS / TAGS
         if "tags" in key_metadata and key_metadata["tags"] is not None:
-            data[_metadata_variable_name][
-                "tags"
-            ] = LiteLLMProxyRequestSetup._merge_tags(
-                request_tags=data[_metadata_variable_name].get("tags"),
-                tags_to_add=key_metadata["tags"],
+            data[_metadata_variable_name]["tags"] = (
+                LiteLLMProxyRequestSetup._merge_tags(
+                    request_tags=data[_metadata_variable_name].get("tags"),
+                    tags_to_add=key_metadata["tags"],
+                )
             )
         if "spend_logs_metadata" in key_metadata and isinstance(
             key_metadata["spend_logs_metadata"], dict
@@ -483,23 +483,6 @@ class LiteLLMProxyRequestSetup:
                 tags = [tag.strip() for tag in _tags]
             elif isinstance(headers["x-litellm-tags"], list):
                 tags = headers["x-litellm-tags"]
-        if "user-agent" in headers:
-            """
-            Allow tracking spend by cli tools like Claude Code - e.g. "claude-cli/1.0.25 (external, cli)"
-            """
-            # add user-agent to tags
-            if tags is None:
-                tags = []
-            user_agent = headers["user-agent"]
-            if user_agent is not None:
-                user_agent_part: Optional[str] = None
-                if "/" in user_agent:
-                    user_agent_part = user_agent.split("/")[
-                        0
-                    ]  # extract "claude-cli" - enables spend tracking acrosss versions
-                if user_agent_part is not None:
-                    tags.append(user_agent_part)
-                tags.append(user_agent)  # append full user-agent
         # Check request body for tags
         if "tags" in data and isinstance(data["tags"], list):
             tags = data["tags"]
@@ -630,9 +613,9 @@ async def add_litellm_data_to_request(  # noqa: PLR0915
     data[_metadata_variable_name]["litellm_api_version"] = version
 
     if general_settings is not None:
-        data[_metadata_variable_name][
-            "global_max_parallel_requests"
-        ] = general_settings.get("global_max_parallel_requests", None)
+        data[_metadata_variable_name]["global_max_parallel_requests"] = (
+            general_settings.get("global_max_parallel_requests", None)
+        )
 
     ### KEY-LEVEL Controls
     key_metadata = user_api_key_dict.metadata

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -55,11 +55,13 @@ router = APIRouter()
 def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> dict:
     if "user_id" in data_json and data_json["user_id"] is None:
         data_json["user_id"] = str(uuid.uuid4())
+
     auto_create_key = data_json.pop("auto_create_key", True)
+
     if auto_create_key is False:
-        data_json[
-            "table_name"
-        ] = "user"  # only create a user, don't create key if 'auto_create_key' set to False
+        data_json["table_name"] = (
+            "user"  # only create a user, don't create key if 'auto_create_key' set to False
+        )
 
     if litellm.default_internal_user_params:
         for key, value in litellm.default_internal_user_params.items():
@@ -91,6 +93,7 @@ def _update_internal_new_user_params(data_json: dict, data: NewUserRequest) -> d
         ):
             data_json["budget_duration"] = litellm.internal_user_budget_duration
 
+    data_json.pop("teams", None)  # handled separately
     return data_json
 
 
@@ -158,6 +161,88 @@ async def _add_user_to_organizations(
     await asyncio.gather(*tasks, return_exceptions=True)
 
 
+async def _add_user_to_team(
+    user_id: str,
+    team_id: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    user_email: Optional[str] = None,
+    max_budget_in_team: Optional[float] = None,
+    user_role: Literal["user", "admin"] = "user",
+):
+    from litellm.proxy.management_endpoints.team_endpoints import team_member_add
+
+    try:
+        await team_member_add(
+            data=TeamMemberAddRequest(
+                team_id=team_id,
+                member=Member(
+                    user_id=user_id,
+                    role=user_role,
+                    user_email=user_email,
+                ),
+                max_budget_in_team=max_budget_in_team,
+            ),
+            user_api_key_dict=user_api_key_dict,
+        )
+    except HTTPException as e:
+        if e.status_code == 400 and (
+            "already exists" in str(e) or "doesn't exist" in str(e)
+        ):
+            verbose_proxy_logger.debug(
+                "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {}".format(
+                    str(e)
+                )
+            )
+        else:
+            verbose_proxy_logger.debug(
+                "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): Exception occured - {}".format(
+                    str(e)
+                )
+            )
+    except Exception as e:
+        if "already exists" in str(e) or "doesn't exist" in str(e):
+            verbose_proxy_logger.debug(
+                "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {}".format(
+                    str(e)
+                )
+            )
+        elif (
+            isinstance(e, ProxyException)
+            and ProxyErrorTypes.team_member_already_in_team in e.type
+        ):
+            verbose_proxy_logger.debug(
+                "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {}".format(
+                    str(e)
+                )
+            )
+        else:
+            raise e
+
+
+def check_if_default_team_set() -> Optional[Union[List[str], List[NewUserRequestTeam]]]:
+    if litellm.default_internal_user_params is None:
+        return None
+    teams = litellm.default_internal_user_params.get("teams")
+    if teams is not None:
+        if all(isinstance(team, str) for team in teams):
+            return teams
+        elif all(isinstance(team, dict) for team in teams):
+            return [
+                NewUserRequestTeam(
+                    team_id=team.get("team_id"),
+                    max_budget_in_team=team.get("max_budget_in_team"),
+                    user_role=team.get("user_role", "user"),
+                )
+                for team in teams
+            ]
+        else:
+            verbose_proxy_logger.error(
+                "Invalid team type in default internal user params: %s",
+                teams,
+            )
+    return None
+
+
 @router.post(
     "/user/new",
     tags=["Internal User management"],
@@ -252,6 +337,9 @@ async def new_user(
 
         data_json = data.json()  # type: ignore
         data_json = _update_internal_new_user_params(data_json, data)
+        teams = data.teams
+        if teams is None:
+            teams = check_if_default_team_set()
         organization_ids = cast(
             Optional[List[str]], data_json.pop("organizations", None)
         )
@@ -260,47 +348,41 @@ async def new_user(
         # Admin UI Logic
         # Add User to Team and Organization
         # if team_id passed add this user to the team
-        if data_json.get("team_id", None) is not None:
-            from litellm.proxy.management_endpoints.team_endpoints import (
-                team_member_add,
+        _team_id = data_json.get("team_id", None)
+        if _team_id is not None:
+            await _add_user_to_team(
+                user_id=cast(str, response.get("user_id")),
+                team_id=_team_id,
+                user_api_key_dict=user_api_key_dict,
+                user_email=data.user_email,
+                max_budget_in_team=None,
+                user_role="user",
             )
+        elif teams is not None:
+            tasks = []
+            for team in teams:
+                max_budget_in_team: Optional[float] = None
+                user_role: Literal["user", "admin"] = "user"
+                if isinstance(team, str):
+                    team_id = team
+                elif isinstance(team, NewUserRequestTeam):
+                    team_id = team.team_id
+                    max_budget_in_team = team.max_budget_in_team
+                    user_role = team.user_role
+                else:
+                    raise ValueError(f"Invalid team type: {type(team)}")
 
-            try:
-                await team_member_add(
-                    data=TeamMemberAddRequest(
-                        team_id=data_json.get("team_id", None),
-                        member=Member(
-                            user_id=data_json.get("user_id", None),
-                            role="user",
-                            user_email=data_json.get("user_email", None),
-                        ),
-                    ),
-                    user_api_key_dict=user_api_key_dict,
+                tasks.append(
+                    _add_user_to_team(
+                        user_id=cast(str, response.get("user_id")),
+                        team_id=team_id,
+                        user_email=data.user_email,
+                        user_api_key_dict=user_api_key_dict,
+                        max_budget_in_team=max_budget_in_team,
+                        user_role=user_role,
+                    )
                 )
-            except HTTPException as e:
-                if e.status_code == 400 and (
-                    "already exists" in str(e) or "doesn't exist" in str(e)
-                ):
-                    verbose_proxy_logger.debug(
-                        "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {}".format(
-                            str(e)
-                        )
-                    )
-                else:
-                    verbose_proxy_logger.debug(
-                        "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): Exception occured - {}".format(
-                            str(e)
-                        )
-                    )
-            except Exception as e:
-                if "already exists" in str(e) or "doesn't exist" in str(e):
-                    verbose_proxy_logger.debug(
-                        "litellm.proxy.management_endpoints.internal_user_endpoints.new_user(): User already exists in team - {}".format(
-                            str(e)
-                        )
-                    )
-                else:
-                    raise e
+            await asyncio.gather(*tasks, return_exceptions=True)
 
         user_id = cast(Optional[str], response.get("user_id", None))
 
@@ -676,9 +758,9 @@ def _update_internal_user_params(data_json: dict, data: UpdateUserRequest) -> di
         "budget_duration" not in non_default_values
     ):  # applies internal user limits, if user role updated
         if is_internal_user and litellm.internal_user_budget_duration is not None:
-            non_default_values[
-                "budget_duration"
-            ] = litellm.internal_user_budget_duration
+            non_default_values["budget_duration"] = (
+                litellm.internal_user_budget_duration
+            )
             from litellm.proxy.common_utils.timezone_utils import get_budget_reset_time
 
             non_default_values["budget_reset_at"] = get_budget_reset_time(
@@ -1322,13 +1404,13 @@ async def ui_view_users(
             }
 
         # Query users with pagination and filters
-        users: Optional[
-            List[BaseModel]
-        ] = await prisma_client.db.litellm_usertable.find_many(
-            where=where_conditions,
-            skip=skip,
-            take=page_size,
-            order={"created_at": "desc"},
+        users: Optional[List[BaseModel]] = (
+            await prisma_client.db.litellm_usertable.find_many(
+                where=where_conditions,
+                skip=skip,
+                take=page_size,
+                order={"created_at": "desc"},
+            )
         )
 
         if not users:

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -91,16 +91,48 @@ if MCP_AVAILABLE:
         --header 'Authorization: Bearer your_api_key_here'
         ```
         """
+        from datetime import datetime
+
         prisma_client = get_prisma_client_or_throw(
             "Database not connected. Connect a database to your proxy"
         )
 
+        LIST_MCP_SERVERS: List[LiteLLM_MCPServerTable] = []
+
         # perform authz check to filter the mcp servers user has access to
         if _user_has_admin_view(user_api_key_dict):
-            return await get_all_mcp_servers(prisma_client)
+            LIST_MCP_SERVERS = await get_all_mcp_servers(prisma_client)
+        else:
+            # Find all mcp servers the user has access to
+            LIST_MCP_SERVERS = await get_all_mcp_servers_for_user(
+                prisma_client, user_api_key_dict
+            )
 
-        # Find all mcp servers the user has access to
-        return await get_all_mcp_servers_for_user(prisma_client, user_api_key_dict)
+        #########################################################
+        # Allowed MCP Servers from config.yaml
+        #########################################################
+        ALLOWED_MCP_SERVER_IDS = (
+            await global_mcp_server_manager.get_allowed_mcp_servers(
+                user_api_key_auth=user_api_key_dict
+            )
+        )
+        ALL_CONFIG_MCP_SERVERS = global_mcp_server_manager.config_mcp_servers
+        for _server_id, _server_config in ALL_CONFIG_MCP_SERVERS.items():
+            if _server_id in ALLOWED_MCP_SERVER_IDS:
+                LIST_MCP_SERVERS.append(
+                    LiteLLM_MCPServerTable(
+                        server_id=_server_id,
+                        alias=_server_config.name,
+                        url=_server_config.url,
+                        transport=_server_config.transport,
+                        spec_version=_server_config.spec_version,
+                        auth_type=_server_config.auth_type,
+                        created_at=datetime.now(),
+                        updated_at=datetime.now(),
+                    )
+                )
+        #########################################################
+        return LIST_MCP_SERVERS
 
     @router.get(
         "/server/{server_id}",

litellm/proxy/management_endpoints/scim/scim_transformations.py

@@ -124,6 +124,8 @@ class ScimTransformations:
         # Get team members
         scim_members: List[SCIMMember] = []
         for member in team.members_with_roles or []:
+            if isinstance(member, dict):
+                member = Member(**member)
             scim_members.append(
                 SCIMMember(
                     value=ScimTransformations._get_scim_member_value(member),

litellm/proxy/management_endpoints/scim/scim_v2.py

@@ -5,7 +5,7 @@ This is an enterprise feature and requires a premium license.
 """
 
 import uuid
-from typing import List, Optional
+from typing import Any, Dict, List, Optional, Set, Tuple, TypedDict
 
 from fastapi import (
     APIRouter,
@@ -19,12 +19,15 @@ from fastapi import (
 )
 
 from litellm._logging import verbose_proxy_logger
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.proxy._types import (
     LiteLLM_UserTable,
     LitellmUserRoles,
     Member,
     NewTeamRequest,
     NewUserRequest,
+    TeamMemberAddRequest,
+    TeamMemberDeleteRequest,
     UserAPIKeyAuth,
 )
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -32,10 +35,69 @@ from litellm.proxy.management_endpoints.internal_user_endpoints import new_user
 from litellm.proxy.management_endpoints.scim.scim_transformations import (
     ScimTransformations,
 )
-from litellm.proxy.management_endpoints.team_endpoints import new_team
+from litellm.proxy.management_endpoints.team_endpoints import (
+    new_team,
+    team_member_add,
+    team_member_delete,
+)
 from litellm.proxy.utils import _premium_user_check, handle_exception_on_proxy
 from litellm.types.proxy.management_endpoints.scim_v2 import *
 
+
+class UserProvisionerHelpers:
+    """Helper methods for user provisioning operations."""
+    
+    @staticmethod
+    async def handle_existing_user_by_email(
+        prisma_client,
+        new_user_request: NewUserRequest
+    ) -> Optional[SCIMUser]:
+        """
+        Check if a user with the given email already exists and update them if found.
+        
+        Args:
+            prisma_client: Database client
+            new_user_request: New user request data
+            
+        Returns:
+            SCIMUser if user was updated, None if no existing user found
+        """
+        if not new_user_request.user_email:
+            return None
+            
+        existing_user = await prisma_client.db.litellm_usertable.find_first(
+            where={"user_email": new_user_request.user_email}
+        )
+        
+        if not existing_user:
+            return None
+            
+        # Update the user
+        updated_user = await prisma_client.db.litellm_usertable.update(
+            where={"user_id": existing_user.user_id},
+            data={
+                "user_id": new_user_request.user_id,
+                "user_email": new_user_request.user_email,
+                "user_alias": new_user_request.user_alias,
+                "teams": new_user_request.teams,
+                "metadata": safe_dumps(new_user_request.metadata),
+            },
+        )
+        
+        return await ScimTransformations.transform_litellm_user_to_scim_user(updated_user)
+
+
+class ScimUserData(TypedDict):
+    """Typed structure for extracted SCIM user data."""
+    user_email: Optional[str]
+    user_alias: Optional[str]
+    sso_user_id: Optional[str]
+    teams: List[str]
+    given_name: Optional[str]
+    family_name: Optional[str]
+    active: Optional[bool]
+
+
 scim_router = APIRouter(
     prefix="/scim/v2",
     tags=["✨ SCIM v2 (Enterprise Only)"],
@@ -43,6 +105,137 @@ scim_router = APIRouter(
 )
 
 
+# Helper functions for common operations
+async def _get_prisma_client_or_raise_exception():
+    """Check if database is connected and raise HTTPException if not."""
+    from litellm.proxy.proxy_server import prisma_client
+    
+    if prisma_client is None:
+        raise HTTPException(status_code=500, detail={"error": "No database connected"})
+    return prisma_client
+
+
+async def _check_user_exists(user_id: str):
+    """Check if user exists and return user, raise 404 if not found."""
+    prisma_client = await _get_prisma_client_or_raise_exception()
+    
+    user = await prisma_client.db.litellm_usertable.find_unique(
+        where={"user_id": user_id}
+    )
+    
+    if not user:
+        raise HTTPException(
+            status_code=404, detail={"error": f"User not found with ID: {user_id}"}
+        )
+    
+    return user
+
+
+async def _check_team_exists(team_id: str):
+    """Check if team exists and return team, raise 404 if not found."""
+    prisma_client = await _get_prisma_client_or_raise_exception()
+    
+    team = await prisma_client.db.litellm_teamtable.find_unique(
+        where={"team_id": team_id}
+    )
+    
+    if not team:
+        raise HTTPException(
+            status_code=404, detail={"error": f"Group not found with ID: {team_id}"}
+        )
+    
+    return team
+
+
+def _extract_scim_user_data(user: SCIMUser) -> ScimUserData:
+    """Extract common data from SCIMUser object."""
+    user_email = None
+    if user.emails and len(user.emails) > 0:
+        user_email = user.emails[0].value
+
+    user_alias = None
+    if user.name and user.name.givenName:
+        user_alias = user.name.givenName
+
+    teams = []
+    if user.groups:
+        teams = [group.value for group in user.groups]
+
+    return {
+        "user_email": user_email,
+        "user_alias": user_alias,
+        "sso_user_id": user.externalId,
+        "teams": teams,
+        "given_name": user.name.givenName if user.name else None,
+        "family_name": user.name.familyName if user.name else None,
+        "active": user.active,
+    }
+
+
+def _build_scim_metadata(given_name: Optional[str], family_name: Optional[str], active: Optional[bool] = None) -> Dict[str, Any]:
+    """Build metadata dictionary with SCIM data."""
+    metadata: Dict[str, Any] = {
+        "scim_metadata": LiteLLM_UserScimMetadata(
+            givenName=given_name,
+            familyName=family_name,
+        ).model_dump()
+    }
+    
+    if active is not None:
+        metadata["scim_active"] = active
+    
+    return metadata
+
+
+async def _extract_group_member_ids(group: SCIMGroup) -> List[str]:
+    """Extract valid member IDs from SCIMGroup, verifying users exist."""
+    prisma_client = await _get_prisma_client_or_raise_exception()
+    member_ids = []
+    
+    if group.members:
+        for member in group.members:
+            # Check if user exists
+            user = await prisma_client.db.litellm_usertable.find_unique(
+                where={"user_id": member.value}
+            )
+            if user:
+                member_ids.append(member.value)
+    
+    return member_ids
+
+
+async def _get_team_members_display(member_ids: List[str]) -> List[SCIMMember]:
+    """Get SCIMMember objects with display names for a list of member IDs."""
+    prisma_client = await _get_prisma_client_or_raise_exception()
+    members: List[SCIMMember] = []
+    
+    for member_id in member_ids:
+        user = await prisma_client.db.litellm_usertable.find_unique(
+            where={"user_id": member_id}
+        )
+        if user:
+            display_name = user.user_email or user.user_id
+            members.append(SCIMMember(value=user.user_id, display=display_name))
+    
+    return members
+
+
+async def _handle_team_membership_changes(user_id: str, existing_teams: List[str], new_teams: List[str]) -> None:
+    """Handle adding/removing user from teams based on changes."""
+    existing_teams_set = set(existing_teams)
+    new_teams_set = set(new_teams)
+    
+    teams_to_add = new_teams_set - existing_teams_set
+    teams_to_remove = existing_teams_set - new_teams_set
+    
+    if teams_to_add or teams_to_remove:
+        await patch_team_membership(
+            user_id=user_id,
+            teams_ids_to_add_user_to=list(teams_to_add),
+            teams_ids_to_remove_user_from=list(teams_to_remove),
+        )
+
+
 # Dependency to set the correct SCIM Content-Type
 async def set_scim_content_type(response: Response):
     """Sets the Content-Type header to application/scim+json"""
@@ -66,12 +259,8 @@ async def get_users(
     """
     Get a list of users according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
+        prisma_client = await _get_prisma_client_or_raise_exception()
         # Parse filter if provided (basic support)
         where_conditions = {}
         if filter:
@@ -129,21 +318,9 @@ async def get_user(
     """
     Get a single user by ID according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
-        user = await prisma_client.db.litellm_usertable.find_unique(
-            where={"user_id": user_id}
-        )
-
-        if not user:
-            raise HTTPException(
-                status_code=404, detail={"error": f"User not found with ID: {user_id}"}
-            )
-
+        user = await _check_user_exists(user_id)
+        
         # Convert to SCIM format
         scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(user)
         return scim_user
@@ -151,7 +328,6 @@ async def get_user(
     except Exception as e:
         raise handle_exception_on_proxy(e)
 
-
 @scim_router.post(
     "/Users",
     response_model=SCIMUser,
@@ -164,52 +340,55 @@ async def create_user(
     """
     Create a user according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
         verbose_proxy_logger.debug("SCIM CREATE USER request: %s", user)
-        # Extract email from SCIM user
-        user_email = None
-        if user.emails and len(user.emails) > 0:
-            user_email = user.emails[0].value
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        
+        # Extract data from SCIM user
+        user_data = _extract_scim_user_data(user)
 
         # Check if user already exists
-        existing_user = None
         if user.userName:
             existing_user = await prisma_client.db.litellm_usertable.find_unique(
                 where={"user_id": user.userName}
             )
-
-        if existing_user:
-            raise HTTPException(
-                status_code=409,
-                detail={"error": f"User already exists with username: {user.userName}"},
-            )
+            if existing_user:
+                raise HTTPException(
+                    status_code=409,
+                    detail={"error": f"User already exists with username: {user.userName}"},
+                )
 
         # Create user in database
         user_id = user.userName or str(uuid.uuid4())
-        created_user = await new_user(
-            data=NewUserRequest(
-                user_id=user_id,
-                user_email=user_email,
-                user_alias=user.name.givenName,
-                teams=[group.value for group in user.groups] if user.groups else None,
-                metadata={
-                    "scim_metadata": LiteLLM_UserScimMetadata(
-                        givenName=user.name.givenName,
-                        familyName=user.name.familyName,
-                    ).model_dump()
-                },
-                auto_create_key=False,
-            ),
+        metadata = _build_scim_metadata(user_data["given_name"], user_data["family_name"])
+        new_user_request = NewUserRequest(
+            user_id=user_id,
+            user_email=user_data["user_email"],
+            user_alias=user_data["user_alias"],
+            teams=user_data["teams"],
+            metadata=metadata,
+            auto_create_key=False,
         )
+
+        # Check if user with email already exists and update if found
+        existing_user_scim = await UserProvisionerHelpers.handle_existing_user_by_email(
+            prisma_client=prisma_client,
+            new_user_request=new_user_request
+        )
+        
+        if existing_user_scim:
+            return existing_user_scim
+
+        created_user = await new_user(
+            data=new_user_request,
+        )
+        
         scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(
             user=created_user
         )
         return scim_user
+    except HTTPException as e: # allow exceptions like SCIMUserAlreadyExists to be raised
+        raise e
     except Exception as e:
         raise handle_exception_on_proxy(e)
 
@@ -225,14 +404,55 @@ async def update_user(
     user: SCIMUser = Body(...),
 ):
     """
-    Update a user according to SCIM v2 protocol
+    Update a user according to SCIM v2 protocol (full replacement)
     """
-    from litellm.proxy.proxy_server import prisma_client
+    verbose_proxy_logger.debug("SCIM PUT USER request: %s", user)
 
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
     try:
-        return None
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_user = await _check_user_exists(user_id)
+
+        # Extract data from SCIM user
+        user_data = _extract_scim_user_data(user)
+
+        # Build metadata with SCIM data  
+        metadata = _build_scim_metadata(
+            user_data["given_name"], 
+            user_data["family_name"], 
+            user_data["active"]
+        )
+
+        # Handle team membership changes
+        await _handle_team_membership_changes(
+            user_id=user_id,
+            existing_teams=existing_user.teams or [],
+            new_teams=user_data["teams"]
+        )
+
+        # Update user with all new data (full replacement)
+        update_data = {
+            "user_email": user_data["user_email"],
+            "user_alias": user_data["user_alias"],
+            "sso_user_id": user_data["sso_user_id"],
+            "teams": user_data["teams"],
+            "metadata": metadata,
+        }
+
+        # Serialize metadata to JSON string for Prisma to avoid GraphQL parsing issues
+        if "metadata" in update_data and isinstance(update_data["metadata"], dict):
+            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+            update_data["metadata"] = safe_dumps(update_data["metadata"])
+
+        updated_user = await prisma_client.db.litellm_usertable.update(
+            where={"user_id": user_id},
+            data=update_data,
+        )
+
+        # Convert back to SCIM format
+        scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(updated_user)
+        
+        return scim_user
+
     except Exception as e:
         raise handle_exception_on_proxy(e)
 
@@ -248,21 +468,9 @@ async def delete_user(
     """
     Delete a user according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
-        # Check if user exists
-        existing_user = await prisma_client.db.litellm_usertable.find_unique(
-            where={"user_id": user_id}
-        )
-
-        if not existing_user:
-            raise HTTPException(
-                status_code=404, detail={"error": f"User not found with ID: {user_id}"}
-            )
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_user = await _check_user_exists(user_id)
 
         # Get teams user belongs to
         teams = []
@@ -291,6 +499,156 @@ async def delete_user(
         raise handle_exception_on_proxy(e)
 
 
+def _extract_group_values(value: Any) -> List[str]:
+    """Return group ids from a SCIM patch value."""
+    group_values: List[str] = []
+    if isinstance(value, list):
+        for v in value:
+            if isinstance(v, dict) and v.get("value"):
+                group_values.append(str(v.get("value")))
+            elif isinstance(v, str):
+                group_values.append(v)
+    elif isinstance(value, dict):
+        if value.get("value"):
+            group_values.append(str(value.get("value")))
+    elif isinstance(value, str):
+        group_values.append(value)
+    return group_values
+
+
+def _handle_displayname_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None:
+    """Handle displayname updates."""
+    if op_type == "remove":
+        update_data["user_alias"] = None
+    else:
+        update_data["user_alias"] = str(value)
+
+
+def _handle_externalid_update(op_type: str, value: Any, update_data: Dict[str, Any]) -> None:
+    """Handle externalid updates."""
+    if op_type == "remove":
+        update_data["sso_user_id"] = None
+    else:
+        update_data["sso_user_id"] = str(value)
+
+
+def _handle_active_update(op_type: str, value: Any, metadata: Dict[str, Any]) -> None:
+    """Handle active status updates."""
+    if op_type == "remove":
+        metadata.pop("scim_active", None)
+    else:
+        bool_val = value
+        if isinstance(value, str):
+            bool_val = value.lower() == "true"
+        else:
+            bool_val = bool(value)
+        metadata["scim_active"] = bool_val
+
+
+def _handle_name_update(path: str, op_type: str, value: Any, scim_metadata: Dict[str, Any]) -> None:
+    """Handle name field updates (givenName, familyName)."""
+    if path == "name.givenname":
+        if op_type == "remove":
+            scim_metadata.pop("givenName", None)
+        else:
+            scim_metadata["givenName"] = str(value)
+    elif path == "name.familyname":
+        if op_type == "remove":
+            scim_metadata.pop("familyName", None)
+        else:
+            scim_metadata["familyName"] = str(value)
+
+
+def _handle_group_operations(op_type: str, value: Any, teams_set: Set[str]) -> Optional[Set[str]]:
+    """Handle group/team membership operations."""
+    group_values = _extract_group_values(value)
+    if op_type == "replace":
+        return set(group_values)
+    elif op_type == "add":
+        teams_set.update(group_values)
+    elif op_type == "remove":
+        for gid in group_values:
+            teams_set.discard(gid)
+    return None
+
+
+def _handle_generic_metadata(path: str, op_type: str, value: Any, metadata: Dict[str, Any]) -> None:
+    """Handle generic metadata operations for unknown paths."""
+    if op_type == "remove":
+        metadata.pop(path, None)
+    else:
+        metadata[path] = value
+
+
+def _apply_patch_ops(
+    existing_user: LiteLLM_UserTable,
+    patch_ops: SCIMPatchOp,
+) -> Tuple[Dict[str, Any], Set[str]]:
+    """Apply patch operations and return update data and final team set."""
+    update_data: Dict[str, Any] = {}
+    metadata = existing_user.metadata or {}
+    scim_metadata = metadata.get("scim_metadata", {})
+
+    teams_set: Set[str] = set(existing_user.teams or [])
+    replace_team_set: Optional[Set[str]] = None
+
+    for op in patch_ops.Operations:
+        path = (op.path or "").lower()
+        value = op.value
+        op_type = op.op
+
+        if path == "displayname":
+            _handle_displayname_update(op_type, value, update_data)
+        elif path == "externalid":
+            _handle_externalid_update(op_type, value, update_data)
+        elif path == "active":
+            _handle_active_update(op_type, value, metadata)
+        elif path in ("name.givenname", "name.familyname"):
+            _handle_name_update(path, op_type, value, scim_metadata)
+        elif path.startswith("groups"):
+            new_replace_set = _handle_group_operations(op_type, value, teams_set)
+            if new_replace_set is not None:
+                replace_team_set = new_replace_set
+        else:
+            _handle_generic_metadata(path, op_type, value, metadata)
+
+    final_team_set = replace_team_set if replace_team_set is not None else teams_set
+    metadata["scim_metadata"] = scim_metadata
+    update_data["metadata"] = metadata
+    return update_data, final_team_set
+
+async def patch_team_membership(
+    user_id: str,
+    teams_ids_to_add_user_to: List[str],
+    teams_ids_to_remove_user_from: List[str],
+) -> bool:
+    """
+    Add or remove user from teams
+    """
+    for _team_id in teams_ids_to_add_user_to:
+            try:
+                await team_member_add(
+                    data=TeamMemberAddRequest(
+                        team_id=_team_id,
+                        member=Member(user_id=user_id, role="user"),
+                    ),
+                    user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+                )
+            except Exception as e:
+                verbose_proxy_logger.exception(f"Error adding user to team {_team_id}: {e}")
+
+    for _team_id in teams_ids_to_remove_user_from:
+        try:
+            await team_member_delete(
+                data=TeamMemberDeleteRequest(team_id=_team_id, user_id=user_id),
+                user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
+            )
+        except Exception as e:
+            verbose_proxy_logger.exception(f"Error removing user from team {_team_id}: {e}")
+    
+
+    return True
+
 @scim_router.patch(
     "/Users/{user_id}",
     response_model=SCIMUser,
@@ -304,25 +662,39 @@ async def patch_user(
     """
     Patch a user according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     verbose_proxy_logger.debug("SCIM PATCH USER request: %s", patch_ops)
 
     try:
-        # Check if user exists
-        existing_user = await prisma_client.db.litellm_usertable.find_unique(
-            where={"user_id": user_id}
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_user = await _check_user_exists(user_id)
+
+        update_data, final_team_set = _apply_patch_ops(
+            existing_user=existing_user,
+            patch_ops=patch_ops,
         )
 
-        if not existing_user:
-            raise HTTPException(
-                status_code=404, detail={"error": f"User not found with ID: {user_id}"}
-            )
+        # Handle team membership changes
+        await _handle_team_membership_changes(
+            user_id=user_id,
+            existing_teams=existing_user.teams or [],
+            new_teams=list(final_team_set)
+        )
 
-        return None
+        update_data["teams"] = list(final_team_set)
+
+        # Serialize metadata to JSON string for Prisma to avoid GraphQL parsing issues
+        if "metadata" in update_data and isinstance(update_data["metadata"], dict):
+            from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+            update_data["metadata"] = safe_dumps(update_data["metadata"])
+
+        updated_user = await prisma_client.db.litellm_usertable.update(
+            where={"user_id": user_id},
+            data=update_data,
+        )
+
+        scim_user = await ScimTransformations.transform_litellm_user_to_scim_user(updated_user)
+
+        return scim_user
 
     except Exception as e:
         raise handle_exception_on_proxy(e)
@@ -343,12 +715,8 @@ async def get_groups(
     """
     Get a list of groups according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
+        prisma_client = await _get_prisma_client_or_raise_exception()
         # Parse filter if provided (basic support)
         where_conditions = {}
         if filter:
@@ -373,18 +741,9 @@ async def get_groups(
         # Convert to SCIM format
         scim_groups = []
         for team in teams:
-            # Get team members
-            members = []
-            for member_id in team.members or []:
-                member = await prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": member_id}
-                )
-                if member:
-                    display_name = member.user_email or member.user_id
-                    members.append(
-                        SCIMMember(value=member.user_id, display=display_name)
-                    )
-
+            # Get team members with display names
+            members = await _get_team_members_display(team.members or [])
+            verbose_proxy_logger.debug(f"SCIM GET GROUPS members: {members}")
             team_alias = getattr(team, "team_alias", team.team_id)
             team_created_at = team.created_at.isoformat() if team.created_at else None
             team_updated_at = team.updated_at.isoformat() if team.updated_at else None
@@ -402,6 +761,7 @@ async def get_groups(
             )
             scim_groups.append(scim_group)
 
+        verbose_proxy_logger.debug(f"SCIM GET GROUPS response: {scim_groups}")
         return SCIMListResponse(
             totalResults=total_count,
             startIndex=startIndex,
@@ -425,25 +785,13 @@ async def get_group(
     """
     Get a single group by ID according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
-        team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": group_id}
-        )
-
-        if not team:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Group not found with ID: {group_id}"},
-            )
+        team = await _check_team_exists(group_id)
 
         scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(
             team
         )
+        verbose_proxy_logger.debug(f"SCIM GET GROUP response: {scim_group}")
         return scim_group
 
     except Exception as e:
@@ -462,12 +810,9 @@ async def create_group(
     """
     Create a group according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        
         # Generate ID if not provided
         team_id = group.id or str(uuid.uuid4())
 
@@ -482,16 +827,9 @@ async def create_group(
                 detail={"error": f"Group already exists with ID: {team_id}"},
             )
 
-        # Extract members
-        members_with_roles: List[Member] = []
-        if group.members:
-            for member in group.members:
-                # Check if user exists
-                user = await prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": member.value}
-                )
-                if user:
-                    members_with_roles.append(Member(user_id=member.value, role="user"))
+        # Extract valid member IDs  
+        member_ids = await _extract_group_member_ids(group)
+        members_with_roles = [Member(user_id=member_id, role="user") for member_id in member_ids]
 
         # Create team in database
         created_team = await new_team(
@@ -525,33 +863,12 @@ async def update_group(
     """
     Update a group according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
-        # Check if team exists
-        existing_team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": group_id}
-        )
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_team = await _check_team_exists(group_id)
 
-        if not existing_team:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Group not found with ID: {group_id}"},
-            )
-
-        # Extract members
-        member_ids = []
-        if group.members:
-            for member in group.members:
-                # Check if user exists
-                user = await prisma_client.db.litellm_usertable.find_unique(
-                    where={"user_id": member.value}
-                )
-                if user:
-                    member_ids.append(member.value)
+        # Extract valid member IDs
+        member_ids = await _extract_group_member_ids(group)
 
         # Update team in database
         existing_metadata = existing_team.metadata if existing_team.metadata else {}
@@ -596,14 +913,7 @@ async def update_group(
                         )
 
         # Get updated members for response
-        members = []
-        for member_id in member_ids:
-            user = await prisma_client.db.litellm_usertable.find_unique(
-                where={"user_id": member_id}
-            )
-            if user:
-                display_name = user.user_email or user.user_id
-                members.append(SCIMMember(value=user.user_id, display=display_name))
+        members = await _get_team_members_display(member_ids)
 
         team_created_at = (
             updated_team.created_at.isoformat() if updated_team.created_at else None
@@ -639,22 +949,9 @@ async def delete_group(
     """
     Delete a group according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     try:
-        # Check if team exists
-        existing_team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": group_id}
-        )
-
-        if not existing_team:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Group not found with ID: {group_id}"},
-            )
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_team = await _check_team_exists(group_id)
 
         # For each member, remove this team from their teams list
         for member_id in existing_team.members or []:
@@ -678,6 +975,122 @@ async def delete_group(
         raise handle_exception_on_proxy(e)
 
 
+async def _process_group_patch_operations(
+    patch_ops: SCIMPatchOp,
+    existing_team,
+    prisma_client
+) -> Tuple[Dict[str, Any], Set[str]]:
+    """Process patch operations for a group and return update data and final members."""
+    update_data: Dict[str, Any] = {}
+    
+    # Create a fresh copy of existing metadata to avoid Prisma issues
+    existing_metadata = existing_team.metadata or {}
+    metadata = dict(existing_metadata) if existing_metadata else {}
+    
+    # Track member changes
+    current_members = set(existing_team.members or [])
+    final_members = current_members.copy()
+    
+    # Process each patch operation
+    for op in patch_ops.Operations:
+        path = (op.path or "").lower()
+        value = op.value
+        op_type = op.op
+
+        if path == "displayname":
+            if op_type == "remove":
+                update_data["team_alias"] = None
+            else:
+                update_data["team_alias"] = str(value)
+        elif path == "externalid":
+            if op_type == "remove":
+                metadata.pop("externalId", None)
+            else:
+                metadata["externalId"] = str(value)
+        elif path.startswith("members"):
+            # Handle member operations
+            member_values = _extract_group_values(value)
+            # Validate that users exist
+            valid_members = []
+            for member_id in member_values:
+                user = await prisma_client.db.litellm_usertable.find_unique(
+                    where={"user_id": member_id}
+                )
+                if user:
+                    valid_members.append(member_id)
+            
+            if op_type == "replace":
+                final_members = set(valid_members)
+            elif op_type == "add":
+                final_members.update(valid_members)
+            elif op_type == "remove":
+                for member_id in valid_members:
+                    final_members.discard(member_id)
+        else:
+            # Handle other generic metadata
+            if op_type == "remove":
+                metadata.pop(path, None)
+            else:
+                metadata[path] = value
+
+    # Include metadata in update data if it exists
+    if metadata:
+        update_data["metadata"] = metadata
+    
+    return update_data, final_members
+
+
+async def _apply_group_patch_updates(
+    group_id: str,
+    update_data: Dict[str, Any],
+    final_members: Set[str],
+    prisma_client
+):
+    """Apply patch updates to the group in the database."""
+    # Serialize metadata if present
+    if "metadata" in update_data and isinstance(update_data["metadata"], dict):
+        update_data["metadata"] = safe_dumps(update_data["metadata"])
+    
+    # Update members list
+    update_data["members"] = list(final_members)
+
+    # Update team in database
+    updated_team = await prisma_client.db.litellm_teamtable.update(
+        where={"team_id": group_id},
+        data=update_data,
+    )
+    
+    return updated_team
+
+
+async def _handle_group_membership_changes(
+    group_id: str,
+    current_members: Set[str],
+    final_members: Set[str]
+):
+    """Handle adding/removing members from the group."""
+    members_to_add = final_members - current_members
+    members_to_remove = current_members - final_members
+    
+    verbose_proxy_logger.debug(f"members_to_add: {members_to_add}")
+    verbose_proxy_logger.debug(f"members_to_remove: {members_to_remove}")
+    
+    # Use existing helper functions for team membership changes
+    for member_id in members_to_add:
+        await patch_team_membership(
+            user_id=member_id,
+            teams_ids_to_add_user_to=[group_id],
+            teams_ids_to_remove_user_from=[],
+        )
+
+    for member_id in members_to_remove:
+        await patch_team_membership(
+            user_id=member_id,
+            teams_ids_to_add_user_to=[],
+            teams_ids_to_remove_user_from=[group_id],
+        )
+
+
 @scim_router.patch(
     "/Groups/{group_id}",
     response_model=SCIMGroup,
@@ -691,24 +1104,35 @@ async def patch_group(
     """
     Patch a group according to SCIM v2 protocol
     """
-    from litellm.proxy.proxy_server import prisma_client
-
-    if prisma_client is None:
-        raise HTTPException(status_code=500, detail={"error": "No database connected"})
-
     verbose_proxy_logger.debug("SCIM PATCH GROUP request: %s", patch_ops)
 
     try:
-        # Check if group exists
-        existing_team = await prisma_client.db.litellm_teamtable.find_unique(
-            where={"team_id": group_id}
+        prisma_client = await _get_prisma_client_or_raise_exception()
+        existing_team = await _check_team_exists(group_id)
+
+        # Process patch operations
+        update_data, final_members = await _process_group_patch_operations(
+            patch_ops, existing_team, prisma_client
+        )
+        
+        # Track current members for comparison
+        current_members = set(existing_team.members or [])
+
+        # Apply updates to the database
+        updated_team = await _apply_group_patch_updates(
+            group_id, update_data, final_members, prisma_client
         )
 
-        if not existing_team:
-            raise HTTPException(
-                status_code=404,
-                detail={"error": f"Group not found with ID: {group_id}"},
-            )
-        return None
+        # Handle user-team relationship changes
+        await _handle_group_membership_changes(
+            group_id, current_members, final_members
+        )
+
+        # Convert to SCIM format and return
+        scim_group = await ScimTransformations.transform_litellm_team_to_scim_group(
+            updated_team
+        )
+        return scim_group
+
     except Exception as e:
         raise handle_exception_on_proxy(e)

litellm/proxy/management_endpoints/tag_management_endpoints.py

@@ -1,18 +1,19 @@
 """
 TAG MANAGEMENT
 
-All /tag management endpoints 
+All /tag management endpoints
 
-/tag/new   
+/tag/new
 /tag/info
 /tag/update
 /tag/delete
 /tag/list
 """
 
+import asyncio
 import datetime
 import json
-from typing import Dict, List, Optional
+from typing import TYPE_CHECKING, Dict, List, Optional
 
 from fastapi import APIRouter, Depends, HTTPException
 
@@ -33,6 +34,10 @@ from litellm.types.tag_management import (
     TagUpdateRequest,
 )
 
+if TYPE_CHECKING:
+    from litellm import Router
+    from litellm.types.router import Deployment
+
 router = APIRouter()
 
 
@@ -111,6 +116,33 @@ async def _save_tags_config(prisma_client, tags_config: Dict[str, TagConfig]):
         )
 
 
+async def get_deployments_by_model(
+    model: str, llm_router: "Router"
+) -> List["Deployment"]:
+    """
+    Get all deployments by model
+    """
+    from litellm.types.router import Deployment, LiteLLM_Params, ModelInfo
+
+    # Check if model id
+    deployment = llm_router.get_deployment(model_id=model)
+    if deployment is not None:
+        return [deployment]
+
+    # Check if model name
+    deployments = llm_router.get_model_list(model_name=model)
+    if deployments is None:
+        return []
+    return [
+        Deployment(
+            model_name=deployment["model_name"],
+            litellm_params=LiteLLM_Params(**deployment["litellm_params"]),  # type: ignore
+            model_info=ModelInfo(**deployment.get("model_info") or {}),
+        )
+        for deployment in deployments
+    ]
+
+
 @router.post(
     "/tag/new",
     tags=["tag management"],
@@ -126,12 +158,19 @@ async def new_tag(
     Parameters:
     - name: str - The name of the tag
     - description: Optional[str] - Description of what this tag represents
-    - models: List[str] - List of LLM models allowed for this tag
+    - models: List[str] - List of either 'model_id' or 'model_name' allowed for this tag
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy._types import CommonProxyErrors
+    from litellm.proxy.proxy_server import llm_router, prisma_client
 
     if prisma_client is None:
-        raise HTTPException(status_code=500, detail="Database not connected")
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
+        )
+    if llm_router is None:
+        raise HTTPException(
+            status_code=500, detail=CommonProxyErrors.no_llm_router.value
+        )
     try:
         # Get existing tags config
         tags_config = await _get_tags_config(prisma_client)
@@ -160,11 +199,19 @@ async def new_tag(
 
         # Update models with new tag
         if tag.models:
-            for model_id in tag.models:
-                await _add_tag_to_deployment(
-                    model_id=model_id,
-                    tag=tag.name,
+            tasks = []
+            for model in tag.models:
+                deployments = await get_deployments_by_model(model, llm_router)
+                tasks.extend(
+                    [
+                        _add_tag_to_deployment(
+                            deployment=deployment,
+                            tag=tag.name,
+                        )
+                        for deployment in deployments
+                    ]
                 )
+            await asyncio.gather(*tasks)
 
         # Get model names for response
         model_info = await _get_model_names(prisma_client, tag.models or [])
@@ -179,27 +226,26 @@ async def new_tag(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-async def _add_tag_to_deployment(model_id: str, tag: str):
+async def _add_tag_to_deployment(deployment: "Deployment", tag: str):
     """Helper function to add tag to deployment"""
     from litellm.proxy.proxy_server import prisma_client
 
     if prisma_client is None:
         raise HTTPException(status_code=500, detail="Database not connected")
 
-    deployment = await prisma_client.db.litellm_proxymodeltable.find_unique(
-        where={"model_id": model_id}
-    )
-    if deployment is None:
-        raise HTTPException(status_code=404, detail=f"Deployment {model_id} not found")
-
     litellm_params = deployment.litellm_params
     if "tags" not in litellm_params:
         litellm_params["tags"] = []
     litellm_params["tags"].append(tag)
-    await prisma_client.db.litellm_proxymodeltable.update(
-        where={"model_id": model_id},
-        data={"litellm_params": safe_dumps(litellm_params)},
-    )
+
+    try:
+        await prisma_client.db.litellm_proxymodeltable.update(
+            where={"model_id": deployment.model_info.id},
+            data={"litellm_params": safe_dumps(litellm_params)},
+        )
+    except Exception as e:
+        verbose_proxy_logger.exception(f"Error adding tag to deployment: {str(e)}")
+        raise HTTPException(status_code=500, detail=str(e))
 
 
 @router.post(

litellm/proxy/pass_through_endpoints/pass_through_endpoints.py

@@ -1,17 +1,19 @@
 import ast
 import asyncio
+import copy
 import json
 import traceback
 import uuid
 from base64 import b64encode
 from datetime import datetime
-from typing import List, Optional, Tuple, Union
+from typing import Dict, List, Optional, Tuple, Union
 from urllib.parse import urlencode, urlparse
 
 import httpx
 from fastapi import (
     APIRouter,
     Depends,
+    FastAPI,
     HTTPException,
     Request,
     Response,
@@ -413,10 +415,10 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
 
         for field_name, field_value in form_data.items():
             if isinstance(field_value, (StarletteUploadFile, UploadFile)):
-                files[
-                    field_name
-                ] = await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
-                    upload_file=field_value
+                files[field_name] = (
+                    await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
+                        upload_file=field_value
+                    )
                 )
             else:
                 form_data_dict[field_name] = field_value
@@ -467,18 +469,52 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
         kwargs = {
             "litellm_params": {
                 "metadata": _metadata,
+                "proxy_server_request": {
+                        "url": str(request.url),
+                        "method": request.method,
+                        "body": copy.copy(_parsed_body),  # use copy instead of deepcopy
+                    }
             },
             "call_type": "pass_through_endpoint",
             "litellm_call_id": litellm_call_id,
             "passthrough_logging_payload": passthrough_logging_payload,
         }
 
-        logging_obj.model_call_details[
-            "passthrough_logging_payload"
-        ] = passthrough_logging_payload
+        logging_obj.model_call_details["passthrough_logging_payload"] = (
+            passthrough_logging_payload
+        )
 
         return kwargs
 
+    @staticmethod
+    def construct_target_url_with_subpath(
+        base_target: str, subpath: str, include_subpath: Optional[bool]
+    ) -> str:
+        """
+        Helper function to construct the full target URL with subpath handling.
+
+        Args:
+            base_target: The base target URL
+            subpath: The captured subpath from the request
+            include_subpath: Whether to include the subpath in the target URL
+
+        Returns:
+            The constructed full target URL
+        """
+        if not include_subpath:
+            return base_target
+
+        if not subpath:
+            return base_target
+
+        # Ensure base_target ends with / and subpath doesn't start with /
+        if not base_target.endswith("/"):
+            base_target = base_target + "/"
+        if subpath.startswith("/"):
+            subpath = subpath[1:]
+
+        return base_target + subpath
+
 
 async def pass_through_request(  # noqa: PLR0915
     request: Request,
@@ -490,9 +526,22 @@ async def pass_through_request(  # noqa: PLR0915
     merge_query_params: Optional[bool] = False,
     query_params: Optional[dict] = None,
     stream: Optional[bool] = None,
+    cost_per_request: Optional[float] = None,
 ):
     """
     Pass through endpoint handler, makes the httpx request for pass-through endpoints and ensures logging hooks are called
+
+    Args:
+        request: The incoming request
+        target: The target URL
+        custom_headers: The custom headers
+        user_api_key_dict: The user API key dictionary
+        custom_body: The custom body
+        forward_headers: Whether to forward headers
+        merge_query_params: Whether to merge query params
+        query_params: The query params
+        stream: Whether to stream the response
+        cost_per_request: Optional field - cost per request to the target endpoint
     """
     from litellm.litellm_core_utils.litellm_logging import Logging
     from litellm.proxy.proxy_server import proxy_logging_obj
@@ -570,6 +619,7 @@ async def pass_through_request(  # noqa: PLR0915
             url=str(url),
             request_body=_parsed_body,
             request_method=getattr(request, "method", None),
+            cost_per_request=cost_per_request,
         )
         kwargs = HttpPassThroughEndpointHelpers._init_kwargs_for_pass_through_endpoint(
             user_api_key_dict=user_api_key_dict,
@@ -818,6 +868,8 @@ def create_pass_through_route(
     _forward_headers: Optional[bool] = False,
     _merge_query_params: Optional[bool] = False,
     dependencies: Optional[List] = None,
+    include_subpath: Optional[bool] = False,
+    cost_per_request: Optional[float] = None,
 ):
     # check if target is an adapter.py or a url
     import uuid
@@ -836,6 +888,7 @@ def create_pass_through_route(
             request: Request,
             fastapi_response: Response,
             user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+            subpath: str = "",  # captures sub-paths when include_subpath=True
         ):
             return await chat_completion_pass_through_endpoint(
                 fastapi_response=fastapi_response,
@@ -856,10 +909,18 @@ def create_pass_through_route(
             stream: Optional[
                 bool
             ] = None,  # if pass-through endpoint is a streaming request
+            subpath: str = "",  # captures sub-paths when include_subpath=True
         ):
+            # Construct the full target URL with subpath if needed
+            full_target = (
+                HttpPassThroughEndpointHelpers.construct_target_url_with_subpath(
+                    base_target=target, subpath=subpath, include_subpath=include_subpath
+                )
+            )
+
             return await pass_through_request(  # type: ignore
                 request=request,
-                target=target,
+                target=full_target,
                 custom_headers=custom_headers or {},
                 user_api_key_dict=user_api_key_dict,
                 forward_headers=_forward_headers,
@@ -867,6 +928,7 @@ def create_pass_through_route(
                 query_params=query_params,
                 stream=stream,
                 custom_body=custom_body,
+                cost_per_request=cost_per_request,
             )
 
     return endpoint_func
@@ -879,14 +941,99 @@ def _is_streaming_response(response: httpx.Response) -> bool:
     return False
 
 
-async def initialize_pass_through_endpoints(pass_through_endpoints: list):
+class InitPassThroughEndpointHelpers:
+    @staticmethod
+    def add_exact_path_route(
+        app: FastAPI,
+        path: str,
+        target: str,
+        custom_headers: Optional[dict],
+        forward_headers: Optional[bool],
+        merge_query_params: Optional[bool],
+        dependencies: Optional[List],
+        cost_per_request: Optional[float],
+    ):
+        """Add exact path route for pass-through endpoint"""
+        verbose_proxy_logger.debug(
+            "adding exact pass through endpoint: %s, dependencies: %s",
+            path,
+            dependencies,
+        )
+
+        app.add_api_route(
+            path=path,
+            endpoint=create_pass_through_route(
+                path,
+                target,
+                custom_headers,
+                forward_headers,
+                merge_query_params,
+                dependencies,
+                cost_per_request=cost_per_request,
+            ),
+            methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+            dependencies=dependencies,
+        )
+
+    @staticmethod
+    def add_subpath_route(
+        app: FastAPI,
+        path: str,
+        target: str,
+        custom_headers: Optional[dict],
+        forward_headers: Optional[bool],
+        merge_query_params: Optional[bool],
+        dependencies: Optional[List],
+        cost_per_request: Optional[float],
+    ):
+        """Add wildcard route for sub-paths"""
+        wildcard_path = f"{path}/{{subpath:path}}"
+        verbose_proxy_logger.debug(
+            "adding wildcard pass through endpoint: %s, dependencies: %s",
+            wildcard_path,
+            dependencies,
+        )
+
+        app.add_api_route(
+            path=wildcard_path,
+            endpoint=create_pass_through_route(
+                path,
+                target,
+                custom_headers,
+                forward_headers,
+                merge_query_params,
+                dependencies,
+                include_subpath=True,
+                cost_per_request=cost_per_request,
+            ),
+            methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+            dependencies=dependencies,
+        )
+
+
+async def initialize_pass_through_endpoints(
+    pass_through_endpoints: Union[List[Dict], List[PassThroughGenericEndpoint]],
+):
+    """
+    Initialize a list of pass-through endpoints by adding them to the FastAPI app routes
+
+    Args:
+        pass_through_endpoints: List of pass-through endpoints to initialize
+
+    Returns:
+        None
+    """
     verbose_proxy_logger.debug("initializing pass through endpoints")
     from litellm.proxy._types import CommonProxyErrors, LiteLLMRoutes
     from litellm.proxy.proxy_server import app, premium_user
 
     for endpoint in pass_through_endpoints:
+        if isinstance(endpoint, PassThroughGenericEndpoint):
+            endpoint = endpoint.model_dump()
         _target = endpoint.get("target", None)
-        _path = endpoint.get("path", None)
+        _path: Optional[str] = endpoint.get("path", None)
+        if _path is None:
+            raise ValueError("Path is required for pass-through endpoint")
         _custom_headers = endpoint.get("headers", None)
         _custom_headers = await set_env_variables_in_header(
             custom_headers=_custom_headers
@@ -908,55 +1055,53 @@ async def initialize_pass_through_endpoints(pass_through_endpoints: list):
         if _target is None:
             continue
 
-        verbose_proxy_logger.debug(
-            "adding pass through endpoint: %s, dependencies: %s", _path, _dependencies
-        )
-        app.add_api_route(  # type: ignore
+        # Add exact path route
+        verbose_proxy_logger.debug("Initializing pass through endpoint: %s", _path)
+        InitPassThroughEndpointHelpers.add_exact_path_route(
+            app=app,
             path=_path,
-            endpoint=create_pass_through_route(  # type: ignore
-                _path,
-                _target,
-                _custom_headers,
-                _forward_headers,
-                _merge_query_params,
-                _dependencies,
-            ),
-            methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
+            target=_target,
+            custom_headers=_custom_headers,
+            forward_headers=_forward_headers,
+            merge_query_params=_merge_query_params,
             dependencies=_dependencies,
+            cost_per_request=endpoint.get("cost_per_request", None),
         )
 
+        # Add wildcard route for sub-paths
+        if endpoint.get("include_subpath", False) is True:
+            InitPassThroughEndpointHelpers.add_subpath_route(
+                app=app,
+                path=_path,
+                target=_target,
+                custom_headers=_custom_headers,
+                forward_headers=_forward_headers,
+                merge_query_params=_merge_query_params,
+                dependencies=_dependencies,
+                cost_per_request=endpoint.get("cost_per_request", None),
+            )
+
         verbose_proxy_logger.debug("Added new pass through endpoint: %s", _path)
 
 
-@router.get(
-    "/config/pass_through_endpoint",
-    dependencies=[Depends(user_api_key_auth)],
-    response_model=PassThroughEndpointResponse,
-)
-async def get_pass_through_endpoints(
+async def _get_pass_through_endpoints_from_db(
     endpoint_id: Optional[str] = None,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
-):
-    """
-    GET configured pass through endpoint.
-
-    If no endpoint_id given, return all configured endpoints.
-    """
+) -> List[PassThroughGenericEndpoint]:
     from litellm.proxy.proxy_server import get_config_general_settings
 
-    ## Get existing pass-through endpoint field value
     try:
         response: ConfigFieldInfo = await get_config_general_settings(
             field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
         )
     except Exception:
-        return PassThroughEndpointResponse(endpoints=[])
+        return []
 
     pass_through_endpoint_data: Optional[List] = response.field_value
     if pass_through_endpoint_data is None:
-        return PassThroughEndpointResponse(endpoints=[])
+        return []
 
-    returned_endpoints = []
+    returned_endpoints: List[PassThroughGenericEndpoint] = []
     if endpoint_id is None:
         for endpoint in pass_through_endpoint_data:
             if isinstance(endpoint, dict):
@@ -973,19 +1118,115 @@ async def get_pass_through_endpoints(
 
             if _endpoint is not None and _endpoint.path == endpoint_id:
                 returned_endpoints.append(_endpoint)
+    return returned_endpoints
 
-    return PassThroughEndpointResponse(endpoints=returned_endpoints)
+
+@router.get(
+    "/config/pass_through_endpoint",
+    dependencies=[Depends(user_api_key_auth)],
+    response_model=PassThroughEndpointResponse,
+)
+async def get_pass_through_endpoints(
+    endpoint_id: Optional[str] = None,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
+    """
+    GET configured pass through endpoint.
+
+    If no endpoint_id given, return all configured endpoints.
+    """  ## Get existing pass-through endpoint field value
+    pass_through_endpoints = await _get_pass_through_endpoints_from_db(
+        endpoint_id=endpoint_id, user_api_key_dict=user_api_key_dict
+    )
+    return PassThroughEndpointResponse(endpoints=pass_through_endpoints)
 
 
 @router.post(
     "/config/pass_through_endpoint/{endpoint_id}",
     dependencies=[Depends(user_api_key_auth)],
 )
-async def update_pass_through_endpoints(request: Request, endpoint_id: str):
+async def update_pass_through_endpoints(
+    endpoint_id: str,
+    data: PassThroughGenericEndpoint,
+    user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+):
     """
     Update a pass-through endpoint
     """
-    pass
+    from litellm.proxy.proxy_server import (
+        get_config_general_settings,
+        update_config_general_settings,
+    )
+
+    ## Get existing pass-through endpoint field value
+    try:
+        response: ConfigFieldInfo = await get_config_general_settings(
+            field_name="pass_through_endpoints", user_api_key_dict=user_api_key_dict
+        )
+    except Exception:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": "No pass-through endpoints found"},
+        )
+
+    pass_through_endpoint_data: Optional[List] = response.field_value
+    if pass_through_endpoint_data is None:
+        raise HTTPException(
+            status_code=404,
+            detail={"error": "No pass-through endpoints found"},
+        )
+
+    # Find and update the endpoint
+    updated_endpoint: Optional[PassThroughGenericEndpoint] = None
+    endpoint_found = False
+    
+    for idx, endpoint in enumerate(pass_through_endpoint_data):
+        _endpoint: Optional[PassThroughGenericEndpoint] = None
+        if isinstance(endpoint, dict):
+            _endpoint = PassThroughGenericEndpoint(**endpoint)
+        elif isinstance(endpoint, PassThroughGenericEndpoint):
+            _endpoint = endpoint
+
+        if _endpoint is not None and _endpoint.path == endpoint_id:
+            endpoint_found = True
+            # Get the update data as dict, excluding None values for partial updates
+            update_data = data.model_dump(exclude_none=True)
+            
+            # Start with existing endpoint data
+            endpoint_dict = _endpoint.model_dump()
+            
+            # Update with new data (only non-None values)
+            endpoint_dict.update(update_data)
+            
+            # Ensure the path stays the same (can't change the endpoint_id)
+            endpoint_dict["path"] = endpoint_id
+            
+            # Create updated endpoint object
+            updated_endpoint = PassThroughGenericEndpoint(**endpoint_dict)
+            
+            # Update the list
+            pass_through_endpoint_data[idx] = endpoint_dict
+            break
+
+    if not endpoint_found:
+        raise HTTPException(
+            status_code=404,
+            detail={
+                "error": f"Endpoint with path '{endpoint_id}' not found"
+            },
+        )
+
+    ## Update db
+    updated_data = ConfigFieldUpdate(
+        field_name="pass_through_endpoints",
+        field_value=pass_through_endpoint_data,
+        config_type="general_settings",
+    )
+    await update_config_general_settings(
+        data=updated_data, user_api_key_dict=user_api_key_dict
+    )
+
+    return PassThroughEndpointResponse(endpoints=[updated_endpoint] if updated_endpoint else [])
 
 
 @router.post(
@@ -1107,3 +1348,13 @@ async def delete_pass_through_endpoints(
             },
         )
     return PassThroughEndpointResponse(endpoints=[response_obj])
+
+
+async def initialize_pass_through_endpoints_in_db():
+    """
+    Gets all pass-through endpoints from db and initializes them in the proxy server.
+    """
+    pass_through_endpoints = await _get_pass_through_endpoints_from_db()
+    await initialize_pass_through_endpoints(
+        pass_through_endpoints=pass_through_endpoints
+    )

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -88,6 +88,8 @@ class PassThroughEndpointLogging:
             cache_hit=False,
             **kwargs,
         )
+    
+
 
     async def pass_through_async_success_handler(
         self,
@@ -192,6 +194,13 @@ class PassThroughEndpointLogging:
             standard_logging_response_object = StandardPassThroughResponseObject(
                 response=httpx_response.text
             )
+        
+        kwargs = self._set_cost_per_request(
+            logging_obj=logging_obj,
+            passthrough_logging_payload=passthrough_logging_payload,
+            kwargs=kwargs,
+        )
+        
 
         await self._handle_logging(
             logging_obj=logging_obj,
@@ -200,6 +209,7 @@ class PassThroughEndpointLogging:
             start_time=start_time,
             end_time=end_time,
             cache_hit=cache_hit,
+            standard_pass_through_logging_payload=passthrough_logging_payload,
             **kwargs,
         )
 
@@ -234,3 +244,29 @@ class PassThroughEndpointLogging:
             if route in parsed_url.path:
                 return True
         return False
+    
+
+    def _set_cost_per_request(
+        self, 
+        logging_obj: LiteLLMLoggingObj, 
+        passthrough_logging_payload: PassthroughStandardLoggingPayload,
+        kwargs: dict,
+    ):
+        """
+        Helper function to set the cost per request in the logging object
+
+        Only set the cost per request if it's set in the passthrough logging payload.
+        If it's not set, don't set it in the logging object.
+        """
+        #########################################################
+        # Check if cost per request is set
+        #########################################################
+        if passthrough_logging_payload.get("cost_per_request") is not None:
+            kwargs["response_cost"] = passthrough_logging_payload.get(
+                "cost_per_request"
+            )
+            logging_obj.model_call_details["response_cost"] = passthrough_logging_payload.get(
+                "cost_per_request"
+            )
+        
+        return kwargs

litellm/proxy/proxy_config.yaml

@@ -8,15 +8,11 @@ model_list:
       model: "anthropic/*"
       api_key: os.environ/ANTHROPIC_API_KEY
 
-litellm_settings:
-  callbacks: ["prometheus"]
-  prometheus_metrics_config:
-    # High-cardinality metrics with minimal labels
-    - group: "proxy_metrics"
-      metrics:
-        - "litellm_proxy_total_requests_metric"
-        - "litellm_proxy_failed_requests_metric"
-      include_labels:
-        - "hashed_api_key"
-        - "requested_model"
-        - "model_group"
+mcp_servers:
+  deepwiki_mcp:
+    url: "https://mcp.deepwiki.com/mcp"
+    transport: "http"
+
+general_settings:
+  store_model_in_db: true
+  store_prompts_in_spend_logs: true

litellm/proxy/proxy_server.py

@@ -421,9 +421,9 @@ except ImportError:
 server_root_path = os.getenv("SERVER_ROOT_PATH", "")
 _license_check = LicenseCheck()
 premium_user: bool = _license_check.is_premium()
-premium_user_data: Optional[
-    "EnterpriseLicenseData"
-] = _license_check.airgapped_license_data
+premium_user_data: Optional["EnterpriseLicenseData"] = (
+    _license_check.airgapped_license_data
+)
 global_max_parallel_request_retries_env: Optional[str] = os.getenv(
     "LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES"
 )
@@ -752,21 +752,6 @@ try:
     current_dir = os.path.dirname(os.path.abspath(__file__))
     ui_path = os.path.join(current_dir, "_experimental", "out")
     litellm_asset_prefix = "/litellm-asset-prefix"
-    # # Mount the _next directory at the root level
-    app.mount(
-        "/_next",
-        StaticFiles(directory=os.path.join(ui_path, "_next")),
-        name="next_static",
-    )
-    app.mount(
-        f"{litellm_asset_prefix}/_next",
-        StaticFiles(directory=os.path.join(ui_path, "_next")),
-        name="next_static",
-    )
-    # print(f"mounted _next at {server_root_path}/ui/_next")
-
-    app.mount("/ui", StaticFiles(directory=ui_path, html=True), name="ui")
-
     # Iterate through files in the UI directory
     for root, dirs, files in os.walk(ui_path):
         for filename in files:
@@ -792,19 +777,37 @@ try:
 
                 # Replace the asset prefix with the server root path
                 modified_content = content.replace(
-                    f"{litellm_asset_prefix}", server_root_path
+                    f"{litellm_asset_prefix}",
+                    f"{server_root_path}",
                 )
+
                 # Replace the /.well-known/litellm-ui-config with the server root path
                 modified_content = modified_content.replace(
                     "/litellm/.well-known/litellm-ui-config",
                     f"{server_root_path}/.well-known/litellm-ui-config",
                 )
+
                 with open(file_path, "w", encoding="utf-8") as f:
                     f.write(modified_content)
             except UnicodeDecodeError:
                 # Skip binary files that can't be decoded
                 continue
 
+    # # Mount the _next directory at the root level
+    app.mount(
+        "/_next",
+        StaticFiles(directory=os.path.join(ui_path, "_next")),
+        name="next_static",
+    )
+    app.mount(
+        f"{litellm_asset_prefix}/_next",
+        StaticFiles(directory=os.path.join(ui_path, "_next")),
+        name="next_static",
+    )
+    # print(f"mounted _next at {server_root_path}/ui/_next")
+
+    app.mount("/ui", StaticFiles(directory=ui_path, html=True), name="ui")
+
     # Handle HTML file restructuring
     for filename in os.listdir(ui_path):
         if filename.endswith(".html") and filename != "index.html":
@@ -883,9 +886,9 @@ model_max_budget_limiter = _PROXY_VirtualKeyModelMaxBudgetLimiter(
     dual_cache=user_api_key_cache
 )
 litellm.logging_callback_manager.add_litellm_callback(model_max_budget_limiter)
-redis_usage_cache: Optional[
-    RedisCache
-] = None  # redis cache used for tracking spend, tpm/rpm limits
+redis_usage_cache: Optional[RedisCache] = (
+    None  # redis cache used for tracking spend, tpm/rpm limits
+)
 user_custom_auth = None
 user_custom_key_generate = None
 user_custom_sso = None
@@ -1212,9 +1215,9 @@ async def update_cache(  # noqa: PLR0915
         _id = "team_id:{}".format(team_id)
         try:
             # Fetch the existing cost for the given user
-            existing_spend_obj: Optional[
-                LiteLLM_TeamTable
-            ] = await user_api_key_cache.async_get_cache(key=_id)
+            existing_spend_obj: Optional[LiteLLM_TeamTable] = (
+                await user_api_key_cache.async_get_cache(key=_id)
+            )
             if existing_spend_obj is None:
                 # do nothing if team not in api key cache
                 return
@@ -2780,6 +2783,7 @@ class ProxyConfig:
         await self._init_guardrails_in_db(prisma_client=prisma_client)
         await self._init_vector_stores_in_db(prisma_client=prisma_client)
         await self._init_mcp_servers_in_db()
+        await self._init_pass_through_endpoints_in_db()
 
     async def _init_guardrails_in_db(self, prisma_client: PrismaClient):
         from litellm.proxy.guardrails.guardrail_registry import (
@@ -2789,10 +2793,10 @@ class ProxyConfig:
         )
 
         try:
-            guardrails_in_db: List[
-                Guardrail
-            ] = await GuardrailRegistry.get_all_guardrails_from_db(
-                prisma_client=prisma_client
+            guardrails_in_db: List[Guardrail] = (
+                await GuardrailRegistry.get_all_guardrails_from_db(
+                    prisma_client=prisma_client
+                )
             )
             verbose_proxy_logger.debug(
                 "guardrails from the DB %s", str(guardrails_in_db)
@@ -2857,6 +2861,13 @@ class ProxyConfig:
                 )
             )
 
+    async def _init_pass_through_endpoints_in_db(self):
+        from litellm.proxy.pass_through_endpoints.pass_through_endpoints import (
+            initialize_pass_through_endpoints_in_db,
+        )
+
+        await initialize_pass_through_endpoints_in_db()
+
     def decrypt_credentials(self, credential: Union[dict, BaseModel]) -> CredentialItem:
         if isinstance(credential, dict):
             credential_object = CredentialItem(**credential)
@@ -3012,9 +3023,9 @@ async def initialize(  # noqa: PLR0915
         user_api_base = api_base
         dynamic_config[user_model]["api_base"] = api_base
     if api_version:
-        os.environ[
-            "AZURE_API_VERSION"
-        ] = api_version  # set this for azure - litellm can read this from the env
+        os.environ["AZURE_API_VERSION"] = (
+            api_version  # set this for azure - litellm can read this from the env
+        )
     if max_tokens:  # model-specific param
         dynamic_config[user_model]["max_tokens"] = max_tokens
     if temperature:  # model-specific param
@@ -4255,8 +4266,16 @@ async def audio_speech(
             user_api_key_dict=user_api_key_dict,
             request_data=data,
         )
+        # Determine media type based on model type
+        media_type = "audio/mpeg"  # Default for OpenAI TTS
+        request_model = data.get("model", "")
+        if "gemini" in request_model.lower() and (
+            "tts" in request_model.lower() or "preview-tts" in request_model.lower()
+        ):
+            media_type = "audio/wav"  # Gemini TTS returns WAV format after conversion
+
         return StreamingResponse(
-            generate(response), media_type="audio/mpeg", headers=custom_headers  # type: ignore
+            generate(response), media_type=media_type, headers=custom_headers  # type: ignore
         )
 
     except Exception as e:
@@ -7874,9 +7893,9 @@ async def get_config_list(
                             hasattr(sub_field_info, "description")
                             and sub_field_info.description is not None
                         ):
-                            nested_fields[
-                                idx
-                            ].field_description = sub_field_info.description
+                            nested_fields[idx].field_description = (
+                                sub_field_info.description
+                            )
                         idx += 1
 
                     _stored_in_db = None

litellm/proxy/schema.prisma

@@ -497,4 +497,24 @@ model LiteLLM_GuardrailsTable {
   guardrail_info Json?
   created_at DateTime @default(now())
   updated_at DateTime @updatedAt
+}
+
+model LiteLLM_HealthCheckTable {
+  health_check_id String @id @default(uuid())
+  model_name String
+  model_id String?
+  status String
+  healthy_count Int @default(0)
+  unhealthy_count Int @default(0)
+  error_message String?
+  response_time_ms Float?
+  details Json?
+  checked_by String?
+  checked_at DateTime @default(now())
+  created_at DateTime @default(now())
+  updated_at DateTime @updatedAt
+
+  @@index([model_name])
+  @@index([checked_at])
+  @@index([status])
 }

litellm/proxy/spend_tracking/spend_management_endpoints.py

@@ -1854,11 +1854,19 @@ async def view_spend_logs(  # noqa: PLR0915
         default=None,
         description="Time till which to view key spend",
     ),
+    summarize: bool = fastapi.Query(
+        default=True,
+        description="When start_date and end_date are provided, summarize=true returns aggregated data by date (legacy behavior), summarize=false returns filtered individual logs",
+    ),
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
 ):
     """
     View all spend logs, if request_id is provided, only logs for that request_id will be returned
 
+    When start_date and end_date are provided:
+    - summarize=true (default): Returns aggregated spend data grouped by date (maintains backward compatibility)
+    - summarize=false: Returns filtered individual log entries within the date range
+
     Example Request for all logs
     ```
     curl -X GET "http://0.0.0.0:8000/spend/logs" \
@@ -1882,6 +1890,12 @@ async def view_spend_logs(  # noqa: PLR0915
     curl -X GET "http://0.0.0.0:8000/spend/logs?user_id=ishaan@berri.ai" \
 -H "Authorization: Bearer sk-1234"
     ```
+
+    Example Request for date range with individual logs (unsummarized)
+    ```
+    curl -X GET "http://0.0.0.0:8000/spend/logs?start_date=2024-01-01&end_date=2024-01-02&summarize=false" \
+-H "Authorization: Bearer sk-1234"
+    ```
     """
     from litellm.proxy.proxy_server import prisma_client
 
@@ -1922,6 +1936,18 @@ async def view_spend_logs(  # noqa: PLR0915
             elif user_id is not None and isinstance(user_id, str):
                 filter_query["user"] = user_id  # type: ignore
 
+            # Check if user wants unsummarized data
+            if not summarize:
+                # Return filtered individual log entries (similar to UI endpoint)
+                data = await prisma_client.db.litellm_spendlogs.find_many(
+                    where=filter_query,  # type: ignore
+                    order={
+                        "startTime": "desc",
+                    },
+                )
+                return data
+
+            # Legacy behavior: return summarized data (when summarize=true)
             # SQL query
             response = await prisma_client.db.litellm_spendlogs.group_by(
                 by=["api_key", "user", "model", "startTime"],

litellm/proxy/spend_tracking/spend_tracking_utils.py

@@ -91,9 +91,9 @@ def _get_spend_logs_metadata(
     clean_metadata["applied_guardrails"] = applied_guardrails
     clean_metadata["batch_models"] = batch_models
     clean_metadata["mcp_tool_call_metadata"] = mcp_tool_call_metadata
-    clean_metadata[
-        "vector_store_request_metadata"
-    ] = _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
+    clean_metadata["vector_store_request_metadata"] = (
+        _get_vector_store_request_for_spend_logs_payload(vector_store_request_metadata)
+    )
     clean_metadata["guardrail_information"] = guardrail_information
     clean_metadata["usage_object"] = usage_object
     clean_metadata["model_map_information"] = model_map_information
@@ -212,6 +212,11 @@ def get_logging_payload(  # noqa: PLR0915
         if isinstance(metadata.get("tags", []), list)
         else "[]"
     )
+    if (
+        standard_logging_payload is not None
+        and standard_logging_payload.get("request_tags") is not None
+    ):  # use 'tags' from standard logging payload instead
+        request_tags = json.dumps(standard_logging_payload["request_tags"])
     if (
         _is_master_key(api_key=api_key, _master_key=master_key)
         and general_settings.get("disable_adding_master_key_hash_to_db") is True

litellm/proxy/utils.py

@@ -54,6 +54,8 @@ from litellm import (
 )
 from litellm._logging import verbose_proxy_logger
 from litellm._service_logger import ServiceLogging, ServiceTypes
+from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
+from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.caching.caching import DualCache, RedisCache
 from litellm.exceptions import RejectedRequestError
 from litellm.integrations.custom_guardrail import CustomGuardrail
@@ -2458,8 +2460,121 @@ class PrismaClient:
             value=_num_spend_logs_rows,
         )
 
+    # Health Check Database Methods
+    def _validate_response_time(self, response_time_ms: Optional[float]) -> Optional[float]:
+        """Validate and clean response time value"""
+        if response_time_ms is None:
+            return None
+        try:
+            value = float(response_time_ms)
+            return value if value == value and value not in (float('inf'), float('-inf')) else None
+        except (ValueError, TypeError):
+            verbose_proxy_logger.warning(f"Invalid response_time_ms value: {response_time_ms}")
+            return None
+
+    def _clean_details(self, details: Optional[dict]) -> Optional[dict]:
+        """Clean and validate details JSON"""
+        if not isinstance(details, dict):
+            return None
+        try:
+            return safe_json_loads(safe_dumps(details))
+        except Exception as e:
+            verbose_proxy_logger.warning(f"Failed to clean details JSON: {e}")
+            return None
+
+    async def save_health_check_result(
+        self,
+        model_name: str,
+        status: str,
+        healthy_count: int = 0,
+        unhealthy_count: int = 0,
+        error_message: Optional[str] = None,
+        response_time_ms: Optional[float] = None,
+        details: Optional[dict] = None,
+        checked_by: Optional[str] = None,
+        model_id: Optional[str] = None,
+    ):
+        """Save health check result to database"""
+        try:
+            # Build base data with required fields
+            health_check_data = {
+                "model_name": str(model_name),
+                "status": str(status),
+                "healthy_count": int(healthy_count),
+                "unhealthy_count": int(unhealthy_count),
+            }
+            
+            # Add optional fields using dict comprehension and helper methods
+            optional_fields = {
+                "error_message": str(error_message)[:500] if error_message else None,
+                "response_time_ms": self._validate_response_time(response_time_ms),
+                "details": self._clean_details(details),
+                "checked_by": str(checked_by) if checked_by else None,
+                "model_id": str(model_id) if model_id else None,
+            }
+            
+            # Add only non-None optional fields
+            health_check_data.update({k: v for k, v in optional_fields.items() if v is not None})
+            
+            verbose_proxy_logger.debug(f"Saving health check data: {health_check_data}")
+            return await self.db.litellm_healthchecktable.create(data=health_check_data)
+            
+        except Exception as e:
+            verbose_proxy_logger.error(f"Error saving health check result for model {model_name}: {e}")
+            return None
+
+    async def get_health_check_history(
+        self,
+        model_name: Optional[str] = None,
+        limit: int = 100,
+        offset: int = 0,
+        status_filter: Optional[str] = None,
+    ):
+        """
+        Get health check history with optional filtering
+        """
+        try:
+            where_clause = {}
+            if model_name:
+                where_clause["model_name"] = model_name
+            if status_filter:
+                where_clause["status"] = status_filter
+
+            results = await self.db.litellm_healthchecktable.find_many(
+                where=where_clause,
+                order={"checked_at": "desc"},
+                take=limit,
+                skip=offset,
+            )
+            return results
+        except Exception as e:
+            verbose_proxy_logger.error(f"Error getting health check history: {e}")
+            return []
+
+    async def get_all_latest_health_checks(self):
+        """
+        Get the latest health check for each model
+        """
+        try:
+            # Get all unique model names first
+            all_checks = await self.db.litellm_healthchecktable.find_many(
+                order={"checked_at": "desc"}
+            )
+            
+            # Group by model_name and get the latest for each
+            latest_checks = {}
+            for check in all_checks:
+                if check.model_name not in latest_checks:
+                    latest_checks[check.model_name] = check
+            
+            return list(latest_checks.values())
+        except Exception as e:
+            verbose_proxy_logger.error(f"Error getting all latest health checks: {e}")
+            return []
+
 
 ### HELPER FUNCTIONS ###
+
 async def _cache_user_row(user_id: str, cache: DualCache, db: PrismaClient):
     """
     Check if a user_id exists in cache,
@@ -2957,32 +3072,40 @@ def is_known_model(model: Optional[str], llm_router: Optional[Router]) -> bool:
 
 
 def join_paths(base_path: str, route: str) -> str:
-    # Remove trailing/leading slashes
+    # Remove trailing slashes from base_path and leading slashes from route
     base_path = base_path.rstrip("/")
     route = route.lstrip("/")
-
-    # Join with a single slash
+    
+    # If base_path is empty, return route with leading slash
+    if not base_path:
+        return f"/{route}" if route else "/"
+    
+    # If route is empty, return just base_path
+    if not route:
+        return base_path
+    
+    # Join with single slash
     return f"{base_path}/{route}"
 
 
 def get_custom_url(request_base_url: str, route: Optional[str] = None) -> str:
-    """
-    Use proxy base url, if set.
-
-    Else, use request base url.
-    """
-    from httpx import URL
-
-    proxy_base_url = os.getenv("PROXY_BASE_URL")
-    server_root_path = os.getenv("SERVER_ROOT_PATH") or ""
-    if route is not None:
-        server_root_path = join_paths(base_path=server_root_path, route=route)
-    if proxy_base_url:
-        ui_link = str(URL(proxy_base_url).join(server_root_path))
+    # Use environment variable value, otherwise use URL from request
+    server_base_url = get_proxy_base_url()
+    if server_base_url is not None:
+        base_url = server_base_url
     else:
-        ui_link = str(URL(request_base_url).join(server_root_path))
-
-    return ui_link
+        base_url = request_base_url
+    
+    server_root_path = get_server_root_path()
+    if route is not None:
+        if server_root_path != "":
+            # First join base_url with server_root_path, then with route
+            intermediate_url = join_paths(base_url, server_root_path)
+            return join_paths(intermediate_url, route)
+        else:
+            return join_paths(base_url, route)
+    else:
+        return join_paths(base_url, server_root_path)
 
 
 def get_proxy_base_url() -> Optional[str]: