Merge pull request #19497 from BerriAI/main

merge main

.circleci/config.yml

@@ -44,8 +44,8 @@ commands:
             pip install "pytest-asyncio==0.21.1"
             pip install "respx==0.22.0"
             pip install "hypercorn==0.17.3"
-            pip install "pydantic==2.10.2"
-            pip install "mcp==1.10.1"
+            pip install "pydantic==2.11.0"
+            pip install "mcp==1.25.0"
             pip install "requests-mock>=1.12.1"
             pip install "responses==0.25.7"
             pip install "pytest-xdist==3.6.1"
@@ -1152,8 +1152,8 @@ jobs:
             pip install "pytest-cov==5.0.0"
             pip install "pytest-asyncio==0.21.1"
             pip install "respx==0.22.0"
-            pip install "pydantic==2.10.2"
-            pip install "mcp==1.21.2"
+            pip install "pydantic==2.11.0"
+            pip install "mcp==1.25.0"
       # Run pytest and generate JUnit XML report
       - run:
           name: Run tests
@@ -1556,8 +1556,8 @@ jobs:
             pip install "pytest-asyncio==0.21.1"
             pip install "respx==0.22.0"
             pip install "hypercorn==0.17.3"
-            pip install "pydantic==2.10.2"
-            pip install "mcp==1.10.1"
+            pip install "pydantic==2.11.0"
+            pip install "mcp==1.25.0"
             pip install "requests-mock>=1.12.1"
             pip install "responses==0.25.7"
             pip install "pytest-xdist==3.6.1"
@@ -1915,7 +1915,7 @@ jobs:
             pip install "pytest-asyncio==0.21.1"
             pip install "pytest-cov==5.0.0"
             pip install "tomli==2.2.1"
-            pip install "mcp==1.10.1"
+            pip install "mcp==1.25.0"
       - run:
           name: Run tests
           command: |

.circleci/requirements.txt

@@ -8,12 +8,12 @@ redis==5.2.1
 redisvl==0.4.1
 anthropic
 orjson==3.10.12 # fast /embedding responses
-pydantic==2.10.2
+pydantic==2.11.0
 google-cloud-aiplatform==1.43.0
 google-cloud-iam==2.19.1
 fastapi-sso==0.16.0 
 uvloop==0.21.0
-mcp==1.10.1    # for MCP server  
+mcp==1.25.0    # for MCP server  
 semantic_router==0.1.10 # for auto-routing with litellm
 fastuuid==0.12.0
 responses==0.25.7  # for proxy client tests

.github/workflows/ghcr_deploy.yml

@@ -320,72 +320,36 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
     
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        shell: bash
-        run: |
-          LATEST_TAG=$(git describe --tags --exclude "*dev*" --abbrev=0)
-          if [ -z "${LATEST_TAG}" ]; then
-            echo "latest_tag=latest" | tee -a $GITHUB_OUTPUT
-          else
-            echo "latest_tag=${LATEST_TAG}" | tee -a $GITHUB_OUTPUT
-          fi
-
-      - name: Get last published chart version
-        id: current_version
-        shell: bash
-        run: |
-          CHART_LIST=$(helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/${{ env.CHART_NAME }} 2>/dev/null || true)
-          if [ -z "${CHART_LIST}" ]; then
-            echo "current-version=1.0.0" | tee -a $GITHUB_OUTPUT
-          else
-            # Extract version and strip any prerelease suffix (e.g., 1.0.5-latest -> 1.0.5)
-            VERSION=$(printf '%s' "${CHART_LIST}" | grep '^version:' | awk 'BEGIN{FS=":"}{print $2}' | tr -d " " | cut -d'-' -f1)
-            echo "current-version=${VERSION}" | tee -a $GITHUB_OUTPUT
-          fi
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
-
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '1.0.0' }}
-          version-fragment: 'bug'
-
-      # Add suffix for non-stable releases (semantic versioning)
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      # This allows users to easily map Helm chart versions to LiteLLM versions
+      # See: https://codefresh.io/docs/docs/ci-cd-guides/helm-best-practices/
       - name: Calculate chart and app versions
         id: chart_version
         shell: bash
         run: |
-          BASE_VERSION="${{ steps.bump_version.outputs.next-version || '1.0.0' }}"
-          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
           INPUT_TAG="${{ github.event.inputs.tag }}"
+          RELEASE_TYPE="${{ github.event.inputs.release_type }}"
 
-          # Chart version (independent Helm chart versioning with release type suffix)
-          if [ "$RELEASE_TYPE" = "stable" ]; then
-            echo "version=${BASE_VERSION}" | tee -a $GITHUB_OUTPUT
-          else
-            echo "version=${BASE_VERSION}-${RELEASE_TYPE}" | tee -a $GITHUB_OUTPUT
+          # Chart version = LiteLLM version without 'v' prefix (Helm semver convention)
+          # v1.81.0 -> 1.81.0, v1.81.0.rc.1 -> 1.81.0.rc.1
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # Add suffix for 'latest' releases (rc already has suffix in tag)
+          if [ "$RELEASE_TYPE" = "latest" ]; then
+            CHART_VERSION="${CHART_VERSION}-latest"
           fi
 
-          # App version (must match Docker tags)
-          # stable/rc releases: Docker creates main-{tag}, so use the tag
-          # latest/dev releases: Docker only creates main-{release_type}, so use release_type
-          if [ "$RELEASE_TYPE" = "stable" ] || [ "$RELEASE_TYPE" = "rc" ]; then
-            APP_VERSION="${INPUT_TAG}"
-          else
-            APP_VERSION="${RELEASE_TYPE}"
-          fi
+          # App version = Docker tag (keeps 'v' prefix to match Docker image tags)
+          APP_VERSION="${INPUT_TAG}"
 
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
           echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - uses: ./.github/actions/helm-oci-chart-releaser
         with:
           name: ${{ env.CHART_NAME }}
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.chart_version.outputs.version || '1.0.0' }}
+          tag: ${{ steps.chart_version.outputs.version }}
           app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/${{ env.CHART_NAME }}
           registry: ${{ env.REGISTRY }}

.github/workflows/ghcr_helm_deploy.yml

@@ -1,10 +1,12 @@
-# this workflow is triggered by an API call when there is a new PyPI release of LiteLLM
+# Standalone workflow to publish LiteLLM Helm Chart
+# Note: The main ghcr_deploy.yml workflow also publishes the Helm chart as part of a full release
 name: Build, Publish LiteLLM Helm Chart. New Release
 on:
   workflow_dispatch:
     inputs:
-      chartVersion:
-        description: "Update the helm chart's version to this"
+      tag:
+        description: "LiteLLM version tag (e.g., v1.81.0)"
+        required: true
 
 # Defines two custom environment variables for the workflow. Used for the Container registry domain, and a name for the Docker image that this workflow builds.
 env:
@@ -31,24 +33,22 @@ jobs:
         run: |
           echo "REPO_OWNER=`echo ${{github.repository_owner}} | tr '[:upper:]' '[:lower:]'`" >>${GITHUB_ENV}
 
-      - name: Get LiteLLM Latest Tag
-        id: current_app_tag
-        uses: WyriHaximus/github-action-get-previous-tag@v1.3.0
-
-      - name: Get last published chart version
-        id: current_version
+      # Sync Helm chart version with LiteLLM release version (1-1 versioning)
+      - name: Calculate chart and app versions
+        id: chart_version
         shell: bash
-        run: helm show chart oci://${{ env.REGISTRY }}/${{ env.REPO_OWNER }}/litellm-helm | grep '^version:' | awk 'BEGIN{FS=":"}{print "current-version="$2}' | tr -d " " | tee -a $GITHUB_OUTPUT
-        env:
-          HELM_EXPERIMENTAL_OCI: '1'
+        run: |
+          INPUT_TAG="${{ github.event.inputs.tag }}"
 
-      # Automatically update the helm chart version one "patch" level
-      - name: Bump release version
-        id: bump_version
-        uses: christian-draeger/increment-semantic-version@1.1.0
-        with:
-          current-version: ${{ steps.current_version.outputs.current-version || '0.1.0' }}
-          version-fragment: 'bug'
+          # Chart version = LiteLLM version without 'v' prefix
+          # v1.81.0 -> 1.81.0
+          CHART_VERSION="${INPUT_TAG#v}"
+
+          # App version = Docker tag (keeps 'v' prefix)
+          APP_VERSION="${INPUT_TAG}"
+
+          echo "version=${CHART_VERSION}" | tee -a $GITHUB_OUTPUT
+          echo "app_version=${APP_VERSION}" | tee -a $GITHUB_OUTPUT
 
       - name: Lint helm chart
         run: helm lint deploy/charts/litellm-helm
@@ -57,8 +57,8 @@ jobs:
         with:
           name: litellm-helm
           repository: ${{ env.REPO_OWNER }}
-          tag: ${{ github.event.inputs.chartVersion || steps.bump_version.outputs.next-version || '0.1.0' }}
-          app_version: ${{ steps.current_app_tag.outputs.tag || 'latest' }}
+          tag: ${{ steps.chart_version.outputs.version }}
+          app_version: ${{ steps.chart_version.outputs.app_version }}
           path: deploy/charts/litellm-helm
           registry: ${{ env.REGISTRY }}
           registry_username: ${{ github.actor }}

.github/workflows/test-mcp.yml

@@ -34,8 +34,8 @@ jobs:
         poetry run pip install "pytest-cov==5.0.0"
         poetry run pip install "pytest-asyncio==0.21.1"
         poetry run pip install "respx==0.22.0"
-        poetry run pip install "pydantic==2.10.2"
-        poetry run pip install "mcp==1.10.1"
+        poetry run pip install "pydantic==2.11.0"
+        poetry run pip install "mcp==1.25.0"
         poetry run pip install pytest-xdist
 
     - name: Setup litellm-enterprise as local package

docker/prod_entrypoint.sh

@@ -2,6 +2,7 @@
 
 if [ "$SEPARATE_HEALTH_APP" = "1" ]; then
     export LITELLM_ARGS="$@"
+    export SUPERVISORD_STOPWAITSECS="${SUPERVISORD_STOPWAITSECS:-3600}"
     exec supervisord -c /etc/supervisord.conf
 fi
 

docker/supervisord.conf

@@ -16,6 +16,7 @@ priority=1
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0
@@ -31,6 +32,7 @@ priority=2
 exitcodes=0
 stopasgroup=true
 killasgroup=true
+stopwaitsecs=%(ENV_SUPERVISORD_STOPWAITSECS)s
 stdout_logfile=/dev/stdout
 stderr_logfile=/dev/stderr
 stdout_logfile_maxbytes = 0

docs/my-website/docs/adding_provider/generic_guardrail_api.md

@@ -237,6 +237,27 @@ litellm_settings:
           language: "en"
 ```
 
+### Example: Pillar Security
+
+[Pillar Security](https://pillar.security) uses the Generic Guardrail API to provide comprehensive AI security scanning including prompt injection protection, PII/PCI detection, secret detection, and content moderation.
+
+```yaml
+guardrails:
+  - guardrail_name: "pillar-security"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: [pre_call, post_call]
+      api_base: https://api.pillar.security/api/v1/integrations/litellm
+      api_key: os.environ/PILLAR_API_KEY
+      default_on: true
+      additional_provider_specific_params:
+        plr_mask: true      # Enable automatic masking of sensitive data
+        plr_evidence: true  # Include detection evidence in response
+        plr_scanners: true  # Include scanner details in response
+```
+
+See the [Pillar Security documentation](../proxy/guardrails/pillar_security.md) for full configuration options.
+
 ## Usage
 
 Users apply your guardrail by name:

docs/my-website/docs/completion/json_mode.md

@@ -341,4 +341,90 @@ curl http://0.0.0.0:4000/v1/chat/completions \
 ```
 
 </TabItem>
-</Tabs>
+</Tabs>
+
+## Gemini - Native JSON Schema Format (Gemini 2.0+)
+
+Gemini 2.0+ models automatically use the native `responseJsonSchema` parameter, which provides better compatibility with standard JSON Schema format.
+
+### Benefits (Gemini 2.0+):
+- Standard JSON Schema format (lowercase types like `string`, `object`)
+- Supports `additionalProperties: false` for stricter validation
+- Better compatibility with Pydantic's `model_json_schema()`
+- No `propertyOrdering` required
+
+### Usage
+
+<Tabs>
+<TabItem value="sdk" label="SDK">
+
+```python
+from litellm import completion
+from pydantic import BaseModel
+
+class UserInfo(BaseModel):
+    name: str
+    age: int
+
+response = completion(
+    model="gemini/gemini-2.0-flash",
+    messages=[{"role": "user", "content": "Extract: John is 25 years old"}],
+    response_format={
+        "type": "json_schema",
+        "json_schema": {
+            "name": "user_info",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"}
+                },
+                "required": ["name", "age"],
+                "additionalProperties": False  # Supported on Gemini 2.0+
+            }
+        }
+    }
+)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer $LITELLM_API_KEY" \
+  -d '{
+    "model": "gemini-2.0-flash",
+    "messages": [
+        {"role": "user", "content": "Extract: John is 25 years old"}
+    ],
+    "response_format": {
+        "type": "json_schema",
+        "json_schema": {
+            "name": "user_info",
+            "schema": {
+                "type": "object",
+                "properties": {
+                    "name": {"type": "string"},
+                    "age": {"type": "integer"}
+                },
+                "required": ["name", "age"],
+                "additionalProperties": false
+            }
+        }
+    }
+  }'
+```
+
+</TabItem>
+</Tabs>
+
+### Model Behavior
+
+| Model | Format Used | `additionalProperties` Support |
+|-------|-------------|-------------------------------|
+| Gemini 2.0+ | `responseJsonSchema` (JSON Schema) | ✅ Yes |
+| Gemini 1.5 | `responseSchema` (OpenAPI) | ❌ No |
+
+LiteLLM automatically selects the appropriate format based on the model version.

docs/my-website/docs/mcp.md

@@ -21,6 +21,11 @@ LiteLLM Proxy provides an MCP Gateway that allows you to use a fixed endpoint fo
 | Supported MCP Transports | • Streamable HTTP<br/>• SSE<br/>• Standard Input/Output (stdio) |
 | LiteLLM Permission Management | • By Key<br/>• By Team<br/>• By Organization |
 
+:::caution MCP protocol update
+Starting in LiteLLM v1.80.18, the LiteLLM MCP protocol version is `2025-11-25`.<br/> 
+LiteLLM namespaces multiple MCP servers by prefixing each tool name with its MCP server name, so newly created servers now must use names that comply with SEP-986—noncompliant names cannot be added anymore. Existing servers that still violate SEP-986 only emit warnings today, but future MCP-side rollouts may block those names entirely, so we recommend updating any legacy server names proactively before MCP enforcement makes them unusable.
+:::
+
 ## Adding your MCP
 
 ### Prerequisites

docs/my-website/docs/providers/chatgpt.md

@@ -0,0 +1,84 @@
+# ChatGPT Subscription
+
+Use ChatGPT Pro/Max subscription models through LiteLLM with OAuth device flow authentication.
+
+| Property | Details |
+|-------|-------|
+| Description | ChatGPT subscription access (Codex + GPT-5.2 family) via ChatGPT backend API |
+| Provider Route on LiteLLM | `chatgpt/` |
+| Supported Endpoints | `/responses`, `/chat/completions` (bridged to Responses for supported models) |
+| API Reference | https://chatgpt.com |
+
+ChatGPT subscription access is native to the Responses API. Chat Completions requests are bridged to Responses for supported models (for example `chatgpt/gpt-5.2`).
+
+Notes:
+- The ChatGPT subscription backend rejects token limit fields (`max_tokens`, `max_output_tokens`, `max_completion_tokens`) and `metadata`. LiteLLM strips these fields for this provider.
+- `/v1/chat/completions` honors `stream`. When `stream` is false (default), LiteLLM aggregates the Responses stream into a single JSON response.
+
+## Authentication
+
+ChatGPT subscription access uses an OAuth device code flow:
+
+1. LiteLLM prints a device code and verification URL
+2. Open the URL, sign in, and enter the code
+3. Tokens are stored locally for reuse
+
+## Usage - LiteLLM Python SDK
+
+### Responses (recommended for Codex models)
+
+```python showLineNumbers title="ChatGPT Responses"
+import litellm
+
+response = litellm.responses(
+    model="chatgpt/gpt-5.2-codex",
+    input="Write a Python hello world"
+)
+
+print(response)
+```
+
+### Chat Completions (bridged to Responses)
+
+```python showLineNumbers title="ChatGPT Chat Completions"
+import litellm
+
+response = litellm.completion(
+    model="chatgpt/gpt-5.2",
+    messages=[{"role": "user", "content": "Write a Python hello world"}]
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: chatgpt/gpt-5.2
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.2
+  - model_name: chatgpt/gpt-5.2-codex
+    model_info:
+      mode: responses
+    litellm_params:
+      model: chatgpt/gpt-5.2-codex
+```
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config config.yaml
+```
+
+## Configuration
+
+### Environment Variables
+
+- `CHATGPT_TOKEN_DIR`: Custom token storage directory
+- `CHATGPT_AUTH_FILE`: Auth file name (default: `auth.json`)
+- `CHATGPT_API_BASE`: Override API base (default: `https://chatgpt.com/backend-api/codex`)
+- `OPENAI_CHATGPT_API_BASE`: Alias for `CHATGPT_API_BASE`
+- `CHATGPT_ORIGINATOR`: Override the `originator` header value
+- `CHATGPT_USER_AGENT`: Override the `User-Agent` header value
+- `CHATGPT_USER_AGENT_SUFFIX`: Optional suffix appended to the `User-Agent` header

docs/my-website/docs/providers/gemini.md

@@ -15,6 +15,17 @@ import TabItem from '@theme/TabItem';
 
 <br />
 
+:::tip Gemini API vs Vertex AI
+| Model Format | Provider | Auth Required |
+|-------------|----------|---------------|
+| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
+| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
+| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
+
+**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix.
+
+Models without a prefix default to Vertex AI which requires full GCP authentication.
+:::
 
 ## API Keys
 
@@ -1547,16 +1558,21 @@ LiteLLM Supports the following image types passed in `url`
 - Images with direct links - https://storage.googleapis.com/github-repo/img/gemini/intro/landmark3.jpg
 - Image in local storage - ./localimage.jpeg
 
-## Image Resolution Control (Gemini 3+)
+## Media Resolution Control (Images & Videos)
 
-For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images in your request.
+For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
 
 **Supported `detail` values:**
 - `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
+- `"medium"` - Maps to `media_resolution: "medium"`
 - `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
+- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
 - `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
 
-**Usage Example:**
+**Usage Examples:**
+
+<Tabs>
+<TabItem value="images" label="Images">
 
 ```python
 from litellm import completion
@@ -1593,10 +1609,193 @@ response = completion(
 )
 ```
 
+</TabItem>
+<TabItem value="videos" label="Videos with Files">
+
+```python
+from litellm import completion
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Analyze this video"
+            },
+            {
+                "type": "file",
+                "file": {
+                    "file_id": "gs://my-bucket/video.mp4",
+                    "format": "video/mp4",
+                    "detail": "high"  # High resolution for detailed video analysis
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="gemini/gemini-3-pro-preview",
+    messages=messages,
+)
+```
+
+</TabItem>
+</Tabs>
+
 :::info
-**Per-Part Resolution:** Each image in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature is only available for Gemini 3+ models.
+**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
 :::
 
+## Video Metadata Control
+
+For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis.
+
+**Supported `video_metadata` parameters:**
+
+| Parameter | Type | Description | Example |
+|-----------|------|-------------|---------|
+| `fps` | Number | Frame extraction rate (frames per second) | `5` |
+| `start_offset` | String | Start time for video clip processing | `"10s"` |
+| `end_offset` | String | End time for video clip processing | `"60s"` |
+
+:::note
+**Field Name Conversion:** LiteLLM automatically converts snake_case field names to camelCase for the Gemini API:
+- `start_offset` → `startOffset`
+- `end_offset` → `endOffset`
+- `fps` remains unchanged
+:::
+
+:::warning
+- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models
+- **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API
+- **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files
+:::
+
+**Usage Examples:**
+
+<Tabs>
+<TabItem value="basic" label="Basic Video Metadata">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3-pro-preview",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Analyze this video clip"},
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "gs://my-bucket/video.mp4",
+                        "format": "video/mp4",
+                        "video_metadata": {
+                            "fps": 5,               # Extract 5 frames per second
+                            "start_offset": "10s",  # Start from 10 seconds
+                            "end_offset": "60s"     # End at 60 seconds
+                        }
+                    }
+                }
+            ]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="combined" label="Combined with Detail">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="gemini/gemini-3-pro-preview",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Provide detailed analysis of this video segment"},
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "https://example.com/presentation.mp4",
+                        "format": "video/mp4",
+                        "detail": "high",  # High resolution for detailed analysis
+                        "video_metadata": {
+                            "fps": 10,              # Extract 10 frames per second
+                            "start_offset": "30s",  # Start from 30 seconds
+                            "end_offset": "90s"     # End at 90 seconds
+                        }
+                    }
+                }
+            ]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: gemini-3-pro
+    litellm_params:
+      model: gemini/gemini-3-pro-preview
+      api_key: os.environ/GEMINI_API_KEY
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Make request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3-pro",
+    "messages": [
+      {
+        "role": "user",
+        "content": [
+          {"type": "text", "text": "Analyze this video clip"},
+          {
+            "type": "file",
+            "file": {
+              "file_id": "gs://my-bucket/video.mp4",
+              "format": "video/mp4",
+              "detail": "high",
+              "video_metadata": {
+                "fps": 5,
+                "start_offset": "10s",
+                "end_offset": "60s"
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
+
 ## Sample Usage
 ```python
 import os

docs/my-website/docs/providers/vertex.md

@@ -14,6 +14,17 @@ import TabItem from '@theme/TabItem';
 | Base URL | 1. Regional endpoints<br/>`https://{vertex_location}-aiplatform.googleapis.com/`<br/>2. Global endpoints (limited availability)<br/>`https://aiplatform.googleapis.com/`|
 | Supported Operations | [`/chat/completions`](#sample-usage), `/completions`, [`/embeddings`](#embedding-models), [`/audio/speech`](#text-to-speech-apis), [`/fine_tuning`](#fine-tuning-apis), [`/batches`](#batch-apis), [`/files`](#batch-apis), [`/images`](#image-generation-models), [`/rerank`](#rerank-api) |
 
+:::tip Vertex AI vs Gemini API
+| Model Format | Provider | Auth Required |
+|-------------|----------|---------------|
+| `vertex_ai/gemini-2.0-flash` | Vertex AI | GCP credentials + project |
+| `gemini-2.0-flash` (no prefix) | Vertex AI | GCP credentials + project |
+| `gemini/gemini-2.0-flash` | Gemini API | `GEMINI_API_KEY` (simple API key) |
+
+**If you just want to use an API key** (like OpenAI), use the `gemini/` prefix instead. See [Gemini - Google AI Studio](./gemini.md).
+
+Models without a prefix default to Vertex AI which requires GCP authentication.
+:::
 
 <br />
 <br />
@@ -1957,6 +1968,244 @@ assert isinstance(
 
 ```
 
+## Media Resolution Control (Images & Videos)
+
+For Gemini 3+ models, LiteLLM supports per-part media resolution control using OpenAI's `detail` parameter. This allows you to specify different resolution levels for individual images and videos in your request, whether using `image_url` or `file` content types.
+
+**Supported `detail` values:**
+- `"low"` - Maps to `media_resolution: "low"` (280 tokens for images, 70 tokens per frame for videos)
+- `"medium"` - Maps to `media_resolution: "medium"`
+- `"high"` - Maps to `media_resolution: "high"` (1120 tokens for images)
+- `"ultra_high"` - Maps to `media_resolution: "ultra_high"`
+- `"auto"` or `None` - Model decides optimal resolution (no `media_resolution` set)
+
+**Usage Examples:**
+
+<Tabs>
+<TabItem value="images" label="Images">
+
+```python
+from litellm import completion
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "https://example.com/chart.png",
+                    "detail": "high"  # High resolution for detailed chart analysis
+                }
+            },
+            {
+                "type": "text",
+                "text": "Analyze this chart"
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "https://example.com/icon.png",
+                    "detail": "low"  # Low resolution for simple icon
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="vertex_ai/gemini-3-pro-preview",
+    messages=messages,
+)
+```
+
+</TabItem>
+<TabItem value="videos" label="Videos with Files">
+
+```python
+from litellm import completion
+
+messages = [
+    {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": "Analyze this video"
+            },
+            {
+                "type": "file",
+                "file": {
+                    "file_id": "gs://my-bucket/video.mp4",
+                    "format": "video/mp4",
+                    "detail": "high"  # High resolution for detailed video analysis
+                }
+            }
+        ]
+    }
+]
+
+response = completion(
+    model="vertex_ai/gemini-3-pro-preview",
+    messages=messages,
+)
+```
+
+</TabItem>
+</Tabs>
+
+:::info
+**Per-Part Resolution:** Each image or video in your request can have its own `detail` setting, allowing mixed-resolution requests (e.g., a high-res chart alongside a low-res icon). This feature works with both `image_url` and `file` content types, and is only available for Gemini 3+ models.
+:::
+
+## Video Metadata Control
+
+For Gemini 3+ models, LiteLLM supports fine-grained video processing control through the `video_metadata` field. This allows you to specify frame extraction rates and time ranges for video analysis.
+
+**Supported `video_metadata` parameters:**
+
+| Parameter | Type | Description | Example |
+|-----------|------|-------------|---------|
+| `fps` | Number | Frame extraction rate (frames per second) | `5` |
+| `start_offset` | String | Start time for video clip processing | `"10s"` |
+| `end_offset` | String | End time for video clip processing | `"60s"` |
+
+:::note
+**Field Name Conversion:** LiteLLM automatically converts snake_case field names to camelCase for the Gemini API:
+- `start_offset` → `startOffset`
+- `end_offset` → `endOffset`
+- `fps` remains unchanged
+:::
+
+:::warning
+- **Gemini 3+ Only:** This feature is only available for Gemini 3.0 and newer models
+- **Video Files Recommended:** While `video_metadata` is designed for video files, error handling for other media types is delegated to the Vertex AI API
+- **File Formats Supported:** Works with `gs://`, `https://`, and base64-encoded video files
+:::
+
+**Usage Examples:**
+
+<Tabs>
+<TabItem value="basic" label="Basic Video Metadata">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-3-pro-preview",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Analyze this video clip"},
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "gs://my-bucket/video.mp4",
+                        "format": "video/mp4",
+                        "video_metadata": {
+                            "fps": 5,               # Extract 5 frames per second
+                            "start_offset": "10s",  # Start from 10 seconds
+                            "end_offset": "60s"     # End at 60 seconds
+                        }
+                    }
+                }
+            ]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="combined" label="Combined with Detail">
+
+```python
+from litellm import completion
+
+response = completion(
+    model="vertex_ai/gemini-3-pro-preview",
+    messages=[
+        {
+            "role": "user",
+            "content": [
+                {"type": "text", "text": "Provide detailed analysis of this video segment"},
+                {
+                    "type": "file",
+                    "file": {
+                        "file_id": "https://example.com/presentation.mp4",
+                        "format": "video/mp4",
+                        "detail": "high",  # High resolution for detailed analysis
+                        "video_metadata": {
+                            "fps": 10,              # Extract 10 frames per second
+                            "start_offset": "30s",  # Start from 30 seconds
+                            "end_offset": "90s"     # End at 90 seconds
+                        }
+                    }
+                }
+            ]
+        }
+    ]
+)
+
+print(response.choices[0].message.content)
+```
+
+</TabItem>
+<TabItem value="proxy" label="PROXY">
+
+1. Setup config.yaml
+
+```yaml
+model_list:
+  - model_name: gemini-3-pro
+    litellm_params:
+      model: vertex_ai/gemini-3-pro-preview
+      vertex_project: your-project
+      vertex_location: us-central1
+```
+
+2. Start proxy
+
+```bash
+litellm --config /path/to/config.yaml
+```
+
+3. Make request
+
+```bash
+curl http://0.0.0.0:4000/v1/chat/completions \
+  -H "Content-Type: application/json" \
+  -H "Authorization: Bearer <YOUR-LITELLM-KEY>" \
+  -d '{
+    "model": "gemini-3-pro",
+    "messages": [
+      {
+        "role": "user",
+        "content": [
+          {"type": "text", "text": "Analyze this video clip"},
+          {
+            "type": "file",
+            "file": {
+              "file_id": "gs://my-bucket/video.mp4",
+              "format": "video/mp4",
+              "detail": "high",
+              "video_metadata": {
+                "fps": 5,
+                "start_offset": "10s",
+                "end_offset": "60s"
+              }
+            }
+          }
+        ]
+      }
+    ]
+  }'
+```
+
+</TabItem>
+</Tabs>
 
 ## Usage - PDF / Videos / Audio etc. Files 
 

docs/my-website/docs/proxy/config_settings.md

@@ -397,6 +397,7 @@ router_settings:
 | AUDIO_SPEECH_CHUNK_SIZE | Chunk size for audio speech processing. Default is 1024
 | ANTHROPIC_API_KEY | API key for Anthropic service
 | ANTHROPIC_API_BASE | Base URL for Anthropic API. Default is https://api.anthropic.com
+| ANTHROPIC_TOKEN_COUNTING_BETA_VERSION | Beta version header for Anthropic token counting API. Default is `token-counting-2024-11-01`
 | AWS_ACCESS_KEY_ID | Access Key ID for AWS services
 | AWS_BATCH_ROLE_ARN | ARN of the AWS IAM role for batch operations
 | AWS_DEFAULT_REGION | Default AWS region for service interactions when AWS_REGION is not set
@@ -412,6 +413,8 @@ router_settings:
 | AWS_WEB_IDENTITY_TOKEN | Web identity token for AWS
 | AWS_WEB_IDENTITY_TOKEN_FILE | Path to file containing web identity token for AWS
 | AZURE_API_VERSION | Version of the Azure API being used
+| AZURE_AI_API_BASE | Base URL for Azure AI services (e.g., Azure AI Anthropic)
+| AZURE_AI_API_KEY | API key for Azure AI services (e.g., Azure AI Anthropic)
 | AZURE_AUTHORITY_HOST | Azure authority host URL
 | AZURE_CERTIFICATE_PASSWORD | Password for Azure OpenAI certificate
 | AZURE_CLIENT_ID | Client ID for Azure services
@@ -449,6 +452,13 @@ router_settings:
 | BRAINTRUST_API_KEY | API key for Braintrust integration
 | BRAINTRUST_API_BASE | Base URL for Braintrust API. Default is https://api.braintrustdata.com/v1
 | CACHED_STREAMING_CHUNK_DELAY | Delay in seconds for cached streaming chunks. Default is 0.02
+| CHATGPT_API_BASE | Base URL for ChatGPT API. Default is https://chatgpt.com/backend-api/codex
+| CHATGPT_AUTH_FILE | Filename for ChatGPT authentication data. Default is "auth.json"
+| CHATGPT_DEFAULT_INSTRUCTIONS | Default system instructions for ChatGPT provider
+| CHATGPT_ORIGINATOR | Originator identifier for ChatGPT API requests. Default is "codex_cli_rs"
+| CHATGPT_TOKEN_DIR | Directory to store ChatGPT authentication tokens. Default is "~/.config/litellm/chatgpt"
+| CHATGPT_USER_AGENT | Custom user agent string for ChatGPT API requests
+| CHATGPT_USER_AGENT_SUFFIX | Suffix to append to the ChatGPT user agent string
 | CIRCLE_OIDC_TOKEN | OpenID Connect token for CircleCI
 | CIRCLE_OIDC_TOKEN_V2 | Version 2 of the OpenID Connect token for CircleCI
 | CLOUDZERO_API_KEY | CloudZero API key for authentication
@@ -794,6 +804,7 @@ router_settings:
 | OPENAI_BASE_URL | Base URL for OpenAI API
 | OPENAI_API_BASE | Base URL for OpenAI API. Default is https://api.openai.com/
 | OPENAI_API_KEY | API key for OpenAI services
+| OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
 | OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
 | OPENAI_ORGANIZATION | Organization identifier for OpenAI
 | OPENID_BASE_URL | Base URL for OpenID Connect services
@@ -867,6 +878,7 @@ router_settings:
 | SECRET_MANAGER_REFRESH_INTERVAL | Refresh interval in seconds for secret manager. Default is 86400 (24 hours)
 | SEPARATE_HEALTH_APP | If set to '1', runs health endpoints on a separate ASGI app and port. Default: '0'.
 | SEPARATE_HEALTH_PORT | Port for the separate health endpoints app. Only used if SEPARATE_HEALTH_APP=1. Default: 4001.
+| SUPERVISORD_STOPWAITSECS | Upper bound timeout in seconds for graceful shutdown when SEPARATE_HEALTH_APP=1. Default: 3600 (1 hour).
 | SERVER_ROOT_PATH | Root path for the server application
 | SEND_USER_API_KEY_ALIAS | Flag to send user API key alias to Zscaler AI Guard. Default is False
 | SEND_USER_API_KEY_TEAM_ID | Flag to send user API key team ID to Zscaler AI Guard. Default is False

docs/my-website/docs/proxy/guardrails/quick_start.md

@@ -59,6 +59,18 @@ guardrails:
       presidio_score_thresholds:  # minimum confidence scores for keeping detections
         CREDIT_CARD: 0.8
         EMAIL_ADDRESS: 0.6
+
+# Example Pillar Security config via Generic Guardrail API
+  - guardrail_name: "pillar-security"
+    litellm_params:
+      guardrail: generic_guardrail_api
+      mode: [pre_call, post_call]
+      api_base: https://api.pillar.security/api/v1/integrations/litellm
+      api_key: os.environ/PILLAR_API_KEY
+      additional_provider_specific_params:
+        plr_mask: true
+        plr_evidence: true
+        plr_scanners: true
 ```
 
 

docs/my-website/docs/proxy/prod.md

@@ -277,8 +277,13 @@ Set the following environment variable(s):
 ```bash
 SEPARATE_HEALTH_APP="1" # Default "0" 
 SEPARATE_HEALTH_PORT="8001" # Default "4001", Works only if `SEPARATE_HEALTH_APP` is "1"
+SUPERVISORD_STOPWAITSECS="3600" # Optional: Upper bound timeout in seconds for graceful shutdown. Default: 3600 (1 hour). Only used when SEPARATE_HEALTH_APP=1.
 ```
 
+**Graceful Shutdown:**
+
+Previously, `stopwaitsecs` was not set, defaulting to 10 seconds and causing in-flight requests to fail. `SUPERVISORD_STOPWAITSECS` (default: 3600) provides an upper bound for graceful shutdown, allowing uvicorn to wait for all in-flight requests to complete.
+
 <video controls width="100%" style={{ borderRadius: '8px', marginBottom: '1em' }}>
   <source src="https://cdn.loom.com/sessions/thumbnails/b08be303331246b88fdc053940d03281-1718990992822.mp4" type="video/mp4" />
   Your browser does not support the video tag.

docs/my-website/docs/proxy/users.md

@@ -545,6 +545,26 @@ You can set:
 - max parallel requests
 - rpm / tpm limits per model for a given key
 
+### TPM Rate Limit Type (Input/Output/Total)
+
+By default, TPM (tokens per minute) rate limits count **total tokens** (input + output). You can configure this to count only input tokens or only output tokens instead.
+
+Set `token_rate_limit_type` in your `config.yaml`:
+
+```yaml
+general_settings:
+  master_key: sk-1234
+  token_rate_limit_type: "output"  # Options: "input", "output", "total" (default)
+```
+
+| Value | Description |
+|-------|-------------|
+| `total` | Count total tokens (prompt + completion). **Default behavior.** |
+| `input` | Count only prompt/input tokens |
+| `output` | Count only completion/output tokens |
+
+This setting applies globally to all TPM rate limit checks (keys, users, teams, etc.).
+
 
 <Tabs>
 <TabItem value="per-team" label="Per Team">

docs/my-website/docs/tutorials/claude_code_max_subscription.md

@@ -0,0 +1,357 @@
+import Image from '@theme/IdealImage';
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Using Claude Code Max Subscription
+
+<div style={{ textAlign: 'center' }}>
+<Image img={require('../../img/claude_code_max.png')} style={{ width: '100%', maxWidth: '800px', height: 'auto' }} />
+
+Route Claude Code Max subscription traffic through LiteLLM AI Gateway.
+</div>
+
+**Why Claude Code Max over direct API?**
+- **Lower costs** — Claude Code Max subscriptions are cheaper for Claude Code power users than per-token API pricing
+
+**Why route through LiteLLM?**
+- **Cost attribution** — Track spend per user, team, or key
+- **Budgets & rate limits** — Set spending caps and request limits
+- **Guardrails** — Apply content filtering and safety controls to all requests
+
+
+
+## Quick Start Video
+
+Watch the end-to-end walkthrough of setting up Claude Code with LiteLLM Gateway:
+
+<iframe width="840" height="500" src="https://www.loom.com/embed/2d069b9e3bcc4cecaa5eb27a72ba7b3c" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
+
+## Prerequisites
+
+- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed
+- Claude Max subscription
+- LiteLLM Gateway running
+
+## Step 1: Configure LiteLLM Proxy
+
+Create a `config.yaml` with the critical `forward_client_headers_to_llm_api: true` setting:
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+  - model_name: claude-3-5-sonnet-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-sonnet-20241022
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+
+general_settings:
+  forward_client_headers_to_llm_api: true  # Required: forwards OAuth token to Anthropic
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+:::info Why `forward_client_headers_to_llm_api`?
+
+This setting forwards the user's OAuth token (in the `Authorization` header) through LiteLLM to the Anthropic API, enabling per-user authentication with their Max subscription while LiteLLM handles tracking and controls.
+
+:::
+
+## Step 2: Start LiteLLM Proxy
+
+```bash showLineNumbers title="Start LiteLLM Proxy"
+litellm --config /path/to/config.yaml
+
+# RUNNING on http://0.0.0.0:4000
+```
+
+## Walkthrough
+
+### Part 1: Create a Virtual Key in LiteLLM
+
+Navigate to the LiteLLM Dashboard and create a new virtual key for Claude Code usage.
+
+#### 1.1 Open Virtual Keys Page
+
+Navigate to the Virtual Keys section in the LiteLLM Dashboard.
+
+<Image img={require('../../img/claude_code_max/step1.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.2 Click "Create New Key"
+
+<Image img={require('../../img/claude_code_max/step2.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.3 Configure Key Details
+
+Enter a key name (e.g., `claude-code-test`) and select the models you want to allow access to.
+
+<Image img={require('../../img/claude_code_max/step3.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.4 Select Models
+
+Choose the Anthropic models that should be accessible via this key (e.g., `anthropic-claude`, `claude-4.5-haiku`).
+
+<Image img={require('../../img/claude_code_max/step5.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.5 Confirm Model Selection
+
+<Image img={require('../../img/claude_code_max/step7.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 1.6 Create the Key
+
+Click "Create Key" to generate your virtual key. Copy the generated key value (e.g., `sk-otsclFlEblQ-6D60ua2IZg`).
+
+<Image img={require('../../img/claude_code_max/step8.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+### Part 2: Sign into Claude Code Max Plan (Client Side)
+
+Set up Claude Code environment variables and authenticate with your Max subscription.
+
+#### 2.1 Set Environment Variables
+
+Configure Claude Code to use LiteLLM Gateway with your virtual key:
+
+```bash showLineNumbers title="Configure Claude Code Environment Variables"
+export ANTHROPIC_BASE_URL=http://localhost:4000
+export ANTHROPIC_MODEL="anthropic-claude"
+export ANTHROPIC_CUSTOM_HEADERS="x-litellm-api-key: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+<Image img={require('../../img/claude_code_max/step15.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### Environment Variables Explained
+
+| Variable | Description |
+|----------|-------------|
+| `ANTHROPIC_BASE_URL` | Points Claude Code to your LiteLLM Gateway endpoint |
+| `ANTHROPIC_MODEL` | The model name configured in your LiteLLM `config.yaml` |
+| `ANTHROPIC_CUSTOM_HEADERS` | The `x-litellm-api-key` header for LiteLLM authentication |
+
+#### 2.2 Launch Claude Code
+
+Start Claude Code:
+
+```bash showLineNumbers title="Launch Claude Code"
+claude
+```
+
+<Image img={require('../../img/claude_code_max/step16.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.3 Select Login Method
+
+Choose "Claude account with subscription" (Pro, Max, Team, or Enterprise).
+
+<Image img={require('../../img/claude_code_max/step17.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.4 Authorize in Browser
+
+Claude Code opens your browser to authenticate. Click "Authorize" to connect your Claude Max account.
+
+<Image img={require('../../img/claude_code_max/step19.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.5 Login Successful
+
+After authorization, you'll see the login success confirmation.
+
+<Image img={require('../../img/claude_code_max/step20.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 2.6 Complete Setup
+
+Press Enter to continue past the security notes and complete the setup.
+
+<Image img={require('../../img/claude_code_max/step21.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+### Part 3: Use Claude Code with LiteLLM
+
+Now you can use Claude Code normally, and all requests will be tracked in LiteLLM.
+
+#### 3.1 Make a Request in Claude Code
+
+Start using Claude Code - requests will flow through LiteLLM Gateway.
+
+<Image img={require('../../img/claude_code_max/step24.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 3.2 View Logs in LiteLLM Dashboard
+
+Navigate to the Logs page in LiteLLM Dashboard to see all Claude Code requests.
+
+<Image img={require('../../img/claude_code_max/step25.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+#### 3.3 View Request Details
+
+Click on a request to see detailed information including tokens, cost, duration, and model used.
+
+<Image img={require('../../img/claude_code_max/step27.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+The logs show:
+- **Key Name**: `claude-code-test` (the virtual key you created)
+- **Model**: `anthropic/claude-sonnet-4-20250514`
+- **Tokens**: 65012 (64679 prompt + 333 completion)
+- **Cost**: $0.249754
+- **Status**: Success
+
+<Image img={require('../../img/claude_code_max/step28.jpeg')} style={{ width: '800px', height: 'auto' }} />
+
+---
+
+## How It Works
+
+LiteLLM Gateway handles two types of authentication:
+1. **`x-litellm-api-key`**: Authenticates the request with LiteLLM (usage tracking, budgets, rate limits)
+2. **OAuth Token (via `Authorization` header)**: Forwarded to Anthropic API for Claude Max authentication
+
+```mermaid
+sequenceDiagram
+    participant User as Claude Code User
+    participant LiteLLM as LiteLLM AI Gateway
+    participant Anthropic as Anthropic API
+
+    User->>LiteLLM: Request with:<br/>- x-litellm-api-key (LiteLLM auth)<br/>- Authorization: Bearer {oauth_token}
+
+    Note over LiteLLM: 1. Validate x-litellm-api-key<br/>2. Check budgets/rate limits<br/>3. Log request for tracking
+
+    LiteLLM->>Anthropic: Forward request with:<br/>- Authorization: Bearer {oauth_token}<br/>(User's Claude Max OAuth token)
+
+    Note over Anthropic: Authenticate user via<br/>OAuth token from Max plan
+
+    Anthropic-->>LiteLLM: Response
+
+    Note over LiteLLM: Log usage, tokens, cost
+
+    LiteLLM-->>User: Response
+```
+
+### Header Flow
+
+| Header | Purpose | Handled By |
+|--------|---------|------------|
+| `x-litellm-api-key` | LiteLLM Gateway authentication, budget tracking, rate limits | LiteLLM |
+| `Authorization: Bearer {oauth_token}` | Claude Max subscription authentication | Anthropic API |
+
+### Complete Request Flow Example
+
+Here's what a typical request looks like when Claude Code makes a call through LiteLLM:
+
+```bash showLineNumbers title="Example Request from Claude Code to LiteLLM"
+curl -X POST "http://localhost:4000/v1/messages" \
+  -H "x-litellm-api-key: Bearer sk-otsclFlEblQ-6D60ua2IZg" \
+  -H "Authorization: Bearer oauth_token_from_max_plan" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model": "anthropic-claude",
+    "max_tokens": 1024,
+    "messages": [{"role": "user", "content": "Hello, Claude!"}]
+  }'
+```
+
+LiteLLM then:
+1. Validates `x-litellm-api-key` for gateway access
+2. Logs the request for usage tracking
+3. Forwards the request to Anthropic with the OAuth `Authorization` header (because of `forward_client_headers_to_llm_api: true`)
+
+## Advanced Configuration
+
+### Per-Model Header Forwarding
+
+For more granular control, you can enable header forwarding only for specific models:
+
+```yaml showLineNumbers title="config.yaml - Per-Model Header Forwarding"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+  - model_name: claude-3-5-haiku-20241022
+    litellm_params:
+      model: anthropic/claude-3-5-haiku-20241022
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+  model_group_settings:
+    forward_client_headers_to_llm_api:
+      - anthropic-claude
+      - claude-3-5-haiku-20241022
+```
+
+### Budget Controls
+
+Set up per-user budgets while using Max subscriptions:
+
+```yaml showLineNumbers title="config.yaml - With Database for Budget Tracking"
+model_list:
+  - model_name: anthropic-claude
+    litellm_params:
+      model: anthropic/claude-sonnet-4-20250514
+
+general_settings:
+  forward_client_headers_to_llm_api: true
+  database_url: "postgresql://..."
+
+litellm_settings:
+  master_key: os.environ/LITELLM_MASTER_KEY
+```
+
+Then create virtual keys with budgets:
+
+```bash showLineNumbers title="Create Virtual Key with Budget"
+curl -X POST "http://localhost:4000/key/generate" \
+  -H "Authorization: Bearer $LITELLM_MASTER_KEY" \
+  -H "Content-Type: application/json" \
+  -d '{
+    "key_alias": "developer-1",
+    "max_budget": 100.00,
+    "budget_duration": "monthly"
+  }'
+```
+
+## Troubleshooting
+
+### OAuth Token Not Being Forwarded
+
+**Symptom**: Authentication errors from Anthropic API
+
+**Solution**: Ensure `forward_client_headers_to_llm_api: true` is set in your config:
+
+```yaml showLineNumbers title="config.yaml - Enable Header Forwarding"
+general_settings:
+  forward_client_headers_to_llm_api: true
+```
+
+### LiteLLM Authentication Failing
+
+**Symptom**: 401 errors from LiteLLM Gateway
+
+**Solution**: Verify `x-litellm-api-key` header is set correctly in `ANTHROPIC_CUSTOM_HEADERS`:
+
+```bash showLineNumbers title="Verify Key Info"
+curl -X GET "http://localhost:4000/key/info" \
+  -H "Authorization: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+### Model Not Found
+
+**Symptom**: Model not found errors
+
+**Solution**: Ensure the `ANTHROPIC_MODEL` matches a model name in your config:
+
+```bash showLineNumbers title="List Available Models"
+curl "http://localhost:4000/v1/models" \
+  -H "Authorization: Bearer sk-otsclFlEblQ-6D60ua2IZg"
+```
+
+## Related Documentation
+
+- [Forward Client Headers](/docs/proxy/forward_client_headers) - Detailed header forwarding configuration
+- [Claude Code Quickstart](/docs/tutorials/claude_responses_api) - Basic Claude Code + LiteLLM setup
+- [Virtual Keys](/docs/proxy/virtual_keys) - Creating and managing API keys
+- [Budgets & Rate Limits](/docs/proxy/users) - Setting up usage controls

docs/my-website/release_notes/v1.81.0/index.md

@@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem';
 docker run \
 -e STORE_MODEL_IN_DB=True \
 -p 4000:4000 \
-docker.litellm.ai/berriai/litellm:v1.81.0
+docker.litellm.ai/berriai/litellm:v1.81.0.rc.1
 ```
 
 </TabItem>

docs/my-website/sidebars.js

@@ -121,6 +121,7 @@ const sidebars = {
           label: "Claude Code",
           items: [
             "tutorials/claude_responses_api",
+            "tutorials/claude_code_max_subscription",
             "tutorials/claude_code_customer_tracking",
             "tutorials/claude_code_websearch",
             "tutorials/claude_mcp",
@@ -718,6 +719,7 @@ const sidebars = {
         "providers/galadriel",
         "providers/github",
         "providers/github_copilot",
+        "providers/chatgpt",
         "providers/gradient_ai",
         "providers/groq",
         "providers/helicone",

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "litellm-proxy-extras"
-version = "0.4.23"
+version = "0.4.25"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 authors = ["BerriAI"]
 readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
 [tool.commitizen]
-version = "0.4.23"
+version = "0.4.25"
 version_files = [
     "pyproject.toml:version",
     "../requirements.txt:litellm-proxy-extras==",

litellm/__init__.py

@@ -557,6 +557,7 @@ docker_model_runner_models: Set = set()
 amazon_nova_models: Set = set()
 stability_models: Set = set()
 github_copilot_models: Set = set()
+chatgpt_models: Set = set()
 minimax_models: Set = set()
 aws_polly_models: Set = set()
 gigachat_models: Set = set()
@@ -812,6 +813,8 @@ def add_known_models():
             stability_models.add(key)
         elif value.get("litellm_provider") == "github_copilot":
             github_copilot_models.add(key)
+        elif value.get("litellm_provider") == "chatgpt":
+            chatgpt_models.add(key)
         elif value.get("litellm_provider") == "minimax":
             minimax_models.add(key)
         elif value.get("litellm_provider") == "aws_polly":
@@ -1025,6 +1028,7 @@ models_by_provider: dict = {
     "amazon_nova": amazon_nova_models,
     "stability": stability_models,
     "github_copilot": github_copilot_models,
+    "chatgpt": chatgpt_models,
     "minimax": minimax_models,
     "aws_polly": aws_polly_models,
     "gigachat": gigachat_models,
@@ -1268,7 +1272,7 @@ if TYPE_CHECKING:
     from litellm.types.utils import ModelInfo as _ModelInfoType
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
     from litellm.caching.caching import Cache
-    
+
     # Type stubs for lazy-loaded configs to help mypy
     from .llms.bedrock.chat.converse_transformation import AmazonConverseConfig as AmazonConverseConfig
     from .llms.openai_like.chat.handler import OpenAILikeChatConfig as OpenAILikeChatConfig
@@ -1374,6 +1378,7 @@ if TYPE_CHECKING:
     from .llms.azure.responses.o_series_transformation import AzureOpenAIOSeriesResponsesAPIConfig as AzureOpenAIOSeriesResponsesAPIConfig
     from .llms.xai.responses.transformation import XAIResponsesAPIConfig as XAIResponsesAPIConfig
     from .llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig as LiteLLMProxyResponsesAPIConfig
+    from .llms.volcengine.responses.transformation import VolcEngineResponsesAPIConfig as VolcEngineResponsesAPIConfig
     from .llms.manus.responses.transformation import ManusResponsesAPIConfig as ManusResponsesAPIConfig
     from .llms.gemini.interactions.transformation import GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig
     from .llms.openai.chat.o_series_transformation import OpenAIOSeriesConfig as OpenAIOSeriesConfig, OpenAIOSeriesConfig as OpenAIO1Config
@@ -1387,7 +1392,7 @@ if TYPE_CHECKING:
     from .llms.openai.chat.gpt_audio_transformation import OpenAIGPTAudioConfig as OpenAIGPTAudioConfig
     from .llms.nvidia_nim.chat.transformation import NvidiaNimConfig as NvidiaNimConfig
     from .llms.nvidia_nim.embed import NvidiaNimEmbeddingConfig as NvidiaNimEmbeddingConfig
-    
+
     # Type stubs for lazy-loaded config instances
     openaiOSeriesConfig: OpenAIOSeriesConfig
     openAIGPTConfig: OpenAIGPTConfig
@@ -1395,7 +1400,7 @@ if TYPE_CHECKING:
     openAIGPT5Config: OpenAIGPT5Config
     nvidiaNimConfig: NvidiaNimConfig
     nvidiaNimEmbeddingConfig: NvidiaNimEmbeddingConfig
-    
+
     # Import config classes that need type stubs (for mypy) - import with _ prefix to avoid circular reference
     from .llms.vllm.completion.transformation import VLLMConfig as _VLLMConfig
     from .llms.deepseek.chat.transformation import DeepSeekChatConfig as _DeepSeekChatConfig
@@ -1413,7 +1418,7 @@ if TYPE_CHECKING:
     from .llms.lm_studio.embed.transformation import LmStudioEmbeddingConfig as _LmStudioEmbeddingConfig
     from .llms.watsonx.embed.transformation import IBMWatsonXEmbeddingConfig as _IBMWatsonXEmbeddingConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig as _VertexGeminiConfig
-    
+
     # Type stubs for lazy-loaded config classes (to help mypy understand types)
     VLLMConfig: Type[_VLLMConfig]
     DeepSeekChatConfig: Type[_DeepSeekChatConfig]
@@ -1431,7 +1436,7 @@ if TYPE_CHECKING:
     LmStudioEmbeddingConfig: Type[_LmStudioEmbeddingConfig]
     IBMWatsonXEmbeddingConfig: Type[_IBMWatsonXEmbeddingConfig]
     VertexAIConfig: Type[_VertexGeminiConfig]  # Alias for VertexGeminiConfig
-    
+
     from .llms.featherless_ai.chat.transformation import FeatherlessAIConfig as FeatherlessAIConfig
     from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig
     from .llms.baseten.chat import BasetenConfig as BasetenConfig
@@ -1458,6 +1463,8 @@ if TYPE_CHECKING:
     from .llms.github_copilot.chat.transformation import GithubCopilotConfig as GithubCopilotConfig
     from .llms.github_copilot.responses.transformation import GithubCopilotResponsesAPIConfig as GithubCopilotResponsesAPIConfig
     from .llms.github_copilot.embedding.transformation import GithubCopilotEmbeddingConfig as GithubCopilotEmbeddingConfig
+    from .llms.chatgpt.chat.transformation import ChatGPTConfig as ChatGPTConfig
+    from .llms.chatgpt.responses.transformation import ChatGPTResponsesAPIConfig as ChatGPTResponsesAPIConfig
     from .llms.gigachat.chat.transformation import GigaChatConfig as GigaChatConfig
     from .llms.gigachat.embedding.transformation import GigaChatEmbeddingConfig as GigaChatEmbeddingConfig
     from .llms.nebius.chat.transformation import NebiusConfig as NebiusConfig
@@ -1551,14 +1558,14 @@ if TYPE_CHECKING:
 
     # Custom logger class (lazy-loaded)
     from litellm.integrations.custom_logger import CustomLogger
-    
+
     # Datadog LLM observability params (lazy-loaded)
     from litellm.types.integrations.datadog_llm_obs import DatadogLLMObsInitParams
-    
+
     # Logging callback manager class and instance (lazy-loaded)
     from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager
     logging_callback_manager: LoggingCallbackManager
-    
+
     # provider_list is lazy-loaded
     from litellm.types.utils import LlmProviders
     provider_list: List[Union[LlmProviders, str]]
@@ -1588,12 +1595,12 @@ def __getattr__(name: str) -> Any:
         from litellm.llms.custom_httpx.async_client_cleanup import register_async_client_cleanup
         register_async_client_cleanup()
         _async_client_cleanup_registered = True
-    
+
     # Use cached registry from _lazy_imports instead of importing tuples every time
     from ._lazy_imports import _get_lazy_import_registry
-    
+
     registry = _get_lazy_import_registry()
-    
+
     # Check if name is in registry and call the cached handler function
     if name in registry:
         handler_func = registry[name]
@@ -1608,7 +1615,7 @@ def __getattr__(name: str) -> Any:
             from .main import encoding as _encoding
             _globals["encoding"] = _encoding
         return _globals["encoding"]
-    
+
     # Lazy load bedrock_tool_name_mappings instance
     if name == "bedrock_tool_name_mappings":
         from ._lazy_imports import _get_litellm_globals
@@ -1618,7 +1625,7 @@ def __getattr__(name: str) -> Any:
             from .llms.bedrock.chat.invoke_handler import bedrock_tool_name_mappings as _bedrock_tool_name_mappings
             _globals["bedrock_tool_name_mappings"] = _bedrock_tool_name_mappings
         return _globals["bedrock_tool_name_mappings"]
-    
+
     # Lazy load AzureOpenAIError exception class
     if name == "AzureOpenAIError":
         from ._lazy_imports import _get_litellm_globals
@@ -1628,7 +1635,7 @@ def __getattr__(name: str) -> Any:
             from .llms.azure.common_utils import AzureOpenAIError as _AzureOpenAIError
             _globals["AzureOpenAIError"] = _AzureOpenAIError
         return _globals["AzureOpenAIError"]
-    
+
     # Lazy load openaiOSeriesConfig instance
     if name == "openaiOSeriesConfig":
         from ._lazy_imports import _get_litellm_globals
@@ -1638,7 +1645,7 @@ def __getattr__(name: str) -> Any:
             config_class = __getattr__("OpenAIOSeriesConfig")
             _globals["openaiOSeriesConfig"] = config_class()
         return _globals["openaiOSeriesConfig"]
-    
+
     # Lazy load other config instances
     _config_instances = {
         "openAIGPTConfig": "OpenAIGPTConfig",
@@ -1655,11 +1662,11 @@ def __getattr__(name: str) -> Any:
             config_class = __getattr__(_config_instances[name])
             _globals[name] = config_class()
         return _globals[name]
-    
+
     # Handle OpenAIO1Config alias
     if name == "OpenAIO1Config":
         return __getattr__("OpenAIOSeriesConfig")
-    
+
     # Lazy load provider_list
     if name == "provider_list":
         from ._lazy_imports import _get_litellm_globals
@@ -1670,7 +1677,7 @@ def __getattr__(name: str) -> Any:
             from litellm.types.utils import LlmProviders
             _globals["provider_list"] = list(LlmProviders)
         return _globals["provider_list"]
-    
+
     # Lazy load priority_reservation_settings instance
     if name == "priority_reservation_settings":
         from ._lazy_imports import _get_litellm_globals
@@ -1681,7 +1688,7 @@ def __getattr__(name: str) -> Any:
             PriorityReservationSettings = __getattr__("PriorityReservationSettings")
             _globals["priority_reservation_settings"] = PriorityReservationSettings()
         return _globals["priority_reservation_settings"]
-    
+
     # Lazy load logging_callback_manager instance
     if name == "logging_callback_manager":
         from ._lazy_imports import _get_litellm_globals
@@ -1692,7 +1699,7 @@ def __getattr__(name: str) -> Any:
             LoggingCallbackManager = __getattr__("LoggingCallbackManager")
             _globals["logging_callback_manager"] = LoggingCallbackManager()
         return _globals["logging_callback_manager"]
-    
+
     # Lazy load _service_logger module
     if name == "_service_logger":
         from ._lazy_imports import _get_litellm_globals

litellm/_lazy_imports_registry.py

@@ -198,6 +198,7 @@ LLM_CONFIG_NAMES = (
     "AzureOpenAIOSeriesResponsesAPIConfig",
     "XAIResponsesAPIConfig",
     "LiteLLMProxyResponsesAPIConfig",
+    "VolcEngineResponsesAPIConfig",
     "GoogleAIStudioInteractionsConfig",
     "OpenAIOSeriesConfig",
     "AnthropicSkillsConfig",
@@ -253,6 +254,8 @@ LLM_CONFIG_NAMES = (
     "IBMWatsonXAudioTranscriptionConfig",
     "GithubCopilotConfig",
     "GithubCopilotResponsesAPIConfig",
+    "ChatGPTConfig",
+    "ChatGPTResponsesAPIConfig",
     "ManusResponsesAPIConfig",
     "GithubCopilotEmbeddingConfig",
     "NebiusConfig",
@@ -591,6 +594,7 @@ _LLM_CONFIGS_IMPORT_MAP = {
     "AzureOpenAIOSeriesResponsesAPIConfig": (".llms.azure.responses.o_series_transformation", "AzureOpenAIOSeriesResponsesAPIConfig"),
     "XAIResponsesAPIConfig": (".llms.xai.responses.transformation", "XAIResponsesAPIConfig"),
     "LiteLLMProxyResponsesAPIConfig": (".llms.litellm_proxy.responses.transformation", "LiteLLMProxyResponsesAPIConfig"),
+    "VolcEngineResponsesAPIConfig": (".llms.volcengine.responses.transformation", "VolcEngineResponsesAPIConfig"),
     "ManusResponsesAPIConfig": (".llms.manus.responses.transformation", "ManusResponsesAPIConfig"),
     "GoogleAIStudioInteractionsConfig": (".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig"),
     "OpenAIOSeriesConfig": (".llms.openai.chat.o_series_transformation", "OpenAIOSeriesConfig"),
@@ -648,6 +652,8 @@ _LLM_CONFIGS_IMPORT_MAP = {
     "GithubCopilotConfig": (".llms.github_copilot.chat.transformation", "GithubCopilotConfig"),
     "GithubCopilotResponsesAPIConfig": (".llms.github_copilot.responses.transformation", "GithubCopilotResponsesAPIConfig"),
     "GithubCopilotEmbeddingConfig": (".llms.github_copilot.embedding.transformation", "GithubCopilotEmbeddingConfig"),
+    "ChatGPTConfig": (".llms.chatgpt.chat.transformation", "ChatGPTConfig"),
+    "ChatGPTResponsesAPIConfig": (".llms.chatgpt.responses.transformation", "ChatGPTResponsesAPIConfig"),
     "NebiusConfig": (".llms.nebius.chat.transformation", "NebiusConfig"),
     "WandbConfig": (".llms.wandb.chat.transformation", "WandbConfig"),
     "GigaChatConfig": (".llms.gigachat.chat.transformation", "GigaChatConfig"),
@@ -774,4 +780,3 @@ __all__ = [
     "_LLM_PROVIDER_LOGIC_IMPORT_MAP",
     "_UTILS_MODULE_IMPORT_MAP",
 ]
-

litellm/_service_logger.py

@@ -145,16 +145,19 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                _otel_logger_to_use: Optional[OpenTelemetry] = None
+                if isinstance(callback, OpenTelemetry):
+                    _otel_logger_to_use = callback
+                else:
+                    from litellm.proxy.proxy_server import open_telemetry_logger
 
-                await self.init_otel_logger_if_none()
+                    if open_telemetry_logger is not None and isinstance(
+                        open_telemetry_logger, OpenTelemetry
+                    ):
+                        _otel_logger_to_use = open_telemetry_logger
 
-                if (
-                    parent_otel_span is not None
-                    and open_telemetry_logger is not None
-                    and isinstance(open_telemetry_logger, OpenTelemetry)
-                ):
-                    await self.otel_logger.async_service_success_hook(
+                if _otel_logger_to_use is not None and parent_otel_span is not None:
+                    await _otel_logger_to_use.async_service_success_hook(
                         payload=payload,
                         parent_otel_span=parent_otel_span,
                         start_time=start_time,
@@ -253,20 +256,24 @@ class ServiceLogging(CustomLogger):
                     event_metadata=event_metadata,
                 )
             elif callback == "otel" or isinstance(callback, OpenTelemetry):
-                from litellm.proxy.proxy_server import open_telemetry_logger
+                _otel_logger_to_use: Optional[OpenTelemetry] = None
+                if isinstance(callback, OpenTelemetry):
+                    _otel_logger_to_use = callback
+                else:
+                    from litellm.proxy.proxy_server import open_telemetry_logger
 
-                await self.init_otel_logger_if_none()
+                    if open_telemetry_logger is not None and isinstance(
+                        open_telemetry_logger, OpenTelemetry
+                    ):
+                        _otel_logger_to_use = open_telemetry_logger
 
                 if not isinstance(error, str):
                     error = str(error)
 
-                if (
-                    parent_otel_span is not None
-                    and open_telemetry_logger is not None
-                    and isinstance(open_telemetry_logger, OpenTelemetry)
-                ):
-                    await self.otel_logger.async_service_success_hook(
+                if _otel_logger_to_use is not None and parent_otel_span is not None:
+                    await _otel_logger_to_use.async_service_failure_hook(
                         payload=payload,
+                        error=error,
                         parent_otel_span=parent_otel_span,
                         start_time=start_time,
                         end_time=end_time,

litellm/completion_extras/litellm_responses_transformation/handler.py

@@ -2,10 +2,12 @@
 Handler for transforming /chat/completions api requests to litellm.responses requests
 """
 
-from typing import TYPE_CHECKING, Any, Coroutine, Union
+from typing import TYPE_CHECKING, Any, Coroutine, Optional, Union
 
 from typing_extensions import TypedDict
 
+from litellm.types.llms.openai import ResponsesAPIResponse
+
 if TYPE_CHECKING:
     from litellm import CustomStreamWrapper, LiteLLMLoggingObj, ModelResponse
 
@@ -28,6 +30,71 @@ class ResponsesToCompletionBridgeHandler:
         super().__init__()
         self.transformation_handler = LiteLLMResponsesTransformationHandler()
 
+    @staticmethod
+    def _resolve_stream_flag(optional_params: dict, litellm_params: dict) -> bool:
+        stream = optional_params.get("stream")
+        if stream is None:
+            stream = litellm_params.get("stream", False)
+        return bool(stream)
+
+    @staticmethod
+    def _coerce_response_object(
+        response_obj: Any,
+        hidden_params: Optional[dict],
+    ) -> "ResponsesAPIResponse":
+        if isinstance(response_obj, ResponsesAPIResponse):
+            response = response_obj
+        elif isinstance(response_obj, dict):
+            try:
+                response = ResponsesAPIResponse(**response_obj)
+            except Exception:
+                response = ResponsesAPIResponse.model_construct(**response_obj)
+        else:
+            raise ValueError("Unexpected responses stream payload")
+
+        if hidden_params:
+            existing = getattr(response, "_hidden_params", None)
+            if not isinstance(existing, dict) or not existing:
+                setattr(response, "_hidden_params", dict(hidden_params))
+            else:
+                for key, value in hidden_params.items():
+                    existing.setdefault(key, value)
+        return response
+
+    def _collect_response_from_stream(
+        self, stream_iter: Any
+    ) -> "ResponsesAPIResponse":
+        for _ in stream_iter:
+            pass
+
+        completed = getattr(stream_iter, "completed_response", None)
+        response_obj = getattr(completed, "response", None) if completed else None
+        if response_obj is None:
+            raise ValueError("Stream ended without a completed response")
+
+        hidden_params = getattr(stream_iter, "_hidden_params", None)
+        response = self._coerce_response_object(response_obj, hidden_params)
+        if not isinstance(response, ResponsesAPIResponse):
+            raise ValueError("Stream completed response is invalid")
+        return response
+
+    async def _collect_response_from_stream_async(
+        self, stream_iter: Any
+    ) -> "ResponsesAPIResponse":
+        async for _ in stream_iter:
+            pass
+
+        completed = getattr(stream_iter, "completed_response", None)
+        response_obj = getattr(completed, "response", None) if completed else None
+        if response_obj is None:
+            raise ValueError("Stream ended without a completed response")
+
+        hidden_params = getattr(stream_iter, "_hidden_params", None)
+        response = self._coerce_response_object(response_obj, hidden_params)
+        if not isinstance(response, ResponsesAPIResponse):
+            raise ValueError("Stream completed response is invalid")
+        return response
+
     def validate_input_kwargs(
         self, kwargs: dict
     ) -> ResponsesToCompletionBridgeHandlerInputKwargs:
@@ -87,7 +154,6 @@ class ResponsesToCompletionBridgeHandler:
 
         from litellm import responses
         from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
-        from litellm.types.llms.openai import ResponsesAPIResponse
 
         validated_kwargs = self.validate_input_kwargs(kwargs)
         model = validated_kwargs["model"]
@@ -113,6 +179,7 @@ class ResponsesToCompletionBridgeHandler:
             **request_data,
         )
 
+        stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
                 model=model,
@@ -127,6 +194,21 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif not stream:
+            responses_api_response = self._collect_response_from_stream(result)
+            return self.transformation_handler.transform_response(
+                model=model,
+                raw_response=responses_api_response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                request_data=request_data,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                encoding=kwargs.get("encoding"),
+                api_key=kwargs.get("api_key"),
+                json_mode=kwargs.get("json_mode"),
+            )
         else:
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore
@@ -146,7 +228,6 @@ class ResponsesToCompletionBridgeHandler:
     ) -> Union["ModelResponse", "CustomStreamWrapper"]:
         from litellm import aresponses
         from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
-        from litellm.types.llms.openai import ResponsesAPIResponse
 
         validated_kwargs = self.validate_input_kwargs(kwargs)
         model = validated_kwargs["model"]
@@ -175,6 +256,7 @@ class ResponsesToCompletionBridgeHandler:
             aresponses=True,
         )
 
+        stream = self._resolve_stream_flag(optional_params, litellm_params)
         if isinstance(result, ResponsesAPIResponse):
             return self.transformation_handler.transform_response(
                 model=model,
@@ -189,6 +271,23 @@ class ResponsesToCompletionBridgeHandler:
                 api_key=kwargs.get("api_key"),
                 json_mode=kwargs.get("json_mode"),
             )
+        elif not stream:
+            responses_api_response = await self._collect_response_from_stream_async(
+                result
+            )
+            return self.transformation_handler.transform_response(
+                model=model,
+                raw_response=responses_api_response,
+                model_response=model_response,
+                logging_obj=logging_obj,
+                request_data=request_data,
+                messages=messages,
+                optional_params=optional_params,
+                litellm_params=litellm_params,
+                encoding=kwargs.get("encoding"),
+                api_key=kwargs.get("api_key"),
+                json_mode=kwargs.get("json_mode"),
+            )
         else:
             completion_stream = self.transformation_handler.get_model_response_iterator(
                 streaming_response=result,  # type: ignore

litellm/completion_extras/litellm_responses_transformation/transformation.py

@@ -779,10 +779,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
     @staticmethod
     def _convert_annotations_to_chat_format(
         annotations: Optional[List[Any]],
-    ) -> Optional[List["ChatCompletionAnnotation"]]:
+    ) -> Optional[List[ChatCompletionAnnotation]]:
         """
         Convert annotations from Responses API to Chat Completions format.
-        
+
         Annotations are already in compatible format between both APIs,
         so we just need to convert Pydantic models to dicts.
         """

litellm/constants.py

@@ -323,6 +323,9 @@ EMAIL_BUDGET_ALERT_TTL = int(os.getenv("EMAIL_BUDGET_ALERT_TTL", 24 * 60 * 60))
 EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE = float(os.getenv("EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE", 0.8))  # 80% of max budget
 ############### LLM Provider Constants ###############
 ### ANTHROPIC CONSTANTS ###
+ANTHROPIC_TOKEN_COUNTING_BETA_VERSION = os.getenv(
+    "ANTHROPIC_TOKEN_COUNTING_BETA_VERSION", "token-counting-2024-11-01"
+)
 ANTHROPIC_SKILLS_API_BETA_VERSION = "skills-2025-10-02"
 ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES = {
     "low": 1,
@@ -415,6 +418,7 @@ LITELLM_CHAT_PROVIDERS = [
     "galadriel",
     "gradient_ai",
     "github_copilot",  # GitHub Copilot Chat API
+    "chatgpt",  # ChatGPT subscription API
     "novita",
     "meta_llama",
     "featherless_ai",
@@ -542,6 +546,10 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = {
     "web_search_options": None,
     "service_tier": None,
     "safety_identifier": None,
+    "prompt_cache_key": None,
+    "prompt_cache_retention": None,
+    "store": None,
+    "metadata": None,
 }
 
 openai_compatible_endpoints: List = [
@@ -613,6 +621,7 @@ openai_compatible_providers: List = [
     "lm_studio",
     "galadriel",
     "github_copilot",  # GitHub Copilot Chat API
+    "chatgpt",  # ChatGPT subscription API
     "novita",
     "meta_llama",
     "publicai",  # PublicAI - JSON-configured provider

litellm/experimental_mcp_client/client.py

@@ -4,14 +4,13 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers.
 
 import asyncio
 import base64
-from datetime import timedelta
 from typing import Awaitable, Callable, Dict, List, Optional, TypeVar, Union
 
 import httpx
 from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters
 from mcp.client.sse import sse_client
 from mcp.client.stdio import stdio_client
-from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.streamable_http import streamable_http_client
 from mcp.types import (
     CallToolRequestParams as MCPCallToolRequestParams,
     GetPromptRequestParams,
@@ -80,6 +79,7 @@ class MCPClient:
     ) -> TSessionResult:
         """Open a session, run the provided coroutine, and clean up."""
         transport_ctx = None
+        http_client: Optional[httpx.AsyncClient] = None
 
         try:
             if self.transport_type == MCPTransport.stdio:
@@ -105,13 +105,15 @@ class MCPClient:
                 headers = self._get_auth_headers()
                 httpx_client_factory = self._create_httpx_client_factory()
                 verbose_logger.debug(
-                    "litellm headers for streamablehttp_client: %s", headers
+                    "litellm headers for streamable_http_client: %s", headers
                 )
-                transport_ctx = streamablehttp_client(
-                    url=self.server_url,
-                    timeout=timedelta(seconds=self.timeout),
+                http_client = httpx_client_factory(
                     headers=headers,
-                    httpx_client_factory=httpx_client_factory,
+                    timeout=httpx.Timeout(self.timeout),
+                )
+                transport_ctx = streamable_http_client(
+                    url=self.server_url,
+                    http_client=http_client,
                 )
 
             if transport_ctx is None:
@@ -128,6 +130,9 @@ class MCPClient:
                 "MCP client run_with_session failed for %s", self.server_url or "stdio"
             )
             raise
+        finally:
+            if http_client is not None:
+                await http_client.aclose()
 
     def update_auth_value(self, mcp_auth_value: Union[str, Dict[str, str]]):
         """

litellm/integrations/langfuse/langfuse_otel.py

@@ -156,7 +156,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         "arguments": arguments_obj,
                     }
                     transformed_tool_calls.append(langfuse_tool_call)
-                safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(transformed_tool_calls))
+                safe_set_attribute(
+                    span,
+                    LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                    safe_dumps(transformed_tool_calls),
+                )
             else:
                 output_data = {}
                 if message.get("role"):
@@ -164,7 +168,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                 if message.get("content") is not None:
                     output_data["content"] = message.get("content")
                 if output_data:
-                    safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_data))
+                    safe_set_attribute(
+                        span,
+                        LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                        safe_dumps(output_data),
+                    )
 
         output = response_obj.get("output", [])
         if output:
@@ -175,15 +183,28 @@ class LangfuseOtelLogger(OpenTelemetry):
                     if item_type == "reasoning" and hasattr(item, "summary"):
                         for summary in item.summary:
                             if hasattr(summary, "text"):
-                                output_items_data.append({"role": "reasoning_summary", "content": summary.text})
+                                output_items_data.append(
+                                    {
+                                        "role": "reasoning_summary",
+                                        "content": summary.text,
+                                    }
+                                )
                     elif item_type == "message":
-                        output_items_data.append({
-                            "role": getattr(item, "role", "assistant"),
-                            "content": getattr(getattr(item, "content", [{}])[0], "text", "")
-                        })
+                        output_items_data.append(
+                            {
+                                "role": getattr(item, "role", "assistant"),
+                                "content": getattr(
+                                    getattr(item, "content", [{}])[0], "text", ""
+                                ),
+                            }
+                        )
                     elif item_type == "function_call":
                         arguments_str = getattr(item, "arguments", "{}")
-                        arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
+                        arguments_obj = (
+                            json.loads(arguments_str)
+                            if isinstance(arguments_str, str)
+                            else arguments_str
+                        )
                         langfuse_tool_call = {
                             "id": getattr(item, "id", ""),
                             "name": getattr(item, "name", ""),
@@ -193,7 +214,11 @@ class LangfuseOtelLogger(OpenTelemetry):
                         }
                         output_items_data.append(langfuse_tool_call)
             if output_items_data:
-                safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_OUTPUT.value, safe_dumps(output_items_data))
+                safe_set_attribute(
+                    span,
+                    LangfuseSpanAttributes.OBSERVATION_OUTPUT.value,
+                    safe_dumps(output_items_data),
+                )
 
     @staticmethod
     def _set_langfuse_specific_attributes(span: Span, kwargs, response_obj):
@@ -210,14 +235,22 @@ class LangfuseOtelLogger(OpenTelemetry):
 
         langfuse_environment = os.environ.get("LANGFUSE_TRACING_ENVIRONMENT")
         if langfuse_environment:
-            safe_set_attribute(span, LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value, langfuse_environment)
+            safe_set_attribute(
+                span,
+                LangfuseSpanAttributes.LANGFUSE_ENVIRONMENT.value,
+                langfuse_environment,
+            )
 
         metadata = LangfuseOtelLogger._extract_langfuse_metadata(kwargs)
         LangfuseOtelLogger._set_metadata_attributes(span=span, metadata=metadata)
 
         messages = kwargs.get("messages")
         if messages:
-            safe_set_attribute(span, LangfuseSpanAttributes.OBSERVATION_INPUT.value, safe_dumps(messages))
+            safe_set_attribute(
+                span,
+                LangfuseSpanAttributes.OBSERVATION_INPUT.value,
+                safe_dumps(messages),
+            )
 
         LangfuseOtelLogger._set_observation_output(span=span, response_obj=response_obj)
 
@@ -319,3 +352,15 @@ class LangfuseOtelLogger(OpenTelemetry):
             dynamic_headers["Authorization"] = auth_header
 
         return dynamic_headers
+
+    async def async_service_success_hook(self, *args, **kwargs):
+        """
+        Langfuse should not receive service success logs.
+        """
+        pass
+
+    async def async_service_failure_hook(self, *args, **kwargs):
+        """
+        Langfuse should not receive service failure logs.
+        """
+        pass

litellm/integrations/opentelemetry.py

@@ -988,7 +988,7 @@ class OpenTelemetry(CustomLogger):
 
         from opentelemetry._logs import SeverityNumber, get_logger, get_logger_provider
         try:
-            from opentelemetry.sdk._logs import LogRecord as SdkLogRecord  # OTEL < 1.39.0
+            from opentelemetry.sdk._logs import LogRecord as SdkLogRecord  # type: ignore[attr-defined]  # OTEL < 1.39.0
         except ImportError:
             from opentelemetry.sdk._logs._internal import LogRecord as SdkLogRecord  # OTEL >= 1.39.0
 

litellm/litellm_core_utils/get_llm_provider_logic.py

@@ -768,6 +768,14 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
         ) = litellm.GithubCopilotConfig()._get_openai_compatible_provider_info(
             model, api_base, api_key, custom_llm_provider
         )
+    elif custom_llm_provider == "chatgpt":
+        (
+            api_base,
+            dynamic_api_key,
+            custom_llm_provider,
+        ) = litellm.ChatGPTConfig()._get_openai_compatible_provider_info(
+            model, api_base, api_key, custom_llm_provider
+        )
     elif custom_llm_provider == "novita":
         api_base = (
             api_base

litellm/litellm_core_utils/litellm_logging.py

@@ -1624,15 +1624,7 @@ class Logging(LiteLLMLoggingBaseClass):
                     result.usage
                 )
             )
-            setattr(
-                result,
-                "usage",
-                (
-                    transformed_usage.model_dump()
-                    if hasattr(transformed_usage, "model_dump")
-                    else dict(transformed_usage)
-                ),
-            )
+            setattr(result, "usage", transformed_usage)
             if (
                 standard_logging_payload := self.model_call_details.get(
                     "standard_logging_object"
@@ -1897,6 +1889,14 @@ class Logging(LiteLLMLoggingBaseClass):
                     status="success",
                     standard_built_in_tools_params=self.standard_built_in_tools_params,
                 )
+                if (
+                    standard_logging_payload := self.model_call_details.get(
+                        "standard_logging_object"
+                    )
+                ) is not None:
+                    # Only emit for sync requests (async_success_handler handles async)
+                    if is_sync_request:
+                        emit_standard_logging_payload(standard_logging_payload)
             callbacks = self.get_combined_callback_list(
                 dynamic_success_callbacks=self.dynamic_success_callbacks,
                 global_callbacks=litellm.success_callback,
@@ -2190,10 +2190,7 @@ class Logging(LiteLLMLoggingBaseClass):
                                 print_verbose=print_verbose,
                             )
 
-                    if (
-                        callback == "openmeter"
-                        and is_sync_request
-                    ):
+                    if callback == "openmeter" and is_sync_request:
                         global openMeterLogger
                         if openMeterLogger is None:
                             print_verbose("Instantiates openmeter client")
@@ -2405,6 +2402,14 @@ class Logging(LiteLLMLoggingBaseClass):
                 status="success",
                 standard_built_in_tools_params=self.standard_built_in_tools_params,
             )
+
+            # print standard logging payload
+            if (
+                standard_logging_payload := self.model_call_details.get(
+                    "standard_logging_object"
+                )
+            ) is not None:
+                emit_standard_logging_payload(standard_logging_payload)
         callbacks = self.get_combined_callback_list(
             dynamic_success_callbacks=self.dynamic_async_success_callbacks,
             global_callbacks=litellm._async_success_callback,
@@ -2799,8 +2804,7 @@ class Logging(LiteLLMLoggingBaseClass):
                             callback_func=callback,
                         )
                     if (
-                        isinstance(callback, CustomLogger)
-                        and is_sync_request
+                        isinstance(callback, CustomLogger) and is_sync_request
                     ):  # custom logger class
                         callback.log_failure_event(
                             start_time=start_time,
@@ -3743,10 +3747,13 @@ def _init_custom_logger_compatible_class(  # noqa: PLR0915
                 OpenTelemetry,
                 OpenTelemetryConfig,
             )
-            logfire_base_url = os.getenv("LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev")
+
+            logfire_base_url = os.getenv(
+                "LOGFIRE_BASE_URL", "https://logfire-api.pydantic.dev"
+            )
             otel_config = OpenTelemetryConfig(
                 exporter="otlp_http",
-                endpoint = f"{logfire_base_url.rstrip('/')}/v1/traces",
+                endpoint=f"{logfire_base_url.rstrip('/')}/v1/traces",
                 headers=f"Authorization={os.getenv('LOGFIRE_TOKEN')}",
             )
             for callback in _in_memory_loggers:
@@ -4342,32 +4349,38 @@ class StandardLoggingPayloadSetup:
     def merge_litellm_metadata(litellm_params: dict) -> dict:
         """
         Merge both litellm_metadata and metadata from litellm_params.
-        
+
         litellm_metadata contains model-related fields, metadata contains user API key fields.
         We need both for complete standard logging payload.
-        
+
         Args:
             litellm_params: Dictionary containing metadata and litellm_metadata
-            
+
         Returns:
             dict: Merged metadata with user API key fields taking precedence
         """
         merged_metadata: dict = {}
-        
+
         # Start with metadata (user API key fields) - but skip non-serializable objects
-        if litellm_params.get("metadata") and isinstance(litellm_params.get("metadata"), dict):
+        if litellm_params.get("metadata") and isinstance(
+            litellm_params.get("metadata"), dict
+        ):
             for key, value in litellm_params["metadata"].items():
                 # Skip non-serializable objects like UserAPIKeyAuth
                 if key == "user_api_key_auth":
                     continue
                 merged_metadata[key] = value
-        
+
         # Then merge litellm_metadata (model-related fields) - this will NOT overwrite existing keys
-        if litellm_params.get("litellm_metadata") and isinstance(litellm_params.get("litellm_metadata"), dict):
+        if litellm_params.get("litellm_metadata") and isinstance(
+            litellm_params.get("litellm_metadata"), dict
+        ):
             for key, value in litellm_params["litellm_metadata"].items():
-                if key not in merged_metadata:  # Don't overwrite existing keys from metadata
+                if (
+                    key not in merged_metadata
+                ):  # Don't overwrite existing keys from metadata
                     merged_metadata[key] = value
-        
+
         return merged_metadata
 
     @staticmethod
@@ -4810,7 +4823,9 @@ class StandardLoggingPayloadSetup:
         """
         Extract additional header tags for spend tracking based on config.
         """
-        extra_headers: List[str] = getattr(litellm, "extra_spend_tag_headers", None) or []
+        extra_headers: List[str] = (
+            getattr(litellm, "extra_spend_tag_headers", None) or []
+        )
         if not extra_headers:
             return None
 
@@ -4959,7 +4974,9 @@ def get_standard_logging_object_payload(
         proxy_server_request = litellm_params.get("proxy_server_request") or {}
 
         # Merge both litellm_metadata and metadata to get complete metadata
-        metadata: dict = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
+        metadata: dict = StandardLoggingPayloadSetup.merge_litellm_metadata(
+            litellm_params
+        )
 
         completion_start_time = kwargs.get("completion_start_time", end_time)
         call_type = kwargs.get("call_type")
@@ -5129,7 +5146,8 @@ def get_standard_logging_object_payload(
             standard_built_in_tools_params=standard_built_in_tools_params,
         )
 
-        emit_standard_logging_payload(payload)
+        # emit_standard_logging_payload(payload) - Moved to success_handler to prevent double emitting
+
         return payload
     except Exception as e:
         verbose_logger.exception(

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -354,7 +354,7 @@ class PromptTokensDetailsResult(TypedDict):
     image_tokens: int
     character_count: int
     image_count: int
-    video_length_seconds: int
+    video_length_seconds: float
 
 
 def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
@@ -400,10 +400,10 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
     )
     video_length_seconds = (
         cast(
-            Optional[int],
+            Optional[float],
             getattr(usage.prompt_tokens_details, "video_length_seconds", 0),
         )
-        or 0
+        or 0.0
     )
 
     return PromptTokensDetailsResult(
@@ -415,7 +415,7 @@ def _parse_prompt_tokens_details(usage: Usage) -> PromptTokensDetailsResult:
         image_tokens=image_tokens,
         character_count=character_count,
         image_count=image_count,
-        video_length_seconds=video_length_seconds,
+        video_length_seconds=float(video_length_seconds),
     )
 
 
@@ -561,7 +561,7 @@ def generic_cost_per_token(  # noqa: PLR0915
         image_tokens=0,
         character_count=0,
         image_count=0,
-        video_length_seconds=0,
+        video_length_seconds=0.0,
     )
     if usage.prompt_tokens_details:
         prompt_tokens_details = _parse_prompt_tokens_details(usage)

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -6,7 +6,7 @@ import mimetypes
 import re
 import xml.etree.ElementTree as ET
 from enum import Enum
-from typing import Any, Dict, List, Optional, Tuple, Union, cast, overload
+from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast, overload
 
 from jinja2.sandbox import ImmutableSandboxedEnvironment
 
@@ -2143,6 +2143,9 @@ def anthropic_messages_pt(  # noqa: PLR0915
         if user_content:
             new_messages.append({"role": "user", "content": user_content})
 
+        # Track unique tool IDs in this merge block to avoid duplication
+        unique_tool_ids: Set[str] = set()
+
         assistant_content: List[AnthropicMessagesAssistantMessageValues] = []
         ## MERGE CONSECUTIVE ASSISTANT CONTENT ##
         while msg_i < len(messages) and messages[msg_i]["role"] == "assistant":
@@ -2236,13 +2239,25 @@ def anthropic_messages_pt(  # noqa: PLR0915
                     assistant_tool_calls,
                     web_search_results=_web_search_results,
                 )
-                # AnthropicMessagesAssistantMessageValues includes AnthropicMessagesToolUseParam
-                assistant_content.extend(
-                    cast(
-                        List[AnthropicMessagesAssistantMessageValues],
-                        tool_invoke_results,
+
+                # Prevent "tool_use ids must be unique" errors by filtering duplicates
+                # This can happen when merging history that already contains the tool calls
+                for item in tool_invoke_results:
+                    # tool_use items are typically dicts, but handle objects just in case
+                    item_id = (
+                        item.get("id")
+                        if isinstance(item, dict)
+                        else getattr(item, "id", None)
+                    )
+
+                    if item_id:
+                        if item_id in unique_tool_ids:
+                            continue
+                        unique_tool_ids.add(item_id)
+
+                    assistant_content.append(
+                        cast(AnthropicMessagesAssistantMessageValues, item)
                     )
-                )
 
             assistant_function_call = assistant_content_block.get("function_call")
 

litellm/llms/anthropic/chat/handler.py

@@ -317,6 +317,7 @@ class AnthropicChatCompletion(BaseLLM):
         stream = optional_params.pop("stream", None)
         json_mode: bool = optional_params.pop("json_mode", False)
         is_vertex_request: bool = optional_params.pop("is_vertex_request", False)
+        optional_params.pop("vertex_count_tokens_location", None)
         _is_function_call = False
         messages = copy.deepcopy(messages)
         headers = AnthropicConfig().validate_environment(

litellm/llms/anthropic/common_utils.py

@@ -2,7 +2,7 @@
 This file contains common utils for anthropic calls.
 """
 
-from typing import Any, Dict, List, Optional, Union
+from typing import Dict, List, Optional, Union
 
 import httpx
 
@@ -14,11 +14,36 @@ from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.anthropic import (
     ANTHROPIC_HOSTED_TOOLS,
+    ANTHROPIC_OAUTH_BETA_HEADER,
+    ANTHROPIC_OAUTH_TOKEN_PREFIX,
     AllAnthropicToolsValues,
     AnthropicMcpServerTool,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import TokenCountResponse
+
+
+def optionally_handle_anthropic_oauth(
+    headers: dict, api_key: Optional[str]
+) -> tuple[dict, Optional[str]]:
+    """
+    Handle Anthropic OAuth token detection and header setup.
+
+    If an OAuth token is detected in the Authorization header, extracts it
+    and sets the required OAuth headers.
+
+    Args:
+        headers: Request headers dict
+        api_key: Current API key (may be None)
+
+    Returns:
+        Tuple of (updated headers, api_key)
+    """
+    auth_header = headers.get("authorization", "")
+    if auth_header and auth_header.startswith(f"Bearer {ANTHROPIC_OAUTH_TOKEN_PREFIX}"):
+        api_key = auth_header.replace("Bearer ", "")
+        headers["anthropic-beta"] = ANTHROPIC_OAUTH_BETA_HEADER
+        headers["anthropic-dangerous-direct-browser-access"] = "true"
+    return headers, api_key
 
 
 class AnthropicError(BaseLLMException):
@@ -372,6 +397,8 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
     ) -> Dict:
+        # Check for Anthropic OAuth token in headers
+        headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
         if api_key is None:
             raise litellm.AuthenticationError(
                 message="Missing Anthropic API Key - A call is being made to anthropic but no key is set either in the environment variables or via params. Please set `ANTHROPIC_API_KEY` in your environment vars",
@@ -476,45 +503,11 @@ class AnthropicModelInfo(BaseLLMModelInfo):
         Returns:
             AnthropicTokenCounter instance for this provider.
         """
-        return AnthropicTokenCounter()
-
-
-class AnthropicTokenCounter(BaseTokenCounter):
-    """Token counter implementation for Anthropic provider."""
-
-    def should_use_token_counting_api(
-        self, 
-        custom_llm_provider: Optional[str] = None,
-    ) -> bool:
-        from litellm.types.utils import LlmProviders
-        return custom_llm_provider == LlmProviders.ANTHROPIC.value
-    
-    async def count_tokens(
-        self,
-        model_to_use: str,
-        messages: Optional[List[Dict[str, Any]]],
-        contents: Optional[List[Dict[str, Any]]],
-        deployment: Optional[Dict[str, Any]] = None,
-        request_model: str = "",
-    ) -> Optional[TokenCountResponse]:
-        from litellm.proxy.utils import count_tokens_with_anthropic_api
-        
-        result = await count_tokens_with_anthropic_api(
-            model_to_use=model_to_use,
-            messages=messages,
-            deployment=deployment,
+        from litellm.llms.anthropic.count_tokens.token_counter import (
+            AnthropicTokenCounter,
         )
-        
-        if result is not None:
-            return TokenCountResponse(
-                total_tokens=result.get("total_tokens", 0),
-                request_model=request_model,
-                model_used=model_to_use,
-                tokenizer_type=result.get("tokenizer_used", ""),
-                original_response=result,
-            )
-        
-        return None
+
+        return AnthropicTokenCounter()
 
 
 def process_anthropic_headers(headers: Union[httpx.Headers, dict]) -> dict:

litellm/llms/anthropic/count_tokens/__init__.py

@@ -0,0 +1,15 @@
+"""
+Anthropic CountTokens API implementation.
+"""
+
+from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
+from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+
+__all__ = [
+    "AnthropicCountTokensHandler",
+    "AnthropicCountTokensConfig",
+    "AnthropicTokenCounter",
+]

litellm/llms/anthropic/count_tokens/handler.py

@@ -0,0 +1,122 @@
+"""
+Anthropic CountTokens API handler.
+
+Uses httpx for HTTP requests instead of the Anthropic SDK.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+
+
+class AnthropicCountTokensHandler(AnthropicCountTokensConfig):
+    """
+    Handler for Anthropic CountTokens API requests.
+
+    Uses httpx for HTTP requests, following the same pattern as BedrockCountTokensHandler.
+    """
+
+    async def handle_count_tokens_request(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+        api_key: str,
+        api_base: Optional[str] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Handle a CountTokens request using httpx.
+
+        Args:
+            model: The model identifier (e.g., "claude-3-5-sonnet-20241022")
+            messages: The messages to count tokens for
+            api_key: The Anthropic API key
+            api_base: Optional custom API base URL
+            timeout: Optional timeout for the request (defaults to litellm.request_timeout)
+
+        Returns:
+            Dictionary containing token count response
+
+        Raises:
+            AnthropicError: If the API request fails
+        """
+        try:
+            # Validate the request
+            self.validate_request(model, messages)
+
+            verbose_logger.debug(
+                f"Processing Anthropic CountTokens request for model: {model}"
+            )
+
+            # Transform request to Anthropic format
+            request_body = self.transform_request_to_count_tokens(
+                model=model,
+                messages=messages,
+            )
+
+            verbose_logger.debug(f"Transformed request: {request_body}")
+
+            # Get endpoint URL
+            endpoint_url = api_base or self.get_anthropic_count_tokens_endpoint()
+
+            verbose_logger.debug(f"Making request to: {endpoint_url}")
+
+            # Get required headers
+            headers = self.get_required_headers(api_key)
+
+            # Use LiteLLM's async httpx client
+            async_client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.ANTHROPIC
+            )
+
+            # Use provided timeout or fall back to litellm.request_timeout
+            request_timeout = timeout if timeout is not None else litellm.request_timeout
+
+            response = await async_client.post(
+                endpoint_url,
+                headers=headers,
+                json=request_body,
+                timeout=request_timeout,
+            )
+
+            verbose_logger.debug(f"Response status: {response.status_code}")
+
+            if response.status_code != 200:
+                error_text = response.text
+                verbose_logger.error(f"Anthropic API error: {error_text}")
+                raise AnthropicError(
+                    status_code=response.status_code,
+                    message=error_text,
+                )
+
+            anthropic_response = response.json()
+
+            verbose_logger.debug(f"Anthropic response: {anthropic_response}")
+
+            # Return Anthropic response directly - no transformation needed
+            return anthropic_response
+
+        except AnthropicError:
+            # Re-raise Anthropic exceptions as-is
+            raise
+        except httpx.HTTPStatusError as e:
+            # HTTP errors - preserve the actual status code
+            verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=e.response.status_code,
+                message=e.response.text,
+            )
+        except Exception as e:
+            verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=500,
+                message=f"CountTokens processing error: {str(e)}",
+            )

litellm/llms/anthropic/count_tokens/token_counter.py

@@ -0,0 +1,104 @@
+"""
+Anthropic Token Counter implementation using the CountTokens API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.count_tokens.handler import AnthropicCountTokensHandler
+from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.types.utils import LlmProviders, TokenCountResponse
+
+# Global handler instance - reuse across all token counting requests
+anthropic_count_tokens_handler = AnthropicCountTokensHandler()
+
+
+class AnthropicTokenCounter(BaseTokenCounter):
+    """Token counter implementation for Anthropic provider using the CountTokens API."""
+
+    def should_use_token_counting_api(
+        self,
+        custom_llm_provider: Optional[str] = None,
+    ) -> bool:
+        return custom_llm_provider == LlmProviders.ANTHROPIC.value
+
+    async def count_tokens(
+        self,
+        model_to_use: str,
+        messages: Optional[List[Dict[str, Any]]],
+        contents: Optional[List[Dict[str, Any]]],
+        deployment: Optional[Dict[str, Any]] = None,
+        request_model: str = "",
+    ) -> Optional[TokenCountResponse]:
+        """
+        Count tokens using Anthropic's CountTokens API.
+
+        Args:
+            model_to_use: The model identifier
+            messages: The messages to count tokens for
+            contents: Alternative content format (not used for Anthropic)
+            deployment: Deployment configuration containing litellm_params
+            request_model: The original request model name
+
+        Returns:
+            TokenCountResponse with token count, or None if counting fails
+        """
+        from litellm.llms.anthropic.common_utils import AnthropicError
+
+        if not messages:
+            return None
+
+        deployment = deployment or {}
+        litellm_params = deployment.get("litellm_params", {})
+
+        # Get Anthropic API key from deployment config or environment
+        api_key = litellm_params.get("api_key")
+        if not api_key:
+            api_key = os.getenv("ANTHROPIC_API_KEY")
+
+        if not api_key:
+            verbose_logger.warning("No Anthropic API key found for token counting")
+            return None
+
+        try:
+            result = await anthropic_count_tokens_handler.handle_count_tokens_request(
+                model=model_to_use,
+                messages=messages,
+                api_key=api_key,
+            )
+
+            if result is not None:
+                return TokenCountResponse(
+                    total_tokens=result.get("input_tokens", 0),
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="anthropic_api",
+                    original_response=result,
+                )
+        except AnthropicError as e:
+            verbose_logger.warning(
+                f"Anthropic CountTokens API error: status={e.status_code}, message={e.message}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="anthropic_api",
+                error=True,
+                error_message=e.message,
+                status_code=e.status_code,
+            )
+        except Exception as e:
+            verbose_logger.warning(f"Error calling Anthropic CountTokens API: {e}")
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="anthropic_api",
+                error=True,
+                error_message=str(e),
+                status_code=500,
+            )
+
+        return None

litellm/llms/anthropic/count_tokens/transformation.py

@@ -0,0 +1,103 @@
+"""
+Anthropic CountTokens API transformation logic.
+
+This module handles the transformation of requests to Anthropic's CountTokens API format.
+"""
+
+from typing import Any, Dict, List
+
+from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
+
+
+class AnthropicCountTokensConfig:
+    """
+    Configuration and transformation logic for Anthropic CountTokens API.
+
+    Anthropic CountTokens API Specification:
+    - Endpoint: POST https://api.anthropic.com/v1/messages/count_tokens
+    - Beta header required: anthropic-beta: token-counting-2024-11-01
+    - Response: {"input_tokens": <number>}
+    """
+
+    def get_anthropic_count_tokens_endpoint(self) -> str:
+        """
+        Get the Anthropic CountTokens API endpoint.
+
+        Returns:
+            The endpoint URL for the CountTokens API
+        """
+        return "https://api.anthropic.com/v1/messages/count_tokens"
+
+    def transform_request_to_count_tokens(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+    ) -> Dict[str, Any]:
+        """
+        Transform request to Anthropic CountTokens format.
+
+        Input:
+        {
+            "model": "claude-3-5-sonnet-20241022",
+            "messages": [{"role": "user", "content": "Hello!"}]
+        }
+
+        Output (Anthropic CountTokens format):
+        {
+            "model": "claude-3-5-sonnet-20241022",
+            "messages": [{"role": "user", "content": "Hello!"}]
+        }
+        """
+        return {
+            "model": model,
+            "messages": messages,
+        }
+
+    def get_required_headers(self, api_key: str) -> Dict[str, str]:
+        """
+        Get the required headers for the CountTokens API.
+
+        Args:
+            api_key: The Anthropic API key
+
+        Returns:
+            Dictionary of required headers
+        """
+        return {
+            "Content-Type": "application/json",
+            "x-api-key": api_key,
+            "anthropic-version": "2023-06-01",
+            "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+        }
+
+    def validate_request(
+        self, model: str, messages: List[Dict[str, Any]]
+    ) -> None:
+        """
+        Validate the incoming count tokens request.
+
+        Args:
+            model: The model name
+            messages: The messages to count tokens for
+
+        Raises:
+            ValueError: If the request is invalid
+        """
+        if not model:
+            raise ValueError("model parameter is required")
+
+        if not messages:
+            raise ValueError("messages parameter is required")
+
+        if not isinstance(messages, list):
+            raise ValueError("messages must be a list")
+
+        for i, message in enumerate(messages):
+            if not isinstance(message, dict):
+                raise ValueError(f"Message {i} must be a dictionary")
+
+            if "role" not in message:
+                raise ValueError(f"Message {i} must have a 'role' field")
+
+            if "content" not in message:
+                raise ValueError(f"Message {i} must have a 'content' field")

litellm/llms/anthropic/experimental_pass_through/messages/transformation.py

@@ -17,7 +17,11 @@ from litellm.types.llms.anthropic_messages.anthropic_response import (
 from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
 from litellm.types.router import GenericLiteLLMParams
 
-from ...common_utils import AnthropicError, AnthropicModelInfo
+from ...common_utils import (
+    AnthropicError,
+    AnthropicModelInfo,
+    optionally_handle_anthropic_oauth,
+)
 
 DEFAULT_ANTHROPIC_API_BASE = "https://api.anthropic.com"
 DEFAULT_ANTHROPIC_API_VERSION = "2023-06-01"
@@ -68,8 +72,11 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
     ) -> Tuple[dict, Optional[str]]:
         import os
 
+        # Check for Anthropic OAuth token in Authorization header
+        headers, api_key = optionally_handle_anthropic_oauth(headers=headers, api_key=api_key)
         if api_key is None:
             api_key = os.getenv("ANTHROPIC_API_KEY")
+
         if "x-api-key" not in headers and api_key:
             headers["x-api-key"] = api_key
         if "anthropic-version" not in headers:

litellm/llms/azure/azure.py

@@ -4,7 +4,13 @@ import time
 from typing import Any, Callable, Coroutine, Dict, List, Optional, Union
 
 import httpx  # type: ignore
-from openai import APITimeoutError, AsyncAzureOpenAI, AzureOpenAI
+from openai import (
+    APITimeoutError,
+    AsyncAzureOpenAI,
+    AsyncOpenAI,
+    AzureOpenAI,
+    OpenAI,
+)
 
 import litellm
 from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
@@ -128,7 +134,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
 
     def make_sync_azure_openai_chat_completion_request(
         self,
-        azure_client: AzureOpenAI,
+        azure_client: Union[AzureOpenAI, OpenAI],
         data: dict,
         timeout: Union[float, httpx.Timeout],
     ):
@@ -151,7 +157,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
     @track_llm_api_timing()
     async def make_azure_openai_chat_completion_request(
         self,
-        azure_client: AsyncAzureOpenAI,
+        azure_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         data: dict,
         timeout: Union[float, httpx.Timeout],
         logging_obj: LiteLLMLoggingObj,
@@ -215,7 +221,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
 
             ### CHECK IF CLOUDFLARE AI GATEWAY ###
             ### if so - set the model as part of the base url
-            if "gateway.ai.cloudflare.com" in api_base:
+            if api_base is not None and "gateway.ai.cloudflare.com" in api_base:
                 client = self._init_azure_client_for_cloudflare_ai_gateway(
                     api_base=api_base,
                     model=model,
@@ -328,10 +334,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     _is_async=False,
                     litellm_params=litellm_params,
                 )
-                if not isinstance(azure_client, AzureOpenAI):
+                if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
                     raise AzureOpenAIError(
                         status_code=500,
-                        message="azure_client is not an instance of AzureOpenAI",
+                        message="azure_client is not an instance of AzureOpenAI or OpenAI",
                     )
 
                 headers, response = self.make_sync_azure_openai_chat_completion_request(
@@ -401,8 +407,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 _is_async=True,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
             ## LOGGING
             logging_obj.pre_call(
                 input=data["messages"],
@@ -412,7 +418,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                         "api_key": api_key,
                         "azure_ad_token": azure_ad_token,
                     },
-                    "api_base": azure_client._base_url._uri_reference,
+                    "api_base": api_base,
                     "acompletion": True,
                     "complete_input_dict": data,
                 },
@@ -520,10 +526,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             _is_async=False,
             litellm_params=litellm_params,
         )
-        if not isinstance(azure_client, AzureOpenAI):
+        if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
             raise AzureOpenAIError(
                 status_code=500,
-                message="azure_client is not an instance of AzureOpenAI",
+                message="azure_client is not an instance of AzureOpenAI or OpenAI",
             )
         ## LOGGING
         logging_obj.pre_call(
@@ -534,7 +540,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     "api_key": api_key,
                     "azure_ad_token": azure_ad_token,
                 },
-                "api_base": azure_client._base_url._uri_reference,
+                "api_base": api_base,
                 "acompletion": True,
                 "complete_input_dict": data,
             },
@@ -578,8 +584,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 _is_async=True,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
 
             ## LOGGING
             logging_obj.pre_call(
@@ -590,7 +596,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                         "api_key": api_key,
                         "azure_ad_token": azure_ad_token,
                     },
-                    "api_base": azure_client._base_url._uri_reference,
+                    "api_base": api_base,
                     "acompletion": True,
                     "complete_input_dict": data,
                 },
@@ -657,8 +663,8 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 client=client,
                 litellm_params=litellm_params,
             )
-            if not isinstance(openai_aclient, AsyncAzureOpenAI):
-                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI")
+            if not isinstance(openai_aclient, (AsyncAzureOpenAI, AsyncOpenAI)):
+                raise ValueError("Azure client is not an instance of AsyncAzureOpenAI or AsyncOpenAI")
 
             raw_response = await openai_aclient.embeddings.with_raw_response.create(
                 **data, timeout=timeout
@@ -776,10 +782,10 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 client=client,
                 litellm_params=litellm_params,
             )
-            if not isinstance(azure_client, AzureOpenAI):
+            if not isinstance(azure_client, (AzureOpenAI, OpenAI)):
                 raise AzureOpenAIError(
                     status_code=500,
-                    message="azure_client is not an instance of AzureOpenAI",
+                    message="azure_client is not an instance of AzureOpenAI or OpenAI",
                 )
 
             ## COMPLETION CALL
@@ -1338,7 +1344,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
         prompt: Optional[str] = None,
     ) -> dict:
         client_session = litellm.client_session or httpx.Client()
-        if "gateway.ai.cloudflare.com" in api_base:
+        if api_base is not None and "gateway.ai.cloudflare.com" in api_base:
             ## build base url - assume api base includes resource name
             if not api_base.endswith("/"):
                 api_base += "/"

litellm/llms/azure/batches/handler.py

@@ -6,6 +6,8 @@ from typing import Any, Coroutine, Optional, Union, cast
 
 import httpx
 
+from openai import AsyncOpenAI, OpenAI
+
 from litellm.llms.azure.azure import AsyncAzureOpenAI, AzureOpenAI
 from litellm.types.llms.openai import (
     Batch,
@@ -33,7 +35,7 @@ class AzureBatchesAPI(BaseAzureLLM):
     async def acreate_batch(
         self,
         create_batch_data: CreateBatchRequest,
-        azure_client: AsyncAzureOpenAI,
+        azure_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> LiteLLMBatch:
         response = await azure_client.batches.create(**create_batch_data)
         return LiteLLMBatch(**response.model_dump())
@@ -47,11 +49,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[LiteLLMBatch, Coroutine[Any, Any, LiteLLMBatch]]:
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -66,20 +68,20 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )
             return self.acreate_batch(  # type: ignore
                 create_batch_data=create_batch_data, azure_client=azure_client
             )
-        response = cast(AzureOpenAI, azure_client).batches.create(**create_batch_data)
+        response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.create(**create_batch_data)
         return LiteLLMBatch(**response.model_dump())
 
     async def aretrieve_batch(
         self,
         retrieve_batch_data: RetrieveBatchRequest,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> LiteLLMBatch:
         response = await client.batches.retrieve(**retrieve_batch_data)
         return LiteLLMBatch(**response.model_dump())
@@ -93,11 +95,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -112,14 +114,14 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )
             return self.aretrieve_batch(  # type: ignore
                 retrieve_batch_data=retrieve_batch_data, client=azure_client
             )
-        response = cast(AzureOpenAI, azure_client).batches.retrieve(
+        response = cast(Union[AzureOpenAI, OpenAI], azure_client).batches.retrieve(
             **retrieve_batch_data
         )
         return LiteLLMBatch(**response.model_dump())
@@ -127,7 +129,7 @@ class AzureBatchesAPI(BaseAzureLLM):
     async def acancel_batch(
         self,
         cancel_batch_data: CancelBatchRequest,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> Batch:
         response = await client.batches.cancel(**cancel_batch_data)
         return response
@@ -141,11 +143,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -163,7 +165,7 @@ class AzureBatchesAPI(BaseAzureLLM):
 
     async def alist_batches(
         self,
-        client: AsyncAzureOpenAI,
+        client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         after: Optional[str] = None,
         limit: Optional[int] = None,
     ):
@@ -180,11 +182,11 @@ class AzureBatchesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         after: Optional[str] = None,
         limit: Optional[int] = None,
-        client: Optional[AzureOpenAI] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         azure_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             api_key=api_key,
             api_base=api_base,
@@ -199,7 +201,7 @@ class AzureBatchesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(azure_client, AsyncAzureOpenAI):
+            if not isinstance(azure_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "OpenAI client is not an instance of AsyncOpenAI. Make sure you passed an AsyncOpenAI client."
                 )

litellm/llms/azure/common_utils.py

@@ -3,7 +3,7 @@ import os
 from typing import Any, Callable, Dict, Literal, Optional, Union, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AzureOpenAI
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 
 import litellm
 from litellm._logging import verbose_logger
@@ -439,12 +439,12 @@ class BaseAzureLLM(BaseOpenAILLM):
         api_key: Optional[str],
         api_base: Optional[str],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
         _is_async: bool = False,
         model: Optional[str] = None,
-    ) -> Optional[Union[AzureOpenAI, AsyncAzureOpenAI]]:
-        openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None
+    ) -> Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]]:
+        openai_client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None
         client_initialization_params: dict = locals()
         client_initialization_params["is_async"] = _is_async
         if client is None:
@@ -453,9 +453,7 @@ class BaseAzureLLM(BaseOpenAILLM):
                 client_type="azure",
             )
             if cached_client:
-                if isinstance(cached_client, AzureOpenAI) or isinstance(
-                    cached_client, AsyncAzureOpenAI
-                ):
+                if isinstance(cached_client, (AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI)):
                     return cached_client
 
             azure_client_params = self.initialize_azure_sdk_client(
@@ -466,15 +464,40 @@ class BaseAzureLLM(BaseOpenAILLM):
                 api_version=api_version,
                 is_async=_is_async,
             )
-            if _is_async is True:
-                openai_client = AsyncAzureOpenAI(**azure_client_params)
+
+            # For Azure v1 API, use standard OpenAI client instead of AzureOpenAI
+            # See: https://learn.microsoft.com/en-us/azure/ai-services/openai/reference#api-specs
+            if self._is_azure_v1_api_version(api_version):
+                # Extract only params that OpenAI client accepts
+                # Always use /openai/v1/ regardless of whether user passed "v1", "latest", or "preview"
+                v1_params = {
+                    "api_key": azure_client_params.get("api_key"),
+                    "base_url": f"{api_base}/openai/v1/",
+                }
+                if "timeout" in azure_client_params:
+                    v1_params["timeout"] = azure_client_params["timeout"]
+                if "max_retries" in azure_client_params:
+                    v1_params["max_retries"] = azure_client_params["max_retries"]
+                if "http_client" in azure_client_params:
+                    v1_params["http_client"] = azure_client_params["http_client"]
+
+                verbose_logger.debug(f"Using Azure v1 API with base_url: {v1_params['base_url']}")
+
+                if _is_async is True:
+                    openai_client = AsyncOpenAI(**v1_params)  # type: ignore
+                else:
+                    openai_client = OpenAI(**v1_params)  # type: ignore
             else:
-                openai_client = AzureOpenAI(**azure_client_params)  # type: ignore
+                # Traditional Azure API uses AzureOpenAI client
+                if _is_async is True:
+                    openai_client = AsyncAzureOpenAI(**azure_client_params)
+                else:
+                    openai_client = AzureOpenAI(**azure_client_params)  # type: ignore
         else:
             openai_client = client
             if api_version is not None and isinstance(
-                openai_client._custom_query, dict
-            ):
+                openai_client, (AzureOpenAI, AsyncAzureOpenAI)
+            ) and isinstance(openai_client._custom_query, dict):
                 # set api_version to version passed by user
                 openai_client._custom_query.setdefault("api-version", api_version)
 

litellm/llms/azure/files/handler.py

@@ -1,7 +1,7 @@
 from typing import Any, Coroutine, Optional, Union, cast
 
 import httpx
-from openai import AsyncAzureOpenAI, AzureOpenAI
+from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI
 from openai.types.file_deleted import FileDeleted
 
 from litellm._logging import verbose_logger
@@ -40,7 +40,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def acreate_file(
         self,
         create_file_data: CreateFileRequest,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> OpenAIFileObject:
         verbose_logger.debug("create_file_data=%s", create_file_data)
         response = await openai_client.files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
@@ -56,11 +56,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         api_version: Optional[str],
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[OpenAIFileObject, Coroutine[Any, Any, OpenAIFileObject]]:
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -75,20 +75,20 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
             return self.acreate_file(
                 create_file_data=create_file_data, openai_client=openai_client
             )
-        response = cast(AzureOpenAI, openai_client).files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
+        response = cast(Union[AzureOpenAI, OpenAI], openai_client).files.create(**self._prepare_create_file_data(create_file_data))  # type: ignore[arg-type]
         return OpenAIFileObject(**response.model_dump())
 
     async def afile_content(
         self,
         file_content_request: FileContentRequest,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> HttpxBinaryResponseContent:
         response = await openai_client.files.content(**file_content_request)
         return HttpxBinaryResponseContent(response=response.response)
@@ -102,13 +102,13 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ) -> Union[
         HttpxBinaryResponseContent, Coroutine[Any, Any, HttpxBinaryResponseContent]
     ]:
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -123,7 +123,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -131,7 +131,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
                 file_content_request=file_content_request,
                 openai_client=openai_client,
             )
-        response = cast(AzureOpenAI, openai_client).files.content(
+        response = cast(Union[AzureOpenAI, OpenAI], openai_client).files.content(
             **file_content_request
         )
 
@@ -140,7 +140,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def aretrieve_file(
         self,
         file_id: str,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> FileObject:
         response = await openai_client.files.retrieve(file_id=file_id)
         return response
@@ -154,11 +154,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         timeout: Union[float, httpx.Timeout],
         max_retries: Optional[int],
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -173,7 +173,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -188,7 +188,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
     async def adelete_file(
         self,
         file_id: str,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
     ) -> FileDeleted:
         response = await openai_client.files.delete(file_id=file_id)
 
@@ -206,11 +206,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         organization: Optional[str] = None,
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -225,7 +225,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )
@@ -242,7 +242,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
 
     async def alist_files(
         self,
-        openai_client: AsyncAzureOpenAI,
+        openai_client: Union[AsyncAzureOpenAI, AsyncOpenAI],
         purpose: Optional[str] = None,
     ):
         if isinstance(purpose, str):
@@ -260,11 +260,11 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
         max_retries: Optional[int],
         purpose: Optional[str] = None,
         api_version: Optional[str] = None,
-        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI]] = None,
+        client: Optional[Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]] = None,
         litellm_params: Optional[dict] = None,
     ):
         openai_client: Optional[
-            Union[AzureOpenAI, AsyncAzureOpenAI]
+            Union[AzureOpenAI, AsyncAzureOpenAI, OpenAI, AsyncOpenAI]
         ] = self.get_azure_openai_client(
             litellm_params=litellm_params or {},
             api_key=api_key,
@@ -279,7 +279,7 @@ class AzureOpenAIFilesAPI(BaseAzureLLM):
             )
 
         if _is_async is True:
-            if not isinstance(openai_client, AsyncAzureOpenAI):
+            if not isinstance(openai_client, (AsyncAzureOpenAI, AsyncOpenAI)):
                 raise ValueError(
                     "AzureOpenAI client is not an instance of AsyncAzureOpenAI. Make sure you passed an AsyncAzureOpenAI client."
                 )

litellm/llms/azure_ai/anthropic/count_tokens/__init__.py

@@ -0,0 +1,19 @@
+"""
+Azure AI Anthropic CountTokens API implementation.
+"""
+
+from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
+    AzureAIAnthropicCountTokensHandler,
+)
+from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import (
+    AzureAIAnthropicTokenCounter,
+)
+from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
+    AzureAIAnthropicCountTokensConfig,
+)
+
+__all__ = [
+    "AzureAIAnthropicCountTokensHandler",
+    "AzureAIAnthropicCountTokensConfig",
+    "AzureAIAnthropicTokenCounter",
+]

litellm/llms/azure_ai/anthropic/count_tokens/handler.py

@@ -0,0 +1,127 @@
+"""
+Azure AI Anthropic CountTokens API handler.
+
+Uses httpx for HTTP requests with Azure authentication.
+"""
+
+from typing import Any, Dict, List, Optional, Union
+
+import httpx
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.llms.anthropic.common_utils import AnthropicError
+from litellm.llms.azure_ai.anthropic.count_tokens.transformation import (
+    AzureAIAnthropicCountTokensConfig,
+)
+from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
+
+
+class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
+    """
+    Handler for Azure AI Anthropic CountTokens API requests.
+
+    Uses httpx for HTTP requests with Azure authentication.
+    """
+
+    async def handle_count_tokens_request(
+        self,
+        model: str,
+        messages: List[Dict[str, Any]],
+        api_key: str,
+        api_base: str,
+        litellm_params: Optional[Dict[str, Any]] = None,
+        timeout: Optional[Union[float, httpx.Timeout]] = None,
+    ) -> Dict[str, Any]:
+        """
+        Handle a CountTokens request using httpx with Azure authentication.
+
+        Args:
+            model: The model identifier (e.g., "claude-3-5-sonnet")
+            messages: The messages to count tokens for
+            api_key: The Azure AI API key
+            api_base: The Azure AI API base URL
+            litellm_params: Optional LiteLLM parameters
+            timeout: Optional timeout for the request (defaults to litellm.request_timeout)
+
+        Returns:
+            Dictionary containing token count response
+
+        Raises:
+            AnthropicError: If the API request fails
+        """
+        try:
+            # Validate the request
+            self.validate_request(model, messages)
+
+            verbose_logger.debug(
+                f"Processing Azure AI Anthropic CountTokens request for model: {model}"
+            )
+
+            # Transform request to Anthropic format
+            request_body = self.transform_request_to_count_tokens(
+                model=model,
+                messages=messages,
+            )
+
+            verbose_logger.debug(f"Transformed request: {request_body}")
+
+            # Get endpoint URL
+            endpoint_url = self.get_count_tokens_endpoint(api_base)
+
+            verbose_logger.debug(f"Making request to: {endpoint_url}")
+
+            # Get required headers with Azure authentication
+            headers = self.get_required_headers(
+                api_key=api_key,
+                litellm_params=litellm_params,
+            )
+
+            # Use LiteLLM's async httpx client
+            async_client = get_async_httpx_client(
+                llm_provider=litellm.LlmProviders.AZURE_AI
+            )
+
+            # Use provided timeout or fall back to litellm.request_timeout
+            request_timeout = timeout if timeout is not None else litellm.request_timeout
+
+            response = await async_client.post(
+                endpoint_url,
+                headers=headers,
+                json=request_body,
+                timeout=request_timeout,
+            )
+
+            verbose_logger.debug(f"Response status: {response.status_code}")
+
+            if response.status_code != 200:
+                error_text = response.text
+                verbose_logger.error(f"Azure AI Anthropic API error: {error_text}")
+                raise AnthropicError(
+                    status_code=response.status_code,
+                    message=error_text,
+                )
+
+            azure_response = response.json()
+
+            verbose_logger.debug(f"Azure AI Anthropic response: {azure_response}")
+
+            # Return Anthropic-compatible response directly - no transformation needed
+            return azure_response
+
+        except AnthropicError:
+            # Re-raise Anthropic exceptions as-is
+            raise
+        except httpx.HTTPStatusError as e:
+            # HTTP errors - preserve the actual status code
+            verbose_logger.error(f"HTTP error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=e.response.status_code,
+                message=e.response.text,
+            )
+        except Exception as e:
+            verbose_logger.error(f"Error in CountTokens handler: {str(e)}")
+            raise AnthropicError(
+                status_code=500,
+                message=f"CountTokens processing error: {str(e)}",
+            )

litellm/llms/azure_ai/anthropic/count_tokens/token_counter.py

@@ -0,0 +1,119 @@
+"""
+Azure AI Anthropic Token Counter implementation using the CountTokens API.
+"""
+
+import os
+from typing import Any, Dict, List, Optional
+
+from litellm._logging import verbose_logger
+from litellm.llms.azure_ai.anthropic.count_tokens.handler import (
+    AzureAIAnthropicCountTokensHandler,
+)
+from litellm.llms.base_llm.base_utils import BaseTokenCounter
+from litellm.types.utils import LlmProviders, TokenCountResponse
+
+# Global handler instance - reuse across all token counting requests
+azure_ai_anthropic_count_tokens_handler = AzureAIAnthropicCountTokensHandler()
+
+
+class AzureAIAnthropicTokenCounter(BaseTokenCounter):
+    """Token counter implementation for Azure AI Anthropic provider using the CountTokens API."""
+
+    def should_use_token_counting_api(
+        self,
+        custom_llm_provider: Optional[str] = None,
+    ) -> bool:
+        return custom_llm_provider == LlmProviders.AZURE_AI.value
+
+    async def count_tokens(
+        self,
+        model_to_use: str,
+        messages: Optional[List[Dict[str, Any]]],
+        contents: Optional[List[Dict[str, Any]]],
+        deployment: Optional[Dict[str, Any]] = None,
+        request_model: str = "",
+    ) -> Optional[TokenCountResponse]:
+        """
+        Count tokens using Azure AI Anthropic's CountTokens API.
+
+        Args:
+            model_to_use: The model identifier
+            messages: The messages to count tokens for
+            contents: Alternative content format (not used for Anthropic)
+            deployment: Deployment configuration containing litellm_params
+            request_model: The original request model name
+
+        Returns:
+            TokenCountResponse with token count, or None if counting fails
+        """
+        from litellm.llms.anthropic.common_utils import AnthropicError
+
+        if not messages:
+            return None
+
+        deployment = deployment or {}
+        litellm_params = deployment.get("litellm_params", {})
+
+        # Get Azure AI API key from deployment config or environment
+        api_key = litellm_params.get("api_key")
+        if not api_key:
+            api_key = os.getenv("AZURE_AI_API_KEY")
+
+        # Get API base from deployment config or environment
+        api_base = litellm_params.get("api_base")
+        if not api_base:
+            api_base = os.getenv("AZURE_AI_API_BASE")
+
+        if not api_key:
+            verbose_logger.warning("No Azure AI API key found for token counting")
+            return None
+
+        if not api_base:
+            verbose_logger.warning("No Azure AI API base found for token counting")
+            return None
+
+        try:
+            result = await azure_ai_anthropic_count_tokens_handler.handle_count_tokens_request(
+                model=model_to_use,
+                messages=messages,
+                api_key=api_key,
+                api_base=api_base,
+                litellm_params=litellm_params,
+            )
+
+            if result is not None:
+                return TokenCountResponse(
+                    total_tokens=result.get("input_tokens", 0),
+                    request_model=request_model,
+                    model_used=model_to_use,
+                    tokenizer_type="azure_ai_anthropic_api",
+                    original_response=result,
+                )
+        except AnthropicError as e:
+            verbose_logger.warning(
+                f"Azure AI Anthropic CountTokens API error: status={e.status_code}, message={e.message}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="azure_ai_anthropic_api",
+                error=True,
+                error_message=e.message,
+                status_code=e.status_code,
+            )
+        except Exception as e:
+            verbose_logger.warning(
+                f"Error calling Azure AI Anthropic CountTokens API: {e}"
+            )
+            return TokenCountResponse(
+                total_tokens=0,
+                request_model=request_model,
+                model_used=model_to_use,
+                tokenizer_type="azure_ai_anthropic_api",
+                error=True,
+                error_message=str(e),
+                status_code=500,
+            )
+
+        return None

litellm/llms/azure_ai/anthropic/count_tokens/transformation.py

@@ -0,0 +1,88 @@
+"""
+Azure AI Anthropic CountTokens API transformation logic.
+
+Extends the base Anthropic CountTokens transformation with Azure authentication.
+"""
+
+from typing import Any, Dict, Optional
+
+from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
+from litellm.llms.anthropic.count_tokens.transformation import (
+    AnthropicCountTokensConfig,
+)
+from litellm.llms.azure.common_utils import BaseAzureLLM
+from litellm.types.router import GenericLiteLLMParams
+
+
+class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
+    """
+    Configuration and transformation logic for Azure AI Anthropic CountTokens API.
+
+    Extends AnthropicCountTokensConfig with Azure authentication.
+    Azure AI Anthropic uses the same endpoint format but with Azure auth headers.
+    """
+
+    def get_required_headers(
+        self,
+        api_key: str,
+        litellm_params: Optional[Dict[str, Any]] = None,
+    ) -> Dict[str, str]:
+        """
+        Get the required headers for the Azure AI Anthropic CountTokens API.
+
+        Uses Azure authentication (api-key header) instead of Anthropic's x-api-key.
+
+        Args:
+            api_key: The Azure AI API key
+            litellm_params: Optional LiteLLM parameters for additional auth config
+
+        Returns:
+            Dictionary of required headers with Azure authentication
+        """
+        # Start with base headers
+        headers = {
+            "Content-Type": "application/json",
+            "anthropic-version": "2023-06-01",
+            "anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
+        }
+
+        # Use Azure authentication
+        litellm_params = litellm_params or {}
+        if "api_key" not in litellm_params:
+            litellm_params["api_key"] = api_key
+
+        litellm_params_obj = GenericLiteLLMParams(**litellm_params)
+
+        # Get Azure auth headers
+        azure_headers = BaseAzureLLM._base_validate_azure_environment(
+            headers={}, litellm_params=litellm_params_obj
+        )
+
+        # Merge Azure auth headers
+        headers.update(azure_headers)
+
+        return headers
+
+    def get_count_tokens_endpoint(self, api_base: str) -> str:
+        """
+        Get the Azure AI Anthropic CountTokens API endpoint.
+
+        Args:
+            api_base: The Azure AI API base URL 
+                      (e.g., https://my-resource.services.ai.azure.com or
+                       https://my-resource.services.ai.azure.com/anthropic)
+
+        Returns:
+            The endpoint URL for the CountTokens API
+        """
+        # Azure AI Anthropic endpoint format:
+        # https://<resource>.services.ai.azure.com/anthropic/v1/messages/count_tokens
+        api_base = api_base.rstrip("/")
+
+        # Ensure the URL has /anthropic path
+        if not api_base.endswith("/anthropic"):
+            if "/anthropic" not in api_base:
+                api_base = f"{api_base}/anthropic"
+
+        # Add the count_tokens path
+        return f"{api_base}/v1/messages/count_tokens"

litellm/llms/azure_ai/common_utils.py

@@ -1,17 +1,22 @@
 from typing import List, Literal, Optional
 
 import litellm
-from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
+from litellm.llms.base_llm.base_utils import BaseLLMModelInfo, BaseTokenCounter
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
 
 
 class AzureFoundryModelInfo(BaseLLMModelInfo):
+    """Model info for Azure AI / Azure Foundry models."""
+
+    def __init__(self, model: Optional[str] = None):
+        self._model = model
+
     @staticmethod
     def get_azure_ai_route(model: str) -> Literal["agents", "default"]:
         """
         Get the Azure AI route for the given model.
-        
+
         Similar to BedrockModelInfo.get_bedrock_route().
         """
         if "agents/" in model:
@@ -20,34 +25,54 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
 
     @staticmethod
     def get_api_base(api_base: Optional[str] = None) -> Optional[str]:
-        return (
-                api_base
-                or litellm.api_base
-                or get_secret_str("AZURE_AI_API_BASE")
-        )
-    
+        return api_base or litellm.api_base or get_secret_str("AZURE_AI_API_BASE")
+
     @staticmethod
     def get_api_key(api_key: Optional[str] = None) -> Optional[str]:
         return (
-                api_key
-                or litellm.api_key
-                or litellm.openai_key
-                or get_secret_str("AZURE_AI_API_KEY")
-            )
-    
+            api_key
+            or litellm.api_key
+            or litellm.openai_key
+            or get_secret_str("AZURE_AI_API_KEY")
+        )
+
     @property
     def api_version(self, api_version: Optional[str] = None) -> Optional[str]:
         api_version = (
-            api_version
-            or litellm.api_version
-            or get_secret_str("AZURE_API_VERSION")
+            api_version or litellm.api_version or get_secret_str("AZURE_API_VERSION")
         )
         return api_version
-    
+
+    def get_token_counter(self) -> Optional[BaseTokenCounter]:
+        """
+        Factory method to create a token counter for Azure AI.
+
+        Returns:
+            AzureAIAnthropicTokenCounter for Claude models, None otherwise.
+        """
+        # Only return token counter for Claude models
+        if self._model and "claude" in self._model.lower():
+            from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import (
+                AzureAIAnthropicTokenCounter,
+            )
+
+            return AzureAIAnthropicTokenCounter()
+        return None
+
+    def get_models(
+        self, api_key: Optional[str] = None, api_base: Optional[str] = None
+    ) -> List[str]:
+        """
+        Returns a list of models supported by Azure AI.
+        
+        Azure AI doesn't have a standard model listing endpoint,
+        so this returns an empty list.
+        """
+        return []
+
     #########################################################
     # Not implemented methods
     #########################################################
-        
 
     @staticmethod
     def get_base_model(model: str) -> Optional[str]:
@@ -64,4 +89,6 @@ class AzureFoundryModelInfo(BaseLLMModelInfo):
         api_base: Optional[str] = None,
     ) -> dict:
         """Azure Foundry sends api key in query params"""
-        raise NotImplementedError("Azure Foundry does not support environment validation")
+        raise NotImplementedError(
+            "Azure Foundry does not support environment validation"
+        )

litellm/llms/bedrock/chat/agentcore/sse_iterator.py

@@ -1,252 +0,0 @@
-"""
-SSE Stream Iterator for Bedrock AgentCore.
-
-Handles Server-Sent Events (SSE) streaming responses from AgentCore.
-"""
-
-import json
-from typing import TYPE_CHECKING, Any, Optional
-
-import httpx
-
-from litellm._logging import verbose_logger
-from litellm._uuid import uuid
-from litellm.types.llms.bedrock_agentcore import AgentCoreUsage
-from litellm.types.utils import Delta, ModelResponse, StreamingChoices, Usage
-
-if TYPE_CHECKING:
-    pass
-
-
-class AgentCoreSSEStreamIterator:
-    """
-    Iterator for AgentCore SSE streaming responses.
-    Supports both sync and async iteration.
-
-    CRITICAL: The line iterators are created lazily on first access and reused.
-    We must NOT create new iterators in __aiter__/__iter__ because
-    CustomStreamWrapper calls __aiter__ on every call to its __anext__,
-    which would create new iterators and cause StreamConsumed errors.
-    """
-
-    def __init__(self, response: httpx.Response, model: str):
-        self.response = response
-        self.model = model
-        self.finished = False
-        self._sync_iter: Any = None
-        self._async_iter: Any = None
-        self._sync_iter_initialized = False
-        self._async_iter_initialized = False
-
-    def __iter__(self):
-        """Initialize sync iteration - create iterator lazily on first call only."""
-        if not self._sync_iter_initialized:
-            self._sync_iter = iter(self.response.iter_lines())
-            self._sync_iter_initialized = True
-        return self
-
-    def __aiter__(self):
-        """Initialize async iteration - create iterator lazily on first call only."""
-        if not self._async_iter_initialized:
-            self._async_iter = self.response.aiter_lines().__aiter__()
-            self._async_iter_initialized = True
-        return self
-
-    def _parse_sse_line(self, line: str) -> Optional[ModelResponse]:
-        """
-        Parse a single SSE line and return a ModelResponse chunk if applicable.
-
-        AgentCore SSE format:
-        - data: {"event": {"contentBlockDelta": {"delta": {"text": "..."}}}}
-        - data: {"event": {"metadata": {"usage": {...}}}}
-        - data: {"message": {...}}
-        """
-        line = line.strip()
-        if not line or not line.startswith("data:"):
-            return None
-
-        json_str = line[5:].strip()
-        if not json_str:
-            return None
-
-        try:
-            data = json.loads(json_str)
-
-            # Skip non-dict data (some lines contain Python repr strings)
-            if not isinstance(data, dict):
-                return None
-
-            # Process content delta events
-            if "event" in data and isinstance(data["event"], dict):
-                event_payload = data["event"]
-                content_block_delta = event_payload.get("contentBlockDelta")
-
-                if content_block_delta:
-                    delta = content_block_delta.get("delta", {})
-                    text = delta.get("text", "")
-
-                    if text:
-                        # Return chunk with text
-                        chunk = ModelResponse(
-                            id=f"chatcmpl-{uuid.uuid4()}",
-                            created=0,
-                            model=self.model,
-                            object="chat.completion.chunk",
-                        )
-
-                        chunk.choices = [
-                            StreamingChoices(
-                                finish_reason=None,
-                                index=0,
-                                delta=Delta(content=text, role="assistant"),
-                            )
-                        ]
-
-                        return chunk
-
-                # Check for metadata/usage - this signals the end
-                metadata = event_payload.get("metadata")
-                if metadata and "usage" in metadata:
-                    chunk = ModelResponse(
-                        id=f"chatcmpl-{uuid.uuid4()}",
-                        created=0,
-                        model=self.model,
-                        object="chat.completion.chunk",
-                    )
-
-                    chunk.choices = [
-                        StreamingChoices(
-                            finish_reason="stop",
-                            index=0,
-                            delta=Delta(),
-                        )
-                    ]
-
-                    usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
-                    setattr(
-                        chunk,
-                        "usage",
-                        Usage(
-                            prompt_tokens=usage_data.get("inputTokens", 0),
-                            completion_tokens=usage_data.get("outputTokens", 0),
-                            total_tokens=usage_data.get("totalTokens", 0),
-                        ),
-                    )
-
-                    self.finished = True
-                    return chunk
-
-            # Check for final message (alternative finish signal)
-            if "message" in data and isinstance(data["message"], dict):
-                if not self.finished:
-                    chunk = ModelResponse(
-                        id=f"chatcmpl-{uuid.uuid4()}",
-                        created=0,
-                        model=self.model,
-                        object="chat.completion.chunk",
-                    )
-
-                    chunk.choices = [
-                        StreamingChoices(
-                            finish_reason="stop",
-                            index=0,
-                            delta=Delta(),
-                        )
-                    ]
-
-                    self.finished = True
-                    return chunk
-
-        except json.JSONDecodeError:
-            verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
-
-        return None
-
-    def _create_final_chunk(self) -> ModelResponse:
-        """Create a final chunk to signal stream completion."""
-        chunk = ModelResponse(
-            id=f"chatcmpl-{uuid.uuid4()}",
-            created=0,
-            model=self.model,
-            object="chat.completion.chunk",
-        )
-
-        chunk.choices = [
-            StreamingChoices(
-                finish_reason="stop",
-                index=0,
-                delta=Delta(),
-            )
-        ]
-
-        return chunk
-
-    def __next__(self) -> ModelResponse:
-        """
-        Sync iteration - parse SSE events and yield ModelResponse chunks.
-        
-        Uses next() on the stored iterator to properly resume between calls.
-        """
-        try:
-            if self._sync_iter is None:
-                raise StopIteration
-
-            # Keep getting lines until we have a result to return
-            while True:
-                try:
-                    line = next(self._sync_iter)
-                except StopIteration:
-                    # Stream ended - send final chunk if not already finished
-                    if not self.finished:
-                        self.finished = True
-                        return self._create_final_chunk()
-                    raise
-
-                result = self._parse_sse_line(line)
-                if result is not None:
-                    return result
-
-        except StopIteration:
-            raise
-        except httpx.StreamConsumed:
-            raise StopIteration
-        except httpx.StreamClosed:
-            raise StopIteration
-        except Exception as e:
-            verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}")
-            raise StopIteration
-
-    async def __anext__(self) -> ModelResponse:
-        """
-        Async iteration - parse SSE events and yield ModelResponse chunks.
-
-        Uses __anext__() on the stored iterator to properly resume between calls.
-        """
-        try:
-            if self._async_iter is None:
-                raise StopAsyncIteration
-
-            # Keep getting lines until we have a result to return
-            while True:
-                try:
-                    line = await self._async_iter.__anext__()
-                except StopAsyncIteration:
-                    # Stream ended - send final chunk if not already finished
-                    if not self.finished:
-                        self.finished = True
-                        return self._create_final_chunk()
-                    raise
-
-                result = self._parse_sse_line(line)
-                if result is not None:
-                    return result
-
-        except StopAsyncIteration:
-            raise
-        except httpx.StreamConsumed:
-            raise StopAsyncIteration
-        except httpx.StreamClosed:
-            raise StopAsyncIteration
-        except Exception as e:
-            verbose_logger.error(f"Error in AgentCore SSE stream: {str(e)}")
-            raise StopAsyncIteration

litellm/llms/bedrock/chat/agentcore/transformation.py

@@ -5,6 +5,7 @@ https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agentcore_InvokeAgen
 """
 
 import json
+from collections.abc import AsyncGenerator
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 from urllib.parse import quote
 
@@ -15,9 +16,9 @@ from litellm._uuid import uuid
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
 )
+from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
-from litellm.llms.bedrock.chat.agentcore.sse_iterator import AgentCoreSSEStreamIterator
 from litellm.llms.bedrock.common_utils import BedrockError
 from litellm.types.llms.bedrock_agentcore import (
     AgentCoreMessage,
@@ -25,19 +26,17 @@ from litellm.types.llms.bedrock_agentcore import (
     AgentCoreUsage,
 )
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import Choices, Message, ModelResponse, Usage
+from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices, Usage
 
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
     from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
-    from litellm.utils import CustomStreamWrapper
 
     LiteLLMLoggingObj = _LiteLLMLoggingObj
 else:
     LiteLLMLoggingObj = Any
     HTTPHandler = Any
     AsyncHTTPHandler = Any
-    CustomStreamWrapper = Any
 
 
 class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
@@ -116,7 +115,8 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         fake_stream: Optional[bool] = None,
     ) -> Tuple[dict, Optional[bytes]]:
         # Check if api_key (bearer token) is provided for Cognito authentication
-        jwt_token = optional_params.get("api_key")
+        # Priority: api_key parameter first, then optional_params
+        jwt_token = api_key or optional_params.get("api_key")
         if jwt_token:
             verbose_logger.debug(
                 f"AgentCore: Using Bearer token authentication (Cognito/JWT) - token: {jwt_token[:50]}..."
@@ -437,22 +437,104 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             content=content, usage=usage_data, final_message=final_message
         )
 
-    def get_streaming_response(
+    def _stream_agentcore_response_sync(
         self,
+        response: httpx.Response,
         model: str,
-        raw_response: httpx.Response,
-    ) -> AgentCoreSSEStreamIterator:
+    ):
         """
-        Return a streaming iterator for SSE responses.
-
-        Args:
-            model: The model name
-            raw_response: Raw HTTP response with streaming data
-
-        Returns:
-            AgentCoreSSEStreamIterator: Iterator that yields ModelResponse chunks
+        Internal sync generator that parses SSE and yields ModelResponse chunks.
         """
-        return AgentCoreSSEStreamIterator(response=raw_response, model=model)
+        buffer = ""
+        for text_chunk in response.iter_text():
+            buffer += text_chunk
+
+            # Process complete lines
+            while '\n' in buffer:
+                line, buffer = buffer.split('\n', 1)
+                line = line.strip()
+
+                if not line or not line.startswith('data:'):
+                    continue
+
+                json_str = line[5:].strip()
+                if not json_str:
+                    continue
+
+                try:
+                    data_obj = json.loads(json_str)
+                    if not isinstance(data_obj, dict):
+                        continue
+
+                    # Process contentBlockDelta events
+                    if "event" in data_obj and isinstance(data_obj["event"], dict):
+                        event_payload = data_obj["event"]
+                        content_block_delta = event_payload.get("contentBlockDelta")
+
+                        if content_block_delta:
+                            delta = content_block_delta.get("delta", {})
+                            text = delta.get("text", "")
+
+                            if text:
+                                chunk = ModelResponse(
+                                    id=f"chatcmpl-{uuid.uuid4()}",
+                                    created=0,
+                                    model=model,
+                                    object="chat.completion.chunk",
+                                )
+                                chunk.choices = [
+                                    StreamingChoices(
+                                        finish_reason=None,
+                                        index=0,
+                                        delta=Delta(content=text, role="assistant"),
+                                    )
+                                ]
+                                yield chunk
+
+                        # Process metadata/usage
+                        metadata = event_payload.get("metadata")
+                        if metadata and "usage" in metadata:
+                            chunk = ModelResponse(
+                                id=f"chatcmpl-{uuid.uuid4()}",
+                                created=0,
+                                model=model,
+                                object="chat.completion.chunk",
+                            )
+                            chunk.choices = [
+                                StreamingChoices(
+                                    finish_reason="stop",
+                                    index=0,
+                                    delta=Delta(),
+                                )
+                            ]
+                            usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
+                            setattr(chunk, "usage", Usage(
+                                prompt_tokens=usage_data.get("inputTokens", 0),
+                                completion_tokens=usage_data.get("outputTokens", 0),
+                                total_tokens=usage_data.get("totalTokens", 0),
+                            ))
+                            yield chunk
+
+                    # Process final message
+                    if "message" in data_obj and isinstance(data_obj["message"], dict):
+                        chunk = ModelResponse(
+                            id=f"chatcmpl-{uuid.uuid4()}",
+                            created=0,
+                            model=model,
+                            object="chat.completion.chunk",
+                        )
+                        chunk.choices = [
+                            StreamingChoices(
+                                finish_reason="stop",
+                                index=0,
+                                delta=Delta(),
+                            )
+                        ]
+                        yield chunk
+
+                except json.JSONDecodeError:
+                    verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
+                    continue
 
     def get_sync_custom_stream_wrapper(
         self,
@@ -466,17 +548,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None,
         json_mode: Optional[bool] = None,
         signed_json_body: Optional[bytes] = None,
-    ) -> CustomStreamWrapper:
+    ) -> "CustomStreamWrapper":
         """
-        Get a CustomStreamWrapper for synchronous streaming.
-
-        This is called when stream=True is passed to completion().
+        Simplified sync streaming - returns a generator that yields ModelResponse chunks.
         """
         from litellm.llms.custom_httpx.http_handler import (
             HTTPHandler,
             _get_httpx_client,
         )
-        from litellm.utils import CustomStreamWrapper
 
         if client is None or not isinstance(client, HTTPHandler):
             client = _get_httpx_client(params={})
@@ -488,7 +567,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             api_base,
             headers=headers,
             data=signed_json_body if signed_json_body else json.dumps(data),
-            stream=True,  # THIS IS KEY - tells httpx to not buffer
+            stream=True,
             logging_obj=logging_obj,
         )
 
@@ -497,18 +576,6 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                 status_code=response.status_code, message=str(response.read())
             )
 
-        # Create iterator for SSE stream
-        completion_stream = self.get_streaming_response(
-            model=model, raw_response=response
-        )
-
-        streaming_response = CustomStreamWrapper(
-            completion_stream=completion_stream,
-            model=model,
-            custom_llm_provider=custom_llm_provider,
-            logging_obj=logging_obj,
-        )
-
         # LOGGING
         logging_obj.post_call(
             input=messages,
@@ -517,7 +584,112 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        return streaming_response
+        # Wrap the generator in CustomStreamWrapper
+        return CustomStreamWrapper(
+            completion_stream=self._stream_agentcore_response_sync(response, model),
+            model=model,
+            custom_llm_provider="bedrock",
+            logging_obj=logging_obj,
+        )
+
+    async def _stream_agentcore_response(
+        self,
+        response: httpx.Response,
+        model: str,
+    ) -> AsyncGenerator[ModelResponse, None]:
+        """
+        Internal async generator that parses SSE and yields ModelResponse chunks.
+        """
+        buffer = ""
+        async for text_chunk in response.aiter_text():
+            buffer += text_chunk
+
+            # Process complete lines
+            while '\n' in buffer:
+                line, buffer = buffer.split('\n', 1)
+                line = line.strip()
+
+                if not line or not line.startswith('data:'):
+                    continue
+
+                json_str = line[5:].strip()
+                if not json_str:
+                    continue
+
+                try:
+                    data_obj = json.loads(json_str)
+                    if not isinstance(data_obj, dict):
+                        continue
+
+                    # Process contentBlockDelta events
+                    if "event" in data_obj and isinstance(data_obj["event"], dict):
+                        event_payload = data_obj["event"]
+                        content_block_delta = event_payload.get("contentBlockDelta")
+
+                        if content_block_delta:
+                            delta = content_block_delta.get("delta", {})
+                            text = delta.get("text", "")
+
+                            if text:
+                                chunk = ModelResponse(
+                                    id=f"chatcmpl-{uuid.uuid4()}",
+                                    created=0,
+                                    model=model,
+                                    object="chat.completion.chunk",
+                                )
+                                chunk.choices = [
+                                    StreamingChoices(
+                                        finish_reason=None,
+                                        index=0,
+                                        delta=Delta(content=text, role="assistant"),
+                                    )
+                                ]
+                                yield chunk
+
+                        # Process metadata/usage
+                        metadata = event_payload.get("metadata")
+                        if metadata and "usage" in metadata:
+                            chunk = ModelResponse(
+                                id=f"chatcmpl-{uuid.uuid4()}",
+                                created=0,
+                                model=model,
+                                object="chat.completion.chunk",
+                            )
+                            chunk.choices = [
+                                StreamingChoices(
+                                    finish_reason="stop",
+                                    index=0,
+                                    delta=Delta(),
+                                )
+                            ]
+                            usage_data: AgentCoreUsage = metadata["usage"]  # type: ignore
+                            setattr(chunk, "usage", Usage(
+                                prompt_tokens=usage_data.get("inputTokens", 0),
+                                completion_tokens=usage_data.get("outputTokens", 0),
+                                total_tokens=usage_data.get("totalTokens", 0),
+                            ))
+                            yield chunk
+
+                    # Process final message
+                    if "message" in data_obj and isinstance(data_obj["message"], dict):
+                        chunk = ModelResponse(
+                            id=f"chatcmpl-{uuid.uuid4()}",
+                            created=0,
+                            model=model,
+                            object="chat.completion.chunk",
+                        )
+                        chunk.choices = [
+                            StreamingChoices(
+                                finish_reason="stop",
+                                index=0,
+                                delta=Delta(),
+                            )
+                        ]
+                        yield chunk
+
+                except json.JSONDecodeError:
+                    verbose_logger.debug(f"Skipping non-JSON SSE line: {line[:100]}")
+                    continue
 
     async def get_async_custom_stream_wrapper(
         self,
@@ -531,17 +703,14 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         client: Optional["AsyncHTTPHandler"] = None,
         json_mode: Optional[bool] = None,
         signed_json_body: Optional[bytes] = None,
-    ) -> CustomStreamWrapper:
+    ) -> "CustomStreamWrapper":
         """
-        Get a CustomStreamWrapper for asynchronous streaming.
-
-        This is called when stream=True is passed to acompletion().
+        Simplified async streaming - returns an async generator that yields ModelResponse chunks.
         """
         from litellm.llms.custom_httpx.http_handler import (
             AsyncHTTPHandler,
             get_async_httpx_client,
         )
-        from litellm.utils import CustomStreamWrapper
 
         if client is None or not isinstance(client, AsyncHTTPHandler):
             client = get_async_httpx_client(
@@ -555,7 +724,7 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             api_base,
             headers=headers,
             data=signed_json_body if signed_json_body else json.dumps(data),
-            stream=True,  # THIS IS KEY - tells httpx to not buffer
+            stream=True,
             logging_obj=logging_obj,
         )
 
@@ -564,18 +733,6 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
                 status_code=response.status_code, message=str(await response.aread())
             )
 
-        # Create iterator for SSE stream
-        completion_stream = self.get_streaming_response(
-            model=model, raw_response=response
-        )
-
-        streaming_response = CustomStreamWrapper(
-            completion_stream=completion_stream,
-            model=model,
-            custom_llm_provider=custom_llm_provider,
-            logging_obj=logging_obj,
-        )
-
         # LOGGING
         logging_obj.post_call(
             input=messages,
@@ -584,7 +741,13 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
             additional_args={"complete_input_dict": data},
         )
 
-        return streaming_response
+        # Wrap the async generator in CustomStreamWrapper
+        return CustomStreamWrapper(
+            completion_stream=self._stream_agentcore_response(response, model),
+            model=model,
+            custom_llm_provider="bedrock",
+            logging_obj=logging_obj,
+        )
 
     @property
     def has_custom_stream_wrapper(self) -> bool:
@@ -692,4 +855,5 @@ class AmazonAgentCoreConfig(BaseConfig, BaseAWSLLM):
         stream: Optional[bool],
         custom_llm_provider: Optional[str] = None,
     ) -> bool:
-        return True
+        # AgentCore supports true streaming - don't buffer
+        return False

litellm/llms/bedrock/chat/converse_transformation.py

@@ -53,7 +53,13 @@ from litellm.types.utils import (
     PromptTokensDetailsWrapper,
     Usage,
 )
-from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning
+from litellm.utils import (
+    add_dummy_tool,
+    any_assistant_message_has_thinking_blocks,
+    has_tool_call_blocks,
+    last_assistant_with_tool_calls_has_no_thinking_blocks,
+    supports_reasoning,
+)
 
 from ..common_utils import (
     BedrockError,
@@ -773,7 +779,7 @@ class AmazonConverseConfig(BaseConfig):
             return optional_params
 
         """
-        Follow similar approach to anthropic - translate to a single tool call. 
+        Follow similar approach to anthropic - translate to a single tool call.
 
         When using tools in this way: - https://docs.anthropic.com/en/docs/build-with-claude/tool-use#json-mode
         - You usually want to provide a single tool
@@ -1070,9 +1076,28 @@ class AmazonConverseConfig(BaseConfig):
                     llm_provider="bedrock",
                 )
 
+        # Drop thinking param if thinking is enabled but thinking_blocks are missing
+        # This prevents the error: "Expected thinking or redacted_thinking, but found tool_use"
+        #
+        # IMPORTANT: Only drop thinking if NO assistant messages have thinking_blocks.
+        # If any message has thinking_blocks, we must keep thinking enabled, otherwise
+        # Related issues: https://github.com/BerriAI/litellm/issues/14194
+        if (
+            optional_params.get("thinking") is not None
+            and messages is not None
+            and last_assistant_with_tool_calls_has_no_thinking_blocks(messages)
+            and not any_assistant_message_has_thinking_blocks(messages)
+        ):
+            if litellm.modify_params:
+                optional_params.pop("thinking", None)
+                litellm.verbose_logger.warning(
+                    "Dropping 'thinking' param because the last assistant message with tool_calls "
+                    "has no thinking_blocks. The model won't use extended thinking for this turn."
+                )
+
         # Prepare and separate parameters
-        inference_params, additional_request_params, request_metadata = (
-            self._prepare_request_params(optional_params, model)
+        inference_params, additional_request_params, request_metadata = self._prepare_request_params(
+            optional_params, model
         )
 
         original_tools = inference_params.pop("tools", [])
@@ -1459,11 +1484,11 @@ class AmazonConverseConfig(BaseConfig):
             )
 
         """
-        Bedrock Response Object has optional message block 
+        Bedrock Response Object has optional message block
 
         completion_response["output"].get("message", None)
 
-        A message block looks like this (Example 1): 
+        A message block looks like this (Example 1):
         "output": {
             "message": {
                 "role": "assistant",

litellm/llms/bedrock/chat/invoke_handler.py

@@ -374,6 +374,29 @@ class BedrockLLM(BaseAWSLLM):
     def __init__(self) -> None:
         super().__init__()
 
+    @staticmethod
+    def is_claude_messages_api_model(model: str) -> bool:
+        """
+        Check if the model uses the Claude Messages API (Claude 3+).
+
+        Handles:
+        - Regional prefixes: eu.anthropic.claude-*, us.anthropic.claude-*
+        - Claude 3 models: claude-3-haiku, claude-3-sonnet, claude-3-opus, claude-3-5-*, claude-3-7-*
+        - Claude 4 models: claude-opus-4, claude-sonnet-4, claude-haiku-4
+        """
+        # Normalize model string to lowercase for matching
+        model_lower = model.lower()
+
+        # Claude 3+ indicators (all use Messages API)
+        messages_api_indicators = [
+            "claude-3",      # Claude 3.x models
+            "claude-opus-4", # Claude Opus 4
+            "claude-sonnet-4", # Claude Sonnet 4
+            "claude-haiku-4",  # Claude Haiku 4
+        ]
+
+        return any(indicator in model_lower for indicator in messages_api_indicators)
+
     def convert_messages_to_prompt(
         self, model, messages, provider, custom_prompt_dict
     ) -> Tuple[str, Optional[list]]:
@@ -465,7 +488,7 @@ class BedrockLLM(BaseAWSLLM):
                         completion_response["generations"][0]["finish_reason"]
                     )
             elif provider == "anthropic":
-                if model.startswith("anthropic.claude-3"):
+                if self.is_claude_messages_api_model(model):
                     json_schemas: dict = {}
                     _is_function_call = False
                     ## Handle Tool Calling
@@ -595,13 +618,12 @@ class BedrockLLM(BaseAWSLLM):
                         outputText = choice["message"].get("content")
                     elif "text" in choice:  # fallback for completion format
                         outputText = choice["text"]
-                    
                     # Set finish reason
                     if "finish_reason" in choice:
                         model_response.choices[0].finish_reason = map_finish_reason(
                             choice["finish_reason"]
                         )
-                    
+
                     # Set usage if available
                     if "usage" in completion_response:
                         usage = completion_response["usage"]
@@ -838,7 +860,7 @@ class BedrockLLM(BaseAWSLLM):
                     ] = True  # cohere requires stream = True in inference params
                 data = json.dumps({"prompt": prompt, **inference_params})
         elif provider == "anthropic":
-            if model.startswith("anthropic.claude-3"):
+            if self.is_claude_messages_api_model(model):
                 # Separate system prompt from rest of message
                 system_prompt_idx: list[int] = []
                 system_messages: list[str] = []
@@ -936,13 +958,13 @@ class BedrockLLM(BaseAWSLLM):
             # Use AmazonBedrockOpenAIConfig for proper OpenAI transformation
             openai_config = AmazonBedrockOpenAIConfig()
             supported_params = openai_config.get_supported_openai_params(model=model)
-            
+
             # Filter to only supported OpenAI params
             filtered_params = {
-                k: v for k, v in inference_params.items() 
+                k: v for k, v in inference_params.items()
                 if k in supported_params
             }
-            
+
             # OpenAI uses messages format, not prompt
             data = json.dumps({"messages": messages, **filtered_params})
         else:

litellm/llms/chatgpt/authenticator.py

@@ -0,0 +1,388 @@
+import base64
+import json
+import os
+import time
+from typing import Any, Dict, Optional
+
+import httpx
+
+from litellm._logging import verbose_logger
+from litellm.llms.custom_httpx.http_handler import _get_httpx_client
+
+from .common_utils import (
+    CHATGPT_API_BASE,
+    CHATGPT_AUTH_BASE,
+    CHATGPT_CLIENT_ID,
+    CHATGPT_DEVICE_CODE_URL,
+    CHATGPT_DEVICE_TOKEN_URL,
+    CHATGPT_DEVICE_VERIFY_URL,
+    CHATGPT_OAUTH_TOKEN_URL,
+    GetAccessTokenError,
+    GetDeviceCodeError,
+    RefreshAccessTokenError,
+)
+
+TOKEN_EXPIRY_SKEW_SECONDS = 60
+DEVICE_CODE_TIMEOUT_SECONDS = 15 * 60
+DEVICE_CODE_COOLDOWN_SECONDS = 5 * 60
+DEVICE_CODE_POLL_SLEEP_SECONDS = 5
+
+
+class Authenticator:
+    def __init__(self) -> None:
+        self.token_dir = os.getenv(
+            "CHATGPT_TOKEN_DIR",
+            os.path.expanduser("~/.config/litellm/chatgpt"),
+        )
+        self.auth_file = os.path.join(
+            self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json")
+        )
+        self._ensure_token_dir()
+
+    def get_api_base(self) -> str:
+        return (
+            os.getenv("CHATGPT_API_BASE")
+            or os.getenv("OPENAI_CHATGPT_API_BASE")
+            or CHATGPT_API_BASE
+        )
+
+    def get_access_token(self) -> str:
+        auth_data = self._read_auth_file()
+        if auth_data:
+            access_token = auth_data.get("access_token")
+            if access_token and not self._is_token_expired(auth_data, access_token):
+                return access_token
+            refresh_token = auth_data.get("refresh_token")
+            if refresh_token:
+                try:
+                    refreshed = self._refresh_tokens(refresh_token)
+                    return refreshed["access_token"]
+                except RefreshAccessTokenError as exc:
+                    verbose_logger.warning(
+                        "ChatGPT refresh token failed, re-login required: %s", exc
+                    )
+
+        cooldown_remaining = self._get_device_code_cooldown_remaining(auth_data)
+        if cooldown_remaining > 0:
+            token = self._wait_for_access_token(cooldown_remaining)
+            if token:
+                return token
+
+        tokens = self._login_device_code()
+        return tokens["access_token"]
+
+    def get_account_id(self) -> Optional[str]:
+        auth_data = self._read_auth_file()
+        if not auth_data:
+            return None
+        account_id = auth_data.get("account_id")
+        if account_id:
+            return account_id
+        id_token = auth_data.get("id_token")
+        access_token = auth_data.get("access_token")
+        derived = self._extract_account_id(id_token or access_token)
+        if derived:
+            auth_data["account_id"] = derived
+            self._write_auth_file(auth_data)
+        return derived
+
+    def _ensure_token_dir(self) -> None:
+        if not os.path.exists(self.token_dir):
+            os.makedirs(self.token_dir, exist_ok=True)
+
+    def _read_auth_file(self) -> Optional[Dict[str, Any]]:
+        try:
+            with open(self.auth_file, "r") as f:
+                return json.load(f)
+        except IOError:
+            return None
+        except json.JSONDecodeError as exc:
+            verbose_logger.warning("Invalid ChatGPT auth file: %s", exc)
+            return None
+
+    def _write_auth_file(self, data: Dict[str, Any]) -> None:
+        try:
+            with open(self.auth_file, "w") as f:
+                json.dump(data, f)
+        except IOError as exc:
+            verbose_logger.error("Failed to write ChatGPT auth file: %s", exc)
+
+    def _is_token_expired(self, auth_data: Dict[str, Any], access_token: str) -> bool:
+        expires_at = auth_data.get("expires_at")
+        if expires_at is None:
+            expires_at = self._get_expires_at(access_token)
+            if expires_at:
+                auth_data["expires_at"] = expires_at
+                self._write_auth_file(auth_data)
+        if expires_at is None:
+            return True
+        return time.time() >= float(expires_at) - TOKEN_EXPIRY_SKEW_SECONDS
+
+    def _get_expires_at(self, token: str) -> Optional[int]:
+        claims = self._decode_jwt_claims(token)
+        exp = claims.get("exp")
+        if isinstance(exp, (int, float)):
+            return int(exp)
+        return None
+
+    def _decode_jwt_claims(self, token: str) -> Dict[str, Any]:
+        try:
+            parts = token.split(".")
+            if len(parts) < 2:
+                return {}
+            payload_b64 = parts[1]
+            payload_b64 += "=" * (-len(payload_b64) % 4)
+            payload_bytes = base64.urlsafe_b64decode(payload_b64)
+            return json.loads(payload_bytes.decode("utf-8"))
+        except Exception:
+            return {}
+
+    def _extract_account_id(self, token: Optional[str]) -> Optional[str]:
+        if not token:
+            return None
+        claims = self._decode_jwt_claims(token)
+        auth_claims = claims.get("https://api.openai.com/auth")
+        if isinstance(auth_claims, dict):
+            account_id = auth_claims.get("chatgpt_account_id")
+            if isinstance(account_id, str) and account_id:
+                return account_id
+        return None
+
+    def _login_device_code(self) -> Dict[str, str]:
+        cooldown_remaining = self._get_device_code_cooldown_remaining(
+            self._read_auth_file()
+        )
+        if cooldown_remaining > 0:
+            token = self._wait_for_access_token(cooldown_remaining)
+            if token:
+                return {"access_token": token}
+
+        device_code = self._request_device_code()
+        self._record_device_code_request()
+        print(  # noqa: T201
+            "Sign in with ChatGPT using device code:\n"
+            f"1) Visit {CHATGPT_DEVICE_VERIFY_URL}\n"
+            f"2) Enter code: {device_code['user_code']}\n"
+            "Device codes are a common phishing target. Never share this code.",
+            flush=True,
+        )
+        auth_code = self._poll_for_authorization_code(device_code)
+        tokens = self._exchange_code_for_tokens(auth_code)
+        auth_data = self._build_auth_record(tokens)
+        self._write_auth_file(auth_data)
+        return tokens
+
+    def _request_device_code(self) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            resp = client.post(
+                CHATGPT_DEVICE_CODE_URL,
+                json={"client_id": CHATGPT_CLIENT_ID},
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise GetDeviceCodeError(
+                message=f"Failed to request device code: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise GetDeviceCodeError(
+                message=f"Failed to request device code: {exc}",
+                status_code=400,
+            )
+
+        device_auth_id = data.get("device_auth_id")
+        user_code = data.get("user_code") or data.get("usercode")
+        interval = data.get("interval")
+        if not device_auth_id or not user_code:
+            raise GetDeviceCodeError(
+                message=f"Device code response missing fields: {data}",
+                status_code=400,
+            )
+        return {
+            "device_auth_id": device_auth_id,
+            "user_code": user_code,
+            "interval": str(interval or "5"),
+        }
+
+    def _poll_for_authorization_code(self, device_code: Dict[str, str]) -> Dict[str, str]:
+        client = _get_httpx_client()
+        interval = int(device_code.get("interval", "5"))
+        start_time = time.time()
+        while time.time() - start_time < DEVICE_CODE_TIMEOUT_SECONDS:
+            try:
+                resp = client.post(
+                    CHATGPT_DEVICE_TOKEN_URL,
+                    json={
+                        "device_auth_id": device_code["device_auth_id"],
+                        "user_code": device_code["user_code"],
+                    },
+                )
+                if resp.status_code == 200:
+                    data = resp.json()
+                    if all(
+                        key in data
+                        for key in (
+                            "authorization_code",
+                            "code_challenge",
+                            "code_verifier",
+                        )
+                    ):
+                        return data
+                if resp.status_code in (403, 404):
+                    time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+                    continue
+                resp.raise_for_status()
+            except httpx.HTTPStatusError as exc:
+                status_code = exc.response.status_code if exc.response else None
+                if status_code in (403, 404):
+                    time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+                    continue
+                raise GetAccessTokenError(
+                    message=f"Polling failed: {exc}",
+                    status_code=exc.response.status_code,
+                )
+            except Exception as exc:
+                raise GetAccessTokenError(
+                    message=f"Polling failed: {exc}",
+                    status_code=400,
+                )
+            time.sleep(max(interval, DEVICE_CODE_POLL_SLEEP_SECONDS))
+
+        raise GetAccessTokenError(
+            message="Timed out waiting for device authorization",
+            status_code=408,
+        )
+
+    def _exchange_code_for_tokens(self, code_data: Dict[str, str]) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            redirect_uri = f"{CHATGPT_AUTH_BASE}/deviceauth/callback"
+            body = (
+                "grant_type=authorization_code"
+                f"&code={code_data['authorization_code']}"
+                f"&redirect_uri={redirect_uri}"
+                f"&client_id={CHATGPT_CLIENT_ID}"
+                f"&code_verifier={code_data['code_verifier']}"
+            )
+            resp = client.post(
+                CHATGPT_OAUTH_TOKEN_URL,
+                headers={"Content-Type": "application/x-www-form-urlencoded"},
+                content=body,
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise GetAccessTokenError(
+                message=f"Token exchange failed: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise GetAccessTokenError(
+                message=f"Token exchange failed: {exc}",
+                status_code=400,
+            )
+
+        if not all(key in data for key in ("access_token", "refresh_token", "id_token")):
+            raise GetAccessTokenError(
+                message=f"Token exchange response missing fields: {data}",
+                status_code=400,
+            )
+        return {
+            "access_token": data["access_token"],
+            "refresh_token": data["refresh_token"],
+            "id_token": data["id_token"],
+        }
+
+    def _refresh_tokens(self, refresh_token: str) -> Dict[str, str]:
+        try:
+            client = _get_httpx_client()
+            resp = client.post(
+                CHATGPT_OAUTH_TOKEN_URL,
+                json={
+                    "client_id": CHATGPT_CLIENT_ID,
+                    "grant_type": "refresh_token",
+                    "refresh_token": refresh_token,
+                    "scope": "openid profile email",
+                },
+            )
+            resp.raise_for_status()
+            data = resp.json()
+        except httpx.HTTPStatusError as exc:
+            raise RefreshAccessTokenError(
+                message=f"Refresh token failed: {exc}",
+                status_code=exc.response.status_code,
+            )
+        except Exception as exc:
+            raise RefreshAccessTokenError(
+                message=f"Refresh token failed: {exc}",
+                status_code=400,
+            )
+
+        access_token = data.get("access_token")
+        id_token = data.get("id_token")
+        if not access_token or not id_token:
+            raise RefreshAccessTokenError(
+                message=f"Refresh response missing fields: {data}",
+                status_code=400,
+            )
+
+        refreshed = {
+            "access_token": access_token,
+            "refresh_token": data.get("refresh_token", refresh_token),
+            "id_token": id_token,
+        }
+        auth_data = self._build_auth_record(refreshed)
+        self._write_auth_file(auth_data)
+        return refreshed
+
+    def _build_auth_record(self, tokens: Dict[str, str]) -> Dict[str, Any]:
+        access_token = tokens.get("access_token")
+        id_token = tokens.get("id_token")
+        expires_at = self._get_expires_at(access_token) if access_token else None
+        account_id = self._extract_account_id(id_token or access_token)
+        return {
+            "access_token": access_token,
+            "refresh_token": tokens.get("refresh_token"),
+            "id_token": id_token,
+            "expires_at": expires_at,
+            "account_id": account_id,
+        }
+
+    def _get_device_code_cooldown_remaining(
+        self, auth_data: Optional[Dict[str, Any]]
+    ) -> float:
+        if not auth_data:
+            return 0.0
+        requested_at = auth_data.get("device_code_requested_at")
+        if not isinstance(requested_at, (int, float, str)):
+            return 0.0
+        try:
+            requested_at = float(requested_at)
+        except (TypeError, ValueError):
+            return 0.0
+        elapsed = time.time() - requested_at
+        remaining = DEVICE_CODE_COOLDOWN_SECONDS - elapsed
+        return max(0.0, remaining)
+
+    def _record_device_code_request(self) -> None:
+        auth_data = self._read_auth_file() or {}
+        auth_data["device_code_requested_at"] = time.time()
+        self._write_auth_file(auth_data)
+
+    def _wait_for_access_token(self, timeout_seconds: float) -> Optional[str]:
+        deadline = time.time() + timeout_seconds
+        while time.time() < deadline:
+            auth_data = self._read_auth_file()
+            if auth_data:
+                access_token = auth_data.get("access_token")
+                if access_token and not self._is_token_expired(
+                    auth_data, access_token
+                ):
+                    return access_token
+            sleep_for = min(DEVICE_CODE_POLL_SLEEP_SECONDS, max(0.0, deadline - time.time()))
+            if sleep_for <= 0:
+                break
+            time.sleep(sleep_for)
+        return None

litellm/llms/chatgpt/chat/transformation.py

@@ -0,0 +1,75 @@
+from typing import List, Optional, Tuple
+
+from litellm.exceptions import AuthenticationError
+from litellm.llms.openai.openai import OpenAIConfig
+from litellm.types.llms.openai import AllMessageValues
+
+from ..authenticator import Authenticator
+from ..common_utils import (
+    GetAccessTokenError,
+    ensure_chatgpt_session_id,
+    get_chatgpt_default_headers,
+)
+
+
+class ChatGPTConfig(OpenAIConfig):
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+        custom_llm_provider: str = "openai",
+    ) -> None:
+        super().__init__()
+        self.authenticator = Authenticator()
+
+    def _get_openai_compatible_provider_info(
+        self,
+        model: str,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        custom_llm_provider: str,
+    ) -> Tuple[Optional[str], Optional[str], str]:
+        dynamic_api_base = self.authenticator.get_api_base()
+        try:
+            dynamic_api_key = self.authenticator.get_access_token()
+        except GetAccessTokenError as e:
+            raise AuthenticationError(
+                model=model,
+                llm_provider=custom_llm_provider,
+                message=str(e),
+            )
+        return dynamic_api_base, dynamic_api_key, custom_llm_provider
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        validated_headers = super().validate_environment(
+            headers, model, messages, optional_params, litellm_params, api_key, api_base
+        )
+
+        account_id = self.authenticator.get_account_id()
+        session_id = ensure_chatgpt_session_id(litellm_params)
+        default_headers = get_chatgpt_default_headers(
+            api_key or "", account_id, session_id
+        )
+        return {**default_headers, **validated_headers}
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        optional_params = super().map_openai_params(
+            non_default_params, optional_params, model, drop_params
+        )
+        optional_params.setdefault("stream", False)
+        return optional_params

litellm/llms/chatgpt/common_utils.py

@@ -0,0 +1,301 @@
+"""
+Constants and helpers for ChatGPT subscription OAuth.
+"""
+import os
+import platform
+from typing import Any, Optional, Union
+from uuid import uuid4
+
+import httpx
+
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+
+# OAuth + API constants (derived from openai/codex)
+CHATGPT_AUTH_BASE = "https://auth.openai.com"
+CHATGPT_DEVICE_CODE_URL = f"{CHATGPT_AUTH_BASE}/api/accounts/deviceauth/usercode"
+CHATGPT_DEVICE_TOKEN_URL = f"{CHATGPT_AUTH_BASE}/api/accounts/deviceauth/token"
+CHATGPT_OAUTH_TOKEN_URL = f"{CHATGPT_AUTH_BASE}/oauth/token"
+CHATGPT_DEVICE_VERIFY_URL = f"{CHATGPT_AUTH_BASE}/codex/device"
+CHATGPT_API_BASE = "https://chatgpt.com/backend-api/codex"
+CHATGPT_CLIENT_ID = "app_EMoamEEZ73f0CkXaXp7hrann"
+
+DEFAULT_ORIGINATOR = "codex_cli_rs"
+DEFAULT_USER_AGENT = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown"
+CHATGPT_DEFAULT_INSTRUCTIONS = """You are Codex, based on GPT-5. You are running as a coding agent in the Codex CLI on a user's computer.
+
+## General
+
+- When searching for text or files, prefer using `rg` or `rg --files` respectively because `rg` is much faster than alternatives like `grep`. (If the `rg` command is not found, then use alternatives.)
+
+## Editing constraints
+
+- Default to ASCII when editing or creating files. Only introduce non-ASCII or other Unicode characters when there is a clear justification and the file already uses them.
+- Add succinct code comments that explain what is going on if code is not self-explanatory. You should not add comments like "Assigns the value to the variable", but a brief comment might be useful ahead of a complex code block that the user would otherwise have to spend time parsing out. Usage of these comments should be rare.
+- Try to use apply_patch for single file edits, but it is fine to explore other options to make the edit if it does not work well. Do not use apply_patch for changes that are auto-generated (i.e. generating package.json or running a lint or format command like gofmt) or when scripting is more efficient (such as search and replacing a string across a codebase).
+- You may be in a dirty git worktree.
+    * NEVER revert existing changes you did not make unless explicitly requested, since these changes were made by the user.
+    * If asked to make a commit or code edits and there are unrelated changes to your work or changes that you didn't make in those files, don't revert those changes.
+    * If the changes are in files you've touched recently, you should read carefully and understand how you can work with the changes rather than reverting them.
+    * If the changes are in unrelated files, just ignore them and don't revert them.
+- Do not amend a commit unless explicitly requested to do so.
+- While you are working, you might notice unexpected changes that you didn't make. If this happens, STOP IMMEDIATELY and ask the user how they would like to proceed.
+- **NEVER** use destructive commands like `git reset --hard` or `git checkout --` unless specifically requested or approved by the user.
+
+## Plan tool
+
+When using the planning tool:
+- Skip using the planning tool for straightforward tasks (roughly the easiest 25%).
+- Do not make single-step plans.
+- When you made a plan, update it after having performed one of the sub-tasks that you shared on the plan.
+
+## Special user requests
+
+- If the user makes a simple request (such as asking for the time) which you can fulfill by running a terminal command (such as `date`), you should do so.
+- If the user asks for a "review", default to a code review mindset: prioritise identifying bugs, risks, behavioural regressions, and missing tests. Findings must be the primary focus of the response - keep summaries or overviews brief and only after enumerating the issues. Present findings first (ordered by severity with file/line references), follow with open questions or assumptions, and offer a change-summary only as a secondary detail. If no findings are discovered, state that explicitly and mention any residual risks or testing gaps.
+
+## Frontend tasks
+When doing frontend design tasks, avoid collapsing into "AI slop" or safe, average-looking layouts.
+Aim for interfaces that feel intentional, bold, and a bit surprising.
+- Typography: Use expressive, purposeful fonts and avoid default stacks (Inter, Roboto, Arial, system).
+- Color & Look: Choose a clear visual direction; define CSS variables; avoid purple-on-white defaults. No purple bias or dark mode bias.
+- Motion: Use a few meaningful animations (page-load, staggered reveals) instead of generic micro-motions.
+- Background: Don't rely on flat, single-color backgrounds; use gradients, shapes, or subtle patterns to build atmosphere.
+- Overall: Avoid boilerplate layouts and interchangeable UI patterns. Vary themes, type families, and visual languages across outputs.
+- Ensure the page loads properly on both desktop and mobile
+
+Exception: If working within an existing website or design system, preserve the established patterns, structure, and visual language.
+
+## Presenting your work and final message
+
+You are producing plain text that will later be styled by the CLI. Follow these rules exactly. Formatting should make results easy to scan, but not feel mechanical. Use judgment to decide how much structure adds value.
+
+- Default: be very concise; friendly coding teammate tone.
+- Ask only when needed; suggest ideas; mirror the user's style.
+- For substantial work, summarize clearly; follow final-answer formatting.
+- Skip heavy formatting for simple confirmations.
+- Don't dump large files you've written; reference paths only.
+- No "save/copy this file" - User is on the same machine.
+- Offer logical next steps (tests, commits, build) briefly; add verify steps if you couldn't do something.
+- For code changes:
+  * Lead with a quick explanation of the change, and then give more details on the context covering where and why a change was made. Do not start this explanation with "summary", just jump right in.
+  * If there are natural next steps the user may want to take, suggest them at the end of your response. Do not make suggestions if there are no natural next steps.
+  * When suggesting multiple options, use numeric lists for the suggestions so the user can quickly respond with a single number.
+- The user does not command execution outputs. When asked to show the output of a command (e.g. `git show`), relay the important details in your answer or summarize the key lines so the user understands the result.
+
+### Final answer structure and style guidelines
+
+- Plain text; CLI handles styling. Use structure only when it helps scanability.
+- Headers: optional; short Title Case (1-3 words) wrapped in **...**; no blank line before the first bullet; add only if they truly help.
+- Bullets: use - ; merge related points; keep to one line when possible; 4-6 per list ordered by importance; keep phrasing consistent.
+- Monospace: backticks for commands/paths/env vars/code ids and inline examples; use for literal keyword bullets; never combine with **.
+- Code samples or multi-line snippets should be wrapped in fenced code blocks; include an info string as often as possible.
+- Structure: group related bullets; order sections general -> specific -> supporting; for subsections, start with a bolded keyword bullet, then items; match complexity to the task.
+- Tone: collaborative, concise, factual; present tense, active voice; self-contained; no "above/below"; parallel wording.
+- Don'ts: no nested bullets/hierarchies; no ANSI codes; don't cram unrelated keywords; keep keyword lists short--wrap/reformat if long; avoid naming formatting styles in answers.
+- Adaptation: code explanations -> precise, structured with code refs; simple tasks -> lead with outcome; big changes -> logical walkthrough + rationale + next actions; casual one-offs -> plain sentences, no headers/bullets.
+- File References: When referencing files in your response follow the below rules:
+  * Use inline code to make file paths clickable.
+  * Each reference should have a stand alone path. Even if it's the same file.
+  * Accepted: absolute, workspace-relative, a/ or b/ diff prefixes, or bare filename/suffix.
+  * Optionally include line/column (1-based): :line[:column] or #Lline[Ccolumn] (column defaults to 1).
+  * Do not use URIs like file://, vscode://, or https://.
+  * Do not provide range of lines
+  * Examples: src/app.ts, src/app.ts:42, b/server/index.js#L10, C:\\repo\\project\\main.rs:12:5
+"""
+
+
+class ChatGPTAuthError(BaseLLMException):
+    def __init__(
+        self,
+        status_code,
+        message,
+        request: Optional[httpx.Request] = None,
+        response: Optional[httpx.Response] = None,
+        headers: Optional[Union[httpx.Headers, dict]] = None,
+        body: Optional[dict] = None,
+    ):
+        super().__init__(
+            status_code=status_code,
+            message=message,
+            request=request,
+            response=response,
+            headers=headers,
+            body=body,
+        )
+
+
+class GetDeviceCodeError(ChatGPTAuthError):
+    pass
+
+
+class GetAccessTokenError(ChatGPTAuthError):
+    pass
+
+
+class RefreshAccessTokenError(ChatGPTAuthError):
+    pass
+
+
+def _safe_header_value(value: str) -> str:
+    if not value:
+        return ""
+    return "".join(ch if 32 <= ord(ch) <= 126 else "_" for ch in value)
+
+
+def _sanitize_user_agent_token(value: str) -> str:
+    if not value:
+        return ""
+    return "".join(
+        ch if (ch.isalnum() or ch in "-_./") else "_" for ch in value
+    )
+
+
+def _terminal_user_agent() -> str:
+    term_program = os.getenv("TERM_PROGRAM")
+    if term_program:
+        version = os.getenv("TERM_PROGRAM_VERSION")
+        token = f"{term_program}/{version}" if version else term_program
+        return _sanitize_user_agent_token(token) or "unknown"
+
+    wezterm_version = os.getenv("WEZTERM_VERSION")
+    if wezterm_version is not None:
+        token = (
+            f"WezTerm/{wezterm_version}" if wezterm_version else "WezTerm"
+        )
+        return _sanitize_user_agent_token(token) or "WezTerm"
+
+    if (
+        os.getenv("ITERM_SESSION_ID")
+        or os.getenv("ITERM_PROFILE")
+        or os.getenv("ITERM_PROFILE_NAME")
+    ):
+        return "iTerm.app"
+
+    if os.getenv("TERM_SESSION_ID"):
+        return "Apple_Terminal"
+
+    if os.getenv("KITTY_WINDOW_ID") or "kitty" in (os.getenv("TERM") or ""):
+        return "kitty"
+
+    if os.getenv("ALACRITTY_SOCKET") or os.getenv("TERM") == "alacritty":
+        return "Alacritty"
+
+    konsole_version = os.getenv("KONSOLE_VERSION")
+    if konsole_version is not None:
+        token = (
+            f"Konsole/{konsole_version}" if konsole_version else "Konsole"
+        )
+        return _sanitize_user_agent_token(token) or "Konsole"
+
+    if os.getenv("GNOME_TERMINAL_SCREEN"):
+        return "gnome-terminal"
+
+    vte_version = os.getenv("VTE_VERSION")
+    if vte_version is not None:
+        token = f"VTE/{vte_version}" if vte_version else "VTE"
+        return _sanitize_user_agent_token(token) or "VTE"
+
+    if os.getenv("WT_SESSION"):
+        return "WindowsTerminal"
+
+    term = os.getenv("TERM")
+    if term:
+        return _sanitize_user_agent_token(term) or "unknown"
+
+    return "unknown"
+
+
+def _get_litellm_version() -> str:
+    try:
+        from importlib.metadata import version
+
+        return version("litellm")
+    except Exception:
+        return "0.0.0"
+
+
+def get_chatgpt_originator() -> str:
+    originator = os.getenv("CHATGPT_ORIGINATOR") or DEFAULT_ORIGINATOR
+    return _safe_header_value(originator) or DEFAULT_ORIGINATOR
+
+
+def get_chatgpt_user_agent(originator: str) -> str:
+    override = os.getenv("CHATGPT_USER_AGENT")
+    if override:
+        return _safe_header_value(override) or DEFAULT_USER_AGENT
+    version = _get_litellm_version()
+    os_type = platform.system() or "Unknown"
+    os_version = platform.release() or "0"
+    arch = platform.machine() or "unknown"
+    terminal_ua = _terminal_user_agent()
+    suffix = os.getenv("CHATGPT_USER_AGENT_SUFFIX", "").strip()
+    suffix = f" ({suffix})" if suffix else ""
+    candidate = (
+        f"{originator}/{version} ({os_type} {os_version}; {arch}) {terminal_ua}{suffix}"
+    )
+    return _safe_header_value(candidate) or DEFAULT_USER_AGENT
+
+
+def get_chatgpt_default_headers(
+    access_token: str,
+    account_id: Optional[str],
+    session_id: Optional[str] = None,
+) -> dict:
+    originator = get_chatgpt_originator()
+    user_agent = get_chatgpt_user_agent(originator)
+    headers = {
+        "Authorization": f"Bearer {access_token}",
+        "content-type": "application/json",
+        "accept": "text/event-stream",
+        "originator": originator,
+        "user-agent": user_agent,
+    }
+    if session_id:
+        headers["session_id"] = session_id
+    if account_id:
+        headers["ChatGPT-Account-Id"] = account_id
+    return headers
+
+
+def get_chatgpt_default_instructions() -> str:
+    return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS
+
+
+def _normalize_litellm_params(litellm_params: Optional[Any]) -> dict:
+    if litellm_params is None:
+        return {}
+    if isinstance(litellm_params, dict):
+        return litellm_params
+    if hasattr(litellm_params, "model_dump"):
+        try:
+            return litellm_params.model_dump()
+        except Exception:
+            return {}
+    if hasattr(litellm_params, "dict"):
+        try:
+            return litellm_params.dict()
+        except Exception:
+            return {}
+    return {}
+
+
+def get_chatgpt_session_id(litellm_params: Optional[Any]) -> Optional[str]:
+    params = _normalize_litellm_params(litellm_params)
+    for key in ("litellm_session_id", "session_id"):
+        value = params.get(key)
+        if value:
+            return str(value)
+    metadata = params.get("metadata")
+    if isinstance(metadata, dict):
+        value = metadata.get("session_id")
+        if value:
+            return str(value)
+    for key in ("litellm_trace_id", "litellm_call_id"):
+        value = params.get(key)
+        if value:
+            return str(value)
+    return None
+
+
+def ensure_chatgpt_session_id(litellm_params: Optional[Any]) -> str:
+    return get_chatgpt_session_id(litellm_params) or str(uuid4())

litellm/llms/chatgpt/responses/transformation.py

@@ -0,0 +1,191 @@
+import json
+from typing import Any, Optional
+
+from litellm.exceptions import AuthenticationError
+from litellm.constants import STREAM_SSE_DONE_STRING
+from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.llms.openai.common_utils import OpenAIError
+from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
+from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
+    _safe_convert_created_field,
+)
+from litellm.types.llms.openai import (
+    ResponsesAPIResponse,
+    ResponsesAPIStreamEvents,
+)
+from litellm.types.router import GenericLiteLLMParams
+from litellm.types.utils import LlmProviders
+from litellm.utils import CustomStreamWrapper
+
+from ..authenticator import Authenticator
+from ..common_utils import (
+    CHATGPT_API_BASE,
+    GetAccessTokenError,
+    ensure_chatgpt_session_id,
+    get_chatgpt_default_headers,
+    get_chatgpt_default_instructions,
+)
+
+
+class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
+    def __init__(self) -> None:
+        super().__init__()
+        self.authenticator = Authenticator()
+
+    @property
+    def custom_llm_provider(self) -> LlmProviders:
+        return LlmProviders.CHATGPT
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        litellm_params: Optional[GenericLiteLLMParams],
+    ) -> dict:
+        try:
+            access_token = self.authenticator.get_access_token()
+        except GetAccessTokenError as e:
+            raise AuthenticationError(
+                model=model,
+                llm_provider="chatgpt",
+                message=str(e),
+            )
+
+        account_id = self.authenticator.get_account_id()
+        session_id = ensure_chatgpt_session_id(litellm_params)
+        default_headers = get_chatgpt_default_headers(
+            access_token, account_id, session_id
+        )
+        return {**default_headers, **headers}
+
+    def transform_responses_api_request(
+        self,
+        model: str,
+        input: Any,
+        response_api_optional_request_params: dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> dict:
+        request = super().transform_responses_api_request(
+            model,
+            input,
+            response_api_optional_request_params,
+            litellm_params,
+            headers,
+        )
+        request.pop("max_output_tokens", None)
+        request.pop("max_tokens", None)
+        request.pop("max_completion_tokens", None)
+        request.pop("metadata", None)
+        base_instructions = get_chatgpt_default_instructions()
+        existing_instructions = request.get("instructions")
+        if existing_instructions:
+            if base_instructions not in existing_instructions:
+                request["instructions"] = (
+                    f"{base_instructions}\n\n{existing_instructions}"
+                )
+        else:
+            request["instructions"] = base_instructions
+        request["store"] = False
+        request["stream"] = True
+        include = list(request.get("include") or [])
+        if "reasoning.encrypted_content" not in include:
+            include.append("reasoning.encrypted_content")
+        request["include"] = include
+        return request
+
+    def transform_response_api_response(
+        self,
+        model: str,
+        raw_response: Any,
+        logging_obj: Any,
+    ):
+        content_type = (raw_response.headers or {}).get("content-type", "")
+        body_text = raw_response.text or ""
+        if "text/event-stream" not in content_type.lower():
+            trimmed_body = body_text.lstrip()
+            if not (
+                trimmed_body.startswith("event:")
+                or trimmed_body.startswith("data:")
+                or "\nevent:" in body_text
+                or "\ndata:" in body_text
+            ):
+                return super().transform_response_api_response(
+                    model=model,
+                    raw_response=raw_response,
+                    logging_obj=logging_obj,
+                )
+
+        logging_obj.post_call(
+            original_response=raw_response.text,
+            additional_args={"complete_input_dict": {}},
+        )
+
+        completed_response = None
+        error_message = None
+        for chunk in body_text.splitlines():
+            stripped_chunk = CustomStreamWrapper._strip_sse_data_from_chunk(chunk)
+            if not stripped_chunk:
+                continue
+            stripped_chunk = stripped_chunk.strip()
+            if not stripped_chunk:
+                continue
+            if stripped_chunk == STREAM_SSE_DONE_STRING:
+                break
+            try:
+                parsed_chunk = json.loads(stripped_chunk)
+            except json.JSONDecodeError:
+                continue
+            if not isinstance(parsed_chunk, dict):
+                continue
+            event_type = parsed_chunk.get("type")
+            if event_type == ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
+                response_payload = parsed_chunk.get("response")
+                if isinstance(response_payload, dict):
+                    response_payload = dict(response_payload)
+                    if "created_at" in response_payload:
+                        response_payload["created_at"] = _safe_convert_created_field(
+                            response_payload["created_at"]
+                        )
+                    try:
+                        completed_response = ResponsesAPIResponse(**response_payload)
+                    except Exception:
+                        completed_response = ResponsesAPIResponse.model_construct(
+                            **response_payload
+                        )
+                break
+            if event_type in (
+                ResponsesAPIStreamEvents.RESPONSE_FAILED,
+                ResponsesAPIStreamEvents.ERROR,
+            ):
+                error_obj = parsed_chunk.get("error") or (
+                    parsed_chunk.get("response") or {}
+                ).get("error")
+                if error_obj is not None:
+                    if isinstance(error_obj, dict):
+                        error_message = error_obj.get("message") or str(error_obj)
+                    else:
+                        error_message = str(error_obj)
+
+        if completed_response is None:
+            raise OpenAIError(
+                message=error_message or raw_response.text,
+                status_code=raw_response.status_code,
+            )
+
+        raw_headers = dict(raw_response.headers)
+        processed_headers = process_response_headers(raw_headers)
+        if not hasattr(completed_response, "_hidden_params"):
+            setattr(completed_response, "_hidden_params", {})
+        completed_response._hidden_params["additional_headers"] = processed_headers
+        completed_response._hidden_params["headers"] = raw_headers
+        return completed_response
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        litellm_params: dict,
+    ) -> str:
+        api_base = api_base or self.authenticator.get_api_base() or CHATGPT_API_BASE
+        api_base = api_base.rstrip("/")
+        return f"{api_base}/responses"

litellm/llms/custom_httpx/aiohttp_handler.py

@@ -134,6 +134,41 @@ class BaseLLMAIOHTTPHandler:
                 # Ignore errors during transport cleanup
                 pass
 
+    def __del__(self):
+        """
+        Cleanup: close aiohttp session on instance destruction.
+
+        Provides defense-in-depth for issue #12443 - ensures cleanup happens
+        even if atexit handler doesn't run (abnormal termination).
+        """
+        if (
+            self.client_session is not None
+            and not self.client_session.closed
+            and self._owns_session
+        ):
+            try:
+                import asyncio
+
+                try:
+                    loop = asyncio.get_event_loop()
+                    if loop.is_running():
+                        # Event loop is running - schedule cleanup task
+                        asyncio.create_task(self.close())
+                    else:
+                        # Event loop exists but not running - run cleanup
+                        loop.run_until_complete(self.close())
+                except RuntimeError:
+                    # No event loop available - create one for cleanup
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+                    try:
+                        loop.run_until_complete(self.close())
+                    finally:
+                        loop.close()
+            except Exception:
+                # Silently ignore errors during __del__ to avoid issues
+                pass
+
     async def _make_common_async_call(
         self,
         async_client_session: Optional[ClientSession],

litellm/llms/custom_httpx/async_client_cleanup.py

@@ -9,7 +9,8 @@ async def close_litellm_async_clients():
     Close all cached async HTTP clients to prevent resource leaks.
 
     This function iterates through all cached clients in litellm's in-memory cache
-    and closes any aiohttp client sessions that are still open.
+    and closes any aiohttp client sessions that are still open. Also closes the
+    global base_llm_aiohttp_handler instance (issue #12443).
     """
     # Import here to avoid circular import
     import litellm
@@ -25,7 +26,7 @@ async def close_litellm_async_clients():
             except Exception:
                 # Silently ignore errors during cleanup
                 pass
-        
+
         # Handle AsyncHTTPHandler instances (used by Gemini and other providers)
         elif hasattr(handler, 'client'):
             client = handler.client
@@ -43,7 +44,7 @@ async def close_litellm_async_clients():
                 except Exception:
                     # Silently ignore errors during cleanup
                     pass
-        
+
         # Handle any other objects with aclose method
         elif hasattr(handler, 'aclose'):
             try:
@@ -52,6 +53,17 @@ async def close_litellm_async_clients():
                 # Silently ignore errors during cleanup
                 pass
 
+    # Close the global base_llm_aiohttp_handler instance (issue #12443)
+    # This is used by Gemini and other providers that use aiohttp
+    if hasattr(litellm, 'base_llm_aiohttp_handler'):
+        base_handler = getattr(litellm, 'base_llm_aiohttp_handler', None)
+        if isinstance(base_handler, BaseLLMAIOHTTPHandler) and hasattr(base_handler, 'close'):
+            try:
+                await base_handler.close()
+            except Exception:
+                # Silently ignore errors during cleanup
+                pass
+
 
 def register_async_client_cleanup():
     """
@@ -62,22 +74,24 @@ def register_async_client_cleanup():
     import atexit
 
     def cleanup_wrapper():
+        """
+        Cleanup wrapper that creates a fresh event loop for atexit cleanup.
+
+        At exit time, the main event loop is often already closed. Creating a new
+        event loop ensures cleanup runs successfully (fixes issue #12443).
+        """
         try:
-            loop = asyncio.get_event_loop()
-            if loop.is_running():
-                # Schedule the cleanup coroutine
-                loop.create_task(close_litellm_async_clients())
-            else:
-                # Run the cleanup coroutine
-                loop.run_until_complete(close_litellm_async_clients())
-        except Exception:
-            # If we can't get an event loop or it's already closed, try creating a new one
+            # Always create a fresh event loop at exit time
+            # Don't use get_event_loop() - it may be closed or unavailable
+            loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(loop)
             try:
-                loop = asyncio.new_event_loop()
                 loop.run_until_complete(close_litellm_async_clients())
+            finally:
+                # Clean up the loop we created
                 loop.close()
-            except Exception:
-                # Silently ignore errors during cleanup
-                pass
+        except Exception:
+            # Silently ignore errors during cleanup to avoid exit handler failures
+            pass
 
     atexit.register(cleanup_wrapper)

litellm/llms/custom_httpx/http_handler.py

@@ -1168,8 +1168,10 @@ def get_async_httpx_client(
         return _cached_client
 
     if params is not None:
-        params["shared_session"] = shared_session
-        _new_client = AsyncHTTPHandler(**params)
+        # Filter out params that are only used for cache key, not for AsyncHTTPHandler.__init__
+        handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
+        handler_params["shared_session"] = shared_session
+        _new_client = AsyncHTTPHandler(**handler_params)
     else:
         _new_client = AsyncHTTPHandler(
             timeout=httpx.Timeout(timeout=600.0, connect=5.0),
@@ -1215,7 +1217,9 @@ def _get_httpx_client(params: Optional[dict] = None) -> HTTPHandler:
         return _cached_client
 
     if params is not None:
-        _new_client = HTTPHandler(**params)
+        # Filter out params that are only used for cache key, not for HTTPHandler.__init__
+        handler_params = {k: v for k, v in params.items() if k != "disable_aiohttp_transport"}
+        _new_client = HTTPHandler(**handler_params)
     else:
         _new_client = HTTPHandler(timeout=httpx.Timeout(timeout=600.0, connect=5.0))
 

litellm/llms/openai/common_utils.py

@@ -15,12 +15,14 @@ if TYPE_CHECKING:
     from aiohttp import ClientSession
 
 import litellm
+from litellm._logging import verbose_logger
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.custom_httpx.http_handler import (
     _DEFAULT_TTL_FOR_HTTPX_CLIENTS,
     AsyncHTTPHandler,
     get_ssl_configuration,
 )
+from litellm.types.utils import LlmProviders
 
 
 class OpenAIError(BaseLLMException):
@@ -203,30 +205,67 @@ class BaseOpenAILLM:
         if litellm.aclient_session is not None:
             return litellm.aclient_session
 
-        # Get unified SSL configuration
-        ssl_config = get_ssl_configuration()
+        # Use the global cached client system to prevent memory leaks (issue #14540)
+        # This routes through get_async_httpx_client() which provides TTL-based caching
+        from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 
-        return httpx.AsyncClient(
-            verify=ssl_config,
-            transport=AsyncHTTPHandler._create_async_transport(
-                ssl_context=ssl_config
-                if isinstance(ssl_config, ssl.SSLContext)
-                else None,
-                ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+        try:
+            # Get SSL config and include in params for proper cache key
+            ssl_config = get_ssl_configuration()
+            params = {"ssl_verify": ssl_config} if ssl_config is not None else {}
+            params["disable_aiohttp_transport"] = litellm.disable_aiohttp_transport
+
+            # Get a cached AsyncHTTPHandler which manages the httpx.AsyncClient
+            cached_handler = get_async_httpx_client(
+                llm_provider=LlmProviders.OPENAI,  # Cache key includes provider
+                params=params,  # Include SSL config in cache key
                 shared_session=shared_session,
-            ),
-            follow_redirects=True,
-        )
+            )
+            # Return the underlying httpx client from the handler
+            return cached_handler.client
+        except (ImportError, AttributeError, KeyError) as e:
+            # Fallback to creating a client directly if caching system unavailable
+            # This preserves backwards compatibility
+            verbose_logger.debug(
+                f"Client caching unavailable ({type(e).__name__}), using direct client creation"
+            )
+            ssl_config = get_ssl_configuration()
+            return httpx.AsyncClient(
+                verify=ssl_config,
+                transport=AsyncHTTPHandler._create_async_transport(
+                    ssl_context=ssl_config
+                    if isinstance(ssl_config, ssl.SSLContext)
+                    else None,
+                    ssl_verify=ssl_config if isinstance(ssl_config, bool) else None,
+                    shared_session=shared_session,
+                ),
+                follow_redirects=True,
+            )
 
     @staticmethod
     def _get_sync_http_client() -> Optional[httpx.Client]:
         if litellm.client_session is not None:
             return litellm.client_session
 
-        # Get unified SSL configuration
-        ssl_config = get_ssl_configuration()
+        # Use the global cached client system to prevent memory leaks (issue #14540)
+        from litellm.llms.custom_httpx.http_handler import _get_httpx_client
 
-        return httpx.Client(
-            verify=ssl_config,
-            follow_redirects=True,
-        )
+        try:
+            # Get SSL config and include in params for proper cache key
+            ssl_config = get_ssl_configuration()
+            params = {"ssl_verify": ssl_config} if ssl_config is not None else None
+
+            # Get a cached HTTPHandler which manages the httpx.Client
+            cached_handler = _get_httpx_client(params=params)
+            # Return the underlying httpx client from the handler
+            return cached_handler.client
+        except (ImportError, AttributeError, KeyError) as e:
+            # Fallback to creating a client directly if caching system unavailable
+            verbose_logger.debug(
+                f"Client caching unavailable ({type(e).__name__}), using direct client creation"
+            )
+            ssl_config = get_ssl_configuration()
+            return httpx.Client(
+                verify=ssl_config,
+                follow_redirects=True,
+            )

litellm/llms/openai/realtime/handler.py

@@ -56,7 +56,9 @@ class OpenAIRealtime(OpenAIChatCompletion):
         url = self._construct_url(api_base, query_params)
 
         try:
-            ssl_context = get_shared_realtime_ssl_context()
+            # Only use SSL context for secure websocket connections (wss://)
+            # websockets library doesn't accept ssl argument for ws:// URIs
+            ssl_context = None if url.startswith("ws://") else get_shared_realtime_ssl_context()
             # Log a masked request preview consistent with other endpoints.
             logging_obj.pre_call(
                 input=None,

litellm/llms/vertex_ai/common_utils.py

@@ -150,6 +150,34 @@ def get_supports_response_schema(
     return _supports_response_schema
 
 
+def supports_response_json_schema(model: str) -> bool:
+    """
+    Check if the model supports responseJsonSchema (JSON Schema format).
+
+    responseJsonSchema is supported by Gemini 2.0+ models and uses standard
+    JSON Schema format with lowercase types (string, object, etc.) instead of
+    the OpenAPI-style responseSchema with uppercase types (STRING, OBJECT, etc.).
+
+    Benefits of responseJsonSchema:
+    - Supports additionalProperties for stricter schema validation
+    - Uses standard JSON Schema format (no type conversion needed)
+    - Better compatibility with Pydantic's model_json_schema()
+
+    Args:
+        model: The model name (e.g., "gemini-2.0-flash", "gemini-2.5-pro")
+
+    Returns:
+        True if the model supports responseJsonSchema, False otherwise
+    """
+    model_lower = model.lower()
+
+    # Gemini 2.0+ and 2.5+ models support responseJsonSchema
+    # Pattern matches: gemini-2.0-*, gemini-2.5-*, gemini-3-*, etc.
+    gemini_2_plus_pattern = re.compile(r"gemini-([2-9]|[1-9]\d+)\.")
+
+    return bool(gemini_2_plus_pattern.search(model_lower))
+
+
 from typing import Literal, Optional
 
 all_gemini_url_modes = Literal[
@@ -487,6 +515,44 @@ def _build_vertex_schema(parameters: dict, add_property_ordering: bool = False):
     return parameters
 
 
+def _build_json_schema(parameters: dict) -> dict:
+    """
+    Build a JSON Schema for use with Gemini's responseJsonSchema parameter.
+
+    Unlike _build_vertex_schema (used for responseSchema), this function:
+    - Does NOT convert types to uppercase (keeps standard JSON Schema format)
+    - Does NOT add propertyOrdering
+    - Does NOT filter fields (allows additionalProperties)
+    - Still unpacks $defs/$ref (Gemini doesn't support JSON Schema references)
+
+    Parameters:
+        parameters: dict - the JSON schema to process
+
+    Returns:
+        dict - the processed schema in standard JSON Schema format
+    """
+    # Unpack $defs references (Gemini doesn't support $ref)
+    defs = parameters.pop("$defs", {})
+    for name, value in defs.items():
+        unpack_defs(value, defs)
+    unpack_defs(parameters, defs)
+
+    # Convert anyOf with null to nullable
+    convert_anyof_null_to_nullable(parameters)
+
+    # Handle empty strings in enum values - Gemini doesn't accept empty strings in enums
+    _fix_enum_empty_strings(parameters)
+
+    # Remove enums for non-string typed fields (Gemini requires enum only on strings)
+    _fix_enum_types(parameters)
+
+    # Handle empty items objects
+    process_items(parameters)
+    add_object_type(parameters)
+
+    return parameters
+
+
 def _filter_anyof_fields(schema_dict: Dict[str, Any]) -> Dict[str, Any]:
     """
     When anyof is present, only keep the anyof field and its contents - otherwise VertexAI will throw an error - https://github.com/BerriAI/litellm/issues/11164

litellm/llms/vertex_ai/gemini/transformation.py

@@ -72,17 +72,64 @@ def _convert_detail_to_media_resolution_enum(
         return {"level": "MEDIA_RESOLUTION_MEDIUM"}
     elif detail == "high":
         return {"level": "MEDIA_RESOLUTION_HIGH"}
+    elif detail == "ultra_high":
+        return {"level": "MEDIA_RESOLUTION_ULTRA_HIGH"}
     return None
 
 
-def _process_gemini_image(
-    image_url: str, 
+def _apply_gemini_3_metadata(
+    part: PartType,
+    model: Optional[str],
+    media_resolution_enum: Optional[Dict[str, str]],
+    video_metadata: Optional[Dict[str, Any]],
+) -> PartType:
+    """
+    Apply the unique media_resolution and video_metadata parameters of Gemini 3+    
+    """
+    if model is None:
+        return part
+
+    from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
+
+    if not VertexGeminiConfig._is_gemini_3_or_newer(model):
+        return part
+
+    part_dict = dict(part)
+
+    if media_resolution_enum is not None:
+        part_dict["media_resolution"] = media_resolution_enum
+
+    if video_metadata is not None:
+        gemini_video_metadata = {}
+        if "fps" in video_metadata:
+            gemini_video_metadata["fps"] = video_metadata["fps"]
+        if "start_offset" in video_metadata:
+            gemini_video_metadata["startOffset"] = video_metadata["start_offset"]
+        if "end_offset" in video_metadata:
+            gemini_video_metadata["endOffset"] = video_metadata["end_offset"]
+        if gemini_video_metadata:
+            part_dict["video_metadata"] = gemini_video_metadata
+
+    return cast(PartType, part_dict)
+
+
+def _process_gemini_media(
+    image_url: str,
     format: Optional[str] = None,
     media_resolution_enum: Optional[Dict[str, str]] = None,
     model: Optional[str] = None,
+    video_metadata: Optional[Dict[str, Any]] = None,
 ) -> PartType:
     """
-    Given an image URL, return the appropriate PartType for Gemini
+    Given a media URL (image, audio, or video), return the appropriate PartType for Gemini
+    By the way, actually video_metadata can only be used with videos; it cannot be used with images, audio, or files. However, I haven't made any special handling because vertex returns a parameter error.
+
+    Args:
+        image_url: The URL or base64 string of the media (image, audio, or video)
+        format: The MIME type of the media
+        media_resolution_enum: Media resolution level (for Gemini 3+)
+        model: The model name (to check version compatibility)
+        video_metadata: Video-specific metadata (fps, start_offset, end_offset)
     """
 
     try:
@@ -104,14 +151,9 @@ def _process_gemini_image(
                 mime_type = format
             file_data = FileDataType(mime_type=mime_type, file_uri=image_url)
             part: PartType = {"file_data": file_data}
-            
-            if media_resolution_enum is not None and model is not None:
-                from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
-                if VertexGeminiConfig._is_gemini_3_or_newer(model):
-                    part_dict = dict(part)
-                    part_dict["media_resolution"] = media_resolution_enum
-                    return cast(PartType, part_dict)
-            return part
+            return _apply_gemini_3_metadata(
+                part, model, media_resolution_enum, video_metadata
+            )
         elif (
             "https://" in image_url
             and (image_type := format or _get_image_mime_type_from_url(image_url))
@@ -119,27 +161,16 @@ def _process_gemini_image(
         ):
             file_data = FileDataType(mime_type=image_type, file_uri=image_url)
             part = {"file_data": file_data}
-            
-            if media_resolution_enum is not None and model is not None:
-                from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
-                if VertexGeminiConfig._is_gemini_3_or_newer(model):
-                    part_dict = dict(part)
-                    part_dict["media_resolution"] = media_resolution_enum
-                    return cast(PartType, part_dict)
-            return part
+            return _apply_gemini_3_metadata(
+                part, model, media_resolution_enum, video_metadata
+            )
         elif "http://" in image_url or "https://" in image_url or "base64" in image_url:
             image = convert_to_anthropic_image_obj(image_url, format=format)
             _blob: BlobType = {"data": image["data"], "mime_type": image["media_type"]}
-            
             part = {"inline_data": cast(BlobType, _blob)}
-            
-            if media_resolution_enum is not None and model is not None:
-                from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
-                if VertexGeminiConfig._is_gemini_3_or_newer(model):
-                    part_dict = dict(part)
-                    part_dict["media_resolution"] = media_resolution_enum
-                    return cast(PartType, part_dict)
-            return part
+            return _apply_gemini_3_metadata(
+                part, model, media_resolution_enum, video_metadata
+            )
         raise Exception("Invalid image received - {}".format(image_url))
     except Exception as e:
         raise e
@@ -253,8 +284,8 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                                 media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
                             else:
                                 image_url = img_element["image_url"]
-                            _part = _process_gemini_image(
-                                image_url=image_url, 
+                            _part = _process_gemini_media(
+                                image_url=image_url,
                                 format=format,
                                 media_resolution_enum=media_resolution_enum,
                                 model=model,
@@ -279,7 +310,7 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                                         )
                                     )
                                 )
-                                _part = _process_gemini_image(
+                                _part = _process_gemini_media(
                                     image_url=openai_image_str,
                                     format=audio_format_modified,
                                     model=model,
@@ -290,16 +321,24 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                             file_id = file_element["file"].get("file_id")
                             format = file_element["file"].get("format")
                             file_data = file_element["file"].get("file_data")
+                            detail = file_element["file"].get("detail")
+                            video_metadata = file_element["file"].get("video_metadata")
                             passed_file = file_id or file_data
                             if passed_file is None:
                                 raise Exception(
                                     "Unknown file type. Please pass in a file_id or file_data"
                                 )
+
+                            # Convert detail to media_resolution_enum
+                            media_resolution_enum = _convert_detail_to_media_resolution_enum(detail)
+
                             try:
-                                _part = _process_gemini_image(
-                                    image_url=passed_file, 
+                                _part = _process_gemini_media(
+                                    image_url=passed_file,
                                     format=format,
                                     model=model,
+                                    media_resolution_enum=media_resolution_enum,
+                                    video_metadata=video_metadata,
                                 )
                                 _parts.append(_part)
                             except Exception:

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -92,7 +92,12 @@ from litellm.utils import (
 )
 
 from ....utils import _remove_additional_properties, _remove_strict_from_schema
-from ..common_utils import VertexAIError, _build_vertex_schema
+from ..common_utils import (
+    VertexAIError,
+    _build_json_schema,
+    _build_vertex_schema,
+    supports_response_json_schema,
+)
 from ..vertex_llm_base import VertexBase
 from .transformation import (
     _gemini_convert_messages_with_history,
@@ -624,30 +629,55 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
             )
         return old_schema
 
-    def apply_response_schema_transformation(self, value: dict, optional_params: dict):
+    def apply_response_schema_transformation(
+        self, value: dict, optional_params: dict, model: str
+    ):
         new_value = deepcopy(value)
-        # remove 'additionalProperties' from json schema
-        new_value = _remove_additional_properties(new_value)
-        # remove 'strict' from json schema
+        # remove 'strict' from json schema (not supported by Gemini)
         new_value = _remove_strict_from_schema(new_value)
-        if new_value["type"] == "json_object":
+
+        # Automatically use responseJsonSchema for Gemini 2.0+ models
+        # responseJsonSchema uses standard JSON Schema format and supports additionalProperties
+        # For older models (Gemini 1.5), fall back to responseSchema (OpenAPI format)
+        use_json_schema = supports_response_json_schema(model)
+
+        if not use_json_schema:
+            # For responseSchema, remove 'additionalProperties' (not supported)
+            new_value = _remove_additional_properties(new_value)
+
+        # Handle response type
+        if new_value.get("type") == "json_object":
             optional_params["response_mime_type"] = "application/json"
-        elif new_value["type"] == "text":
+        elif new_value.get("type") == "text":
             optional_params["response_mime_type"] = "text/plain"
+
+        # Extract schema from response_format
+        schema = None
         if "response_schema" in new_value:
             optional_params["response_mime_type"] = "application/json"
-            optional_params["response_schema"] = new_value["response_schema"]
-        elif new_value["type"] == "json_schema":  # type: ignore
-            if "json_schema" in new_value and "schema" in new_value["json_schema"]:  # type: ignore
+            schema = new_value["response_schema"]
+        elif new_value.get("type") == "json_schema":
+            if "json_schema" in new_value and "schema" in new_value["json_schema"]:
                 optional_params["response_mime_type"] = "application/json"
-                optional_params["response_schema"] = new_value["json_schema"]["schema"]  # type: ignore
+                schema = new_value["json_schema"]["schema"]
 
-        if "response_schema" in optional_params and isinstance(
-            optional_params["response_schema"], dict
-        ):
-            optional_params["response_schema"] = self._map_response_schema(
-                value=optional_params["response_schema"]
-            )
+        if schema and isinstance(schema, dict):
+            if use_json_schema:
+                # Use responseJsonSchema (Gemini 2.0+ only, opt-in)
+                # - Standard JSON Schema format (lowercase types)
+                # - Supports additionalProperties
+                # - No propertyOrdering needed
+                optional_params["response_json_schema"] = _build_json_schema(
+                    deepcopy(schema)
+                )
+            else:
+                # Use responseSchema (default, backwards compatible)
+                # - OpenAPI-style format (uppercase types)
+                # - No additionalProperties support
+                # - Requires propertyOrdering
+                optional_params["response_schema"] = self._map_response_schema(
+                    value=schema
+                )
 
     @staticmethod
     def _map_reasoning_effort_to_thinking_budget(
@@ -947,7 +977,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                 optional_params["max_output_tokens"] = value
             elif param == "response_format" and isinstance(value, dict):  # type: ignore
                 self.apply_response_schema_transformation(
-                    value=value, optional_params=optional_params
+                    value=value, optional_params=optional_params, model=model
                 )
             elif param == "frequency_penalty":
                 if self._supports_penalty_parameters(model):
@@ -988,25 +1018,34 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     optional_params["parallel_tool_calls"] = value
             elif param == "seed":
                 optional_params["seed"] = value
-            elif param == "reasoning_effort" and isinstance(value, str):
-                # Validate no conflict with thinking_level
-                VertexGeminiConfig._validate_thinking_config_conflicts(
-                    optional_params=optional_params,
-                    param_name="reasoning_effort",
-                    param_description="thinking_budget",
-                )
-                if VertexGeminiConfig._is_gemini_3_or_newer(model):
-                    optional_params["thinkingConfig"] = (
-                        VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
-                            value, model
-                        )
+            elif param == "reasoning_effort":
+                # Extract effort value - handle both string and dict formats
+                # Dict format comes from OpenAI Agents SDK: {"effort": "high", "summary": "auto"}
+                effort_value: Optional[str] = None
+                if isinstance(value, str):
+                    effort_value = value
+                elif isinstance(value, dict):
+                    effort_value = value.get("effort")
+
+                if effort_value is not None:
+                    # Validate no conflict with thinking_level
+                    VertexGeminiConfig._validate_thinking_config_conflicts(
+                        optional_params=optional_params,
+                        param_name="reasoning_effort",
+                        param_description="thinking_budget",
                     )
-                else:
-                    optional_params["thinkingConfig"] = (
-                        VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
-                            value, model
+                    if VertexGeminiConfig._is_gemini_3_or_newer(model):
+                        optional_params["thinkingConfig"] = (
+                            VertexGeminiConfig._map_reasoning_effort_to_thinking_level(
+                                effort_value, model
+                            )
+                        )
+                    else:
+                        optional_params["thinkingConfig"] = (
+                            VertexGeminiConfig._map_reasoning_effort_to_thinking_budget(
+                                effort_value, model
+                            )
                         )
-                    )
             elif param == "thinking":
                 # Validate no conflict with thinking_level
                 VertexGeminiConfig._validate_thinking_config_conflicts(

litellm/llms/vertex_ai/multimodal_embeddings/transformation.py

@@ -265,7 +265,7 @@ class VertexAIMultimodalEmbeddingConfig(BaseEmbeddingConfig):
                 image_count += 1
 
         ## Calculate video embeddings usage
-        video_length_seconds = 0
+        video_length_seconds = 0.0
         for prediction in vertex_predictions["predictions"]:
             video_embeddings = prediction.get("videoEmbeddings")
             if video_embeddings:

litellm/llms/vertex_ai/vertex_llm_base.py

@@ -23,6 +23,11 @@ from .common_utils import (
     is_global_only_vertex_model,
 )
 
+GOOGLE_IMPORT_ERROR_MESSAGE = (
+    "Google Cloud SDK not found. Install it with: pip install 'litellm[google]' "
+    "or pip install google-cloud-aiplatform"
+)
+
 if TYPE_CHECKING:
     from google.auth.credentials import Credentials as GoogleCredentialsObject
 else:
@@ -138,7 +143,10 @@ class VertexBase:
 
     # Google Auth Helpers -- extracted for mocking purposes in tests
     def _credentials_from_identity_pool(self, json_obj, scopes):
-        from google.auth import identity_pool
+        try:
+            from google.auth import identity_pool
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         creds = identity_pool.Credentials.from_info(json_obj)
         if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
@@ -146,7 +154,10 @@ class VertexBase:
         return creds
 
     def _credentials_from_identity_pool_with_aws(self, json_obj, scopes):
-        from google.auth import aws
+        try:
+            from google.auth import aws
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         creds = aws.Credentials.from_info(json_obj)
         if scopes and hasattr(creds, "requires_scopes") and creds.requires_scopes:
@@ -154,22 +165,30 @@ class VertexBase:
         return creds
 
     def _credentials_from_authorized_user(self, json_obj, scopes):
-        import google.oauth2.credentials
+        try:
+            import google.oauth2.credentials
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google.oauth2.credentials.Credentials.from_authorized_user_info(
             json_obj, scopes=scopes
         )
 
     def _credentials_from_service_account(self, json_obj, scopes):
-        import google.oauth2.service_account
+        try:
+            import google.oauth2.service_account
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google.oauth2.service_account.Credentials.from_service_account_info(
             json_obj, scopes=scopes
         )
 
     def _credentials_from_default_auth(self, scopes):
-
-        import google.auth as google_auth
+        try:
+            import google.auth as google_auth
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         return google_auth.default(scopes=scopes)
 
@@ -261,9 +280,12 @@ class VertexBase:
         return api_base
 
     def refresh_auth(self, credentials: Any) -> None:
-        from google.auth.transport.requests import (
-            Request,  # type: ignore[import-untyped]
-        )
+        try:
+            from google.auth.transport.requests import (
+                Request,  # type: ignore[import-untyped]
+            )
+        except ImportError:
+            raise ImportError(GOOGLE_IMPORT_ERROR_MESSAGE)
 
         credentials.refresh(Request())
 

litellm/llms/volcengine/__init__.py

@@ -1,6 +1,6 @@
 """
 Volcengine LLM Provider
-Support for Volcengine (ByteDance) chat and embedding models
+Support for Volcengine (ByteDance) chat, embedding, and responses models.
 """
 
 from .chat.transformation import VolcEngineChatConfig
@@ -10,6 +10,7 @@ from .common_utils import (
     get_volcengine_headers,
 )
 from .embedding import VolcEngineEmbeddingConfig
+from .responses.transformation import VolcEngineResponsesAPIConfig
 
 # For backward compatibility, keep the old class name
 VolcEngineConfig = VolcEngineChatConfig
@@ -18,6 +19,7 @@ __all__ = [
     "VolcEngineChatConfig",
     "VolcEngineConfig",  # backward compatibility
     "VolcEngineEmbeddingConfig",
+    "VolcEngineResponsesAPIConfig",
     "VolcEngineError",
     "get_volcengine_base_url",
     "get_volcengine_headers",