Merge pull request #26211 from BerriAI/litellm_internal_staging

[Infra] Promote internal staging to main

.circleci/config.yml

@@ -439,7 +439,14 @@ jobs:
         auth:
           username: ${DOCKERHUB_USERNAME}
           password: ${DOCKERHUB_PASSWORD}
+      - image: cimg/postgres:16.0
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
     working_directory: ~/project
+    environment:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
 
     steps:
       - checkout
@@ -463,12 +470,14 @@ jobs:
           paths:
             - ./.venv
           key: v2-dependencies-{{ checksum "uv.lock" }}-{{ checksum ".circleci/config.yml" }}
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
       - run:
-          name: Run prisma ./docker/entrypoint.sh
+          name: Seed DB schema via prisma db push
           command: |
             set +e
-            chmod +x docker/entrypoint.sh
-            ./docker/entrypoint.sh
+            uv run --no-sync litellm --skip_server_startup --use_prisma_db_push
             set -e
       - run:
           name: Generate Prisma Client
@@ -1520,7 +1529,50 @@ jobs:
           command: |
             pwd
             ls
-            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py
+            uv run --no-sync python -m pytest -vv tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
+
+  installing_litellm_on_python_v2_migration_resolver:
+    docker:
+      - image: cimg/python:3.11
+        auth:
+          username: ${DOCKERHUB_USERNAME}
+          password: ${DOCKERHUB_PASSWORD}
+      - image: cimg/postgres:16.0
+        environment:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: litellm_test
+    working_directory: ~/project
+    environment:
+      DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test"
+
+    steps:
+      - checkout
+      - setup_google_dns
+      - run:
+          name: Install Dependencies
+          command: |
+            curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
+            echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f  /tmp/uv-install.sh" | sha256sum -c -
+            env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
+            rm -f /tmp/uv-install.sh
+            echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
+            export PATH="$HOME/.local/bin:$PATH"
+            if [ -f "$HOME/miniconda/etc/profile.d/conda.sh" ]; then
+              export PATH="$HOME/miniconda/bin:$PATH"
+              source "$HOME/miniconda/etc/profile.d/conda.sh"
+              conda activate myenv
+            fi
+            uv sync --frozen --all-groups --all-extras --python "$(which python)"
+      - setup_litellm_enterprise_pip
+      - wait_for_service:
+          url: tcp://localhost:5432
+          timeout: "60"
+      - run:
+          name: Run v2 migration resolver proxy smoke test
+          command: |
+            uv run --no-sync python -m pytest -vv \
+              tests/local_testing/test_basic_python_version.py::test_litellm_proxy_server_config_no_general_settings_v2_resolver
 
   installing_litellm_on_python_3_13:
     docker:
@@ -1554,7 +1606,7 @@ jobs:
           command: |
             pwd
             ls
-            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py
+            uv run --no-sync python -m pytest -v tests/local_testing/test_basic_python_version.py -k "not v2_resolver"
   helm_chart_testing:
     machine:
       image: ubuntu-2204:2023.10.1 # Use machine executor instead of docker
@@ -3042,10 +3094,19 @@ jobs:
             - ui/litellm-dashboard/node_modules
       - run:
           name: Build UI from source
+          # Prior version used `cp -r out/ ../../litellm/proxy/_experimental/out/`.
+          # GNU cp (used on CircleCI's Ubuntu image) interprets that as "copy the
+          # source directory as a child of the destination" when the destination
+          # already exists — silently creating `_experimental/out/out/` instead of
+          # replacing the served bundle. The proxy continued serving whatever was
+          # checked into `_experimental/out/*`, so this job was effectively testing
+          # the pre-build bundle on every run. Replace-and-move guarantees the
+          # freshly built bundle is what the proxy actually serves.
           command: |
             cd ui/litellm-dashboard
             npm run build
-            cp -r out/ ../../litellm/proxy/_experimental/out/
+            rm -rf ../../litellm/proxy/_experimental/out
+            mv out ../../litellm/proxy/_experimental/out
             # Restructure HTML so extensionless routes work (login.html -> login/index.html)
             find ../../litellm/proxy/_experimental/out -name '*.html' ! -name 'index.html' | while read -r f; do
               d="${f%.html}"; mkdir -p "$d"; mv "$f" "$d/index.html"
@@ -3526,6 +3587,12 @@ workflows:
               only:
                 - main
                 - /litellm_.*/
+      - installing_litellm_on_python_v2_migration_resolver:
+          filters:
+            branches:
+              only:
+                - main
+                - /litellm_.*/
       - helm_chart_testing:
           requires:
             - build_docker_database_image

.github/workflows/test-unit-proxy-db.yml

@@ -31,8 +31,15 @@ jobs:
             test-path: "tests/proxy_unit_tests/test_auth_checks.py tests/proxy_unit_tests/test_user_api_key_auth.py"
             workers: 8
             timeout: 20
+          # test_proxy_utils.py is large (168+ parametrized tests) — run it on its
+          # own matrix so --dist=loadscope doesn't pin all of it to a single xdist
+          # worker and push the "remaining" group past the job timeout.
+          - test-group: proxy-utils
+            test-path: "tests/proxy_unit_tests/test_proxy_utils.py"
+            workers: 8
+            timeout: 20
           - test-group: remaining
-            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py"
+            test-path: "tests/proxy_unit_tests --ignore=tests/proxy_unit_tests/test_key_generate_prisma.py --ignore=tests/proxy_unit_tests/test_auth_checks.py --ignore=tests/proxy_unit_tests/test_user_api_key_auth.py --ignore=tests/proxy_unit_tests/test_proxy_utils.py"
             workers: 8
             timeout: 30
     uses: ./.github/workflows/_test-unit-services-base.yml

ci_cd/run_migration.py

@@ -1,22 +1,231 @@
+import argparse
 import os
-import subprocess
-from pathlib import Path
-from datetime import datetime
-import testing.postgresql
+import re
 import shutil
+import subprocess
+import sys
+from datetime import datetime
+from pathlib import Path
+
+import testing.postgresql
 
 
-def create_migration(migration_name: str = None):
+DESTRUCTIVE_PATTERN = re.compile(r"\bDROP\s+(COLUMN|TABLE|INDEX)\b", re.IGNORECASE)
+DEFAULT_BASE_BRANCH = "litellm_internal_staging"
+
+
+def _find_destructive_statements(sql: str) -> list:
+    """Return SQL lines containing DROP COLUMN, DROP TABLE, or DROP INDEX."""
+    return [
+        line.strip() for line in sql.splitlines() if DESTRUCTIVE_PATTERN.search(line)
+    ]
+
+
+def _print_freshness_failure(
+    base_branch: str, reason: str, stderr_text: str = ""
+) -> None:
+    """Loudly refuse to run when the freshness check can't be completed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(f"  FRESHNESS CHECK FAILED — COULD NOT VERIFY origin/{base_branch}", file=out)
+    print(banner, file=out)
+    print("", file=out)
+    print(f"Reason: {reason}", file=out)
+    if stderr_text:
+        print("", file=out)
+        print("git stderr:", file=out)
+        for line in stderr_text.rstrip().splitlines():
+            print(f"    {line}", file=out)
+    print("", file=out)
+    print("Common causes:", file=out)
+    print("  - No network access (offline)", file=out)
+    print("  - 'origin' remote not configured, or base branch name is wrong", file=out)
+    print("  - Not a git repository", file=out)
+    print("", file=out)
+    print("Options:", file=out)
+    print(
+        f"  - Fix the above and re-run, OR pass --base-branch <name> if your", file=out
+    )
+    print(
+        f"    base branch is not '{base_branch}', OR pass --skip-freshness-check",
+        file=out,
+    )
+    print("    to bypass (only if you fully understand the risk).", file=out)
+    print(banner, file=out)
+
+
+def _print_stale_branch_refusal(base_branch: str, behind: int) -> None:
+    """Loudly refuse to run when HEAD is behind the base branch."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        f"  STALE BRANCH — {behind} commit(s) behind origin/{base_branch}",
+        file=out,
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        f"Your branch is {behind} commit(s) behind origin/{base_branch}. Generating a\n"
+        "migration from a stale branch is how newly-added columns get silently\n"
+        "dropped — the exact incident this guard exists to prevent.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(f"  git fetch origin && git rebase origin/{base_branch}", file=out)
+    print("  # or merge, whichever matches your workflow", file=out)
+    print("", file=out)
+    print(
+        "If you have a genuine reason to bypass (e.g. intentionally generating",
+        file=out,
+    )
+    print("a migration against an older base), pass --skip-freshness-check.", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically run `git rebase` or pass\n"
+        "  --skip-freshness-check. Surface this error to the human operator and\n"
+        "  wait for their explicit confirmation. Auto-rebasing can drop the\n"
+        "  human's in-progress schema edits via a bad conflict resolution.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def _check_branch_freshness(root_dir: Path, base_branch: str) -> None:
+    """Fetch origin/<base_branch> and exit 3 if HEAD is behind it."""
+    cwd = str(root_dir)
+    try:
+        subprocess.run(
+            ["git", "fetch", "origin", base_branch],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+    except FileNotFoundError:
+        _print_freshness_failure(base_branch, "git executable not found on PATH")
+        sys.exit(3)
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git fetch origin {base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-list", "--count", f"HEAD..origin/{base_branch}"],
+            check=True,
+            capture_output=True,
+            text=True,
+            cwd=cwd,
+        )
+        behind = int(result.stdout.strip())
+    except subprocess.CalledProcessError as e:
+        _print_freshness_failure(
+            base_branch,
+            f"`git rev-list HEAD..origin/{base_branch}` failed",
+            e.stderr or "",
+        )
+        sys.exit(3)
+    except ValueError:
+        _print_freshness_failure(
+            base_branch,
+            "could not parse commit count from `git rev-list`",
+        )
+        sys.exit(3)
+
+    if behind > 0:
+        _print_stale_branch_refusal(base_branch, behind)
+        sys.exit(3)
+
+    print(f"Branch freshness OK: up to date with origin/{base_branch}.")
+
+
+def _print_destructive_refusal(destructive_lines: list) -> None:
+    """Loudly refuse to write a destructive migration and explain how to proceed."""
+    banner = "=" * 72
+    out = sys.stderr
+    print(banner, file=out)
+    print(
+        "  DESTRUCTIVE MIGRATION DETECTED — REFUSING TO WRITE MIGRATION FILE", file=out
+    )
+    print(banner, file=out)
+    print("", file=out)
+    print(
+        "The generated migration contains the following destructive statements:",
+        file=out,
+    )
+    print("", file=out)
+    for line in destructive_lines:
+        print(f"    {line}", file=out)
+    print("", file=out)
+    print(
+        "This very often means your branch is OUT OF DATE, or your local\n"
+        "schema.prisma is inconsistent with the migrations/ directory. A\n"
+        "previous incident saw newly-added columns silently dropped this way\n"
+        "and merged to main. Stop and verify before proceeding.",
+        file=out,
+    )
+    print("", file=out)
+    print("Before re-running:", file=out)
+    print(
+        "  1. git fetch origin && git status   # confirm branch is up to date", file=out
+    )
+    print(
+        "  2. Re-check all schema.prisma files are in sync (runbook Step 0)", file=out
+    )
+    print("  3. Review EACH DROP above — is it actually intended?", file=out)
+    print("  4. If (and only if) the drops are intentional, re-run with:", file=out)
+    print("         --allow-destructive", file=out)
+    print("", file=out)
+    print(banner, file=out)
+    print(
+        "  AI AGENTS / ASSISTANTS: DO NOT automatically re-run this command\n"
+        "  with --allow-destructive. Surface this error to the human operator\n"
+        "  and wait for their explicit confirmation before passing the flag.\n"
+        "  Passing the flag without human review is the exact failure mode\n"
+        "  this guard exists to prevent.",
+        file=out,
+    )
+    print(banner, file=out)
+
+
+def create_migration(
+    migration_name: str = None,
+    allow_destructive: bool = False,
+    base_branch: str = DEFAULT_BASE_BRANCH,
+    skip_freshness_check: bool = False,
+):
     """
     Create a new migration SQL file in the migrations directory by comparing
-    current database state with schema
+    current database state with schema.
 
     Args:
         migration_name (str): Name for the migration
+        allow_destructive (bool): Required to write a migration that contains
+            DROP COLUMN, DROP TABLE, or DROP INDEX statements. Without this
+            flag, the script exits non-zero and prints guidance.
+        base_branch (str): Branch to check freshness against
+            (default: "litellm_internal_staging").
+        skip_freshness_check (bool): Skip the "branch is up to date" check.
+            Only for intentional migrations against an older base.
     """
+    root_dir = Path(__file__).parent.parent
+
+    if skip_freshness_check:
+        print(
+            "WARNING: freshness check skipped (--skip-freshness-check). "
+            "Generating a migration from a stale branch can silently drop columns."
+        )
+    else:
+        _check_branch_freshness(root_dir, base_branch)
+
     try:
-        # Get paths
-        root_dir = Path(__file__).parent.parent
         migrations_dir = (
             root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
         )
@@ -59,7 +268,27 @@ def create_migration(migration_name: str = None):
                     check=True,
                 )
 
-                if result.stdout.strip():
+                # Prisma emits the literal "-- This is an empty migration." when
+                # there's no real drift. Treat that as "no changes".
+                diff_sql = result.stdout
+                stripped = diff_sql.strip()
+                is_empty_diff = (
+                    not stripped or stripped == "-- This is an empty migration."
+                )
+
+                if not is_empty_diff:
+                    destructive_lines = _find_destructive_statements(diff_sql)
+                    if destructive_lines and not allow_destructive:
+                        _print_destructive_refusal(destructive_lines)
+                        sys.exit(2)
+                    if destructive_lines and allow_destructive:
+                        print(
+                            "WARNING: writing destructive migration "
+                            "(--allow-destructive passed). Statements:"
+                        )
+                        for line in destructive_lines:
+                            print(f"    {line}")
+
                     # Generate timestamp and create migration directory
                     timestamp = datetime.now().strftime("%Y%m%d%H%M%S")
                     migration_name = migration_name or "unnamed_migration"
@@ -68,7 +297,7 @@ def create_migration(migration_name: str = None):
 
                     # Write the SQL to migration.sql
                     migration_file = migration_dir / "migration.sql"
-                    migration_file.write_text(result.stdout)
+                    migration_file.write_text(diff_sql)
 
                     print(f"Created migration in {migration_dir}")
                     return True
@@ -90,8 +319,48 @@ def create_migration(migration_name: str = None):
 
 
 if __name__ == "__main__":
-    # If running directly, can optionally pass migration name as argument
-    import sys
-
-    migration_name = sys.argv[1] if len(sys.argv) > 1 else None
-    create_migration(migration_name)
+    parser = argparse.ArgumentParser(
+        description=(
+            "Generate a Prisma migration by diffing the temp DB "
+            "(existing migrations applied) against schema.prisma."
+        )
+    )
+    parser.add_argument(
+        "migration_name",
+        nargs="?",
+        default=None,
+        help="Name for the migration (used in the generated directory name).",
+    )
+    parser.add_argument(
+        "--allow-destructive",
+        action="store_true",
+        help=(
+            "Required to write a migration that contains DROP COLUMN, "
+            "DROP TABLE, or DROP INDEX. Without this flag, destructive "
+            "diffs are refused."
+        ),
+    )
+    parser.add_argument(
+        "--base-branch",
+        default=DEFAULT_BASE_BRANCH,
+        help=(
+            f"Branch to check freshness against (default: {DEFAULT_BASE_BRANCH}). "
+            "The script fetches origin/<base-branch> and refuses to run if HEAD "
+            "is behind it."
+        ),
+    )
+    parser.add_argument(
+        "--skip-freshness-check",
+        action="store_true",
+        help=(
+            "Bypass the 'branch is up to date' check. Only for intentional "
+            "migrations against an older base. Pairs poorly with automation."
+        ),
+    )
+    args = parser.parse_args()
+    create_migration(
+        args.migration_name,
+        allow_destructive=args.allow_destructive,
+        base_branch=args.base_branch,
+        skip_freshness_check=args.skip_freshness_check,
+    )

docker/Dockerfile.non_root

@@ -15,29 +15,21 @@ COPY --from=uvbin /uv /usr/local/bin/uv
 COPY --from=uvbin /uvx /usr/local/bin/uvx
 
 RUN for i in 1 2 3; do \
-    apk add --no-cache \
-      python3 \
-      python3-dev \
-      clang \
-      llvm \
-      lld \
-      gcc \
-      linux-headers \
-      build-base \
-      bash \
-      coreutils \
-      curl \
-      openssl \
-      openssl-dev \
-      nodejs \
-      npm \
-      libsndfile && break || sleep 5; \
+      apk add --no-cache \
+        python3 \
+        python3-dev \
+        gcc \
+        bash \
+        coreutils \
+        curl \
+        openssl \
+        libsndfile \
+        nodejs && break || sleep 5; \
     done
 
 ENV UV_PROJECT_ENVIRONMENT=/app/.venv \
     UV_LINK_MODE=copy \
-    NVM_DIR=/root/.nvm \
-    PATH="/root/.nvm/versions/node/v20.20.2/bin:/app/.venv/bin:${PATH}" \
+    PATH="/app/.venv/bin:${PATH}" \
     LITELLM_NON_ROOT=true \
     PRISMA_BINARY_CACHE_DIR=/app/.cache/prisma-python/binaries \
     PRISMA_CLI_BINARY_TARGETS="debian-openssl-3.0.x" \
@@ -49,7 +41,8 @@ COPY enterprise/pyproject.toml enterprise/
 COPY litellm-proxy-extras/pyproject.toml litellm-proxy-extras/
 
 # Install third-party dependencies (cached unless pyproject.toml/uv.lock change)
-RUN uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    uv sync --frozen --no-install-project --no-install-workspace --no-default-groups --no-editable \
     --extra proxy \
     --extra proxy-runtime \
     --extra extra_proxy \
@@ -62,38 +55,12 @@ COPY . .
 # Set non-root flag for build time consistency
 ENV LITELLM_NON_ROOT=true
 
-# Build Admin UI once and stage the static output for the runtime image.
-# NOTE: .npmrc files (which may set ignore-scripts=true and min-release-age=3d)
-# are temporarily renamed during npm install/ci so they don't block lifecycle
-# scripts needed by the build. This is safe because npm ci installs from
-# package-lock.json with pinned versions + integrity hashes.
+# Stage the pre-built Admin UI from the checked-in Next.js static export.
+# _experimental/out/ is regenerated as part of the release runbook.
+# Restructure extensionless routes (foo.html -> foo/index.html) to match the layout
+# proxy_server.py expects, and drop a readiness marker.
 RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
-    ([ -f /app/.npmrc ] && mv /app/.npmrc /app/.npmrc.bak || true) && \
-    NVM_VERSION="v0.40.4" && \
-    NVM_CHECKSUM="4b7412c49960c7d31e8df72da90c1fb5b8cccb419ac99537b737028d497aba4f" && \
-    NODE_VERSION="v20.20.2" && \
-    NVM_SCRIPT="/tmp/install-nvm.sh" && \
-    curl -fsSL "https://raw.githubusercontent.com/nvm-sh/nvm/${NVM_VERSION}/install.sh" -o "$NVM_SCRIPT" && \
-    echo "${NVM_CHECKSUM}  ${NVM_SCRIPT}" | sha256sum -c - && \
-    bash "$NVM_SCRIPT" && \
-    export NVM_DIR="$HOME/.nvm" && \
-    . "$NVM_DIR/nvm.sh" && \
-    nvm install "${NODE_VERSION}" && \
-    nvm use "${NODE_VERSION}" && \
-    npm install -g npm@11.12.1 && \
-    npm install -g node-gyp@12.2.0 && \
-    ln -sf "$(npm root -g)/node-gyp" "$(npm root -g)/npm/node_modules/node-gyp" && \
-    npm cache clean --force && \
-    cd /app/ui/litellm-dashboard && \
-    if [ -f "/app/enterprise/enterprise_ui/enterprise_colors.json" ]; then \
-      cp /app/enterprise/enterprise_ui/enterprise_colors.json ./ui_colors.json; \
-    fi && \
-    ([ -f .npmrc ] && mv .npmrc .npmrc.bak || true) && \
-    npm ci --no-audit --no-fund && \
-    ([ -f .npmrc.bak ] && mv .npmrc.bak .npmrc || true) && \
-    ([ -f /app/.npmrc.bak ] && mv /app/.npmrc.bak /app/.npmrc || true) && \
-    npm run build && \
-    cp -r /app/ui/litellm-dashboard/out/* /var/lib/litellm/ui/ && \
+    cp -r /app/litellm/proxy/_experimental/out/. /var/lib/litellm/ui/ && \
     cp /app/litellm/proxy/logo.jpg /var/lib/litellm/assets/logo.jpg && \
     ( cd /var/lib/litellm/ui && \
       for html_file in *.html; do \
@@ -103,10 +70,10 @@ RUN mkdir -p /var/lib/litellm/ui /var/lib/litellm/assets && \
           mv "$html_file" "$folder_name/index.html"; \
         fi; \
       done && \
-      touch .litellm_ui_ready ) && \
-    cd /app/ui/litellm-dashboard && rm -rf ./out
+      touch .litellm_ui_ready )
 
-RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
+RUN --mount=type=cache,target=/app/.cache/uv,id=litellm-uv-cache \
+    if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
       uv sync --frozen --no-default-groups --no-editable \
         --extra proxy \
         --extra proxy-runtime \
@@ -123,10 +90,7 @@ RUN if [ "$PROXY_EXTRAS_SOURCE" = "published" ]; then \
         --python python3; \
     fi
 
-RUN mkdir -p /app/.cache/npm && \
-    prisma generate --schema=./schema.prisma && \
-    prisma --version && \
-    prisma migrate diff --from-empty --to-schema-datamodel ./schema.prisma --script > /dev/null 2>&1 || true
+RUN prisma generate --schema=./schema.prisma
 
 RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
     sed -i 's/\r$//' docker/prod_entrypoint.sh && chmod +x docker/prod_entrypoint.sh
@@ -137,33 +101,11 @@ WORKDIR /app
 USER root
 
 RUN for i in 1 2 3; do \
-    apk upgrade --no-cache && break || sleep 5; \
+      apk upgrade --no-cache && break || sleep 5; \
     done && \
     for i in 1 2 3; do \
-      apk add --no-cache python3 bash openssl tzdata nodejs npm supervisor libsndfile && break || sleep 5; \
-    done && \
-    apk upgrade --no-cache nodejs && \
-    npm install -g npm@11.12.1 tar@7.5.11 glob@11.1.0 @isaacs/brace-expansion@5.0.1 minimatch@10.2.4 diff@8.0.3 && \
-    GLOBAL="$(npm root -g)" && \
-    find "$GLOBAL/npm" -type d -name "tar" -path "*/node_modules/tar" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/tar" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "glob" -path "*/node_modules/glob" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/glob" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "brace-expansion" -path "*/node_modules/@isaacs/brace-expansion" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/@isaacs/brace-expansion" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "minimatch" -path "*/node_modules/minimatch" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/minimatch" "$d"; \
-    done && \
-    find "$GLOBAL/npm" -type d -name "diff" -path "*/node_modules/diff" | while read d; do \
-      rm -rf "$d" && cp -rL "$GLOBAL/diff" "$d"; \
-    done && \
-    find /usr/local/lib /usr/lib -path "*/node_modules/npm/package.json" -exec \
-      sed -i 's/"tar": "\^7\.5\.[0-9]*"/"tar": "^7.5.10"/g; s/"minimatch": "\^10\.[0-9.]*"/"minimatch": "^10.2.4"/g' {} + 2>/dev/null && \
-    npm cache clean --force && \
-    { apk del --no-cache npm 2>/dev/null || true; }
+      apk add --no-cache python3 bash openssl tzdata supervisor libsndfile nodejs && break || sleep 5; \
+    done
 
 COPY --from=builder /app /app
 COPY --from=builder /var/lib/litellm/ui /var/lib/litellm/ui
@@ -179,15 +121,10 @@ ENV PATH="/app/.venv/bin:${PATH}" \
     PRISMA_SKIP_POSTINSTALL_GENERATE=1 \
     PRISMA_HIDE_UPDATE_MESSAGE=1 \
     PRISMA_ENGINES_CHECKSUM_IGNORE_MISSING=1 \
-    NPM_CONFIG_CACHE=/app/.cache/npm \
-    NPM_CONFIG_PREFER_OFFLINE=true \
     PRISMA_OFFLINE_MODE=true
 
-RUN sed -i 's/\r$//' docker/entrypoint.sh && \
-    sed -i 's/\r$//' docker/prod_entrypoint.sh && \
-    chmod +x docker/entrypoint.sh docker/prod_entrypoint.sh && \
-    mkdir -p /nonexistent /.npm /var/lib/litellm/assets /var/lib/litellm/ui /tmp/.npm && \
-    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent /.npm /tmp/.npm && \
+RUN mkdir -p /nonexistent /var/lib/litellm/assets /var/lib/litellm/ui && \
+    chown -R nobody:nogroup /app /var/lib/litellm/ui /var/lib/litellm/assets /nonexistent && \
     PRISMA_PATH=$(python -c "import os, prisma; print(os.path.dirname(prisma.__file__))") && \
     chown -R nobody:nogroup "$PRISMA_PATH" && \
     LITELLM_PKG_MIGRATIONS_PATH="$(python -c 'import os, litellm_proxy_extras; print(os.path.dirname(litellm_proxy_extras.__file__))' 2>/dev/null || echo '')/migrations" && \

docs/my-website/docs/completion/prompt_caching.md

@@ -10,6 +10,7 @@ Supported Providers:
 - Vertex AI (`vertex_ai/`, `vertex_ai_beta/`)
 - Bedrock (`bedrock/`, `bedrock/invoke/`, `bedrock/converse`) ([All models bedrock supports prompt caching on](https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html))
 - Deepseek API (`deepseek/`)
+- xAI (`xai/`)
 
 For the supported providers, LiteLLM follows the OpenAI prompt caching usage object format:
 

docs/my-website/docs/completion/prompt_compression.md

@@ -8,6 +8,7 @@ The function keeps high-relevance and recent context, replaces low-relevance con
 
 ```python
 import litellm
+from litellm.types.utils import CallTypes
 
 messages = [
     {"role": "system", "content": "You are a coding assistant."},
@@ -19,6 +20,7 @@ messages = [
 compressed = litellm.compress(
     messages=messages,
     model="gpt-4o",
+    call_type=CallTypes.completion,
     compression_trigger=1000,
     compression_target=500,
 )
@@ -45,6 +47,7 @@ response = litellm.completion(
 
 - `messages` (`List[dict]`, required): input conversation messages
 - `model` (`str`, required): model name used for token counting
+- `call_type` (`CallTypes`, default `CallTypes.completion`): the LiteLLM call type whose message schema these messages follow. Supported values: `CallTypes.completion` / `CallTypes.acompletion` (OpenAI chat-completions shape) and `CallTypes.anthropic_messages` (Anthropic Messages shape)
 - `compression_trigger` (`int`, default `200000`): compress only if input token count exceeds this
 - `compression_target` (`Optional[int]`, default `70% of compression_trigger`): desired post-compression token budget
 - `embedding_model` (`Optional[str]`): if set, combines BM25 + embedding relevance scoring
@@ -70,6 +73,28 @@ args = json.loads(tool_call.function.arguments)
 full_content = compressed["cache"][args["key"]]
 ```
 
+## Server-side Callback Loop (`/v1/messages`)
+
+You can enable callback-based compression interception to make retrieval loops
+transparent for Anthropic Messages calls:
+
+```yaml
+litellm_settings:
+  callbacks: ["compression_interception"]
+  compression_interception_params:
+    enabled: true
+    compression_trigger: 10000
+    compression_target: 7000
+```
+
+With this enabled, LiteLLM runs the following server-side flow:
+
+1. Compresses inbound messages before the first provider call.
+2. Injects the `litellm_content_retrieve` tool.
+3. Detects retrieval `tool_use` blocks in the model response.
+4. Resolves retrieval keys from the compression cache.
+5. Reruns the model via agentic loop and returns the final answer.
+
 ## Performance
 
 Benchmarked on [SWE-bench Lite](https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_bm25_27K) (real GitHub issues with ~27k tokens of BM25-retrieved repo context per problem).

docs/my-website/docs/providers/scaleway.md

@@ -60,3 +60,44 @@ curl http://localhost:4000/chat/completions \
 ## Supported features
 
 Scaleway provider supports all features in [Generative APIs reference documentation ↗](https://www.scaleway.com/en/developers/api/generative-apis/), such as streaming, structured outputs and tool calling.
+
+## Audio transcription
+
+Scaleway's `/audio/transcriptions` endpoint is OpenAI-compatible and works with Whisper models.
+
+### Python SDK
+
+```python
+import os
+from litellm import transcription
+
+os.environ["SCW_SECRET_KEY"] = "your-scaleway-secret-key"
+
+with open("speech.mp3", "rb") as audio_file:
+    response = transcription(
+        model="scaleway/whisper-large-v3",
+        file=audio_file,
+    )
+print(response.text)
+```
+
+### Proxy config
+
+```yaml
+model_list:
+  - model_name: scaleway-whisper
+    litellm_params:
+      model: scaleway/whisper-large-v3
+      api_key: "os.environ/SCW_SECRET_KEY"
+```
+
+### Proxy request
+
+```bash
+curl http://localhost:4000/v1/audio/transcriptions \
+  -H "Authorization: Bearer YOUR_LITELLM_MASTER_KEY" \
+  -F model="scaleway-whisper" \
+  -F file="@speech.mp3"
+```
+
+Supported optional params: `language`, `prompt`, `response_format`, `temperature`, `timestamp_granularities`.

docs/my-website/docs/proxy/agentic_loop_hook.md

@@ -0,0 +1,95 @@
+# Agentic Loop Hook
+
+Build a `CustomLogger` callback that intercepts a model response, fulfills tool calls server-side, and reruns the model — transparently to the caller.
+
+:::info Supported call types
+- `async` only (sync calls do not trigger the hook)
+- Non-streaming only (streaming responses cannot be inspected for tool calls)
+- Works on both `/v1/messages` and `/v1/chat/completions`
+:::
+
+## Implement the callback
+
+Override two methods on `CustomLogger`:
+
+```python
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.custom_logger import AgenticLoopPlan, AgenticLoopRequestPatch
+
+MY_TOOL = "my_tool"
+
+class MyToolCallback(CustomLogger):
+
+    async def async_should_run_agentic_loop(
+        self, response, model, messages, tools, stream, custom_llm_provider, kwargs
+    ):
+        # Return (True, context_dict) if there are tool calls to handle
+        content = getattr(response, "content", None) or []
+        calls = [b for b in content if isinstance(b, dict)
+                 and b.get("type") == "tool_use" and b.get("name") == MY_TOOL]
+        if not calls:
+            return False, {}
+        return True, {"tool_calls": calls}
+
+    async def async_build_agentic_loop_plan(
+        self, tools, model, messages, response,
+        anthropic_messages_provider_config,
+        anthropic_messages_optional_request_params,
+        logging_obj, stream, kwargs,
+    ):
+        calls = tools["tool_calls"]
+        results = [f"result for {c['input']}" for c in calls]  # your logic here
+
+        follow_up = messages + [
+            {"role": "assistant", "content": [
+                {"type": "tool_use", "id": c["id"], "name": c["name"], "input": c["input"]}
+                for c in calls
+            ]},
+            {"role": "user", "content": [
+                {"type": "tool_result", "tool_use_id": c["id"], "content": results[i]}
+                for i, c in enumerate(calls)
+            ]},
+        ]
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=AgenticLoopRequestPatch(messages=follow_up),
+        )
+```
+
+For `/v1/chat/completions`, override `async_build_chat_completion_agentic_loop_plan` instead — same idea, `optional_params` replaces `anthropic_messages_optional_request_params`.
+
+## Register it
+
+```python
+import litellm
+litellm.callbacks = [MyToolCallback()]
+```
+
+Or in `config.yaml`:
+
+```yaml
+litellm_settings:
+  callbacks: ["my_module.MyToolCallback"]
+```
+
+## `AgenticLoopPlan` fields
+
+| Field | Effect |
+|---|---|
+| `run_agentic_loop=True` + `request_patch` | Reruns the model with the patched request |
+| `response_override` | Returns this value directly to the caller (no rerun) |
+| `terminate=True` | Stops the loop, returns the current response |
+| `run_agentic_loop=False` (default) | Skips; next callback is checked |
+
+`AgenticLoopRequestPatch` accepts: `model`, `messages`, `tools`, `max_tokens`, `optional_params`, `kwargs`.
+
+## Loop safety
+
+- Default max reruns: `3` — override per-request with `kwargs["max_agentic_loops"]`
+- Identical tool-call fingerprints abort the loop automatically
+- Current depth is in `kwargs["_agentic_loop_depth"]`
+
+## Examples in this repo
+
+- `litellm/integrations/compression_interception/handler.py`
+- `litellm/integrations/websearch_interception/handler.py`

docs/my-website/docs/tutorials/prompt_caching.md

@@ -8,6 +8,22 @@ Reduce costs by up to 90% by using LiteLLM to auto-inject prompt caching checkpo
 
 <Image img={require('../../img/auto_prompt_caching.png')}  style={{ width: '800px', height: 'auto' }} />
 
+Supported Providers (`cache_control` marker):
+- Anthropic API (`anthropic/`)
+- AWS Bedrock - Claude (`bedrock/`)
+- Vertex AI - Claude and Gemini (`vertex_ai/`)
+- Google AI Studio - Gemini (`gemini/`)
+- Azure AI - Claude (`azure_ai/`)
+- OpenRouter - Claude, Gemini, MiniMax, GLM, z-ai routes (`openrouter/`)
+- Databricks - Claude (`databricks/`)
+- DashScope / Qwen (`dashscope/`)
+- MiniMax (`minimax/`)
+- Z.ai / GLM (`zai/`)
+
+Provider Managed (automatic, no marker needed):
+- OpenAI (`openai/`)
+- DeepSeek (`deepseek/`)
+- xAI (`xai/`)
 
 ## How it works
 

docs/my-website/sidebars.js

@@ -536,6 +536,7 @@ const sidebars = {
           description: "Modify requests, responses, and more",
           items: [
             "proxy/call_hooks",
+            "proxy/agentic_loop_hook",
             "proxy/rules",
           ]
         },

litellm-proxy-extras/litellm_proxy_extras/utils.py

@@ -30,6 +30,26 @@ def _get_prisma_env() -> dict:
     return prisma_env
 
 
+_MIGRATION_TS_RE = re.compile(r"^(\d{14})_")
+
+
+def _migration_timestamp(name: str) -> int:
+    """Extract the leading `YYYYMMDDHHMMSS` timestamp from a migration name.
+
+    Returns 0 if the name doesn't match the Prisma pattern — unexpected-format
+    entries sort as "oldest" and are treated as historical.
+    """
+    m = _MIGRATION_TS_RE.match(name)
+    return int(m.group(1)) if m else 0
+
+
+def _max_migration_timestamp(names) -> int:
+    """Max timestamp in a set/list of migration names (0 if empty)."""
+    if not names:
+        return 0
+    return max(_migration_timestamp(n) for n in names)
+
+
 def _get_prisma_command() -> str:
     """Get the Prisma command to use, bypassing Python wrapper in offline mode."""
     if str_to_bool(os.getenv("PRISMA_OFFLINE_MODE")):
@@ -383,18 +403,301 @@ class ProxyExtrasDBManager:
                     )
 
     @staticmethod
-    def setup_database(use_migrate: bool = False) -> bool:
+    def _strip_prisma_query_params(url: str) -> str:
+        """Remove Prisma-specific query params (connection_limit, pool_timeout,
+        schema, etc.) from DATABASE_URL so psycopg can parse it."""
+        from urllib.parse import urlparse, urlunparse, parse_qsl, urlencode
+
+        parsed = urlparse(url)
+        if not parsed.query:
+            return url
+        libpq_params = {
+            "sslmode",
+            "sslcert",
+            "sslkey",
+            "sslrootcert",
+            "sslpassword",
+            "application_name",
+            "connect_timeout",
+            "client_encoding",
+            "options",
+            "service",
+            "gssencmode",
+            "krbsrvname",
+            "target_session_attrs",
+        }
+        kept = [(k, v) for k, v in parse_qsl(parsed.query) if k in libpq_params]
+        return urlunparse(parsed._replace(query=urlencode(kept)))
+
+    @staticmethod
+    def _warn_if_db_ahead_of_head(migrations_dir: str) -> None:
+        """
+        Log a warning if _prisma_migrations contains applied migrations with
+        timestamps newer than every migration this build ships.
+
+        This is informational only for the v2 resolver — it tells the operator
+        the DB was likely migrated by a newer deployment, which is usually a
+        signal that this (older) version shouldn't run against it. We do NOT
+        block startup: many users have weird _prisma_migrations state from
+        prior thrashing bugs, and blocking them would be a breaking change.
+
+        Safe no-op if psycopg isn't installed or DB isn't reachable.
+        """
+        database_url = os.getenv("DATABASE_URL")
+        if not database_url:
+            return
+
+        try:
+            import psycopg
+        except ImportError:
+            return
+
+        cleaned_url = ProxyExtrasDBManager._strip_prisma_query_params(database_url)
+        known = set(ProxyExtrasDBManager._get_migration_names(migrations_dir))
+
+        try:
+            # autocommit=True keeps the SELECT outside a transaction. Without
+            # it, psycopg3's `with conn` calls COMMIT on clean exit — which
+            # fails after `UndefinedTable` (fresh DB) leaves the transaction
+            # in an aborted state.
+            with psycopg.connect(
+                cleaned_url, connect_timeout=10, autocommit=True
+            ) as conn:
+                try:
+                    rows = conn.execute(
+                        "SELECT migration_name FROM _prisma_migrations "
+                        "WHERE finished_at IS NOT NULL AND rolled_back_at IS NULL"
+                    ).fetchall()
+                except psycopg.errors.UndefinedTable:
+                    return
+        except (psycopg.OperationalError, psycopg.DatabaseError):
+            # Swallow connection failures AND any other DB-layer error
+            # (e.g. InsufficientPrivilege if the runtime user lacks SELECT
+            # on _prisma_migrations). This is an informational check —
+            # never block startup on it.
+            return
+
+        applied = {r[0] for r in rows}
+        unknown = applied - known
+        if not unknown:
+            return
+
+        head_newest_ts = _max_migration_timestamp(known)
+        hostile = {
+            name for name in unknown if _migration_timestamp(name) > head_newest_ts
+        }
+        if not hostile:
+            return
+
+        sorted_hostile = sorted(hostile)
+        logger.warning(
+            "Database has %d migration(s) applied that are NEWER than any "
+            "migration this LiteLLM version ships. This usually means the "
+            "database was migrated by a newer LiteLLM deployment. Some API "
+            "endpoints may fail because this proxy's Prisma client does not "
+            "know about those schema changes. Consider upgrading this "
+            "deployment. Unknown: %s",
+            len(hostile),
+            ", ".join(sorted_hostile[:5]) + (" ..." if len(sorted_hostile) > 5 else ""),
+        )
+
+    @staticmethod
+    def _setup_database_v2(use_migrate: bool) -> bool:
+        """
+        v2 migration resolver (opt-in via --use_v2_migration_resolver).
+
+        Runs `prisma migrate deploy` and handles standard recovery paths
+        (P3005 baseline, P3009/P3018 idempotent errors). Critically, it does
+        NOT call `_resolve_all_migrations` — the diff-and-force recovery that
+        caused schema thrashing when two LiteLLM versions contended for the
+        same DB during rolling deploys.
+
+        Ahead-of-HEAD state (DB has migrations newer than this build ships)
+        is logged as a warning, not a fatal error — users whose DBs got into
+        weird shapes from the old thrashing should still be able to start.
+        """
+        schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
+        migrations_dir = ProxyExtrasDBManager._get_prisma_dir()
+
+        if not use_migrate:
+            # Preserve `prisma db push` path unchanged.
+            original_dir = os.getcwd()
+            os.chdir(migrations_dir)
+            try:
+                subprocess.run(
+                    [_get_prisma_command(), "db", "push", "--accept-data-loss"],
+                    timeout=60,
+                    check=True,
+                    env=_get_prisma_env(),
+                )
+                return True
+            except (
+                subprocess.CalledProcessError,
+                subprocess.TimeoutExpired,
+            ) as e:
+                # Re-raise as RuntimeError so proxy_cli.py's
+                # `except RuntimeError` catches it and exits cleanly.
+                raise RuntimeError(f"prisma db push failed.\n\nDetail: {e}") from e
+            finally:
+                os.chdir(original_dir)
+
+        # Informational — never blocks.
+        ProxyExtrasDBManager._warn_if_db_ahead_of_head(migrations_dir)
+
+        original_dir = os.getcwd()
+        os.chdir(migrations_dir)
+        try:
+            for attempt in range(4):
+                try:
+                    result = subprocess.run(
+                        [_get_prisma_command(), "migrate", "deploy"],
+                        timeout=60,
+                        check=True,
+                        capture_output=True,
+                        text=True,
+                        env=_get_prisma_env(),
+                    )
+                    logger.info(f"prisma migrate deploy stdout: {result.stdout}")
+                    return True
+
+                except subprocess.TimeoutExpired:
+                    logger.info(
+                        f"prisma migrate deploy attempt {attempt + 1} timed out, retrying"
+                    )
+                    time.sleep(random.randrange(5, 15))
+                    continue
+
+                except subprocess.CalledProcessError as e:
+                    stderr = e.stderr or ""
+
+                    if "P3005" in stderr and "database schema is not empty" in stderr:
+                        logger.info(
+                            "Schema exists but no migrations ledger — creating baseline"
+                        )
+                        ProxyExtrasDBManager._create_baseline_migration(schema_path)
+                        continue
+
+                    if "P3009" in stderr:
+                        migration_match = re.search(r"`(\d+_\S+?)`", stderr)
+                        if (
+                            migration_match
+                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
+                        ):
+                            name = migration_match.group(1)
+                            logger.info(
+                                f"Migration {name} failed idempotently — marking applied and retrying"
+                            )
+                            try:
+                                ProxyExtrasDBManager._roll_back_migration(name)
+                            except (
+                                subprocess.CalledProcessError,
+                                subprocess.TimeoutExpired,
+                            ):
+                                pass  # may already be rolled-back
+                            try:
+                                ProxyExtrasDBManager._resolve_specific_migration(name)
+                            except (
+                                subprocess.CalledProcessError,
+                                subprocess.TimeoutExpired,
+                            ) as resolve_err:
+                                # We're already inside the outer
+                                # `except CalledProcessError` handler —
+                                # re-raising CalledProcessError from here
+                                # would escape as itself, bypassing
+                                # proxy_cli.py's `except RuntimeError`.
+                                raise RuntimeError(
+                                    f"Failed to mark migration {name} as applied "
+                                    f"after idempotent recovery. Manual "
+                                    f"intervention may be required.\n\n"
+                                    f"Detail: {resolve_err}"
+                                ) from resolve_err
+                            continue
+                        raise RuntimeError(
+                            "Database migration failed and cannot be auto-recovered. "
+                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+                        ) from e
+
+                    if "P3018" in stderr:
+                        if ProxyExtrasDBManager._is_permission_error(stderr):
+                            raise RuntimeError(
+                                "Database migration failed due to insufficient "
+                                "permissions. Please grant the required privileges "
+                                f"and retry.\n\nPrisma error:\n{stderr}"
+                            ) from e
+
+                        migration_match = re.search(
+                            r"Migration name: (\d+_\S+)", stderr
+                        )
+                        if (
+                            migration_match
+                            and ProxyExtrasDBManager._is_idempotent_error(stderr)
+                        ):
+                            name = migration_match.group(1)
+                            logger.info(
+                                f"Migration {name} SQL hit idempotent error — marking applied and retrying"
+                            )
+                            try:
+                                ProxyExtrasDBManager._roll_back_migration(name)
+                            except (
+                                subprocess.CalledProcessError,
+                                subprocess.TimeoutExpired,
+                            ):
+                                pass  # may already be rolled-back
+                            try:
+                                ProxyExtrasDBManager._resolve_specific_migration(name)
+                            except (
+                                subprocess.CalledProcessError,
+                                subprocess.TimeoutExpired,
+                            ) as resolve_err:
+                                raise RuntimeError(
+                                    f"Failed to mark migration {name} as applied "
+                                    f"after idempotent recovery. Manual "
+                                    f"intervention may be required.\n\n"
+                                    f"Detail: {resolve_err}"
+                                ) from resolve_err
+                            continue
+
+                        raise RuntimeError(
+                            "Database migration failed and cannot be auto-recovered. "
+                            f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+                        ) from e
+
+                    raise RuntimeError(
+                        "Database migration failed and cannot be auto-recovered. "
+                        f"Manual intervention required.\n\nPrisma error:\n{stderr}"
+                    ) from e
+
+            raise RuntimeError(
+                "Database migration failed after 4 attempts (retry loop "
+                "exhausted by timeouts or repeated idempotent-recovery "
+                "continues). Check database connectivity, load, and "
+                "_prisma_migrations ledger state."
+            )
+        finally:
+            os.chdir(original_dir)
+
+    @staticmethod
+    def setup_database(
+        use_migrate: bool = False, use_v2_resolver: bool = False
+    ) -> bool:
         """
         Set up the database using either prisma migrate or prisma db push
         Uses migrations from litellm-proxy-extras package
 
         Args:
-            schema_path (str): Path to the Prisma schema file
-            use_migrate (bool): Whether to use prisma migrate instead of db push
+            use_migrate: Whether to use prisma migrate instead of db push
+            use_v2_resolver: Opt into the v2 migration resolver (safer during
+                rolling deploys; does not run the diff-and-force recovery
+                that causes schema thrashing). Defaults to False for
+                backwards compatibility.
 
         Returns:
             bool: True if setup was successful, False otherwise
         """
+        if use_v2_resolver:
+            logger.info("Using v2 migration resolver (--use_v2_migration_resolver)")
+            return ProxyExtrasDBManager._setup_database_v2(use_migrate=use_migrate)
+
         schema_path = ProxyExtrasDBManager._get_prisma_dir() + "/schema.prisma"
         for attempt in range(4):
             original_dir = os.getcwd()

litellm-proxy-extras/migration_runbook.md

@@ -2,6 +2,8 @@
 
 This is a runbook for creating and running database migrations for the LiteLLM proxy. For use for litellm engineers only.
 
+> **AI AGENTS / ASSISTANTS:** If the script refuses with either a "STALE BRANCH" or "DESTRUCTIVE MIGRATION DETECTED" error, **do NOT** bypass it on your own (no `git rebase`, no `--skip-freshness-check`, no `--allow-destructive`). Surface the error to the human operator and wait for their explicit confirmation. See the [Branch freshness](#branch-freshness-check) and [Destructive migrations](#destructive-migrations-drop-column--drop-table) sections below.
+
 ## Step 0: Sync All `schema.prisma` Files
 
 Before doing anything else, make sure all `schema.prisma` files in the repo are in sync. There are multiple copies that must match:
@@ -46,10 +48,50 @@ uv run --with testing.postgresql python ci_cd/run_migration.py "your_migration_n
 
 ## What It Does
 
-1. Creates temp PostgreSQL DB
-2. Applies existing migrations
-3. Compares with `schema.prisma`
-4. Generates new migration if changes found
+1. **Verifies the current branch is up to date with `origin/litellm_internal_staging`** (see [Branch freshness](#branch-freshness-check))
+2. Creates temp PostgreSQL DB
+3. Applies existing migrations
+4. Compares with `schema.prisma`
+5. Generates new migration if changes found
+6. **Refuses to write if the generated SQL contains `DROP COLUMN`, `DROP TABLE`, or `DROP INDEX`**, unless `--allow-destructive` is passed
+
+## Branch Freshness Check
+
+Before generating anything, `run_migration.py` runs `git fetch origin <base>` and refuses to proceed if `HEAD` is behind `origin/<base>`. Default base is `litellm_internal_staging` (the branch PRs target). A previous incident saw a stale branch silently drop production columns; freshness is the first-line defense.
+
+Flags:
+
+- `--base-branch <name>` — check against a different base (e.g. `main`). Default is `litellm_internal_staging`.
+- `--skip-freshness-check` — bypass entirely. Only for intentional migrations against an older base.
+
+When the guard fires:
+
+1. Update your branch:
+
+   ```bash
+   git fetch origin && git rebase origin/litellm_internal_staging
+   # or git merge origin/litellm_internal_staging — whichever matches your workflow
+   ```
+2. Re-run `run_migration.py`.
+
+> **AI AGENTS / ASSISTANTS:** Do **not** auto-rebase or auto-pass `--skip-freshness-check`. A bad conflict resolution during rebase can itself drop the human's in-progress schema edits. Surface the error and wait for explicit confirmation.
+
+## Destructive Migrations (DROP COLUMN / DROP TABLE / DROP INDEX)
+
+If the generated diff contains `DROP COLUMN`, `DROP TABLE`, or `DROP INDEX`, `run_migration.py` exits non-zero and refuses to write the migration file. A previous incident saw newly-added columns silently dropped by a stale branch and merged to main — this guard exists to prevent a repeat.
+
+When the guard fires:
+
+1. Run `git fetch origin && git status` — confirm your branch is up to date with the base branch.
+2. Re-check all `schema.prisma` files are in sync (Step 0).
+3. Review EACH `DROP` statement printed in the error — is it actually intended?
+4. Only if the drops are genuinely intentional, re-run with the flag:
+
+   ```bash
+   uv run --with testing.postgresql python ci_cd/run_migration.py "your_migration_name" --allow-destructive
+   ```
+
+> **AI AGENTS / ASSISTANTS:** Do **not** automatically re-run the command with `--allow-destructive`. If the guard fires while you are driving the runbook for a human, stop, show them the error, and wait for their explicit confirmation before passing the flag. Auto-passing `--allow-destructive` is the exact failure mode this guard exists to prevent.
 
 ## Common Fixes
 

litellm-proxy-extras/tests/test_setup_database_fail_fast.py

@@ -0,0 +1,242 @@
+"""Regression tests for ProxyExtrasDBManager v2 migration resolver.
+
+The v2 resolver is opt-in via `--use_v2_migration_resolver` / the
+`use_v2_resolver=True` kwarg. These tests exercise the v2 path; the v1
+(default) behavior is unchanged from pre-fix.
+"""
+
+import subprocess
+from unittest.mock import patch
+
+import pytest
+
+from litellm_proxy_extras.utils import (
+    ProxyExtrasDBManager,
+    _max_migration_timestamp,
+    _migration_timestamp,
+)
+
+
+def _fake_migrate_deploy_failure(returncode: int, stderr: str):
+    def _run(*args, **kwargs):
+        raise subprocess.CalledProcessError(
+            returncode=returncode,
+            cmd=args[0],
+            stderr=stderr,
+            output="",
+        )
+
+    return _run
+
+
+def test_v2_p3018_permission_error_raises_runtime_error(monkeypatch, tmp_path):
+    """v2: a permission failure during migrate deploy raises RuntimeError."""
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
+    )
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    stderr = (
+        "Error: P3018\nMigration name: 20250326162113_baseline\n"
+        "Database error code: 42501\npermission denied for schema public"
+    )
+    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+        with pytest.raises(RuntimeError, match="permission"):
+            ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+
+
+def test_v2_non_idempotent_p3009_raises_runtime_error(monkeypatch, tmp_path):
+    """v2: a non-idempotent migration failure raises (no silent recovery)."""
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
+    )
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    stderr = (
+        "Error: P3009\nMigration `20260101000000_genuinely_broken` failed\n"
+        'Reason: syntax error at or near "BRKN" LINE 42'
+    )
+    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+        with pytest.raises(RuntimeError, match="cannot be auto-recovered"):
+            ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+
+
+def test_strip_prisma_query_params_removes_connection_limit():
+    """DATABASE_URLs with Prisma-specific params should be parseable by psycopg."""
+    url = "postgresql://u:p@h:5432/db?connection_limit=100&pool_timeout=60&sslmode=require"
+    stripped = ProxyExtrasDBManager._strip_prisma_query_params(url)
+    assert "connection_limit" not in stripped
+    assert "pool_timeout" not in stripped
+    assert "sslmode=require" in stripped
+
+
+def test_strip_prisma_query_params_passthrough_no_query():
+    """URLs without query strings are returned unchanged."""
+    url = "postgresql://u:p@h:5432/db"
+    assert ProxyExtrasDBManager._strip_prisma_query_params(url) == url
+
+
+def test_migration_timestamp_extracts_leading_digits():
+    assert _migration_timestamp("20260101000000_add_foo") == 20260101000000
+    assert _migration_timestamp("20250326162113_baseline") == 20250326162113
+
+
+def test_migration_timestamp_returns_zero_on_malformed():
+    assert _migration_timestamp("0_init") == 0
+    assert _migration_timestamp("not_a_migration") == 0
+
+
+def test_max_migration_timestamp():
+    names = {"20250326000000_a", "20260415000000_b", "20251115000000_c"}
+    assert _max_migration_timestamp(names) == 20260415000000
+
+
+def test_max_migration_timestamp_empty_set():
+    assert _max_migration_timestamp(set()) == 0
+
+
+def test_v1_default_still_calls_resolve_all_migrations(monkeypatch, tmp_path):
+    """v1 (default) continues to call _resolve_all_migrations on the happy path.
+
+    This is the existing buggy behavior — we're not fixing it in v1, only
+    offering v2 as opt-in. This test pins the default so that a future
+    inadvertent default flip is caught.
+    """
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    # Stub `prisma migrate deploy` to claim success with pending migrations
+    # applied, which is the code path that triggers the legacy post-migration
+    # sanity check (a call to _resolve_all_migrations).
+    class FakeResult:
+        stdout = "Applied migration.\n"
+        stderr = ""
+
+    def fake_run(cmd, *args, **kwargs):
+        return FakeResult()
+
+    resolve_called = {"n": 0}
+
+    def fake_resolve(*args, **kwargs):
+        resolve_called["n"] += 1
+
+    monkeypatch.setattr("subprocess.run", fake_run)
+    monkeypatch.setattr(ProxyExtrasDBManager, "_resolve_all_migrations", fake_resolve)
+
+    ok = ProxyExtrasDBManager.setup_database(use_migrate=True)  # v2 flag NOT set
+    assert ok is True
+    assert resolve_called["n"] == 1, "v1 default should still invoke the legacy path"
+
+
+def test_v2_db_push_wraps_subprocess_error_as_runtime_error(monkeypatch, tmp_path):
+    """v2: a failing `prisma db push` must raise RuntimeError, not leak
+    CalledProcessError past proxy_cli.py's `except RuntimeError`."""
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    stderr = "db push error"
+    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+        with pytest.raises(RuntimeError, match="prisma db push failed"):
+            ProxyExtrasDBManager.setup_database(use_migrate=False, use_v2_resolver=True)
+
+
+def test_v2_warn_ahead_of_head_swallows_db_errors(monkeypatch, tmp_path):
+    """_warn_if_db_ahead_of_head must never raise — it's informational.
+
+    Non-connection DB errors (e.g. InsufficientPrivilege from a user
+    without SELECT on _prisma_migrations) must be caught, not propagated.
+    """
+    import psycopg
+
+    monkeypatch.setenv("DATABASE_URL", "postgresql://u:p@localhost:9/x")
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    class _FakeConn:
+        def __enter__(self):
+            return self
+
+        def __exit__(self, *a):
+            return False
+
+        def execute(self, *a, **kw):
+            # Simulate an InsufficientPrivilege (subclass of DatabaseError).
+            raise psycopg.errors.InsufficientPrivilege("permission denied")
+
+    def _fake_connect(*a, **kw):
+        return _FakeConn()
+
+    monkeypatch.setattr("psycopg.connect", _fake_connect)
+
+    # Must not raise.
+    ProxyExtrasDBManager._warn_if_db_ahead_of_head(str(tmp_path))
+
+
+def test_v2_resolve_specific_migration_failure_raises_runtime_error(
+    monkeypatch, tmp_path
+):
+    """If marking a migration as applied fails inside P3009 idempotent
+    recovery, the subprocess error must be re-raised as RuntimeError so
+    proxy_cli.py catches it cleanly (instead of leaking CalledProcessError)."""
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
+    )
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_roll_back_migration", lambda *a, **kw: None
+    )
+
+    # First call: migrate deploy -> P3009 idempotent error.
+    # Recovery path tries _resolve_specific_migration; that also raises.
+    def _failing_resolve(*a, **kw):
+        raise subprocess.CalledProcessError(
+            returncode=1,
+            cmd="prisma migrate resolve --applied",
+            stderr="resolve failed",
+            output="",
+        )
+
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_resolve_specific_migration", _failing_resolve
+    )
+
+    stderr = (
+        "Error: P3009\nMigration `20260101000000_some_migration` failed\n"
+        "relation already exists"
+    )
+    with patch("subprocess.run", side_effect=_fake_migrate_deploy_failure(1, stderr)):
+        with pytest.raises(
+            RuntimeError, match="Failed to mark migration .* as applied"
+        ):
+            ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+
+
+def test_v2_does_not_call_resolve_all_migrations(monkeypatch, tmp_path):
+    """v2 must never call _resolve_all_migrations — that's the bug it fixes."""
+    monkeypatch.setattr(
+        ProxyExtrasDBManager, "_warn_if_db_ahead_of_head", lambda _: None
+    )
+    monkeypatch.setattr(ProxyExtrasDBManager, "_get_prisma_dir", lambda: str(tmp_path))
+    (tmp_path / "schema.prisma").write_text("// stub")
+
+    class FakeResult:
+        stdout = "Applied migration.\n"
+        stderr = ""
+
+    monkeypatch.setattr("subprocess.run", lambda *a, **kw: FakeResult())
+
+    resolve_called = {"n": 0}
+    monkeypatch.setattr(
+        ProxyExtrasDBManager,
+        "_resolve_all_migrations",
+        lambda *a, **kw: resolve_called.__setitem__("n", resolve_called["n"] + 1),
+    )
+
+    ok = ProxyExtrasDBManager.setup_database(use_migrate=True, use_v2_resolver=True)
+    assert ok is True
+    assert resolve_called["n"] == 0, "v2 must not invoke the diff-and-force recovery"

litellm/__init__.py

@@ -148,6 +148,7 @@ _custom_logger_compatible_callbacks_literal = Literal[
     "vantage",
     "posthog",
     "levo",
+    "compression_interception",
 ]
 cold_storage_custom_logger: Optional[_custom_logger_compatible_callbacks_literal] = None
 logged_real_time_event_types: Optional[Union[List[str], Literal["*"]]] = None
@@ -1501,6 +1502,9 @@ if TYPE_CHECKING:
     from .llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
         AmazonAnthropicClaudeMessagesConfig as AmazonAnthropicClaudeMessagesConfig,
     )
+    from .llms.bedrock.messages.mantle_transformation import (
+        AmazonMantleMessagesConfig as AmazonMantleMessagesConfig,
+    )
     from .llms.together_ai.chat import TogetherAIConfig as TogetherAIConfig
     from .llms.nlp_cloud.chat.handler import NLPCloudConfig as NLPCloudConfig
     from .llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (

litellm/_lazy_imports_registry.py

@@ -171,6 +171,7 @@ LLM_CONFIG_NAMES = (
     "CohereChatConfig",
     "AnthropicMessagesConfig",
     "AmazonAnthropicClaudeMessagesConfig",
+    "AmazonMantleMessagesConfig",
     "TogetherAIConfig",
     "NLPCloudConfig",
     "VertexGeminiConfig",
@@ -715,6 +716,10 @@ _LLM_CONFIGS_IMPORT_MAP = {
         ".llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation",
         "AmazonAnthropicClaudeMessagesConfig",
     ),
+    "AmazonMantleMessagesConfig": (
+        ".llms.bedrock.messages.mantle_transformation",
+        "AmazonMantleMessagesConfig",
+    ),
     "TogetherAIConfig": (".llms.together_ai.chat", "TogetherAIConfig"),
     "NLPCloudConfig": (".llms.nlp_cloud.chat.handler", "NLPCloudConfig"),
     "VertexGeminiConfig": (

litellm/compression/compress.py

@@ -1,9 +1,9 @@
 """
-Main compress() function — orchestrates BM25/embedding scoring, message stubbing,
-and retrieval tool injection.
+Main compress() function — normalizes input messages, orchestrates BM25/embedding
+scoring, message stubbing, and retrieval tool injection.
 """
 
-from typing import Any, Dict, List, Optional, Set, Union, cast
+from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
 
 from litellm.caching.dual_cache import DualCache
 from litellm.compression.message_stubbing import (
@@ -15,27 +15,196 @@ from litellm.compression.retrieval_tool import build_retrieval_tool
 from litellm.compression.scoring.bm25 import bm25_score_messages
 from litellm.litellm_core_utils.token_counter import token_counter
 from litellm.types.compression import CompressedResult
-from litellm.types.utils import AllMessageValues, Message
+from litellm.types.utils import CallTypes
+
+# CallTypes that produce Anthropic-shaped messages (structured content blocks).
+# Everything else is treated as OpenAI chat-completions shape.
+_ANTHROPIC_CALL_TYPES = frozenset({CallTypes.anthropic_messages.value})
+# CallTypes that are valid targets for compression.  Compression operates on
+# message-shaped inputs, so we only accept call types whose payload is a list
+# of role/content messages.
+_SUPPORTED_CALL_TYPES = frozenset(
+    {
+        CallTypes.completion.value,
+        CallTypes.acompletion.value,
+        CallTypes.anthropic_messages.value,
+    }
+)
+
+
+def _normalize_call_type(call_type: Union[CallTypes, str]) -> str:
+    """Return the string value for a ``CallTypes`` enum or a raw string."""
+    if isinstance(call_type, CallTypes):
+        return call_type.value
+    return call_type
+
+
+def _is_anthropic_call_type(call_type: str) -> bool:
+    return call_type in _ANTHROPIC_CALL_TYPES
+
+
+def _build_retrieval_tools(keys: List[str], call_type: str) -> List[dict]:
+    """
+    Build retrieval tool definitions in the target request schema.
+
+    - Chat-completions call types: keep OpenAI function-tool schema.
+    - Anthropic messages call type: remap to Anthropic's custom tool schema.
+    """
+    if not keys:
+        return []
+
+    openai_tools = [build_retrieval_tool(keys)]
+    if not _is_anthropic_call_type(call_type):
+        return openai_tools
+
+    # Lazy import to avoid introducing provider transformation imports during
+    # module import for non-Anthropic call paths.
+    from litellm.llms.anthropic.chat.transformation import AnthropicConfig
+
+    anthropic_tools, _mcp_servers = AnthropicConfig()._map_tools(openai_tools)
+    return cast(List[dict], anthropic_tools)
+
+
+def _content_to_text(content: Any) -> str:
+    """
+    Convert OpenAI/Anthropic message content blocks to plain text.
+
+    Text extraction policy:
+    - Include text-bearing fields only (`text` blocks + string values).
+    - For `tool_result`, expand into nested `content` items.
+    - Ignore non-textual blocks (images/documents/tool metadata/thinking metadata).
+
+    Implemented iteratively (stack-based) to avoid unbounded recursion.
+    """
+    parts: List[str] = []
+    stack: List[Any] = [content]
+    while stack:
+        item = stack.pop()
+        if isinstance(item, str):
+            parts.append(item)
+        elif isinstance(item, list):
+            # Push list items in reverse order so they are processed left-to-right.
+            for element in reversed(item):
+                stack.append(element)
+        elif isinstance(item, dict):
+            item_type = item.get("type")
+            if item_type == "text":
+                parts.append(str(item.get("text", "")))
+            elif item_type == "tool_result":
+                stack.append(item.get("content", ""))
+    return " ".join(parts)
+
+
+def _normalize_messages_for_compression(
+    messages: List[dict],
+    call_type: str,
+) -> Tuple[List[dict], List[dict]]:
+    """
+    Normalize each original message to a text-surrogate content for scoring.
+
+    Returns:
+        (normalized_messages, original_messages_copy)
+    """
+    if call_type not in _SUPPORTED_CALL_TYPES:
+        raise ValueError(
+            f"Unsupported call_type={call_type!r} for compression. "
+            f"Expected one of: {sorted(_SUPPORTED_CALL_TYPES)}."
+        )
+
+    original_messages: List[Dict[str, Any]] = [dict(m) for m in messages]
+
+    normalized_messages: List[dict] = []
+    for msg in original_messages:
+        normalized_messages.append(
+            {
+                **msg,
+                "content": _content_to_text(msg.get("content", "")),
+            }
+        )
+    return normalized_messages, original_messages
 
 
 def _extract_last_user_message(messages: List[dict]) -> str:
     """Return the text content of the last user message."""
     for msg in reversed(messages):
         if msg.get("role") == "user":
-            content = msg.get("content", "")
-            if isinstance(content, str):
-                return content
-            if isinstance(content, list):
-                parts = []
-                for part in content:
-                    if isinstance(part, dict) and part.get("type") == "text":
-                        parts.append(part.get("text", ""))
-                    elif isinstance(part, str):
-                        parts.append(part)
-                return " ".join(parts)
+            return _content_to_text(msg.get("content", ""))
     return ""
 
 
+def _extract_tool_use_ids(content: Any) -> List[str]:
+    if not isinstance(content, list):
+        return []
+    tool_use_ids: List[str] = []
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        if part.get("type") != "tool_use":
+            continue
+        tool_use_id = part.get("id")
+        if isinstance(tool_use_id, str) and tool_use_id:
+            tool_use_ids.append(tool_use_id)
+    return tool_use_ids
+
+
+def _extract_tool_result_ids(content: Any) -> Set[str]:
+    if not isinstance(content, list):
+        return set()
+    tool_result_ids: Set[str] = set()
+    for part in content:
+        if not isinstance(part, dict):
+            continue
+        if part.get("type") != "tool_result":
+            continue
+        tool_use_id = part.get("tool_use_id")
+        if isinstance(tool_use_id, str) and tool_use_id:
+            tool_result_ids.add(tool_use_id)
+    return tool_result_ids
+
+
+def _extract_anthropic_tool_exchange_spans(
+    messages: List[dict],
+) -> Tuple[List[Set[int]], Optional[str]]:
+    """
+    Return atomic 2-message spans for Anthropic tool exchanges.
+
+    Each assistant message containing `tool_use` must be immediately followed by a
+    user message containing matching `tool_result` blocks for all tool_use ids.
+    """
+    spans: List[Set[int]] = []
+    i = 0
+    while i < len(messages):
+        current = messages[i]
+        if current.get("role") != "assistant":
+            i += 1
+            continue
+
+        tool_use_ids = _extract_tool_use_ids(current.get("content"))
+        if not tool_use_ids:
+            i += 1
+            continue
+
+        if i + 1 >= len(messages):
+            return [], "invalid_anthropic_tool_sequence"
+
+        next_msg = messages[i + 1]
+        if next_msg.get("role") != "user":
+            return [], "invalid_anthropic_tool_sequence"
+
+        tool_result_ids = _extract_tool_result_ids(next_msg.get("content"))
+        if not tool_result_ids:
+            return [], "invalid_anthropic_tool_sequence"
+
+        for tool_use_id in tool_use_ids:
+            if tool_use_id not in tool_result_ids:
+                return [], "invalid_anthropic_tool_sequence"
+
+        spans.append({i, i + 1})
+        i += 2
+
+    return spans, None
+
+
 def _get_protected_indices(messages: List[dict]) -> List[int]:
     """
     Return indices of messages that must never be compressed:
@@ -87,9 +256,98 @@ def _combine_scores(
     return [bm25_weight * b + emb_weight * e for b, e in zip(norm_bm25, norm_emb)]
 
 
+def _select_kept_indices_for_budget(
+    normalized_messages: List[dict],
+    original_messages: List[dict],
+    combined_scores: List[float],
+    compression_target: int,
+    model: str,
+    initial_kept_indices: Set[int],
+    tool_exchange_spans: List[Set[int]],
+) -> Tuple[Set[int], Dict[int, dict]]:
+    kept_indices = set(initial_kept_indices)
+    current_tokens = 0
+    for i in kept_indices:
+        current_tokens += token_counter(
+            model=model,
+            text=cast(str, normalized_messages[i].get("content", "") or ""),
+        )
+
+    # Fill token budget from highest-scoring units.
+    # A unit is either:
+    # 1) a single message index, or
+    # 2) an Anthropic tool-exchange span that must be kept/dropped atomically.
+    truncated_overrides: Dict[int, dict] = {}  # idx -> truncated message dict
+    span_id_by_index: Dict[int, int] = {}
+    for span_id, span in enumerate(tool_exchange_spans):
+        for idx in span:
+            span_id_by_index[idx] = span_id
+
+    # Build single-message candidate units (non-span messages).
+    candidate_units: List[Tuple[float, Tuple[int, ...], bool]] = []
+    for idx in range(len(normalized_messages)):
+        if idx in span_id_by_index or idx in kept_indices:
+            continue
+        candidate_units.append((combined_scores[idx], (idx,), True))
+
+    # Build span candidate units (atomic keep/drop for tool exchanges).
+    for span in tool_exchange_spans:
+        span_indices = tuple(sorted(span))
+        if any(idx in kept_indices for idx in span_indices):
+            continue
+        span_score = max(combined_scores[idx] for idx in span_indices)
+        candidate_units.append((span_score, span_indices, False))
+
+    # Sort by descending relevance score.
+    candidate_units.sort(key=lambda item: item[0], reverse=True)
+
+    for _score, indices, can_truncate in candidate_units:
+        if any(idx in kept_indices for idx in indices):
+            continue
+        msg_tokens = 0
+        for idx in indices:
+            msg_tokens += token_counter(
+                model=model,
+                text=cast(str, normalized_messages[idx].get("content", "") or ""),
+            )
+        remaining = compression_target - current_tokens
+
+        if remaining <= 0:
+            break  # budget exhausted
+
+        if current_tokens + msg_tokens <= compression_target:
+            # Fits entirely
+            kept_indices.update(indices)
+            current_tokens += msg_tokens
+        elif can_truncate and len(indices) == 1 and remaining >= 100:
+            # Too large to fit whole single message, but we have budget — truncate it.
+            idx = indices[0]
+            truncated = truncate_message(original_messages[idx], remaining)
+            truncated_tokens = token_counter(
+                model=model,
+                text=truncated.get("content", "") or "",
+            )
+            truncated_overrides[idx] = truncated
+            kept_indices.add(idx)
+            current_tokens += truncated_tokens
+
+    return kept_indices, truncated_overrides
+
+
+def _get_dropped_tool_span_indices(
+    kept_indices: Set[int], tool_exchange_spans: List[Set[int]]
+) -> Set[int]:
+    dropped_tool_span_indices: Set[int] = set()
+    for span in tool_exchange_spans:
+        if not any(idx in kept_indices for idx in span):
+            dropped_tool_span_indices.update(span)
+    return dropped_tool_span_indices
+
+
 def compress(
     messages: List[dict],
     model: str,
+    call_type: Union[CallTypes, str] = CallTypes.completion,
     compression_trigger: int = 200_000,
     compression_target: Optional[int] = None,
     embedding_model: Optional[str] = None,
@@ -108,6 +366,12 @@ def compress(
     Parameters:
         messages: The conversation messages to (potentially) compress.
         model: The LLM model name — used for token counting.
+        call_type: The LiteLLM call type whose message schema these messages
+            follow.  Supported values:
+            - ``CallTypes.completion`` / ``CallTypes.acompletion`` — OpenAI
+              chat-completions shape (default)
+            - ``CallTypes.anthropic_messages`` — Anthropic Messages shape
+              (structured content blocks + atomic tool exchanges)
         compression_trigger: Only compress if input exceeds this token count.
         compression_target: Target token count after compression.
             Defaults to ``compression_trigger // 2``.
@@ -122,29 +386,37 @@ def compress(
         A ``CompressedResult`` dict containing compressed messages, token
         counts, a cache of original content, and the retrieval tool definition.
     """
+    call_type_str = _normalize_call_type(call_type)
+    normalized_messages, original_messages = _normalize_messages_for_compression(
+        messages=messages,
+        call_type=call_type_str,
+    )
+
     if compression_target is None:
         compression_target = compression_trigger * 7 // 10
 
     original_tokens = token_counter(
-        model=model, messages=cast(List[Union[AllMessageValues, Message]], messages)
+        model=model,
+        messages=cast(List[Any], original_messages),
     )
 
     # Pass through if below trigger
     if original_tokens <= compression_trigger:
         return CompressedResult(
-            messages=messages,
+            messages=original_messages,
             original_tokens=original_tokens,
             compressed_tokens=original_tokens,
             compression_ratio=0.0,
             cache={},
             tools=[],
+            compression_skipped_reason="below_trigger",
         )
 
     # Extract query for relevance scoring
-    query = _extract_last_user_message(messages)
+    query = _extract_last_user_message(normalized_messages)
 
     # Score each message
-    bm25_scores = bm25_score_messages(query, messages)
+    bm25_scores = bm25_score_messages(query, normalized_messages)
 
     if embedding_model:
         from litellm.compression.scoring.embedding_scorer import (
@@ -153,7 +425,7 @@ def compress(
 
         emb_scores = embedding_score_messages(
             query,
-            messages,
+            normalized_messages,
             model=embedding_model,
             cache=compression_cache,
             embedding_model_params=embedding_model_params,
@@ -162,85 +434,69 @@ def compress(
     else:
         combined_scores = bm25_scores
 
-    # Sort message indices by score descending
-    ranked_indices = sorted(
-        range(len(messages)),
-        key=lambda i: combined_scores[i],
-        reverse=True,
-    )
-
     # Protected messages are never compressed
-    protected_indices = _get_protected_indices(messages)
+    protected_indices = _get_protected_indices(normalized_messages)
     kept_indices: Set[int] = set(protected_indices)
 
-    # Count tokens for protected messages
-    current_tokens = 0
-    for i in kept_indices:
-        current_tokens += token_counter(
-            model=model, text=messages[i].get("content", "") or ""
+    tool_exchange_spans: List[Set[int]] = []
+    if _is_anthropic_call_type(call_type_str):
+        tool_exchange_spans, tool_sequence_error = (
+            _extract_anthropic_tool_exchange_spans(original_messages)
         )
-
-    # Fill token budget from highest-scoring messages.
-    # For each candidate (ranked by relevance):
-    #   - If it fits entirely → keep it as-is.
-    #   - If it doesn't fit but there's meaningful remaining budget → truncate it
-    #     to fill as much of the budget as possible.
-    #   - Otherwise → stub it (pointer only, content goes to cache).
-    # Multiple messages may be truncated so we preserve partial content from
-    # several high-scoring messages rather than fully stubbing all but one.
-    truncated_overrides: Dict[int, dict] = {}  # idx -> truncated message dict
-
-    for idx in ranked_indices:
-        if idx in kept_indices:
-            continue
-        msg_content = messages[idx].get("content", "") or ""
-        msg_tokens = token_counter(model=model, text=msg_content)
-        remaining = compression_target - current_tokens
-
-        if remaining <= 0:
-            break  # budget exhausted
-
-        if current_tokens + msg_tokens <= compression_target:
-            # Fits entirely
-            kept_indices.add(idx)
-            current_tokens += msg_tokens
-        elif remaining >= 100:
-            # Too large to fit whole, but we have budget — truncate it.
-            truncated = truncate_message(messages[idx], remaining)
-            truncated_tokens = token_counter(
-                model=model,
-                text=truncated.get("content", "") or "",
+        if tool_sequence_error is not None:
+            return CompressedResult(
+                messages=original_messages,
+                original_tokens=original_tokens,
+                compressed_tokens=original_tokens,
+                compression_ratio=0.0,
+                cache={},
+                tools=[],
+                compression_skipped_reason=tool_sequence_error,
             )
-            truncated_overrides[idx] = truncated
-            kept_indices.add(idx)
-            current_tokens += truncated_tokens
+
+        for span in tool_exchange_spans:
+            # If any message in the span is protected, keep the whole span.
+            if any(idx in kept_indices for idx in span):
+                kept_indices.update(span)
+
+    kept_indices, truncated_overrides = _select_kept_indices_for_budget(
+        normalized_messages=normalized_messages,
+        original_messages=original_messages,
+        combined_scores=combined_scores,
+        compression_target=compression_target,
+        model=model,
+        initial_kept_indices=kept_indices,
+        tool_exchange_spans=tool_exchange_spans,
+    )
 
     # Build compressed messages and cache
     compressed_messages: List[dict] = []
     cache: Dict[str, str] = {}
     used_keys: Set[str] = set()
+    dropped_tool_span_indices = _get_dropped_tool_span_indices(
+        kept_indices=kept_indices, tool_exchange_spans=tool_exchange_spans
+    )
 
-    for i, msg in enumerate(messages):
+    for i, msg in enumerate(original_messages):
+        if i in dropped_tool_span_indices:
+            continue
         if i in kept_indices:
             # Use the truncated version if we made one, otherwise the original
             compressed_messages.append(truncated_overrides.get(i, msg))
         else:
-            key = extract_key(msg, fallback_index=i, used_keys=used_keys)
-            content = msg.get("content", "")
-            if isinstance(content, list):
-                content = " ".join(
-                    p.get("text", "") if isinstance(p, dict) else str(p)
-                    for p in content
-                )
+            key = extract_key(
+                normalized_messages[i], fallback_index=i, used_keys=used_keys
+            )
+            content = _content_to_text(msg.get("content", ""))
             cache[key] = content
             compressed_messages.append(stub_message(msg, key))
 
-    # Build retrieval tool
-    tools = [build_retrieval_tool(list(cache.keys()))] if cache else []
+    # Build retrieval tool in the target request schema
+    tools = _build_retrieval_tools(list(cache.keys()), call_type=call_type_str)
 
     compressed_tokens = token_counter(
         model=model,
-        messages=cast(List[Union[AllMessageValues, Message]], compressed_messages),
+        messages=cast(List[Any], compressed_messages),
     )
 
     return CompressedResult(

litellm/integrations/compression_interception/__init__.py

@@ -0,0 +1,14 @@
+"""
+Compression Interception Module
+
+Provides server-side prompt compression + retrieval tool fulfillment for
+Anthropic Messages agentic loops.
+"""
+
+from litellm.integrations.compression_interception.handler import (
+    CompressionInterceptionLogger,
+)
+
+__all__ = [
+    "CompressionInterceptionLogger",
+]

litellm/integrations/compression_interception/handler.py

@@ -0,0 +1,399 @@
+"""
+Compression Interception Handler
+
+CustomLogger that compresses inbound Anthropic Messages requests and fulfills
+litellm_content_retrieve tool calls server-side via the typed agentic loop plan.
+"""
+
+import time
+import uuid
+from typing import Any, Dict, List, Optional, Tuple, cast
+
+from litellm._logging import verbose_logger
+from litellm.compression import compress
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.integrations.compression_interception import (
+    CompressionInterceptionConfig,
+)
+from litellm.types.integrations.custom_logger import (
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+)
+from litellm.types.utils import CallTypes
+
+LITELLM_CONTENT_RETRIEVE_TOOL_NAME = "litellm_content_retrieve"
+_CACHE_TTL_SECONDS = 15 * 60
+
+
+class CompressionInterceptionLogger(CustomLogger):
+    """
+    CustomLogger that implements transparent prompt compression + retrieval loops.
+
+    Flow:
+    1. Compress inbound /v1/messages requests in pre-call hook.
+    2. Inject litellm_content_retrieve tool and persist compressed cache by call_id.
+    3. Detect retrieval tool_use blocks in first model response.
+    4. Build typed rerun plan with tool_result blocks from the compressed cache.
+    """
+
+    def __init__(
+        self,
+        enabled: bool = True,
+        compression_trigger: int = 200_000,
+        compression_target: Optional[int] = None,
+        embedding_model: Optional[str] = None,
+        embedding_model_params: Optional[Dict[str, Any]] = None,
+    ):
+        super().__init__()
+        self.enabled = enabled
+        self.compression_trigger = compression_trigger
+        self.compression_target = compression_target
+        self.embedding_model = embedding_model
+        self.embedding_model_params = embedding_model_params
+        self._compression_cache_by_call_id: Dict[str, Tuple[Dict[str, str], float]] = {}
+
+    @classmethod
+    def from_config_yaml(
+        cls, config: CompressionInterceptionConfig
+    ) -> "CompressionInterceptionLogger":
+        return cls(
+            enabled=bool(config.get("enabled", True)),
+            compression_trigger=int(config.get("compression_trigger", 200_000)),
+            compression_target=config.get("compression_target"),
+            embedding_model=config.get("embedding_model"),
+            embedding_model_params=config.get("embedding_model_params"),
+        )
+
+    @staticmethod
+    def initialize_from_proxy_config(
+        litellm_settings: Dict[str, Any],
+        callback_specific_params: Dict[str, Any],
+    ) -> "CompressionInterceptionLogger":
+        compression_params: CompressionInterceptionConfig = {}
+        if "compression_interception_params" in litellm_settings:
+            compression_params = litellm_settings["compression_interception_params"]
+        elif "compression_interception" in callback_specific_params:
+            compression_params = callback_specific_params["compression_interception"]
+        return CompressionInterceptionLogger.from_config_yaml(compression_params)
+
+    async def async_pre_call_deployment_hook(
+        self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
+    ) -> Optional[dict]:
+        if not self.enabled:
+            return None
+        if call_type is not None and call_type != CallTypes.anthropic_messages:
+            return None
+        if int(kwargs.get("_agentic_loop_depth", 0) or 0) > 0:
+            return None
+
+        messages = kwargs.get("messages")
+        model = kwargs.get("model")
+        if not isinstance(messages, list) or not isinstance(model, str):
+            return None
+
+        if self._has_retrieval_tool(kwargs.get("tools")):
+            return None
+
+        self._prune_expired_cache()
+
+        compressed = compress(  # type: ignore
+            messages=messages,
+            model=model,
+            call_type=CallTypes.anthropic_messages,
+            compression_trigger=self.compression_trigger,
+            compression_target=self.compression_target,
+            embedding_model=self.embedding_model,
+            embedding_model_params=self.embedding_model_params,
+        )
+
+        cache = cast(Dict[str, str], compressed.get("cache", {}))
+        skip_reason = cast(Optional[str], compressed.get("compression_skipped_reason"))
+        compressed_tools = cast(List[Dict[str, Any]], compressed.get("tools", []))
+
+        # Only mutate kwargs when compression actually produced a result.
+        # If compression was a no-op (below trigger, invalid tool sequence, etc.),
+        # leave ``messages`` and ``tools`` untouched — injecting an empty
+        # ``tools: []`` onto a request that originally had no tools breaks
+        # Anthropic Messages requests.
+        if cache:
+            kwargs["messages"] = compressed["messages"]
+            if compressed_tools:
+                kwargs["tools"] = self._merge_tools(
+                    existing_tools=cast(
+                        Optional[List[Dict[str, Any]]], kwargs.get("tools")
+                    ),
+                    compressed_tools=compressed_tools,
+                )
+            call_id = cast(Optional[str], kwargs.get("litellm_call_id"))
+            if not call_id:
+                call_id = str(uuid.uuid4())
+                kwargs["litellm_call_id"] = call_id
+            self._compression_cache_by_call_id[call_id] = (cache, time.time())
+            verbose_logger.debug(
+                "CompressionInterception: compressed request [call_id=%s original=%d compressed=%d cached_keys=%d]",
+                call_id,
+                compressed.get("original_tokens"),
+                compressed.get("compressed_tokens"),
+                len(cache),
+            )
+        elif skip_reason is not None:
+            verbose_logger.debug(
+                "CompressionInterception: compression skipped [reason=%s original=%d compressed=%d]",
+                skip_reason,
+                compressed.get("original_tokens"),
+                compressed.get("compressed_tokens"),
+            )
+
+        return kwargs
+
+    async def async_should_run_agentic_loop(
+        self,
+        response: Any,
+        model: str,
+        messages: List[Dict],
+        tools: Optional[List[Dict]],
+        stream: bool,
+        custom_llm_provider: str,
+        kwargs: Dict,
+    ) -> Tuple[bool, Dict]:
+        if not self.enabled:
+            return False, {}
+        if not self._has_retrieval_tool(tools):
+            return False, {}
+
+        tool_calls, thinking_blocks = self._extract_retrieval_tool_calls(
+            response=response
+        )
+        if not tool_calls:
+            return False, {}
+
+        return True, {
+            "tool_calls": tool_calls,
+            "thinking_blocks": thinking_blocks,
+            "tool_type": "compression_retrieval",
+        }
+
+    async def async_build_agentic_loop_plan(
+        self,
+        tools: Dict,
+        model: str,
+        messages: List[Dict],
+        response: Any,
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: Dict,
+    ) -> AgenticLoopPlan:
+        self._prune_expired_cache()
+        tool_calls = cast(List[Dict[str, Any]], tools.get("tool_calls", []))
+        thinking_blocks = cast(List[Dict[str, Any]], tools.get("thinking_blocks", []))
+
+        call_id = self._resolve_call_id(logging_obj=logging_obj, kwargs=kwargs)
+        cache = self._get_cache(call_id=call_id)
+        retrieval_results = [
+            self._resolve_retrieval_content(tc, cache) for tc in tool_calls
+        ]
+
+        assistant_message = {
+            "role": "assistant",
+            "content": thinking_blocks
+            + [
+                {
+                    "type": "tool_use",
+                    "id": tc.get("id"),
+                    "name": tc.get("name", LITELLM_CONTENT_RETRIEVE_TOOL_NAME),
+                    "input": tc.get("input", {}),
+                }
+                for tc in tool_calls
+            ],
+        }
+        user_message = {
+            "role": "user",
+            "content": [
+                {
+                    "type": "tool_result",
+                    "tool_use_id": tool_calls[i].get("id"),
+                    "content": retrieval_results[i],
+                }
+                for i in range(len(tool_calls))
+            ],
+        }
+        follow_up_messages = messages + [assistant_message, user_message]
+
+        max_tokens = cast(
+            Optional[int],
+            anthropic_messages_optional_request_params.get("max_tokens")
+            or kwargs.get("max_tokens"),
+        )
+        optional_params_without_max_tokens = {
+            k: v
+            for k, v in anthropic_messages_optional_request_params.items()
+            if k != "max_tokens"
+        }
+
+        full_model_name = model
+        if logging_obj is not None:
+            agentic_params = logging_obj.model_call_details.get(
+                "agentic_loop_params", {}
+            )
+            full_model_name = cast(str, agentic_params.get("model", model))
+
+        request_patch = AgenticLoopRequestPatch(
+            model=full_model_name,
+            messages=follow_up_messages,
+            max_tokens=max_tokens,
+            optional_params=optional_params_without_max_tokens,
+            kwargs=self._prepare_followup_kwargs(kwargs=kwargs),
+        )
+
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={"tool_type": "compression_retrieval", "call_id": call_id or ""},
+        )
+
+    def _prune_expired_cache(self) -> None:
+        now = time.time()
+        self._compression_cache_by_call_id = {
+            call_id: (cache, created_at)
+            for call_id, (
+                cache,
+                created_at,
+            ) in self._compression_cache_by_call_id.items()
+            if now - created_at <= _CACHE_TTL_SECONDS
+        }
+
+    def _get_cache(self, call_id: Optional[str]) -> Dict[str, str]:
+        if not call_id:
+            return {}
+        cache_entry = self._compression_cache_by_call_id.get(call_id)
+        if cache_entry is None:
+            return {}
+        return cache_entry[0]
+
+    def _resolve_call_id(
+        self, logging_obj: Any, kwargs: Dict[str, Any]
+    ) -> Optional[str]:
+        if logging_obj is not None:
+            logging_call_id = getattr(logging_obj, "litellm_call_id", None)
+            if isinstance(logging_call_id, str) and logging_call_id:
+                return logging_call_id
+        kwargs_call_id = kwargs.get("litellm_call_id")
+        return cast(
+            Optional[str], kwargs_call_id if isinstance(kwargs_call_id, str) else None
+        )
+
+    def _resolve_retrieval_content(
+        self, tool_call: Dict[str, Any], cache: Dict[str, str]
+    ) -> str:
+        raw_input = tool_call.get("input", {})
+        key = ""
+        if isinstance(raw_input, dict):
+            key = str(raw_input.get("key", "") or "")
+        if not key:
+            return "No retrieval key provided."
+        if key in cache:
+            return cache[key]
+        return f"[compressed content key '{key}' not found]"
+
+    def _extract_retrieval_tool_calls(
+        self, response: Any
+    ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]]]:
+        if isinstance(response, dict):
+            content = response.get("content", [])
+        else:
+            content = getattr(response, "content", []) or []
+
+        if not isinstance(content, list):
+            return [], []
+
+        tool_calls: List[Dict[str, Any]] = []
+        thinking_blocks: List[Dict[str, Any]] = []
+
+        for block in content:
+            if isinstance(block, dict):
+                block_type = block.get("type")
+                block_name = block.get("name")
+                if block_type in ("thinking", "redacted_thinking"):
+                    thinking_blocks.append(block)
+                if (
+                    block_type == "tool_use"
+                    and block_name == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
+                ):
+                    tool_calls.append(
+                        {
+                            "id": block.get("id"),
+                            "type": "tool_use",
+                            "name": block_name,
+                            "input": block.get("input", {}),
+                        }
+                    )
+            else:
+                block_type = getattr(block, "type", None)
+                block_name = getattr(block, "name", None)
+                if block_type == "thinking":
+                    thinking_blocks.append(
+                        {
+                            "type": "thinking",
+                            "thinking": getattr(block, "thinking", ""),
+                            "signature": getattr(block, "signature", ""),
+                        }
+                    )
+                elif block_type == "redacted_thinking":
+                    thinking_blocks.append(
+                        {
+                            "type": "redacted_thinking",
+                            "data": getattr(block, "data", ""),
+                        }
+                    )
+                if (
+                    block_type == "tool_use"
+                    and block_name == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
+                ):
+                    tool_calls.append(
+                        {
+                            "id": getattr(block, "id", None),
+                            "type": "tool_use",
+                            "name": block_name,
+                            "input": getattr(block, "input", {}) or {},
+                        }
+                    )
+
+        return tool_calls, thinking_blocks
+
+    def _prepare_followup_kwargs(self, kwargs: Dict[str, Any]) -> Dict[str, Any]:
+        internal_keys = {"litellm_logging_obj"}
+        return {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_compression_interception") and k not in internal_keys
+        }
+
+    def _has_retrieval_tool(self, tools: Any) -> bool:
+        if not isinstance(tools, list):
+            return False
+        for tool in tools:
+            if not isinstance(tool, dict):
+                continue
+            function = tool.get("function")
+            if tool.get("type") == "function" and isinstance(function, dict):
+                if function.get("name") == LITELLM_CONTENT_RETRIEVE_TOOL_NAME:
+                    return True
+            if (
+                tool.get("type") == "custom"
+                and tool.get("name") == LITELLM_CONTENT_RETRIEVE_TOOL_NAME
+            ):
+                return True
+        return False
+
+    def _merge_tools(
+        self,
+        existing_tools: Optional[List[Dict[str, Any]]],
+        compressed_tools: List[Dict[str, Any]],
+    ) -> List[Dict[str, Any]]:
+        merged = list(existing_tools or [])
+        if self._has_retrieval_tool(merged):
+            return merged
+        merged.extend(compressed_tools)
+        return merged

litellm/integrations/custom_logger.py

@@ -20,6 +20,7 @@ from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH_SENSITIVE_DATA_MASKER
 from litellm.types.integrations.argilla import ArgillaItem
 from litellm.types.llms.openai import AllMessageValues, ChatCompletionRequest
 from litellm.types.prompts.init_prompts import PromptSpec
+from litellm.types.integrations.custom_logger import AgenticLoopPlan
 from litellm.types.utils import (
     AdapterCompletionStreamWrapper,
     CallTypes,
@@ -239,7 +240,7 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         self,
         model: str,
         request_kwargs: Dict,
-        messages: Optional[List[Dict[str, str]]] = None,
+        messages: Optional[List[Dict[str, Any]]] = None,
         input: Optional[Union[str, List]] = None,
         specific_deployment: Optional[bool] = False,
     ) -> Optional[PreRoutingHookResponse]:
@@ -676,6 +677,26 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         """
         pass
 
+    async def async_build_agentic_loop_plan(
+        self,
+        tools: Dict,
+        model: str,
+        messages: List[Dict],
+        response: Any,
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: "LiteLLMLoggingObj",
+        stream: bool,
+        kwargs: Dict,
+    ) -> AgenticLoopPlan:
+        """
+        Build a typed rerun plan for Anthropic Messages agentic loops.
+
+        Override this method to separate callback decision/tool execution from
+        follow-up request execution (handled by BaseLLMHTTPHandler).
+        """
+        return AgenticLoopPlan(run_agentic_loop=False)
+
     async def async_should_run_chat_completion_agentic_loop(
         self,
         response: Any,
@@ -707,6 +728,22 @@ class CustomLogger:  # https://docs.litellm.ai/docs/observability/custom_callbac
         """
         pass
 
+    async def async_build_chat_completion_agentic_loop_plan(
+        self,
+        tools: Dict,
+        model: str,
+        messages: List[Dict],
+        response: Any,
+        optional_params: Dict,
+        logging_obj: "LiteLLMLoggingObj",
+        stream: bool,
+        kwargs: Dict,
+    ) -> AgenticLoopPlan:
+        """
+        Build a typed rerun plan for chat-completions agentic loops.
+        """
+        return AgenticLoopPlan(run_agentic_loop=False)
+
     # Useful helpers for custom logger classes
 
     def truncate_standard_logging_payload_content(

litellm/integrations/opentelemetry.py

@@ -1615,6 +1615,14 @@ class OpenTelemetry(CustomLogger):
                     value=response_id,
                 )
 
+            litellm_call_id = standard_logging_payload.get("litellm_call_id")
+            if litellm_call_id:
+                self.safe_set_attribute(
+                    span=span,
+                    key="litellm.call_id",
+                    value=litellm_call_id,
+                )
+
             # The model used to generate the response.
             if response_obj and response_obj.get("model"):
                 self.safe_set_attribute(

litellm/integrations/prometheus.py

@@ -51,6 +51,7 @@ if TYPE_CHECKING:
 else:
     AsyncIOScheduler = Any
 
+
 class PrometheusLogger(CustomLogger):
     # Class variables or attributes
 
@@ -991,9 +992,7 @@ class PrometheusLogger(CustomLogger):
         amount: float = 1.0,
     ) -> None:
         _labels = prometheus_label_factory(
-            supported_enum_labels=self.get_labels_for_metric(
-                metric_name=metric_name
-            ),
+            supported_enum_labels=self.get_labels_for_metric(metric_name=metric_name),
             enum_values=enum_values,
             label_context=label_context,
         )
@@ -1118,7 +1117,9 @@ class PrometheusLogger(CustomLogger):
 
             user_api_key = hash_token(user_api_key)
 
-        label_context = PrometheusLabelFactoryContext(enum_values) #amortized per request.
+        label_context = PrometheusLabelFactoryContext(
+            enum_values
+        )  # amortized per request.
 
         # increment total LLM requests and spend metric
         self._increment_top_level_request_and_spend_metrics(
@@ -3490,7 +3491,9 @@ def _prometheus_labels_from_context(
     }
 
     if UserAPIKeyLabelNames.END_USER.value in filtered_labels:
-        filtered_labels[UserAPIKeyLabelNames.END_USER.value] = ctx.get_resolved_end_user()
+        filtered_labels[UserAPIKeyLabelNames.END_USER.value] = (
+            ctx.get_resolved_end_user()
+        )
 
     for sk, val in ctx._custom_by_sanitized_key.items():
         if sk in supported_enum_labels:

litellm/integrations/prometheus_helpers/__init__.py

@@ -51,8 +51,7 @@ class PrometheusLabelFactoryContext:
         self.enum_values = enum_values
         enum_dict = enum_values.model_dump()
         self._sanitized_enum: Dict[str, Optional[str]] = {
-            k: _sanitize_prometheus_label_value(v)
-            for k, v in enum_dict.items()
+            k: _sanitize_prometheus_label_value(v) for k, v in enum_dict.items()
         }
         self._custom_by_sanitized_key: Dict[str, Optional[str]] = {}
         if enum_values.custom_metadata_labels is not None:

litellm/integrations/websearch_interception/handler.py

@@ -28,6 +28,10 @@ from litellm.integrations.websearch_interception.transformation import (
 from litellm.types.integrations.websearch_interception import (
     WebSearchInterceptionConfig,
 )
+from litellm.types.integrations.custom_logger import (
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+)
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import LlmProviders
 from litellm.utils import ProviderConfigManager
@@ -573,6 +577,35 @@ class WebSearchInterceptionLogger(CustomLogger):
             kwargs=kwargs,
         )
 
+    async def async_build_agentic_loop_plan(
+        self,
+        tools: Dict,
+        model: str,
+        messages: List[Dict],
+        response: Any,
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: Dict,
+    ) -> AgenticLoopPlan:
+        tool_calls = tools["tool_calls"]
+        thinking_blocks = tools.get("thinking_blocks", [])
+        request_patch = await self._build_anthropic_request_patch(
+            model=model,
+            messages=messages,
+            tool_calls=tool_calls,
+            thinking_blocks=thinking_blocks,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            logging_obj=logging_obj,
+            kwargs=kwargs,
+        )
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={"tool_type": "websearch", "response_format": "anthropic"},
+        )
+
     async def async_run_chat_completion_agentic_loop(
         self,
         tools: Dict,
@@ -608,6 +641,33 @@ class WebSearchInterceptionLogger(CustomLogger):
             response_format=response_format,
         )
 
+    async def async_build_chat_completion_agentic_loop_plan(
+        self,
+        tools: Dict,
+        model: str,
+        messages: List[Dict],
+        response: Any,
+        optional_params: Dict,
+        logging_obj: Any,
+        stream: bool,
+        kwargs: Dict,
+    ) -> AgenticLoopPlan:
+        tool_calls = tools["tool_calls"]
+        response_format = tools.get("response_format", "openai")
+        request_patch = await self._build_chat_completion_request_patch(
+            model=model,
+            messages=messages,
+            tool_calls=tool_calls,
+            optional_params=optional_params,
+            kwargs=kwargs,
+            response_format=response_format,
+        )
+        return AgenticLoopPlan(
+            run_agentic_loop=True,
+            request_patch=request_patch,
+            metadata={"tool_type": "websearch", "response_format": response_format},
+        )
+
     @staticmethod
     def _resolve_max_tokens(
         optional_params: Dict,
@@ -672,7 +732,48 @@ class WebSearchInterceptionLogger(CustomLogger):
         stream: bool,
         kwargs: Dict,
     ) -> Any:
-        """Execute litellm.search() and make follow-up request"""
+        """Legacy path: execute search + build patch + run follow-up call."""
+        request_patch = await self._build_anthropic_request_patch(
+            model=model,
+            messages=messages,
+            tool_calls=tool_calls,
+            thinking_blocks=thinking_blocks,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            logging_obj=logging_obj,
+            kwargs=kwargs,
+        )
+        if request_patch.messages is None:
+            raise ValueError("WebSearchInterception: missing follow-up messages")
+
+        optional_params = dict(anthropic_messages_optional_request_params)
+        optional_params.update(request_patch.optional_params)
+        max_tokens = request_patch.max_tokens
+        if max_tokens is None:
+            max_tokens = cast(Optional[int], optional_params.pop("max_tokens", None))
+        else:
+            optional_params.pop("max_tokens", None)
+        if max_tokens is None:
+            max_tokens = cast(int, kwargs.get("max_tokens", 1024))
+
+        return await anthropic_messages.acreate(
+            max_tokens=max_tokens,
+            messages=request_patch.messages,
+            model=request_patch.model or model,
+            **optional_params,
+            **request_patch.kwargs,
+        )
+
+    async def _build_anthropic_request_patch(
+        self,
+        model: str,
+        messages: List[Dict],
+        tool_calls: List[Dict],
+        thinking_blocks: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: Any,
+        kwargs: Dict,
+    ) -> AgenticLoopRequestPatch:
+        """Execute litellm.search() and build follow-up request patch."""
 
         # Extract search queries from tool_use blocks
         search_tasks = []
@@ -721,20 +822,8 @@ class WebSearchInterceptionLogger(CustomLogger):
             thinking_blocks=thinking_blocks,
         )
 
-        # Make follow-up request with search results
-        # Type cast: user_message is a Dict for Anthropic format (default response_format)
         follow_up_messages = messages + [assistant_message, cast(Dict, user_message)]
 
-        verbose_logger.debug(
-            "WebSearchInterception: Making follow-up request with search results"
-        )
-        verbose_logger.debug(
-            f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
-        )
-        verbose_logger.debug(
-            f"WebSearchInterception: Last message (tool_result): {user_message}"
-        )
-
         # Correlation context for structured logging
         _call_id = getattr(logging_obj, "litellm_call_id", None) or kwargs.get(
             "litellm_call_id", "unknown"
@@ -742,61 +831,41 @@ class WebSearchInterceptionLogger(CustomLogger):
 
         full_model_name = model  # safe default before try block
 
-        # Use anthropic_messages.acreate for follow-up request
-        try:
-            max_tokens = self._resolve_max_tokens(
-                anthropic_messages_optional_request_params, kwargs
-            )
+        max_tokens = self._resolve_max_tokens(
+            anthropic_messages_optional_request_params, kwargs
+        )
 
-            verbose_logger.debug(
-                f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
-            )
+        verbose_logger.debug(
+            f"WebSearchInterception: Using max_tokens={max_tokens} for follow-up request"
+        )
 
-            # Create a copy of optional params without max_tokens (since we pass it explicitly)
-            optional_params_without_max_tokens = {
-                k: v
-                for k, v in anthropic_messages_optional_request_params.items()
-                if k != "max_tokens"
-            }
+        optional_params_without_max_tokens = {
+            k: v
+            for k, v in anthropic_messages_optional_request_params.items()
+            if k != "max_tokens"
+        }
+        kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
 
-            kwargs_for_followup = self._prepare_followup_kwargs(kwargs)
-
-            # Get model from logging_obj.model_call_details["agentic_loop_params"]
-            # This preserves the full model name with provider prefix (e.g., "bedrock/invoke/...")
-            if logging_obj is not None:
-                agentic_params = logging_obj.model_call_details.get(
-                    "agentic_loop_params", {}
-                )
-                full_model_name = agentic_params.get("model", model)
-            verbose_logger.debug(
-                f"WebSearchInterception: Using model name: {full_model_name}"
+        if logging_obj is not None:
+            agentic_params = logging_obj.model_call_details.get(
+                "agentic_loop_params", {}
             )
-
-            final_response = await anthropic_messages.acreate(
-                max_tokens=max_tokens,
-                messages=follow_up_messages,
-                model=full_model_name,
-                **optional_params_without_max_tokens,
-                **kwargs_for_followup,
-            )
-            verbose_logger.debug(
-                f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
-            )
-            verbose_logger.debug(
-                f"WebSearchInterception: Final response: {final_response}"
-            )
-            return final_response
-        except Exception as e:
-            verbose_logger.exception(
-                "WebSearchInterception: Follow-up request failed "
-                "[call_id=%s model=%s messages=%d searches=%d]: %s",
-                _call_id,
-                full_model_name,
-                len(follow_up_messages),
-                len(final_search_results),
-                str(e),
-            )
-            raise
+            full_model_name = agentic_params.get("model", model)
+        verbose_logger.debug(
+            "WebSearchInterception: Built anthropic request patch "
+            "[call_id=%s model=%s messages=%d searches=%d]",
+            _call_id,
+            full_model_name,
+            len(follow_up_messages),
+            len(final_search_results),
+        )
+        return AgenticLoopRequestPatch(
+            model=full_model_name,
+            messages=follow_up_messages,
+            max_tokens=max_tokens,
+            optional_params=optional_params_without_max_tokens,
+            kwargs=kwargs_for_followup,
+        )
 
     async def _execute_search(self, query: str) -> str:
         """Execute a single web search using router's search tools"""
@@ -883,7 +952,36 @@ class WebSearchInterceptionLogger(CustomLogger):
         kwargs: Dict,
         response_format: str = "openai",
     ) -> Any:
-        """Execute litellm.search() and make follow-up chat completion request"""
+        """Legacy path: execute search + build patch + run follow-up call."""
+        request_patch = await self._build_chat_completion_request_patch(
+            model=model,
+            messages=messages,
+            tool_calls=tool_calls,
+            optional_params=optional_params,
+            kwargs=kwargs,
+            response_format=response_format,
+        )
+        if request_patch.messages is None:
+            raise ValueError("WebSearchInterception: missing follow-up messages")
+        params = dict(optional_params)
+        params.update(request_patch.optional_params)
+        return await litellm.acompletion(
+            model=request_patch.model or model,
+            messages=request_patch.messages,
+            **params,
+            **request_patch.kwargs,
+        )
+
+    async def _build_chat_completion_request_patch(  # noqa: PLR0915
+        self,
+        model: str,
+        messages: List[Dict],
+        tool_calls: List[Dict],
+        optional_params: Dict,
+        kwargs: Dict,
+        response_format: str = "openai",
+    ) -> AgenticLoopRequestPatch:
+        """Execute litellm.search() and build chat-completion rerun patch."""
 
         # Extract search queries from tool_calls
         search_tasks = []
@@ -963,74 +1061,56 @@ class WebSearchInterceptionLogger(CustomLogger):
             f"WebSearchInterception: Follow-up messages count: {len(follow_up_messages)}"
         )
 
-        # Use litellm.acompletion for follow-up request
-        try:
-            # Remove internal parameters that shouldn't be passed to follow-up request
-            internal_params = {
-                "_websearch_interception",
-                "acompletion",
-                "litellm_logging_obj",
-                "custom_llm_provider",
+        # Remove internal parameters that shouldn't be passed to follow-up request
+        internal_params = {
+            "_websearch_interception",
+            "acompletion",
+            "litellm_logging_obj",
+            "custom_llm_provider",
+            "model_alias_map",
+            "stream_response",
+            "custom_prompt_dict",
+        }
+        kwargs_for_followup = {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_websearch_interception") and k not in internal_params
+        }
+
+        full_model_name = model
+        if "custom_llm_provider" in kwargs:
+            custom_llm_provider = kwargs["custom_llm_provider"]
+            if not model.startswith(custom_llm_provider) and "/" not in model:
+                full_model_name = f"{custom_llm_provider}/{model}"
+
+        verbose_logger.debug(
+            "WebSearchInterception: Built chat completion request patch model=%s messages=%d",
+            full_model_name,
+            len(follow_up_messages),
+        )
+
+        tools_param = optional_params.get("tools")
+        optional_params_clean = {
+            k: v
+            for k, v in optional_params.items()
+            if k
+            not in {
+                "tools",
+                "extra_body",
                 "model_alias_map",
                 "stream_response",
                 "custom_prompt_dict",
             }
-            kwargs_for_followup = {
-                k: v
-                for k, v in kwargs.items()
-                if not k.startswith("_websearch_interception")
-                and k not in internal_params
-            }
+        }
+        if tools_param is not None:
+            optional_params_clean["tools"] = tools_param
 
-            # Get full model name from kwargs
-            full_model_name = model
-            if "custom_llm_provider" in kwargs:
-                custom_llm_provider = kwargs["custom_llm_provider"]
-                # Reconstruct full model name with provider prefix if needed
-                if not model.startswith(custom_llm_provider):
-                    # Check if model already has a provider prefix
-                    if "/" not in model:
-                        full_model_name = f"{custom_llm_provider}/{model}"
-
-            verbose_logger.debug(
-                f"WebSearchInterception: Using model name: {full_model_name}"
-            )
-
-            # Prepare tools for follow-up request (same as original)
-            tools_param = optional_params.get("tools")
-
-            # Remove tools and extra_body from optional_params to avoid issues
-            # extra_body often contains internal LiteLLM params that shouldn't be forwarded
-            optional_params_clean = {
-                k: v
-                for k, v in optional_params.items()
-                if k
-                not in {
-                    "tools",
-                    "extra_body",
-                    "model_alias_map",
-                    "stream_response",
-                    "custom_prompt_dict",
-                }
-            }
-
-            final_response = await litellm.acompletion(
-                model=full_model_name,
-                messages=follow_up_messages,
-                tools=tools_param,
-                **optional_params_clean,
-                **kwargs_for_followup,
-            )
-
-            verbose_logger.debug(
-                f"WebSearchInterception: Follow-up request completed, response type: {type(final_response)}"
-            )
-            return final_response
-        except Exception as e:
-            verbose_logger.exception(
-                f"WebSearchInterception: Follow-up request failed: {str(e)}"
-            )
-            raise
+        return AgenticLoopRequestPatch(
+            model=full_model_name,
+            messages=follow_up_messages,
+            optional_params=optional_params_clean,
+            kwargs=kwargs_for_followup,
+        )
 
     async def _create_empty_search_result(self) -> str:
         """Create an empty search result for tool calls without queries"""

litellm/litellm_core_utils/get_supported_openai_params.py

@@ -296,6 +296,15 @@ def get_supported_openai_params(  # noqa: PLR0915
             return OVHCloudAudioTranscriptionConfig().get_supported_openai_params(
                 model=model
             )
+    elif custom_llm_provider == "scaleway":
+        if request_type == "transcription":
+            from litellm.llms.scaleway.audio_transcription.transformation import (
+                ScalewayAudioTranscriptionConfig,
+            )
+
+            return ScalewayAudioTranscriptionConfig().get_supported_openai_params(
+                model=model
+            )
     elif custom_llm_provider == "elevenlabs":
         if request_type == "transcription":
             from litellm.llms.elevenlabs.audio_transcription.transformation import (

litellm/litellm_core_utils/litellm_logging.py

@@ -5512,6 +5512,8 @@ def get_standard_logging_object_payload(
 
         payload: StandardLoggingPayload = StandardLoggingPayload(
             id=str(id),
+            litellm_call_id=kwargs.get("litellm_call_id")
+            or litellm_params.get("litellm_call_id"),
             trace_id=StandardLoggingPayloadSetup._get_standard_logging_payload_trace_id(
                 logging_obj=logging_obj,
                 litellm_params=litellm_params,

litellm/litellm_core_utils/llm_cost_calc/utils.py

@@ -684,7 +684,7 @@ def generic_cost_per_token(  # noqa: PLR0915
             - cache_creation
             - image_tokens
         )
-        # Clamp to zero: inconsistent streaming usage 
+        # Clamp to zero: inconsistent streaming usage
         if text_tokens < 0:
             text_tokens = 0
         prompt_tokens_details["text_tokens"] = text_tokens

litellm/llms/anthropic/chat/guardrail_translation/handler.py

@@ -34,6 +34,7 @@ from litellm.types.llms.anthropic import (
 )
 from litellm.types.llms.openai import (
     AllMessageValues,
+    ChatCompletionRequest,
     ChatCompletionToolCallChunk,
     ChatCompletionToolParam,
 )
@@ -67,6 +68,32 @@ class AnthropicMessagesHandler(BaseTranslation):
         super().__init__()
         self.adapter = LiteLLMAnthropicMessagesAdapter()
 
+    def _translate_to_openai(self, data: dict) -> ChatCompletionRequest:
+        """Translate Anthropic request to OpenAI chat completion format."""
+        (
+            chat_completion_compatible_request,
+            _tool_name_mapping,
+        ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
+            anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
+        )
+        return chat_completion_compatible_request
+
+    def get_structured_messages(self, data: dict) -> Optional[List[AllMessageValues]]:
+        """
+        Convert Anthropic messages request data to OpenAI-spec structured messages.
+
+        Uses the Anthropic-to-OpenAI adapter to translate message format.
+        """
+        messages = data.get("messages")
+        if messages is None:
+            return None
+        chat_completion_compatible_request = self._translate_to_openai(data)
+        result = cast(
+            List[AllMessageValues],
+            chat_completion_compatible_request.get("messages", []),
+        )
+        return result if result else None
+
     async def process_input_messages(
         self,
         data: dict,
@@ -82,13 +109,7 @@ class AnthropicMessagesHandler(BaseTranslation):
 
         skip_system = effective_skip_system_message_for_guardrail(guardrail_to_apply)
 
-        (
-            chat_completion_compatible_request,
-            _tool_name_mapping,
-        ) = LiteLLMAnthropicMessagesAdapter().translate_anthropic_to_openai(
-            # Use a shallow copy to avoid mutating request data (pop on litellm_metadata).
-            anthropic_message_request=cast(AnthropicMessagesRequest, data.copy())
-        )
+        chat_completion_compatible_request = self._translate_to_openai(data)
 
         structured_messages = cast(
             List[AllMessageValues],
@@ -103,8 +124,6 @@ class AnthropicMessagesHandler(BaseTranslation):
             chat_completion_compatible_request.get("tools", [])
         )
         task_mappings: List[Tuple[int, Optional[int]]] = []
-        # Track (message_index, content_index) for each text
-        # content_index is None for string content, int for list content
 
         # Step 1: Extract all text content and images
         for msg_idx, message in enumerate(messages):

litellm/llms/anthropic/experimental_pass_through/messages/agentic_streaming_iterator.py

@@ -0,0 +1,322 @@
+"""
+Agentic Streaming Iterator for Anthropic Messages
+
+Wraps the raw SSE byte stream from the Anthropic pass-through endpoint,
+yields every chunk to the caller (preserving real streaming), collects
+all bytes, and on stream exhaustion rebuilds the full Anthropic response
+to run through agentic completion hooks. If an agentic hook fires, the
+follow-up response is chained as Phase 2 of the same iterator.
+"""
+
+import json
+from typing import Any, AsyncIterator, Dict, List, Optional, cast
+
+from litellm._logging import verbose_logger
+
+
+# ---------------------------------------------------------------------------
+# SSE parsing helpers (module-level to keep the class lean)
+# ---------------------------------------------------------------------------
+
+
+def _parse_sse_events(raw: bytes) -> List[tuple]:
+    """Return a list of (event_type, parsed_data_dict) from raw SSE bytes."""
+    text = raw.decode("utf-8", errors="replace")
+    lines = text.split("\n")
+    events: List[tuple] = []
+    current_event_type: Optional[str] = None
+
+    for line in lines:
+        stripped = line.strip()
+        if stripped.startswith("event:"):
+            current_event_type = stripped[len("event:") :].strip()
+            continue
+        if not stripped.startswith("data:"):
+            continue
+        data_str = stripped[len("data:") :].strip()
+        try:
+            data = json.loads(data_str)
+        except (json.JSONDecodeError, ValueError):
+            continue
+        event_type = current_event_type or data.get("type", "")
+        current_event_type = None
+        events.append((event_type, data))
+    return events
+
+
+def _handle_message_start(data: Dict, response: Dict) -> None:
+    msg = data.get("message", {})
+    response["id"] = msg.get("id", response["id"])
+    response["model"] = msg.get("model", response["model"])
+    response["role"] = msg.get("role", response["role"])
+    usage = msg.get("usage", {})
+    if usage:
+        response["usage"]["input_tokens"] = usage.get("input_tokens", 0)
+        for key in ("cache_creation_input_tokens", "cache_read_input_tokens"):
+            if key in usage:
+                response["usage"][key] = usage[key]
+
+
+def _handle_content_block_start(data: Dict, content_blocks: Dict[int, Dict]) -> None:
+    idx = data.get("index", len(content_blocks))
+    block = data.get("content_block", {})
+    block_type = block.get("type", "text")
+
+    _BLOCK_TEMPLATES: Dict[str, Dict] = {
+        "text": {"type": "text", "text": ""},
+        "thinking": {"type": "thinking", "thinking": "", "signature": ""},
+        "redacted_thinking": {
+            "type": "redacted_thinking",
+            "data": block.get("data", ""),
+        },
+    }
+    if block_type == "tool_use":
+        content_blocks[idx] = {
+            "type": "tool_use",
+            "id": block.get("id", ""),
+            "name": block.get("name", ""),
+            "input": {},
+            "_partial_json": "",
+        }
+    elif block_type in _BLOCK_TEMPLATES:
+        content_blocks[idx] = dict(_BLOCK_TEMPLATES[block_type])
+    else:
+        content_blocks[idx] = dict(block)
+
+
+def _handle_content_block_delta(data: Dict, content_blocks: Dict[int, Dict]) -> None:
+    idx = data.get("index", 0)
+    delta = data.get("delta", {})
+    delta_type = delta.get("type", "")
+    block = content_blocks.get(idx)
+    if block is None:
+        return
+
+    if delta_type == "text_delta":
+        block["text"] = block.get("text", "") + delta.get("text", "")
+    elif delta_type == "input_json_delta":
+        block["_partial_json"] = block.get("_partial_json", "") + delta.get(
+            "partial_json", ""
+        )
+    elif delta_type == "thinking_delta":
+        block["thinking"] = block.get("thinking", "") + delta.get("thinking", "")
+    elif delta_type == "signature_delta":
+        block["signature"] = delta.get("signature", block.get("signature", ""))
+
+
+def _handle_content_block_stop(data: Dict, content_blocks: Dict[int, Dict]) -> None:
+    idx = data.get("index", 0)
+    block = content_blocks.get(idx)
+    if block and block.get("type") == "tool_use":
+        partial = block.pop("_partial_json", "")
+        if partial:
+            try:
+                block["input"] = json.loads(partial)
+            except (json.JSONDecodeError, ValueError):
+                block["input"] = {"_raw": partial}
+
+
+def _handle_message_delta(data: Dict, response: Dict) -> None:
+    delta = data.get("delta", {})
+    if "stop_reason" in delta:
+        response["stop_reason"] = delta["stop_reason"]
+    if "stop_sequence" in delta:
+        response["stop_sequence"] = delta["stop_sequence"]
+    usage = data.get("usage", {})
+    if usage.get("output_tokens") is not None:
+        response["usage"]["output_tokens"] = usage["output_tokens"]
+    for key in (
+        "input_tokens",
+        "cache_creation_input_tokens",
+        "cache_read_input_tokens",
+    ):
+        if key in usage:
+            response["usage"][key] = usage[key]
+
+
+class AgenticAnthropicStreamingIterator:
+    """
+    Two-phase async iterator that enables agentic hooks on streaming
+    Anthropic Messages pass-through responses.
+
+    Phase 1: Yield raw SSE bytes from the upstream response while
+             accumulating them. When the inner iterator is exhausted,
+             rebuild the full Anthropic response dict and call agentic hooks.
+
+    Phase 2: If an agentic hook fires and returns a follow-up response
+             (streaming or non-streaming), yield those bytes to the caller.
+    """
+
+    def __init__(
+        self,
+        completion_stream: AsyncIterator,
+        http_handler: Any,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_provider_config: Any,
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: Any,
+        custom_llm_provider: str,
+        kwargs: Dict,
+    ):
+        self._inner = completion_stream.__aiter__()
+        self._http_handler = http_handler
+        self._model = model
+        self._messages = messages
+        self._anthropic_messages_provider_config = anthropic_messages_provider_config
+        self._anthropic_messages_optional_request_params = (
+            anthropic_messages_optional_request_params
+        )
+        self._logging_obj = logging_obj
+        self._custom_llm_provider = custom_llm_provider
+        self._kwargs = kwargs
+
+        self._collected_bytes: List[bytes] = []
+        self._stream_exhausted = False
+        self._hook_processing_done = False
+        self._follow_up_iterator: Optional[AsyncIterator] = None
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> bytes:
+        # Phase 1: yield from upstream, collect bytes
+        if not self._stream_exhausted:
+            try:
+                chunk = await self._inner.__anext__()
+                self._collected_bytes.append(chunk)
+                return chunk
+            except StopAsyncIteration:
+                self._stream_exhausted = True
+                await self._process_agentic_hooks()
+                # Fall through to Phase 2
+
+        # Phase 2: yield from follow-up stream if one was created
+        if self._follow_up_iterator is not None:
+            chunk = await self._follow_up_iterator.__anext__()
+            return chunk
+
+        raise StopAsyncIteration
+
+    async def _process_agentic_hooks(self) -> None:
+        """Rebuild the Anthropic response from collected SSE bytes and call hooks."""
+        if self._hook_processing_done:
+            return
+        self._hook_processing_done = True
+
+        if not self._collected_bytes:
+            return
+
+        try:
+            rebuilt = self._rebuild_anthropic_response_from_sse(self._collected_bytes)
+            if rebuilt is None:
+                verbose_logger.debug(
+                    "AgenticStreamingIterator: Could not rebuild response from SSE bytes"
+                )
+                return
+
+            [
+                (
+                    f"{b.get('type')}({b.get('name', '')})"
+                    if b.get("type") == "tool_use"
+                    else b.get("type")
+                )
+                for b in rebuilt.get("content", [])
+            ]
+
+            result = await self._http_handler._call_agentic_completion_hooks(
+                response=rebuilt,
+                model=self._model,
+                messages=self._messages,
+                anthropic_messages_provider_config=self._anthropic_messages_provider_config,
+                anthropic_messages_optional_request_params=self._anthropic_messages_optional_request_params,
+                logging_obj=self._logging_obj,
+                stream=True,
+                custom_llm_provider=self._custom_llm_provider,
+                kwargs=self._kwargs,
+            )
+
+            if result is None:
+                return
+
+            if hasattr(result, "__aiter__"):
+                self._follow_up_iterator = result.__aiter__()
+            elif isinstance(result, dict):
+                from litellm.llms.anthropic.experimental_pass_through.messages.fake_stream_iterator import (
+                    FakeAnthropicMessagesStreamIterator,
+                )
+                from litellm.types.llms.anthropic_messages.anthropic_response import (
+                    AnthropicMessagesResponse,
+                )
+
+                fake = FakeAnthropicMessagesStreamIterator(
+                    response=cast(AnthropicMessagesResponse, result)
+                )
+                self._follow_up_iterator = fake.__aiter__()
+            else:
+                verbose_logger.warning(
+                    "AgenticStreamingIterator: Unexpected result type from hooks: %s",
+                    type(result).__name__,
+                )
+        except Exception as e:
+            _call_id = getattr(self._logging_obj, "litellm_call_id", "unknown")
+            verbose_logger.exception(
+                "AgenticStreamingIterator: Error in agentic hook processing "
+                "[call_id=%s model=%s]: %s",
+                _call_id,
+                self._model,
+                str(e),
+            )
+
+    @staticmethod
+    def _rebuild_anthropic_response_from_sse(
+        raw_bytes: List[bytes],
+    ) -> Optional[Dict[str, Any]]:
+        """
+        Parse collected SSE bytes into an Anthropic Messages response dict.
+
+        Processes SSE events in order:
+        - message_start   -> envelope (id, model, role, usage)
+        - content_block_start -> new content block
+        - content_block_delta -> accumulate text/json/thinking deltas
+        - content_block_stop  -> finalize block
+        - message_delta   -> stop_reason, output usage
+        - message_stop    -> end
+        """
+        events = _parse_sse_events(b"".join(raw_bytes))
+
+        response: Dict[str, Any] = {
+            "id": "",
+            "type": "message",
+            "role": "assistant",
+            "model": "",
+            "content": [],
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": {"input_tokens": 0, "output_tokens": 0},
+        }
+        content_blocks: Dict[int, Dict[str, Any]] = {}
+        saw_message_start = False
+
+        for event_type, data in events:
+            if event_type == "message_start":
+                saw_message_start = True
+                _handle_message_start(data, response)
+            elif event_type == "content_block_start":
+                _handle_content_block_start(data, content_blocks)
+            elif event_type == "content_block_delta":
+                _handle_content_block_delta(data, content_blocks)
+            elif event_type == "content_block_stop":
+                _handle_content_block_stop(data, content_blocks)
+            elif event_type == "message_delta":
+                _handle_message_delta(data, response)
+
+        if not saw_message_start:
+            return None
+
+        for idx in sorted(content_blocks.keys()):
+            block = content_blocks[idx]
+            block.pop("_partial_json", None)
+            response["content"].append(block)
+
+        return response

litellm/llms/base_llm/guardrail_translation/base_translation.py

@@ -5,6 +5,7 @@ if TYPE_CHECKING:
     from litellm.integrations.custom_guardrail import CustomGuardrail
     from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
     from litellm.proxy._types import UserAPIKeyAuth
+    from litellm.types.llms.openai import AllMessageValues
 
 
 class BaseTranslation(ABC):
@@ -101,6 +102,16 @@ class BaseTranslation(ABC):
         """
         return responses_so_far
 
+    def get_structured_messages(self, data: dict) -> Optional[List["AllMessageValues"]]:
+        """
+        Convert request data to OpenAI-spec structured messages.
+
+        Override in subclasses for format-specific conversion.
+
+        Returns None if no convertible content is found.
+        """
+        return None
+
     def extract_request_tool_names(self, data: dict) -> List[str]:
         """
         Extract tool names from the request body for allowlist/policy checks.

litellm/llms/bedrock/chat/mantle/transformation.py

@@ -0,0 +1,91 @@
+"""
+Transformation for Bedrock Mantle (Claude Mythos Preview)
+
+https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-mythos-preview.html
+
+The bedrock-mantle endpoint uses the Anthropic Messages API format but is served
+at a different endpoint (bedrock-mantle.{region}.api.aws) with AWS SigV4 auth.
+"""
+
+from typing import TYPE_CHECKING, Any, List, Optional
+
+from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import (
+    AmazonAnthropicClaudeConfig,
+)
+from litellm.types.llms.openai import AllMessageValues
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = Any
+
+MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
+
+
+class AmazonMantleConfig(AmazonAnthropicClaudeConfig):
+    """
+    Config for the bedrock-mantle endpoint (Claude Mythos Preview).
+
+    Uses the Anthropic Messages API format with AWS SigV4 auth, but at a
+    different endpoint from bedrock-runtime. Model ID goes in the request body.
+
+    Usage: model="bedrock/mantle/anthropic.claude-mythos-preview"
+    """
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        region = self._get_aws_region_name(optional_params=optional_params, model=model)
+        return MANTLE_ENDPOINT_TEMPLATE.format(region=region)
+
+    def transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        # Strip the "mantle/" routing prefix to get the real model ID
+        model_id = model.replace("mantle/", "", 1)
+
+        request = self._build_bedrock_anthropic_request_base(
+            model=model_id,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        # The parent strips "model" from the body (Invoke API puts it in URL).
+        # The mantle endpoint (Messages API) requires "model" in the body.
+        request["model"] = model_id
+        return request
+
+    async def async_transform_request(
+        self,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        headers: dict,
+    ) -> dict:
+        model_id = model.replace("mantle/", "", 1)
+
+        request = self._build_bedrock_anthropic_request_base(
+            model=model_id,
+            messages=messages,
+            optional_params=optional_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+        await self._async_convert_document_url_sources_to_base64(request)
+        request["model"] = model_id
+        return request

litellm/llms/bedrock/common_utils.py

@@ -696,6 +696,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
         "agentcore",
         "async_invoke",
         "openai",
+        "mantle",
     ]:
         """
         Get the bedrock route for the given model.
@@ -710,6 +711,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
                 "agentcore",
                 "async_invoke",
                 "openai",
+                "mantle",
             ],
         ] = {
             "invoke/": "invoke",
@@ -719,6 +721,7 @@ class BedrockModelInfo(BaseLLMModelInfo):
             "agentcore/": "agentcore",
             "async_invoke/": "async_invoke",
             "openai/": "openai",
+            "mantle/": "mantle",
         }
 
         # Check explicit routes first
@@ -770,6 +773,13 @@ class BedrockModelInfo(BaseLLMModelInfo):
         """
         return "agentcore/" in model
 
+    @staticmethod
+    def _explicit_mantle_route(model: str) -> bool:
+        """
+        Check if the model is an explicit mantle route (bedrock-mantle endpoint).
+        """
+        return "mantle/" in model
+
     @staticmethod
     def _explicit_converse_like_route(model: str) -> bool:
         """
@@ -809,6 +819,16 @@ class BedrockModelInfo(BaseLLMModelInfo):
         if BedrockModelInfo._explicit_converse_route(model):
             return None
 
+        #########################################################
+        # Mantle route uses the bedrock-mantle endpoint (not bedrock-runtime)
+        #########################################################
+        if BedrockModelInfo._explicit_mantle_route(model):
+            from litellm.llms.bedrock.messages.mantle_transformation import (
+                AmazonMantleMessagesConfig,
+            )
+
+            return AmazonMantleMessagesConfig()
+
         #########################################################
         # This goes through litellm.AmazonAnthropicClaude3MessagesConfig()
         # Since bedrock Invoke supports Native Anthropic Messages API
@@ -855,6 +875,12 @@ def get_bedrock_chat_config(model: str):
         )
 
         return AmazonAgentCoreConfig()
+    elif bedrock_route == "mantle":
+        from litellm.llms.bedrock.chat.mantle.transformation import (
+            AmazonMantleConfig,
+        )
+
+        return AmazonMantleConfig()
 
     # Handle provider-specific configs
     if bedrock_invoke_provider == "amazon":

litellm/llms/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.py

@@ -34,6 +34,7 @@ from litellm.llms.bedrock.common_utils import (
     remove_custom_field_from_tools,
 )
 from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER
+from litellm.types.llms.bedrock import BedrockInvokeAnthropicMessagesRequest
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.router import GenericLiteLLMParams
 from litellm.types.utils import GenericStreamingChunk
@@ -59,6 +60,10 @@ class AmazonAnthropicClaudeMessagesConfig(
 
     DEFAULT_BEDROCK_ANTHROPIC_API_VERSION = "bedrock-2023-05-31"
 
+    BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS = frozenset(
+        BedrockInvokeAnthropicMessagesRequest.__annotations__.keys()
+    )
+
     def __init__(self, **kwargs):
         BaseAnthropicMessagesConfig.__init__(self, **kwargs)
         AmazonInvokeConfig.__init__(self, **kwargs)
@@ -500,10 +505,6 @@ class AmazonAnthropicClaudeMessagesConfig(
                 anthropic_messages_request=anthropic_messages_request,
             )
 
-        # 5b. Strip `output_config` — Bedrock Invoke doesn't support it
-        # Fixes: https://github.com/BerriAI/litellm/issues/22797
-        anthropic_messages_request.pop("output_config", None)
-
         # 5a. Remove `custom` field from tools (Bedrock doesn't support it)
         # Claude Code sends `custom: {defer_loading: true}` on tool definitions,
         # which causes Bedrock to reject the request with "Extra inputs are not permitted"
@@ -550,14 +551,43 @@ class AmazonAnthropicClaudeMessagesConfig(
         if "tool-search-tool-2025-10-19" in beta_set:
             beta_set.add("tool-examples-2025-10-29")
 
-        filtered_auto_betas = filter_and_transform_beta_headers(
-            beta_headers=list(beta_set - user_beta_set),
-            provider="bedrock",
+        filtered_betas = sorted(
+            filter_and_transform_beta_headers(
+                beta_headers=list(beta_set),
+                provider="bedrock",
+            )
         )
-        filtered_betas = sorted(user_beta_set.union(set(filtered_auto_betas)))
+
+        dropped_user_betas = sorted(
+            b
+            for b in user_beta_set
+            if not filter_and_transform_beta_headers([b], provider="bedrock")
+        )
+        if dropped_user_betas:
+            verbose_logger.warning(
+                "Bedrock Invoke: dropping unsupported anthropic-beta values "
+                "from client headers: %s. Bedrock has no mapping entry for "
+                "these; forwarding them would cause a 400.",
+                dropped_user_betas,
+            )
+
         if filtered_betas:
             anthropic_messages_request["anthropic_beta"] = filtered_betas
 
+        # 7. Final safety net: filter top-level fields to the Bedrock Invoke allowlist.
+        # Catches Anthropic-only extensions (context_management, output_config, speed,
+        # mcp_servers, ...) and any future additions Claude Code may start sending.
+        allowed = self.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS
+        stripped = sorted(k for k in anthropic_messages_request if k not in allowed)
+        if stripped:
+            verbose_logger.debug(
+                "Bedrock Invoke: stripping unsupported top-level request fields: %s",
+                stripped,
+            )
+        anthropic_messages_request = {
+            k: v for k, v in anthropic_messages_request.items() if k in allowed
+        }
+
         return anthropic_messages_request
 
     def get_async_streaming_response_iterator(

litellm/llms/bedrock/messages/mantle_transformation.py

@@ -0,0 +1,69 @@
+"""
+Transformation for Bedrock Mantle (Claude Mythos Preview) - /messages endpoint
+
+Inherits all Messages API request/response transformations from
+AmazonAnthropicClaudeMessagesConfig. Overrides only the URL and model-prefix
+stripping that are specific to the bedrock-mantle endpoint.
+"""
+
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude3_transformation import (
+    AmazonAnthropicClaudeMessagesConfig,
+)
+from litellm.types.router import GenericLiteLLMParams
+
+if TYPE_CHECKING:
+    from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
+
+    LiteLLMLoggingObj = _LiteLLMLoggingObj
+else:
+    LiteLLMLoggingObj = Any
+
+MANTLE_ENDPOINT_TEMPLATE = "https://bedrock-mantle.{region}.api.aws/v1/messages"
+
+
+class AmazonMantleMessagesConfig(AmazonAnthropicClaudeMessagesConfig):
+    """
+    Config for the bedrock-mantle /messages endpoint (Claude Mythos Preview).
+
+    The mantle endpoint uses the Anthropic Messages API format and requires the
+    model ID in the request body (unlike Bedrock Invoke which puts it in the URL).
+    """
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        region = self._get_aws_region_name(optional_params=optional_params, model=model)
+        return MANTLE_ENDPOINT_TEMPLATE.format(region=region)
+
+    def transform_anthropic_messages_request(
+        self,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        litellm_params: GenericLiteLLMParams,
+        headers: dict,
+    ) -> Dict:
+        # Strip "mantle/" routing prefix to get the real model ID
+        model_id = model.replace("mantle/", "", 1)
+
+        request = super().transform_anthropic_messages_request(
+            model=model_id,
+            messages=messages,
+            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+            litellm_params=litellm_params,
+            headers=headers,
+        )
+
+        # Parent (AmazonAnthropicClaudeMessagesConfig) removes "model" from the
+        # body (Bedrock Invoke puts model in the URL). The mantle endpoint
+        # (Messages API) requires "model" in the request body.
+        request["model"] = model_id
+        return request

litellm/llms/custom_httpx/llm_http_handler.py

@@ -78,6 +78,10 @@ from litellm.types.containers.main import (
     DeleteContainerResult,
 )
 from litellm.types.files import TwoStepFileUploadConfig
+from litellm.types.integrations.custom_logger import (
+    AgenticLoopPlan,
+    AgenticLoopRequestPatch,
+)
 from litellm.types.llms.anthropic_messages.anthropic_response import (
     AnthropicMessagesResponse,
 )
@@ -2047,7 +2051,23 @@ class BaseLLMHTTPHandler:
                 request_body=request_body,
                 litellm_logging_obj=logging_obj,
             )
-            initial_response = completion_stream
+
+            from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+                AgenticAnthropicStreamingIterator,
+            )
+
+            initial_response = AgenticAnthropicStreamingIterator(
+                completion_stream=completion_stream,
+                http_handler=self,
+                model=model,
+                messages=messages,
+                anthropic_messages_provider_config=anthropic_messages_provider_config,
+                anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+                logging_obj=logging_obj,
+                custom_llm_provider=custom_llm_provider,
+                kwargs=kwargs,
+            )
+            return initial_response
         else:
             initial_response = anthropic_messages_provider_config.transform_anthropic_messages_response(
                 model=model,
@@ -2055,7 +2075,7 @@ class BaseLLMHTTPHandler:
                 logging_obj=logging_obj,
             )
 
-        # Call agentic completion hooks
+        # Call agentic completion hooks (non-streaming path only)
         final_response = await self._call_agentic_completion_hooks(
             response=initial_response,
             model=model,
@@ -2063,7 +2083,7 @@ class BaseLLMHTTPHandler:
             anthropic_messages_provider_config=anthropic_messages_provider_config,
             anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
             logging_obj=logging_obj,
-            stream=stream or False,
+            stream=False,
             custom_llm_provider=custom_llm_provider,
             kwargs=kwargs,
         )
@@ -4516,6 +4536,167 @@ class BaseLLMHTTPHandler:
             return stream, data
         return stream, data
 
+    @staticmethod
+    def _get_agentic_loop_settings(kwargs: Dict) -> Tuple[int, int, List[str]]:
+        depth = int(kwargs.get("_agentic_loop_depth", 0) or 0)
+        max_loops = int(kwargs.get("max_agentic_loops", 3) or 3)
+        fingerprints = list(kwargs.get("_agentic_loop_fingerprints", []) or [])
+        return depth, max(max_loops, 1), fingerprints
+
+    @staticmethod
+    def _check_agentic_loop_safety(
+        tool_calls: Any,
+        fingerprints: List[str],
+        depth: int,
+        max_loops: int,
+        model: str,
+    ) -> str:
+        """
+        Evaluate agentic-loop safety guards (fingerprint cycle / max depth).
+
+        Raises ValueError on abort.  Returns the current fingerprint on success.
+
+        These checks must not be swallowed by the per-callback ``except Exception``
+        block that wraps callback dispatch — they are bounded-loop / cycle-break
+        safety rails and must abort the agentic dispatch when they trip.
+        """
+        fingerprint = BaseLLMHTTPHandler._fingerprint_agentic_tools(tool_calls)
+        if fingerprint in fingerprints:
+            raise ValueError(
+                "Agentic loop detected repeated tool-call fingerprint; aborting rerun"
+            )
+        if depth >= max_loops:
+            raise ValueError(
+                f"Exceeded max_agentic_loops={max_loops} for model={model}"
+            )
+        return fingerprint
+
+    @staticmethod
+    def _fingerprint_agentic_tools(tools: Dict) -> str:
+        try:
+            return json.dumps(tools, sort_keys=True, default=str)
+        except Exception:
+            return str(tools)
+
+    async def _execute_anthropic_agentic_plan(
+        self,
+        plan: AgenticLoopPlan,
+        model: str,
+        messages: List[Dict],
+        anthropic_messages_optional_request_params: Dict,
+        logging_obj: "LiteLLMLoggingObj",
+        kwargs: Dict,
+        depth: int,
+        max_loops: int,
+        fingerprints: List[str],
+        fingerprint: str,
+        stream: bool = False,
+    ) -> Any:
+        from litellm.anthropic_interface import messages as anthropic_messages
+
+        patch = plan.request_patch or AgenticLoopRequestPatch()
+        if patch.messages is None:
+            raise ValueError("Agentic loop plan missing patched messages")
+
+        full_model_name = model
+        if logging_obj is not None:
+            agentic_params = logging_obj.model_call_details.get(
+                "agentic_loop_params", {}
+            )
+            full_model_name = cast(str, agentic_params.get("model", model))
+
+        optional_params = dict(anthropic_messages_optional_request_params)
+        optional_params.update(patch.optional_params)
+        if patch.tools is not None:
+            optional_params["tools"] = patch.tools
+
+        max_tokens = patch.max_tokens
+        if max_tokens is None:
+            max_tokens = cast(Optional[int], optional_params.pop("max_tokens", None))
+        else:
+            optional_params.pop("max_tokens", None)
+        if max_tokens is None:
+            max_tokens = cast(int, kwargs.get("max_tokens", 1024))
+
+        internal_keys = {"litellm_logging_obj"}
+        kwargs_for_followup = {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_websearch_interception")
+            and not k.startswith("_compression_interception")
+            and k not in internal_keys
+            and k not in optional_params
+        }
+        kwargs_for_followup.update(patch.kwargs)
+        kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+        kwargs_for_followup["max_agentic_loops"] = max_loops
+        kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+
+        return await anthropic_messages.acreate(
+            **{
+                "max_tokens": max_tokens,
+                "messages": patch.messages,
+                "model": patch.model or full_model_name,
+                "stream": stream,
+                **optional_params,
+                **kwargs_for_followup,
+            }
+        )
+
+    async def _execute_chat_completion_agentic_plan(
+        self,
+        plan: AgenticLoopPlan,
+        model: str,
+        messages: List[Dict],
+        optional_params: Dict,
+        kwargs: Dict,
+        custom_llm_provider: str,
+        depth: int,
+        max_loops: int,
+        fingerprints: List[str],
+        fingerprint: str,
+    ) -> Any:
+        patch = plan.request_patch or AgenticLoopRequestPatch()
+        if patch.messages is None:
+            raise ValueError("Agentic loop plan missing patched messages")
+
+        full_model_name = patch.model or model
+        if "/" not in full_model_name:
+            full_model_name = f"{custom_llm_provider}/{full_model_name}"
+
+        optional_params_for_followup = dict(optional_params)
+        optional_params_for_followup.update(patch.optional_params)
+        if patch.tools is not None:
+            optional_params_for_followup["tools"] = patch.tools
+
+        internal_params = {
+            "_websearch_interception",
+            "acompletion",
+            "litellm_logging_obj",
+            "custom_llm_provider",
+            "model_alias_map",
+            "stream_response",
+            "custom_prompt_dict",
+        }
+        kwargs_for_followup = {
+            k: v
+            for k, v in kwargs.items()
+            if not k.startswith("_websearch_interception")
+            and not k.startswith("_compression_interception")
+            and k not in internal_params
+        }
+        kwargs_for_followup.update(patch.kwargs)
+        kwargs_for_followup["_agentic_loop_depth"] = depth + 1
+        kwargs_for_followup["max_agentic_loops"] = max_loops
+        kwargs_for_followup["_agentic_loop_fingerprints"] = fingerprints + [fingerprint]
+
+        return await litellm.acompletion(
+            model=full_model_name,
+            messages=patch.messages,
+            **optional_params_for_followup,
+            **kwargs_for_followup,
+        )
+
     async def _call_agentic_completion_hooks(
         self,
         response: Any,
@@ -4541,45 +4722,111 @@ class BaseLLMHTTPHandler:
 
         callbacks = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
         tools = anthropic_messages_optional_request_params.get("tools", [])
+        depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
 
         for callback in callbacks:
+            if not isinstance(callback, CustomLogger):
+                continue
+
+            should_run: bool = False
+            tool_calls: Any = None
             try:
-                if isinstance(callback, CustomLogger):
-                    # First: Check if agentic loop should run
-                    (
-                        should_run,
-                        tool_calls,
-                    ) = await callback.async_should_run_agentic_loop(
-                        response=response,
+                # First: Check if agentic loop should run.  Wrap in try/except
+                # to shield from buggy user callbacks — a callback crash should
+                # not abort the whole request.
+                (
+                    should_run,
+                    tool_calls,
+                ) = await callback.async_should_run_agentic_loop(
+                    response=response,
+                    model=model,
+                    messages=messages,
+                    tools=tools,
+                    stream=stream,
+                    custom_llm_provider=custom_llm_provider,
+                    kwargs=kwargs,
+                )
+            except Exception as e:
+                _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
+                verbose_logger.exception(
+                    "LiteLLM.AgenticHookError: Exception in "
+                    "async_should_run_agentic_loop [call_id=%s model=%s]: %s",
+                    _call_id,
+                    model,
+                    str(e),
+                )
+                continue
+
+            if not should_run:
+                continue
+
+            # Safety guards must run OUTSIDE the callback try/except — they are
+            # bounded-loop / cycle-break rails that must propagate to the caller.
+            fingerprint = self._check_agentic_loop_safety(
+                tool_calls=tool_calls,
+                fingerprints=fingerprints,
+                depth=depth,
+                max_loops=max_loops,
+                model=model,
+            )
+
+            try:
+                kwargs_with_provider = kwargs.copy() if kwargs else {}
+                kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
+                build_plan_overridden = (
+                    callback.__class__.async_build_agentic_loop_plan
+                    is not CustomLogger.async_build_agentic_loop_plan
+                )
+                if not build_plan_overridden:
+                    return await callback.async_run_agentic_loop(
+                        tools=tool_calls,
                         model=model,
                         messages=messages,
-                        tools=tools,
+                        response=response,
+                        anthropic_messages_provider_config=anthropic_messages_provider_config,
+                        anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+                        logging_obj=logging_obj,
                         stream=stream,
-                        custom_llm_provider=custom_llm_provider,
-                        kwargs=kwargs,
+                        kwargs=kwargs_with_provider,
                     )
 
-                    if should_run:
-                        # Second: Execute agentic loop
-                        # Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
-                        kwargs_with_provider = kwargs.copy() if kwargs else {}
-                        kwargs_with_provider["custom_llm_provider"] = (
-                            custom_llm_provider
-                        )
-                        agentic_response = await callback.async_run_agentic_loop(
-                            tools=tool_calls,
-                            model=model,
-                            messages=messages,
-                            response=response,
-                            anthropic_messages_provider_config=anthropic_messages_provider_config,
-                            anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
-                            logging_obj=logging_obj,
-                            stream=stream,
-                            kwargs=kwargs_with_provider,
-                        )
-                        # First hook that runs agentic loop wins
-                        return agentic_response
+                plan = await callback.async_build_agentic_loop_plan(
+                    tools=tool_calls,
+                    model=model,
+                    messages=messages,
+                    response=response,
+                    anthropic_messages_provider_config=anthropic_messages_provider_config,
+                    anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+                    logging_obj=logging_obj,
+                    stream=stream,
+                    kwargs=kwargs_with_provider,
+                )
 
+                if plan.response_override is not None:
+                    return plan.response_override
+                if plan.terminate:
+                    verbose_logger.debug(
+                        "Agentic loop terminated by callback=%s reason=%s",
+                        callback.__class__.__name__,
+                        plan.stop_reason,
+                    )
+                    return response
+                if not plan.run_agentic_loop:
+                    continue
+
+                return await self._execute_anthropic_agentic_plan(
+                    plan=plan,
+                    model=model,
+                    messages=messages,
+                    anthropic_messages_optional_request_params=anthropic_messages_optional_request_params,
+                    logging_obj=logging_obj,
+                    kwargs=kwargs_with_provider,
+                    depth=depth,
+                    max_loops=max_loops,
+                    fingerprints=fingerprints,
+                    fingerprint=fingerprint,
+                    stream=stream,
+                )
             except Exception as e:
                 _call_id = getattr(logging_obj, "litellm_call_id", "unknown")
                 verbose_logger.exception(
@@ -4653,52 +4900,104 @@ class BaseLLMHTTPHandler:
 
         callbacks = litellm.callbacks + (logging_obj.dynamic_success_callbacks or [])
         tools = optional_params.get("tools", [])
+        depth, max_loops, fingerprints = self._get_agentic_loop_settings(kwargs=kwargs)
 
         for callback in callbacks:
-            try:
-                if isinstance(callback, CustomLogger):
-                    # Check if callback has the chat completion agentic loop method
-                    if not hasattr(
-                        callback, "async_should_run_chat_completion_agentic_loop"
-                    ):
-                        continue
+            if not isinstance(callback, CustomLogger):
+                continue
+            if not hasattr(callback, "async_should_run_chat_completion_agentic_loop"):
+                continue
 
-                    # First: Check if agentic loop should run
-                    (
-                        should_run,
-                        tool_calls,
-                    ) = await callback.async_should_run_chat_completion_agentic_loop(
-                        response=response,
+            should_run: bool = False
+            tool_calls: Any = None
+            try:
+                (
+                    should_run,
+                    tool_calls,
+                ) = await callback.async_should_run_chat_completion_agentic_loop(
+                    response=response,
+                    model=model,
+                    messages=messages,
+                    tools=tools,
+                    stream=stream,
+                    custom_llm_provider=custom_llm_provider,
+                    kwargs=kwargs,
+                )
+            except Exception as e:
+                verbose_logger.exception(
+                    "LiteLLM.AgenticHookError: Exception in "
+                    "async_should_run_chat_completion_agentic_loop: %s",
+                    str(e),
+                )
+                continue
+
+            if not should_run:
+                continue
+
+            # Safety guards must run OUTSIDE the callback try/except — they are
+            # bounded-loop / cycle-break rails that must propagate to the caller.
+            fingerprint = self._check_agentic_loop_safety(
+                tool_calls=tool_calls,
+                fingerprints=fingerprints,
+                depth=depth,
+                max_loops=max_loops,
+                model=model,
+            )
+
+            try:
+                kwargs_with_provider = kwargs.copy() if kwargs else {}
+                kwargs_with_provider["custom_llm_provider"] = custom_llm_provider
+                build_plan_overridden = (
+                    callback.__class__.async_build_chat_completion_agentic_loop_plan
+                    is not CustomLogger.async_build_chat_completion_agentic_loop_plan
+                )
+                if not build_plan_overridden:
+                    return await callback.async_run_chat_completion_agentic_loop(
+                        tools=tool_calls,
                         model=model,
                         messages=messages,
-                        tools=tools,
+                        response=response,
+                        optional_params=optional_params,
+                        logging_obj=logging_obj,
                         stream=stream,
-                        custom_llm_provider=custom_llm_provider,
-                        kwargs=kwargs,
+                        kwargs=kwargs_with_provider,
                     )
 
-                    if should_run:
-                        # Second: Execute agentic loop
-                        # Add custom_llm_provider to kwargs so the agentic loop can reconstruct the full model name
-                        kwargs_with_provider = kwargs.copy() if kwargs else {}
-                        kwargs_with_provider["custom_llm_provider"] = (
-                            custom_llm_provider
-                        )
-                        agentic_response = (
-                            await callback.async_run_chat_completion_agentic_loop(
-                                tools=tool_calls,
-                                model=model,
-                                messages=messages,
-                                response=response,
-                                optional_params=optional_params,
-                                logging_obj=logging_obj,
-                                stream=stream,
-                                kwargs=kwargs_with_provider,
-                            )
-                        )
-                        # First hook that runs agentic loop wins
-                        return agentic_response
+                plan = await callback.async_build_chat_completion_agentic_loop_plan(
+                    tools=tool_calls,
+                    model=model,
+                    messages=messages,
+                    response=response,
+                    optional_params=optional_params,
+                    logging_obj=logging_obj,
+                    stream=stream,
+                    kwargs=kwargs_with_provider,
+                )
 
+                if plan.response_override is not None:
+                    return plan.response_override
+                if plan.terminate:
+                    verbose_logger.debug(
+                        "Agentic chat loop terminated by callback=%s reason=%s",
+                        callback.__class__.__name__,
+                        plan.stop_reason,
+                    )
+                    return response
+                if not plan.run_agentic_loop:
+                    continue
+
+                return await self._execute_chat_completion_agentic_plan(
+                    plan=plan,
+                    model=model,
+                    messages=messages,
+                    optional_params=optional_params,
+                    kwargs=kwargs_with_provider,
+                    custom_llm_provider=custom_llm_provider,
+                    depth=depth,
+                    max_loops=max_loops,
+                    fingerprints=fingerprints,
+                    fingerprint=fingerprint,
+                )
             except Exception as e:
                 verbose_logger.exception(
                     f"LiteLLM.AgenticHookError: Exception in chat completion agentic hooks: {str(e)}"

litellm/llms/github_copilot/authenticator.py

@@ -294,9 +294,7 @@ class Authenticator:
         access_token_url = os.getenv(
             "GITHUB_COPILOT_ACCESS_TOKEN_URL", DEFAULT_GITHUB_ACCESS_TOKEN_URL
         )
-        client_id = os.getenv(
-            "GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID
-        )
+        client_id = os.getenv("GITHUB_COPILOT_CLIENT_ID", DEFAULT_GITHUB_CLIENT_ID)
 
         for attempt in range(max_attempts):
             try:

litellm/llms/openai/chat/guardrail_translation/handler.py

@@ -48,6 +48,17 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
     Methods can be overridden to customize behavior for different message formats.
     """
 
+    def get_structured_messages(self, data: dict) -> Optional[List[AllMessageValues]]:
+        """
+        Convert chat completions request data to OpenAI-spec structured messages.
+
+        Messages are already in OpenAI format, so this is a simple extraction.
+        """
+        messages = data.get("messages")
+        if messages is None:
+            return None
+        return cast(List[AllMessageValues], messages)
+
     async def process_input_messages(
         self,
         data: dict,
@@ -68,9 +79,6 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
         tool_calls_to_check: List[ChatCompletionToolParam] = []
         text_task_mappings: List[Tuple[int, Optional[int]]] = []
         tool_call_task_mappings: List[Tuple[int, int]] = []
-        # text_task_mappings: Track (message_index, content_index) for each text
-        # content_index is None for string content, int for list content
-        # tool_call_task_mappings: Track (message_index, tool_call_index) for each tool call
 
         # Step 1: Extract all text content, images, and tool calls
         for msg_idx, message in enumerate(messages):
@@ -92,12 +100,12 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
                 inputs["images"] = images_to_check
             if tool_calls_to_check:
                 inputs["tool_calls"] = tool_calls_to_check  # type: ignore
-            if messages:
-                msg_list = cast(List[AllMessageValues], messages)
+            structured_messages = self.get_structured_messages(data)
+            if structured_messages:
                 inputs["structured_messages"] = (
-                    openai_messages_without_system(msg_list)
+                    openai_messages_without_system(structured_messages)
                     if skip_system
-                    else msg_list
+                    else structured_messages
                 )
             # Pass tools (function definitions) to the guardrail
             tools = data.get("tools")

litellm/llms/openai/responses/guardrail_translation/handler.py

@@ -43,6 +43,7 @@ from litellm.responses.litellm_completion_transformation.transformation import (
     LiteLLMCompletionResponsesConfig,
 )
 from litellm.types.llms.openai import (
+    AllMessageValues,
     ChatCompletionToolCallChunk,
     ChatCompletionToolParam,
 )
@@ -70,6 +71,24 @@ class OpenAIResponsesHandler(BaseTranslation):
     Methods can be overridden to customize behavior for different message formats.
     """
 
+    def get_structured_messages(self, data: dict) -> Optional[List[AllMessageValues]]:
+        """
+        Convert Responses API request data to OpenAI-spec structured messages.
+
+        Transforms `input` (string or ResponseInputParam) and optional
+        `instructions` into chat completion messages.
+        """
+        input_data = data.get("input")
+        if input_data is None:
+            return None
+        messages = (
+            LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
+                input=input_data,
+                responses_api_request=data,
+            )
+        )
+        return cast(List[AllMessageValues], messages) if messages else None
+
     async def process_input_messages(
         self,
         data: dict,
@@ -86,12 +105,7 @@ class OpenAIResponsesHandler(BaseTranslation):
         if input_data is None:
             return data
 
-        structured_messages = (
-            LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages(
-                input=input_data,
-                responses_api_request=data,
-            )
-        )
+        structured_messages = self.get_structured_messages(data)
 
         # Handle simple string input
         if isinstance(input_data, str):

litellm/llms/scaleway/audio_transcription/transformation.py

@@ -0,0 +1,158 @@
+"""
+Support for Scaleway's OpenAI-compatible `/v1/audio/transcriptions` endpoint.
+
+API reference: https://www.scaleway.com/en/developers/api/generative-apis/#path-audio-create-an-audio-transcription
+"""
+
+from typing import List, Optional, Union
+
+import httpx
+
+from litellm.litellm_core_utils.audio_utils.utils import process_audio_file
+from litellm.llms.base_llm.audio_transcription.transformation import (
+    AudioTranscriptionRequestData,
+    BaseAudioTranscriptionConfig,
+)
+from litellm.llms.base_llm.chat.transformation import BaseLLMException
+from litellm.secret_managers.main import get_secret_str
+from litellm.types.llms.openai import (
+    AllMessageValues,
+    OpenAIAudioTranscriptionOptionalParams,
+)
+from litellm.types.utils import FileTypes, TranscriptionResponse
+
+
+class ScalewayAudioTranscriptionException(BaseLLMException):
+    pass
+
+
+class ScalewayAudioTranscriptionConfig(BaseAudioTranscriptionConfig):
+    def get_supported_openai_params(
+        self, model: str
+    ) -> List[OpenAIAudioTranscriptionOptionalParams]:
+        return [
+            "language",
+            "prompt",
+            "response_format",
+            "temperature",
+            "timestamp_granularities",
+        ]
+
+    def map_openai_params(
+        self,
+        non_default_params: dict,
+        optional_params: dict,
+        model: str,
+        drop_params: bool,
+    ) -> dict:
+        supported_params = self.get_supported_openai_params(model)
+        for k, v in non_default_params.items():
+            if k in supported_params:
+                optional_params[k] = v
+        return optional_params
+
+    def get_complete_url(
+        self,
+        api_base: Optional[str],
+        api_key: Optional[str],
+        model: str,
+        optional_params: dict,
+        litellm_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        api_base = (
+            "https://api.scaleway.ai/v1" if api_base is None else api_base.rstrip("/")
+        )
+        return f"{api_base}/audio/transcriptions"
+
+    def get_error_class(
+        self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
+    ) -> BaseLLMException:
+        return ScalewayAudioTranscriptionException(
+            message=error_message,
+            status_code=status_code,
+            headers=headers,
+        )
+
+    def validate_environment(
+        self,
+        headers: dict,
+        model: str,
+        messages: List[AllMessageValues],
+        optional_params: dict,
+        litellm_params: dict,
+        api_key: Optional[str] = None,
+        api_base: Optional[str] = None,
+    ) -> dict:
+        if api_key is None:
+            api_key = get_secret_str("SCW_SECRET_KEY")
+
+        if not api_key:
+            raise ScalewayAudioTranscriptionException(
+                message=(
+                    "Scaleway API key not found. Pass `api_key=...` or set the "
+                    "SCW_SECRET_KEY environment variable."
+                ),
+                status_code=401,
+                headers={},
+            )
+
+        default_headers = {
+            "Authorization": f"Bearer {api_key}",
+            "accept": "application/json",
+        }
+        default_headers.update(headers or {})
+        return default_headers
+
+    def transform_audio_transcription_request(
+        self,
+        model: str,
+        audio_file: FileTypes,
+        optional_params: dict,
+        litellm_params: dict,
+    ) -> AudioTranscriptionRequestData:
+        processed_audio = process_audio_file(audio_file)
+
+        form_fields: dict = {"model": model}
+        for key in self.get_supported_openai_params(model):
+            value = optional_params.get(key)
+            if value is not None:
+                form_fields[key] = value
+
+        files = {
+            "file": (
+                processed_audio.filename,
+                processed_audio.file_content,
+                processed_audio.content_type,
+            )
+        }
+
+        return AudioTranscriptionRequestData(data=form_fields, files=files)
+
+    def transform_audio_transcription_response(
+        self,
+        raw_response: httpx.Response,
+    ) -> TranscriptionResponse:
+        content_type = (raw_response.headers.get("content-type") or "").lower()
+        if "application/json" not in content_type:
+            return TranscriptionResponse(text=raw_response.text)
+
+        try:
+            response_json = raw_response.json()
+        except Exception:
+            raise ScalewayAudioTranscriptionException(
+                message=raw_response.text,
+                status_code=raw_response.status_code,
+                headers=raw_response.headers,
+            )
+
+        text = response_json.get("text") or ""
+        response = TranscriptionResponse(text=text)
+
+        if "segments" in response_json:
+            response["segments"] = response_json["segments"]
+        if "language" in response_json:
+            response["language"] = response_json["language"]
+
+        response._hidden_params = response_json
+        return response

litellm/passthrough/utils.py

@@ -79,7 +79,9 @@ class BasePassthroughUtils:
         for header_name, header_value in request_headers.items():
             if header_name.lower().startswith(PASS_THROUGH_HEADER_PREFIX):
                 # Strip the 'x-pass-' prefix and normalize to lowercase
-                actual_header_name = header_name[len(PASS_THROUGH_HEADER_PREFIX) :].lower()
+                actual_header_name = header_name[
+                    len(PASS_THROUGH_HEADER_PREFIX) :
+                ].lower()
                 if actual_header_name in _PASS_THROUGH_PROTECTED_HEADERS or any(
                     actual_header_name.startswith(p)
                     for p in _PASS_THROUGH_PROTECTED_HEADER_PREFIXES

litellm/provider_endpoints_support_backup.json

@@ -1950,7 +1950,7 @@
         "responses": true,
         "embeddings": false,
         "image_generations": false,
-        "audio_transcriptions": false,
+        "audio_transcriptions": true,
         "audio_speech": false,
         "moderations": false,
         "batches": false,

litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py

@@ -323,6 +323,14 @@ async def authorize_with_server(
         )
 
     parsed = urlparse(redirect_uri)
+    if parsed.scheme not in ("http", "https"):
+        raise HTTPException(
+            status_code=400,
+            detail={
+                "error": "invalid_redirect_uri",
+                "message": "redirect_uri must use http or https scheme",
+            },
+        )
     base_url = urlunparse(parsed._replace(query=""))
     request_base_url = get_request_base_url(request)
     encoded_state = encode_state_with_base_url(

litellm/proxy/_new_secret_config.yaml

@@ -22,11 +22,21 @@ model_list:
       output_cost_per_token: 10         # 100x standard ($10.00/1M = $0.00001)
 
   # Anthropic model for /v1/messages test — 100x custom pricing
-  - model_name: "claude-sonnet-4-20250514"
+  - model_name: "claude-sonnet-4-6"
     litellm_params:
-      model: anthropic/claude-sonnet-4-20250514
+      model: anthropic/claude-sonnet-4-6
       api_key: os.environ/ANTHROPIC_API_KEY
     model_info:
       id: claude-sonnet-4-custom-pricing
       input_cost_per_token: 0.0003         # 100x standard ($0.000003)
-      output_cost_per_token: 0.0015         # 100x standard ($0.000015)
+      output_cost_per_token: 0.0015         # 100x standard ($0.000015)
+  - model_name: my-auto
+    litellm_params:
+      model: auto_router/complexity_router
+      complexity_router_config:
+        tiers:
+          SIMPLE: "gpt-4.1-mini"
+          COMPLEX: claude-sonnet-4-6
+        tier_boundaries:
+          simple_medium: 0.30
+      complexity_router_default_model: small-model

litellm/proxy/auth/auth_checks.py

@@ -626,11 +626,17 @@ async def common_checks(  # noqa: PLR0915
             and user_object.max_budget is not None
         ):
             user_budget = user_object.max_budget
-            if user_budget < user_object.spend:
+            from litellm.proxy.proxy_server import get_current_spend
+
+            user_spend = await get_current_spend(
+                counter_key=f"spend:user:{user_object.user_id}",
+                fallback_spend=user_object.spend or 0.0,
+            )
+            if user_spend >= user_budget:
                 raise litellm.BudgetExceededError(
-                    current_cost=user_object.spend,
+                    current_cost=user_spend,
                     max_budget=user_budget,
-                    message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_object.spend}, Budget={user_budget}",
+                    message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
                 )
 
         ## 4.2 check team member budget, if team key
@@ -3126,9 +3132,7 @@ async def _virtual_key_max_budget_alert_check(
         alert_email_config: Optional[Dict[str, List[str]]] = (
             _merge_budget_alert_email_configs(
                 global_cfg=litellm.default_key_max_budget_alert_emails,
-                per_key_cfg=(valid_token.metadata or {}).get(
-                    "max_budget_alert_emails"
-                ),
+                per_key_cfg=(valid_token.metadata or {}).get("max_budget_alert_emails"),
             )
         )
 
@@ -3138,7 +3142,9 @@ async def _virtual_key_max_budget_alert_check(
                 (int(k) for k in alert_email_config if k.isdigit()),
                 default=None,
             )
-            if min_pct is None or valid_token.spend < valid_token.max_budget * (min_pct / 100.0):
+            if min_pct is None or valid_token.spend < valid_token.max_budget * (
+                min_pct / 100.0
+            ):
                 return
 
             call_info = CallInfo(
@@ -3164,8 +3170,7 @@ async def _virtual_key_max_budget_alert_check(
         else:
             # Old path: existing single 80% threshold — completely unchanged
             alert_threshold = (
-                valid_token.max_budget
-                * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
+                valid_token.max_budget * EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE
             )
 
             if (
@@ -3666,12 +3671,20 @@ async def _organization_max_budget_check(
     if org_max_budget is None or org_max_budget <= 0:
         return
 
+    # Read spend from cross-pod counter (Redis-first) or cached object (fallback)
+    from litellm.proxy.proxy_server import get_current_spend
+
+    org_spend = await get_current_spend(
+        counter_key=f"spend:org:{org_id}",
+        fallback_spend=org_table.spend or 0.0,
+    )
+
     # Check if organization spend exceeds max budget
-    if org_table.spend >= org_max_budget:
+    if org_spend >= org_max_budget:
         # Trigger budget alert
         call_info = CallInfo(
             token=valid_token.token,
-            spend=org_table.spend,
+            spend=org_spend,
             max_budget=org_max_budget,
             user_id=valid_token.user_id,
             team_id=valid_token.team_id,
@@ -3687,9 +3700,9 @@ async def _organization_max_budget_check(
         )
 
         raise litellm.BudgetExceededError(
-            current_cost=org_table.spend,
+            current_cost=org_spend,
             max_budget=org_max_budget,
-            message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_table.spend}, Max budget: {org_max_budget}",
+            message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
         )
 
 

litellm/proxy/common_utils/callback_utils.py

@@ -37,6 +37,20 @@ def initialize_callbacks_on_proxy(  # noqa: PLR0915
     if isinstance(value, list):
         imported_list: List[Any] = []
         for callback in value:  # ["presidio", <my-custom-callback>]
+            if isinstance(callback, str) and callback == "compression_interception":
+                from litellm.integrations.compression_interception.handler import (
+                    CompressionInterceptionLogger,
+                )
+
+                compression_interception_obj = (
+                    CompressionInterceptionLogger.initialize_from_proxy_config(
+                        litellm_settings=litellm_settings,
+                        callback_specific_params=callback_specific_params,
+                    )
+                )
+                imported_list.append(compression_interception_obj)
+                continue
+
             # check if callback is a custom logger compatible callback
             if isinstance(callback, str):
                 callback = LoggingCallbackManager._add_custom_callback_generic_api_str(

litellm/proxy/db/prisma_client.py

@@ -403,10 +403,18 @@ class PrismaManager:
         return dname
 
     @staticmethod
-    def setup_database(use_migrate: bool = False) -> bool:
+    def setup_database(
+        use_migrate: bool = False, use_v2_resolver: bool = False
+    ) -> bool:
         """
         Set up the database using either prisma migrate or prisma db push
 
+        Args:
+            use_migrate: Use `prisma migrate deploy` instead of `db push`.
+            use_v2_resolver: Opt into the v2 migration resolver that avoids
+                the diff-and-force recovery behavior (which caused schema
+                thrashing during rolling deploys). Defaults to False.
+
         Returns:
             bool: True if setup was successful, False otherwise
         """
@@ -427,7 +435,10 @@ class PrismaManager:
 
                     prisma_dir = PrismaManager._get_prisma_dir()
 
-                    return ProxyExtrasDBManager.setup_database(use_migrate=use_migrate)
+                    return ProxyExtrasDBManager.setup_database(
+                        use_migrate=use_migrate,
+                        use_v2_resolver=use_v2_resolver,
+                    )
                 else:
                     # Use prisma db push with increased timeout
                     subprocess.run(

litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py

@@ -62,6 +62,7 @@ from litellm.types.utils import (
     CallTypesLiteral,
     Choices,
     GuardrailStatus,
+    Message,
     ModelResponse,
     ModelResponseStream,
     StreamingChoices,
@@ -1563,11 +1564,43 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
 
             # Bedrock will throw an error if there is no text to process
             if filtered_messages:
-                bedrock_response = await self.make_bedrock_api_request(
-                    source="INPUT",
-                    messages=filtered_messages,
-                    request_data=request_data,
+                # Map the abstract input_type to the Bedrock source parameter.
+                # "request"  -> INPUT  (scan user-supplied content)
+                # "response" -> OUTPUT (scan model-generated content)
+                # Bedrock guardrail policies are often configured differently
+                # for Input vs Output (e.g. PII blocking only on Output), so
+                # the source MUST match where the text originated.
+                bedrock_source: Literal["INPUT", "OUTPUT"] = (
+                    "OUTPUT" if input_type == "response" else "INPUT"
                 )
+                if bedrock_source == "OUTPUT":
+                    # Build a synthetic ModelResponse whose choices carry the
+                    # text(s) to scan, so _create_bedrock_output_content_request
+                    # can produce the correct Bedrock OUTPUT payload.
+                    synthetic_response = ModelResponse(
+                        choices=[
+                            Choices(
+                                index=_idx,
+                                message=Message(
+                                    role="assistant",
+                                    content=str(_msg.get("content") or ""),
+                                ),
+                                finish_reason="stop",
+                            )
+                            for _idx, _msg in enumerate(filtered_messages)
+                        ]
+                    )
+                    bedrock_response = await self.make_bedrock_api_request(
+                        source="OUTPUT",
+                        response=synthetic_response,
+                        request_data=request_data,
+                    )
+                else:
+                    bedrock_response = await self.make_bedrock_api_request(
+                        source="INPUT",
+                        messages=filtered_messages,
+                        request_data=request_data,
+                    )
 
                 # Apply any masking that was applied by the guardrail
                 output_list = bedrock_response.get("output")

litellm/proxy/health_check.py

@@ -306,7 +306,9 @@ def _health_check_deployment_is_wildcard(litellm_params: dict) -> bool:
     return "*" in _deployment_model_string_for_health_check(litellm_params)
 
 
-def _resolve_health_check_max_tokens(model_info: dict, litellm_params: dict) -> Optional[int]:
+def _resolve_health_check_max_tokens(
+    model_info: dict, litellm_params: dict
+) -> Optional[int]:
     """
     Pick max_tokens for the health check request.
 
@@ -341,10 +343,7 @@ def _resolve_health_check_max_tokens(model_info: dict, litellm_params: dict) ->
                 return int(tokens_reasoning)
             if not is_reasoning and tokens_non_reasoning is not None:
                 return int(tokens_non_reasoning)
-        if (
-            is_reasoning
-            and BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING is not None
-        ):
+        if is_reasoning and BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING is not None:
             return int(BACKGROUND_HEALTH_CHECK_MAX_TOKENS_REASONING)
 
     if BACKGROUND_HEALTH_CHECK_MAX_TOKENS is not None:

litellm/proxy/health_endpoints/_health_endpoints.py

@@ -1121,14 +1121,14 @@ async def _db_health_readiness_check():
         return db_health_cache
     except Exception as e:
         db_health_cache = {"status": "disconnected", "last_updated": datetime.now()}
-        PrismaDBExceptionHandler.handle_db_exception(e)
         if PrismaDBExceptionHandler.is_database_transport_error(e):
             try:
                 verbose_proxy_logger.warning(
                     "_db_health_readiness_check: health_check failed, attempting reconnect"
                 )
-                await prisma_client.disconnect()
-                await prisma_client.connect()
+                await prisma_client.attempt_db_reconnect(
+                    reason="health_readiness_check"
+                )
                 await prisma_client.health_check()
                 verbose_proxy_logger.info(
                     "_db_health_readiness_check: reconnect succeeded"

litellm/proxy/hooks/max_budget_limiter.py

@@ -21,20 +21,30 @@ class _PROXY_MaxBudgetLimiter(CustomLogger):
     ):
         try:
             verbose_proxy_logger.debug("Inside Max Budget Limiter Pre-Call Hook")
-            cache_key = f"{user_api_key_dict.user_id}_user_api_key_user_id"
-            user_row = await cache.async_get_cache(
-                cache_key, parent_otel_span=user_api_key_dict.parent_otel_span
+            max_budget = user_api_key_dict.user_max_budget
+            user_id = user_api_key_dict.user_id
+
+            if max_budget is None or user_id is None:
+                return
+
+            # Personal budget applies only to non-team requests, matching
+            # the explicit team-key exemption in common_checks section 4.1.
+            if user_api_key_dict.team_id is not None:
+                return
+
+            from litellm.proxy.proxy_server import get_current_spend
+
+            curr_spend = await get_current_spend(
+                counter_key=f"spend:user:{user_id}",
+                fallback_spend=user_api_key_dict.user_spend or 0.0,
             )
-            if user_row is None:  # value not yet cached
-                return
-            max_budget = user_row["max_budget"]
-            curr_spend = user_row["spend"]
 
-            if max_budget is None:
-                return
-
-            if curr_spend is None:
-                return
+            verbose_proxy_logger.debug(
+                "MaxBudgetLimiter: user_id=%s, spend=%.6f, max=%.6f",
+                user_id,
+                curr_spend,
+                max_budget,
+            )
 
             # CHECK IF REQUEST ALLOWED
             if curr_spend >= max_budget:

litellm/proxy/hooks/parallel_request_limiter_v3.py

@@ -1570,9 +1570,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
         user_api_key_project_id = standard_logging_metadata.get(
             "user_api_key_project_id"
         )
-        user_api_key_end_user_id = kwargs.get(
-            "user"
-        ) or standard_logging_metadata.get("user_api_key_end_user_id")
+        user_api_key_end_user_id = kwargs.get("user") or standard_logging_metadata.get(
+            "user_api_key_end_user_id"
+        )
         model_group = get_model_group_from_litellm_kwargs(kwargs)
 
         # Get total tokens from response

litellm/proxy/hooks/proxy_track_cost_callback.py

@@ -213,6 +213,7 @@ class _ProxyDBLogger(CustomLogger):
                         team_id=team_id,
                         user_id=user_id,
                         response_cost=response_cost,
+                        org_id=org_id,
                     )
 
                     # update cache (fire-and-forget for backward compat:

litellm/proxy/litellm_pre_call_utils.py

@@ -1,5 +1,6 @@
 import asyncio
 import copy
+import re
 import time
 from collections import OrderedDict
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
@@ -28,6 +29,14 @@ _SPECIAL_HEADERS_CACHE = frozenset(
     v.value.lower() for v in SpecialHeaders._member_map_.values()
 )
 
+# Matches any header of the form x-<something>-session-id (case-insensitive).
+# Excludes the two explicit litellm headers which are handled with higher priority.
+_GENERIC_SESSION_ID_HEADER_RE = re.compile(r"^x-.+-session-id$", re.IGNORECASE)
+_EXPLICIT_SESSION_HEADERS = frozenset({"x-litellm-trace-id", "x-litellm-session-id"})
+# Session-id values must be non-empty strings of alphanumerics, hyphens, or underscores
+# (covers UUIDs and most common session-id formats).
+_SESSION_ID_VALUE_RE = re.compile(r"^[a-zA-Z0-9_\-]{8,}$")
+
 
 def _sanitize_for_log(value: Any) -> str:
     """
@@ -115,13 +124,43 @@ def _get_metadata_variable_name(request: Request) -> str:
     return "metadata"
 
 
+def _extract_generic_session_id_from_headers(
+    normalized: Dict[str, str],
+) -> Optional[str]:
+    """
+    Scan a normalised (lower-cased keys) header dict for any header that looks
+    like ``x-<vendor>-session-id`` and whose value is a plausible session/trace
+    identifier (alphanumeric + hyphens/underscores, at least 8 chars).
+
+    The two explicit LiteLLM headers (``x-litellm-trace-id`` /
+    ``x-litellm-session-id``) are excluded here because they are handled with
+    higher priority by the caller.
+
+    Example: ``x-claude-code-session-id: e96634a3-fa28-4083-b354-55542e2dca01``
+    """
+    for key, value in normalized.items():
+        if (
+            key not in _EXPLICIT_SESSION_HEADERS
+            and _GENERIC_SESSION_ID_HEADER_RE.match(key)
+            and isinstance(value, str)
+            and _SESSION_ID_VALUE_RE.match(value)
+        ):
+            return value
+    return None
+
+
 def get_chain_id_from_headers(headers: Optional[Dict[str, str]]) -> Optional[str]:
     """
     Extract chain id for call chaining from request headers.
 
-    x-litellm-trace-id and x-litellm-session-id are interchangeable; when both
-    are present, x-litellm-trace-id takes precedence. Header keys are matched
-    case-insensitively so this works with raw header dicts from any transport.
+    Priority order:
+    1. ``x-litellm-trace-id`` (explicit, highest priority)
+    2. ``x-litellm-session-id`` (explicit)
+    3. Any ``x-<vendor>-session-id`` header whose value looks like a session id
+       (alphanumeric / UUID, at least 8 chars).  E.g. ``x-claude-code-session-id``.
+
+    Header keys are matched case-insensitively so this works with raw header
+    dicts from any transport.
 
     Used by MCP (and other paths that have raw_headers but no Request) to set
     litellm_trace_id/litellm_session_id for spend logs and logging consistency.
@@ -129,8 +168,10 @@ def get_chain_id_from_headers(headers: Optional[Dict[str, str]]) -> Optional[str
     if not headers:
         return None
     normalized = {k.lower(): v for k, v in headers.items() if isinstance(k, str)}
-    return normalized.get("x-litellm-trace-id") or normalized.get(
-        "x-litellm-session-id"
+    return (
+        normalized.get("x-litellm-trace-id")
+        or normalized.get("x-litellm-session-id")
+        or _extract_generic_session_id_from_headers(normalized)
     )
 
 
@@ -649,10 +690,8 @@ class LiteLLMProxyRequestSetup:
         #########################################################################################
 
         agent_id_from_header = headers.get("x-litellm-agent-id")
-        # x-litellm-trace-id and x-litellm-session-id are interchangeable for call chaining
-        chain_id = headers.get("x-litellm-trace-id") or headers.get(
-            "x-litellm-session-id"
-        )
+        # Explicit litellm headers take precedence; fall back to any x-*-session-id header.
+        chain_id = get_chain_id_from_headers(dict(headers))
 
         if agent_id_from_header:
             metadata_from_headers["agent_id"] = agent_id_from_header

litellm/proxy/management_endpoints/common_utils.py

@@ -355,6 +355,7 @@ async def _upsert_budget_and_membership(
     tpm_limit: Optional[int] = None,
     rpm_limit: Optional[int] = None,
     allowed_models: Optional[List[str]] = None,
+    team_default_budget_id: Optional[str] = None,
 ):
     """
     Helper function to Create/Update or Delete the budget within the team membership
@@ -368,6 +369,11 @@ async def _upsert_budget_and_membership(
         tpm_limit: Tokens per minute limit for the team member
         rpm_limit: Requests per minute limit for the team member
         allowed_models: Per-member model scope. None = don't change. [] = remove restrictions. Non-empty list = enforce.
+        team_default_budget_id: The team's shared default member budget id (from
+            team metadata.team_member_budget_id), if any. When the membership's
+            existing_budget_id matches this, we clone-on-write so editing one
+            member's budget does not mutate the shared default (and therefore
+            every other member who still points at it).
 
     If max_budget, tpm_limit, rpm_limit, and allowed_models are all None, the user's budget is removed from the team membership.
     If any of these values exist, a budget is updated or created and linked to the team membership.
@@ -385,7 +391,13 @@ async def _upsert_budget_and_membership(
         )
         return
 
-    if existing_budget_id is not None:
+    is_shared_default = (
+        existing_budget_id is not None
+        and team_default_budget_id is not None
+        and existing_budget_id == team_default_budget_id
+    )
+
+    if existing_budget_id is not None and not is_shared_default:
         # Update the existing budget in-place to preserve fields not being changed.
         # Only write fields that the caller explicitly provided (non-None).
         update_data: Dict[str, Any] = {
@@ -405,11 +417,40 @@ async def _upsert_budget_and_membership(
         )
         return
 
-    # No existing budget — create a new one and link it to the membership.
+    # Either there is no existing budget, OR the membership is still pointing
+    # at the team's shared default member budget. In both cases we create a
+    # NEW private budget for this user and (re)link the membership to it.
     create_data: Dict[str, Any] = {
         "created_by": user_api_key_dict.user_id or "",
         "updated_by": user_api_key_dict.user_id or "",
     }
+
+    # If we're forking off the shared default, seed the new row with the
+    # default's values so fields the caller did not change carry over.
+    if is_shared_default:
+        default_budget_row = await tx.litellm_budgettable.find_unique(
+            where={"budget_id": existing_budget_id}
+        )
+        if default_budget_row is not None:
+            default_budget_dict = default_budget_row.model_dump()
+            for field in (
+                "max_budget",
+                "soft_budget",
+                "max_parallel_requests",
+                "tpm_limit",
+                "rpm_limit",
+                "model_max_budget",
+                "budget_duration",
+                "allowed_models",
+            ):
+                value = default_budget_dict.get(field)
+                if value is None:
+                    continue
+                if isinstance(value, list) and len(value) == 0:
+                    continue
+                create_data[field] = value
+
+    # Caller-provided values take precedence over the cloned defaults.
     if max_budget is not None:
         create_data["max_budget"] = max_budget
     if tpm_limit is not None:

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -2120,9 +2120,7 @@ async def delete_user(
         for m in all_target_memberships:
             if not m.organization_id:
                 continue
-            target_org_ids_by_user.setdefault(m.user_id, set()).add(
-                m.organization_id
-            )
+            target_org_ids_by_user.setdefault(m.user_id, set()).add(m.organization_id)
 
     # check that all teams passed exist
     for user_id in data.user_ids:
@@ -2141,9 +2139,7 @@ async def delete_user(
             # Org-admin may only delete users whose entire org membership is
             # within their admin scope. A target with ANY org outside the
             # caller's scope (or no org at all) requires PROXY_ADMIN.
-            if not target_org_ids or not target_org_ids.issubset(
-                caller_admin_org_ids
-            ):
+            if not target_org_ids or not target_org_ids.issubset(caller_admin_org_ids):
                 raise HTTPException(
                     status_code=403,
                     detail={

litellm/proxy/management_endpoints/mcp_management_endpoints.py

@@ -1336,7 +1336,9 @@ if MCP_AVAILABLE:
 
         return _redact_mcp_credentials(temp_record)
 
-    def _get_cached_temporary_mcp_server_or_404(server_id: str) -> MCPServer:
+    def _get_cached_temporary_mcp_server_or_404(
+        server_id: str, request: Optional[Request] = None
+    ) -> MCPServer:
         server = get_cached_temporary_mcp_server(server_id)
         if server is None:
             # Fall back to real DB/config server (e.g. for the user-side OAuth flow
@@ -1344,10 +1346,14 @@ if MCP_AVAILABLE:
             from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
                 global_mcp_server_manager,
             )
+            from litellm.proxy.auth.ip_address_utils import IPAddressUtils
 
+            client_ip = IPAddressUtils.get_mcp_client_ip(request) if request else None
             server = global_mcp_server_manager.get_mcp_server_by_id(
                 server_id
-            ) or global_mcp_server_manager.get_mcp_server_by_name(server_id)
+            ) or global_mcp_server_manager.get_mcp_server_by_name(
+                server_id, client_ip=client_ip
+            )
         if server is None:
             raise HTTPException(
                 status_code=status.HTTP_404_NOT_FOUND,
@@ -1358,10 +1364,12 @@ if MCP_AVAILABLE:
     @router.get(
         "/server/oauth/{server_id}/authorize",
         include_in_schema=False,
+        dependencies=[Depends(user_api_key_auth)],
     )
     async def mcp_authorize(
         request: Request,
         server_id: str,
+        user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
         client_id: Optional[str] = None,
         redirect_uri: str = Query(...),
         state: str = "",
@@ -1370,7 +1378,7 @@ if MCP_AVAILABLE:
         response_type: Optional[str] = None,
         scope: Optional[str] = None,
     ):
-        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id)
+        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id, request=request)
         # Use the server's stored client_id when the caller doesn't supply one
         resolved_client_id = mcp_server.client_id or client_id or ""
         if not resolved_client_id:
@@ -1399,10 +1407,12 @@ if MCP_AVAILABLE:
     @router.post(
         "/server/oauth/{server_id}/token",
         include_in_schema=False,
+        dependencies=[Depends(user_api_key_auth)],
     )
     async def mcp_token(
         request: Request,
         server_id: str,
+        user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
         grant_type: str = Form(...),
         code: Optional[str] = Form(None),
         redirect_uri: Optional[str] = Form(None),
@@ -1412,7 +1422,7 @@ if MCP_AVAILABLE:
         refresh_token: Optional[str] = Form(None),
         scope: Optional[str] = Form(None),
     ):
-        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id)
+        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id, request=request)
         resolved_client_id = mcp_server.client_id or client_id or ""
         if not resolved_client_id:
             raise HTTPException(
@@ -1441,9 +1451,14 @@ if MCP_AVAILABLE:
     @router.post(
         "/server/oauth/{server_id}/register",
         include_in_schema=False,
+        dependencies=[Depends(user_api_key_auth)],
     )
-    async def mcp_register(request: Request, server_id: str):
-        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id)
+    async def mcp_register(
+        request: Request,
+        server_id: str,
+        user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    ):
+        mcp_server = _get_cached_temporary_mcp_server_or_404(server_id, request=request)
         request_data = await _read_request_body(request=request)
         data: dict = {**request_data}
 

litellm/proxy/management_endpoints/organization_endpoints.py

@@ -1078,10 +1078,7 @@ async def organization_member_update(
             LitellmUserRoles.PROXY_ADMIN.value,
             LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY.value,
         ):
-            if (
-                user_api_key_dict.user_role
-                != LitellmUserRoles.PROXY_ADMIN.value
-            ):
+            if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value:
                 raise HTTPException(
                     status_code=403,
                     detail={

litellm/proxy/management_endpoints/team_endpoints.py

@@ -1570,8 +1570,7 @@ async def update_team(  # noqa: PLR0915
             current_org_id = getattr(existing_team_row, "organization_id", None)
             if (
                 data.organization_id != current_org_id
-                and user_api_key_dict.user_role
-                != LitellmUserRoles.PROXY_ADMIN.value
+                and user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN.value
             ):
                 # Is the caller org_admin of the destination org?
                 caller_memberships = (
@@ -2609,6 +2608,15 @@ async def team_member_update(
             identified_budget_id = tm.budget_id
             break
 
+    # If this membership still points at the team's shared default member
+    # budget, _upsert_budget_and_membership will clone-on-write so that the
+    # update only touches this user (not every member sharing the default).
+    team_default_budget_id: Optional[str] = None
+    if team_table.metadata is not None:
+        raw_default_budget_id = team_table.metadata.get("team_member_budget_id")
+        if isinstance(raw_default_budget_id, str):
+            team_default_budget_id = raw_default_budget_id
+
     ### upsert new budget
     async with prisma_client.db.tx() as tx:
         await _upsert_budget_and_membership(
@@ -2621,6 +2629,7 @@ async def team_member_update(
             tpm_limit=data.tpm_limit,
             rpm_limit=data.rpm_limit,
             allowed_models=data.allowed_models,
+            team_default_budget_id=team_default_budget_id,
         )
 
     ### update team member role

litellm/proxy/management_helpers/utils.py

@@ -140,6 +140,62 @@ async def handle_budget_for_entity(
         return existing_budget_id
 
 
+# Fields on LiteLLM_BudgetTable that represent the budget's *configuration*
+# (i.e. the values an admin sets). We copy these when cloning a team's
+# default member-budget into an individual member-budget so that the new
+# row starts with the same limits as the default.
+_CLONABLE_BUDGET_FIELDS: Tuple[str, ...] = (
+    "max_budget",
+    "soft_budget",
+    "max_parallel_requests",
+    "tpm_limit",
+    "rpm_limit",
+    "model_max_budget",
+    "budget_duration",
+    "allowed_models",
+)
+
+
+async def _clone_team_default_budget_for_member(
+    prisma_client: PrismaClient,
+    default_team_budget_id: str,
+    user_api_key_dict: UserAPIKeyAuth,
+    litellm_proxy_admin_name: str,
+) -> Optional[str]:
+    """
+    Create a new budget row that copies the values from the team's default
+    member budget. Returns the new budget_id, or None if the default budget
+    no longer exists in the DB.
+
+    Used when adding a new team member without an explicit per-member budget,
+    so the member starts with the team default's values but gets their own
+    private budget row (which can be edited independently).
+    """
+    default_budget = await prisma_client.db.litellm_budgettable.find_unique(
+        where={"budget_id": default_team_budget_id}
+    )
+    if default_budget is None:
+        return None
+
+    default_budget_dict = default_budget.model_dump()
+    cloned_data: dict = {
+        "created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+        "updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
+    }
+    for field in _CLONABLE_BUDGET_FIELDS:
+        value = default_budget_dict.get(field)
+        if value is None:
+            continue
+        # Skip empty list defaults (e.g. allowed_models = []) so the cloned
+        # row matches the "no value set" shape rather than carrying a default.
+        if isinstance(value, list) and len(value) == 0:
+            continue
+        cloned_data[field] = value
+
+    new_budget = await prisma_client.db.litellm_budgettable.create(data=cloned_data)
+    return new_budget.budget_id
+
+
 async def add_new_member(
     new_member: Member,
     max_budget_in_team: Optional[float],
@@ -221,8 +277,20 @@ async def add_new_member(
         response = await prisma_client.db.litellm_budgettable.create(data=budget_data)
 
         _budget_id = response.budget_id
+    elif default_team_budget_id is not None:
+        # No per-member budget was provided, but the team has a default member
+        # budget. Clone the default budget into a new row for this user so that
+        # later edits to one member's budget do not bleed into other members.
+        # If the default no longer exists in the DB, fall back to no budget.
+        _budget_id = await _clone_team_default_budget_for_member(
+            prisma_client=prisma_client,
+            default_team_budget_id=default_team_budget_id,
+            user_api_key_dict=user_api_key_dict,
+            litellm_proxy_admin_name=litellm_proxy_admin_name,
+        )
     else:
-        _budget_id = default_team_budget_id
+        # No per-member budget and no team default → member gets no budget.
+        _budget_id = None
 
     if _budget_id and returned_user is not None and returned_user.user_id is not None:
         _returned_team_membership = (

litellm/proxy/proxy_cli.py

@@ -577,6 +577,22 @@ class ProxyInitializationHelpers:
     help="Exit with error if database migration fails on startup.",
     envvar="ENFORCE_PRISMA_MIGRATION_CHECK",
 )
+@click.option(
+    "--use_v2_migration_resolver",
+    is_flag=True,
+    default=False,
+    help=(
+        "Opt into the v2 migration resolver. Avoids the diff-and-force recovery "
+        "path that can cause schema thrashing during rolling deploys where two "
+        "LiteLLM versions contend for the same DB. Default is the v1 resolver."
+    ),
+)
+@click.option(
+    "--reload",
+    is_flag=True,
+    default=False,
+    help="Enable uvicorn hot reload (dev only). Incompatible with --num_workers>1, --run_gunicorn, and --run_hypercorn.",
+)
 def run_server(  # noqa: PLR0915
     host,
     port,
@@ -618,6 +634,8 @@ def run_server(  # noqa: PLR0915
     keepalive_timeout,
     max_requests_before_restart,
     enforce_prisma_migration_check: bool,
+    use_v2_migration_resolver: bool,
+    reload: bool,
 ):
     if setup:
         from litellm.setup_wizard import run_setup_wizard
@@ -886,9 +904,31 @@ def run_server(  # noqa: PLR0915
                 ):
                     check_prisma_schema_diff(db_url=None)
                 else:
-                    if not PrismaManager.setup_database(
-                        use_migrate=not use_prisma_db_push
-                    ):
+                    if not use_v2_migration_resolver:
+                        print(  # noqa
+                            "\033[1;33mLiteLLM Proxy: Using default (v1) migration resolver. "
+                            "If your deployment has seen schema thrashing during rolling "
+                            "deploys, try --use_v2_migration_resolver (safer: avoids the "
+                            "diff-and-force recovery that caused the thrash).\033[0m"
+                        )
+                    try:
+                        setup_ok = PrismaManager.setup_database(
+                            use_migrate=not use_prisma_db_push,
+                            use_v2_resolver=use_v2_migration_resolver,
+                        )
+                    except RuntimeError as e:
+                        # v2 resolver raises on unrecoverable migration errors
+                        # (e.g. non-idempotent failures, permission issues).
+                        # v1 never raises here, so this only fires when the
+                        # operator opted into v2.
+                        print(  # noqa
+                            "\033[1;31mLiteLLM Proxy: Database migration cannot proceed. "
+                            f"{e}\033[0m",
+                            file=sys.stderr,
+                            flush=True,
+                        )
+                        sys.exit(2)
+                    if not setup_ok:
                         if enforce_prisma_migration_check:
                             print(  # noqa
                                 "\033[1;31mLiteLLM Proxy: Database setup failed after multiple retries. "
@@ -954,6 +994,9 @@ def run_server(  # noqa: PLR0915
             if loop_type:
                 uvicorn_args["loop"] = loop_type
 
+            if reload:
+                uvicorn_args["reload"] = True
+
             uvicorn.run(
                 **uvicorn_args,
                 workers=num_workers,

litellm/proxy/proxy_server.py

@@ -1795,6 +1795,7 @@ async def increment_spend_counters(
     team_id: Optional[str],
     user_id: Optional[str],
     response_cost: Optional[float],
+    org_id: Optional[str] = None,
 ):
     """
     Atomically increment spend counters for budget enforcement.
@@ -1881,6 +1882,20 @@ async def increment_spend_counters(
             increment=response_cost,
         )
 
+    if user_id is not None:
+        await _init_and_increment_spend_counter(
+            counter_key=f"spend:user:{user_id}",
+            source_cache_key=user_id,
+            increment=response_cost,
+        )
+
+    if org_id is not None:
+        await _init_and_increment_spend_counter(
+            counter_key=f"spend:org:{org_id}",
+            source_cache_key=f"org_id:{org_id}",
+            increment=response_cost,
+        )
+
 
 async def _init_and_increment_spend_counter(
     counter_key: str,

litellm/router.py

@@ -200,12 +200,16 @@ if TYPE_CHECKING:
     from litellm.router_strategy.complexity_router.complexity_router import (
         ComplexityRouter,
     )
+    from litellm.router_strategy.quality_router.quality_router import (
+        QualityRouter,
+    )
 
     Span = Union[_Span, Any]
 else:
     Span = Any
     AutoRouter = Any
     ComplexityRouter = Any
+    QualityRouter = Any
     PreRoutingHookResponse = Any
 
 
@@ -464,6 +468,7 @@ class Router:
         )  # {"TEAM_ID": PatternMatchRouter}
         self.auto_routers: Dict[str, "AutoRouter"] = {}
         self.complexity_routers: Dict[str, "ComplexityRouter"] = {}
+        self.quality_routers: Dict[str, "QualityRouter"] = {}
 
         # Initialize model_group_alias early since it's used in set_model_list
         self.model_group_alias: Dict[str, Union[str, RouterModelGroupAliasItem]] = (
@@ -5884,7 +5889,7 @@ class Router:
             response = await response
         ## PROCESS RESPONSE HEADERS
         response = await self.set_response_headers(
-            response=response, model_group=model_group
+            response=response, model_group=model_group, request_kwargs=kwargs
         )
 
         return response
@@ -6814,6 +6819,8 @@ class Router:
         """
         if litellm_params.model.startswith("auto_router/complexity_router"):
             return False  # This is handled by complexity_router
+        if litellm_params.model.startswith("auto_router/quality_router"):
+            return False  # This is handled by quality_router
         if litellm_params.model.startswith("auto_router/"):
             return True
         return False
@@ -6920,6 +6927,58 @@ class Router:
             )
         self.complexity_routers[deployment.model_name] = complexity_router
 
+    def _is_quality_router_deployment(self, litellm_params: LiteLLM_Params) -> bool:
+        """
+        Check if the deployment is a quality-router deployment.
+
+        Returns True if the litellm_params model starts with "auto_router/quality_router".
+        """
+        if litellm_params.model.startswith("auto_router/quality_router"):
+            return True
+        return False
+
+    def init_quality_router_deployment(self, deployment: Deployment):
+        """
+        Initialize the quality-router deployment.
+
+        Resolves the default model from either `quality_router_default_model` or
+        `quality_router_config["default_model"]`, then instantiates the
+        QualityRouter and stores it in `self.quality_routers`.
+        """
+        # Import here to mirror the AutoRouter / ComplexityRouter init pattern
+        # and avoid circular imports.
+        from litellm.router_strategy.quality_router.quality_router import (
+            QualityRouter,
+        )
+
+        quality_router_config: Optional[dict] = (
+            deployment.litellm_params.quality_router_config
+        )
+
+        default_model: Optional[str] = (
+            deployment.litellm_params.quality_router_default_model
+        )
+        if default_model is None and quality_router_config:
+            default_model = quality_router_config.get("default_model")
+
+        if default_model is None:
+            raise ValueError(
+                "quality_router_default_model is required for quality-router deployments, "
+                "or set default_model in quality_router_config. Please configure it in the litellm_params"
+            )
+
+        quality_router: QualityRouter = QualityRouter(
+            model_name=deployment.model_name,
+            default_model=default_model,
+            litellm_router_instance=self,
+            quality_router_config=quality_router_config,
+        )
+        if deployment.model_name in self.quality_routers:
+            raise ValueError(
+                f"Quality-router deployment {deployment.model_name} already exists. Please use a different model name."
+            )
+        self.quality_routers[deployment.model_name] = quality_router
+
     def deployment_is_active_for_environment(self, deployment: Deployment) -> bool:
         """
         Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments
@@ -6966,6 +7025,11 @@ class Router:
         self.model_id_to_deployment_index_map = {}  # Reset the index
         self.model_name_to_deployment_indices = {}  # Reset the model_name index
         self.team_model_to_deployment_indices = {}  # Reset the team_model index
+        # Reset per-strategy router registries so hot-reload doesn't leave
+        # stale routers pointing at the old model_list.
+        self.quality_routers = {}
+        self.complexity_routers = {}
+        self.auto_routers = {}
         self._invalidate_model_group_info_cache()
         self._invalidate_access_groups_cache()
         # we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
@@ -7140,6 +7204,12 @@ class Router:
         ):
             self.init_complexity_router_deployment(deployment=deployment)
 
+        #########################################################
+        # Check if this is a quality-router deployment
+        #########################################################
+        if self._is_quality_router_deployment(litellm_params=deployment.litellm_params):
+            self.init_quality_router_deployment(deployment=deployment)
+
         return deployment
 
     def _initialize_deployment_for_pass_through(
@@ -8143,7 +8213,10 @@ class Router:
         return returned_dict
 
     async def set_response_headers(
-        self, response: Any, model_group: Optional[str] = None
+        self,
+        response: Any,
+        model_group: Optional[str] = None,
+        request_kwargs: Optional[dict] = None,
     ) -> Any:
         """
         Add the most accurate rate limit headers for a given model response.
@@ -8164,6 +8237,45 @@ class Router:
 
             additional_headers = response._hidden_params["additional_headers"]  # type: ignore
 
+            # Lift QualityRouter routing decision into response headers for
+            # transparency. The decision is stashed in request_kwargs.metadata
+            # by QualityRouter.async_pre_routing_hook.
+            metadata = (
+                (request_kwargs.get("metadata") or {})
+                if isinstance(request_kwargs, dict)
+                else {}
+            )
+            decision = (
+                metadata.get("quality_router_decision")
+                if isinstance(metadata, dict)
+                else None
+            )
+            if isinstance(decision, dict):
+                # Only emit headers for fields that have a meaningful value.
+                # `complexity_tier` and `matched_keyword` are mutually exclusive
+                # (the keyword path short-circuits classification), so each
+                # request emits one or the other but not both.
+                if decision.get("routed_model") is not None:
+                    additional_headers["x-litellm-quality-router-model"] = str(
+                        decision["routed_model"]
+                    )
+                if decision.get("quality_tier") is not None:
+                    additional_headers["x-litellm-quality-router-tier"] = str(
+                        decision["quality_tier"]
+                    )
+                if decision.get("routed_via") is not None:
+                    additional_headers["x-litellm-quality-router-via"] = str(
+                        decision["routed_via"]
+                    )
+                if decision.get("matched_keyword") is not None:
+                    additional_headers["x-litellm-quality-router-keyword"] = str(
+                        decision["matched_keyword"]
+                    )
+                if decision.get("complexity_tier") is not None:
+                    additional_headers["x-litellm-quality-router-complexity"] = str(
+                        decision["complexity_tier"]
+                    )
+
             if (
                 "x-ratelimit-remaining-tokens" not in additional_headers
                 and "x-ratelimit-remaining-requests" not in additional_headers
@@ -8708,8 +8820,6 @@ class Router:
                 and self.routing_strategy == "latency-based-routing"
             ):
                 _settings_to_return[var] = self.lowestlatency_logger.routing_args.json()
-            elif var == "routing_strategy_args":
-                _settings_to_return[var] = None
         return _settings_to_return
 
     def update_settings(self, **kwargs):
@@ -9620,7 +9730,7 @@ class Router:
         self,
         model: str,
         request_kwargs: Dict,
-        messages: Optional[List[Dict[str, str]]] = None,
+        messages: Optional[List[Dict[str, Any]]] = None,
         input: Optional[Union[str, List]] = None,
         specific_deployment: Optional[bool] = False,
     ) -> Optional[PreRoutingHookResponse]:
@@ -9653,6 +9763,18 @@ class Router:
                 specific_deployment=specific_deployment,
             )
 
+        #########################################################
+        # Check if any quality-router should be used
+        #########################################################
+        if model in self.quality_routers:
+            return await self.quality_routers[model].async_pre_routing_hook(
+                model=model,
+                request_kwargs=request_kwargs,
+                messages=messages,
+                input=input,
+                specific_deployment=specific_deployment,
+            )
+
         return None
 
     def get_available_deployment(

litellm/router_strategy/auto_router/auto_router.py

@@ -82,11 +82,34 @@ class AutoRouter(CustomLogger):
             )
         return auto_router_routes
 
+    @staticmethod
+    def _extract_text_from_messages(messages: List[Dict[str, Any]]) -> str:
+        """
+        Extract text content from the last user message for routing.
+
+        Handles tool-call conversations (where the last message may be an
+        assistant or tool message with non-string content) and multimodal
+        messages (where content is a list of content blocks).
+        """
+        for msg in reversed(messages):
+            if msg.get("role") == "user":
+                content = msg.get("content")
+                if content is None:
+                    return ""
+                if isinstance(content, list):
+                    return " ".join(
+                        block.get("text", "")
+                        for block in content
+                        if isinstance(block, dict) and block.get("type") == "text"
+                    )
+                return str(content)
+        return ""
+
     async def async_pre_routing_hook(
         self,
         model: str,
         request_kwargs: Dict,
-        messages: Optional[List[Dict[str, str]]] = None,
+        messages: Optional[List[Dict[str, Any]]] = None,
         input: Optional[Union[str, List]] = None,
         specific_deployment: Optional[bool] = False,
     ) -> Optional["PreRoutingHookResponse"]:
@@ -120,8 +143,7 @@ class AutoRouter(CustomLogger):
                 auto_sync=self.auto_sync_value,
             )
 
-        user_message: Dict[str, str] = messages[-1]
-        message_content: str = user_message.get("content", "")
+        message_content = self._extract_text_from_messages(messages)
         route_choice: Optional[Union[RouteChoice, List[RouteChoice]]] = self.routelayer(
             text=message_content
         )

litellm/router_strategy/complexity_router/complexity_router.py

@@ -332,45 +332,68 @@ class ComplexityRouter(CustomLogger):
             f"No model configured for tier {tier_key} and no default_model set"
         )
 
-    async def async_pre_routing_hook(
+    def _resolve_messages(
         self,
-        model: str,
+        messages: Optional[List[Dict[str, Any]]],
         request_kwargs: Dict,
-        messages: Optional[List[Dict[str, Any]]] = None,
-        input: Optional[Union[str, List]] = None,
-        specific_deployment: Optional[bool] = False,
-    ) -> Optional["PreRoutingHookResponse"]:
+    ) -> Optional[List[Dict[str, Any]]]:
         """
-        Pre-routing hook called before the routing decision.
+        Resolve messages from the request, converting from other formats if needed.
 
-        Classifies the request by complexity and returns the appropriate model.
-
-        Args:
-            model: The original model name requested.
-            request_kwargs: The request kwargs.
-            messages: The messages in the request.
-            input: Optional input for embeddings.
-            specific_deployment: Whether a specific deployment was requested.
-
-        Returns:
-            PreRoutingHookResponse with the routed model, or None if no routing needed.
+        Uses the guardrail translation handler dispatch to convert Responses API
+        ``input`` (or other non-chat-completions formats) into OpenAI-spec messages.
         """
-        from litellm.types.router import PreRoutingHookResponse
+        if messages:
+            return messages
 
-        if messages is None or len(messages) == 0:
-            verbose_router_logger.debug(
-                "ComplexityRouter: No messages provided, skipping routing"
-            )
-            return None
+        from litellm.litellm_core_utils.api_route_to_call_types import (
+            get_call_types_for_route,
+        )
+        from litellm.llms import load_guardrail_translation_mappings
+        from litellm.types.utils import CallTypes
 
-        # Extract the last user message and the last system prompt
+        mappings = load_guardrail_translation_mappings()
+        call_type: Optional[CallTypes] = None
+
+        # 1. Try route-based inference from proxy metadata
+        route = request_kwargs.get("litellm_metadata", {}).get(
+            "user_api_key_request_route"
+        )
+        if route:
+            call_types_list = get_call_types_for_route(route)
+            if call_types_list:
+                for ct in call_types_list:
+                    if ct in mappings:
+                        call_type = ct
+                        break
+
+        # 2. Fallback: try each mapped handler until one produces messages
+        handlers_to_try: List[Any] = []
+        if call_type is not None and call_type in mappings:
+            handlers_to_try.append(mappings[call_type]())
+        else:
+            handlers_to_try.extend(handler_cls() for handler_cls in mappings.values())
+
+        for handler in handlers_to_try:
+            structured = handler.get_structured_messages(request_kwargs)
+            if structured:
+                return [
+                    msg if isinstance(msg, dict) else msg.model_dump()  # type: ignore
+                    for msg in structured
+                ]
+        return None
+
+    @staticmethod
+    def _extract_user_message_and_system_prompt(
+        messages: List[Dict[str, Any]],
+    ) -> Tuple[Optional[str], Optional[str]]:
+        """Extract the last user message text and last system prompt from messages."""
         user_message: Optional[str] = None
         system_prompt: Optional[str] = None
 
         for msg in reversed(messages):
             role = msg.get("role", "")
             content = msg.get("content") or ""
-            # content may be a list of content parts (e.g. [{"type": "text", "text": "..."}])
             if isinstance(content, list):
                 text_parts = [
                     part.get("text", "")
@@ -383,6 +406,52 @@ class ComplexityRouter(CustomLogger):
                     user_message = content
                 elif role == "system" and system_prompt is None:
                     system_prompt = content
+            if user_message is not None and system_prompt is not None:
+                break
+
+        return user_message, system_prompt
+
+    async def async_pre_routing_hook(
+        self,
+        model: str,
+        request_kwargs: Dict,
+        messages: Optional[List[Dict[str, Any]]] = None,
+        input: Optional[Union[str, List]] = None,
+        specific_deployment: Optional[bool] = False,
+    ) -> Optional["PreRoutingHookResponse"]:
+        """
+        Pre-routing hook called before the routing decision.
+
+        Classifies the request by complexity and returns the appropriate model.
+        Supports chat completions (messages), Responses API (input), and other
+        formats via the guardrail translation handler dispatch.
+
+        Args:
+            model: The original model name requested.
+            request_kwargs: The request kwargs.
+            messages: The messages in the request.
+            input: Optional input for Responses API or embeddings.
+            specific_deployment: Whether a specific deployment was requested.
+
+        Returns:
+            PreRoutingHookResponse with the routed model, or None if no routing needed.
+        """
+        from litellm.types.router import PreRoutingHookResponse
+
+        resolved_messages = self._resolve_messages(messages, request_kwargs)
+
+        if not resolved_messages:
+            verbose_router_logger.debug(
+                "ComplexityRouter: No messages could be resolved, skipping routing"
+            )
+            return None
+
+        # Determine whether the original request used messages directly
+        has_original_messages = messages is not None and len(messages) > 0
+
+        user_message, system_prompt = self._extract_user_message_and_system_prompt(
+            resolved_messages
+        )
 
         if user_message is None:
             verbose_router_logger.debug(
@@ -391,13 +460,10 @@ class ComplexityRouter(CustomLogger):
             return PreRoutingHookResponse(
                 model=self.config.default_model
                 or self.get_model_for_tier(ComplexityTier.MEDIUM),
-                messages=messages,
+                messages=messages if has_original_messages else None,
             )
 
-        # Classify the request
         tier, score, signals = self.classify(user_message, system_prompt)
-
-        # Get the model for this tier
         routed_model = self.get_model_for_tier(tier)
 
         verbose_router_logger.info(
@@ -407,5 +473,5 @@ class ComplexityRouter(CustomLogger):
 
         return PreRoutingHookResponse(
             model=routed_model,
-            messages=messages,
+            messages=messages if has_original_messages else None,
         )

litellm/router_strategy/quality_router/__init__.py

@@ -0,0 +1,21 @@
+"""
+Quality-tier auto-router.
+
+Re-uses the ComplexityRouter's classification to decide a request's complexity,
+then maps that complexity to an admin-configured quality tier and resolves the
+target model from each candidate's `model_info.litellm_routing_preferences`.
+"""
+
+from .config import (
+    DEFAULT_COMPLEXITY_TO_QUALITY,
+    QualityRouterConfig,
+    RoutingPreferences,
+)
+from .quality_router import QualityRouter
+
+__all__ = [
+    "QualityRouter",
+    "QualityRouterConfig",
+    "RoutingPreferences",
+    "DEFAULT_COMPLEXITY_TO_QUALITY",
+]

litellm/router_strategy/quality_router/config.py

@@ -0,0 +1,74 @@
+"""
+Configuration models for the QualityRouter.
+"""
+
+from typing import Dict, List, Optional
+
+from pydantic import BaseModel, ConfigDict, Field
+
+# Default mapping from ComplexityTier name (string) to quality tier (int).
+# Higher tier = higher capability requirement.
+DEFAULT_COMPLEXITY_TO_QUALITY: Dict[str, int] = {
+    "SIMPLE": 1,
+    "MEDIUM": 2,
+    "COMPLEX": 3,
+    "REASONING": 4,
+}
+
+
+class QualityRouterConfig(BaseModel):
+    """Configuration for the QualityRouter."""
+
+    available_models: List[str] = Field(
+        default_factory=list,
+        description=(
+            "List of candidate model names this router may route to. Each model "
+            "must declare its quality_tier in model_info.litellm_routing_preferences."
+        ),
+    )
+
+    default_model: Optional[str] = Field(
+        default=None,
+        description="Fallback model when no quality tier resolves.",
+    )
+
+    complexity_to_quality: Dict[str, int] = Field(
+        default_factory=lambda: DEFAULT_COMPLEXITY_TO_QUALITY.copy(),
+        description="Mapping from ComplexityTier name to quality tier (int).",
+    )
+
+    model_config = ConfigDict(extra="allow")
+
+
+class RoutingPreferences(BaseModel):
+    """Per-deployment routing preferences declared on model_info."""
+
+    quality_tier: int = Field(
+        ...,
+        description="The quality tier this deployment satisfies.",
+    )
+
+    keywords: List[str] = Field(
+        default_factory=list,
+        description=(
+            "Substring keywords (case-insensitive) that, when present in the "
+            "user message, route the request to this deployment. See `order` "
+            "for explicit collision handling, otherwise ties fall through to "
+            "(highest quality_tier, then cheapest model_info.input_cost_per_token)."
+        ),
+    )
+
+    order: Optional[int] = Field(
+        default=None,
+        description=(
+            "Explicit priority used to break ties between deployments at the "
+            "same quality tier. Lower values win. Applies both to keyword "
+            "collisions and to picking between multiple deployments at the "
+            "same quality_tier. Tiebreak order is "
+            "(quality_tier DESC, order ASC, input_cost_per_token ASC, "
+            "model_name ASC) — quality always wins first, then explicit "
+            "order, then price."
+        ),
+    )
+
+    model_config = ConfigDict(extra="allow")

litellm/router_strategy/quality_router/quality_router.py

@@ -0,0 +1,446 @@
+"""
+Quality-tier Auto Router.
+
+Routes a request to a model at a target quality tier. The quality tier is
+inferred by re-using the existing ComplexityRouter's classification, then
+mapped through an admin-configured `complexity_to_quality` table. Each
+candidate model declares its own `quality_tier` in
+`model_info.litellm_routing_preferences`.
+
+Optional keyword override: deployments may also declare `keywords` in
+`litellm_routing_preferences`. If any declared keyword appears in the user
+message (case-insensitive substring match), the router short-circuits the
+complexity-classification flow and routes to the matching deployment. When
+multiple deployments match, ties are broken by (highest quality_tier first,
+then cheapest `model_info.input_cost_per_token`).
+"""
+
+import math
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
+
+from litellm._logging import verbose_router_logger
+from litellm.integrations.custom_logger import CustomLogger
+from litellm.router_strategy.complexity_router.complexity_router import (
+    ComplexityRouter,
+)
+
+from .config import QualityRouterConfig, RoutingPreferences
+
+if TYPE_CHECKING:
+    from litellm.router import Router
+    from litellm.types.router import PreRoutingHookResponse
+else:
+    Router = Any
+    PreRoutingHookResponse = Any
+
+
+class QualityRouter(CustomLogger):
+    """
+    Routes requests to a model at a target quality tier, with an optional
+    keyword override.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        litellm_router_instance: "Router",
+        default_model: Optional[str] = None,
+        quality_router_config: Optional[Dict[str, Any]] = None,
+    ):
+        self.model_name = model_name
+        self.litellm_router_instance = litellm_router_instance
+
+        if quality_router_config:
+            self.config = QualityRouterConfig(**quality_router_config)
+        else:
+            self.config = QualityRouterConfig()
+
+        # Explicit default_model arg overrides anything in the config dict.
+        if default_model:
+            self.config.default_model = default_model
+
+        # Internal scorer — re-use the existing rule-based classifier.
+        self._scorer = ComplexityRouter(
+            model_name=f"{model_name}::scorer",
+            litellm_router_instance=litellm_router_instance,
+        )
+
+        # Per-model indices populated alongside the tier index. `_model_keywords`
+        # stores keywords lowercased so we can substring-match against the
+        # lowercased user message in O(total-keyword-count). `_model_quality`,
+        # `_model_cost`, and `_model_order` drive tiebreaking — `_model_order`
+        # is the explicit priority (lower wins, unset = +inf).
+        self._model_keywords: Dict[str, List[str]] = {}
+        self._model_quality: Dict[str, int] = {}
+        self._model_cost: Dict[str, Optional[float]] = {}
+        self._model_order: Dict[str, Optional[int]] = {}
+
+        # Tier → models index. Built lazily on first access so the QualityRouter
+        # deployment does NOT need to appear after all its referenced models in
+        # the config — when `_build_tier_index` runs eagerly in `__init__`, the
+        # router instance's `model_list` is still being assembled incrementally
+        # by `_create_deployment`, and any `available_models` defined AFTER the
+        # router entry in config.yaml would silently be reported as missing.
+        self._tier_to_models_cache: Optional[Dict[int, List[str]]] = None
+
+        verbose_router_logger.debug(
+            f"QualityRouter initialized for {model_name} with "
+            f"available_models={self.config.available_models}, "
+            f"default_model={self.config.default_model}"
+        )
+
+    @property
+    def _tier_to_models(self) -> Dict[int, List[str]]:
+        """Lazy tier→models index; built on first access."""
+        if self._tier_to_models_cache is None:
+            self._tier_to_models_cache = self._build_tier_index()
+        return self._tier_to_models_cache
+
+    def _get_routing_preferences(self, deployment: Any) -> Optional[Dict[str, Any]]:
+        """
+        Extract litellm_routing_preferences from a deployment, handling both
+        dict-shaped and Pydantic-object-shaped deployments.
+        """
+        # Dict-shaped deployment.
+        if isinstance(deployment, dict):
+            model_info = deployment.get("model_info") or {}
+            if isinstance(model_info, dict):
+                return model_info.get("litellm_routing_preferences")
+            # Pydantic ModelInfo nested in a dict.
+            return getattr(model_info, "litellm_routing_preferences", None)
+
+        # Pydantic-object deployment.
+        model_info = getattr(deployment, "model_info", None)
+        if model_info is None:
+            return None
+        if isinstance(model_info, dict):
+            return model_info.get("litellm_routing_preferences")
+        return getattr(model_info, "litellm_routing_preferences", None)
+
+    def _get_deployment_input_cost(self, deployment: Any) -> Optional[float]:
+        """
+        Extract `input_cost_per_token` from a deployment's model_info.
+
+        Returns None when not declared — None is treated as "infinite cost"
+        for the cheapest-tiebreak ordering, so unpriced models lose ties to
+        priced ones. (Admins who want a model to win on price must declare it.)
+        """
+        if isinstance(deployment, dict):
+            model_info = deployment.get("model_info") or {}
+        else:
+            model_info = getattr(deployment, "model_info", None) or {}
+
+        if isinstance(model_info, dict):
+            cost = model_info.get("input_cost_per_token")
+        else:
+            cost = getattr(model_info, "input_cost_per_token", None)
+
+        if cost is None:
+            return None
+        try:
+            return float(cost)
+        except (TypeError, ValueError):
+            return None
+
+    def _get_deployment_model_name(self, deployment: Any) -> Optional[str]:
+        """Extract `model_name` from a dict- or object-shaped deployment."""
+        if isinstance(deployment, dict):
+            return deployment.get("model_name")
+        return getattr(deployment, "model_name", None)
+
+    def _build_tier_index(self) -> Dict[int, List[str]]:
+        """
+        Build {quality_tier: [model_name, ...]} for every model in
+        `available_models`, plus side indices `_model_keywords`,
+        `_model_quality`, and `_model_cost`. Raises if any listed model is
+        missing `litellm_routing_preferences`.
+        """
+        model_list = getattr(self.litellm_router_instance, "model_list", None) or []
+        available = set(self.config.available_models)
+
+        # Track which available models we've matched so we can error on missing.
+        seen: Dict[str, bool] = {name: False for name in available}
+        tier_to_models: Dict[int, List[str]] = {}
+
+        for deployment in model_list:
+            name = self._get_deployment_model_name(deployment)
+            if name is None or name not in available:
+                continue
+
+            raw_prefs = self._get_routing_preferences(deployment)
+            if raw_prefs is None:
+                raise ValueError(
+                    f"QualityRouter: model '{name}' is listed in available_models "
+                    f"but has no model_info.litellm_routing_preferences"
+                )
+
+            # Validate via the Pydantic model so we get a clear error for
+            # missing quality_tier, wrong types, etc. This also means
+            # `RoutingPreferences` is the single source of truth for the
+            # accepted shape — readers relied on raw dicts before.
+            try:
+                if isinstance(raw_prefs, RoutingPreferences):
+                    prefs = raw_prefs
+                elif isinstance(raw_prefs, dict):
+                    prefs = RoutingPreferences(**raw_prefs)
+                else:
+                    # A Pydantic object of some other shape — coerce via its dict.
+                    prefs = RoutingPreferences(
+                        **(
+                            raw_prefs.model_dump()
+                            if hasattr(raw_prefs, "model_dump")
+                            else dict(raw_prefs)
+                        )
+                    )
+            except Exception as e:
+                raise ValueError(
+                    f"QualityRouter: model '{name}' has invalid "
+                    f"litellm_routing_preferences: {e}"
+                ) from e
+
+            tier_int = int(prefs.quality_tier)
+            tier_to_models.setdefault(tier_int, []).append(name)
+            self._model_keywords[name] = [str(k).lower() for k in prefs.keywords if k]
+            self._model_quality[name] = tier_int
+            self._model_cost[name] = self._get_deployment_input_cost(deployment)
+            self._model_order[name] = prefs.order
+            seen[name] = True
+
+        missing = [name for name, found in seen.items() if not found]
+        if missing:
+            raise ValueError(
+                f"QualityRouter: the following available_models are not present in "
+                f"the router's model_list (or are missing routing preferences): {missing}"
+            )
+
+        # Sort each tier's model list so `_resolve_model_for_quality_tier`
+        # (which picks index [0]) honors (order ASC, cost ASC, name ASC).
+        # Quality is moot within a single tier; keep parity with the keyword
+        # tiebreak by ordering on (order, cost, name) here.
+        for models in tier_to_models.values():
+            models.sort(key=lambda n: (self._order_key(n), self._cost_key(n), n))
+
+        return tier_to_models
+
+    def _order_key(self, model_name: str) -> float:
+        """`order` lookup as a float — unset becomes +inf so explicit wins."""
+        order = self._model_order.get(model_name)
+        return float(order) if order is not None else math.inf
+
+    def _cost_key(self, model_name: str) -> float:
+        """`input_cost_per_token` as a float — unset becomes +inf."""
+        cost = self._model_cost.get(model_name)
+        return float(cost) if cost is not None else math.inf
+
+    def _keyword_override(self, user_message: str) -> Optional[Tuple[str, str]]:
+        """
+        Find a deployment whose declared keywords appear in `user_message`.
+
+        Returns (model_name, matched_keyword) or None when no keyword matches.
+        When multiple deployments match, sorts by:
+            1. quality_tier DESC (best quality always wins first)
+            2. `order` ASC (explicit priority — unset = +inf so explicit wins
+               within the same tier)
+            3. input_cost_per_token ASC (unpriced = +inf so priced wins)
+            4. model_name ASC (deterministic stability)
+        """
+        # Touch the lazy index so `_model_keywords` / `_model_quality` /
+        # `_model_cost` / `_model_order` are populated.
+        _ = self._tier_to_models
+
+        text = user_message.lower()
+
+        matches: List[Tuple[str, str]] = []  # (model_name, matched_keyword)
+        for model_name, keywords in self._model_keywords.items():
+            for kw in keywords:
+                if kw and kw in text:
+                    matches.append((model_name, kw))
+                    break  # one match per model is enough
+
+        if not matches:
+            return None
+
+        def sort_key(match: Tuple[str, str]) -> Tuple[int, float, float, str]:
+            name = match[0]
+            quality = self._model_quality.get(name, 0)
+            order_val = self._order_key(name)
+            cost = self._model_cost.get(name)
+            cost_val = cost if cost is not None else math.inf
+            # Negate quality so higher tier sorts first under ASC sort.
+            return (-quality, order_val, cost_val, name)
+
+        matches.sort(key=sort_key)
+        return matches[0]
+
+    def _resolve_model_for_quality_tier(self, tier: int) -> str:
+        """
+        Resolve a quality tier to a concrete model name.
+
+        Strategy:
+            1. Exact tier match → first model registered at that tier.
+            2. Round UP to the next higher tier that has a model (closer to a
+               request we might lack capacity for).
+            3. Round DOWN to the closest lower tier that has a model (degrade
+               gracefully instead of jumping straight to `default_model`,
+               which may be off-tier).
+            4. Fall back to `config.default_model`.
+            5. Otherwise raise.
+        """
+        tier_index = self._tier_to_models
+        if tier in tier_index and tier_index[tier]:
+            return tier_index[tier][0]
+
+        # Round up.
+        higher_tiers = sorted(t for t in tier_index if t > tier)
+        for t in higher_tiers:
+            if tier_index[t]:
+                return tier_index[t][0]
+
+        # Round down — closest lower tier first.
+        lower_tiers = sorted((t for t in tier_index if t < tier), reverse=True)
+        for t in lower_tiers:
+            if tier_index[t]:
+                return tier_index[t][0]
+
+        if self.config.default_model:
+            return self.config.default_model
+
+        raise ValueError(
+            f"QualityRouter: no model available for quality tier {tier} and "
+            f"no default_model configured"
+        )
+
+    def _stash_decision(
+        self,
+        request_kwargs: Optional[Dict[str, Any]],
+        decision: Dict[str, Any],
+    ) -> None:
+        """
+        Stash the routing decision in request_kwargs.metadata so the Router can
+        lift it into response headers (`x-litellm-quality-router-*`). The same
+        dict object flows from here through to `make_call.set_response_headers`.
+        """
+        if request_kwargs is None:
+            return
+        metadata = request_kwargs.setdefault("metadata", {})
+        if isinstance(metadata, dict):
+            metadata["quality_router_decision"] = decision
+
+    async def async_pre_routing_hook(
+        self,
+        model: str,
+        request_kwargs: Dict,
+        messages: Optional[List[Dict[str, Any]]] = None,
+        input: Optional[Union[str, List]] = None,
+        specific_deployment: Optional[bool] = False,
+    ) -> Optional["PreRoutingHookResponse"]:
+        """Try keyword override first; fall back to complexity-tier routing."""
+        from litellm.types.router import PreRoutingHookResponse
+
+        if messages is None or len(messages) == 0:
+            verbose_router_logger.debug(
+                "QualityRouter: No messages provided, skipping routing"
+            )
+            return None
+
+        # Extract last user message and last system prompt — same rules as
+        # ComplexityRouter.async_pre_routing_hook.
+        user_message: Optional[str] = None
+        system_prompt: Optional[str] = None
+
+        for msg in reversed(messages):
+            role = msg.get("role", "")
+            content = msg.get("content") or ""
+            if isinstance(content, list):
+                text_parts = [
+                    part.get("text", "")
+                    for part in content
+                    if isinstance(part, dict) and part.get("type") == "text"
+                ]
+                content = " ".join(text_parts).strip()
+            if isinstance(content, str) and content:
+                if role == "user" and user_message is None:
+                    user_message = content
+                elif role == "system" and system_prompt is None:
+                    system_prompt = content
+
+        if user_message is None:
+            verbose_router_logger.debug(
+                "QualityRouter: No user message found, routing to default model"
+            )
+            if not self.config.default_model:
+                raise ValueError(
+                    "QualityRouter: no user message and no default_model configured"
+                )
+            return PreRoutingHookResponse(
+                model=self.config.default_model,
+                messages=messages,
+            )
+
+        # Try keyword override first — it short-circuits complexity classification.
+        keyword_match = self._keyword_override(user_message)
+        if keyword_match is not None:
+            routed_model, matched_keyword = keyword_match
+            verbose_router_logger.info(
+                f"QualityRouter: keyword override matched='{matched_keyword}' "
+                f"routed_model={routed_model} "
+                f"(quality_tier={self._model_quality.get(routed_model)}, "
+                f"input_cost_per_token={self._model_cost.get(routed_model)})"
+            )
+            self._stash_decision(
+                request_kwargs,
+                {
+                    "router_model_name": self.model_name,
+                    "routed_model": routed_model,
+                    "routed_via": "keyword",
+                    "matched_keyword": matched_keyword,
+                    "quality_tier": self._model_quality.get(routed_model),
+                    "complexity_tier": None,
+                },
+            )
+            return PreRoutingHookResponse(
+                model=routed_model,
+                messages=messages,
+            )
+
+        # No keyword match → complexity classification flow.
+        complexity_tier, score, signals = self._scorer.classify(
+            user_message, system_prompt
+        )
+        complexity_name = (
+            complexity_tier.value
+            if hasattr(complexity_tier, "value")
+            else str(complexity_tier)
+        )
+
+        quality_tier = self.config.complexity_to_quality.get(complexity_name)
+        if quality_tier is None:
+            raise ValueError(
+                f"QualityRouter: complexity tier '{complexity_name}' not present "
+                f"in complexity_to_quality mapping {self.config.complexity_to_quality}"
+            )
+
+        routed_model = self._resolve_model_for_quality_tier(int(quality_tier))
+
+        verbose_router_logger.info(
+            f"QualityRouter: complexity={complexity_name}, score={score:.3f}, "
+            f"signals={signals}, quality_tier={quality_tier}, "
+            f"routed_model={routed_model}"
+        )
+
+        self._stash_decision(
+            request_kwargs,
+            {
+                "router_model_name": self.model_name,
+                "routed_model": routed_model,
+                "routed_via": "quality_tier",
+                "matched_keyword": None,
+                "quality_tier": int(quality_tier),
+                "complexity_tier": complexity_name,
+            },
+        )
+
+        return PreRoutingHookResponse(
+            model=routed_model,
+            messages=messages,
+        )

litellm/types/compression.py

@@ -2,7 +2,14 @@
 Type definitions for litellm.compress().
 """
 
-from typing import Dict, List, TypedDict
+import sys
+
+if sys.version_info >= (3, 11):
+    from typing import Dict, List, NotRequired, TypedDict
+else:
+    from typing import Dict, List, TypedDict
+
+    from typing_extensions import NotRequired
 
 
 class CompressedResult(TypedDict):
@@ -12,3 +19,4 @@ class CompressedResult(TypedDict):
     compression_ratio: float  # fraction reduced, e.g. 0.6 means 60% reduction
     cache: Dict[str, str]  # key -> original content (for retrieval tool responses)
     tools: List[dict]  # [litellm_content_retrieve tool definition]
+    compression_skipped_reason: NotRequired[str]

litellm/types/integrations/compression_interception.py

@@ -0,0 +1,27 @@
+"""
+Type definitions for Compression Interception integration.
+"""
+
+from typing import Any, Dict, Optional, TypedDict
+
+
+class CompressionInterceptionConfig(TypedDict, total=False):
+    """
+    Configuration parameters for CompressionInterceptionLogger.
+
+    Used in proxy_config.yaml under litellm_settings:
+        litellm_settings:
+          compression_interception_params:
+            enabled: true
+            compression_trigger: 100000
+            compression_target: 70000
+            embedding_model: "text-embedding-3-small"
+            embedding_model_params:
+              dimensions: 512
+    """
+
+    enabled: bool
+    compression_trigger: int
+    compression_target: Optional[int]
+    embedding_model: Optional[str]
+    embedding_model_params: Optional[Dict[str, Any]]

litellm/types/integrations/custom_logger.py

@@ -1,6 +1,6 @@
-from typing import Optional
+from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 class StandardCustomLoggerInitParams(BaseModel):
@@ -9,3 +9,29 @@ class StandardCustomLoggerInitParams(BaseModel):
     """
 
     turn_off_message_logging: Optional[bool] = False
+
+
+class AgenticLoopRequestPatch(BaseModel):
+    """
+    Patch returned by callbacks to request a follow-up LLM call.
+    """
+
+    model: Optional[str] = None
+    messages: Optional[List[Dict[str, Any]]] = None
+    tools: Optional[List[Dict[str, Any]]] = None
+    max_tokens: Optional[int] = None
+    optional_params: Dict[str, Any] = Field(default_factory=dict)
+    kwargs: Dict[str, Any] = Field(default_factory=dict)
+
+
+class AgenticLoopPlan(BaseModel):
+    """
+    Typed callback response for agentic-loop reruns.
+    """
+
+    run_agentic_loop: bool = False
+    request_patch: Optional[AgenticLoopRequestPatch] = None
+    response_override: Optional[Any] = None
+    terminate: bool = False
+    stop_reason: Optional[str] = None
+    metadata: Dict[str, Any] = Field(default_factory=dict)

litellm/types/integrations/prometheus.py

@@ -784,7 +784,7 @@ class UserAPIKeyLabelValues:
     org_id: Optional[str] = None
     org_alias: Optional[str] = None
 
-    #Added for test compatibility.
+    # Added for test compatibility.
     def __init__(self, **kwargs: Any) -> None:
         """
         Match former Pydantic behavior: unknown keys are ignored; ``api_key_hash`` maps to

litellm/types/llms/bedrock.py

@@ -997,3 +997,47 @@ class BedrockToolBlock(TypedDict, total=False):
     toolSpec: Optional[ToolSpecBlock]
     systemTool: Optional[SystemToolBlock]  # For Nova grounding
     cachePoint: Optional[CachePointBlock]
+
+
+class BedrockInvokeAnthropicMessagesRequest(TypedDict, total=False):
+    """
+    Top-level request body accepted by AWS Bedrock `InvokeModel` /
+    `InvokeModelWithResponseStream` when calling an Anthropic Claude model with
+    the Messages API format. The LiteLLM /v1/messages → Bedrock Invoke
+    transformation filters outgoing requests to the keys of this TypedDict; any
+    other field (Anthropic-only extension, internal metadata, future addition)
+    is dropped before signing so Bedrock doesn't 400 with
+    "Extra inputs are not permitted".
+
+    Reference:
+        https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages.html
+        https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-anthropic-claude-messages-request-response.html
+
+    Editing this type is the single source of truth — the runtime allowlist in
+    `AmazonAnthropicClaudeMessagesConfig.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS`
+    is derived from `__annotations__`, and a test asserts the resolved set
+    exactly, so any edit forces a conscious review.
+
+    Value types are intentionally loose (`list`, `dict`) — this type exists to
+    pin the allowed field names, not to validate nested structure.
+    """
+
+    # Required by Bedrock
+    anthropic_version: str
+    max_tokens: int
+    messages: list
+
+    # Documented optional fields
+    anthropic_beta: List[str]
+    system: object  # str or list[TextBlock]
+    stop_sequences: List[str]
+    temperature: float
+    top_p: float
+    top_k: int
+    tools: list
+    tool_choice: dict
+
+    # `thinking` is required for Opus 4.5 / Sonnet 4 extended thinking,
+    # `metadata` is part of the common Anthropic Messages API shape.
+    thinking: dict
+    metadata: dict

litellm/types/router.py

@@ -221,6 +221,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams):
     complexity_router_config: Optional[Dict] = None
     complexity_router_default_model: Optional[str] = None
 
+    # quality-router params
+    quality_router_config: Optional[Dict] = None
+    quality_router_default_model: Optional[str] = None
+
     # Batch/File API Params
     s3_bucket_name: Optional[str] = None
     s3_encryption_key_id: Optional[str] = None

litellm/types/utils.py

@@ -2851,6 +2851,7 @@ class StandardAuditLogPayload(TypedDict):
 class StandardLoggingPayload(TypedDict):
     id: str
     trace_id: str  # Trace multiple LLM calls belonging to same overall request (e.g. fallbacks/retries)
+    litellm_call_id: Optional[str]  # UUID returned in x-litellm-call-id response header
     call_type: str
     stream: Optional[bool]
     response_cost: float
@@ -3290,6 +3291,7 @@ class LlmProviders(str, Enum):
     MANUS = "manus"
     WANDB = "wandb"
     OVHCLOUD = "ovhcloud"
+    SCALEWAY = "scaleway"
     LEMONADE = "lemonade"
     AMAZON_NOVA = "amazon_nova"
     A2A_AGENT = "a2a_agent"

litellm/utils.py

@@ -8472,6 +8472,12 @@ class ProviderConfigManager:
             )
 
             return OVHCloudAudioTranscriptionConfig()
+        elif litellm.LlmProviders.SCALEWAY == provider:
+            from litellm.llms.scaleway.audio_transcription.transformation import (
+                ScalewayAudioTranscriptionConfig,
+            )
+
+            return ScalewayAudioTranscriptionConfig()
         elif litellm.LlmProviders.MISTRAL == provider:
             from litellm.llms.mistral.audio_transcription.transformation import (
                 MistralAudioTranscriptionConfig,

model_prices_and_context_window.json

@@ -1148,6 +1148,20 @@
         "tool_use_system_prompt_tokens": 346,
         "supports_native_structured_output": true
     },
+    "anthropic.claude-mythos-preview": {
+        "input_cost_per_token": 0,
+        "output_cost_per_token": 0,
+        "litellm_provider": "bedrock",
+        "max_input_tokens": 1000000,
+        "max_output_tokens": 128000,
+        "max_tokens": 128000,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_vision": true,
+        "supports_prompt_caching": false,
+        "supports_reasoning": true,
+        "supports_tool_choice": true
+    },
     "global.anthropic.claude-opus-4-7": {
         "cache_creation_input_token_cost": 6.25e-06,
         "cache_read_input_token_cost": 5e-07,
@@ -33302,6 +33316,7 @@
         "output_cost_per_token": 1.5e-05,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
         "supports_web_search": true
@@ -33317,6 +33332,7 @@
         "output_cost_per_token": 1.5e-05,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
         "supports_web_search": true
@@ -33332,6 +33348,7 @@
         "output_cost_per_token": 2.5e-05,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
         "supports_web_search": true
@@ -33347,6 +33364,7 @@
         "output_cost_per_token": 2.5e-05,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
         "supports_web_search": true
@@ -33362,6 +33380,7 @@
         "output_cost_per_token": 1.5e-05,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
         "supports_web_search": true
@@ -33378,6 +33397,7 @@
         "output_cost_per_token": 5e-07,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33395,6 +33415,7 @@
         "output_cost_per_token": 5e-07,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33411,6 +33432,7 @@
         "output_cost_per_token": 4e-06,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33427,6 +33449,7 @@
         "output_cost_per_token": 4e-06,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33443,6 +33466,7 @@
         "output_cost_per_token": 4e-06,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33459,6 +33483,7 @@
         "output_cost_per_token": 5e-07,
         "source": "https://x.ai/api#pricing",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": false,
         "supports_tool_choice": true,
@@ -33474,38 +33499,41 @@
         "output_cost_per_token": 1.5e-05,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_web_search": true
     },
     "xai/grok-4-fast-reasoning": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 2e-07,
+        "input_cost_per_token_above_128k_tokens": 4e-07,
         "litellm_provider": "xai",
         "max_input_tokens": 2000000.0,
         "max_output_tokens": 2000000.0,
         "max_tokens": 2000000.0,
         "mode": "chat",
-        "input_cost_per_token": 2e-07,
-        "input_cost_per_token_above_128k_tokens": 4e-07,
         "output_cost_per_token": 5e-07,
         "output_cost_per_token_above_128k_tokens": 1e-06,
-        "cache_read_input_token_cost": 5e-08,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_web_search": true
     },
     "xai/grok-4-fast-non-reasoning": {
+        "cache_read_input_token_cost": 5e-08,
+        "input_cost_per_token": 2e-07,
+        "input_cost_per_token_above_128k_tokens": 4e-07,
         "litellm_provider": "xai",
         "max_input_tokens": 2000000.0,
         "max_output_tokens": 2000000.0,
-        "cache_read_input_token_cost": 5e-08,
         "max_tokens": 2000000.0,
         "mode": "chat",
-        "input_cost_per_token": 2e-07,
-        "input_cost_per_token_above_128k_tokens": 4e-07,
         "output_cost_per_token": 5e-07,
         "output_cost_per_token_above_128k_tokens": 1e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_web_search": true
     },
@@ -33521,6 +33549,7 @@
         "output_cost_per_token_above_128k_tokens": 3e-05,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_web_search": true
     },
@@ -33536,6 +33565,7 @@
         "output_cost_per_token_above_128k_tokens": 3e-05,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_web_search": true
     },
@@ -33553,6 +33583,7 @@
         "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning",
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -33573,6 +33604,7 @@
         "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning",
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -33593,6 +33625,7 @@
         "source": "https://docs.x.ai/docs/models/grok-4-1-fast-reasoning",
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
@@ -33613,6 +33646,7 @@
         "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -33632,6 +33666,7 @@
         "source": "https://docs.x.ai/docs/models/grok-4-1-fast-non-reasoning",
         "supports_audio_input": true,
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_response_schema": true,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -33648,6 +33683,7 @@
         "output_cost_per_token": 6e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -33664,6 +33700,7 @@
         "output_cost_per_token": 6e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true,
         "supports_vision": true,
@@ -33696,6 +33733,7 @@
         "output_cost_per_token": 6e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_tool_choice": true,
         "supports_vision": true,
         "supports_web_search": true
@@ -33724,6 +33762,7 @@
         "output_cost_per_token": 1.5e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
@@ -33738,6 +33777,7 @@
         "output_cost_per_token": 1.5e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
     },
@@ -33752,6 +33792,7 @@
         "output_cost_per_token": 1.5e-06,
         "source": "https://docs.x.ai/docs/models",
         "supports_function_calling": true,
+        "supports_prompt_caching": true,
         "supports_reasoning": true,
         "supports_tool_choice": true
     },

provider_endpoints_support.json

@@ -1968,7 +1968,7 @@
         "responses": true,
         "embeddings": false,
         "image_generations": false,
-        "audio_transcriptions": false,
+        "audio_transcriptions": true,
         "audio_speech": false,
         "moderations": false,
         "batches": false,

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.83.10"
+version = "1.83.11"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.14"
@@ -208,7 +208,7 @@ build-backend = "uv_build"
 
 [tool.uv]
 default-groups = ["dev"]
-required-version = "==0.10.9"
+required-version = ">=0.10.9"
 exclude-newer = "3 days"
 
 [tool.uv.sources]
@@ -236,7 +236,7 @@ source-exclude = [
 profile = "black"
 
 [tool.commitizen]
-version = "1.83.10"
+version = "1.83.11"
 version_files = [
     "pyproject.toml:^version",
 ]

scripts/eval_compression.py

@@ -33,6 +33,7 @@ from dataclasses import asdict, dataclass, field
 from typing import Optional
 
 import litellm
+from litellm.types.utils import CallTypes
 
 # ---------------------------------------------------------------------------
 # Problem definitions (HumanEval-style)
@@ -880,6 +881,7 @@ def eval_problem(
         result = litellm.compress(
             messages=messages,
             model=model,
+            call_type=CallTypes.completion,
             compression_trigger=compression_trigger,
             embedding_model=embedding_model,
         )

tests/eval_swe_bench.py

@@ -40,6 +40,7 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
 
 import litellm  # noqa: E402
 from litellm.compression import compress as litellm_compress  # noqa: E402
+from litellm.types.utils import CallTypes  # noqa: E402
 
 # ---------------------------------------------------------------------------
 # Prompts
@@ -445,7 +446,7 @@ def eval_instance(
         compress_kwargs: dict = {
             "messages": messages,
             "model": model,
-            "input_type": "openai_chat_completions",
+            "call_type": CallTypes.completion,
             "compression_trigger": compression_trigger,
             "embedding_model": embedding_model,
         }

tests/llm_translation/test_bedrock_mantle.py

@@ -0,0 +1,149 @@
+"""
+E2E tests for Bedrock Mantle (Claude Mythos Preview) integration.
+
+Tests use a fake/mocked HTTP layer to verify the full request pipeline:
+- correct endpoint URL
+- model ID in the request body
+- AWS SigV4 Authorization header present
+- response parsing
+"""
+
+import json
+import os
+import sys
+from unittest.mock import MagicMock, patch
+
+import httpx
+import pytest
+
+sys.path.insert(0, os.path.abspath("../.."))
+
+import litellm
+from litellm.llms.custom_httpx.http_handler import HTTPHandler
+
+MODEL = "bedrock/mantle/anthropic.claude-mythos-preview"
+REGION = "us-east-1"
+EXPECTED_URL = f"https://bedrock-mantle.{REGION}.api.aws/v1/messages"
+
+FAKE_ANTHROPIC_RESPONSE = {
+    "id": "msg_fake123",
+    "type": "message",
+    "role": "assistant",
+    "model": "anthropic.claude-mythos-preview",
+    "content": [{"type": "text", "text": "Hello from Mythos!"}],
+    "stop_reason": "end_turn",
+    "stop_sequence": None,
+    "usage": {"input_tokens": 10, "output_tokens": 5},
+}
+
+
+def _make_fake_response(body: dict) -> MagicMock:
+    mock_resp = MagicMock(spec=httpx.Response)
+    mock_resp.status_code = 200
+    mock_resp.headers = httpx.Headers({"content-type": "application/json"})
+    mock_resp.text = json.dumps(body)
+    mock_resp.json.return_value = body
+    mock_resp.is_error = False
+    mock_resp.raise_for_status = MagicMock()
+    return mock_resp
+
+
+def test_mantle_request_url_and_body():
+    """Verify the correct URL is called and model appears in the request body."""
+    client = HTTPHandler()
+
+    with patch.object(
+        client, "post", return_value=_make_fake_response(FAKE_ANTHROPIC_RESPONSE)
+    ) as mock_post:
+        try:
+            litellm.completion(
+                model=MODEL,
+                messages=[{"role": "user", "content": "Hello"}],
+                max_tokens=50,
+                aws_region_name=REGION,
+                aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
+                aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+                client=client,
+            )
+        except Exception:
+            pass  # response parsing may fail on mock; we only care about the outgoing call
+
+        mock_post.assert_called_once()
+        call_kwargs = mock_post.call_args.kwargs
+
+        # Correct endpoint
+        assert (
+            call_kwargs["url"] == EXPECTED_URL
+        ), f"Expected {EXPECTED_URL}, got {call_kwargs['url']}"
+
+        # Request body has model ID (without "mantle/" prefix)
+        raw_data = call_kwargs.get("data") or call_kwargs.get("json")
+        body = json.loads(raw_data) if isinstance(raw_data, (str, bytes)) else raw_data
+        assert (
+            body["model"] == "anthropic.claude-mythos-preview"
+        ), f"body['model'] = {body.get('model')}"
+        assert "messages" in body
+        assert body["max_tokens"] == 50
+
+        # AWS SigV4 Authorization header must be present
+        headers = call_kwargs.get("headers", {})
+        assert "Authorization" in headers, f"No Authorization header in {headers}"
+        assert headers["Authorization"].startswith(
+            "AWS4-HMAC-SHA256"
+        ), f"Expected SigV4 auth, got: {headers['Authorization'][:50]}"
+
+
+def test_mantle_request_does_not_include_mantle_prefix_in_body():
+    """Ensure 'mantle/' never leaks into the request body."""
+    client = HTTPHandler()
+
+    with patch.object(
+        client, "post", return_value=_make_fake_response(FAKE_ANTHROPIC_RESPONSE)
+    ) as mock_post:
+        try:
+            litellm.completion(
+                model=MODEL,
+                messages=[{"role": "user", "content": "Hi"}],
+                max_tokens=10,
+                aws_region_name=REGION,
+                aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
+                aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+                client=client,
+            )
+        except Exception:
+            pass
+
+        call_kwargs = mock_post.call_args.kwargs
+        raw_data = call_kwargs.get("data") or call_kwargs.get("json")
+        body = json.loads(raw_data) if isinstance(raw_data, (str, bytes)) else raw_data
+
+        body_str = json.dumps(body)
+        assert "mantle/" not in body_str, f"'mantle/' leaked into body: {body_str}"
+
+
+def test_mantle_region_reflected_in_url():
+    """The region from aws_region_name must appear in the endpoint URL."""
+    client = HTTPHandler()
+
+    for region in ["us-east-1", "us-west-2", "eu-west-1"]:
+        with patch.object(
+            client, "post", return_value=_make_fake_response(FAKE_ANTHROPIC_RESPONSE)
+        ) as mock_post:
+            try:
+                litellm.completion(
+                    model=MODEL,
+                    messages=[{"role": "user", "content": "Hi"}],
+                    max_tokens=10,
+                    aws_region_name=region,
+                    aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
+                    aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+                    client=client,
+                )
+            except Exception:
+                pass
+
+            call_kwargs = mock_post.call_args.kwargs
+            expected = f"https://bedrock-mantle.{region}.api.aws/v1/messages"
+            assert (
+                call_kwargs["url"] == expected
+            ), f"region={region}: expected URL {expected}, got {call_kwargs['url']}"

tests/local_testing/test_basic_python_version.py

@@ -100,8 +100,12 @@ import pytest
 import requests
 
 
-def test_litellm_proxy_server_config_no_general_settings():
-    # Sync the local litellm packages into the project environment
+def _run_proxy_server_smoke_test(extra_proxy_args=None):
+    """Sync deps, generate Prisma client, start proxy with optional extra args,
+    send a health check + chat/completions request, and tear down."""
+    if extra_proxy_args is None:
+        extra_proxy_args = []
+
     server_process = None
     try:
         _run_uv(
@@ -144,6 +148,7 @@ def test_litellm_proxy_server_config_no_general_settings():
                 "litellm.proxy.proxy_cli",
                 "--config",
                 config_fp,
+                *extra_proxy_args,
             ],
             cwd=PROJECT_ROOT,
         )
@@ -182,3 +187,17 @@ def test_litellm_proxy_server_config_no_general_settings():
 
     # Additional assertions can be added here
     assert True
+
+
+def test_litellm_proxy_server_config_no_general_settings():
+    """Exercises the default (v1) migration resolver."""
+    _run_proxy_server_smoke_test()
+
+
+def test_litellm_proxy_server_config_no_general_settings_v2_resolver():
+    """Exercises the opt-in v2 migration resolver.
+
+    Runs in a separate CI job against a local Postgres to avoid collisions
+    with the v1 variant when they share a database.
+    """
+    _run_proxy_server_smoke_test(extra_proxy_args=["--use_v2_migration_resolver"])

tests/local_testing/test_batch_completions.py

@@ -72,7 +72,7 @@ def test_batch_completions_models():
 def test_batch_completion_models_all_responses():
     try:
         responses = batch_completion_models_all_responses(
-            models=["gemini/gemini-2.5-flash-lite", "claude-3-haiku-20240307"],
+            models=["gemini/gemini-2.5-flash-lite", "claude-haiku-4-5-20251001"],
             messages=[{"role": "user", "content": "write a poem"}],
             max_tokens=10,
         )

tests/local_testing/test_function_call_parsing.py

@@ -142,7 +142,7 @@ def trade(model_name: str) -> List[Trade]:  # type: ignore
 
 
 @pytest.mark.parametrize(
-    "model", ["claude-3-haiku-20240307", "anthropic.claude-3-haiku-20240307-v1:0"]
+    "model", ["claude-haiku-4-5-20251001", "anthropic.claude-3-haiku-20240307-v1:0"]
 )
 @pytest.mark.flaky(retries=6, delay=10)
 def test_function_call_parsing(model):

tests/local_testing/test_function_calling.py

@@ -47,7 +47,7 @@ def get_current_weather(location, unit="fahrenheit"):
     [
         "gpt-3.5-turbo-1106",
         "mistral/mistral-large-latest",
-        "claude-3-haiku-20240307",
+        "claude-haiku-4-5-20251001",
         "gemini/gemini-2.5-flash-lite",
         "anthropic.claude-3-sonnet-20240229-v1:0",
     ],
@@ -275,7 +275,7 @@ from litellm.types.utils import ChatCompletionMessageToolCall, Function, Message
             "anthropic.claude-3-sonnet-20240229-v1:0",
             "bedrock",
         ),
-        ("claude-3-haiku-20240307", "anthropic"),
+        ("claude-haiku-4-5-20251001", "anthropic"),
     ],
 )
 @pytest.mark.parametrize(

tests/local_testing/test_router_fallbacks.py

@@ -1509,7 +1509,7 @@ def test_router_fallbacks_with_wildcard_model_name():
             {
                 "model_name": "claude-3-haiku",
                 "litellm_params": {
-                    "model": "claude-3-haiku-20240307",
+                    "model": "claude-haiku-4-5-20251001",
                     "api_key": os.getenv("ANTHROPIC_API_KEY"),
                     "mock_response": "Hi this is claude!",
                 },
@@ -1555,7 +1555,7 @@ def test_fallbacks_with_different_messages():
             {
                 "model_name": "claude-3-haiku",
                 "litellm_params": {
-                    "model": "claude-3-haiku-20240307",
+                    "model": "claude-haiku-4-5-20251001",
                     "api_key": os.getenv("ANTHROPIC_API_KEY"),
                 },
             },

tests/local_testing/test_streaming.py

@@ -1727,7 +1727,7 @@ def test_openai_chat_completion_complete_response_call():
     "model",
     [
         "gpt-3.5-turbo",
-        "claude-3-haiku-20240307",
+        "claude-haiku-4-5-20251001",
         "o1",
     ],
 )
@@ -2247,7 +2247,7 @@ def streaming_and_function_calling_format_tests(idx, chunk):
     [
         # "gpt-3.5-turbo",
         # "anthropic.claude-3-sonnet-20240229-v1:0",
-        "claude-3-haiku-20240307",
+        "claude-haiku-4-5-20251001",
     ],
 )
 def test_streaming_and_function_calling(model):

tests/logging_callback_tests/test_otel_logging.py

@@ -253,6 +253,7 @@ def validate_redacted_message_span_attributes(span):
             or attr.startswith("gen_ai.cost.")
             or attr.startswith("gen_ai.operation.")
             or attr.startswith("gen_ai.request.")
+            or attr.startswith("litellm.")
         ), f"Non-metadata attribute found: {attr}"
 
     pass

tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py

@@ -77,7 +77,7 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest):
     @property
     def model_config(self) -> Dict[str, Any]:
         return {
-            "model": "claude-3-haiku-20240307",
+            "model": "claude-haiku-4-5-20251001",
             "api_key": os.getenv("ANTHROPIC_API_KEY"),
         }
 
@@ -86,7 +86,7 @@ class TestAnthropicDirectAPI(BaseAnthropicMessagesTest):
         """
         This is the model name that is expected to be in the logging payload
         """
-        return "claude-3-haiku-20240307"
+        return "claude-haiku-4-5-20251001"
 
 
 class TestAnthropicBedrockAPI(BaseAnthropicMessagesTest):
@@ -140,7 +140,7 @@ async def test_anthropic_messages_streaming_with_bad_request():
         response = await litellm.anthropic.messages.acreate(
             messages=[{"role": "user", "content": "hi"}],
             api_key=os.getenv("ANTHROPIC_API_KEY"),
-            model="claude-3-haiku-20240307",
+            model="claude-haiku-4-5-20251001",
             max_tokens=100,
             stream=True,
         )
@@ -168,7 +168,7 @@ async def test_anthropic_messages_router_streaming_with_bad_request():
                 {
                     "model_name": "claude-special-alias",
                     "litellm_params": {
-                        "model": "claude-3-haiku-20240307",
+                        "model": "claude-haiku-4-5-20251001",
                         "api_key": os.getenv("ANTHROPIC_API_KEY"),
                     },
                 }
@@ -205,7 +205,7 @@ async def test_anthropic_messages_litellm_router_non_streaming():
             {
                 "model_name": "claude-special-alias",
                 "litellm_params": {
-                    "model": "claude-3-haiku-20240307",
+                    "model": "claude-haiku-4-5-20251001",
                     "api_key": os.getenv("ANTHROPIC_API_KEY"),
                 },
             }
@@ -243,7 +243,7 @@ async def test_anthropic_messages_litellm_router_routing_strategy():
             {
                 "model_name": "claude-special-alias",
                 "litellm_params": {
-                    "model": "claude-3-haiku-20240307",
+                    "model": "claude-haiku-4-5-20251001",
                     "api_key": os.getenv("ANTHROPIC_API_KEY"),
                 },
             }
@@ -341,7 +341,7 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking():
             "type": "message",
             "role": "assistant",
             "content": [{"type": "text", "text": "Here's a joke for you!"}],
-            "model": "claude-3-haiku-20240307",
+            "model": "claude-haiku-4-5-20251001",
             "stop_reason": "end_turn",
             "usage": {"input_tokens": 10, "output_tokens": 20},
         }
@@ -355,7 +355,7 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking():
                 {
                     "model_name": MODEL_GROUP,
                     "litellm_params": {
-                        "model": "claude-3-haiku-20240307",
+                        "model": "claude-haiku-4-5-20251001",
                         "api_key": os.getenv("ANTHROPIC_API_KEY"),
                     },
                 }
@@ -419,7 +419,7 @@ async def test_anthropic_messages_litellm_router_latency_metadata_tracking():
         assert "model_info" in litellm_metadata
 
         # Verify other call parameters
-        assert call_kwargs["model"] == "claude-3-haiku-20240307"
+        assert call_kwargs["model"] == "claude-haiku-4-5-20251001"
         assert call_kwargs["messages"] == messages
         assert call_kwargs["max_tokens"] == 100
         assert call_kwargs["metadata"] == {"user_id": "hello"}
@@ -459,7 +459,7 @@ async def test_anthropic_messages_litellm_router_non_streaming_with_logging():
             {
                 "model_name": MODEL_GROUP,
                 "litellm_params": {
-                    "model": "claude-3-haiku-20240307",
+                    "model": "claude-haiku-4-5-20251001",
                     "api_key": os.getenv("ANTHROPIC_API_KEY"),
                 },
             }
@@ -496,7 +496,7 @@ async def test_anthropic_messages_litellm_router_non_streaming_with_logging():
     assert test_custom_logger.logged_standard_logging_payload["response"] is not None
     assert (
         test_custom_logger.logged_standard_logging_payload["model"]
-        == "claude-3-haiku-20240307"
+        == "claude-haiku-4-5-20251001"
     )
 
     # check logged usage + spend
@@ -543,7 +543,7 @@ async def test_anthropic_messages_with_extra_headers():
                 "text": "Why did the chicken cross the road? To get to the other side!",
             }
         ],
-        "model": "claude-3-haiku-20240307",
+        "model": "claude-haiku-4-5-20251001",
         "stop_reason": "end_turn",
         "usage": {"input_tokens": 10, "output_tokens": 20},
     }
@@ -556,7 +556,7 @@ async def test_anthropic_messages_with_extra_headers():
     response = await litellm.anthropic.messages.acreate(
         messages=messages,
         api_key=api_key,
-        model="claude-3-haiku-20240307",
+        model="claude-haiku-4-5-20251001",
         max_tokens=100,
         client=mock_client,
         provider_specific_header={
@@ -689,7 +689,7 @@ async def test_anthropic_messages_with_thinking():
                 "text": "Why did the chicken cross the road? To get to the other side!",
             }
         ],
-        "model": "claude-3-haiku-20240307",
+        "model": "claude-haiku-4-5-20251001",
         "stop_reason": "end_turn",
         "usage": {"input_tokens": 10, "output_tokens": 20},
     }
@@ -702,7 +702,7 @@ async def test_anthropic_messages_with_thinking():
     response = await litellm.anthropic.messages.acreate(
         messages=messages,
         api_key=api_key,
-        model="claude-3-haiku-20240307",
+        model="claude-haiku-4-5-20251001",
         max_tokens=100,
         client=mock_client,
         thinking={"budget_tokens": 100},
@@ -717,7 +717,7 @@ async def test_anthropic_messages_with_thinking():
     request_body = json.loads(call_kwargs.get("data", {}))
     print("REQUEST BODY", request_body)
     assert request_body["max_tokens"] == 100
-    assert request_body["model"] == "claude-3-haiku-20240307"
+    assert request_body["model"] == "claude-haiku-4-5-20251001"
     assert request_body["messages"] == messages
     assert request_body["thinking"] == {"budget_tokens": 100}
 

tests/proxy_unit_tests/conftest.py

@@ -1,48 +1,143 @@
 # conftest.py
 
-import importlib
+import asyncio
+import copy
+import inspect
 import os
 import sys
+import warnings
 
 import pytest
 
 sys.path.insert(
     0, os.path.abspath("../..")
 )  # Adds the parent directory to the system path
+
 import litellm
+import litellm.proxy.proxy_server
+
+
+# Top-level assignments of these types are the ones importlib.reload(litellm)
+# would have effectively reset. We snapshot them at conftest import time and
+# deep-copy the snapshot back before every test.
+_SNAPSHOT_TYPES = (list, dict, set, tuple, str, int, float, bool, bytes)
+
+
+def _snapshot_mutable_state(module):
+    """Capture a per-module snapshot of primitive and collection attributes."""
+    snapshot = {}
+    for attr in list(vars(module)):
+        if attr.startswith("_"):
+            continue
+        try:
+            value = getattr(module, attr)
+        except Exception as exc:
+            warnings.warn(
+                f"conftest: could not read {module.__name__}.{attr} during snapshot: {exc}",
+                stacklevel=2,
+            )
+            continue
+        if value is None or isinstance(value, _SNAPSHOT_TYPES):
+            try:
+                snapshot[attr] = copy.deepcopy(value)
+            except Exception as exc:
+                warnings.warn(
+                    f"conftest: could not snapshot {module.__name__}.{attr}: {exc}",
+                    stacklevel=2,
+                )
+    return snapshot
+
+
+def _restore_mutable_state(module, snapshot):
+    for attr, default in snapshot.items():
+        try:
+            setattr(module, attr, copy.deepcopy(default))
+        except Exception as exc:
+            warnings.warn(
+                f"conftest: could not restore {module.__name__}.{attr}: {exc}",
+                stacklevel=2,
+            )
+
+
+def _collect_flushable_caches():
+    """Return (module, attr) pairs whose values expose flush_cache()."""
+    targets = []
+    for module in (litellm, litellm.proxy.proxy_server):
+        for attr in list(vars(module)):
+            if attr.startswith("_"):
+                continue
+            try:
+                value = getattr(module, attr)
+            except Exception:
+                continue
+            # Only instances — a class reference has an unbound flush_cache
+            # that can't be called without a self argument.
+            if inspect.isclass(value) or inspect.ismodule(value):
+                continue
+            if callable(getattr(value, "flush_cache", None)):
+                targets.append((module, attr))
+    return targets
+
+
+def _flush_caches(targets):
+    for module, attr in targets:
+        try:
+            value = getattr(module, attr)
+        except Exception:
+            continue
+        flush = getattr(value, "flush_cache", None)
+        if callable(flush):
+            try:
+                flush()
+            except Exception as exc:
+                warnings.warn(
+                    f"conftest: flush_cache failed on {module.__name__}.{attr}: {exc}",
+                    stacklevel=2,
+                )
+
+
+# Snapshot once at conftest import — these are the "clean" module states.
+_LITELLM_STATE = _snapshot_mutable_state(litellm)
+_PROXY_SERVER_STATE = _snapshot_mutable_state(litellm.proxy.proxy_server)
+_FLUSHABLE_CACHES = _collect_flushable_caches()
 
 
 @pytest.fixture(scope="function", autouse=True)
 def setup_and_teardown():
+    """Reset mutable module state on litellm and proxy_server before each test.
+
+    Replaces a previous importlib.reload(litellm) approach that cost ~17s
+    per test (re-executing the full litellm __init__ import chain).
+
+    What IS reset:
+      - Top-level module attributes of type list / dict / set / tuple
+        / str / int / float / bool / bytes, and None-valued attributes.
+        These cover callback lists, general_settings, master_key,
+        premium_user, prisma_client, etc. — anything the old reload() reset
+        by re-executing the module body.
+      - Any module-level object instance that exposes flush_cache() (the
+        DualCache and LLMClientCache family), which handles cache state
+        that can't round-trip through deepcopy because of internal locks.
+
+    What is NOT reset:
+      - Class instances without flush_cache() (e.g. ProxyLogging,
+        JWTHandler, FastAPI routers, loggers). If a test mutates such an
+        instance in-place (setattr on the instance, appending to one of
+        its internal lists, etc.), the mutation will leak into later tests.
+        Use pytest's monkeypatch.setattr() or a local fixture for those
+        cases — don't rely on this autouse fixture to undo them.
     """
-    This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained.
-    """
-    curr_dir = os.getcwd()  # Get the current working directory
-    sys.path.insert(
-        0, os.path.abspath("../..")
-    )  # Adds the project directory to the system path
-
-    import litellm
-    from litellm import Router
-
-    importlib.reload(litellm)
-    try:
-        if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"):
-            importlib.reload(litellm.proxy.proxy_server)
-    except Exception as e:
-        print(f"Error reloading litellm.proxy.proxy_server: {e}")
-
-    import asyncio
+    _restore_mutable_state(litellm, _LITELLM_STATE)
+    _restore_mutable_state(litellm.proxy.proxy_server, _PROXY_SERVER_STATE)
+    _flush_caches(_FLUSHABLE_CACHES)
 
     loop = asyncio.get_event_loop_policy().new_event_loop()
     asyncio.set_event_loop(loop)
-    print(litellm)
-    # from litellm import Router, completion, aembedding, acompletion, embedding
-    yield
-
-    # Teardown code (executes after the yield point)
-    loop.close()  # Close the loop created earlier
-    asyncio.set_event_loop(None)  # Remove the reference to the loop
+    try:
+        yield
+    finally:
+        loop.close()
+        asyncio.set_event_loop(None)
 
 
 def pytest_collection_modifyitems(config, items):

tests/test_litellm/integrations/compression_interception/test_compression_interception_handler.py

@@ -0,0 +1,364 @@
+"""
+Unit tests for Compression Interception Handler.
+"""
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from litellm.integrations.compression_interception.handler import (
+    CompressionInterceptionLogger,
+)
+from litellm.types.utils import CallTypes
+
+
+def test_initialize_from_proxy_config():
+    """Test initialization from proxy config with litellm_settings."""
+    litellm_settings = {
+        "compression_interception_params": {
+            "enabled": True,
+            "compression_trigger": 1234,
+            "compression_target": 789,
+        }
+    }
+
+    logger = CompressionInterceptionLogger.initialize_from_proxy_config(
+        litellm_settings=litellm_settings,
+        callback_specific_params={},
+    )
+
+    assert logger.enabled is True
+    assert logger.compression_trigger == 1234
+    assert logger.compression_target == 789
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_compresses_messages_and_injects_tool(monkeypatch):
+    """Test pre-call hook compresses and stores per-call cache."""
+    logger = CompressionInterceptionLogger()
+    compressed_result = {
+        "messages": [{"role": "user", "content": "stubbed"}],
+        "original_tokens": 12000,
+        "compressed_tokens": 5000,
+        "compression_ratio": 0.58,
+        "cache": {"auth.py": "full file content"},
+        "tools": [
+            {
+                "type": "function",
+                "function": {
+                    "name": "litellm_content_retrieve",
+                    "parameters": {
+                        "type": "object",
+                        "properties": {"key": {"type": "string"}},
+                    },
+                },
+            }
+        ],
+    }
+
+    def _fake_compress(**kwargs):
+        return compressed_result
+
+    # The handler does ``from litellm.compression import compress`` at module
+    # scope, so we must patch the binding on the handler module — patching
+    # ``litellm.compress`` has no effect on the already-bound reference.
+    monkeypatch.setattr(
+        "litellm.integrations.compression_interception.handler.compress",
+        _fake_compress,
+    )
+
+    kwargs = {
+        "model": "bedrock/us.anthropic.claude-sonnet-4-5",
+        "messages": [{"role": "user", "content": "very large context"}],
+        "tools": [
+            {
+                "type": "function",
+                "function": {"name": "existing_tool", "parameters": {"type": "object"}},
+            }
+        ],
+    }
+
+    result = await logger.async_pre_call_deployment_hook(
+        kwargs=kwargs, call_type=CallTypes.anthropic_messages
+    )
+
+    assert result is not None
+    assert result["messages"] == compressed_result["messages"]
+    tool_names = [t.get("function", {}).get("name") for t in result["tools"]]
+    assert "existing_tool" in tool_names
+    assert "litellm_content_retrieve" in tool_names
+    assert result["litellm_call_id"] in logger._compression_cache_by_call_id
+
+
+@pytest.mark.asyncio
+async def test_pre_call_hook_below_trigger_does_not_inject_empty_tools(monkeypatch):
+    """
+    When compression is a no-op (below trigger / invalid tool sequence), the
+    hook must NOT replace ``messages`` or inject an empty ``tools: []`` onto
+    a request that originally had no tools — Anthropic Messages rejects
+    ``tools: []``.
+    """
+    logger = CompressionInterceptionLogger()
+    original_messages = [{"role": "user", "content": "short prompt"}]
+
+    def _fake_compress_noop(**kwargs):
+        return {
+            "messages": original_messages,
+            "original_tokens": 42,
+            "compressed_tokens": 42,
+            "compression_ratio": 0.0,
+            "cache": {},
+            "tools": [],
+            "compression_skipped_reason": "below_trigger",
+        }
+
+    monkeypatch.setattr(
+        "litellm.integrations.compression_interception.handler.compress",
+        _fake_compress_noop,
+    )
+
+    kwargs = {
+        "model": "bedrock/us.anthropic.claude-sonnet-4-5",
+        "messages": original_messages,
+    }
+
+    result = await logger.async_pre_call_deployment_hook(
+        kwargs=kwargs, call_type=CallTypes.anthropic_messages
+    )
+
+    assert result is not None
+    # Original request had no ``tools`` — skipped compression must leave it that way.
+    assert "tools" not in result
+    # Cache must not be populated for a no-op.
+    assert result.get("litellm_call_id") not in logger._compression_cache_by_call_id
+
+
+@pytest.mark.asyncio
+async def test_should_run_agentic_loop_detects_retrieval_tool_use():
+    """Test should-run hook returns tool calls for retrieval tool_use blocks."""
+    logger = CompressionInterceptionLogger()
+    response = {
+        "content": [
+            {
+                "type": "tool_use",
+                "id": "toolu_123",
+                "name": "litellm_content_retrieve",
+                "input": {"key": "auth.py"},
+            }
+        ]
+    }
+
+    should_run, tools_dict = await logger.async_should_run_agentic_loop(
+        response=response,
+        model="bedrock/claude",
+        messages=[],
+        tools=[
+            {
+                "type": "function",
+                "function": {
+                    "name": "litellm_content_retrieve",
+                    "parameters": {"type": "object"},
+                },
+            }
+        ],
+        stream=False,
+        custom_llm_provider="bedrock",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert len(tools_dict["tool_calls"]) == 1
+    assert tools_dict["tool_calls"][0]["input"]["key"] == "auth.py"
+
+
+@pytest.mark.asyncio
+async def test_build_agentic_loop_plan_returns_request_patch():
+    """Callback should return typed patch with tool_result content."""
+    logger = CompressionInterceptionLogger()
+    call_id = "call_123"
+    logger._compression_cache_by_call_id[call_id] = (
+        {"auth.py": "full auth file"},
+        9999999999.0,
+    )
+
+    logging_obj = MagicMock()
+    logging_obj.litellm_call_id = call_id
+    logging_obj.model_call_details = {
+        "agentic_loop_params": {"model": "bedrock/invoke/claude-3-5-sonnet"}
+    }
+
+    plan = await logger.async_build_agentic_loop_plan(
+        tools={
+            "tool_calls": [
+                {
+                    "id": "toolu_abc",
+                    "type": "tool_use",
+                    "name": "litellm_content_retrieve",
+                    "input": {"key": "auth.py"},
+                }
+            ]
+        },
+        model="claude-3-5-sonnet",
+        messages=[{"role": "user", "content": "read auth.py"}],
+        response=None,
+        anthropic_messages_provider_config=None,
+        anthropic_messages_optional_request_params={
+            "max_tokens": 1024,
+            "tools": [{"name": "litellm_content_retrieve"}],
+        },
+        logging_obj=logging_obj,
+        stream=False,
+        kwargs={
+            "temperature": 0.1,
+            "_compression_interception_internal": True,
+            "litellm_logging_obj": object(),
+        },
+    )
+
+    assert plan.run_agentic_loop is True
+    assert plan.request_patch is not None
+    assert plan.request_patch.model == "bedrock/invoke/claude-3-5-sonnet"
+    assert plan.request_patch.max_tokens == 1024
+    assert plan.request_patch.messages is not None
+    assert len(plan.request_patch.messages) == 3
+    tool_result_content = plan.request_patch.messages[-1]["content"][0]["content"]
+    assert tool_result_content == "full auth file"
+    assert "_compression_interception_internal" not in plan.request_patch.kwargs
+    assert "litellm_logging_obj" not in plan.request_patch.kwargs
+    assert plan.request_patch.kwargs["temperature"] == 0.1
+    assert "max_tokens" not in plan.request_patch.optional_params
+
+
+@pytest.mark.asyncio
+async def test_should_run_agentic_loop_with_custom_type_tools():
+    """Test that async_should_run_agentic_loop returns True when tools contain
+    litellm_content_retrieve as a custom-typed tool (e.g. Claude Code tool list)
+    and the model response includes a matching tool_use block."""
+    logger = CompressionInterceptionLogger()
+
+    # Exact tools payload produced by Claude Code – litellm_content_retrieve is
+    # the final entry and uses type="custom" (not type="function").
+    tools = [
+        {
+            "name": "Agent",
+            "description": "Launch a new agent to handle complex, multi-step tasks.",
+            "input_schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {
+                    "description": {"type": "string"},
+                    "prompt": {"type": "string"},
+                },
+                "required": ["description", "prompt"],
+                "additionalProperties": False,
+            },
+        },
+        {
+            "name": "AskUserQuestion",
+            "description": "Use this tool when you need to ask the user questions.",
+            "input_schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {
+                    "questions": {"type": "array", "items": {"type": "object"}},
+                },
+                "required": ["questions"],
+                "additionalProperties": False,
+            },
+        },
+        {
+            "name": "Bash",
+            "description": "Executes a given bash command and returns its output.",
+            "input_schema": {
+                "$schema": "https://json-schema.org/draft/2020-12/schema",
+                "type": "object",
+                "properties": {"command": {"type": "string"}},
+                "required": ["command"],
+                "additionalProperties": False,
+            },
+        },
+        {
+            "name": "litellm_content_retrieve",
+            "description": "Retrieve the full content of a file or message that was compressed to save tokens.",
+            "input_schema": {
+                "type": "object",
+                "properties": {
+                    "key": {
+                        "type": "string",
+                        "description": "The identifier of the content to retrieve",
+                        "enum": [
+                            "message_0",
+                            "HA_UPTIME_ROUTER_SPEC.md",
+                            "message_159",
+                            "message_160",
+                        ],
+                    }
+                },
+                "required": ["key"],
+            },
+            "type": "custom",
+        },
+    ]
+
+    response = {
+        "content": [
+            {
+                "type": "tool_use",
+                "id": "toolu_abc",
+                "name": "litellm_content_retrieve",
+                "input": {"key": "message_0"},
+            }
+        ]
+    }
+
+    should_run, tools_dict = await logger.async_should_run_agentic_loop(
+        response=response,
+        model="claude-3-5-sonnet",
+        messages=[],
+        tools=tools,
+        stream=False,
+        custom_llm_provider="anthropic",
+        kwargs={},
+    )
+
+    assert should_run is True
+    assert tools_dict["tool_type"] == "compression_retrieval"
+    assert len(tools_dict["tool_calls"]) == 1
+    assert tools_dict["tool_calls"][0]["input"]["key"] == "message_0"
+
+
+@pytest.mark.asyncio
+async def test_build_agentic_loop_plan_missing_key_fallback():
+    """Missing cache keys should produce deterministic fallback content."""
+    logger = CompressionInterceptionLogger()
+
+    logging_obj = MagicMock()
+    logging_obj.litellm_call_id = "missing_call"
+    logging_obj.model_call_details = {"agentic_loop_params": {}}
+
+    plan = await logger.async_build_agentic_loop_plan(
+        tools={
+            "tool_calls": [
+                {
+                    "id": "toolu_missing",
+                    "type": "tool_use",
+                    "name": "litellm_content_retrieve",
+                    "input": {"key": "not_found.py"},
+                }
+            ]
+        },
+        model="claude-3-5-sonnet",
+        messages=[{"role": "user", "content": "read file"}],
+        response=None,
+        anthropic_messages_provider_config=None,
+        anthropic_messages_optional_request_params={},
+        logging_obj=logging_obj,
+        stream=False,
+        kwargs={},
+    )
+
+    assert plan.request_patch is not None
+    assert (
+        plan.request_patch.messages[-1]["content"][0]["content"]
+        == "[compressed content key 'not_found.py' not found]"
+    )

tests/test_litellm/integrations/test_opentelemetry.py

@@ -2752,3 +2752,26 @@ class TestResponseIdFallback(unittest.TestCase):
         mock_span.set_attribute.assert_any_call(
             "gen_ai.response.id", "litellm-img-call-101"
         )
+
+    def test_litellm_call_id_emitted_as_span_attribute(self):
+        """litellm.call_id must be set on the span from standard_logging_payload."""
+        otel = OpenTelemetry()
+        mock_span = MagicMock()
+
+        call_id = "my-litellm-call-uuid-456"
+        kwargs = {
+            "model": "gpt-4o",
+            "optional_params": {},
+            "litellm_params": {"custom_llm_provider": "openai"},
+            "standard_logging_object": {
+                "id": "chatcmpl-provider-id",
+                "litellm_call_id": call_id,
+                "call_type": "completion",
+                "metadata": {},
+            },
+        }
+        response_obj = {"id": "chatcmpl-provider-id", "model": "gpt-4o"}
+
+        otel.set_attributes(mock_span, kwargs, response_obj)
+
+        mock_span.set_attribute.assert_any_call("litellm.call_id", call_id)

tests/test_litellm/integrations/websearch_interception/test_websearch_interception_handler.py

@@ -4,7 +4,7 @@ Unit tests for WebSearch Interception Handler
 Tests the WebSearchInterceptionLogger class and helper functions.
 """
 
-from unittest.mock import MagicMock, Mock
+from unittest.mock import AsyncMock, MagicMock, Mock
 
 import pytest
 
@@ -69,6 +69,61 @@ async def test_async_should_run_agentic_loop():
     assert tools_dict == {}
 
 
+@pytest.mark.asyncio
+async def test_async_build_agentic_loop_plan_returns_request_patch():
+    """Callback should return a typed patch for base handler reruns."""
+    logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
+    logger._execute_search = AsyncMock(  # type: ignore
+        return_value="Title: LiteLLM\nURL: docs\nSnippet: test"
+    )
+
+    tools_dict = {
+        "tool_calls": [
+            {
+                "id": "toolu_123",
+                "type": "tool_use",
+                "name": "litellm_web_search",
+                "input": {"query": "what is litellm"},
+            }
+        ],
+        "response_format": "anthropic",
+    }
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {
+        "agentic_loop_params": {"model": "bedrock/invoke/claude-3-5-sonnet"}
+    }
+    kwargs = {
+        "temperature": 0.2,
+        "_websearch_interception_converted_stream": True,
+        "litellm_logging_obj": object(),
+    }
+
+    plan = await logger.async_build_agentic_loop_plan(
+        tools=tools_dict,
+        model="claude-3-5-sonnet",
+        messages=[{"role": "user", "content": "search LiteLLM"}],
+        response=None,
+        anthropic_messages_provider_config=None,
+        anthropic_messages_optional_request_params={
+            "max_tokens": 1024,
+            "tools": [{"name": "litellm_web_search"}],
+        },
+        logging_obj=logging_obj,
+        stream=False,
+        kwargs=kwargs,
+    )
+
+    assert plan.run_agentic_loop is True
+    assert plan.request_patch is not None
+    assert plan.request_patch.model == "bedrock/invoke/claude-3-5-sonnet"
+    assert plan.request_patch.max_tokens == 1024
+    assert plan.request_patch.messages is not None
+    assert len(plan.request_patch.messages) == 3
+    assert "_websearch_interception_converted_stream" not in plan.request_patch.kwargs
+    assert "litellm_logging_obj" not in plan.request_patch.kwargs
+    assert plan.request_patch.kwargs["temperature"] == 0.2
+
+
 @pytest.mark.asyncio
 async def test_internal_flags_filtered_from_followup_kwargs():
     """Test that internal _websearch_interception flags are filtered from follow-up request kwargs.

tests/test_litellm/litellm_core_utils/test_litellm_logging.py

@@ -2410,3 +2410,29 @@ def test_get_additional_headers_reset_fields_preserved():
     assert result is not None
     assert result["x_ratelimit_reset_requests"] == "1s"  # type: ignore
     assert result["x_ratelimit_reset_tokens"] == "100ms"  # type: ignore
+
+
+# ── litellm_call_id propagation ───────────────────────────────────────────────
+
+
+def test_get_standard_logging_object_payload_includes_litellm_call_id(logging_obj):
+    """litellm_call_id from kwargs must appear in the returned StandardLoggingPayload."""
+    import datetime
+
+    from litellm.litellm_core_utils.litellm_logging import (
+        get_standard_logging_object_payload,
+    )
+
+    call_id = "test-call-id-abc-123"
+    now = datetime.datetime.now()
+    payload = get_standard_logging_object_payload(
+        kwargs={"litellm_call_id": call_id, "model": "gpt-4o", "messages": []},
+        init_response_obj={},
+        start_time=now,
+        end_time=now,
+        logging_obj=logging_obj,
+        status="success",
+    )
+
+    assert payload is not None
+    assert payload["litellm_call_id"] == call_id

tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_agentic_streaming_iterator.py

@@ -0,0 +1,792 @@
+"""
+Tests for AgenticAnthropicStreamingIterator and SSE rebuild helpers.
+"""
+
+import json
+import os
+import sys
+from typing import Any, Dict, List, Optional, Tuple
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+sys.path.insert(0, os.path.abspath("../../../../.."))
+
+from litellm.llms.anthropic.experimental_pass_through.messages.agentic_streaming_iterator import (
+    AgenticAnthropicStreamingIterator,
+    _handle_content_block_delta,
+    _handle_content_block_start,
+    _handle_content_block_stop,
+    _handle_message_delta,
+    _handle_message_start,
+    _parse_sse_events,
+)
+
+
+# ---------------------------------------------------------------------------
+# Helpers to build SSE byte payloads
+# ---------------------------------------------------------------------------
+
+
+def _sse_event(event_type: str, data: dict) -> bytes:
+    return f"event: {event_type}\ndata: {json.dumps(data)}\n\n".encode()
+
+
+def _build_simple_text_stream() -> List[bytes]:
+    """Produce SSE bytes for a simple text response (no tool calls)."""
+    chunks = []
+    chunks.append(
+        _sse_event(
+            "message_start",
+            {
+                "type": "message_start",
+                "message": {
+                    "id": "msg_123",
+                    "type": "message",
+                    "role": "assistant",
+                    "model": "claude-sonnet-4-20250514",
+                    "content": [],
+                    "stop_reason": None,
+                    "stop_sequence": None,
+                    "usage": {"input_tokens": 10, "output_tokens": 0},
+                },
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_start",
+            {
+                "type": "content_block_start",
+                "index": 0,
+                "content_block": {"type": "text", "text": ""},
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_delta",
+            {
+                "type": "content_block_delta",
+                "index": 0,
+                "delta": {"type": "text_delta", "text": "Hello, world!"},
+            },
+        )
+    )
+    chunks.append(
+        _sse_event("content_block_stop", {"type": "content_block_stop", "index": 0})
+    )
+    chunks.append(
+        _sse_event(
+            "message_delta",
+            {
+                "type": "message_delta",
+                "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+                "usage": {"output_tokens": 5},
+            },
+        )
+    )
+    chunks.append(_sse_event("message_stop", {"type": "message_stop"}))
+    return chunks
+
+
+def _build_tool_use_stream() -> List[bytes]:
+    """Produce SSE bytes for a response with a tool_use block."""
+    chunks = []
+    chunks.append(
+        _sse_event(
+            "message_start",
+            {
+                "type": "message_start",
+                "message": {
+                    "id": "msg_tool_456",
+                    "type": "message",
+                    "role": "assistant",
+                    "model": "claude-sonnet-4-20250514",
+                    "content": [],
+                    "stop_reason": None,
+                    "usage": {"input_tokens": 50, "output_tokens": 0},
+                },
+            },
+        )
+    )
+    # thinking block
+    chunks.append(
+        _sse_event(
+            "content_block_start",
+            {
+                "type": "content_block_start",
+                "index": 0,
+                "content_block": {
+                    "type": "thinking",
+                    "thinking": "",
+                    "signature": "",
+                },
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_delta",
+            {
+                "type": "content_block_delta",
+                "index": 0,
+                "delta": {
+                    "type": "thinking_delta",
+                    "thinking": "I need to retrieve...",
+                },
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_delta",
+            {
+                "type": "content_block_delta",
+                "index": 0,
+                "delta": {"type": "signature_delta", "signature": "sig_abc"},
+            },
+        )
+    )
+    chunks.append(
+        _sse_event("content_block_stop", {"type": "content_block_stop", "index": 0})
+    )
+    # tool_use block
+    chunks.append(
+        _sse_event(
+            "content_block_start",
+            {
+                "type": "content_block_start",
+                "index": 1,
+                "content_block": {
+                    "type": "tool_use",
+                    "id": "toolu_001",
+                    "name": "litellm_content_retrieve",
+                    "input": {},
+                },
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_delta",
+            {
+                "type": "content_block_delta",
+                "index": 1,
+                "delta": {
+                    "type": "input_json_delta",
+                    "partial_json": '{"key": "section_',
+                },
+            },
+        )
+    )
+    chunks.append(
+        _sse_event(
+            "content_block_delta",
+            {
+                "type": "content_block_delta",
+                "index": 1,
+                "delta": {"type": "input_json_delta", "partial_json": '1"}'},
+            },
+        )
+    )
+    chunks.append(
+        _sse_event("content_block_stop", {"type": "content_block_stop", "index": 1})
+    )
+    chunks.append(
+        _sse_event(
+            "message_delta",
+            {
+                "type": "message_delta",
+                "delta": {"stop_reason": "tool_use"},
+                "usage": {"output_tokens": 20},
+            },
+        )
+    )
+    chunks.append(_sse_event("message_stop", {"type": "message_stop"}))
+    return chunks
+
+
+# ---------------------------------------------------------------------------
+# Mock async stream
+# ---------------------------------------------------------------------------
+
+
+class MockAsyncStream:
+    """Async iterator that yields a list of byte chunks."""
+
+    def __init__(self, chunks: List[bytes]):
+        self._chunks = list(chunks)
+        self._idx = 0
+
+    def __aiter__(self):
+        return self
+
+    async def __anext__(self) -> bytes:
+        if self._idx >= len(self._chunks):
+            raise StopAsyncIteration
+        chunk = self._chunks[self._idx]
+        self._idx += 1
+        return chunk
+
+
+# ---------------------------------------------------------------------------
+# Tests for _parse_sse_events
+# ---------------------------------------------------------------------------
+
+
+class TestParseSSEEvents:
+    def test_should_parse_single_event(self):
+        raw = _sse_event(
+            "message_start", {"type": "message_start", "message": {"id": "1"}}
+        )
+        events = _parse_sse_events(raw)
+        assert len(events) == 1
+        assert events[0][0] == "message_start"
+        assert events[0][1]["message"]["id"] == "1"
+
+    def test_should_parse_multiple_events(self):
+        raw = b"".join(_build_simple_text_stream())
+        events = _parse_sse_events(raw)
+        event_types = [e[0] for e in events]
+        assert "message_start" in event_types
+        assert "content_block_start" in event_types
+        assert "content_block_delta" in event_types
+        assert "content_block_stop" in event_types
+        assert "message_delta" in event_types
+        assert "message_stop" in event_types
+
+    def test_should_skip_malformed_json(self):
+        raw = b"event: message_start\ndata: {invalid json}\n\n"
+        events = _parse_sse_events(raw)
+        assert len(events) == 0
+
+    def test_should_handle_empty_bytes(self):
+        events = _parse_sse_events(b"")
+        assert events == []
+
+
+# ---------------------------------------------------------------------------
+# Tests for _handle_* helpers
+# ---------------------------------------------------------------------------
+
+
+class TestHandleMessageStart:
+    def test_should_populate_envelope(self):
+        response: Dict[str, Any] = {
+            "id": "",
+            "model": "",
+            "role": "assistant",
+            "usage": {"input_tokens": 0, "output_tokens": 0},
+        }
+        data = {
+            "message": {
+                "id": "msg_abc",
+                "model": "claude-sonnet-4-20250514",
+                "role": "assistant",
+                "usage": {
+                    "input_tokens": 42,
+                    "cache_creation_input_tokens": 100,
+                },
+            }
+        }
+        _handle_message_start(data, response)
+        assert response["id"] == "msg_abc"
+        assert response["model"] == "claude-sonnet-4-20250514"
+        assert response["usage"]["input_tokens"] == 42
+        assert response["usage"]["cache_creation_input_tokens"] == 100
+
+
+class TestHandleContentBlockStart:
+    def test_should_create_text_block(self):
+        blocks: Dict[int, Dict] = {}
+        data = {"index": 0, "content_block": {"type": "text", "text": ""}}
+        _handle_content_block_start(data, blocks)
+        assert blocks[0] == {"type": "text", "text": ""}
+
+    def test_should_create_tool_use_block(self):
+        blocks: Dict[int, Dict] = {}
+        data = {
+            "index": 1,
+            "content_block": {
+                "type": "tool_use",
+                "id": "toolu_x",
+                "name": "my_tool",
+                "input": {},
+            },
+        }
+        _handle_content_block_start(data, blocks)
+        assert blocks[1]["type"] == "tool_use"
+        assert blocks[1]["name"] == "my_tool"
+        assert blocks[1]["_partial_json"] == ""
+
+    def test_should_create_thinking_block(self):
+        blocks: Dict[int, Dict] = {}
+        data = {
+            "index": 0,
+            "content_block": {"type": "thinking", "thinking": "", "signature": ""},
+        }
+        _handle_content_block_start(data, blocks)
+        assert blocks[0]["type"] == "thinking"
+
+
+class TestHandleContentBlockDelta:
+    def test_should_accumulate_text(self):
+        blocks = {0: {"type": "text", "text": "Hello"}}
+        _handle_content_block_delta(
+            {"index": 0, "delta": {"type": "text_delta", "text": " World"}},
+            blocks,
+        )
+        assert blocks[0]["text"] == "Hello World"
+
+    def test_should_accumulate_json(self):
+        blocks = {0: {"type": "tool_use", "_partial_json": '{"key":'}}
+        _handle_content_block_delta(
+            {
+                "index": 0,
+                "delta": {"type": "input_json_delta", "partial_json": '"val"}'},
+            },
+            blocks,
+        )
+        assert blocks[0]["_partial_json"] == '{"key":"val"}'
+
+    def test_should_ignore_missing_block(self):
+        blocks: Dict[int, Dict] = {}
+        _handle_content_block_delta(
+            {"index": 99, "delta": {"type": "text_delta", "text": "x"}},
+            blocks,
+        )
+        assert 99 not in blocks
+
+
+class TestHandleContentBlockStop:
+    def test_should_parse_tool_input_json(self):
+        blocks = {
+            0: {
+                "type": "tool_use",
+                "input": {},
+                "_partial_json": '{"key": "section_1"}',
+            }
+        }
+        _handle_content_block_stop({"index": 0}, blocks)
+        assert blocks[0]["input"] == {"key": "section_1"}
+        assert "_partial_json" not in blocks[0]
+
+    def test_should_handle_invalid_json_gracefully(self):
+        blocks = {
+            0: {
+                "type": "tool_use",
+                "input": {},
+                "_partial_json": "not valid json",
+            }
+        }
+        _handle_content_block_stop({"index": 0}, blocks)
+        assert blocks[0]["input"] == {"_raw": "not valid json"}
+
+
+class TestHandleMessageDelta:
+    def test_should_set_stop_reason_and_usage(self):
+        response: Dict[str, Any] = {
+            "stop_reason": None,
+            "stop_sequence": None,
+            "usage": {"input_tokens": 0, "output_tokens": 0},
+        }
+        _handle_message_delta(
+            {
+                "delta": {"stop_reason": "end_turn", "stop_sequence": None},
+                "usage": {"output_tokens": 15},
+            },
+            response,
+        )
+        assert response["stop_reason"] == "end_turn"
+        assert response["usage"]["output_tokens"] == 15
+
+
+# ---------------------------------------------------------------------------
+# Tests for _rebuild_anthropic_response_from_sse
+# ---------------------------------------------------------------------------
+
+
+class TestRebuildAnthropicResponse:
+    def test_should_rebuild_simple_text_response(self):
+        raw_bytes = _build_simple_text_stream()
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            raw_bytes
+        )
+        assert result is not None
+        assert result["id"] == "msg_123"
+        assert result["model"] == "claude-sonnet-4-20250514"
+        assert result["stop_reason"] == "end_turn"
+        assert len(result["content"]) == 1
+        assert result["content"][0]["type"] == "text"
+        assert result["content"][0]["text"] == "Hello, world!"
+        assert result["usage"]["input_tokens"] == 10
+        assert result["usage"]["output_tokens"] == 5
+
+    def test_should_rebuild_tool_use_response(self):
+        raw_bytes = _build_tool_use_stream()
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            raw_bytes
+        )
+        assert result is not None
+        assert result["id"] == "msg_tool_456"
+        assert result["stop_reason"] == "tool_use"
+        assert len(result["content"]) == 2
+
+        thinking = result["content"][0]
+        assert thinking["type"] == "thinking"
+        assert thinking["thinking"] == "I need to retrieve..."
+        assert thinking["signature"] == "sig_abc"
+
+        tool = result["content"][1]
+        assert tool["type"] == "tool_use"
+        assert tool["id"] == "toolu_001"
+        assert tool["name"] == "litellm_content_retrieve"
+        assert tool["input"] == {"key": "section_1"}
+
+    def test_should_return_none_without_message_start(self):
+        raw_bytes = [
+            _sse_event(
+                "content_block_start",
+                {
+                    "type": "content_block_start",
+                    "index": 0,
+                    "content_block": {"type": "text"},
+                },
+            )
+        ]
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            raw_bytes
+        )
+        assert result is None
+
+    def test_should_handle_empty_bytes(self):
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            []
+        )
+        assert result is None
+
+    def test_should_handle_multi_event_chunks(self):
+        """When multiple SSE events arrive in a single bytes chunk."""
+        combined = b"".join(_build_simple_text_stream())
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            [combined]
+        )
+        assert result is not None
+        assert result["content"][0]["text"] == "Hello, world!"
+
+    def test_should_preserve_cache_usage_fields(self):
+        raw_bytes = [
+            _sse_event(
+                "message_start",
+                {
+                    "type": "message_start",
+                    "message": {
+                        "id": "msg_cache",
+                        "model": "claude-sonnet-4-20250514",
+                        "role": "assistant",
+                        "usage": {
+                            "input_tokens": 100,
+                            "cache_creation_input_tokens": 50,
+                            "cache_read_input_tokens": 30,
+                        },
+                    },
+                },
+            ),
+            _sse_event(
+                "message_delta",
+                {
+                    "type": "message_delta",
+                    "delta": {"stop_reason": "end_turn"},
+                    "usage": {"output_tokens": 10},
+                },
+            ),
+            _sse_event("message_stop", {"type": "message_stop"}),
+        ]
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            raw_bytes
+        )
+        assert result is not None
+        assert result["usage"]["cache_creation_input_tokens"] == 50
+        assert result["usage"]["cache_read_input_tokens"] == 30
+
+    def test_should_handle_redacted_thinking_block(self):
+        raw_bytes = [
+            _sse_event(
+                "message_start",
+                {
+                    "type": "message_start",
+                    "message": {
+                        "id": "msg_redact",
+                        "model": "claude-sonnet-4-20250514",
+                        "role": "assistant",
+                        "usage": {"input_tokens": 5},
+                    },
+                },
+            ),
+            _sse_event(
+                "content_block_start",
+                {
+                    "type": "content_block_start",
+                    "index": 0,
+                    "content_block": {"type": "redacted_thinking", "data": "abc123"},
+                },
+            ),
+            _sse_event(
+                "content_block_stop",
+                {"type": "content_block_stop", "index": 0},
+            ),
+            _sse_event(
+                "message_delta",
+                {
+                    "type": "message_delta",
+                    "delta": {"stop_reason": "end_turn"},
+                    "usage": {"output_tokens": 1},
+                },
+            ),
+            _sse_event("message_stop", {"type": "message_stop"}),
+        ]
+        result = AgenticAnthropicStreamingIterator._rebuild_anthropic_response_from_sse(
+            raw_bytes
+        )
+        assert result is not None
+        assert result["content"][0]["type"] == "redacted_thinking"
+
+
+# ---------------------------------------------------------------------------
+# Tests for AgenticAnthropicStreamingIterator (Phase 1 / Phase 2)
+# ---------------------------------------------------------------------------
+
+
+class TestAgenticStreamingIteratorPhase1:
+    @pytest.mark.asyncio
+    async def test_should_yield_all_chunks_when_no_hook_fires(self):
+        """When hooks return None, the wrapper should yield all original chunks."""
+        chunks = _build_simple_text_stream()
+        mock_stream = MockAsyncStream(chunks)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[{"role": "user", "content": "hi"}],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert len(collected) == len(chunks)
+        for orig, got in zip(chunks, collected):
+            assert orig == got
+
+        mock_handler._call_agentic_completion_hooks.assert_awaited_once()
+
+    @pytest.mark.asyncio
+    async def test_should_pass_rebuilt_response_to_hooks(self):
+        """The rebuilt dict passed to hooks should match the original stream content."""
+        chunks = _build_tool_use_stream()
+        mock_stream = MockAsyncStream(chunks)
+
+        captured_response = {}
+
+        async def mock_hooks(**kwargs):
+            captured_response.update(kwargs["response"])
+            return None
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = mock_hooks
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        async for _ in iterator:
+            pass
+
+        assert captured_response["id"] == "msg_tool_456"
+        assert captured_response["stop_reason"] == "tool_use"
+        assert captured_response["content"][1]["name"] == "litellm_content_retrieve"
+
+
+class TestAgenticStreamingIteratorPhase2:
+    @pytest.mark.asyncio
+    async def test_should_chain_follow_up_async_iterator(self):
+        """When hooks return an async iterator, Phase 2 should yield from it."""
+        phase1_chunks = _build_simple_text_stream()
+        phase2_chunks = [b"follow-up-chunk-1", b"follow-up-chunk-2"]
+
+        mock_stream = MockAsyncStream(phase1_chunks)
+        follow_up = MockAsyncStream(phase2_chunks)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=follow_up)
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert len(collected) == len(phase1_chunks) + len(phase2_chunks)
+        assert collected[-2:] == phase2_chunks
+
+    @pytest.mark.asyncio
+    async def test_should_convert_dict_response_to_fake_stream(self):
+        """When hooks return a dict, it should be wrapped in FakeAnthropicMessagesStreamIterator."""
+        phase1_chunks = _build_simple_text_stream()
+        mock_stream = MockAsyncStream(phase1_chunks)
+
+        fake_response = {
+            "id": "msg_followup",
+            "type": "message",
+            "role": "assistant",
+            "model": "claude-sonnet-4-20250514",
+            "content": [{"type": "text", "text": "follow-up answer"}],
+            "stop_reason": "end_turn",
+            "stop_sequence": None,
+            "usage": {"input_tokens": 100, "output_tokens": 20},
+        }
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(
+            return_value=fake_response
+        )
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        # Phase 1 chunks + Phase 2 fake-stream chunks
+        assert len(collected) > len(phase1_chunks)
+        # The follow-up chunks should contain the text from the dict response
+        phase2_bytes = b"".join(collected[len(phase1_chunks) :])
+        assert b"follow-up answer" in phase2_bytes
+
+
+class TestAgenticStreamingIteratorErrorHandling:
+    @pytest.mark.asyncio
+    async def test_should_swallow_hook_errors(self):
+        """Errors in hook processing should be swallowed; Phase 1 chunks are still yielded."""
+        chunks = _build_simple_text_stream()
+        mock_stream = MockAsyncStream(chunks)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(
+            side_effect=RuntimeError("hook exploded")
+        )
+
+        mock_logging = MagicMock()
+        mock_logging.litellm_call_id = "test_call_123"
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=mock_logging,
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        # All Phase 1 chunks should still have been yielded
+        assert len(collected) == len(chunks)
+
+    @pytest.mark.asyncio
+    async def test_should_handle_empty_stream(self):
+        """An empty upstream stream should not crash."""
+        mock_stream = MockAsyncStream([])
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        collected = []
+        async for chunk in iterator:
+            collected.append(chunk)
+
+        assert collected == []
+        # hooks should not be called since no bytes were collected
+        mock_handler._call_agentic_completion_hooks.assert_not_awaited()
+
+    @pytest.mark.asyncio
+    async def test_should_pass_stream_true_to_hooks(self):
+        """The wrapper should always pass stream=True to hooks."""
+        chunks = _build_simple_text_stream()
+        mock_stream = MockAsyncStream(chunks)
+
+        mock_handler = MagicMock()
+        mock_handler._call_agentic_completion_hooks = AsyncMock(return_value=None)
+
+        iterator = AgenticAnthropicStreamingIterator(
+            completion_stream=mock_stream,
+            http_handler=mock_handler,
+            model="claude-sonnet-4-20250514",
+            messages=[],
+            anthropic_messages_provider_config=MagicMock(),
+            anthropic_messages_optional_request_params={},
+            logging_obj=MagicMock(),
+            custom_llm_provider="anthropic",
+            kwargs={},
+        )
+
+        async for _ in iterator:
+            pass
+
+        call_kwargs = mock_handler._call_agentic_completion_hooks.call_args
+        assert call_kwargs.kwargs["stream"] is True

tests/test_litellm/llms/bedrock/messages/invoke_transformations/test_anthropic_claude3_transformation.py

@@ -579,6 +579,155 @@ def test_bedrock_messages_strips_output_config_with_output_format():
     assert "output_format" not in result
 
 
+def test_bedrock_messages_strips_context_management():
+    """
+    Ensure context_management is stripped from the request before sending to
+    Bedrock Invoke, which doesn't support this Anthropic-specific parameter.
+
+    Claude Code sends context_management on every request; leaving it in the body
+    causes a 400 "context_management: Extra inputs are not permitted" from Bedrock.
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {
+        "max_tokens": 4096,
+        "context_management": {
+            "edits": [{"type": "clear_thinking_20251015", "keep": "all"}]
+        },
+    }
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    assert (
+        "context_management" not in result
+    ), "context_management should be stripped — Bedrock Invoke rejects it"
+    assert result.get("max_tokens") == 4096
+
+
+def test_bedrock_messages_allowlist_filters_anthropic_only_fields():
+    """
+    Bedrock Invoke rejects any top-level body field it doesn't recognize with
+    "Extra inputs are not permitted". Defend against that by filtering the
+    outgoing body to a Bedrock-supported allowlist — catches Anthropic-only
+    extensions (speed, mcp_servers, container, ...) and any future additions
+    Claude Code starts sending before we learn about them.
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {
+        "max_tokens": 4096,
+        "temperature": 0.5,
+        "speed": "fast",
+        "mcp_servers": [{"type": "url", "url": "https://example.com"}],
+        "container": {"skills": []},
+        "inference_geo": "us",
+        "output_config": {"effort": "low"},
+        "context_management": {"edits": []},
+    }
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers={},
+    )
+
+    for bad in (
+        "speed",
+        "mcp_servers",
+        "container",
+        "inference_geo",
+        "output_config",
+        "context_management",
+        "model",
+        "stream",
+    ):
+        assert bad not in result, f"{bad} should be stripped by the allowlist"
+
+    # Supported fields pass through.
+    assert result["max_tokens"] == 4096
+    assert result["temperature"] == 0.5
+    assert result["anthropic_version"] == cfg.DEFAULT_BEDROCK_ANTHROPIC_API_VERSION
+    # Every surviving key is in the allowlist.
+    assert set(result).issubset(cfg.BEDROCK_INVOKE_ALLOWED_TOP_LEVEL_FIELDS)
+
+
+def test_bedrock_messages_filters_user_provided_unsupported_beta_header():
+    """
+    In proxy deployments the client (e.g. Claude Code) doesn't know the backend
+    is Bedrock and may send Anthropic-direct beta headers Bedrock can't handle.
+    All betas must go through the provider mapping, not just auto-injected ones
+    — otherwise Bedrock 400s on the unsupported value.
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {"max_tokens": 128}
+    # `advisor-tool-2026-03-01` has no bedrock mapping entry → must be dropped.
+    # `context-1m-2025-08-07` does → must pass through.
+    headers = {
+        "anthropic-beta": "advisor-tool-2026-03-01,context-1m-2025-08-07",
+    }
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers=headers,
+    )
+
+    betas = result.get("anthropic_beta") or []
+    assert (
+        "advisor-tool-2026-03-01" not in betas
+    ), "user-provided beta not in the Bedrock mapping must be dropped"
+    assert (
+        "context-1m-2025-08-07" in betas
+    ), "user-provided beta that IS in the Bedrock mapping should survive"
+
+
+def test_bedrock_messages_renames_user_provided_aliased_beta_header():
+    """
+    Bedrock's config maps `advanced-tool-use-2025-11-20` to
+    `tool-search-tool-2025-10-19`. User-provided betas must go through the
+    rename too, not be forwarded under their Anthropic-direct spelling.
+    """
+    from litellm.types.router import GenericLiteLLMParams
+
+    cfg = AmazonAnthropicClaudeMessagesConfig()
+    messages = [{"role": "user", "content": [{"type": "text", "text": "Hello"}]}]
+    optional_params = {"max_tokens": 128}
+    headers = {"anthropic-beta": "advanced-tool-use-2025-11-20"}
+
+    result = cfg.transform_anthropic_messages_request(
+        model="anthropic.claude-3-haiku-20240307-v1:0",
+        messages=messages,
+        anthropic_messages_optional_request_params=optional_params,
+        litellm_params=GenericLiteLLMParams(),
+        headers=headers,
+    )
+
+    betas = result.get("anthropic_beta") or []
+    assert (
+        "advanced-tool-use-2025-11-20" not in betas
+    ), "Anthropic-direct spelling should be rewritten, not forwarded verbatim"
+    assert (
+        "tool-search-tool-2025-10-19" in betas
+    ), "user-provided beta should be renamed to the Bedrock-side spelling"
+
+
 @pytest.mark.asyncio
 async def test_promote_message_stop_usage_preserves_message_delta_output_tokens():
     """

tests/test_litellm/llms/bedrock/test_anthropic_beta_support.py

@@ -95,7 +95,7 @@ class TestAnthropicBetaHeaderSupport:
     def test_messages_transformation_anthropic_beta(self):
         """Test that Messages API transformation includes anthropic_beta in request."""
         config = AmazonAnthropicClaudeMessagesConfig()
-        headers = {"anthropic-beta": "output-128k-2025-02-19"}
+        headers = {"anthropic-beta": "context-1m-2025-08-07"}
 
         result = config.transform_anthropic_messages_request(
             model="anthropic.claude-haiku-4-5-20251001-v1:0",
@@ -107,7 +107,7 @@ class TestAnthropicBetaHeaderSupport:
 
         assert "anthropic_beta" in result
         # Sort both arrays before comparing to avoid flakiness from ordering differences
-        assert sorted(result["anthropic_beta"]) == sorted(["output-128k-2025-02-19"])
+        assert sorted(result["anthropic_beta"]) == sorted(["context-1m-2025-08-07"])
 
     def test_converse_computer_use_compatibility(self):
         """Test that user anthropic_beta headers work with computer use tools."""

tests/test_litellm/llms/bedrock/test_mantle.py

@@ -0,0 +1,105 @@
+"""
+Unit tests for the Bedrock Mantle (Claude Mythos Preview) integration.
+
+Tests cover route detection, URL construction, and config dispatch for both
+the /chat/completions and /messages endpoints.
+"""
+
+from litellm.llms.bedrock.common_utils import BedrockModelInfo, get_bedrock_chat_config
+from litellm.llms.bedrock.chat.mantle.transformation import AmazonMantleConfig
+from litellm.llms.bedrock.messages.mantle_transformation import (
+    AmazonMantleMessagesConfig,
+)
+
+
+def test_get_bedrock_route_mantle():
+    assert (
+        BedrockModelInfo.get_bedrock_route("mantle/anthropic.claude-mythos-preview")
+        == "mantle"
+    )
+
+
+def test_get_bedrock_route_mantle_does_not_match_other_routes():
+    assert (
+        BedrockModelInfo.get_bedrock_route("anthropic.claude-3-sonnet-20240229-v1:0")
+        != "mantle"
+    )
+    assert (
+        BedrockModelInfo.get_bedrock_route("converse/anthropic.claude-3-sonnet")
+        != "mantle"
+    )
+
+
+def test_explicit_mantle_route_flag():
+    assert (
+        BedrockModelInfo._explicit_mantle_route(
+            "mantle/anthropic.claude-mythos-preview"
+        )
+        is True
+    )
+    assert BedrockModelInfo._explicit_mantle_route("anthropic.claude-3-sonnet") is False
+    assert (
+        BedrockModelInfo._explicit_mantle_route("converse/anthropic.claude-3-sonnet")
+        is False
+    )
+
+
+def test_mantle_url_construction():
+    config = AmazonMantleConfig()
+    url = config.get_complete_url(
+        api_base=None,
+        api_key=None,
+        model="mantle/anthropic.claude-mythos-preview",
+        optional_params={"aws_region_name": "us-east-1"},
+        litellm_params={},
+    )
+    assert url == "https://bedrock-mantle.us-east-1.api.aws/v1/messages"
+
+
+def test_mantle_url_construction_different_region():
+    config = AmazonMantleConfig()
+    url = config.get_complete_url(
+        api_base=None,
+        api_key=None,
+        model="mantle/anthropic.claude-mythos-preview",
+        optional_params={"aws_region_name": "us-west-2"},
+        litellm_params={},
+    )
+    assert url == "https://bedrock-mantle.us-west-2.api.aws/v1/messages"
+
+
+def test_get_bedrock_chat_config_returns_mantle_config():
+    config = get_bedrock_chat_config("mantle/anthropic.claude-mythos-preview")
+    assert isinstance(config, AmazonMantleConfig)
+
+
+def test_get_bedrock_provider_config_for_messages_api_mantle():
+    config = BedrockModelInfo.get_bedrock_provider_config_for_messages_api(
+        "mantle/anthropic.claude-mythos-preview"
+    )
+    assert isinstance(config, AmazonMantleMessagesConfig)
+
+
+def test_mantle_messages_url_construction():
+    config = AmazonMantleMessagesConfig()
+    url = config.get_complete_url(
+        api_base=None,
+        api_key=None,
+        model="mantle/anthropic.claude-mythos-preview",
+        optional_params={"aws_region_name": "us-east-1"},
+        litellm_params={},
+    )
+    assert url == "https://bedrock-mantle.us-east-1.api.aws/v1/messages"
+
+
+def test_mantle_transform_request_strips_prefix_and_adds_model():
+    config = AmazonMantleConfig()
+    request = config.transform_request(
+        model="mantle/anthropic.claude-mythos-preview",
+        messages=[{"role": "user", "content": "Hello"}],
+        optional_params={"max_tokens": 100},
+        litellm_params={},
+        headers={},
+    )
+    assert request["model"] == "anthropic.claude-mythos-preview"
+    assert "mantle/" not in request["model"]

tests/test_litellm/llms/custom_httpx/test_llm_http_handler.py

@@ -74,6 +74,34 @@ def test_prepare_fake_stream_request():
     assert result_data["messages"] == [{"role": "user", "content": "Hello"}]
 
 
+def test_get_agentic_loop_settings_defaults_and_overrides():
+    handler = BaseLLMHTTPHandler()
+
+    depth, max_loops, fingerprints = handler._get_agentic_loop_settings(kwargs={})
+    assert depth == 0
+    assert max_loops == 3
+    assert fingerprints == []
+
+    depth, max_loops, fingerprints = handler._get_agentic_loop_settings(
+        kwargs={
+            "_agentic_loop_depth": 2,
+            "max_agentic_loops": 7,
+            "_agentic_loop_fingerprints": ["fp-1", "fp-2"],
+        }
+    )
+    assert depth == 2
+    assert max_loops == 7
+    assert fingerprints == ["fp-1", "fp-2"]
+
+
+def test_fingerprint_agentic_tools_is_deterministic():
+    handler = BaseLLMHTTPHandler()
+    tools_a = {"tool_calls": [{"id": "1", "input": {"q": "abc"}, "name": "web_search"}]}
+    tools_b = {"tool_calls": [{"name": "web_search", "input": {"q": "abc"}, "id": "1"}]}
+
+    assert handler._fingerprint_agentic_tools(tools_a) == handler._fingerprint_agentic_tools(tools_b)
+
+
 @pytest.mark.asyncio
 async def test_async_anthropic_messages_handler_extra_headers():
     """

tests/test_litellm/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py

@@ -891,6 +891,61 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
         assert result == responses_so_far
 
 
+class TestGetStructuredMessages:
+    """Test the get_structured_messages method."""
+
+    def test_should_return_messages_from_chat_completions_request(self):
+        """Test that messages are returned from a chat completions request."""
+        handler = OpenAIChatCompletionsHandler()
+        data = {
+            "messages": [
+                {"role": "system", "content": "You are helpful."},
+                {"role": "user", "content": "Hello"},
+            ]
+        }
+        result = handler.get_structured_messages(data)
+        assert result is not None
+        assert len(result) == 2
+        assert result[0]["role"] == "system"
+        assert result[1]["role"] == "user"
+
+    def test_should_return_none_when_no_messages(self):
+        """Test that None is returned when no messages key exists."""
+        handler = OpenAIChatCompletionsHandler()
+        data = {"model": "gpt-4"}
+        result = handler.get_structured_messages(data)
+        assert result is None
+
+    def test_should_return_none_for_none_messages(self):
+        """Test that None is returned when messages is explicitly None."""
+        handler = OpenAIChatCompletionsHandler()
+        data = {"messages": None}
+        result = handler.get_structured_messages(data)
+        assert result is None
+
+    def test_should_handle_multimodal_content(self):
+        """Test that messages with multimodal content are returned."""
+        handler = OpenAIChatCompletionsHandler()
+        data = {
+            "messages": [
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What's in this image?"},
+                        {
+                            "type": "image_url",
+                            "image_url": {"url": "https://example.com/image.png"},
+                        },
+                    ],
+                }
+            ]
+        }
+        result = handler.get_structured_messages(data)
+        assert result is not None
+        assert len(result) == 1
+        assert isinstance(result[0]["content"], list)
+
+
 if __name__ == "__main__":
     # Run the tests
     pytest.main([__file__, "-v"])