diff --git a/.circleci/config.yml b/.circleci/config.yml index 1462891fa7..a8a33335ad 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -111,6 +111,28 @@ commands: - wait_for_service: url: tcp://localhost:6379 timeout: "60" + start_openai_record_replay_proxy: + description: "Start the record/replay proxy (tests/_openai_record_replay_proxy.py) on host port 8090 and wait until healthy. Models whose api_base points here replay recorded provider responses, so the E2E run neither pays for nor depends on the live provider. The default upstream is OpenAI; a non-OpenAI model must point its api_base at /__recorder_upstream// so the recorder forwards there instead of defaulting to OpenAI. Run after uv deps are synced." + steps: + - run: + name: Start record/replay proxy + background: true + command: | + CASSETTE_REDIS_URL="$CASSETTE_REDIS_URL" \ + RECORDER_UPSTREAM_BASE_URL="https://api.openai.com" \ + uv run --no-sync python tests/_openai_record_replay_proxy.py --host 0.0.0.0 --port 8090 + - run: + name: Wait for record/replay proxy + command: | + for i in $(seq 1 30); do + if curl -sf http://localhost:8090/__recorder_health >/dev/null 2>&1; then + echo "record/replay proxy is up" + exit 0 + fi + sleep 1 + done + echo "record/replay proxy did not become ready" >&2 + exit 1 setup_litellm_enterprise_pip: steps: - run: @@ -182,7 +204,14 @@ jobs: - run: name: Run Windows-specific test command: | - uv run --no-sync python -m pytest tests/windows_tests/test_litellm_on_windows.py -v + uv run --no-sync python -m pytest tests/windows_tests/ -v + - run: + name: Guard against MAX_PATH-busting packaged wheel paths + environment: + UV_HTTP_TIMEOUT: "300" + command: | + uv build --wheel --out-dir dist + uv run --no-sync python tests/windows_tests/check_windows_wheel_install.py local_testing_part1: docker: @@ -228,7 +257,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm \ --cov-report=xml \ @@ -242,8 +271,15 @@ jobs: - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_part1_coverage.xml - mv .coverage local_testing_part1_coverage + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part1_coverage.xml + mv .coverage local_testing_part1_coverage + else + touch local_testing_part1_coverage.xml local_testing_part1_coverage + fi # Store test results - store_test_results: @@ -293,7 +329,7 @@ jobs: echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm \ --cov-report=xml \ @@ -307,8 +343,15 @@ jobs: - run: name: Rename the coverage files command: | - mv coverage.xml local_testing_part2_coverage.xml - mv .coverage local_testing_part2_coverage + # When CI reruns only the failed tests, a parallel node can receive + # zero tests and pytest never writes coverage. Emit empty placeholders + # so persist_to_workspace and the downstream coverage combine stay green. + if [ -f coverage.xml ]; then + mv coverage.xml local_testing_part2_coverage.xml + mv .coverage local_testing_part2_coverage + else + touch local_testing_part2_coverage.xml local_testing_part2_coverage + fi # Store test results - store_test_results: @@ -356,7 +399,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -409,7 +452,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/proxy_admin_ui_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -431,6 +474,120 @@ jobs: - auth_ui_unit_tests_coverage.xml - auth_ui_unit_tests_coverage + proxy_behavior_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy management behavior tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_behavior \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + proxy_security_tests: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Seed DB schema via prisma db push + command: | + uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Run proxy security tests + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_security_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + + schema_migration_check: + docker: + - *python312_image + - image: cimg/postgres:16.0@sha256:b125148bc76e8e8eee5eb3ad6020a3a14110a14e8192f1c645128afebe2e2f84 + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: litellm_test + working_directory: ~/project + environment: + # An empty database; the test applies every committed migration itself. + DATABASE_URL: "postgresql://postgres:postgres@localhost:5432/litellm_test" + steps: + - checkout + - setup_google_dns + - install_uv + - run: + name: Install Dependencies + command: | + uv sync --frozen --all-groups --all-extras --python 3.12 + - wait_for_service: + url: tcp://localhost:5432 + timeout: "60" + - run: + name: Generate Prisma Client + command: uv run --no-sync python -m prisma generate + - run: + name: Check schema.prisma is in sync with committed migrations + command: | + mkdir -p test-results + uv run --no-sync python -m pytest tests/proxy_migration_tests \ + -v --junitxml=test-results/junit.xml --durations=10 + no_output_timeout: 15m + - store_test_results: + path: test-results + litellm_router_testing: # Runs all tests with the "router" keyword docker: - *python312_image @@ -457,12 +614,17 @@ jobs: - run: name: Run tests command: | + # On a "rerun failed tests" build a parallel node can receive no + # tests, so the test command never creates test-results. Pre-create it + # so store_test_results doesn't fail the node on a missing path. + mkdir -p test-results + TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --split-by=timings \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ -k 'router' \ -n 4 \ @@ -504,7 +666,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/router_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -547,7 +709,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/local_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -589,7 +751,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/llm_translation/**/test_*.py" | grep -v "^tests/llm_translation/realtime/") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=20 \ @@ -625,7 +787,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/llm_translation/realtime/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -668,7 +830,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/agent_tests/**/test_*.py" | grep -v "^tests/agent_tests/local_only_agent_tests/") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -710,7 +872,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/guardrails_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -754,7 +916,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/unified_google_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -805,7 +967,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -836,7 +998,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/ocr_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -878,7 +1040,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/search_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -922,7 +1084,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/enterprise/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-enterprise.xml \ --durations=10 \ @@ -952,7 +1114,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/batches_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -994,7 +1156,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/litellm_utils_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -1037,7 +1199,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/pass_through_unit_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -1080,7 +1242,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/image_gen_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5 \ @@ -1112,7 +1274,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/logging_callback_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv \ --cov=./litellm --cov-report=xml \ -n 4 \ @@ -1155,7 +1317,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/audio_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -1206,7 +1368,7 @@ jobs: tests/local_testing/test_router_utils.py) echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --cov=./litellm --cov-report=xml \ --junitxml=test-results/junit.xml \ @@ -1456,7 +1618,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-2.xml \ --durations=5" @@ -1485,6 +1647,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker tag litellm-docker-database:ci my-app:latest + - start_openai_record_replay_proxy - run: name: Run Docker container command: | @@ -1515,6 +1678,7 @@ jobs: -e LANGFUSE_PROJECT2_PUBLIC=$LANGFUSE_PROJECT2_PUBLIC \ -e LANGFUSE_PROJECT1_SECRET=$LANGFUSE_PROJECT1_SECRET \ -e LANGFUSE_PROJECT2_SECRET=$LANGFUSE_PROJECT2_SECRET \ + -e RECORDER_OPENAI_BASE_URL=http://host.docker.internal:8090/v1 \ --add-host host.docker.internal:host-gateway \ --name my-app \ -v $(pwd)/proxy_server_config.yaml:/app/config.yaml \ @@ -1539,7 +1703,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -v -x \ --junitxml=test-results/junit.xml \ -n 4 \ @@ -1622,7 +1786,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/openai_endpoints_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -s -vv \ --junitxml=test-results/junit.xml \ --durations=5" @@ -1652,6 +1816,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database + - start_openai_record_replay_proxy - run: name: Run Docker container # intentionally give bad redis credentials here @@ -1675,6 +1840,7 @@ jobs: -e DD_SITE=$DD_SITE \ -e AWS_REGION_NAME=$AWS_REGION_NAME \ -e COHERE_API_KEY=$COHERE_API_KEY \ + -e RECORDER_COHERE_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.cohere.com \ -e GCS_FLUSH_INTERVAL="1" \ --add-host host.docker.internal:host-gateway \ --name my-app \ @@ -1698,7 +1864,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/otel_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v \ --junitxml=test-results/junit.xml \ --durations=5" @@ -1748,7 +1914,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit-2.xml \ --durations=5" @@ -1824,7 +1990,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" @@ -1922,7 +2088,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/multi_instance_e2e_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" @@ -1985,7 +2151,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/store_model_in_db_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit.xml \ --durations=5" @@ -2065,7 +2231,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/basic_proxy_startup_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x \ --junitxml=test-results/junit-2.xml \ --durations=5" @@ -2209,7 +2375,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/pass_through_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -v -x \ --junitxml=test-results/junit.xml \ --durations=5" @@ -2240,6 +2406,7 @@ jobs: command: | zstd -d litellm-docker-database.tar.zst --stdout | docker load docker images | grep litellm-docker-database + - start_openai_record_replay_proxy - run: name: Run Docker container with test config command: | @@ -2248,6 +2415,7 @@ jobs: -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ -e LITELLM_MASTER_KEY="sk-1234" \ -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e RECORDER_ANTHROPIC_BASE_URL=http://host.docker.internal:8090/__recorder_upstream/api.anthropic.com \ -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ -e AWS_REGION_NAME="us-east-1" \ @@ -2275,7 +2443,7 @@ jobs: TEST_FILES=$(circleci tests glob "tests/proxy_e2e_anthropic_messages_tests/**/test_*.py") echo "$TEST_FILES" | circleci tests run \ --verbose \ - --command="awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ + --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ -vv -x -s \ --junitxml=test-results/junit.xml \ --durations=5" @@ -2617,6 +2785,12 @@ workflows: filters: *main_branches - auth_ui_unit_tests: filters: *main_branches + - proxy_behavior_tests: + filters: *main_branches + - proxy_security_tests: + filters: *main_branches + - schema_migration_check: + filters: *main_branches - build_docker_database_image: filters: *main_branches - e2e_ui_testing: diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index f0ced6bedb..23b520e2ad 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -8,3 +8,6 @@ # Update pydantic code to fix warnings (GH-3600) 876840e9957bc7e9f7d6a2b58c4d7c53dad16481 + +# style(ui): run prettier --write across the dashboard (#29622) +7edf3a9cb55548b143df1692f4ed7c4681d7fcf7 diff --git a/.gitattributes b/.gitattributes index 9030923a78..5c9061f52a 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ -*.ipynb linguist-vendored \ No newline at end of file +*.ipynb linguist-vendored +ui/litellm-dashboard/src/lib/http/schema.d.ts linguist-generated \ No newline at end of file diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index 7e91341ac7..a42b2f8f9d 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -27,6 +27,11 @@ on: required: false type: number default: 10 + dist: + description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" + required: false + type: string + default: "loadscope" artifact-name: description: "Unique name for the coverage artifact (must be unique per run)" required: true @@ -82,18 +87,31 @@ jobs: MAX_FAILURES: ${{ inputs.max-failures }} WORKERS: ${{ inputs.workers }} RERUNS: ${{ inputs.reruns }} + DIST: ${{ inputs.dist }} run: | - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - -n "${WORKERS}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --dist=loadscope \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml + if [ "${WORKERS}" = "0" ]; then + uv run --no-sync pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --durations=20 \ + --cov=./litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + else + uv run --no-sync pytest ${TEST_PATH:?} \ + --tb=short -vv \ + --maxfail="${MAX_FAILURES}" \ + -n "${WORKERS}" \ + --reruns "${RERUNS}" \ + --reruns-delay 1 \ + --dist="${DIST}" \ + --durations=20 \ + --cov=./litellm \ + --cov-report=xml:coverage.xml \ + --cov-config=pyproject.toml + fi - name: Save coverage report if: always() diff --git a/.github/workflows/_test-unit-services-base.yml b/.github/workflows/_test-unit-services-base.yml deleted file mode 100644 index 7f973d8caf..0000000000 --- a/.github/workflows/_test-unit-services-base.yml +++ /dev/null @@ -1,190 +0,0 @@ -name: _Unit Test Services Base (Reusable) - -on: - workflow_call: - inputs: - test-path: - description: "Pytest path(s) to run" - required: true - type: string - workers: - description: "Number of pytest-xdist workers (0 = no parallelism)" - required: false - type: number - default: 2 - reruns: - description: "Number of reruns for flaky tests" - required: false - type: number - default: 2 - timeout-minutes: - description: "Job timeout in minutes" - required: false - type: number - default: 20 - max-failures: - description: "Stop after this many failures" - required: false - type: number - default: 10 - enable-postgres: - description: "Start a local Postgres service container and run Prisma migrations" - required: false - type: boolean - default: false - dist: - description: "pytest-xdist distribution mode (loadscope|load|worksteal|loadfile|no)" - required: false - type: string - default: "loadscope" - artifact-name: - description: "Unique name for the coverage artifact (must be unique per run)" - required: false - type: string - default: "run" - -permissions: - contents: read - -# The postgres service container below is spawned per-job on localhost and -# destroyed with the job. Nothing outside the runner can reach it. The -# user/password/database here are not secrets — they're bootstrap values -# for a throwaway container — so we hardcode them instead of attaching -# every matrix shard to a GHA environment just to read three "secrets" -# (which also produces a "temporarily deployed to …" notification on the -# PR timeline per shard per push). -jobs: - run: - name: Run tests - runs-on: ubuntu-latest - timeout-minutes: ${{ inputs.timeout-minutes }} - - services: - postgres: - image: postgres@sha256:705a5d5b5836f3fcba0d02c4d281e6a7dd9ed2dd4078640f08a1e1e9896e097d # postgres:14 - env: - POSTGRES_USER: litellm - POSTGRES_PASSWORD: litellm - POSTGRES_DB: litellm_test - ports: - - 5432:5432 - options: >- - --health-cmd "pg_isready" - --health-interval 10s - --health-timeout 5s - --health-retries 5 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 - with: - version: "0.10.9" - - - name: Cache uv dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: | - ~/.cache/uv - .venv - key: ${{ runner.os }}-uv-services-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv-services- - - - name: Install dependencies - run: | - uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router - - - name: Generate Prisma client - env: - PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache - run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Run Prisma migrations - if: ${{ inputs.enable-postgres }} - env: - DATABASE_URL: "postgresql://litellm:litellm@localhost:5432/litellm_test" - run: | - uv run --no-sync prisma db push --schema litellm/proxy/schema.prisma --accept-data-loss - - - name: Run tests - env: - TEST_PATH: ${{ inputs.test-path }} - MAX_FAILURES: ${{ inputs.max-failures }} - WORKERS: ${{ inputs.workers }} - RERUNS: ${{ inputs.reruns }} - DIST: ${{ inputs.dist }} - DATABASE_URL: ${{ inputs.enable-postgres && 'postgresql://litellm:litellm@localhost:5432/litellm_test' || '' }} - run: | - if [ "${WORKERS}" = "0" ]; then - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml - else - uv run --no-sync pytest ${TEST_PATH:?} \ - --tb=short -vv \ - --maxfail="${MAX_FAILURES}" \ - -n "${WORKERS}" \ - --reruns "${RERUNS}" \ - --reruns-delay 1 \ - --dist="${DIST}" \ - --durations=20 \ - --cov=./litellm \ - --cov-report=xml:coverage.xml \ - --cov-config=pyproject.toml - fi - - - name: Save coverage report - if: always() - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} - path: coverage.xml - retention-days: 1 - - upload-coverage: - name: Upload coverage to Codecov - needs: run - if: always() - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - pull-requests: write - - steps: - - name: Checkout code - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - persist-credentials: false - - - name: Download coverage report - uses: actions/download-artifact@95815c38cf2ff2164869cbab79da8d1f422bc89e # v4.2.1 - with: - pattern: coverage-${{ inputs.artifact-name }}-${{ github.run_id }}-${{ github.run_attempt }} - path: coverage-reports - merge-multiple: true - - - name: Upload to Codecov - uses: codecov/codecov-action@75cd11691c0faa626561e295848008c8a7dddffe # v5.5.4 - with: - use_oidc: true - directory: coverage-reports - root_dir: ${{ github.workspace }} - flags: ${{ inputs.artifact-name }} - fail_ci_if_error: false diff --git a/.github/workflows/check-ui-api-types.yml b/.github/workflows/check-ui-api-types.yml new file mode 100644 index 0000000000..eeb5545b15 --- /dev/null +++ b/.github/workflows/check-ui-api-types.yml @@ -0,0 +1,84 @@ +name: Check UI API Types Sync + +on: + pull_request: + paths: + - "litellm/proxy/**" + - "litellm/types/**" + - "ui/litellm-dashboard/src/lib/http/schema.d.ts" + - "ui/litellm-dashboard/scripts/gen-api-types.mjs" + - "ui/litellm-dashboard/package.json" + - "ui/litellm-dashboard/package-lock.json" + - ".github/workflows/check-ui-api-types.yml" + +permissions: + contents: read + +jobs: + check-sync: + name: Verify schema.d.ts matches the proxy OpenAPI spec + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7 + with: + version: "0.10.9" + + - name: Cache uv dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 + with: + path: | + ~/.cache/uv + .venv + key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }} + restore-keys: | + ${{ runner.os }}-uv- + + - name: Install backend dependencies + run: uv sync --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router + + - name: Generate Prisma client + env: + PRISMA_BINARY_CACHE_DIR: ${{ runner.temp }}/prisma-cache + run: uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma + + - name: Set up Node.js + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dashboard dependencies + working-directory: ui/litellm-dashboard + run: npm ci + + - name: Regenerate types from the live spec + working-directory: ui/litellm-dashboard + env: + LITELLM_PYTHON: "uv run --no-sync python" + run: npm run gen:api + + - name: Fail if types are stale + run: | + if ! git diff --exit-code -- ui/litellm-dashboard/src/lib/http/schema.d.ts; then + echo "::error file=ui/litellm-dashboard/src/lib/http/schema.d.ts::Generated API types are out of sync with the proxy OpenAPI spec." + echo "" + echo "A backend route or model changed without regenerating the dashboard types." + echo "To fix, run from ui/litellm-dashboard:" + echo " npm run gen:api" + echo "then commit the updated src/lib/http/schema.d.ts." + exit 1 + fi + echo "schema.d.ts is in sync with the proxy OpenAPI spec." diff --git a/.github/workflows/create-release-branch.yml b/.github/workflows/create-release-branch.yml index ec2651306f..1d145184b6 100644 --- a/.github/workflows/create-release-branch.yml +++ b/.github/workflows/create-release-branch.yml @@ -63,3 +63,28 @@ jobs: sha: commitHash, }); core.info(`Created branch ${branchName} at ${commitHash}`); + + - name: Create stable line branch + env: + TAG: ${{ inputs.tag }} + COMMIT_HASH: ${{ inputs.commit_hash }} + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + script: | + const tag = process.env.TAG; + const commitHash = process.env.COMMIT_HASH; + + const match = tag.match(/^v?(\d+)\.(\d+)\.0$/); + if (!match) { + core.info(`Tag ${tag} is not the X.Y.0 stable opener; skipping stable line branch`); + return; + } + const lineBranch = `stable/${match[1]}.${match[2]}.x`; + + await github.rest.git.createRef({ + owner: context.repo.owner, + repo: context.repo.repo, + ref: `refs/heads/${lineBranch}`, + sha: commitHash, + }); + core.info(`Created branch ${lineBranch} at ${commitHash}`); diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index 862f98e30f..68497b10db 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -36,3 +36,79 @@ jobs: - name: Build run: npm run build + + frontend-lint: + runs-on: ubuntu-latest + timeout-minutes: 8 + defaults: + run: + working-directory: ui/litellm-dashboard + + steps: + - name: Checkout repository + uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Collect changed files + id: changed + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + run: | + : > "$RUNNER_TEMP/prettier_files.txt" + : > "$RUNNER_TEMP/eslint_files.txt" + while IFS= read -r f; do + [ -f "$f" ] || continue + case "$f" in + *.js | *.jsx | *.ts | *.tsx | *.mjs | *.cjs) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" + printf '%s\n' "$f" >> "$RUNNER_TEMP/eslint_files.txt" ;; + *.json | *.css | *.scss | *.md | *.mdx | *.yml | *.yaml | *.html) + printf '%s\n' "$f" >> "$RUNNER_TEMP/prettier_files.txt" ;; + esac + done < <(git diff --name-only --diff-filter=ACMR --relative "$BASE_SHA"...HEAD -- .) + if [ -s "$RUNNER_TEMP/prettier_files.txt" ] || [ -s "$RUNNER_TEMP/eslint_files.txt" ]; then + echo "has_files=true" >> "$GITHUB_OUTPUT" + else + echo "has_files=false" >> "$GITHUB_OUTPUT" + echo "No lintable UI files changed in this PR; nothing to check." + fi + + - name: Setup Node.js + if: steps.changed.outputs.has_files == 'true' + uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: ui/litellm-dashboard/package-lock.json + + - name: Install dependencies + if: steps.changed.outputs.has_files == 'true' + run: npm ci + + - name: Lint changed files (prettier + eslint) + if: steps.changed.outputs.has_files == 'true' + run: | + prettier_files=() + eslint_files=() + while IFS= read -r f; do prettier_files+=("$f"); done < "$RUNNER_TEMP/prettier_files.txt" + while IFS= read -r f; do eslint_files+=("$f"); done < "$RUNNER_TEMP/eslint_files.txt" + status=0 + if [ ${#prettier_files[@]} -gt 0 ]; then + echo "::group::Prettier (${#prettier_files[@]} files)" + npx prettier --check "${prettier_files[@]}" || { status=1; echo "::error::Unformatted files. Fix with: npm run format"; } + echo "::endgroup::" + fi + if [ ${#eslint_files[@]} -gt 0 ]; then + echo "::group::ESLint (${#eslint_files[@]} files)" + npx eslint --no-warn-ignored --pass-on-unpruned-suppressions "${eslint_files[@]}" || status=1 + echo "::endgroup::" + fi + exit $status + + - name: Check lint budgets + if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} + run: | + npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true + node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index 2d4e85630d..2ac9a3b7c1 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -1,9 +1,10 @@ name: "Unit Tests: Proxy DB Operations" -# Uses DATABASE_URL secret — only runs on trusted branches, not PRs. on: - push: - branches: [main, "litellm_**"] + pull_request: + branches: + - main + - litellm_internal_staging permissions: contents: read @@ -30,9 +31,6 @@ concurrency: # xdist balances its 188 parametrized cases across workers instead of # pinning the whole file to one worker (the default --dist=loadscope # behavior for single-file targets). -# * test_db_schema_migration.py is isolated because one test in it -# (test_aaaasschema_migration_check) takes ~170s — by itself it -# determines the shard's wall-clock floor. jobs: # Fast guard — fails the workflow if a test_*.py file under # tests/proxy_unit_tests/ is not referenced by any matrix entry below. @@ -166,18 +164,6 @@ jobs: dist: loadscope timeout: 15 - # ---- db-and-spend: isolate the 170s schema-migration test ---- - # test_db_schema_migration.py has exactly one test, and that test - # is mostly waiting on `prisma migrate deploy` / `prisma migrate - # diff` subprocesses (~170s). It does no CPU-bound Python work - # inside the test. Running with workers=0 (serial, no xdist) - # skips the 4-worker cold-start cost we'd otherwise pay for a - # single test, saving ~4 minutes of wall-clock. - - test-group: schema-migration - test-path: "tests/proxy_unit_tests/test_db_schema_migration.py" - workers: 0 - dist: loadscope - timeout: 15 - test-group: db-and-spend test-path: >- tests/proxy_unit_tests/test_prisma_client_backoff_retry.py @@ -232,12 +218,11 @@ jobs: workers: 4 dist: loadscope timeout: 15 - uses: ./.github/workflows/_test-unit-services-base.yml + uses: ./.github/workflows/_test-unit-base.yml with: test-path: ${{ matrix.test-path }} workers: ${{ matrix.workers }} reruns: 2 timeout-minutes: ${{ matrix.timeout }} - enable-postgres: true dist: ${{ matrix.dist }} artifact-name: proxy-db-${{ matrix.test-group }} diff --git a/.github/workflows/test-unit-proxy-endpoints.yml b/.github/workflows/test-unit-proxy-endpoints.yml index 6b34a08a8e..0a9513ec02 100644 --- a/.github/workflows/test-unit-proxy-endpoints.yml +++ b/.github/workflows/test-unit-proxy-endpoints.yml @@ -36,11 +36,13 @@ jobs: tests/test_litellm/proxy/a2a tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/health_endpoints + tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/rag_endpoints tests/test_litellm/proxy/realtime_endpoints tests/test_litellm/proxy/ui_crud_endpoints + tests/test_litellm/proxy/utils workers: 2 reruns: 2 artifact-name: proxy-endpoints diff --git a/.github/workflows/test-unit-proxy-mgmt-behavior.yml b/.github/workflows/test-unit-proxy-mgmt-behavior.yml deleted file mode 100644 index e73997323a..0000000000 --- a/.github/workflows/test-unit-proxy-mgmt-behavior.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: "Unit Tests: Proxy Management-Endpoint Behavior Pinning" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_branch - - "litellm_**" - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -jobs: - proxy-mgmt-behavior: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - test-path: tests/proxy_behavior - # workers=0 (no xdist): the world seed is a single shared Postgres - # state — two xdist workers both call seed_world() and race on the - # ``behavior-pin-budget`` row, producing UniqueViolation + cascading - # missing-membership FK failures. The whole suite is ~7s sequentially, - # so the cost of disabling parallelism here is negligible. - workers: 0 - reruns: 0 - enable-postgres: true - artifact-name: proxy-mgmt-behavior - timeout-minutes: 15 diff --git a/.github/workflows/test-unit-security.yml b/.github/workflows/test-unit-security.yml deleted file mode 100644 index 4ee8989702..0000000000 --- a/.github/workflows/test-unit-security.yml +++ /dev/null @@ -1,28 +0,0 @@ -name: "Unit Tests: Security" - -# Kept push-only (was previously required by DATABASE_URL secret scoping; -# now the postgres credentials are ephemeral localhost values but the -# push-trigger stays to match the proxy-db workflow cadence). -on: - push: - branches: [main, "litellm_**"] - -permissions: - contents: read - id-token: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - security: - uses: ./.github/workflows/_test-unit-services-base.yml - with: - test-path: "tests/proxy_security_tests/" - workers: 1 - reruns: 2 - timeout-minutes: 20 - enable-postgres: true - artifact-name: security diff --git a/CLAUDE.md b/CLAUDE.md index 3477b71a62..02a9630b48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -9,12 +9,15 @@ Don't assume that the existing code is correct or the right way of doing things - readable - easy to maintain/change - modern + In that order of importance When adding new features, add meaningful tests. Don't add tests that don't check anything substantial and is there just to make the code coverage pass. Yes, code coverage is important, but I'd rather have no signal whether the code is working than tests that don't fail when code is broken. The goal is to have tests that would fail before the feature was added/if the code was mutated in a way that breaks the feature and succeed only when the feature is fully working. I should run mutation testing and see > 90% kill rate Same thing for bug fixes. The tests should make it so that this specific bug can never happen again without failing tests (i.e., regression) +`tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones + When creating PRs, don't set base to `main`. `litellm_internal_staging` serves that purpose Always use @.github/pull_request_template.md as a guide for your PR body @@ -26,6 +29,8 @@ If you ever make public-facing PR descriptions, comments, issues, commit message - don't use "—". Instead, reach for ";", ".", etc. - don't use the pattern "It's not X, it's Y", "You're not X, you're Y", etc. - don't use bulleted or numbered lists unless it would be nonsensical not to. Instead, prefer prose +- don't add a trailing "." at the end of paragraphs (just like this file) +- don't use →. Instead, prefer not to use arrows, and if need be, use -> instead Don't hesitate to use values in .env to get needed API keys and other secrets, as long as you never add them to conversation history, commit them, or include them in GitHub issues / PRs @@ -37,7 +42,7 @@ When you must use real LLM models to, for example, write e2e tests, write a QA r If you're an internal contributor, when creating a new PR, the typical flow is to branch off litellm_internal_staging and create a branch prefixed with litellm_. Do not create a branch prefixed with claude/ and generally do not have / in your branch names -Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch +Do not add `Co-Authored-By: Claude` or any Claude attribution to commit messages. Never use a `claude/` prefix or put a `/` in a branch name. Do not add "Generated with Claude Code" (or any similar attribution) to PR descriptions or comments. Do not create a new PR/branch off the existing PR to fix/add something that is related and could've just been committed directly to the existing PR's branch When working on a PR, keep the PR description in sync with new commits being made @@ -49,22 +54,22 @@ CI supply-chain safety: Never pipe a remote script into a shell (`curl ... | bas ## Think Before Coding -**Don't assume. Don't hide confusion. Surface tradeoffs.** +**Don't assume. Don't hide confusion. Surface tradeoffs** Before implementing: -- State your assumptions explicitly. If uncertain, ask. -- If multiple interpretations exist, present them. Don't pick silently. -- If a simpler approach exists, say so. Push back when warranted. -- If something is unclear, stop. Name what's confusing. Ask. +- State your assumptions explicitly. If uncertain, ask +- If multiple interpretations exist, present them. Don't pick silently +- If a simpler approach exists, say so. Push back when warranted +- If something is unclear, stop. Name what's confusing. Ask ## Simplicity First -**Minimum code that solves the problem. Nothing speculative.** +**Minimum code that solves the problem. Nothing speculative** -- No features beyond what was asked. -- No abstractions for single-use code. -- No "flexibility" or "configurability" that wasn't requested. -- No error handling for impossible scenarios. -- If you write 200 lines and it could be 50, rewrite it. +- No features beyond what was asked +- No abstractions for single-use code +- No "flexibility" or "configurability" that wasn't requested +- No error handling for impossible scenarios +- If you write 200 lines and it could be 50, rewrite it -Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify diff --git a/Makefile b/Makefile index 5dbd308a3e..a00a90da60 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ test-unit-proxy-core: install-test-deps $(UV_RUN) pytest tests/test_litellm/proxy/auth tests/test_litellm/proxy/client tests/test_litellm/proxy/db tests/test_litellm/proxy/hooks tests/test_litellm/proxy/policy_engine --tb=short -vv -n 4 --durations=20 test-unit-proxy-misc: install-test-deps - $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 + $(UV_RUN) pytest tests/test_litellm/proxy/_experimental tests/test_litellm/proxy/agent_endpoints tests/test_litellm/proxy/anthropic_endpoints tests/test_litellm/proxy/common_utils tests/test_litellm/proxy/discovery_endpoints tests/test_litellm/proxy/experimental tests/test_litellm/proxy/google_endpoints tests/test_litellm/proxy/health_endpoints tests/test_litellm/proxy/image_endpoints tests/test_litellm/proxy/middleware tests/test_litellm/proxy/openai_files_endpoint tests/test_litellm/proxy/pass_through_endpoints tests/test_litellm/proxy/prompts tests/test_litellm/proxy/public_endpoints tests/test_litellm/proxy/response_api_endpoints tests/test_litellm/proxy/shutdown tests/test_litellm/proxy/spend_tracking tests/test_litellm/proxy/ui_crud_endpoints tests/test_litellm/proxy/vector_store_endpoints tests/test_litellm/proxy/test_*.py --tb=short -vv -n 4 --durations=20 test-unit-integrations: install-test-deps $(UV_RUN) pytest tests/test_litellm/integrations --tb=short -vv -n 4 --durations=20 diff --git a/README.md b/README.md index 8df351e930..d600f3952c 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ -Group 7154 (1) +LiteLLM AI Gateway --- @@ -407,7 +407,7 @@ Support for more providers. Missing a provider or LLM Platform, raise a [feature ### Run in Developer Mode #### Services 1. Setup .env file in root -2. Run dependant services `docker-compose up db prometheus` +2. Run dependent services `docker-compose up db prometheus` #### Backend 1. (In root) create virtual environment `python -m venv .venv` diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index a9cdf28f0e..6e30a6af44 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -285,11 +285,31 @@ db: deployStandalone: true # Lifecycle hooks for the LiteLLM container +# +# Prefer the native /health/drain preStop hook over a fixed `sleep`: it marks +# the pod NotReady and blocks only until in-flight requests actually finish +# (bounded by GRACEFUL_SHUTDOWN_TIMEOUT, default 30s), instead of always +# waiting the worst-case duration. The drain runs once (the preStop hook and +# the SIGTERM handler share it), so set terminationGracePeriodSeconds a few +# seconds above GRACEFUL_SHUTDOWN_TIMEOUT to leave room for teardown before +# SIGKILL. +# +# /health/drain is off by default; enable it with +# general_settings.enable_drain_endpoint: true. The kubelet calls preStop +# hooks without proxy credentials, so when the health port is reachable from +# other pods (the common case) also set +# general_settings.drain_endpoint_token (or the DRAIN_ENDPOINT_TOKEN env +# var) and send the same value on the X-Drain-Token header from the hook. +# Calls missing/wrong the token get a 401 and have no side effect. # Example: # lifecycle: # preStop: -# exec: -# command: ["/bin/sh", "-c", "sleep 10"] +# httpGet: +# path: /health/drain +# port: 4000 +# httpHeaders: +# - name: X-Drain-Token +# value: lifecycle: {} # Settings for Bitnami postgresql chart (if db.deployStandalone is true, ignored diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index cbbf55c987..144bb4c473 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -106,6 +106,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( # Health & ops "/health", "/metrics", + "/watsonx" ) GATEWAY_EXACT_PATHS: frozenset[str] = frozenset( diff --git a/helm/litellm/templates/backend/deployment.yaml b/helm/litellm/templates/backend/deployment.yaml index 3b59c58c8b..b355db4354 100644 --- a/helm/litellm/templates/backend/deployment.yaml +++ b/helm/litellm/templates/backend/deployment.yaml @@ -12,9 +12,14 @@ spec: {{- include "litellm.backend.selectorLabels" . | nindent 6 }} template: metadata: - {{- with .Values.backend.podAnnotations }} + {{- if or .Values.gateway.config.create .Values.backend.podAnnotations }} annotations: + {{- if .Values.gateway.config.create }} + checksum/config: {{ include (print $.Template.BasePath "/gateway/configmap.yaml") . | sha256sum }} + {{- end }} + {{- with .Values.backend.podAnnotations }} {{- toYaml . | nindent 8 }} + {{- end }} {{- end }} labels: {{- include "litellm.backend.selectorLabels" . | nindent 8 }} @@ -35,7 +40,17 @@ spec: protocol: TCP env: {{- include "litellm.serverEnv" (dict "root" $ "component" .Values.backend) | nindent 12 }} + {{- if .Values.gateway.config.create }} + - name: CONFIG_FILE_PATH + value: /app/config/config.yaml + {{- end }} {{- include "litellm.envFrom" .Values.backend | nindent 10 }} + {{- if .Values.gateway.config.create }} + volumeMounts: + - name: gateway-config + mountPath: /app/config/config.yaml + subPath: config.yaml + {{- end }} {{- with .Values.backend.livenessProbe }} livenessProbe: {{- toYaml . | nindent 12 }} @@ -46,6 +61,12 @@ spec: {{- end }} resources: {{- toYaml .Values.backend.resources | nindent 12 }} + {{- if .Values.gateway.config.create }} + volumes: + - name: gateway-config + configMap: + name: {{ include "litellm.gateway.fullname" . }}-config + {{- end }} {{- with .Values.backend.nodeSelector }} nodeSelector: {{- toYaml . | nindent 8 }} diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql new file mode 100644 index 0000000000..08d35cd74a --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260520120000_add_mcp_env_vars/migration.sql @@ -0,0 +1,23 @@ +-- AlterTable: add admin-configured env_vars to MCP server table +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "env_vars" JSONB DEFAULT '[]'; + +-- CreateTable: per-user env var values for MCP servers +CREATE TABLE IF NOT EXISTS "LiteLLM_MCPUserEnvVars" ( + "id" TEXT NOT NULL, + "user_id" TEXT NOT NULL, + "server_id" TEXT NOT NULL, + "values_b64" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "updated_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + + CONSTRAINT "LiteLLM_MCPUserEnvVars_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_user_id_server_id_key" ON "LiteLLM_MCPUserEnvVars"("user_id", "server_id"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_user_id_idx" ON "LiteLLM_MCPUserEnvVars"("user_id"); + +-- CreateIndex +CREATE INDEX IF NOT EXISTS "LiteLLM_MCPUserEnvVars_server_id_idx" ON "LiteLLM_MCPUserEnvVars"("server_id"); diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260526120000_add_oauth_passthrough_to_mcp_servers/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260526120000_add_oauth_passthrough_to_mcp_servers/migration.sql new file mode 100644 index 0000000000..3c387891a5 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260526120000_add_oauth_passthrough_to_mcp_servers/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "oauth_passthrough" BOOLEAN NOT NULL DEFAULT false; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql new file mode 100644 index 0000000000..fee6926d96 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260604120000_add_oauth2_flow_to_mcp_servers/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "oauth2_flow" TEXT; diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql new file mode 100644 index 0000000000..845ad017cb --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260605182307_add_timeout_to_mcp_server_table/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN "timeout" DOUBLE PRECISION; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 78143fe041..e21c001649 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -311,6 +311,11 @@ model LiteLLM_MCPServerTable { tool_name_to_description Json? @default("{}") extra_headers String[] @default([]) static_headers Json? @default("{}") + // Admin-configured environment variables interpolated into static_headers + // via ${NAME} syntax. Stored as an array of + // {name, value, scope, description}. scope is "global" (value used as-is) + // or "user" (value supplied per-user via LiteLLM_MCPUserEnvVars). + env_vars Json? @default("[]") // Health check status status String? @default("unknown") last_health_check DateTime? @@ -322,13 +327,16 @@ model LiteLLM_MCPServerTable { authorization_url String? token_url String? registration_url String? + oauth2_flow String? allow_all_keys Boolean @default(false) available_on_public_internet Boolean @default(true) delegate_auth_to_upstream Boolean @default(false) + oauth_passthrough Boolean @default(false) is_byok Boolean @default(false) byok_description String[] @default([]) byok_api_key_help_url String? source_url String? + timeout Float? // BYOM submission lifecycle approval_status String? @default("active") submitted_by String? @@ -363,6 +371,21 @@ model LiteLLM_MCPUserCredentials { @@unique([user_id, server_id]) } +// Per-user environment variable values for MCP servers. +// values_b64 is an encrypted JSON object: {VAR_NAME: "value", ...}. +model LiteLLM_MCPUserEnvVars { + id String @id @default(uuid()) + user_id String + server_id String + values_b64 String + created_at DateTime @default(now()) + updated_at DateTime @default(now()) @updatedAt + + @@unique([user_id, server_id]) + @@index([user_id]) + @@index([server_id]) +} + // Generate Tokens for Proxy model LiteLLM_VerificationToken { token String @id diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 0654f17ec6..e2a86205fc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.73" +version = "0.4.74" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.73" +version = "0.4.74" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 56d516536e..f22971dfa1 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -16,8 +16,17 @@ import os # Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available import dotenv as _dotenv + +def _dev_env_hot_reload_enabled() -> bool: + """The proxy exports this flag when started with ``--reload``. A reloaded + worker is a fresh process that inherits the reloader's environment, so an + edited ``.env`` value stays masked by the stale inherited one unless we + let the file win; overriding makes the edit take effect on reload.""" + return os.getenv("LITELLM_DEV_ENV_HOT_RELOAD") == "True" + + if os.getenv("LITELLM_MODE", "DEV") == "DEV": - _dotenv.load_dotenv() + _dotenv.load_dotenv(override=_dev_env_hot_reload_enabled()) from typing import ( Callable, @@ -240,6 +249,7 @@ api_key: Optional[str] = None openai_key: Optional[str] = None groq_key: Optional[str] = None gigachat_key: Optional[str] = None +xai_key: Optional[str] = None databricks_key: Optional[str] = None openai_like_key: Optional[str] = None azure_key: Optional[str] = None @@ -277,6 +287,7 @@ ovhcloud_key: Optional[str] = None lemonade_key: Optional[str] = None sap_service_key: Optional[str] = None amazon_nova_api_key: Optional[str] = None +inception_key: Optional[str] = None common_cloud_provider_auth_params: dict = { "params": ["project", "region_name", "token"], "providers": ["vertex_ai", "bedrock", "watsonx", "azure", "vertex_ai_beta"], @@ -442,6 +453,7 @@ disable_copilot_system_to_assistant: bool = ( False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior. ) public_mcp_servers: Optional[List[str]] = None +public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None public_agent_groups: Optional[List[str]] = None # Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]]) @@ -550,6 +562,7 @@ cohere_models: Set = set() cohere_chat_models: Set = set() mistral_chat_models: Set = set() text_completion_codestral_models: Set = set() +text_completion_inception_models: Set = set() anthropic_models: Set = set() openrouter_models: Set = set() datarobot_models: Set = set() @@ -608,6 +621,7 @@ cerebras_models: Set = set() galadriel_models: Set = set() nvidia_nim_models: Set = set() nvidia_riva_models: Set = set() +soniox_models: Set = set() sambanova_models: Set = set() sambanova_embedding_models: Set = set() novita_models: Set = set() @@ -627,6 +641,7 @@ publicai_models: Set = set() v0_models: Set = set() morph_models: Set = set() lambda_ai_models: Set = set() +inception_models: Set = set() hyperbolic_models: Set = set() black_forest_labs_models: Set = set() recraft_models: Set = set() @@ -791,6 +806,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): fireworks_ai_embedding_models.add(key) elif value.get("litellm_provider") == "text-completion-codestral": text_completion_codestral_models.add(key) + elif value.get("litellm_provider") == "text-completion-inception": + text_completion_inception_models.add(key) elif value.get("litellm_provider") == "xai": xai_models.add(key) elif value.get("litellm_provider") == "zai": @@ -837,6 +854,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): nvidia_nim_models.add(key) elif value.get("litellm_provider") == "nvidia_riva": nvidia_riva_models.add(key) + elif value.get("litellm_provider") == "soniox": + soniox_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": @@ -877,6 +896,8 @@ def add_known_models(model_cost_map: Optional[Dict] = None): morph_models.add(key) elif value.get("litellm_provider") == "lambda_ai": lambda_ai_models.add(key) + elif value.get("litellm_provider") == "inception": + inception_models.add(key) elif value.get("litellm_provider") == "hyperbolic": hyperbolic_models.add(key) elif value.get("litellm_provider") == "black_forest_labs": @@ -979,6 +1000,7 @@ model_list = list( | watsonx_models | gemini_models | text_completion_codestral_models + | text_completion_inception_models | xai_models | zai_models | fal_ai_models @@ -999,6 +1021,7 @@ model_list = list( | galadriel_models | nvidia_nim_models | nvidia_riva_models + | soniox_models | sambanova_models | azure_text_models | novita_models @@ -1017,6 +1040,7 @@ model_list = list( | v0_models | morph_models | lambda_ai_models + | inception_models | black_forest_labs_models | recraft_models | cometapi_models @@ -1073,6 +1097,7 @@ models_by_provider: dict = { "fireworks_ai": fireworks_ai_models | fireworks_ai_embedding_models, "aleph_alpha": aleph_alpha_models, "text-completion-codestral": text_completion_codestral_models, + "text-completion-inception": text_completion_inception_models, "xai": xai_models, "zai": zai_models, "fal_ai": fal_ai_models, @@ -1097,6 +1122,7 @@ models_by_provider: dict = { "galadriel": galadriel_models, "nvidia_nim": nvidia_nim_models, "nvidia_riva": nvidia_riva_models, + "soniox": soniox_models, "sambanova": sambanova_models | sambanova_embedding_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, @@ -1117,6 +1143,7 @@ models_by_provider: dict = { "v0": v0_models, "morph": morph_models, "lambda_ai": lambda_ai_models, + "inception": inception_models, "hyperbolic": hyperbolic_models, "black_forest_labs": black_forest_labs_models, "recraft": recraft_models, @@ -1727,6 +1754,9 @@ if TYPE_CHECKING: from .llms.openrouter.responses.transformation import ( OpenRouterResponsesAPIConfig as OpenRouterResponsesAPIConfig, ) + from .llms.bedrock_mantle.responses.transformation import ( + BedrockMantleResponsesAPIConfig as BedrockMantleResponsesAPIConfig, + ) from .llms.gemini.interactions.transformation import ( GoogleAIStudioInteractionsConfig as GoogleAIStudioInteractionsConfig, ) @@ -1868,6 +1898,9 @@ if TYPE_CHECKING: from .llms.codestral.completion.transformation import ( CodestralTextCompletionConfig as CodestralTextCompletionConfig, ) + from .llms.inception.completion.transformation import ( + InceptionTextCompletionConfig as InceptionTextCompletionConfig, + ) from .llms.azure.azure import ( AzureOpenAIAssistantsAPIConfig as AzureOpenAIAssistantsAPIConfig, ) @@ -1936,6 +1969,9 @@ if TYPE_CHECKING: from .llms.lambda_ai.chat.transformation import ( LambdaAIChatConfig as LambdaAIChatConfig, ) + from .llms.inception.chat.transformation import ( + InceptionChatConfig as InceptionChatConfig, + ) from .llms.hyperbolic.chat.transformation import ( HyperbolicChatConfig as HyperbolicChatConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 17eb660929..bace54ffad 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -237,6 +237,7 @@ LLM_CONFIG_NAMES = ( "PerplexityResponsesConfig", "DatabricksResponsesAPIConfig", "OpenRouterResponsesAPIConfig", + "BedrockMantleResponsesAPIConfig", "GoogleAIStudioInteractionsConfig", "OpenAIOSeriesConfig", "AnthropicSkillsConfig", @@ -267,6 +268,7 @@ LLM_CONFIG_NAMES = ( "AIMLChatConfig", "VolcEngineChatConfig", "CodestralTextCompletionConfig", + "InceptionTextCompletionConfig", "AzureOpenAIAssistantsAPIConfig", "HerokuChatConfig", "CometAPIConfig", @@ -310,6 +312,7 @@ LLM_CONFIG_NAMES = ( "MorphChatConfig", "RAGFlowConfig", "LambdaAIChatConfig", + "InceptionChatConfig", "HyperbolicChatConfig", "VercelAIGatewayConfig", "OVHCloudChatConfig", @@ -318,6 +321,7 @@ LLM_CONFIG_NAMES = ( "LemonadeChatConfig", "SnowflakeEmbeddingConfig", "AmazonNovaChatConfig", + "SonioxAudioTranscriptionConfig", ) # Types that support lazy loading via _lazy_import_types @@ -956,6 +960,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.openrouter.responses.transformation", "OpenRouterResponsesAPIConfig", ), + "BedrockMantleResponsesAPIConfig": ( + ".llms.bedrock_mantle.responses.transformation", + "BedrockMantleResponsesAPIConfig", + ), "GoogleAIStudioInteractionsConfig": ( ".llms.gemini.interactions.transformation", "GoogleAIStudioInteractionsConfig", @@ -1040,6 +1048,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.codestral.completion.transformation", "CodestralTextCompletionConfig", ), + "InceptionTextCompletionConfig": ( + ".llms.inception.completion.transformation", + "InceptionTextCompletionConfig", + ), "AzureOpenAIAssistantsAPIConfig": ( ".llms.azure.azure", "AzureOpenAIAssistantsAPIConfig", @@ -1154,6 +1166,10 @@ _LLM_CONFIGS_IMPORT_MAP = { "MorphChatConfig": (".llms.morph.chat.transformation", "MorphChatConfig"), "RAGFlowConfig": (".llms.ragflow.chat.transformation", "RAGFlowConfig"), "LambdaAIChatConfig": (".llms.lambda_ai.chat.transformation", "LambdaAIChatConfig"), + "InceptionChatConfig": ( + ".llms.inception.chat.transformation", + "InceptionChatConfig", + ), "HyperbolicChatConfig": ( ".llms.hyperbolic.chat.transformation", "HyperbolicChatConfig", @@ -1180,6 +1196,10 @@ _LLM_CONFIGS_IMPORT_MAP = { ".llms.amazon_nova.chat.transformation", "AmazonNovaChatConfig", ), + "SonioxAudioTranscriptionConfig": ( + ".llms.soniox.audio_transcription.transformation", + "SonioxAudioTranscriptionConfig", + ), } # Import map for utils module lazy imports diff --git a/litellm/_service_logger.py b/litellm/_service_logger.py index 5531c41879..b290b4340e 100644 --- a/litellm/_service_logger.py +++ b/litellm/_service_logger.py @@ -371,6 +371,8 @@ class ServiceLogging(CustomLogger): service=ServiceTypes.LITELLM, duration=_duration, call_type=kwargs.get("call_type", "unknown"), + start_time=start_time, + end_time=end_time, ) except Exception as e: raise e diff --git a/litellm/a2a_protocol/litellm_completion_bridge/handler.py b/litellm/a2a_protocol/litellm_completion_bridge/handler.py index 67ffcf4f8f..52e471ff70 100644 --- a/litellm/a2a_protocol/litellm_completion_bridge/handler.py +++ b/litellm/a2a_protocol/litellm_completion_bridge/handler.py @@ -20,9 +20,20 @@ from litellm.a2a_protocol.litellm_completion_bridge.transformation import ( ) from litellm.a2a_protocol.providers.config_manager import A2AProviderConfigManager +# litellm_params key carrying the authenticated principal (hashed virtual key) so +# A2A provider configs can scope provider-side state (e.g. LangFlow session memory) +# per key instead of trusting the client-supplied A2A contextId. +A2A_USER_API_KEY_HASH_PARAM = "litellm_a2a_user_api_key_hash" + # Agent metadata fields stored in litellm_params that are not valid litellm.acompletion() kwargs _AGENT_ONLY_PARAMS = frozenset( - {"is_public", "agent_name", "agent_id", "agent_card_params"} + { + "is_public", + "agent_name", + "agent_id", + "agent_card_params", + A2A_USER_API_KEY_HASH_PARAM, + } ) @@ -37,6 +48,8 @@ class A2ACompletionBridgeHandler: params: Dict[str, Any], litellm_params: Dict[str, Any], api_base: Optional[str] = None, + *, + _skip_a2a_provider_routing: bool = False, ) -> Dict[str, Any]: """ Handle non-streaming A2A request via litellm.acompletion. @@ -50,25 +63,24 @@ class A2ACompletionBridgeHandler: Returns: A2A SendMessageResponse dict """ - # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") - a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider, - model=litellm_params.get("model"), - ) - - # If provider config exists, use it - if a2a_provider_config is not None: - verbose_logger.info(f"A2A: Using provider config for {custom_llm_provider}") - - response_data = await a2a_provider_config.handle_non_streaming( - request_id=request_id, - params=params, - api_base=api_base, - litellm_params=litellm_params, + if not _skip_a2a_provider_routing: + a2a_provider_config = A2AProviderConfigManager.get_provider_config( + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) - return response_data + if a2a_provider_config is not None: + verbose_logger.info( + f"A2A: Using provider config for {custom_llm_provider}" + ) + + return await a2a_provider_config.handle_non_streaming( + request_id=request_id, + params=params, + api_base=api_base, + litellm_params=litellm_params, + ) # Extract message from params message = params.get("message", {}) @@ -137,6 +149,8 @@ class A2ACompletionBridgeHandler: params: Dict[str, Any], litellm_params: Dict[str, Any], api_base: Optional[str] = None, + *, + _skip_a2a_provider_routing: bool = False, ) -> AsyncIterator[Dict[str, Any]]: """ Handle streaming A2A request via litellm.acompletion with stream=True. @@ -156,28 +170,27 @@ class A2ACompletionBridgeHandler: Yields: A2A streaming response events """ - # Get provider config for custom_llm_provider custom_llm_provider = litellm_params.get("custom_llm_provider") - a2a_provider_config = A2AProviderConfigManager.get_provider_config( - custom_llm_provider=custom_llm_provider, - model=litellm_params.get("model"), - ) - - # If provider config exists, use it - if a2a_provider_config is not None: - verbose_logger.info( - f"A2A: Using provider config for {custom_llm_provider} (streaming)" + if not _skip_a2a_provider_routing: + a2a_provider_config = A2AProviderConfigManager.get_provider_config( + custom_llm_provider=custom_llm_provider, + model=litellm_params.get("model"), ) - async for chunk in a2a_provider_config.handle_streaming( - request_id=request_id, - params=params, - api_base=api_base, - litellm_params=litellm_params, - ): - yield chunk + if a2a_provider_config is not None: + verbose_logger.info( + f"A2A: Using provider config for {custom_llm_provider} (streaming)" + ) - return + async for chunk in a2a_provider_config.handle_streaming( + request_id=request_id, + params=params, + api_base=api_base, + litellm_params=litellm_params, + ): + yield chunk + + return # Extract message from params message = params.get("message", {}) diff --git a/litellm/a2a_protocol/main.py b/litellm/a2a_protocol/main.py index 3ad5485dea..6979e1ac65 100644 --- a/litellm/a2a_protocol/main.py +++ b/litellm/a2a_protocol/main.py @@ -159,7 +159,9 @@ async def _send_message_via_completion_bridge( api_base=api_base, ) - return LiteLLMSendMessageResponse.from_dict(response_dict) + return LiteLLMSendMessageResponse.from_dict( + response_dict, request_id=str(request.id) + ) async def _execute_a2a_send_with_retry( @@ -317,15 +319,6 @@ async def asend_message( ) card_url = getattr(agent_card, "url", None) if agent_card else None - context_id = trace_id or str(uuid.uuid4()) - message = request.params.message - if isinstance(message, dict): - if message.get("context_id") is None: - message["context_id"] = context_id - else: - if getattr(message, "context_id", None) is None: - message.context_id = context_id - a2a_response = await _execute_a2a_send_with_retry( a2a_client=a2a_client, request=request, @@ -338,7 +331,9 @@ async def asend_message( verbose_logger.info(f"A2A send_message completed, request_id={request.id}") # Wrap in LiteLLM response type for _hidden_params support - response = LiteLLMSendMessageResponse.from_a2a_response(a2a_response) + response = LiteLLMSendMessageResponse.from_a2a_response( + a2a_response, request_id=str(request.id) + ) # Calculate token usage from request and response response_dict = a2a_response.model_dump(mode="json", exclude_none=True) diff --git a/litellm/a2a_protocol/providers/config_manager.py b/litellm/a2a_protocol/providers/config_manager.py index d684efd475..a421afec18 100644 --- a/litellm/a2a_protocol/providers/config_manager.py +++ b/litellm/a2a_protocol/providers/config_manager.py @@ -48,4 +48,16 @@ class A2AProviderConfigManager: return BedrockAgentCoreA2AConfig() + if custom_llm_provider == "langflow": + from litellm.a2a_protocol.providers.langflow.config import LangFlowA2AConfig + + return LangFlowA2AConfig() + + if custom_llm_provider == "watsonx_orchestrate": + from litellm.a2a_protocol.providers.watsonx_orchestrate.config import ( + WatsonxOrchestrateA2AConfig, + ) + + return WatsonxOrchestrateA2AConfig() + return None diff --git a/litellm/a2a_protocol/providers/langflow/__init__.py b/litellm/a2a_protocol/providers/langflow/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/a2a_protocol/providers/langflow/config.py b/litellm/a2a_protocol/providers/langflow/config.py new file mode 100644 index 0000000000..9302c38126 --- /dev/null +++ b/litellm/a2a_protocol/providers/langflow/config.py @@ -0,0 +1,62 @@ +from typing import Any, AsyncIterator, Dict, Optional + +from litellm.a2a_protocol.litellm_completion_bridge.handler import ( + A2A_USER_API_KEY_HASH_PARAM, + A2ACompletionBridgeHandler, +) +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.llms.langflow.a2a import merge_a2a_session_into_litellm_params + + +class LangFlowA2AConfig(BaseA2AProviderConfig): + """A2A bridge for LangFlow: scopes contextId to the authenticated key as the + LangFlow session_id, then uses completion.""" + + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> Dict[str, Any]: + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for LangFlowA2AConfig " + "(must contain custom_llm_provider and model)" + ) + litellm_params = merge_a2a_session_into_litellm_params( + litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM) + ) + return await A2ACompletionBridgeHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + api_base=api_base, + _skip_a2a_provider_routing=True, + ) + + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs, + ) -> AsyncIterator[Dict[str, Any]]: + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for LangFlowA2AConfig " + "(must contain custom_llm_provider and model)" + ) + litellm_params = merge_a2a_session_into_litellm_params( + litellm_params, params, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM) + ) + async for chunk in A2ACompletionBridgeHandler.handle_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + api_base=api_base, + _skip_a2a_provider_routing=True, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/__init__.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/__init__.py new file mode 100644 index 0000000000..096bcc0121 --- /dev/null +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/__init__.py @@ -0,0 +1,3 @@ +""" +IBM watsonx Orchestrate (WXO) A2A provider. +""" diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/config.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/config.py new file mode 100644 index 0000000000..dbd4a0558f --- /dev/null +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/config.py @@ -0,0 +1,55 @@ +""" +A2A provider configuration for IBM watsonx Orchestrate (WXO). +""" + +from typing import Any, AsyncIterator, Dict, Optional + +from litellm.a2a_protocol.providers.base import BaseA2AProviderConfig +from litellm.a2a_protocol.providers.watsonx_orchestrate.handler import ( + WatsonxOrchestrateHandler, +) + + +class WatsonxOrchestrateA2AConfig(BaseA2AProviderConfig): + """A2A bridge for IBM watsonx Orchestrate (REST runs API + poll/SSE).""" + + async def handle_non_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs: Any, + ) -> Dict[str, Any]: + """Handle a non-streaming A2A request via WXO runs API.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for WatsonxOrchestrateA2AConfig " + "(must contain cp4d_host, instance_id, wxo_agent_id, api_key)" + ) + return await WatsonxOrchestrateHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ) + + async def handle_streaming( + self, + request_id: str, + params: Dict[str, Any], + api_base: Optional[str] = None, + **kwargs: Any, + ) -> AsyncIterator[Dict[str, Any]]: + """Handle a streaming A2A request via WXO streaming runs API.""" + litellm_params = kwargs.get("litellm_params") + if not litellm_params: + raise ValueError( + "litellm_params is required for WatsonxOrchestrateA2AConfig " + "(must contain cp4d_host, instance_id, wxo_agent_id, api_key)" + ) + async for chunk in WatsonxOrchestrateHandler.handle_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py new file mode 100644 index 0000000000..dbc0247618 --- /dev/null +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/handler.py @@ -0,0 +1,373 @@ +""" +Handler for IBM watsonx Orchestrate (WXO) agent provider. +""" + +import asyncio +import hashlib +import json +import time +from typing import Any, AsyncIterator, Dict, NamedTuple, Optional, Tuple, cast + +import httpx + +from litellm._logging import verbose_logger +from litellm.a2a_protocol.providers.watsonx_orchestrate.transformation import ( + WatsonxOrchestrateTransformation, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, +) +from litellm.types.llms.custom_http import httpxSpecialProvider + +_IBM_CLOUD_IAM_URL = "https://iam.cloud.ibm.com/identity/token" +_POLL_INTERVAL_S = 2.0 +_MAX_POLL_ATTEMPTS = 90 +_TOKEN_CACHE_TTL_BUFFER_S = 60 +_token_cache: Dict[str, Tuple[str, float]] = {} + + +class WXORequestParams(NamedTuple): + cp4d_host: str + instance_id: str + wxo_agent_id: str + api_key: str + username: Optional[str] + auth_mode: str + thread_id: Optional[str] + + +class WatsonxOrchestrateHandler: + @staticmethod + def _http_client(timeout: float = 90.0) -> AsyncHTTPHandler: + return get_async_httpx_client( + llm_provider=cast(Any, httpxSpecialProvider.A2AProvider), + params={"timeout": timeout}, + ) + + @staticmethod + def _token_cache_key( + auth_mode: str, + cp4d_host: str, + api_key: str, + username: Optional[str], + ) -> str: + material = f"{auth_mode}:{cp4d_host}:{username or ''}:{api_key}" + return hashlib.sha256(material.encode()).hexdigest() + + @staticmethod + def _cp4d_token_ttl_seconds( + expiration: Any, now_wall: Optional[float] = None + ) -> int: + # CP4D returns expiration as absolute Unix epoch seconds, not a duration. + expires_at = int(expiration) + wall = now_wall if now_wall is not None else time.time() + return max(expires_at - int(wall), 0) + + @staticmethod + async def _get_bearer_token( + cp4d_host: str, + auth_mode: str, + api_key: str, + username: Optional[str] = None, + client: Optional[AsyncHTTPHandler] = None, + ) -> str: + cache_key = WatsonxOrchestrateHandler._token_cache_key( + auth_mode, cp4d_host, api_key, username + ) + now = time.monotonic() + cached = _token_cache.get(cache_key) + if cached and cached[1] > now: + return cached[0] + + if client is None: + client = WatsonxOrchestrateHandler._http_client(timeout=30.0) + + if auth_mode == "ibm_cloud": + response = await client.post( + _IBM_CLOUD_IAM_URL, + data={ + "grant_type": "urn:ibm:params:oauth:grant-type:apikey", + "apikey": api_key, + }, + headers={"Content-Type": "application/x-www-form-urlencoded"}, + ) + response.raise_for_status() + payload = response.json() + token = str(payload["access_token"]) + ttl_s = int(payload.get("expires_in", 3600)) + else: + if not username: + raise ValueError( + "'username' is required in litellm_params when auth_mode='cp4d'" + ) + token_url = f"{cp4d_host.rstrip('/')}/icp4d-api/v1/authorize" + response = await client.post( + token_url, + json={"username": username, "api_key": api_key}, + headers={"Content-Type": "application/json"}, + ) + response.raise_for_status() + payload = response.json() + token = str(payload["token"]) + expiration = payload.get("expiration") + if expiration is None: + ttl_s = 3600 + else: + ttl_s = WatsonxOrchestrateHandler._cp4d_token_ttl_seconds(expiration) + + expires_at = now + max(ttl_s - _TOKEN_CACHE_TTL_BUFFER_S, 0) + _token_cache[cache_key] = (token, expires_at) + for stale_key, (_, stale_expires_at) in list(_token_cache.items()): + if stale_expires_at <= now: + del _token_cache[stale_key] + return token + + @staticmethod + async def _poll_run( + base_url: str, + run_id: str, + auth_headers: Dict[str, str], + client: AsyncHTTPHandler, + max_attempts: int = _MAX_POLL_ATTEMPTS, + interval_s: float = _POLL_INTERVAL_S, + ) -> Dict[str, Any]: + url = f"{base_url}/v1/orchestrate/runs/{run_id}" + + for attempt in range(max_attempts): + await asyncio.sleep(interval_s) + response = await client.get(url, headers=auth_headers) + response.raise_for_status() + result: Dict[str, Any] = response.json() + status = result.get("status", "") + verbose_logger.debug( + f"WXO: Poll {attempt + 1}/{max_attempts} run='{run_id}' status='{status}'" + ) + if status in WatsonxOrchestrateTransformation.TERMINAL_STATES: + return result + + raise asyncio.TimeoutError( + f"WXO run '{run_id}' did not reach a terminal state after " + f"{max_attempts * interval_s:.0f}s" + ) + + @staticmethod + async def _get_successful_run_data( + run_data: Dict[str, Any], + base_url: str, + auth_headers: Dict[str, str], + client: AsyncHTTPHandler, + ) -> Dict[str, Any]: + status = run_data.get("status", "") + if status not in WatsonxOrchestrateTransformation.TERMINAL_STATES: + run_id = run_data.get("run_id") or run_data.get("id") or "" + if not run_id: + raise ValueError(f"WXO: No run_id in response: {run_data}") + run_data = await WatsonxOrchestrateHandler._poll_run( + base_url=base_url, + run_id=run_id, + auth_headers=auth_headers, + client=client, + ) + status = run_data.get("status", "") + + if status not in WatsonxOrchestrateTransformation.SUCCESS_STATES: + raise RuntimeError( + f"WXO run ended with non-success status '{status}': {run_data}" + ) + + return run_data + + @staticmethod + async def _accumulate_wxo_sse_text(response: Any) -> str: + accumulated_text = "" + async for line in response.aiter_lines(): + if not line.startswith("data:"): + continue + data_str = line[5:].strip() + if not data_str or data_str == "[DONE]": + continue + try: + event = json.loads(data_str) + except json.JSONDecodeError: + continue + chunk_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result( + event + ) + if chunk_text: + accumulated_text += chunk_text + return accumulated_text + + @staticmethod + def _extract_litellm_params(litellm_params: Dict[str, Any]) -> WXORequestParams: + cp4d_host = litellm_params.get("cp4d_host") or "" + instance_id = litellm_params.get("instance_id") or "" + wxo_agent_id = litellm_params.get("wxo_agent_id") or "" + api_key = litellm_params.get("api_key") or "" + + if not cp4d_host: + raise ValueError("'cp4d_host' is required in litellm_params for WXO agents") + if not instance_id: + raise ValueError( + "'instance_id' is required in litellm_params for WXO agents" + ) + if not wxo_agent_id: + raise ValueError( + "'wxo_agent_id' is required in litellm_params for WXO agents" + ) + if not api_key: + raise ValueError("'api_key' is required in litellm_params for WXO agents") + + return WXORequestParams( + cp4d_host=cp4d_host, + instance_id=instance_id, + wxo_agent_id=wxo_agent_id, + api_key=api_key, + username=litellm_params.get("username") or None, + auth_mode=litellm_params.get("auth_mode") or "cp4d", + thread_id=litellm_params.get("thread_id") or None, + ) + + @staticmethod + async def handle_non_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + ) -> Dict[str, Any]: + wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params) + + client = WatsonxOrchestrateHandler._http_client(timeout=90.0) + token = await WatsonxOrchestrateHandler._get_bearer_token( + cp4d_host=wxo.cp4d_host, + auth_mode=wxo.auth_mode, + api_key=wxo.api_key, + username=wxo.username, + client=client, + ) + base_url = WatsonxOrchestrateTransformation.get_api_base_url( + wxo.cp4d_host, wxo.instance_id + ) + auth_headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Accept": "application/json", + } + + text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params) + body = WatsonxOrchestrateTransformation.build_wxo_run_body( + wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id + ) + + run_response = await client.post( + f"{base_url}/v1/orchestrate/runs", + json=body, + headers=auth_headers, + ) + run_response.raise_for_status() + run_data: Dict[str, Any] = run_response.json() + + run_data = await WatsonxOrchestrateHandler._get_successful_run_data( + run_data=run_data, + base_url=base_url, + auth_headers=auth_headers, + client=client, + ) + + response_text = WatsonxOrchestrateTransformation.extract_text_from_wxo_result( + run_data + ) + return WatsonxOrchestrateTransformation.build_a2a_message_response( + request_id=request_id, text=response_text + ) + + @staticmethod + async def handle_streaming( + request_id: str, + params: Dict[str, Any], + litellm_params: Dict[str, Any], + chunk_size: int = 50, + delay_ms: int = 10, + ) -> AsyncIterator[Dict[str, Any]]: + wxo = WatsonxOrchestrateHandler._extract_litellm_params(litellm_params) + + client = WatsonxOrchestrateHandler._http_client(timeout=120.0) + token = await WatsonxOrchestrateHandler._get_bearer_token( + cp4d_host=wxo.cp4d_host, + auth_mode=wxo.auth_mode, + api_key=wxo.api_key, + username=wxo.username, + client=client, + ) + base_url = WatsonxOrchestrateTransformation.get_api_base_url( + wxo.cp4d_host, wxo.instance_id + ) + auth_headers = { + "Authorization": f"Bearer {token}", + "Content-Type": "application/json", + "Accept": "text/event-stream, application/json", + } + text = WatsonxOrchestrateTransformation.extract_text_from_a2a_params(params) + body = WatsonxOrchestrateTransformation.build_wxo_run_body( + wxo_agent_id=wxo.wxo_agent_id, text=text, thread_id=wxo.thread_id + ) + + try: + response = await client.post( + f"{base_url}/v1/orchestrate/runs/stream", + json=body, + headers=auth_headers, + stream=True, + ) + response.raise_for_status() + except httpx.TransportError as exc: + verbose_logger.warning( + f"WXO: Streaming request failed before a run was submitted " + f"({exc!r}), falling back to non-streaming + fake streaming", + exc_info=True, + ) + result = await WatsonxOrchestrateHandler.handle_non_streaming( + request_id=request_id, + params=params, + litellm_params=litellm_params, + ) + response_text = ( + WatsonxOrchestrateTransformation.extract_text_from_a2a_message_response( + result + ) + ) + async for ( + chunk + ) in WatsonxOrchestrateTransformation.fake_streaming_from_text( + text=response_text, + request_id=request_id, + chunk_size=chunk_size, + delay_ms=delay_ms, + ): + yield chunk + return + + content_type = response.headers.get("content-type", "").lower() + if "text/event-stream" not in content_type: + response_body = await response.aread() + result = json.loads(response_body) + result = await WatsonxOrchestrateHandler._get_successful_run_data( + run_data=result, + base_url=base_url, + auth_headers=auth_headers, + client=client, + ) + accumulated_text = ( + WatsonxOrchestrateTransformation.extract_text_from_wxo_result(result) + ) + else: + accumulated_text = await WatsonxOrchestrateHandler._accumulate_wxo_sse_text( + response + ) + + async for chunk in WatsonxOrchestrateTransformation.fake_streaming_from_text( + text=accumulated_text, + request_id=request_id, + chunk_size=chunk_size, + delay_ms=delay_ms, + ): + yield chunk diff --git a/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py new file mode 100644 index 0000000000..824e9dbcdd --- /dev/null +++ b/litellm/a2a_protocol/providers/watsonx_orchestrate/transformation.py @@ -0,0 +1,224 @@ +""" +Transformation layer for IBM watsonx Orchestrate (WXO) agent provider. + +WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model: + POST /v1/orchestrate/runs → submit run, get run_id + GET /v1/orchestrate/runs/{id} → poll until terminal state + POST /v1/orchestrate/runs/stream → native SSE streaming +""" + +import asyncio +from typing import Any, AsyncIterator, Dict, Optional +from uuid import uuid4 + +from litellm._logging import verbose_logger + + +class WatsonxOrchestrateTransformation: + """ + Handles request/response transformation between A2A and the WXO REST API. + """ + + TERMINAL_STATES = frozenset( + {"completed", "succeeded", "failed", "error", "cancelled"} + ) + SUCCESS_STATES = frozenset({"completed", "succeeded"}) + + @staticmethod + def get_api_base_url(cp4d_host: str, instance_id: str) -> str: + """Build the WXO API base URL from host and instance ID.""" + return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}" + + @staticmethod + def extract_text_from_a2a_params(params: Dict[str, Any]) -> str: + """ + Extract user message text from A2A MessageSendParams. + + A2A format: params.message.parts[*] where part.kind == "text" + """ + message = params.get("message", {}) + parts = message.get("parts", []) + texts = [] + for part in parts: + if not isinstance(part, dict): + continue + kind = part.get("kind") + if kind in (None, "", "text") and part.get("text"): + texts.append(part["text"]) + return " ".join(texts) or "" + + @staticmethod + def build_wxo_run_body( + wxo_agent_id: str, + text: str, + thread_id: Optional[str] = None, + ) -> Dict[str, Any]: + """Build the WXO POST /v1/orchestrate/runs request body.""" + body: Dict[str, Any] = { + "agent_id": wxo_agent_id, + "message": { + "role": "user", + "content": [ + { + "response_type": "text", + "text": text, + } + ], + }, + } + if thread_id: + body["thread_id"] = thread_id + return body + + @staticmethod + def extract_text_from_wxo_result(result: Any) -> str: + """ + Extract response text from a WXO run result. + + WXO can return text in several locations; checks in priority order per the API spec. + """ + if not isinstance(result, dict): + return "" + + # Primary: last_message.content[0].text + try: + text = result["last_message"]["content"][0]["text"] + if text: + return str(text) + except (KeyError, IndexError, TypeError): + pass + + # Secondary: result.data.message.content[0].text + try: + text = result["result"]["data"]["message"]["content"][0]["text"] + if text: + return str(text) + except (KeyError, IndexError, TypeError): + pass + + # Tertiary: results as a raw string + results = result.get("results") + if results and isinstance(results, str): + return results + + return "" + + @staticmethod + def extract_text_from_a2a_message_response(a2a_response: Dict[str, Any]) -> str: + result = a2a_response.get("result") + if not isinstance(result, dict): + verbose_logger.warning("WXO: A2A response missing result object") + return "" + parts = result.get("parts") + if not isinstance(parts, list): + verbose_logger.warning("WXO: A2A result has no parts list") + return "" + for part in parts: + if ( + isinstance(part, dict) + and part.get("kind") == "text" + and part.get("text") + ): + return str(part["text"]) + verbose_logger.warning("WXO: A2A result parts contained no text") + return "" + + @staticmethod + def build_a2a_message_response(request_id: str, text: str) -> Dict[str, Any]: + """ + Build a standard A2A non-streaming SendMessageResponse (kind=message). + """ + return { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "kind": "message", + "role": "agent", + "parts": [{"kind": "text", "text": text}], + "messageId": str(uuid4()), + }, + } + + @staticmethod + async def fake_streaming_from_text( + text: str, + request_id: str, + chunk_size: int = 50, + delay_ms: int = 10, + ) -> AsyncIterator[Dict[str, Any]]: + """ + Emit standard A2A streaming events from a completed text response. + + Event sequence: + 1. task (kind="task", state="submitted") + 2. status-update (kind="status-update", state="working") + 3. artifact-update chunks + 4. status-update (kind="status-update", state="completed", final=True) + """ + task_id = str(uuid4()) + context_id = str(uuid4()) + artifact_id = str(uuid4()) + + # 1. Task submitted + yield { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "id": task_id, + "kind": "task", + "status": {"state": "submitted"}, + }, + } + + # 2. Working + yield { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "final": False, + "kind": "status-update", + "status": {"state": "working"}, + "taskId": task_id, + }, + } + await asyncio.sleep(delay_ms / 1000.0) + + # 3. Artifact chunks (always emit at least one chunk, even for empty text) + text_to_chunk = text or "" + for i in range(0, max(len(text_to_chunk), 1), chunk_size): + chunk_text = text_to_chunk[i : i + chunk_size] + is_last = (i + chunk_size) >= max(len(text_to_chunk), 1) + yield { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "kind": "artifact-update", + "taskId": task_id, + "artifact": { + "artifactId": artifact_id, + "parts": [{"kind": "text", "text": chunk_text}], + }, + }, + } + if not is_last: + await asyncio.sleep(delay_ms / 1000.0) + + # 4. Completed + yield { + "jsonrpc": "2.0", + "id": request_id, + "result": { + "contextId": context_id, + "final": True, + "kind": "status-update", + "status": {"state": "completed"}, + "taskId": task_id, + }, + } + + verbose_logger.debug( + f"WXO: Fake streaming completed for request_id={request_id}" + ) diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index 2de7bda646..87c26b776e 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -182,6 +182,14 @@ class ResponsesToCompletionBridgeHandler: client=kwargs.get("client"), ) + # Pin the resolved provider so `responses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). request_data already + # carries `custom_llm_provider` via the spread of + # `sanitized_litellm_params`; overwriting it on the dict (rather + # than adding an explicit kwarg) avoids the duplicate-keyword + # TypeError that would otherwise fire on the real bridge path. + request_data["custom_llm_provider"] = custom_llm_provider result = responses( **request_data, ) @@ -268,6 +276,13 @@ class ResponsesToCompletionBridgeHandler: except Exception as e: raise e + # Pin the resolved provider so `aresponses()` doesn't re-run + # `get_llm_provider()` on the model string and strip a second + # provider prefix (see GitHub issue #28505). Set on request_data + # rather than passed as a separate kwarg to avoid the duplicate- + # keyword TypeError when `sanitized_litellm_params` already + # carries `custom_llm_provider`. + request_data["custom_llm_provider"] = custom_llm_provider result = await aresponses( **request_data, aresponses=True, diff --git a/litellm/constants.py b/litellm/constants.py index ae98b37d6e..36e578bd32 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -585,6 +585,7 @@ LITELLM_CHAT_PROVIDERS = [ "volcengine", "codestral", "text-completion-codestral", + "text-completion-inception", "deepseek", "sambanova", "maritalk", @@ -620,6 +621,7 @@ LITELLM_CHAT_PROVIDERS = [ "oci", "morph", "lambda_ai", + "inception", "vercel_ai_gateway", "wandb", "ovhcloud", @@ -676,6 +678,7 @@ OPENAI_CHAT_COMPLETION_PARAMS = [ "extra_headers", "thinking", "web_search_options", + "include_server_side_tool_invocations", "service_tier", "prompt_cache_key", "prompt_cache_retention", @@ -737,6 +740,7 @@ DEFAULT_CHAT_COMPLETION_PARAM_VALUES = { "verbosity": None, "thinking": None, "web_search_options": None, + "include_server_side_tool_invocations": None, "service_tier": None, "safety_identifier": None, "prompt_cache_key": None, @@ -771,6 +775,7 @@ openai_compatible_endpoints: List = [ "https://api.moonshot.ai/v1", "https://api.publicai.co/v1", "https://api.synthetic.new/openai/v1", + "https://serverless.tensormesh.ai/v1", "https://api.stima.tech/v1", "https://nano-gpt.com/api/v1", "https://api.poe.com/v1", @@ -778,6 +783,7 @@ openai_compatible_endpoints: List = [ "https://api.v0.dev/v1", "https://api.morphllm.com/v1", "https://api.lambda.ai/v1", + "https://api.inceptionlabs.ai/v1", "https://api.hyperbolic.xyz/v1", "https://ai-gateway.helicone.ai/", "https://ai-gateway.vercel.sh/v1", @@ -820,6 +826,7 @@ openai_compatible_providers: List = [ "meta_llama", "publicai", # PublicAI - JSON-configured provider "synthetic", # Synthetic - JSON-configured provider + "tensormesh", # Tensormesh - JSON-configured provider "apertis", # Apertis - JSON-configured provider "nano-gpt", # Nano-GPT - JSON-configured provider "poe", # Poe - JSON-configured provider @@ -833,6 +840,7 @@ openai_compatible_providers: List = [ "helicone", "morph", "lambda_ai", + "inception", "hyperbolic", "vercel_ai_gateway", "aiml", @@ -855,6 +863,7 @@ openai_text_completion_compatible_providers: List = ( "moonshot", "publicai", "synthetic", + "tensormesh", "apertis", "nano-gpt", "poe", @@ -868,6 +877,7 @@ openai_text_completion_compatible_providers: List = ( _openai_like_providers: List = [ "predibase", "databricks", + "lemonade", "watsonx", ] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk # well supported replicate llms diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 9a4b158b62..88029615ba 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -2425,12 +2425,11 @@ class BaseTokenUsageProcessor: if not attr.startswith("_") and not callable( getattr(usage.completion_tokens_details, attr) ): - current_val = getattr( - combined.completion_tokens_details, attr, 0 + current_val = ( + getattr(combined.completion_tokens_details, attr, 0) or 0 ) - new_val = getattr(usage.completion_tokens_details, attr, 0) - - if new_val is not None and current_val is not None: + new_val = getattr(usage.completion_tokens_details, attr, 0) or 0 + if isinstance(new_val, (int, float)): setattr( combined.completion_tokens_details, attr, diff --git a/litellm/exceptions.py b/litellm/exceptions.py index 17f5b43c27..15f6030d4a 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -1062,3 +1062,37 @@ class GuardrailInterventionNormalStringError( def __repr__(self): return self.__str__() + + +class SensitiveDataRouteException(Exception): + """ + Exception raised when a guardrail detects sensitive data and wants to reroute the request. + + Instead of blocking the request, this exception signals that the request should be + routed to a different model (typically an on-premise model for data privacy). + + The proxy catches this exception and: + 1. Reroutes the current request to the specified model + 2. When sticky_session_routing is True, stores the routing decision in session + cache so all subsequent requests in the same session are routed to the same model + """ + + def __init__( + self, + route_to_model: str, + session_id: str, + guardrail_name: Optional[str] = None, + detection_info: Optional[Dict[str, Any]] = None, + message: Optional[str] = None, + sticky_session_routing: bool = True, + ): + self.route_to_model = route_to_model + self.session_id = session_id + self.guardrail_name = guardrail_name + self.detection_info = detection_info or {} + self.sticky_session_routing = sticky_session_routing + self.message = ( + message + or f"Sensitive data detected by {guardrail_name}. Routing to model: {route_to_model}" + ) + super().__init__(self.message) diff --git a/litellm/experimental_mcp_client/client.py b/litellm/experimental_mcp_client/client.py index 0dc56b6a3b..c6d427e7f0 100644 --- a/litellm/experimental_mcp_client/client.py +++ b/litellm/experimental_mcp_client/client.py @@ -4,6 +4,7 @@ LiteLLM Proxy uses this MCP Client to connnect to other MCP servers. import asyncio import base64 +import os from typing import ( Any, Awaitable, @@ -16,7 +17,6 @@ from typing import ( TypeVar, Union, ) - import httpx from mcp import ClientSession, ReadResourceResult, Resource, StdioServerParameters from mcp.client.sse import sse_client @@ -42,9 +42,8 @@ from mcp.types import ( ) from mcp.types import Tool as MCPTool from pydantic import AnyUrl - from litellm._logging import verbose_logger -from litellm.constants import MCP_CLIENT_TIMEOUT +from litellm.constants import MCP_CLIENT_TIMEOUT, MCP_NPM_CACHE_DIR from litellm.llms.custom_httpx.http_handler import get_ssl_configuration from litellm.types.llms.custom_http import VerifyTypes from litellm.types.mcp import ( @@ -61,13 +60,33 @@ def to_basic_auth(auth_value: str) -> str: return base64.b64encode(auth_value.encode("utf-8")).decode() +def _strip_header_whitespace(headers: Dict[str, str]) -> Dict[str, str]: + return { + (key.strip() if isinstance(key, str) else key): ( + value.strip() if isinstance(value, str) else value + ) + for key, value in headers.items() + } + + +def _first_non_cancelled_cause(exc: BaseException) -> Optional[BaseException]: + queue: List[BaseException] = [exc] + while queue: + current = queue.pop(0) + nested = getattr(current, "exceptions", None) + if nested: + queue.extend(nested) + elif not isinstance(current, asyncio.CancelledError): + return current + return None + + TSessionResult = TypeVar("TSessionResult") class MCPSigV4Auth(httpx.Auth): """ httpx Auth class that signs each request with AWS SigV4. - This is used for MCP servers that require AWS SigV4 authentication, such as AWS Bedrock AgentCore MCP servers. httpx calls auth_flow() for every outgoing request, enabling per-request signature computation. @@ -92,10 +111,8 @@ class MCPSigV4Auth(httpx.Auth): "Missing botocore to use AWS SigV4 authentication. " "Run 'pip install boto3'." ) - self.service_name = aws_service_name or "bedrock-agentcore" self.region_name = aws_region_name or "us-east-1" - # Note: os.environ/ prefixed values are already resolved by # ProxyConfig._check_for_os_environ_vars() at config load time. # Values arrive here as plain strings. @@ -143,20 +160,17 @@ class MCPSigV4Auth(httpx.Auth): session_name = ( aws_session_name or f"litellm-mcp-{int(__import__('time').time())}" ) - sts_kwargs: dict = {"region_name": aws_region_name} if aws_access_key_id and aws_secret_access_key: sts_kwargs["aws_access_key_id"] = aws_access_key_id sts_kwargs["aws_secret_access_key"] = aws_secret_access_key if aws_session_token: sts_kwargs["aws_session_token"] = aws_session_token - sts_client = boto3.client("sts", **sts_kwargs) sts_response = sts_client.assume_role( RoleArn=aws_role_name, RoleSessionName=session_name, ) - sts_creds = sts_response["Credentials"] return Credentials( access_key=sts_creds["AccessKeyId"], @@ -178,17 +192,14 @@ class MCPSigV4Auth(httpx.Auth): data=request.content, headers=dict(request.headers), ) - # Sign the request — SigV4Auth.add_auth() adds Authorization, # X-Amz-Date, and X-Amz-Security-Token (if session token present). # Host header is derived automatically from the URL. sigv4 = SigV4Auth(self.credentials, self.service_name, self.region_name) sigv4.add_auth(aws_request) - # Copy SigV4 headers back to the httpx request for header_name, header_value in aws_request.headers.items(): request.headers[header_name] = header_value - yield request @@ -198,6 +209,8 @@ class MCPClient: SSE and HTTP transports Authentication via Bearer token, Basic Auth, or API Key Tool calling with error handling and result parsing + Sampling callbacks for upstream server LLM requests + Elicitation callbacks for upstream server user-input requests """ def __init__( @@ -211,6 +224,9 @@ class MCPClient: extra_headers: Optional[Dict[str, str]] = None, ssl_verify: Optional[VerifyTypes] = None, aws_auth: Optional[httpx.Auth] = None, + sampling_callback: Optional[Callable] = None, + elicitation_callback: Optional[Callable] = None, + logging_callback: Optional[Callable] = None, ): self.server_url: str = server_url self.transport_type: MCPTransport = transport_type @@ -222,6 +238,9 @@ class MCPClient: self.ssl_verify: Optional[VerifyTypes] = ssl_verify self._aws_auth: Optional[httpx.Auth] = aws_auth self._last_initialize_instructions: Optional[str] = None + self._sampling_callback: Optional[Callable] = sampling_callback + self._elicitation_callback: Optional[Callable] = elicitation_callback + self._logging_callback: Optional[Callable] = logging_callback # handle the basic auth value if provided if auth_value: self.update_auth_value(auth_value) @@ -231,23 +250,20 @@ class MCPClient: ) -> Tuple[Any, Optional[httpx.AsyncClient]]: """ Create the appropriate transport context based on transport type. - Returns: Tuple of (transport_context, http_client). http_client is only set for HTTP transport and needs cleanup. """ http_client: Optional[httpx.AsyncClient] = None - if self.transport_type == MCPTransport.stdio: if not self.stdio_config: raise ValueError("stdio_config is required for stdio transport") server_params = StdioServerParameters( command=self.stdio_config.get("command", ""), args=self.stdio_config.get("args", []), - env=self.stdio_config.get("env", {}), + env=self._get_safe_stdio_env(self.stdio_config.get("env")), ) return stdio_client(server_params), None - if self.transport_type == MCPTransport.sse: headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() @@ -260,14 +276,12 @@ class MCPClient: ), None, ) - # HTTP transport (default) if streamable_http_client is None: raise ImportError( "streamable_http_client is not available. " "Please install mcp with HTTP support." ) - headers = self._get_auth_headers() httpx_client_factory = self._create_httpx_client_factory() verbose_logger.debug("litellm headers for streamable_http_client: %s", headers) @@ -281,6 +295,54 @@ class MCPClient: ) return transport_ctx, http_client + def _get_safe_stdio_env( + self, provided_env: Optional[Dict[str, str]] + ) -> Optional[Dict[str, str]]: + """ + Return a safe environment for the stdio subprocess. + + If provided_env is set, we use it as-is. + If provided_env is None, we return a minimal allowlist from the parent environment + to avoid leaking sensitive LiteLLM keys (OPENAI_API_KEY, etc.) to sub-processes. + """ + if provided_env is not None: + return provided_env + + # Minimal allowlist of safe/standard environment variables + safe_keys = { + "PATH", + "HOME", + "USER", + "LOGNAME", + "TMPDIR", + "TMP", + "TEMP", + "SHELL", + "LANG", + "LC_ALL", + # Node/Package manager caches + "NPM_CONFIG_CACHE", + "PNPM_HOME", + "XDG_CACHE_HOME", + "XDG_CONFIG_HOME", + "XDG_DATA_HOME", + # System info + "SYSTEMROOT", + "COMSPEC", + "PATHEXT", + "WINDIR", + } + + safe_env = {} + for key in safe_keys: + if key in os.environ: + safe_env[key] = os.environ[key] + + if "NPM_CONFIG_CACHE" not in safe_env: + safe_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR + + return safe_env + async def _execute_session_operation( self, transport_ctx: Any, @@ -288,13 +350,24 @@ class MCPClient: ) -> TSessionResult: """ Execute an operation within a transport and session context. - Handles entering/exiting contexts and running the operation. + Passes sampling/elicitation/logging callbacks to the ClientSession + so that upstream MCP servers can request LLM inference (sampling), + user input (elicitation), or send log messages. """ transport = await transport_ctx.__aenter__() + in_flight_error: Optional[BaseException] = None try: read_stream, write_stream = transport[0], transport[1] - session_ctx = ClientSession(read_stream, write_stream) + # Build session kwargs with optional callbacks + session_kwargs: Dict[str, Any] = {} + if self._sampling_callback is not None: + session_kwargs["sampling_callback"] = self._sampling_callback + if self._elicitation_callback is not None: + session_kwargs["elicitation_callback"] = self._elicitation_callback + if self._logging_callback is not None: + session_kwargs["logging_callback"] = self._logging_callback + session_ctx = ClientSession(read_stream, write_stream, **session_kwargs) session = await session_ctx.__aenter__() try: init_result = await session.initialize() @@ -309,11 +382,21 @@ class MCPClient: await session_ctx.__aexit__(None, None, None) except BaseException as e: verbose_logger.debug(f"Error during session context exit: {e}") + except BaseException as e: + in_flight_error = e + raise finally: try: await transport_ctx.__aexit__(None, None, None) - except BaseException as e: - verbose_logger.debug(f"Error during transport context exit: {e}") + except BaseException as exit_error: + verbose_logger.debug( + f"Error during transport context exit: {exit_error}" + ) + root_cause = _first_non_cancelled_cause(exit_error) + if root_cause is not None and isinstance( + in_flight_error, asyncio.CancelledError + ): + raise root_cause from in_flight_error async def run_with_session( self, operation: Callable[[ClientSession], Awaitable[TSessionResult]] @@ -351,7 +434,6 @@ class MCPClient: def _get_auth_headers(self) -> dict: """Generate authentication headers based on auth type.""" headers = {} - if self._mcp_auth_value: if isinstance(self._mcp_auth_value, str): if self.auth_type == MCPAuth.bearer_token: @@ -373,17 +455,14 @@ class MCPClient: # Note: aws_sigv4 auth is not handled here — SigV4 requires per-request # signing (including the body hash), so it uses httpx.Auth flow instead # of static headers. See MCPSigV4Auth and _create_httpx_client_factory(). - # update the headers with the extra headers if self.extra_headers: headers.update(self.extra_headers) - - return headers + return _strip_header_whitespace(headers) def _create_httpx_client_factory(self) -> Callable[..., httpx.AsyncClient]: """ Create a custom httpx client factory that uses LiteLLM's SSL configuration. - This factory follows the same CA bundle path logic as http_handler.py: 1. Check ssl_verify parameter (can be SSLContext, bool, or path to CA bundle) 2. Check SSL_VERIFY environment variable @@ -400,17 +479,14 @@ class MCPClient: """Create an httpx.AsyncClient with LiteLLM's SSL configuration.""" # Get unified SSL configuration using the same logic as http_handler.py ssl_config = get_ssl_configuration(self.ssl_verify) - verbose_logger.debug( f"MCP client using SSL configuration: {type(ssl_config).__name__}" ) - # Use SigV4 auth if configured and no explicit auth provided. # The MCP SDK's sse_client and streamable_http_client call this # factory without passing auth=, so self._aws_auth is used. # For non-SigV4 clients, self._aws_auth is None — no behavior change. effective_auth = auth if auth is not None else self._aws_auth - return httpx.AsyncClient( headers=headers, timeout=timeout, @@ -421,8 +497,16 @@ class MCPClient: return factory - async def list_tools(self) -> List[MCPTool]: - """List available tools from the server.""" + async def list_tools(self, raise_on_error: bool = False) -> List[MCPTool]: + """List available tools from the server. + + Args: + raise_on_error: When True, re-raise exceptions instead of returning + an empty list. Used by the proxy's pass-through MCP flow so it + can surface upstream HTTP 401 responses as a proper 401 to the + MCP client (triggering the upstream OAuth flow) rather than + masking them as "connected, no tools". + """ verbose_logger.debug( f"MCP client listing tools from {self.server_url or 'stdio'}" ) @@ -450,7 +534,6 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( @@ -458,6 +541,8 @@ class MCPClient: "the MCP server may have crashed, disconnected, or timed out" ) + if raise_on_error: + raise # Return empty list instead of raising to allow graceful degradation return [] @@ -481,7 +566,6 @@ class MCPClient: f"MCP Tool '{call_tool_request_params.name}' progress: " f"{progress}/{total} ({percentage:.0f}%) - {message or ''}" ) - # Forward to Host if callback provided if host_progress_callback: try: @@ -504,14 +588,15 @@ class MCPClient: ) return tool_result except asyncio.CancelledError: - verbose_logger.warning("MCP client tool call was cancelled") + verbose_logger.warning( + f"MCP client tool call timed out after {self.timeout}s for {self.server_url}" + ) raise except Exception as e: import traceback error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client tool call traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -522,14 +607,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream - " "the MCP server may have crashed, disconnected, or timed out." ) - # Return a default error result instead of raising return MCPCallToolResult( content=[ @@ -567,14 +650,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_tools - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -607,7 +688,6 @@ class MCPClient: error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client get_prompt traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -618,14 +698,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during get_prompt - " "the MCP server may have crashed, disconnected, or timed out." ) - raise async def list_resources(self) -> list[Resource]: @@ -657,14 +735,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_resources - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -699,14 +775,12 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during list_resource_templates - " "the MCP server may have crashed, disconnected, or timed out" ) - # Return empty list instead of raising to allow graceful degradation return [] @@ -732,7 +806,6 @@ class MCPClient: error_trace = traceback.format_exc() verbose_logger.debug(f"MCP client read_resource traceback:\n{error_trace}") - # Log detailed error information error_type = type(e).__name__ verbose_logger.error( @@ -743,12 +816,10 @@ class MCPClient: f"Server: {self.server_url or 'stdio'}, " f"Transport: {self.transport_type}" ) - # Check if it's a stream/connection error if "BrokenResourceError" in error_type or "Broken" in error_type: verbose_logger.error( "MCP client detected broken connection/stream during read_resource - " "the MCP server may have crashed, disconnected, or timed out." ) - raise diff --git a/litellm/integrations/arize/_utils.py b/litellm/integrations/arize/_utils.py index a1bf65141c..75710e1049 100644 --- a/litellm/integrations/arize/_utils.py +++ b/litellm/integrations/arize/_utils.py @@ -8,18 +8,23 @@ from litellm.integrations.opentelemetry_utils.base_otel_llm_obs_attributes impor BaseLLMObsOTELAttributes, safe_set_attribute, ) +from litellm.litellm_core_utils.redact_messages import ( + should_redact_message_logging, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.types.utils import StandardLoggingPayload if TYPE_CHECKING: from opentelemetry.trace import Span from litellm.integrations._types.open_inference import ( - MessageAttributes, - ImageAttributes, - SpanAttributes, AudioAttributes, EmbeddingAttributes, + ImageAttributes, + MessageAttributes, + MessageContentAttributes, OpenInferenceSpanKindValues, + SpanAttributes, + ToolCallAttributes, ) @@ -53,40 +58,24 @@ class ArizeOTELAttributes(BaseLLMObsOTELAttributes): msg.get("content", ""), ) - @staticmethod - @override - def set_response_output_messages(span: "Span", response_obj): - """ - Sets output message attributes on the span from the LLM response. - Args: - span: The OpenTelemetry span to set attributes on - response_obj: The response object containing choices with messages - """ - from litellm.integrations._types.open_inference import ( - MessageAttributes, - SpanAttributes, - ) + # Additive: emit structured tool_calls / multimodal content + # so Arize/Phoenix can render tool-using and image-bearing + # turns. These set NEW attribute keys (MESSAGE_TOOL_CALLS / + # MESSAGE_NAME / MESSAGE_TOOL_CALL_ID / MESSAGE_CONTENTS.*) — + # never replace the MESSAGE_CONTENT write above. + _safe_emit( + f"input message extras (idx={idx})", + _emit_input_message_extras, + span, + prefix, + msg, + ) - for idx, choice in enumerate(response_obj.get("choices", [])): - response_message = choice.get("message", {}) - safe_set_attribute( - span, - SpanAttributes.OUTPUT_VALUE, - response_message.get("content", ""), - ) - - # This shows up under `output_messages` tab on the span page. - prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.{idx}" - safe_set_attribute( - span, - f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", - response_message.get("role"), - ) - safe_set_attribute( - span, - f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", - response_message.get("content", ""), - ) + # Note: `BaseLLMObsOTELAttributes.set_response_output_messages` is not + # overridden here. The live code path uses `_set_choice_outputs` (called + # via `_set_response_attributes` from `set_attributes`) which handles + # tool_calls, multimodal output, embeddings, audio, images, and structured + # outputs in a single place. def _set_response_attributes(span: "Span", response_obj): @@ -106,11 +95,17 @@ def _set_response_attributes(span: "Span", response_obj): def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs): for idx, choice in enumerate(response_obj.get("choices", [])): response_message = choice.get("message", {}) - safe_set_attribute( - span, - span_attrs.OUTPUT_VALUE, - response_message.get("content", ""), - ) + content = response_message.get("content", "") + + # Tool-only assistant responses have empty content; serialize the + # tool_calls into OUTPUT_VALUE so Arize's "Output" pane isn't blank. + output_value = content + if not output_value: + tool_calls = _get_tool_calls(response_message) + if tool_calls: + output_value = _summarize_tool_calls_for_output(tool_calls) + + safe_set_attribute(span, span_attrs.OUTPUT_VALUE, output_value) prefix = f"{span_attrs.LLM_OUTPUT_MESSAGES}.{idx}" safe_set_attribute( span, @@ -120,7 +115,18 @@ def _set_choice_outputs(span: "Span", response_obj, msg_attrs, span_attrs): safe_set_attribute( span, f"{prefix}.{msg_attrs.MESSAGE_CONTENT}", - response_message.get("content", ""), + content, + ) + + # Additive: emit assistant tool_calls so tool-using turns render in + # Arize/Phoenix. Sets new MESSAGE_TOOL_CALLS keys only — does not + # change MESSAGE_CONTENT/MESSAGE_ROLE writes above. + _safe_emit( + f"output tool_calls (idx={idx})", + _emit_message_tool_calls, + span, + prefix, + response_message, ) @@ -278,6 +284,43 @@ def _set_usage_outputs(span: "Span", response_obj, span_attrs): reasoning_tokens, ) + # Additive: cache token breakdown so prompt-caching savings render in + # Arize. Sources covered: + # - OpenAI Chat Completions: `prompt_tokens_details.cached_tokens` + # - Anthropic / Bedrock-Anthropic: `cache_read_input_tokens`, + # `cache_creation_input_tokens` + # All emits are conditional, so when none of these fields exist (the + # situation in the existing test fixtures) no extra attributes are set. + prompt_token_details = _safe_get(usage, "prompt_tokens_details") or _safe_get( + usage, "input_tokens_details" + ) + cache_read = _safe_get(prompt_token_details, "cached_tokens") or _safe_get( + usage, "cache_read_input_tokens" + ) + if cache_read: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_READ, + cache_read, + ) + # Anthropic / Bedrock-Anthropic only — OpenAI's `prompt_tokens_details` + # does not expose a cache-write count, so we read straight off `usage`. + cache_write = _safe_get(usage, "cache_creation_input_tokens") + if cache_write: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_CACHE_WRITE, + cache_write, + ) + + audio_prompt_tokens = _safe_get(prompt_token_details, "audio_tokens") + if audio_prompt_tokens: + safe_set_attribute( + span, + span_attrs.LLM_TOKEN_COUNT_PROMPT_DETAILS_AUDIO, + audio_prompt_tokens, + ) + def _infer_open_inference_span_kind(call_type: Optional[str]) -> str: """ @@ -321,6 +364,10 @@ def _infer_open_inference_span_kind(call_type: Optional[str]) -> str: "videos", "realtime", "pass_through", + # `passthrough` (no underscore) is what real call_types use: + # `allm_passthrough_route`, `llm_passthrough_route`. Without + # this they fell through to UNKNOWN, blanking span.kind. + "passthrough", "anthropic_messages", "ocr", ) @@ -396,6 +443,18 @@ def set_attributes( """ Populates span with OpenInference-compliant LLM attributes for Arize and Phoenix tracing. """ + # Coerce non-dict response objects (e.g. httpx.Response from passthrough + # routes) into a dict so downstream `.get()` calls don't crash. Existing + # dict / `.get()`-bearing objects (incl. Pydantic OpenAI Responses API + # models) are returned unchanged, preserving the existing test behavior. + response_obj_for_attrs = _coerce_response_obj_for_attrs(response_obj) + + # Set span.kind defensively before anything else. If a downstream step + # throws, the span still has a kind so Arize can render it correctly + # (an LLM call instead of UNKNOWN). This is the single source of truth + # for span.kind — no late re-write happens below. + _safe_emit("early span kind", _set_early_span_kind, span, kwargs) + try: optional_params = _sanitize_optional_params(kwargs.get("optional_params")) litellm_params = kwargs.get("litellm_params", {}) or {} @@ -415,25 +474,22 @@ def set_attributes( metadata_tools = _extract_metadata_tools(metadata) optional_tools = _extract_optional_tools(optional_params) - call_type = standard_logging_payload.get("call_type") _set_request_attributes( span=span, kwargs=kwargs, standard_logging_payload=standard_logging_payload, optional_params=optional_params, litellm_params=litellm_params, - response_obj=response_obj, + response_obj=response_obj_for_attrs, span_attrs=SpanAttributes, ) - span_kind = _infer_open_inference_span_kind(call_type=call_type) + # span.kind was already set above by `_set_early_span_kind`. We do + # NOT re-write it here based on tool presence: a chat completion + # that passes `tools=[...]` (or returns `tool_calls`) is still an + # LLM call per the OpenInference spec — TOOL is reserved for actual + # tool execution spans, not LLM calls that request tools. _set_tool_attributes(span, optional_tools, metadata_tools) - if ( - optional_tools or metadata_tools - ) and span_kind != OpenInferenceSpanKindValues.TOOL.value: - span_kind = OpenInferenceSpanKindValues.TOOL.value - - safe_set_attribute(span, SpanAttributes.OPENINFERENCE_SPAN_KIND, span_kind) attributes.set_messages(span, kwargs) model_params = ( @@ -443,7 +499,7 @@ def set_attributes( ) _set_model_params(span, model_params, SpanAttributes) - _set_response_attributes(span=span, response_obj=response_obj) + _set_response_attributes(span=span, response_obj=response_obj_for_attrs) except Exception as e: verbose_logger.error( @@ -452,6 +508,22 @@ def set_attributes( if hasattr(span, "record_exception"): span.record_exception(e) + # Additive emitters. Each is independently guarded so a failure can never + # blank the attributes set by the main try-block above. New attributes are + # written under new keys; existing attributes are not overwritten. + slp = kwargs.get("standard_logging_object") + _safe_emit("session/user attrs", _set_session_and_user_attrs, span, kwargs, slp) + _safe_emit("response cost", _set_response_cost_attr, span, slp) + _safe_emit( + "passthrough normalization", + _maybe_normalize_passthrough, + span, + kwargs, + response_obj, + response_obj_for_attrs, + slp, + ) + def _sanitize_optional_params(optional_params: Optional[dict]) -> dict: if not isinstance(optional_params, dict): @@ -534,3 +606,529 @@ def _set_model_params(span: "Span", model_params: Optional[dict], span_attrs) -> user_id = model_params.get("user") if user_id is not None: safe_set_attribute(span, span_attrs.USER_ID, user_id) + + +# --------------------------------------------------------------------------- +# Additive rendering helpers (introduced to enhance Arize/Phoenix rendering +# without changing any previously-emitted attribute keys or values). +# --------------------------------------------------------------------------- + + +def _safe_emit(label: str, fn, *args, **kwargs) -> None: + """Run an additive attribute emitter, swallowing any error so it cannot + blank attributes set elsewhere on the span. Failures are logged at debug. + """ + try: + fn(*args, **kwargs) + except Exception as e: + verbose_logger.debug("[Arize] %s skipped: %s", label, e) + + +def _set_early_span_kind(span: "Span", kwargs: dict) -> None: + """Defensively set OPENINFERENCE_SPAN_KIND before any other logic runs.""" + slp = kwargs.get("standard_logging_object") + call_type = slp.get("call_type") if isinstance(slp, dict) else None + safe_set_attribute( + span, + SpanAttributes.OPENINFERENCE_SPAN_KIND, + _infer_open_inference_span_kind(call_type=call_type), + ) + + +def _coerce_response_obj_for_attrs(response_obj): + """Return a `.get`-compatible view of `response_obj` when possible. + + - dicts and Pydantic models that already expose `.get` are returned + unchanged (preserves all current behavior, including the Responses API + flow which relies on Pydantic attribute access). + - `httpx.Response` and other text-only responses (passthrough routes) + are JSON-decoded so the standard extraction paths can read fields like + `id`, `model`, and `usage`. On failure the original object is returned + so behavior is no worse than today. + """ + if response_obj is None or hasattr(response_obj, "get"): + return response_obj + text = getattr(response_obj, "text", None) + if isinstance(text, str) and text: + try: + parsed = json.loads(text) + if isinstance(parsed, dict): + return parsed + except Exception: + pass + return response_obj + + +def _coerce_text(value) -> Optional[str]: + """Best-effort text extraction from a message-content value. + + Returns None when no textual portion can be derived. Handles: + - plain strings + - lists of OpenAI-style content parts (`{"type": "text", "text": ...}`) + - lists of Anthropic-style content parts (`{"type": "text", "text": ...}` + or `{"type": "input_text", "text": ...}`) + """ + if value is None: + return None + if isinstance(value, str): + return value + if isinstance(value, list): + parts = [] + for part in value: + if isinstance(part, str): + parts.append(part) + elif isinstance(part, dict): + text = part.get("text") or part.get("input_text") + if isinstance(text, str): + parts.append(text) + if parts: + return "\n".join(parts) + return None + + +def _to_plain_dict(value): + """Best-effort: coerce a value (Pydantic model / dict / None) to a dict. + + Returns the original value when no safe conversion exists. Used to bridge + OpenAI Pydantic message/tool_call objects into the dict-based helpers. + """ + if value is None or isinstance(value, dict): + return value + model_dump = getattr(value, "model_dump", None) + if callable(model_dump): + try: + return model_dump() + except Exception: + pass + return value + + +def _get_tool_calls(message) -> Optional[list]: + """Return ``message.tool_calls`` only when it's a non-empty list. + + Works for dicts and Pydantic message objects via ``_safe_get``. + """ + tool_calls = _safe_get(message, "tool_calls") + return tool_calls if isinstance(tool_calls, list) and tool_calls else None + + +def _normalize_tool_call(raw_tc) -> Optional[Dict[str, Any]]: + """Normalize a single tool_call (dict or Pydantic) into a stable shape: + + {"id": str|None, "type": str, "function": {"name": str|None, "arguments": str|None}} + + Arguments are coerced to a JSON string per OpenInference convention. + Returns ``None`` when ``raw_tc`` cannot be coerced to a dict. + """ + tc = _to_plain_dict(raw_tc) + if not isinstance(tc, dict): + return None + function = _to_plain_dict(tc.get("function")) + name = function.get("name") if isinstance(function, dict) else None + args = function.get("arguments") if isinstance(function, dict) else None + if args is not None and not isinstance(args, str): + try: + args = json.dumps(args) + except Exception: + args = str(args) + return { + "id": tc.get("id"), + "type": tc.get("type", "function"), + "function": {"name": name, "arguments": args}, + } + + +def _summarize_tool_calls_for_output(tool_calls) -> str: + """Render a tool_calls list as a compact JSON string for OUTPUT_VALUE. + + Best-effort: returns ``str(tool_calls)`` if anything unexpected happens + so OUTPUT_VALUE is never blanked on a malformed payload. + """ + try: + normalized = [n for n in (_normalize_tool_call(tc) for tc in tool_calls) if n] + return json.dumps({"tool_calls": normalized}) + except Exception: + return str(tool_calls) + + +def _emit_message_tool_calls(span: "Span", prefix: str, message) -> None: + """Emit ``MESSAGE_TOOL_CALLS.*`` for an assistant message that requested + tool calls. Pure addition: only writes when ``tool_calls`` is non-empty. + + Accepts dicts or Pydantic message objects (e.g. ``litellm.Message``); the + same applies to each tool_call entry. + """ + tool_calls = _get_tool_calls(message) + if not tool_calls: + return + for tc_idx, raw_tc in enumerate(tool_calls): + tc = _normalize_tool_call(raw_tc) + if tc is None: + continue + tc_prefix = f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALLS}.{tc_idx}" + if tc["id"]: + safe_set_attribute( + span, f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_ID}", tc["id"] + ) + fn = tc["function"] + if fn["name"]: + safe_set_attribute( + span, + f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_NAME}", + fn["name"], + ) + if fn["arguments"] is not None: + safe_set_attribute( + span, + f"{tc_prefix}.{ToolCallAttributes.TOOL_CALL_FUNCTION_ARGUMENTS_JSON}", + fn["arguments"], + ) + + +def _emit_input_message_extras(span: "Span", prefix: str, message: dict) -> None: + """Emit additive attributes for an input message: + + - `MESSAGE_NAME` and `MESSAGE_TOOL_CALL_ID` (commonly set on tool-result + messages so traces show which tool produced which result). + - `MESSAGE_TOOL_CALLS.*` when an assistant message requested tools. + - `MESSAGE_CONTENTS.*` structured content for list-shaped content + (multimodal text + image parts). The plain `MESSAGE_CONTENT` write is + still performed by the caller, so renderers that only read the legacy + key continue to work. + """ + if not isinstance(message, dict): + return + + name = message.get("name") + if name: + safe_set_attribute(span, f"{prefix}.{MessageAttributes.MESSAGE_NAME}", name) + + tool_call_id = message.get("tool_call_id") + if tool_call_id: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_TOOL_CALL_ID}", + tool_call_id, + ) + + _emit_message_tool_calls(span, prefix, message) + + content = message.get("content") + if isinstance(content, list): + contents_prefix = f"{prefix}.{MessageAttributes.MESSAGE_CONTENTS}" + for part_idx, part in enumerate(content): + if not isinstance(part, dict): + continue + part_prefix = f"{contents_prefix}.{part_idx}" + part_type = part.get("type") + if part_type in ("text", "input_text"): + text = part.get("text") + if isinstance(text, str): + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", + "text", + ) + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TEXT}", + text, + ) + elif part_type in ("image_url", "image", "input_image"): + url = None + image = part.get("image_url") + if isinstance(image, dict): + url = image.get("url") + elif isinstance(image, str): + url = image + if not url: + # Anthropic-style source.{type=base64,media_type,data} + source = part.get("source") + if isinstance(source, dict) and source.get("data"): + media_type = source.get("media_type", "image/jpeg") + url = f"data:{media_type};base64,{source['data']}" + elif isinstance(part.get("url"), str): + url = part["url"] + if url: + safe_set_attribute( + span, + f"{part_prefix}.{MessageContentAttributes.MESSAGE_CONTENT_TYPE}", + "image", + ) + safe_set_attribute( + span, + f"{part_prefix}.message_content.image.image.url", + url, + ) + + +def _set_session_and_user_attrs( + span: "Span", kwargs: dict, standard_logging_payload +) -> None: + """Emit `SESSION_ID` / `USER_ID` / team metadata when source data exists. + + `SESSION_ID` is emitted only when an explicit end-user identifier exists + (`metadata.user_api_key_end_user_id`). We deliberately do NOT fall back + to `trace_id`, because that would create a distinct "session" for every + single request and distort Arize's Session-grouping analytics. The + `trace_id` is still emitted under its own `litellm.trace_id` key so + spans remain filterable by trace. + + USER_ID is *only* emitted when no upstream path (model_params.user or + optional_params.user) has already set it, to avoid overwriting an + existing value with a possibly-different one from API-key metadata. + """ + if not isinstance(standard_logging_payload, dict): + return + metadata = standard_logging_payload.get("metadata") or {} + if not isinstance(metadata, dict): + return + + session_id = metadata.get("user_api_key_end_user_id") + if session_id: + safe_set_attribute(span, SpanAttributes.SESSION_ID, str(session_id)) + + trace_id = standard_logging_payload.get("trace_id") + if trace_id: + safe_set_attribute(span, "litellm.trace_id", str(trace_id)) + + optional_params = kwargs.get("optional_params") or {} + model_params = standard_logging_payload.get("model_parameters") or {} + has_user_already = bool( + (isinstance(optional_params, dict) and optional_params.get("user")) + or (isinstance(model_params, dict) and model_params.get("user")) + ) + if not has_user_already: + user_id = metadata.get("user_api_key_user_id") + if user_id: + safe_set_attribute(span, SpanAttributes.USER_ID, str(user_id)) + + team_id = metadata.get("user_api_key_team_id") + if team_id: + safe_set_attribute(span, "litellm.team_id", str(team_id)) + team_alias = metadata.get("user_api_key_team_alias") + if team_alias: + safe_set_attribute(span, "litellm.team_alias", str(team_alias)) + key_alias = metadata.get("user_api_key_alias") + if key_alias: + safe_set_attribute(span, "litellm.key_alias", str(key_alias)) + + +def _set_response_cost_attr(span: "Span", standard_logging_payload) -> None: + """Emit cost attributes from the StandardLoggingPayload when present. + + Uses the OpenInference `llm.cost.total` key so Arize / Phoenix can + surface the cost in their "Total Cost" column. LiteLLM only tracks a + single total in `StandardLoggingPayload.response_cost`, so we cannot + split it into prompt/completion. We also keep the legacy + `llm.response.cost` key for back-compat with any consumer querying it. + """ + if not isinstance(standard_logging_payload, dict): + return + cost = standard_logging_payload.get("response_cost") + if cost is None: + return + try: + cost_value = float(cost) + except (TypeError, ValueError): + return + safe_set_attribute(span, "llm.cost.total", cost_value) + safe_set_attribute(span, "llm.response.cost", cost_value) + + +def _is_passthrough_call_type(call_type: Optional[str]) -> bool: + if not call_type: + return False + lowered = str(call_type).lower() + return "passthrough" in lowered or "pass_through" in lowered + + +def _maybe_normalize_passthrough( + span: "Span", + kwargs: dict, + raw_response_obj, + coerced_response_obj, + standard_logging_payload, +) -> None: + """Surface input/output text for passthrough routes (e.g. Bedrock + InvokeModel) so the parent span renders as more than `usage` numbers. + + Only runs when `call_type` is a passthrough variant. Reads from: + - `kwargs["additional_args"]["complete_input_dict"]` for input + - the coerced response (or `kwargs["original_response"]`) for output + + All emits are best-effort: if the provider shape isn't recognized the + helper exits silently. Existing chat/completion paths never enter this + helper because their call_type doesn't contain "passthrough". + + TEMPORARY BRIDGE: passthrough handlers don't populate the + StandardLoggingPayload `messages` field today (they call + `transform_response(messages=[])`), so the input is only available via + `additional_args.complete_input_dict`. The proper fix is upstream in + `base_passthrough_logging_handler._create_response_logging_payload()`: + once that populates SLP `messages`/`response`, every callback gets + passthrough I/O (with central redaction) for free and this helper's + `complete_input_dict` fallback can be deleted. See follow-up issue. + """ + call_type = ( + standard_logging_payload.get("call_type") + if isinstance(standard_logging_payload, dict) + else None + ) + if not _is_passthrough_call_type(call_type): + return + + # Respect LiteLLM's central message-redaction contract. The normal + # chat/completion path is redacted by `perform_redaction` before + # callbacks run, but `complete_input_dict` (read below) is NOT covered by + # that layer — so without this gate, an operator who enabled redaction + # would still see raw passthrough prompts in Arize. Skip entirely when + # redaction is on so neither input nor output leaks through this bridge. + if should_redact_message_logging(kwargs): + return + + # --- INPUT -------------------------------------------------------------- + additional_args = kwargs.get("additional_args") or {} + complete_input_dict = ( + additional_args.get("complete_input_dict") + if isinstance(additional_args, dict) + else None + ) + if isinstance(complete_input_dict, dict): + _set_passthrough_input_attributes(span, complete_input_dict.get("messages")) + + # --- OUTPUT ------------------------------------------------------------- + parsed_response = _parse_passthrough_response( + raw_response_obj, coerced_response_obj, kwargs + ) + if not isinstance(parsed_response, dict): + return + + _set_passthrough_output_attributes(span, parsed_response) + + +def _set_passthrough_input_attributes(span: "Span", messages) -> None: + """Render passthrough request messages into INPUT_VALUE + LLM_INPUT_MESSAGES.""" + if not (isinstance(messages, list) and messages): + return + # Set INPUT_VALUE from the last user message text if discoverable. + last_text = None + for msg in reversed(messages): + if isinstance(msg, dict): + last_text = _coerce_text(msg.get("content")) + if last_text: + break + if last_text: + safe_set_attribute(span, SpanAttributes.INPUT_VALUE, last_text) + # Mirror messages into LLM_INPUT_MESSAGES so the input pane renders. + for idx, msg in enumerate(messages): + if not isinstance(msg, dict): + continue + prefix = f"{SpanAttributes.LLM_INPUT_MESSAGES}.{idx}" + role = msg.get("role") + if role: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + role, + ) + text = _coerce_text(msg.get("content")) + if text is not None: + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + text, + ) + + +def _set_passthrough_output_attributes(span: "Span", parsed_response: dict) -> None: + """Render passthrough response into OUTPUT_VALUE + LLM_OUTPUT_MESSAGES.""" + # Anthropic / Bedrock-Anthropic: `content` is a list of typed parts. + content_list = parsed_response.get("content") + if isinstance(content_list, list) and content_list: + texts = [] + for part in content_list: + if isinstance(part, dict) and isinstance(part.get("text"), str): + texts.append(part["text"]) + joined = "\n\n".join(t for t in texts if t) + if joined: + safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, joined) + prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0" + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + parsed_response.get("role", "assistant"), + ) + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + joined, + ) + + # OpenAI-style passthrough: `choices[0].message.content` + choices = parsed_response.get("choices") + if isinstance(choices, list) and choices: + first = choices[0] + if isinstance(first, dict): + msg = first.get("message") + if isinstance(msg, dict): + text = _coerce_text(msg.get("content")) + if text: + safe_set_attribute(span, SpanAttributes.OUTPUT_VALUE, text) + prefix = f"{SpanAttributes.LLM_OUTPUT_MESSAGES}.0" + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_ROLE}", + msg.get("role", "assistant"), + ) + safe_set_attribute( + span, + f"{prefix}.{MessageAttributes.MESSAGE_CONTENT}", + text, + ) + + +def _parse_passthrough_response(raw_response_obj, coerced_response_obj, kwargs): + """Return a dict view of the provider response for passthrough routes.""" + # Prefer the coerced view (already JSON-parsed for httpx.Response). + candidates = [] + if isinstance(coerced_response_obj, dict): + candidates.append(coerced_response_obj) + if ( + isinstance(raw_response_obj, dict) + and raw_response_obj is not coerced_response_obj + ): + candidates.append(raw_response_obj) + + for candidate in candidates: + # StandardPassThroughResponseObject wrapper: {"response": "..."}. + if ( + "response" in candidate + and "content" not in candidate + and "choices" not in candidate + ): + inner = candidate.get("response") + if isinstance(inner, str): + try: + parsed = json.loads(inner) + if isinstance(parsed, dict): + return parsed + except Exception: + continue + if isinstance(inner, dict): + return inner + else: + return candidate + + # Fallback: kwargs["original_response"] from the OTel base path. + original = kwargs.get("original_response") if isinstance(kwargs, dict) else None + if isinstance(original, dict): + return original + if isinstance(original, str): + try: + parsed = json.loads(original) + if isinstance(parsed, dict): + return parsed + except Exception: + return None + return None diff --git a/litellm/integrations/callback_configs.json b/litellm/integrations/callback_configs.json index c2b0c4ddce..3a69c9a793 100644 --- a/litellm/integrations/callback_configs.json +++ b/litellm/integrations/callback_configs.json @@ -104,6 +104,51 @@ }, "description": "Datadog Custom Metrics Integration" }, + { + "id": "galileo", + "displayName": "Galileo", + "logo": "galileo.ico", + "supports_key_team_logging": false, + "dynamic_params": { + "GALILEO_API_KEY": { + "type": "password", + "ui_name": "API Key", + "description": "Galileo Cloud API key (app.galileo.ai). Omit for enterprise username/password auth.", + "required": false + }, + "GALILEO_PROJECT_ID": { + "type": "text", + "ui_name": "Project ID", + "description": "Galileo project ID to log traces to", + "required": true + }, + "GALILEO_LOG_STREAM_ID": { + "type": "text", + "ui_name": "Log Stream ID", + "description": "Galileo log stream ID for v2 spans logging (optional)", + "required": false + }, + "GALILEO_BASE_URL": { + "type": "text", + "ui_name": "Base URL", + "description": "Galileo API base URL (e.g. https://api.galileo.ai for Cloud, or your enterprise API URL)", + "required": false + }, + "GALILEO_USERNAME": { + "type": "text", + "ui_name": "Username", + "description": "Galileo enterprise username (legacy Observe auth; use instead of API key)", + "required": false + }, + "GALILEO_PASSWORD": { + "type": "password", + "ui_name": "Password", + "description": "Galileo enterprise password (legacy Observe auth)", + "required": false + } + }, + "description": "Galileo AI Observability Integration" + }, { "id": "datadog_cost_management", "displayName": "Datadog Cost Management", diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 82a35f2eed..fc5f0429b6 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -47,9 +47,29 @@ from litellm.exceptions import ( BlockedPiiEntityError, GuardrailRaisedException, ModifyResponseException, + SensitiveDataRouteException, ) +def get_session_id_from_request_data(request_data: Dict[str, Any]) -> Optional[str]: + """Extract session_id from request data (litellm_session_id or metadata).""" + session_id = request_data.get("litellm_session_id") + if session_id: + return str(session_id) + + metadata = request_data.get("metadata") or {} + session_id = metadata.get("session_id") + if session_id: + return str(session_id) + + litellm_metadata = request_data.get("litellm_metadata") or {} + session_id = litellm_metadata.get("session_id") + if session_id: + return str(session_id) + + return None + + class CustomGuardrail(CustomLogger): # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. use_native_during_call_hook: ClassVar[bool] = False @@ -68,6 +88,9 @@ class CustomGuardrail(CustomLogger): end_session_after_n_fails: Optional[int] = None, on_violation: Optional[str] = None, realtime_violation_message: Optional[str] = None, + on_sensitive_data: Optional[str] = None, + sensitive_data_route_to_model: Optional[str] = None, + sticky_session_routing: bool = True, **kwargs, ): """ @@ -83,6 +106,9 @@ class CustomGuardrail(CustomLogger): end_session_after_n_fails: For /v1/realtime sessions, end the session after this many violations on_violation: For /v1/realtime sessions, 'warn' or 'end_session' realtime_violation_message: Message the bot speaks aloud when a /v1/realtime guardrail fires + on_sensitive_data: Action when sensitive data is detected. 'block' (default) or 'route' + sensitive_data_route_to_model: Model to route to when on_sensitive_data='route' + sticky_session_routing: When True, all subsequent requests in the session use the same model """ self.guardrail_name = guardrail_name self.supported_event_hooks = supported_event_hooks @@ -96,6 +122,11 @@ class CustomGuardrail(CustomLogger): self.end_session_after_n_fails: Optional[int] = end_session_after_n_fails self.on_violation: Optional[str] = on_violation self.realtime_violation_message: Optional[str] = realtime_violation_message + self.on_sensitive_data: Optional[str] = on_sensitive_data + self.sensitive_data_route_to_model: Optional[str] = ( + sensitive_data_route_to_model + ) + self.sticky_session_routing: bool = sticky_session_routing if supported_event_hooks: ## validate event_hook is in supported_event_hooks @@ -167,6 +198,108 @@ class CustomGuardrail(CustomLogger): detection_info=detection_info, ) + def raise_sensitive_data_route_exception( + self, + route_to_model: str, + request_data: Dict[str, Any], + detection_info: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Raise an exception to reroute the request to a different model. + + Use this when sensitive data is detected and the guardrail is configured + to route to an on-premise model instead of blocking. + + The exception will reroute this request to the specified model. When + sticky_session_routing is enabled (the default), it also stores the + routing decision so subsequent requests in this session reuse the model. + + Args: + route_to_model: The model to route this request (and session) to + request_data: The original request data dictionary + detection_info: Optional non-sensitive detection metadata (e.g. matched + entity types, rule ids, scores). This is surfaced in request metadata + and logs, so it must not contain the raw detected sensitive values. + + Raises: + SensitiveDataRouteException: Always raises to trigger rerouting + """ + session_id = self._get_session_id_from_request_data(request_data) + if not session_id: + raise ValueError( + "Cannot route sensitive data without a session_id. " + "Ensure the request includes a session_id in metadata or headers." + ) + + raise SensitiveDataRouteException( + route_to_model=route_to_model, + session_id=session_id, + guardrail_name=self.guardrail_name, + detection_info=detection_info, + sticky_session_routing=self.sticky_session_routing, + ) + + def _get_session_id_from_request_data( + self, request_data: Dict[str, Any] + ) -> Optional[str]: + """Extract session_id from request data.""" + return get_session_id_from_request_data(request_data) + + def should_route_on_sensitive_data(self) -> bool: + """ + Returns True if this guardrail is configured to route requests + to a different model when sensitive data is detected. + """ + return ( + self.on_sensitive_data == "route" + and self.sensitive_data_route_to_model is not None + ) + + def handle_sensitive_data_detection( + self, + request_data: Dict[str, Any], + detection_info: Optional[Dict[str, Any]] = None, + ) -> None: + """ + Handle sensitive data detection based on guardrail configuration. + + If on_sensitive_data='route', raises SensitiveDataRouteException to reroute. + Otherwise, raises GuardrailRaisedException to block. When routing is + configured but the request carries no session_id, routing is not possible + so the request falls back to a graceful block. + + Args: + request_data: The request data dictionary + detection_info: Optional non-sensitive detection metadata. When routing, + this is surfaced in request metadata and logs, so it must not contain + the raw detected sensitive values. + + Raises: + SensitiveDataRouteException: When configured to route and a session_id is present + GuardrailRaisedException: When configured to block, or when routing is + configured but no session_id is available + """ + if self.should_route_on_sensitive_data(): + try: + self.raise_sensitive_data_route_exception( + route_to_model=self.sensitive_data_route_to_model, # type: ignore + request_data=request_data, + detection_info=detection_info, + ) + except ValueError: + raise GuardrailRaisedException( + message=( + f"Sensitive data detected by {self.guardrail_name} " + "(routing skipped: request has no session_id)" + ), + guardrail_name=self.guardrail_name, + ) + else: + raise GuardrailRaisedException( + message=f"Sensitive data detected by {self.guardrail_name}", + guardrail_name=self.guardrail_name, + ) + @staticmethod def get_config_model() -> Optional[Type["GuardrailConfigModel"]]: """ @@ -662,6 +795,16 @@ class CustomGuardrail(CustomLogger): request_data["metadata"] = {} _append_guardrail_info(request_data["metadata"]) + # Emit the otel guardrail span here, where every guardrail execution lands, + # rather than relying on a post-call hook that does not fire on every path + # (e.g. a pass-through request that passes its guardrails). + try: + from litellm.integrations.otel.logger import emit_guardrail_span + + emit_guardrail_span(slg) + except Exception: + pass + async def apply_guardrail( self, inputs: GenericGuardrailAPIInputs, @@ -743,12 +886,20 @@ class CustomGuardrail(CustomLogger): Guardrails signal intentional blocks by raising: - GuardrailRaisedException (generic guardrail API, tool permission) - BlockedPiiEntityError (Presidio PII detection) + - SensitiveDataRouteException (sensitive-data reroute to on-premise model) - HTTPException with status 400 (content policy violation) - ModifyResponseException (passthrough mode violation) """ if isinstance(e, ModifyResponseException): return True - if isinstance(e, (GuardrailRaisedException, BlockedPiiEntityError)): + if isinstance( + e, + ( + GuardrailRaisedException, + BlockedPiiEntityError, + SensitiveDataRouteException, + ), + ): return True if ( HTTPException is not None diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index c3e555f6e8..79a9219a39 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -41,6 +41,7 @@ from litellm.integrations.datadog.datadog_handler import ( ) from litellm.litellm_core_utils.dd_tracing import tracer from litellm.llms.custom_httpx.http_handler import ( + MaskedHTTPStatusError, _get_httpx_client, get_async_httpx_client, httpxSpecialProvider, @@ -68,6 +69,22 @@ DD_LOGGED_SUCCESS_SERVICE_TYPES = [ ] +def _resolve_dd_batch_size() -> int: + raw = os.getenv("DD_BATCH_SIZE") + if raw is None: + return DD_MAX_BATCH_SIZE + try: + value = int(raw) + except ValueError: + verbose_logger.warning( + "Datadog: ignoring invalid DD_BATCH_SIZE=%r, using %s", + raw, + DD_MAX_BATCH_SIZE, + ) + return DD_MAX_BATCH_SIZE + return max(1, min(value, DD_MAX_BATCH_SIZE)) + + class DataDogLogger( CustomBatchLogger, AdditionalLoggingUtils, @@ -128,7 +145,9 @@ class DataDogLogger( asyncio.create_task(self.periodic_flush()) self.flush_lock = asyncio.Lock() super().__init__( - **kwargs, flush_lock=self.flush_lock, batch_size=DD_MAX_BATCH_SIZE + **kwargs, + flush_lock=self.flush_lock, + batch_size=_resolve_dd_batch_size(), ) except Exception as e: verbose_logger.exception( @@ -339,28 +358,14 @@ class DataDogLogger( "[DATADOG MOCK] Mock mode enabled - API calls will be intercepted" ) - response = await self.async_send_compressed_data(batch_to_send) - if response.status_code == 413: - verbose_logger.exception(DD_ERRORS.DATADOG_413_ERROR.value) - self.log_queue = batch_to_send + self.log_queue - return - - response.raise_for_status() - if response.status_code != 202: - raise Exception( - f"Response from datadog API status_code: {response.status_code}, text: {response.text}" - ) + undelivered = await self._send_with_413_split(batch_to_send) + if undelivered: + self.log_queue = undelivered + self.log_queue if self.is_mock_mode: verbose_logger.debug( f"[DATADOG MOCK] Batch of {len(batch_to_send)} events successfully mocked" ) - else: - verbose_logger.debug( - "Datadog: Response from datadog API status_code: %s, text: %s", - response.status_code, - response.text, - ) except Exception as e: self.log_queue = batch_to_send + self.log_queue @@ -368,6 +373,62 @@ class DataDogLogger( f"Datadog Error sending batch API - {str(e)}\n{traceback.format_exc()}" ) + async def _send_with_413_split(self, batch: List) -> List: + """ + Send a batch, halving any sub-batch that 413s (payload too large) and retrying the + halves, since Datadog enforces a 5MB uncompressed limit per request. + + A 413 surfaces as a raised MaskedHTTPStatusError (httpx raise_for_status), not a + returned response, so both paths are handled. A lone event that still 413s is + dropped to avoid wedging the queue on an undeliverable payload. Returns the events + that could not be delivered because of a non-413 (transient) error, so the caller + re-queues only those and never the events already accepted by Datadog. + """ + pending: List[List] = [batch] + while pending: + chunk = pending.pop() + if not chunk: + continue + try: + response = await self.async_send_compressed_data(chunk) + except Exception as e: + if isinstance(e, MaskedHTTPStatusError) and e.status_code == 413: + response = e.response + else: + verbose_logger.exception( + f"Datadog Error sending batch API - {str(e)}" + ) + return self._undelivered(chunk, pending) + + if response.status_code == 413: + if len(chunk) == 1: + verbose_logger.error(DD_ERRORS.DATADOG_413_ERROR.value) + continue + mid = len(chunk) // 2 + pending.append(chunk[mid:]) + pending.append(chunk[:mid]) + continue + + if response.status_code != 202: + verbose_logger.error( + "Datadog: unexpected response status_code=%s, text=%s", + response.status_code, + response.text, + ) + return self._undelivered(chunk, pending) + + verbose_logger.debug( + "Datadog: delivered %s events, status_code=%s, text=%s", + len(chunk), + response.status_code, + response.text, + ) + return [] + + @staticmethod + def _undelivered(chunk: List, pending: List[List]) -> List: + return chunk + [event for remaining in reversed(pending) for event in remaining] + async def flush_queue(self): if self.flush_lock is None: return diff --git a/litellm/integrations/focus/transformer.py b/litellm/integrations/focus/transformer.py index 6f4433b4a0..8496b7ec15 100644 --- a/litellm/integrations/focus/transformer.py +++ b/litellm/integrations/focus/transformer.py @@ -95,7 +95,9 @@ class FocusTransformer: pl.lit("Usage-Based").alias("ChargeFrequency"), fmt(pl.col("ChargePeriodEnd")).alias("ChargePeriodEnd"), fmt(pl.col("ChargePeriodStart")).alias("ChargePeriodStart"), - dec(pl.lit(1.0)).alias("ConsumedQuantity"), + dec( + pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0) + ).alias("ConsumedQuantity"), pl.lit("Requests").alias("ConsumedUnit"), dec(pl.col("spend").fill_null(0.0)).alias("ContractedCost"), none_str.alias("ContractedUnitPrice"), @@ -107,7 +109,9 @@ class FocusTransformer: none_str.alias("AvailabilityZone"), pl.lit("USD").alias("PricingCurrency"), none_str.alias("PricingCategory"), - dec(pl.lit(1.0)).alias("PricingQuantity"), + dec( + pl.col("api_requests").cast(pl.Int64).cast(pl.Float64).fill_null(0.0) + ).alias("PricingQuantity"), none_dec.alias("PricingCurrencyContractedUnitPrice"), dec(pl.col("spend").fill_null(0.0)).alias("PricingCurrencyEffectiveCost"), none_dec.alias("PricingCurrencyListUnitPrice"), diff --git a/litellm/integrations/galileo.py b/litellm/integrations/galileo.py index a598124f61..8fef90c24e 100644 --- a/litellm/integrations/galileo.py +++ b/litellm/integrations/galileo.py @@ -1,8 +1,13 @@ +from __future__ import annotations + import json import os import re -from typing import Any, Dict, List, Optional, Tuple, cast +import uuid +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional, Tuple, Union, cast +import httpx from pydantic import BaseModel, Field import litellm @@ -12,11 +17,15 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( convert_content_list_to_str, get_content_from_model_response, ) +from litellm.types.llms.openai import ( + AllMessageValues, + HttpxBinaryResponseContent, + ResponsesAPIResponse, +) from litellm.llms.custom_httpx.http_handler import ( get_async_httpx_client, httpxSpecialProvider, ) -from litellm.types.llms.openai import AllMessageValues GALILEO_CLOUD_API_BASE_URL = "https://api.galileo.ai" # Cap the in-memory buffer so persistent flush failures (e.g. Galileo @@ -33,6 +42,11 @@ class LLMResponse(BaseModel): model: str num_input_tokens: int num_output_tokens: int + num_total_tokens: int + cost: Optional[float] = Field( + default=None, + description="Total cost of the LLM call in USD as computed by LiteLLM.", + ) output_logprobs: Optional[Dict[str, Any]] = Field( default=None, description="Optional. When available, logprobs are used to compute Uncertainty.", @@ -121,10 +135,14 @@ class GalileoObserve(CustomLogger): @staticmethod def _galileo_input_messages( - messages: Optional[List[Any]], input_text: str + messages: Optional[Any], input_text: str ) -> List[Dict[str, str]]: + if isinstance(messages, dict): + messages = messages.get("messages") if not messages: return [{"role": "user", "content": input_text}] + if not isinstance(messages, list): + return [{"role": "user", "content": input_text}] galileo_messages: List[Dict[str, str]] = [] for message in messages: @@ -147,13 +165,59 @@ class GalileoObserve(CustomLogger): return [{"role": "user", "content": input_text}] @staticmethod - def _record_to_v2_span(record: Dict[str, Any]) -> Dict[str, Any]: - created_at = record.get("created_at", "") + def _local_timezone(): + return datetime.now().astimezone().tzinfo or timezone.utc + + @staticmethod + def _format_created_at(dt: Union[datetime, Any]) -> str: + """Serialize timestamps as UTC ISO-8601 for Galileo.""" + if not isinstance(dt, datetime): + return str(dt) + + if dt.tzinfo is None: + # LiteLLM often passes naive datetimes in local time; convert to UTC + # instead of appending Z to local time (which shifts Traces tab sorting). + dt = dt.replace(tzinfo=GalileoObserve._local_timezone()) + + return dt.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + @staticmethod + def _normalize_created_at(created_at: str) -> str: if created_at and not re.search(r"(Z|[+-]\d{2}:?\d{2})$", created_at): - created_at = f"{created_at}Z" + return f"{created_at}Z" + return created_at + + @staticmethod + def _token_metrics_from_record(record: Dict[str, Any]) -> Dict[str, Any]: + num_input_tokens = int(record.get("num_input_tokens") or 0) + num_output_tokens = int(record.get("num_output_tokens") or 0) + num_total_tokens = int(record.get("num_total_tokens") or 0) + if num_total_tokens == 0 and (num_input_tokens or num_output_tokens): + num_total_tokens = num_input_tokens + num_output_tokens + metrics: Dict[str, Any] = { + "num_input_tokens": num_input_tokens, + "num_output_tokens": num_output_tokens, + "num_total_tokens": num_total_tokens, + } + cost = record.get("cost") + if cost is not None: + metrics["cost"] = float(cost) + return metrics + + @staticmethod + def _record_to_v2_span( + record: Dict[str, Any], + *, + trace_id: str, + span_id: str, + ) -> Dict[str, Any]: + created_at = GalileoObserve._normalize_created_at(record.get("created_at", "")) span: Dict[str, Any] = { "type": "llm", + "id": span_id, + "trace_id": trace_id, + "parent_id": trace_id, "name": record.get("node_type", "litellm"), "created_at": created_at, "input": GalileoObserve._galileo_input_messages( @@ -167,14 +231,49 @@ class GalileoObserve(CustomLogger): "model": record.get("model"), "metrics": { "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, - "num_input_tokens": record.get("num_input_tokens"), - "num_output_tokens": record.get("num_output_tokens"), + **GalileoObserve._token_metrics_from_record(record), }, } if record.get("tags"): span["tags"] = record["tags"] return span + @staticmethod + def _record_to_v2_trace(record: Dict[str, Any]) -> Dict[str, Any]: + trace_id = str(uuid.uuid4()) + span_id = str(uuid.uuid4()) + created_at = GalileoObserve._normalize_created_at(record.get("created_at", "")) + + return { + "type": "trace", + "id": trace_id, + "name": record.get("node_type", "litellm"), + "created_at": created_at, + "input": record.get("input_text", ""), + "output": record.get("output_text", ""), + "status_code": record.get("status_code", 200), + "metrics": { + "duration_ns": int(record.get("latency_ms", 0)) * 1_000_000, + **GalileoObserve._token_metrics_from_record(record), + }, + "spans": [ + GalileoObserve._record_to_v2_span( + record, trace_id=trace_id, span_id=span_id + ) + ], + } + + def _build_traces_payload(self, records: List[dict]) -> Dict[str, Any]: + payload: Dict[str, Any] = { + "traces": [self._record_to_v2_trace(record) for record in records], + "logging_method": "api_direct", + "reliable": False, + "is_complete": True, + } + if self.log_stream_id: + payload["log_stream_id"] = self.log_stream_id + return payload + def _get_ingest_request(self) -> Optional[Tuple[str, Dict[str, Any]]]: if not self.base_url or not self.project_id: return None @@ -184,105 +283,457 @@ class GalileoObserve(CustomLogger): # flush_in_memory_records) aren't silently dropped when we later clear # the in-memory buffer. records = list(self.in_memory_records) + payload = self._build_traces_payload(records) if self.use_v2_api: - payload: Dict[str, Any] = { - "spans": [self._record_to_v2_span(record) for record in records], - "reliable": False, - } - if self.log_stream_id: - payload["log_stream_id"] = self.log_stream_id return ( - f"{self.base_url}/v2/projects/{self.project_id}/spans", + f"{self.base_url}/ingest/traces/{self.project_id}", payload, ) + # Username/password auth logs in for a JWT and uses the standard v2 traces API. return ( - f"{self.base_url}/projects/{self.project_id}/observe/ingest", - {"records": records}, + f"{self.base_url}/v2/projects/{self.project_id}/traces", + payload, ) + @staticmethod + def _redact_headers(headers: Optional[Dict[str, str]]) -> Dict[str, str]: + if not headers: + return {} + redacted: Dict[str, str] = {} + for key, value in headers.items(): + if key.lower() in {"authorization", "galileo-api-key"} and value: + redacted[key] = ( + f"{value[:8]}...{value[-4:]}" if len(value) > 12 else "***" + ) + else: + redacted[key] = value + return redacted + + def _log_flush_config(self) -> None: + verbose_logger.debug( + "Galileo Logger flush config: use_v2_api=%s base_url=%s project_id=%s " + "log_stream_id=%s api_key_set=%s username_set=%s record_count=%s", + self.use_v2_api, + self.base_url, + self.project_id, + self.log_stream_id, + bool(self.api_key), + bool(self.username), + len(self.in_memory_records), + ) + + @staticmethod + def _log_v2_payload_validation(payload: Dict[str, Any]) -> None: + missing_fields: List[str] = [] + traces = payload.get("traces", []) + if not traces: + missing_fields.append("traces") + + for trace_index, trace in enumerate(traces): + if not isinstance(trace, dict): + continue + for field in ("id", "type", "spans"): + if field not in trace: + missing_fields.append(f"traces[{trace_index}].{field}") + + trace_id = trace.get("id") + for span_index, span in enumerate(trace.get("spans", [])): + if not isinstance(span, dict): + continue + for field in ("id", "trace_id", "parent_id"): + if field not in span: + missing_fields.append( + f"traces[{trace_index}].spans[{span_index}].{field}" + ) + if trace_id and span.get("trace_id") != trace_id: + missing_fields.append( + f"traces[{trace_index}].spans[{span_index}].trace_id mismatch" + ) + + if missing_fields: + verbose_logger.debug( + "Galileo Logger: ingest /traces payload validation issues: %s", + missing_fields, + ) + + def _log_flush_payload(self, url: str, payload: Dict[str, Any]) -> None: + traces = payload.get("traces", []) + verbose_logger.debug( + "Galileo Logger flush URL: %s trace_count=%s", + url, + len(traces) if isinstance(traces, list) else 0, + ) + if self.use_v2_api and "/ingest/traces/" in url: + self._log_v2_payload_validation(payload) + + @staticmethod + def _log_http_status_error(error: httpx.HTTPStatusError, url: str) -> None: + response = error.response + verbose_logger.debug( + "Galileo Logger HTTP error: status=%s url=%s", + response.status_code, + url, + ) + verbose_logger.debug( + "Galileo Logger HTTP error response body: %s", + response.text, + ) + try: + verbose_logger.debug( + "Galileo Logger HTTP error response json: %s", + response.json(), + ) + except Exception: + pass + + @staticmethod + def _build_prompt(kwargs: Dict[str, Any]) -> Dict[str, Any]: + optional_params = kwargs.get("optional_params", {}) or {} + prompt: Dict[str, Any] = {"messages": kwargs.get("messages")} + if optional_params.get("functions") is not None: + prompt["functions"] = optional_params["functions"] + if optional_params.get("tools") is not None: + prompt["tools"] = optional_params["tools"] + return prompt + + @staticmethod + def _serialize_galileo_output(value: Any) -> Optional[str]: + if value is None: + return None + if isinstance(value, str): + return value + + def _json_default(obj: Any) -> Any: + if hasattr(obj, "model_dump"): + return obj.model_dump() + return str(obj) + + return json.dumps(value, default=_json_default) + + @staticmethod + def _prompt_to_input_text(prompt: Dict[str, Any]) -> str: + messages = prompt.get("messages") + if messages is not None: + text = GalileoObserve._input_text_from_messages(messages) + if text: + return text + return json.dumps(prompt, default=str) + + @staticmethod + def _get_chat_content_for_galileo(response_obj: litellm.ModelResponse) -> Any: + if response_obj.choices and len(response_obj.choices) > 0: + message = response_obj["choices"][0]["message"] + if hasattr(message, "json"): + message_json = message.json() + if isinstance(message_json, str): + return json.loads(message_json) + return message_json + return message + return None + + @staticmethod + def _get_text_completion_content_for_galileo( + response_obj: litellm.TextCompletionResponse, + ) -> Optional[str]: + if response_obj.choices and len(response_obj.choices) > 0: + return response_obj.choices[0].text + return None + + @staticmethod + def _get_responses_api_content_for_galileo( + response_obj: ResponsesAPIResponse, + ) -> Any: + if hasattr(response_obj, "output") and response_obj.output: + return response_obj.output + return None + + @staticmethod + def _langfuse_style_rerank_prompt(kwargs: Dict[str, Any]) -> Dict[str, Any]: + """Match Langfuse rerank input: prompt = {"messages": kwargs.get("messages")}.""" + return {"messages": kwargs.get("messages")} + + def _get_galileo_input_output_content( + self, + kwargs: Dict[str, Any], + response_obj: Any, + level: str = "DEFAULT", + status_message: Optional[str] = None, + ) -> Tuple[str, Optional[str], Any]: + """ + Mirror Langfuse _get_langfuse_input_output_content for Galileo ingest. + + Returns (input_text, output_text, messages_for_span). output_text None skips ingest. + """ + call_type = kwargs.get("call_type") + prompt = self._build_prompt(kwargs) + + if ( + level == "ERROR" + and status_message is not None + and isinstance(status_message, str) + ): + return self._prompt_to_input_text(prompt), status_message, prompt + + if response_obj is not None and ( + call_type == "embedding" + or isinstance(response_obj, litellm.EmbeddingResponse) + ): + return self._prompt_to_input_text(prompt), None, prompt + + if response_obj is not None and isinstance(response_obj, litellm.ModelResponse): + output = self._get_chat_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if response_obj is not None and isinstance( + response_obj, HttpxBinaryResponseContent + ): + return self._prompt_to_input_text(prompt), "speech-output", prompt + + if response_obj is not None and isinstance( + response_obj, litellm.TextCompletionResponse + ): + output = self._get_text_completion_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if response_obj is not None and isinstance(response_obj, litellm.ImageResponse): + output = response_obj.get("data", None) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance( + response_obj, litellm.TranscriptionResponse + ): + output = response_obj.get("text", None) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance( + response_obj, litellm.RerankResponse + ): + output = response_obj.results + rerank_prompt = self._langfuse_style_rerank_prompt(kwargs) + return ( + json.dumps(rerank_prompt, default=str), + self._serialize_galileo_output(output), + rerank_prompt, + ) + + if response_obj is not None and isinstance(response_obj, ResponsesAPIResponse): + output = self._get_responses_api_content_for_galileo(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + if ( + call_type == "_arealtime" + and response_obj is not None + and isinstance(response_obj, list) + ): + input_val = kwargs.get("input") + return ( + self._serialize_galileo_output(input_val) or "", + self._serialize_galileo_output(response_obj), + input_val, + ) + + if ( + call_type == "pass_through_endpoint" + and response_obj is not None + and isinstance(response_obj, dict) + ): + output = response_obj.get("response", "") + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + prompt, + ) + + if response_obj is not None and isinstance(response_obj, dict): + output = get_content_from_model_response(response_obj) + return ( + self._prompt_to_input_text(prompt), + self._serialize_galileo_output(output), + kwargs.get("messages") or [], + ) + + return self._prompt_to_input_text(prompt), None, kwargs.get("messages") or [] + def get_output_str_from_response( self, response_obj: Any, kwargs: Dict[str, Any] ) -> Optional[str]: - if response_obj is None: - return None - if kwargs.get("call_type", None) == "embedding" or isinstance( - response_obj, litellm.EmbeddingResponse - ): - return None - if isinstance(response_obj, litellm.TextCompletionResponse): - return response_obj.choices[0].text - if isinstance(response_obj, litellm.ImageResponse): - return json.dumps(response_obj["data"], default=str) - if isinstance(response_obj, (litellm.ModelResponse, dict)): - return get_content_from_model_response(response_obj) - return None + _, output_text, _ = self._get_galileo_input_output_content( + kwargs=kwargs, response_obj=response_obj + ) + return output_text + + @staticmethod + def _input_text_from_messages(messages: Any) -> str: + """Return a plain-string summary of the input suitable for the trace-level input field.""" + if isinstance(messages, str): + return messages + if not isinstance(messages, list): + return "" + # Use the last user/human message so the trace table shows the actual prompt + for msg in reversed(messages): + if not isinstance(msg, dict): + continue + if str(msg.get("role", "")).lower() in ("user", "human"): + content = msg.get("content") or "" + if isinstance(content, list): + content = " ".join( + b.get("text", "") if isinstance(b, dict) else str(b) + for b in content + ) + if content: + return str(content) + # Fallback: first non-empty content of any role + for msg in messages: + if isinstance(msg, dict): + content = msg.get("content") or "" + if isinstance(content, list): + content = " ".join( + b.get("text", "") if isinstance(b, dict) else str(b) + for b in content + ) + if content: + return str(content) + return "" async def async_log_success_event( self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any ): verbose_logger.debug("On Async Success") + try: + await self._async_log_success_event_impl( + kwargs=kwargs, + response_obj=response_obj, + start_time=start_time, + end_time=end_time, + ) + except Exception: + verbose_logger.exception( + "Galileo Logger: unexpected error in async_log_success_event" + ) + async def _async_log_success_event_impl( + self, kwargs: Any, response_obj: Any, start_time: Any, end_time: Any + ): if not self._is_configured(): verbose_logger.debug( - "Galileo Logger: skipping flush — set GALILEO_PROJECT_ID and " - "either GALILEO_API_KEY (hosted) or GALILEO_USERNAME/GALILEO_PASSWORD " - "(enterprise Observe)." + "Galileo Logger: skipping — GALILEO_PROJECT_ID=%s GALILEO_API_KEY=%s GALILEO_BASE_URL=%s", + bool(self.project_id), + bool(self.api_key), + bool(self.base_url), ) return - _latency_ms = int((end_time - start_time).total_seconds() * 1000) - _call_type = kwargs.get("call_type", "litellm") - input_text = litellm.utils.get_formatted_prompt( - data=kwargs, call_type=_call_type + slo: Optional[Dict[str, Any]] = kwargs.get("standard_logging_object") + if slo is None: + verbose_logger.debug( + "Galileo Logger: no standard_logging_object in kwargs, skipping" + ) + return + + _call_type: str = str( + slo.get("call_type") or kwargs.get("call_type") or "litellm" ) - _usage = response_obj.get("usage", {}) or {} - num_input_tokens = _usage.get("prompt_tokens", 0) - num_output_tokens = _usage.get("completion_tokens", 0) + input_text, output_text, messages = self._get_galileo_input_output_content( + kwargs=kwargs, response_obj=response_obj + ) + if output_text is None: + verbose_logger.debug( + "Galileo Logger: skipping %s — no text output to log", _call_type + ) + return - output_text = self.get_output_str_from_response( - response_obj=response_obj, kwargs=kwargs + raw_start = slo.get("startTime") + raw_end = slo.get("endTime") + if raw_start is None or raw_end is None: + verbose_logger.debug( + "Galileo Logger: standard_logging_object missing startTime/endTime, " + "falling back to start_time/end_time params" + ) + if not isinstance(start_time, datetime) or not isinstance( + end_time, datetime + ): + return + start_ts = start_time + end_ts = end_time + if start_ts.tzinfo is None: + start_ts = start_ts.replace(tzinfo=GalileoObserve._local_timezone()) + if end_ts.tzinfo is None: + end_ts = end_ts.replace(tzinfo=GalileoObserve._local_timezone()) + start_ts = start_ts.astimezone(timezone.utc) + end_ts = end_ts.astimezone(timezone.utc) + else: + start_ts = datetime.fromtimestamp(float(raw_start), tz=timezone.utc) + end_ts = datetime.fromtimestamp(float(raw_end), tz=timezone.utc) + _latency_ms = max(0, int((end_ts - start_ts).total_seconds() * 1000)) + num_input_tokens = int(slo.get("prompt_tokens") or 0) + num_output_tokens = int(slo.get("completion_tokens") or 0) + num_total_tokens = int(slo.get("total_tokens") or 0) + if num_total_tokens == 0 and (num_input_tokens or num_output_tokens): + num_total_tokens = num_input_tokens + num_output_tokens + + request_record = LLMResponse( + latency_ms=_latency_ms, + status_code=200, + input_text=input_text, + output_text=output_text, + node_type=_call_type, + model=str(slo.get("model") or kwargs.get("model") or "-"), + num_input_tokens=num_input_tokens, + num_output_tokens=num_output_tokens, + num_total_tokens=num_total_tokens, + cost=slo.get("response_cost"), + created_at=GalileoObserve._format_created_at(start_ts), ) - if output_text is not None: - request_record = LLMResponse( - latency_ms=_latency_ms, - status_code=200, - input_text=input_text, - output_text=output_text, - node_type=_call_type, - model=kwargs.get("model", "-"), - num_input_tokens=num_input_tokens, - num_output_tokens=num_output_tokens, - created_at=start_time.strftime( - "%Y-%m-%dT%H:%M:%S" - ), # timestamp str constructed in "%Y-%m-%dT%H:%M:%S" format + request_dict = request_record.model_dump() + if isinstance(messages, dict): + messages = messages.get("messages") + if isinstance(messages, list) and messages: + request_dict["messages"] = messages + self.in_memory_records.append(request_dict) + verbose_logger.debug( + "Galileo Logger: queued record, in_memory=%d", len(self.in_memory_records) + ) + + # Bound the buffer so persistent flush failures cannot grow it + # without limit. Drop the oldest records once we exceed the cap. + if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: + dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS + self.in_memory_records = self.in_memory_records[ + -GALILEO_MAX_IN_MEMORY_RECORDS: + ] + verbose_logger.warning( + "Galileo Logger: in-memory buffer exceeded %s records; " + "dropped %s oldest record(s). Check Galileo connectivity/credentials.", + GALILEO_MAX_IN_MEMORY_RECORDS, + dropped, ) - request_dict = request_record.model_dump() - messages = kwargs.get("messages") - if messages: - request_dict["messages"] = messages - self.in_memory_records.append(request_dict) - - # Bound the buffer so persistent flush failures cannot grow it - # without limit. Drop the oldest records once we exceed the cap. - if len(self.in_memory_records) > GALILEO_MAX_IN_MEMORY_RECORDS: - dropped = len(self.in_memory_records) - GALILEO_MAX_IN_MEMORY_RECORDS - self.in_memory_records = self.in_memory_records[ - -GALILEO_MAX_IN_MEMORY_RECORDS: - ] - verbose_logger.warning( - "Galileo Logger: in-memory buffer exceeded %s records; " - "dropped %s oldest record(s). Check Galileo connectivity/credentials.", - GALILEO_MAX_IN_MEMORY_RECORDS, - dropped, - ) - - if len(self.in_memory_records) >= self.batch_size: - await self.flush_in_memory_records() + if len(self.in_memory_records) >= self.batch_size: + await self.flush_in_memory_records() async def flush_in_memory_records(self): if not self.in_memory_records: @@ -296,15 +747,23 @@ class GalileoObserve(CustomLogger): ingest_request = self._get_ingest_request() if ingest_request is None: verbose_logger.debug( - "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID" + "Galileo Logger: missing GALILEO_BASE_URL or GALILEO_PROJECT_ID — skipping flush" ) return if not await self._ensure_headers(): - verbose_logger.debug("Galileo Logger: could not set request headers") + verbose_logger.debug( + "Galileo Logger: could not set request headers — skipping flush" + ) return url, payload = ingest_request + self._log_flush_config() + self._log_flush_payload(url=url, payload=payload) + verbose_logger.debug( + "Galileo Logger flush headers: %s", + self._redact_headers(self.headers), + ) verbose_logger.debug("flushing in memory records to %s", url) try: @@ -313,6 +772,12 @@ class GalileoObserve(CustomLogger): headers=self.headers, json=payload, ) + except httpx.HTTPStatusError as e: + self._log_http_status_error(error=e, url=url) + verbose_logger.debug( + "Galileo Logger: failed to flush in memory records: %s", e + ) + return except Exception as e: verbose_logger.debug( "Galileo Logger: failed to flush in memory records: %s", e @@ -323,6 +788,11 @@ class GalileoObserve(CustomLogger): verbose_logger.debug( "Galileo Logger: successfully flushed in memory records" ) + verbose_logger.debug( + "Galileo Logger flush response: status=%s body=%s", + response.status_code, + response.text, + ) del self.in_memory_records[:records_in_payload] else: verbose_logger.debug("Galileo Logger: failed to flush in memory records") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index b7a565512c..cae5929563 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -102,6 +102,18 @@ def langfuse_client_init( if Version(langfuse.version.__version__) >= Version("2.6.0"): parameters["sdk_integration"] = "litellm" + if Version(langfuse.version.__version__) >= Version("2.7.3"): + import httpx + + import litellm + + from ...llms.custom_httpx.http_handler import get_ssl_configuration + + parameters["httpx_client"] = httpx.Client( + verify=get_ssl_configuration(), + cert=os.getenv("SSL_CERTIFICATE", litellm.ssl_certificate), + ) + client = Langfuse(**parameters) return client diff --git a/litellm/integrations/openmeter.py b/litellm/integrations/openmeter.py index 5a8ab4bcc9..b234ab11dd 100644 --- a/litellm/integrations/openmeter.py +++ b/litellm/integrations/openmeter.py @@ -65,7 +65,15 @@ class OpenMeterLogger(CustomLogger): "total_tokens": response_obj["usage"].get("total_tokens"), } - user_param = kwargs.get("user", None) # end-user passed in via 'user' param + # OPENMETER_TRUST_REQUEST_USER (default "true"): when set to "false", + # the request-supplied `user` field is ignored and the subject is + # resolved solely from the key-bound user_api_key_user_id. Proxies + # serving multi-tenant traffic enable this to prevent clients from + # forging attribution by setting `user` in the request body. + trust_request_user = ( + os.getenv("OPENMETER_TRUST_REQUEST_USER", "true").lower() != "false" + ) + user_param = kwargs.get("user", None) if trust_request_user else None # If no user provided directly, try to get it from token user_id if user_param is None: diff --git a/litellm/integrations/opentelemetry.py b/litellm/integrations/opentelemetry.py index 814da344f0..24780eb4bf 100644 --- a/litellm/integrations/opentelemetry.py +++ b/litellm/integrations/opentelemetry.py @@ -83,6 +83,19 @@ _VALID_CAPTURE_MODES = { } +def _normalize_team_metadata_keys(value: Any) -> List[str]: + """Coerce a team-metadata allowlist from a list or comma-separated string. + + config.yaml passes a YAML list; an env var passes a comma-separated string. + Both collapse to a list of stripped, non-empty keys. + """ + if value is None: + return [] + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return [str(item).strip() for item in value if str(item).strip()] + + @dataclass class OpenTelemetryConfig: exporter: Union[str, SpanExporter] = "console" @@ -100,6 +113,10 @@ class OpenTelemetryConfig: # One of NO_CONTENT, SPAN_ONLY, EVENT_ONLY, SPAN_AND_EVENT (or "true" as legacy alias). capture_message_content: Optional[str] = None semconv_stability_opt_in: Set[OTELSemconvCategory] = field(default_factory=set) + # Sub-keys of the team's free-form metadata stamped onto the inference span + # under ``litellm.team.metadata``. Empty by default so none of a team's + # metadata leaves the process until explicitly allowlisted. + baggage_team_metadata_keys: List[str] = field(default_factory=list) def __post_init__(self) -> None: # If endpoint is specified but exporter is still the default "console", @@ -130,6 +147,11 @@ class OpenTelemetryConfig: self.semconv_stability_opt_in |= parse_semconv_opt_in( os.getenv(OTEL_SEMCONV_STABILITY_OPT_IN_ENV) ) + self.baggage_team_metadata_keys = _normalize_team_metadata_keys( + self.baggage_team_metadata_keys + ) or _normalize_team_metadata_keys( + os.getenv("LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS") + ) @classmethod def from_env(cls): @@ -188,8 +210,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): meter_provider: Optional[Any] = None, **kwargs, ): + team_metadata_keys_override = kwargs.pop("baggage_team_metadata_keys", None) if config is None: config = OpenTelemetryConfig.from_env() + if team_metadata_keys_override is not None: + config.baggage_team_metadata_keys = _normalize_team_metadata_keys( + team_metadata_keys_override + ) self.config = config self.callback_name = callback_name @@ -985,6 +1012,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): litellm_params = kwargs.get("litellm_params", {}) or {} _metadata = litellm_params.get("metadata", {}) or {} proxy_span = _metadata.get("litellm_parent_otel_span", None) + + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES). + if proxy_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + proxy_span = _litellm_metadata.get("litellm_parent_otel_span", None) + if ( proxy_span is not None and getattr(proxy_span, "name", None) == LITELLM_PROXY_REQUEST_SPAN_NAME @@ -1245,7 +1279,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): or {} ) team_metadata = self._team_metadata_json( - raw_metadata.get("user_api_key_team_metadata") + raw_metadata.get("user_api_key_team_metadata"), + self.config.baggage_team_metadata_keys, ) if team_metadata: self.safe_set_attribute( @@ -1268,15 +1303,20 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) @staticmethod - def _team_metadata_json(value: Any) -> Optional[str]: - """JSON-serialize a team's metadata dict for a single span attribute. + def _team_metadata_json(value: Any, allowed_keys: List[str]) -> Optional[str]: + """JSON-serialize only the allowlisted sub-keys of a team's metadata. - Returns ``None`` for a missing, non-dict, or empty mapping so the - empty case is dropped rather than stamping a useless ``"{}"``. + Returns ``None`` when nothing is allowlisted or no allowlisted key is + present, so the empty case is dropped rather than stamping a useless + ``"{}"`` (and so a team's metadata never leaves the process until an + operator opts each sub-key in via ``baggage_team_metadata_keys``). """ - if not isinstance(value, dict) or not value: + if not isinstance(value, dict) or not value or not allowed_keys: return None - return safe_dumps(value) + filtered = {key: value[key] for key in allowed_keys if key in value} + if not filtered: + return None + return safe_dumps(filtered) def _record_metrics(self, kwargs, response_obj, start_time, end_time): duration_s = (end_time - start_time).total_seconds() @@ -2635,6 +2675,10 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): ) def _to_ns(self, dt): + if dt is None: + return int(datetime.now().timestamp() * 1e9) + if isinstance(dt, (int, float)): + return int(dt * 1e9) return int(dt.timestamp() * 1e9) def _get_span_name(self, kwargs): @@ -2681,6 +2725,13 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): _metadata = litellm_params.get("metadata", {}) or {} parent_otel_span = _metadata.get("litellm_parent_otel_span", None) + # Fallback: check litellm_metadata (used by /v1/messages and other + # LITELLM_METADATA_ROUTES that store proxy-internal metadata + # separately from the provider's native "metadata" field). + if parent_otel_span is None: + _litellm_metadata = litellm_params.get("litellm_metadata", {}) or {} + parent_otel_span = _litellm_metadata.get("litellm_parent_otel_span", None) + # Priority 1: Explicit parent span from metadata if parent_otel_span is not None: verbose_logger.debug( @@ -3254,6 +3305,32 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger): value=int(status_code), ) + def record_error_attributes_on_span( + self, + span: Optional[Span], + exception: Optional[Exception], + status_code: int, + ) -> None: + """Stamp structured ``error.*`` attributes on the SERVER span from the + exception returned to the client, with ``error.code`` pinned to the real + response status. Idempotent (overwrites); emits no exception event.""" + if span is None or exception is None: + return + from litellm.litellm_core_utils.litellm_logging import ( + StandardLoggingPayloadSetup, + ) + + error_information = StandardLoggingPayloadSetup.get_error_information( + original_exception=exception + ) + error_information["error_code"] = str(status_code) + self._record_exception_on_span( + span=span, + kwargs={ + "standard_logging_object": {"error_information": error_information} + }, + ) + def set_preprocessing_duration_attribute( self, span: Optional[Span], container: Any ) -> None: diff --git a/litellm/integrations/opik/opik_payload_builder/extractors.py b/litellm/integrations/opik/opik_payload_builder/extractors.py index 9779ccddac..1e3a664acc 100644 --- a/litellm/integrations/opik/opik_payload_builder/extractors.py +++ b/litellm/integrations/opik/opik_payload_builder/extractors.py @@ -39,20 +39,32 @@ def extract_opik_metadata( standard_logging_metadata: Dict[str, Any], ) -> Dict[str, Any]: """ - Extract and merge Opik metadata from request and requester. + Merge Opik metadata from three sources in increasing priority order: + + 1. user_api_key_auth_metadata– lowest priority (operator-level defaults) + 2. litellm_metadata (request)– overrides auth-key defaults + 3. requester_metadata – highest priority (e.g. proxy header overrides) Args: - litellm_metadata: Metadata from litellm_params - standard_logging_metadata: Metadata from standard_logging_object + litellm_metadata: Metadata from litellm_params.mak + standard_logging_metadata: Metadata from standard_logging_object. Returns: - Merged Opik metadata dictionary + Merged Opik metadata dictionary. """ - opik_meta = litellm_metadata.get("opik", {}).copy() + # Start with auth-key defaults (lowest priority). + auth_meta = standard_logging_metadata.get("user_api_key_auth_metadata") or {} + opik_meta = (auth_meta.get("opik") or {}).copy() + # Request-level values override auth-key defaults. + request_opik = litellm_metadata.get("opik") or {} + opik_meta.update(request_opik) + + # Requester-level values win over everything else. requester_metadata = standard_logging_metadata.get("requester_metadata", {}) or {} requester_opik = requester_metadata.get("opik", {}) or {} - opik_meta.update(requester_opik) + if requester_opik: + opik_meta.update(requester_opik) _logging.verbose_logger.debug( f"litellm_opik_metadata - {json.dumps(opik_meta, default=str)}" diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 99b3ecea16..3edb96ed8d 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -172,10 +172,13 @@ nothing here imports outside it: `capture_span_content` gates whether prompt/response bodies may be written as span attributes; it defaults **off** (`no_content`). The Baggage allowlists are configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` / - `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` (comma-separated) as env vars, or - `baggage_promoted_keys` / `baggage_metadata_keys` (YAML lists) under - `callback_settings.otel` in `config.yaml` — the latter reach the config through - the logger's constructor kwargs. + `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` / + `LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or + `baggage_promoted_keys` / `baggage_metadata_keys` / + `baggage_team_metadata_keys` (YAML lists) under `callback_settings.otel` in + `config.yaml` — the latter reach the config through the logger's constructor + kwargs. `baggage_team_metadata_keys` is empty by default, so none of a team's + free-form metadata is promoted until each sub-key is explicitly allowlisted. - [`baggage.py`](./model/baggage.py) — the single definition of which request-identity values are promoted into Baggage (so child spans inherit them) and under which attribute keys. diff --git a/litellm/integrations/otel/__init__.py b/litellm/integrations/otel/__init__.py index 42a84a85fb..da3ce4af3e 100644 --- a/litellm/integrations/otel/__init__.py +++ b/litellm/integrations/otel/__init__.py @@ -32,20 +32,28 @@ from litellm.integrations.otel.model.payloads import ( LLMCallSpanData, LLMRequestParams, LLMUsage, + MCPToolCallSpanData, ProxyRequestSpanData, ServerInfo, ServiceSpanData, SpanError, + is_mcp_tool_call, ) from litellm.integrations.otel.model.semconv import ( DB, + HTTP, + MCP, + Client, Error, GenAI, GenAIOperation, GenAIProvider, - HTTP, + JsonRpc, LiteLLM, + MCPMethod, Metric, + Network, + NetworkTransport, Server, resolve_operation, resolve_provider, @@ -69,13 +77,19 @@ __all__ = [ "BAGGAGE_PROMOTED_KEYS", "DB", "DEFAULT_BAGGAGE_METADATA_KEYS", + "Client", "Error", "GenAI", "GenAIOperation", "GenAIProvider", "HTTP", + "JsonRpc", "LiteLLM", + "MCP", + "MCPMethod", "Metric", + "Network", + "NetworkTransport", "Server", "resolve_operation", "resolve_provider", @@ -92,11 +106,13 @@ __all__ = [ "LLMCallSpanData", "LLMRequestParams", "LLMUsage", + "MCPToolCallSpanData", "ProxyRequestSpanData", "RequestContext", "RequestIdentity", "ServerInfo", "ServiceSpanData", "SpanError", + "is_mcp_tool_call", "promoted_baggage", ] diff --git a/litellm/integrations/otel/emitter.py b/litellm/integrations/otel/emitter.py index cae6514efd..7fb7be7ab8 100644 --- a/litellm/integrations/otel/emitter.py +++ b/litellm/integrations/otel/emitter.py @@ -13,6 +13,7 @@ from litellm.integrations.otel.mappers.base import AttributeMapper, SpanData from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, + MCPToolCallSpanData, ServiceSpanData, ) from litellm.integrations.otel.plumbing.providers import to_otel_span_kind @@ -22,6 +23,7 @@ from litellm.integrations.otel.model.spans import ( SpanRole, guardrail_span_name, llm_call_span_name, + mcp_tool_call_span_name, service_span_name, ) @@ -30,6 +32,7 @@ from litellm.integrations.otel.model.spans import ( # have no builder here. _NAME_BUILDERS: dict[SpanRole, Callable[..., str]] = { SpanRole.LLM_CALL: llm_call_span_name, + SpanRole.MCP_TOOL_CALL: mcp_tool_call_span_name, SpanRole.GUARDRAIL: guardrail_span_name, # DB_CALL and SERVICE are both built from ServiceSpanData; they differ only in # span kind (CLIENT vs INTERNAL) and attribute vocabulary, not in naming. @@ -121,10 +124,14 @@ class SpanEmitter: Return the span, or ``None`` if it was deduplicated away. ``tracer`` overrides the bound tracer for this span, used for per-request routing. """ - # Only LLM-call spans carry a dedup key; LLM-call and service spans - # carry an ``error`` field. ``isinstance`` narrows the type for mypy and - # keeps the engine free of duck-typed attribute reads. - dedup_key = data.identity.call_id if isinstance(data, LLMCallSpanData) else None + # LLM-call and MCP tool-call spans carry a dedup key (their request's + # call id), so a sync+async double-firing coalesces. ``isinstance`` narrows + # the type for mypy and keeps the engine free of duck-typed attribute reads. + dedup_key = ( + data.identity.call_id + if isinstance(data, (LLMCallSpanData, MCPToolCallSpanData)) + else None + ) if self._seen(dedup_key, role): return None span = self.start_span( @@ -160,7 +167,15 @@ class SpanEmitter: span.set_attribute(key, value) error = ( data.error - if isinstance(data, (LLMCallSpanData, ServiceSpanData, GuardrailSpanData)) + if isinstance( + data, + ( + LLMCallSpanData, + MCPToolCallSpanData, + ServiceSpanData, + GuardrailSpanData, + ), + ) else None ) if error and (error.error_type or error.message): diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 007b41df0a..57738c356f 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -25,14 +25,15 @@ from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.metadata import ( LLMCallEvent, RequestIdentity, - guardrail_entries_from_request_data, model_from_request_data, ) from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, + MCPToolCallSpanData, ServiceSpanData, SpanError, + is_mcp_tool_call, ) from litellm.integrations.otel.plumbing.providers import ( build_tracer_provider, @@ -43,7 +44,10 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic from litellm.integrations.otel.model.utils import to_ns if TYPE_CHECKING: - from litellm.types.utils import StandardLoggingGuardrailInformation + from litellm.types.utils import ( + StandardLoggingGuardrailInformation, + StandardLoggingPayload, + ) LITELLM_TRACER_NAME = "litellm" @@ -200,11 +204,53 @@ class OpenTelemetryV2(CustomLogger): self._open_llm_calls.popitem(last=False) async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + if self._emit_mcp_tool_call(kwargs, start_time, end_time): + return self._close_llm_call(kwargs, start_time, end_time) async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time): + if self._emit_mcp_tool_call(kwargs, start_time, end_time): + return self._close_llm_call(kwargs, start_time, end_time) + def _emit_mcp_tool_call( + self, + kwargs: Mapping[str, Any], + start_time: datetime | float | None, + end_time: datetime | float | None, + ) -> bool: + """Emit an MCP tool-call span when the closed request was a tool call. + + MCP tool calls reach the success/failure callbacks like any other request + (with ``call_type`` ``call_mcp_tool``), but they are not LLM calls and have + no ``pre_call`` carrier — so they get their own CLIENT span here, parented + to the request's server span. Returns whether it handled the event, so the + caller skips the LLM-call path. The whole span is emitted at once (there is + no boundary to open it at), deduped on the call id by the emitter. + """ + raw_payload = kwargs.get("standard_logging_object") + if not raw_payload or not is_mcp_tool_call( + cast(Mapping[str, object], raw_payload) + ): + return False + payload = cast("StandardLoggingPayload", raw_payload) + data = MCPToolCallSpanData.from_standard_logging_payload( + payload, capture_content=self.config.capture_span_content + ) + # A stray LLM carrier from a ``pre_call`` that mis-fired for this id would + # otherwise linger until evicted; drop it so it's neither leaked nor closed + # as a phantom LLM span. + if data.identity.call_id: + self._open_llm_calls.pop(data.identity.call_id, None) + self._emitter.emit( + SpanRole.MCP_TOOL_CALL, + data, + parent_context=resolve_request_span_context(), + start_time_ns=to_ns(start_time), + end_time_ns=to_ns(end_time), + ) + return True + def _close_llm_call( self, kwargs: Mapping[str, Any], @@ -254,6 +300,7 @@ class OpenTelemetryV2(CustomLogger): data.request_model, promoted_keys=tuple(self.config.baggage_promoted_keys), metadata_keys=tuple(self.config.baggage_metadata_keys), + team_metadata_keys=tuple(self.config.baggage_team_metadata_keys), ) if bag: parent_ctx = set_request_baggage(bag, context=parent_ctx) @@ -380,6 +427,7 @@ class OpenTelemetryV2(CustomLogger): model, promoted_keys=tuple(self.config.baggage_promoted_keys), metadata_keys=tuple(self.config.baggage_metadata_keys), + team_metadata_keys=tuple(self.config.baggage_team_metadata_keys), ) if bag: # Attach (no detach): the contextvar is scoped to this request's @@ -419,46 +467,27 @@ class OpenTelemetryV2(CustomLogger): ) return data - async def async_post_call_success_hook( - self, - data: Mapping[str, Any], - user_api_key_dict: Any, - response: Any, - ) -> Any: - self._emit_guardrail_spans(data) - return response - - async def async_post_call_failure_hook( - self, - request_data: Mapping[str, Any], - original_exception: BaseException | None, - user_api_key_dict: Any, - traceback_str: str | None = None, - ) -> None: - self._emit_guardrail_spans(request_data) - - def _emit_guardrail_spans(self, request_data: Mapping[str, Any]) -> None: + def emit_guardrail_span(self, entry: "StandardLoggingGuardrailInformation") -> None: + # Emitted by the guardrail-recording code the moment a guardrail finishes, + # not from a post-call hook — that hook does not fire on every path (a + # pass-through request that passes its guardrails never reaches it), which + # left passing guardrails without a span. + # # A guardrail is a sibling of the LLM call under the request's root span, - # so parent it to the explicit anchor — not the active span, which on the - # failure path can be the live ``auth`` phase span (post-call failure hooks - # run from inside it on an auth rejection). Emit with the guardrail's actual - # execution window so a pre_call guardrail is placed before the LLM call - # rather than at post-call emission time. - guardrails = guardrail_entries_from_request_data(request_data) - if not guardrails: - return - parent_ctx = resolve_request_span_context() - for entry in guardrails: - data = GuardrailSpanData.from_logging_entry( - cast("StandardLoggingGuardrailInformation", entry) - ) - self._emitter.emit( - SpanRole.GUARDRAIL, - data, - parent_context=parent_ctx, - start_time_ns=to_ns(data.start_time), - end_time_ns=to_ns(data.end_time), - ) + # so parent it to the explicit anchor — never the active span, which during + # a pre_call guardrail can be the live ``auth`` phase span. Emit with the + # guardrail's actual execution window so a pre_call guardrail is placed + # before the LLM call rather than at emission time. One entry in, one span + # out — the module-level entry point routes each entry to this single + # registered logger so a guardrail is never emitted more than once. + data = GuardrailSpanData.from_logging_entry(entry) + self._emitter.emit( + SpanRole.GUARDRAIL, + data, + parent_context=resolve_request_span_context(), + start_time_ns=to_ns(data.start_time), + end_time_ns=to_ns(data.end_time), + ) def create_litellm_proxy_request_started_span( self, start_time: datetime, headers: Mapping[str, str] | None @@ -479,6 +508,26 @@ def _registered_v2_logger() -> "OpenTelemetryV2 | None": return logger if isinstance(logger, OpenTelemetryV2) else None +def emit_guardrail_span(entry: "StandardLoggingGuardrailInformation") -> None: + """Emit a guardrail span on the registered v2 OTel logger. + + Called by the guardrail-recording code the moment a guardrail finishes, so a + span is produced regardless of whether a post-call hook later runs (it does + not on the pass-through allow path). Routes through the single canonical + logger — the same one every other v2 entry point uses — so a guardrail + recorded once yields exactly one span; fanning out across every reachable + ``OpenTelemetryV2`` instance double-emits the same entry. Best-effort: span + emission must never break guardrail evaluation. + """ + logger = _registered_v2_logger() + if logger is None: + return + try: + logger.emit_guardrail_span(entry) + except Exception: + pass + + def seed_request_identity(user_api_key_dict: Any, model: Any = None) -> None: logger = _registered_v2_logger() if logger is not None: diff --git a/litellm/integrations/otel/mappers/base.py b/litellm/integrations/otel/mappers/base.py index e8fb5af979..dfdaf77a83 100644 --- a/litellm/integrations/otel/mappers/base.py +++ b/litellm/integrations/otel/mappers/base.py @@ -7,6 +7,7 @@ from typing_extensions import Protocol, runtime_checkable from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, + MCPToolCallSpanData, ServiceSpanData, ) @@ -21,7 +22,7 @@ AttributeMap = dict[str, AttrValue] # The closed set of span-data types the engine routes through the mapper chain. # Server spans (PROXY_REQUEST + management routes) belong to the mounted FastAPI # instrumentor, not the mapper chain. -SpanData = LLMCallSpanData | GuardrailSpanData | ServiceSpanData +SpanData = LLMCallSpanData | MCPToolCallSpanData | GuardrailSpanData | ServiceSpanData @runtime_checkable diff --git a/litellm/integrations/otel/mappers/genai.py b/litellm/integrations/otel/mappers/genai.py index 57fa51ea1f..6c61feced4 100644 --- a/litellm/integrations/otel/mappers/genai.py +++ b/litellm/integrations/otel/mappers/genai.py @@ -14,10 +14,18 @@ from litellm.integrations.otel.mappers.utils import collect, drop_none from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, + MCPToolCallSpanData, ServiceSpanData, ToolDefinition, ) -from litellm.integrations.otel.model.semconv import DB, Error, GenAI, LiteLLM, Server +from litellm.integrations.otel.model.semconv import ( + DB, + MCP, + Error, + GenAI, + LiteLLM, + Server, +) from litellm.integrations.otel.model.spans import db_system @@ -64,6 +72,18 @@ class GenAIMapper: "parameters": lambda t: t.parameters_json or None, } + _MCP_ATTRS: dict[str, Callable[[MCPToolCallSpanData], AttrValue | None]] = { + GenAI.OPERATION_NAME: lambda d: d.operation.value, + MCP.METHOD_NAME: lambda d: d.method, + MCP.SESSION_ID: lambda d: d.session_id, + GenAI.TOOL_NAME: lambda d: d.tool_name or None, + GenAI.TOOL_CALL_ARGUMENTS: lambda d: d.arguments_json, + GenAI.TOOL_CALL_RESULT: lambda d: d.result_json, + LiteLLM.MCP_SERVER_NAME: lambda d: d.server_name, + LiteLLM.CALL_ID: lambda d: d.identity.call_id or None, + f"{LiteLLM.COST_PREFIX}total": lambda d: d.response_cost, + } + _GUARDRAIL_ATTRS: dict[str, Callable[[GuardrailSpanData], AttrValue | None]] = { LiteLLM.GUARDRAIL_NAME: lambda d: d.guardrail_name, LiteLLM.GUARDRAIL_MODE: lambda d: d.mode, @@ -92,6 +112,8 @@ class GenAIMapper: match data: case LLMCallSpanData(): return self._llm_call(data) + case MCPToolCallSpanData(): + return collect(self._MCP_ATTRS, data) case GuardrailSpanData(): return self._guardrail(data) case ServiceSpanData(): diff --git a/litellm/integrations/otel/model/baggage.py b/litellm/integrations/otel/model/baggage.py index 67dd64e391..ecab643a26 100644 --- a/litellm/integrations/otel/model/baggage.py +++ b/litellm/integrations/otel/model/baggage.py @@ -6,26 +6,36 @@ LLM-call span so that child spans (guardrail, service) inherit them. the allowlisted keys onto every span. This module is the single place baggage is defined: ``_PROMOTABLE`` maps each -promotable attribute key to how its value is read, and the two ``*_KEYS`` -defaults select what is promoted unless the config overrides them. +promotable attribute key to how its value is read, and the ``*_KEYS`` defaults +select what is promoted unless the config overrides them. ``TEAM_METADATA``'s +extractor filters the team's free-form metadata to the sub-keys an operator +allowlists via ``baggage_team_metadata_keys`` (default none), so the blob is +never promoted whole. """ -from collections.abc import Callable +import json +from collections.abc import Callable, Mapping from typing import Final from litellm.integrations.otel.model.metadata import RequestIdentity from litellm.integrations.otel.model.semconv import GenAI, LiteLLM -# Attribute key -> value extractor over (identity, request_model). The single -# definition of what may be promoted and under which key. -_PROMOTABLE: Final[dict[str, Callable[[RequestIdentity, str | None], str | None]]] = { - LiteLLM.TEAM_ID: lambda identity, model: identity.team_id, - LiteLLM.TEAM_ALIAS: lambda identity, model: identity.team_alias, - LiteLLM.TEAM_METADATA: lambda identity, model: identity.team_metadata, - LiteLLM.KEY_HASH: lambda identity, model: identity.key_hash, - LiteLLM.END_USER: lambda identity, model: identity.end_user, - GenAI.REQUEST_MODEL: lambda identity, model: model, - LiteLLM.PROVIDER_MODEL: lambda identity, model: identity.provider_model, +# Attribute key -> value extractor over (identity, request_model, +# team_metadata_keys). The single definition of what may be promoted and under +# which key. Only the ``TEAM_METADATA`` extractor consults team_metadata_keys +# (to filter the team's metadata to an allowlist); the rest ignore it. +_PROMOTABLE: Final[ + dict[str, Callable[[RequestIdentity, str | None, tuple[str, ...]], str | None]] +] = { + LiteLLM.TEAM_ID: lambda identity, model, team_metadata_keys: identity.team_id, + LiteLLM.TEAM_ALIAS: lambda identity, model, team_metadata_keys: identity.team_alias, + LiteLLM.TEAM_METADATA: lambda identity, model, team_metadata_keys: _filtered_team_metadata_json( + identity.team_metadata, team_metadata_keys + ), + LiteLLM.KEY_HASH: lambda identity, model, team_metadata_keys: identity.key_hash, + LiteLLM.END_USER: lambda identity, model, team_metadata_keys: identity.end_user, + GenAI.REQUEST_MODEL: lambda identity, model, team_metadata_keys: model, + LiteLLM.PROVIDER_MODEL: lambda identity, model, team_metadata_keys: identity.provider_model, } # Keys promoted by default (a subset of ``_PROMOTABLE``). ``END_USER`` is @@ -50,23 +60,31 @@ DEFAULT_BAGGAGE_METADATA_KEYS: Final[tuple[str, ...]] = ( "requester_ip_address", ) +# Sub-keys of the team's free-form metadata eligible for promotion under +# ``litellm.team.metadata``. Empty by default: a team's metadata can hold +# arbitrary operator data, so none of it is promoted until each key is +# explicitly allowlisted via ``config.baggage_team_metadata_keys``. +DEFAULT_BAGGAGE_TEAM_METADATA_KEYS: Final[tuple[str, ...]] = () + def promoted_baggage( identity: RequestIdentity, request_model: str | None, promoted_keys: tuple[str, ...], metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_METADATA_KEYS, + team_metadata_keys: tuple[str, ...] = DEFAULT_BAGGAGE_TEAM_METADATA_KEYS, ) -> dict[str, str]: """Identity values to write into Baggage, filtered to ``promoted_keys``. ``promoted_keys`` selects from ``_PROMOTABLE``; ``metadata_keys`` selects - sub-keys of ``identity.metadata`` to promote under ``litellm.metadata.*``. - Empty values are dropped. + sub-keys of ``identity.metadata`` to promote under ``litellm.metadata.*``; + ``team_metadata_keys`` selects sub-keys of the team's metadata to promote + under ``litellm.team.metadata``. Empty values are dropped. """ out: dict[str, str] = {} for key, extract in _PROMOTABLE.items(): if key in promoted_keys: - value = extract(identity, request_model) + value = extract(identity, request_model, team_metadata_keys) if value: out[key] = value for meta_key in metadata_keys: @@ -74,3 +92,21 @@ def promoted_baggage( if value: out[f"{LiteLLM.METADATA_PREFIX}{meta_key}"] = value return out + + +def _filtered_team_metadata_json( + metadata: Mapping[str, object] | None, + allowed_keys: tuple[str, ...], +) -> str | None: + """JSON-serialize only the allowlisted sub-keys of a team's metadata. + + Returns ``None`` when nothing is allowlisted or no allowlisted key is + present, so the empty case is dropped rather than promoting ``"{}"``. Keys + are sorted for a stable, diff-friendly value. + """ + if not isinstance(metadata, Mapping) or not allowed_keys: + return None + filtered = {key: metadata[key] for key in allowed_keys if key in metadata} + if not filtered: + return None + return json.dumps(filtered, default=str, sort_keys=True) diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index f78e1515ea..ca46182bc6 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -9,6 +9,7 @@ from typing_extensions import Annotated from litellm.integrations.otel.model.baggage import ( BAGGAGE_PROMOTED_KEYS, DEFAULT_BAGGAGE_METADATA_KEYS, + DEFAULT_BAGGAGE_TEAM_METADATA_KEYS, ) #: Master feature-flag env var. The logger is inert until this is truthy. @@ -168,10 +169,25 @@ class OpenTelemetryV2Config(BaseSettings): "``callback_settings.otel.baggage_metadata_keys`` in config.yaml." ), ) + baggage_team_metadata_keys: Annotated[List[str], NoDecode] = Field( + default_factory=lambda: list(DEFAULT_BAGGAGE_TEAM_METADATA_KEYS), + validation_alias=AliasChoices( + "baggage_team_metadata_keys", "LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS" + ), + description=( + "Sub-keys of the team's free-form metadata promoted under " + "``litellm.team.metadata``. Empty by default so none of a team's " + "metadata leaves the process until explicitly allowlisted. Configure " + "via the ``LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS`` env var " + "(comma-separated) or " + "``callback_settings.otel.baggage_team_metadata_keys`` in config.yaml." + ), + ) @field_validator( "baggage_promoted_keys", "baggage_metadata_keys", + "baggage_team_metadata_keys", "mapper_names", mode="before", ) diff --git a/litellm/integrations/otel/model/metadata.py b/litellm/integrations/otel/model/metadata.py index f0ea0a608c..4c9cecfef5 100644 --- a/litellm/integrations/otel/model/metadata.py +++ b/litellm/integrations/otel/model/metadata.py @@ -36,7 +36,6 @@ model. They coincide on the SDK path, which is correct. from __future__ import annotations -import json from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Mapping, cast @@ -53,8 +52,10 @@ class RequestIdentity: call_id: str | None = None team_id: str | None = None team_alias: str | None = None - # The team's free-form metadata dict, JSON-serialized (empty/missing -> None). - team_metadata: str | None = None + # The team's free-form metadata, carried raw (empty/missing -> None) and + # filtered to an operator allowlist only at Baggage-promotion time, so an + # unconfigured deployment never promotes any of it. + team_metadata: Mapping[str, Any] | None = None key_hash: str | None = None end_user: str | None = None # The model litellm dispatched to the provider. Only known once the call @@ -86,7 +87,7 @@ class RequestIdentity: or as_str(raw_meta.get("team_id")), team_alias=as_str(raw_meta.get("user_api_key_team_alias")) or as_str(raw_meta.get("team_alias")), - team_metadata=_team_metadata_json( + team_metadata=_team_metadata_dict( raw_meta.get("user_api_key_team_metadata") ), key_hash=as_str(raw_meta.get("user_api_key_hash")), @@ -121,7 +122,7 @@ class RequestIdentity: return cls( team_id=as_str(get("team_id")), team_alias=as_str(get("team_alias")), - team_metadata=_team_metadata_json(get("team_metadata")), + team_metadata=_team_metadata_dict(get("team_metadata")), key_hash=as_str(get("api_key")), end_user=as_str(get("end_user_id")), # ``provider_model`` is unknown at the auth boundary — routing hasn't @@ -254,26 +255,6 @@ def model_from_request_data(data: object) -> str | None: return None -def guardrail_entries_from_request_data( - request_data: Mapping[str, Any], -) -> list[dict]: - """The guardrail-information dicts buried in ``metadata`` of a post-call dict. - - ``standard_logging_guardrail_information`` is stored as either a single dict - or a list of them; normalize to a list of dicts (dropping non-dict noise) so - the caller just iterates. Empty list when none are present. - """ - metadata = request_data.get("metadata") - if not isinstance(metadata, Mapping): - return [] - info = metadata.get("standard_logging_guardrail_information") - if isinstance(info, Mapping): - return [cast(dict, info)] - if isinstance(info, list): - return [entry for entry in info if isinstance(entry, dict)] - return [] - - def resolve_provider_model(payload: "StandardLoggingPayload") -> str | None: """The model litellm dispatched to the provider, from the payload. @@ -300,16 +281,14 @@ def _model_info_id(model_info: object) -> str | None: return None -def _team_metadata_json(value: object) -> str | None: - """JSON-serialize a team's metadata dict for a single Baggage value. +def _team_metadata_dict(value: object) -> Mapping[str, Any] | None: + """The team's free-form metadata as a raw mapping, or ``None`` when missing + or empty. - Returns ``None`` for a missing, non-dict, or empty mapping so the empty case - is dropped rather than promoting a useless ``"{}"``. Keys are sorted for a - stable, diff-friendly serialization. + Carried raw on the identity and filtered to an operator allowlist only at + Baggage-promotion time (see ``baggage.promoted_baggage``), so an empty case + is dropped rather than carrying a useless ``{}``. """ - if not isinstance(value, Mapping) or not value: - return None - try: - return json.dumps(value, default=str, sort_keys=True) - except Exception: - return None + if isinstance(value, Mapping) and value: + return dict(value) + return None diff --git a/litellm/integrations/otel/model/payloads.py b/litellm/integrations/otel/model/payloads.py index 65b50d0fc1..bbef40ba37 100644 --- a/litellm/integrations/otel/model/payloads.py +++ b/litellm/integrations/otel/model/payloads.py @@ -14,6 +14,7 @@ from litellm.integrations.otel.model.metadata import ( ) from litellm.integrations.otel.model.semconv import ( GenAIOperation, + MCPMethod, resolve_operation, resolve_provider, ) @@ -35,11 +36,13 @@ __all__ = [ "LLMCallSpanData", "LLMRequestParams", "LLMUsage", + "MCPToolCallSpanData", "ProxyRequestSpanData", "ServerInfo", "ServiceSpanData", "SpanError", "ToolDefinition", + "is_mcp_tool_call", ] if TYPE_CHECKING: @@ -309,6 +312,77 @@ class LLMCallSpanData: ) +# --- the MCP tool-call model ------------------------------------------------- # + + +@dataclass(frozen=True) +class MCPToolCallSpanData: + """One MCP ``tools/call`` execution, parsed from a closed request's payload. + + The proxy is an MCP *client* to the upstream server it forwards the call to, + so this is a CLIENT span. ``arguments_json``/``result_json`` are the tool's + input/output — sensitive content, so they're only retained when content + capture is enabled, mirroring ``LLMCallSpanData``'s message bodies. + """ + + operation: GenAIOperation + method: str + tool_name: str + server_name: str | None + session_id: str | None + arguments_json: str | None + result_json: str | None + error: SpanError | None + response_cost: float | None + identity: RequestIdentity + + @classmethod + def from_standard_logging_payload( + cls, payload: "StandardLoggingPayload", capture_content: bool = False + ) -> "MCPToolCallSpanData": + meta = _mcp_tool_call_metadata(cast(Mapping[str, object], payload)) + return cls( + operation=resolve_operation(as_str(payload.get("call_type"))), + method=MCPMethod.TOOLS_CALL.value, + tool_name=as_str(meta.get("name")) or "", + server_name=as_str(meta.get("mcp_server_name")), + session_id=as_str(meta.get("mcp_session_id")), + arguments_json=( + _json_or_none(meta.get("arguments")) + if capture_content and meta.get("arguments") is not None + else None + ), + result_json=( + _json_or_none(meta.get("result")) + if capture_content and meta.get("result") is not None + else None + ), + error=_parse_error(payload), + response_cost=as_float(payload.get("response_cost")), + identity=RequestContext.from_standard_logging_payload(payload).identity, + ) + + +def _mcp_tool_call_metadata(payload: Mapping[str, object]) -> Mapping[str, object]: + """The MCP gateway's tool-call metadata, which lives under + ``StandardLoggingPayload.metadata`` (a ``StandardLoggingMetadata`` key), not + at the payload's top level.""" + metadata = payload.get("metadata") + if not isinstance(metadata, Mapping): + return {} + meta = metadata.get("mcp_tool_call_metadata") + return meta if isinstance(meta, Mapping) else {} + + +def is_mcp_tool_call(payload: Mapping[str, object]) -> bool: + """Whether a closed request's payload is an MCP tool call rather than an LLM + call — true when the MCP gateway stamped its tool-call metadata, or the call + type says so on a path that hasn't populated the metadata yet.""" + return bool(_mcp_tool_call_metadata(payload)) or ( + payload.get("call_type") == "call_mcp_tool" + ) + + # --- service event_metadata sanitization ------------------------------------ # # Substrings (case-insensitive) of keys that must never reach a span: secrets, diff --git a/litellm/integrations/otel/model/semconv.py b/litellm/integrations/otel/model/semconv.py index 1c6c30eda0..7df07f30a0 100644 --- a/litellm/integrations/otel/model/semconv.py +++ b/litellm/integrations/otel/model/semconv.py @@ -16,7 +16,7 @@ class GenAIOperation(str, Enum): GENERATE_CONTENT = "generate_content" CREATE_AGENT = "create_agent" # reserved for future agent spans INVOKE_AGENT = "invoke_agent" # reserved for future agent spans - EXECUTE_TOOL = "execute_tool" # reserved for future tool spans + EXECUTE_TOOL = "execute_tool" # MCP tool-call spans class GenAIProvider(str, Enum): @@ -38,6 +38,16 @@ class GenAIProvider(str, Enum): IBM_WATSONX_AI = "ibm.watsonx.ai" +class MCPMethod(str, Enum): + """Well-known values for ``mcp.method.name`` that litellm's MCP gateway + serves. The value is the JSON-RPC method exactly as it travels on the wire.""" + + TOOLS_CALL = "tools/call" + TOOLS_LIST = "tools/list" + PROMPTS_GET = "prompts/get" + PROMPTS_LIST = "prompts/list" + + class GenAI: """Canonical OTel GenAI span-attribute keys.""" @@ -68,11 +78,68 @@ class GenAI: SYSTEM_INSTRUCTIONS: Final = "gen_ai.system_instructions" OUTPUT_TYPE: Final = "gen_ai.output.type" CONVERSATION_ID: Final = "gen_ai.conversation.id" - # agent / tool (reserved) + # agent (reserved) AGENT_ID: Final = "gen_ai.agent.id" AGENT_NAME: Final = "gen_ai.agent.name" + # tool / tool-call (stamped on MCP tool-call spans). Arguments and result are + # the tool's input/output payloads — sensitive, so they're opt-in and gated by + # the same content-capture mode as prompt/response content. TOOL_NAME: Final = "gen_ai.tool.name" TOOL_CALL_ID: Final = "gen_ai.tool.call.id" + TOOL_CALL_ARGUMENTS: Final = "gen_ai.tool.call.arguments" + TOOL_CALL_RESULT: Final = "gen_ai.tool.call.result" + # prompt (MCP ``prompts/get`` etc.) + PROMPT_NAME: Final = "gen_ai.prompt.name" + + +class MCP: + """OTel GenAI MCP (Model Context Protocol) span-attribute keys. + + ``METHOD_NAME`` is the only key litellm populates from a closed request today; + the rest are part of the convention's vocabulary and are stamped when the + corresponding signal (session, protocol version, resource) is available. + """ + + METHOD_NAME: Final = "mcp.method.name" + SESSION_ID: Final = "mcp.session.id" + PROTOCOL_VERSION: Final = "mcp.protocol.version" + RESOURCE_URI: Final = "mcp.resource.uri" + + +class JsonRpc: + """JSON-RPC keys carried on MCP spans. The error/status code lives in the + ``rpc.*`` namespace per semconv, not ``jsonrpc.*``.""" + + REQUEST_ID: Final = "jsonrpc.request.id" + PROTOCOL_VERSION: Final = "jsonrpc.protocol.version" + RESPONSE_STATUS_CODE: Final = "rpc.response.status_code" + + +class NetworkTransport(str, Enum): + """Well-known values for ``network.transport``.""" + + TCP = "tcp" + UDP = "udp" + QUIC = "quic" + UNIX = "unix" + PIPE = "pipe" + + +class Network: + """OTel network keys, recommended on MCP spans to describe the transport + carrying the JSON-RPC messages (stdio pipe, HTTP, websocket, …).""" + + PROTOCOL_NAME: Final = "network.protocol.name" + PROTOCOL_VERSION: Final = "network.protocol.version" + TRANSPORT: Final = "network.transport" + + +class Client: + """Peer (client) network keys, stamped on MCP *server* spans the same way + ``server.*`` is stamped on client spans.""" + + ADDRESS: Final = "client.address" + PORT: Final = "client.port" class Error: @@ -137,6 +204,10 @@ class LiteLLM: SERVICE_NAME: Final = "litellm.service.name" SERVICE_CALL_TYPE: Final = "litellm.service.call_type" PREPROCESSING_MS: Final = "litellm.preprocessing.duration_ms" + # The logical name of the MCP server a tool call was routed to. There is no + # semconv key for an MCP server's *name* (the convention uses ``server.address`` + # for its network location), so it lives under the vendor namespace. + MCP_SERVER_NAME: Final = "litellm.mcp.server.name" class Metric: @@ -179,6 +250,7 @@ _OPERATION_BY_CALL_TYPE: dict[str, GenAIOperation] = { "aembedding": GenAIOperation.EMBEDDINGS, "responses": GenAIOperation.CHAT, "aresponses": GenAIOperation.CHAT, + "call_mcp_tool": GenAIOperation.EXECUTE_TOOL, } diff --git a/litellm/integrations/otel/model/spans.py b/litellm/integrations/otel/model/spans.py index e4876f4ee5..1adc1d68dd 100644 --- a/litellm/integrations/otel/model/spans.py +++ b/litellm/integrations/otel/model/spans.py @@ -46,6 +46,7 @@ if TYPE_CHECKING: from litellm.integrations.otel.model.payloads import ( GuardrailSpanData, LLMCallSpanData, + MCPToolCallSpanData, ProxyRequestSpanData, ServiceSpanData, ) @@ -54,6 +55,7 @@ if TYPE_CHECKING: class SpanRole(str, Enum): PROXY_REQUEST = "proxy_request" LLM_CALL = "llm_call" + MCP_TOOL_CALL = "mcp_tool_call" GUARDRAIL = "guardrail" DB_CALL = "db_call" SERVICE = "service" @@ -81,6 +83,11 @@ SPAN_REGISTRY: dict[SpanRole, SpanSpec] = { SpanRole.LLM_CALL: SpanSpec( SpanRole.LLM_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST ), + # The proxy is an MCP client to the upstream server it dispatches the tool + # call to, so this is a CLIENT span, sibling of the LLM call under the request. + SpanRole.MCP_TOOL_CALL: SpanSpec( + SpanRole.MCP_TOOL_CALL, LiteLLMSpanKind.CLIENT, parent=SpanRole.PROXY_REQUEST + ), SpanRole.GUARDRAIL: SpanSpec( SpanRole.GUARDRAIL, LiteLLMSpanKind.INTERNAL, parent=SpanRole.PROXY_REQUEST ), @@ -165,6 +172,11 @@ def llm_call_span_name(data: "LLMCallSpanData") -> str: return f"{data.operation.value} {model}".strip() +def mcp_tool_call_span_name(data: "MCPToolCallSpanData") -> str: + """``"{mcp.method.name} {tool}"`` e.g. ``"tools/call get-weather"`` (MCP semconv).""" + return f"{data.method} {data.tool_name}".strip() + + def proxy_request_span_name(data: "ProxyRequestSpanData") -> str: """``"{method} {route}"`` (HTTP semconv).""" return f"{data.http_method} {data.route}".strip() diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 5f05284212..648fe67114 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -511,6 +511,23 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_cached_tokens_metric"), ) + # Provider prompt-caching metrics + self.litellm_provider_cache_read_input_tokens_metric = self._counter_factory( + name="litellm_provider_cache_read_input_tokens_metric", + documentation="Total prompt/input tokens read from provider prompt cache (e.g. OpenAI/Anthropic/Gemini/Bedrock)", + labelnames=self.get_labels_for_metric( + "litellm_provider_cache_read_input_tokens_metric" + ), + ) + + self.litellm_provider_cache_creation_input_tokens_metric = self._counter_factory( + name="litellm_provider_cache_creation_input_tokens_metric", + documentation="Total prompt/input tokens written to provider prompt cache (e.g. Anthropic/Bedrock)", + labelnames=self.get_labels_for_metric( + "litellm_provider_cache_creation_input_tokens_metric" + ), + ) + # User and Team count metrics self.litellm_total_users_metric = self._gauge_factory( "litellm_total_users", @@ -1458,11 +1475,11 @@ class PrometheusLogger(CustomLogger): """ cache_hit = standard_logging_payload.get("cache_hit") - # Only track if cache_hit has a definite value (True or False) if cache_hit is None: - return - - if cache_hit is True: + # Historically these metrics only tracked LiteLLM caching. + # Provider prompt-caching metrics are still emitted below. + pass + elif cache_hit is True: # Increment cache hits counter PrometheusLogger._inc_labeled_counter( self, @@ -1493,6 +1510,51 @@ class PrometheusLogger(CustomLogger): label_context=label_context, ) + # Provider prompt caching metrics are independent of LiteLLM cache_hit. + provider_cache_read_tokens = 0 + provider_cache_creation_tokens = 0 + usage_obj = (standard_logging_payload.get("metadata", {}) or {}).get( + "usage_object" + ) + if isinstance(usage_obj, dict): + # Prefer explicit provider cache fields when available. + _read = usage_obj.get("cache_read_input_tokens") + _write = usage_obj.get("cache_creation_input_tokens") + + if isinstance(_read, int): + provider_cache_read_tokens = _read + if isinstance(_write, int): + provider_cache_creation_tokens = _write + + # Fallback to prompt_tokens_details.cached_tokens (common normalization point). + # Only fallback when the explicit field is genuinely absent (None). + if _read is None: + prompt_details = usage_obj.get("prompt_tokens_details") + if isinstance(prompt_details, dict): + cached_tokens = prompt_details.get("cached_tokens") + if isinstance(cached_tokens, int): + provider_cache_read_tokens = cached_tokens + + if provider_cache_read_tokens > 0: + PrometheusLogger._inc_labeled_counter( + self, + self.litellm_provider_cache_read_input_tokens_metric, + "litellm_provider_cache_read_input_tokens_metric", + enum_values, + label_context=label_context, + amount=float(provider_cache_read_tokens), + ) + + if provider_cache_creation_tokens > 0: + PrometheusLogger._inc_labeled_counter( + self, + self.litellm_provider_cache_creation_input_tokens_metric, + "litellm_provider_cache_creation_input_tokens_metric", + enum_values, + label_context=label_context, + amount=float(provider_cache_creation_tokens), + ) + async def _increment_remaining_budget_metrics( self, user_api_team: Optional[str], @@ -2628,7 +2690,7 @@ class PrometheusLogger(CustomLogger): Args: guardrail_name: Name of the guardrail latency_seconds: Execution latency in seconds - status: "success" or "error" + status: "success", "error", or "intervened" error_type: Type of error if any, None otherwise hook_type: "pre_call", "during_call", or "post_call" """ diff --git a/litellm/integrations/websearch_interception/ARCHITECTURE.md b/litellm/integrations/websearch_interception/ARCHITECTURE.md index 3aa0a1558d..ce7f01c5a2 100644 --- a/litellm/integrations/websearch_interception/ARCHITECTURE.md +++ b/litellm/integrations/websearch_interception/ARCHITECTURE.md @@ -244,6 +244,9 @@ search_tools: - search_tool_name: "my-tavily-tool" litellm_params: search_provider: "tavily" + - search_tool_name: "my-you-com-tool" + litellm_params: + search_provider: "you_com" ``` --- diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 2c1d92920a..95658d0876 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -87,6 +87,7 @@ class ExceptionCheckers: "is longer than the model's context length", "input tokens exceed the configured limit", "`inputs` tokens + `max_new_tokens` must be", + "exceeds the available context size", # llama.cpp/Lemonade "exceeds the maximum number of tokens allowed", # Gemini ] for substring in known_exception_substrings: @@ -891,12 +892,14 @@ def exception_type( # type: ignore # noqa: PLR0915 response=getattr(original_exception, "response", None), litellm_debug_info=extra_information, ) - elif "model's maximum context limit" in error_str: + elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str): exception_mapping_worked = True raise ContextWindowExceededError( message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}", model=model, llm_provider=custom_llm_provider, + response=getattr(original_exception, "response", None), + litellm_debug_info=extra_information, ) elif "token_quota_reached" in error_str: exception_mapping_worked = True diff --git a/litellm/litellm_core_utils/fallback_utils.py b/litellm/litellm_core_utils/fallback_utils.py index 52eb35663b..daacca85c8 100644 --- a/litellm/litellm_core_utils/fallback_utils.py +++ b/litellm/litellm_core_utils/fallback_utils.py @@ -47,8 +47,9 @@ async def async_completion_with_fallbacks(**kwargs): completion_kwargs = safe_deep_copy(base_kwargs) # Handle dictionary fallback configurations if isinstance(fallback, dict): - model = fallback.pop("model", original_model) - completion_kwargs.update(fallback) + fallback_config = safe_deep_copy(dict(fallback)) + model = fallback_config.pop("model", original_model) + completion_kwargs.update(fallback_config) else: model = fallback diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index ba6d438f16..de65ed9331 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -1,3 +1,4 @@ +import re from typing import Optional, Tuple from urllib.parse import urlparse @@ -71,6 +72,25 @@ def _is_azure_claude_model(model: str) -> bool: return False +_CLAUDE_PATTERN = re.compile(r"^claude-[a-z]+-\d+-\d+(?:-\d{8})?$", re.IGNORECASE) + + +def _matches_claude_model_pattern(model: str) -> bool: + """ + Check if a model string matches the Claude model naming pattern. + + Matches patterns like: + - claude-opus-4-7 + - claude-sonnet-4-6 + - claude-haiku-4-5 + - claude-opus-5-1-20270101 (with optional date suffix) + + This allows future Claude models to be routed to the Anthropic provider + without requiring updates to model_prices_and_context_window.json. + """ + return _CLAUDE_PATTERN.match(model) is not None + + def handle_cohere_chat_model_custom_llm_provider( model: str, custom_llm_provider: Optional[str] = None ) -> Tuple[str, Optional[str]]: @@ -353,6 +373,9 @@ def get_llm_provider( # noqa: PLR0915 elif endpoint == "https://api.lambda.ai/v1": custom_llm_provider = "lambda_ai" dynamic_api_key = get_secret_str("LAMBDA_API_KEY") + elif endpoint == "https://api.inceptionlabs.ai/v1": + custom_llm_provider = "inception" + dynamic_api_key = get_secret_str("INCEPTION_API_KEY") elif endpoint == "https://api.hyperbolic.xyz/v1": custom_llm_provider = "hyperbolic" dynamic_api_key = get_secret_str("HYPERBOLIC_API_KEY") @@ -398,6 +421,9 @@ def get_llm_provider( # noqa: PLR0915 custom_llm_provider = "anthropic_text" else: custom_llm_provider = "anthropic" + ## anthropic - pattern-based matching for future Claude models + elif _matches_claude_model_pattern(model): + custom_llm_provider = "anthropic" ## cohere elif model in litellm.cohere_models or model in litellm.cohere_embedding_models: custom_llm_provider = "cohere" @@ -633,6 +659,11 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 or get_secret_str("NVIDIA_RIVA_API_KEY") or get_secret_str("NVIDIA_NIM_API_KEY") ) + elif custom_llm_provider == "soniox": + api_base = ( + api_base or get_secret_str("SONIOX_API_BASE") or "https://api.soniox.com" + ) + dynamic_api_key = api_key or get_secret_str("SONIOX_API_KEY") elif custom_llm_provider == "cerebras": api_base = ( api_base or get_secret("CEREBRAS_API_BASE") or "https://api.cerebras.ai/v1" @@ -931,6 +962,13 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.LambdaAIChatConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "inception": + ( + api_base, + dynamic_api_key, + ) = litellm.InceptionChatConfig()._get_openai_compatible_provider_info( + api_base, api_key + ) elif custom_llm_provider == "hyperbolic": ( api_base, diff --git a/litellm/litellm_core_utils/get_supported_openai_params.py b/litellm/litellm_core_utils/get_supported_openai_params.py index b8cdc8210f..23b51faafc 100644 --- a/litellm/litellm_core_utils/get_supported_openai_params.py +++ b/litellm/litellm_core_utils/get_supported_openai_params.py @@ -22,9 +22,11 @@ def get_supported_openai_params( # noqa: PLR0915 ``` Args: - base_model: For Azure, the true underlying model (e.g. ``"azure/gpt-5.2"``) - when the deployment name differs. Used for model-type detection so that - non-standard deployment names route to the correct config. + base_model: An optional capability hint for deployments whose ``model`` + label isn't recognized on its own (e.g. an Azure deployment name, or a + friendly Bedrock alias). It is additive: the result is the union of the + params supported by ``model`` and by ``base_model``, so a hint can only + add capabilities, never strip ones the real model already supports. Returns: - List if custom_llm_provider is mapped @@ -52,7 +54,15 @@ def get_supported_openai_params( # noqa: PLR0915 provider_config = None if provider_config and request_type == "chat_completion": - return provider_config.get_supported_openai_params(model=base_model or model) + supported_params = provider_config.get_supported_openai_params(model=model) + if base_model and base_model != model: + base_model_params = provider_config.get_supported_openai_params( + model=base_model + ) + supported_params = list( + dict.fromkeys([*supported_params, *base_model_params]) + ) + return supported_params if custom_llm_provider == "bedrock": return litellm.AmazonConverseConfig().get_supported_openai_params(model=model) @@ -331,6 +341,11 @@ def get_supported_openai_params( # noqa: PLR0915 return ElevenLabsAudioTranscriptionConfig().get_supported_openai_params( model=model ) + elif custom_llm_provider == "soniox": + if request_type == "transcription": + return litellm.SonioxAudioTranscriptionConfig().get_supported_openai_params( + model=model + ) elif custom_llm_provider in litellm._custom_providers: if request_type == "chat_completion": provider_config = litellm.ProviderConfigManager.get_provider_chat_config( diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index c127b3873a..f20b66790c 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -3503,7 +3503,9 @@ class Logging(LiteLLMLoggingBaseClass): else: return None - def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelResponse: + def _handle_anthropic_messages_response_logging( + self, result: Any + ) -> Union[ModelResponse, ResponsesAPIResponse]: """ Handles logging for Anthropic messages responses. @@ -3522,6 +3524,15 @@ class Logging(LiteLLMLoggingBaseClass): return result elif isinstance(result, ModelResponse): return result + elif isinstance( + result, + (ResponseCompletedEvent, ResponseIncompleteEvent, ResponseFailedEvent), + ): + # anthropic_messages() can route to OpenAI Responses API; in that path + # the assembled streaming result is one of these terminal events rather than + # a ModelResponse. Return the inner response so downstream handlers + # (_transform_usage_objects, normalize_logging_result) can process it. + return result.response httpx_response = self.model_call_details.get("httpx_response", None) if httpx_response and isinstance(httpx_response, httpx.Response): @@ -5300,8 +5311,12 @@ class StandardLoggingPayloadSetup: tb_lines[:MAXIMUM_TRACEBACK_LINES_TO_LOG] ) # Limit to first 100 lines - # Get additional error details - error_message = str(original_exception) + explicit_message = getattr(original_exception, "message", None) + error_message = ( + explicit_message + if isinstance(explicit_message, str) and explicit_message + else str(original_exception) + ) return StandardLoggingPayloadErrorInformation( error_code=error_status, diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index 882561ed2e..f39c942f90 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -34,6 +34,14 @@ _IMAGE_RESPONSE_CALL_TYPES = frozenset( _VALID_DATA_RESIDENCIES = frozenset(r.value for r in DataResidency) +def _get_token_detail_value(details: object, key: str) -> Optional[int]: + if isinstance(details, dict): + value = details.get(key) + else: + value = getattr(details, key, None) + return value if isinstance(value, int) else None + + def _is_above_128k(tokens: float) -> bool: if tokens > 128000: return True @@ -870,17 +878,47 @@ def calculate_image_response_cost_from_usage( cached_tokens=0, ) + output_tokens_details = getattr(usage, "completion_tokens_details", None) + if output_tokens_details is None: + output_tokens_details = getattr(usage, "output_tokens_details", None) + + if output_tokens_details is None: + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ) + else: + text_tokens = _get_token_detail_value(output_tokens_details, "text_tokens") or 0 + image_tokens = ( + _get_token_detail_value(output_tokens_details, "image_tokens") or 0 + ) + audio_tokens = ( + _get_token_detail_value(output_tokens_details, "audio_tokens") or 0 + ) + reasoning_tokens = ( + _get_token_detail_value(output_tokens_details, "reasoning_tokens") or 0 + ) + known_output_tokens = ( + text_tokens + image_tokens + audio_tokens + reasoning_tokens + ) + if completion_tokens > known_output_tokens: + text_tokens += completion_tokens - known_output_tokens + + completion_tokens_details = CompletionTokensDetailsWrapper( + text_tokens=text_tokens, + image_tokens=image_tokens, + reasoning_tokens=reasoning_tokens, + audio_tokens=audio_tokens, + ) + normalized_usage = Usage( prompt_tokens=prompt_tokens, completion_tokens=completion_tokens, total_tokens=total_tokens, prompt_tokens_details=prompt_tokens_details, - completion_tokens_details=CompletionTokensDetailsWrapper( - text_tokens=0, - image_tokens=completion_tokens, - reasoning_tokens=0, - audio_tokens=0, - ), + completion_tokens_details=completion_tokens_details, ) prompt_cost, completion_cost = generic_cost_per_token( diff --git a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py index 5fd42fe0d3..2547fd4d8c 100644 --- a/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py +++ b/litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py @@ -144,6 +144,19 @@ async def convert_to_streaming_response_async(response_object: Optional[dict] = choice_list: List[StreamingChoices] = [] + if not response_object.get("choices"): + from litellm.exceptions import APIError + + raise APIError( + status_code=500, + message=( + "LiteLLM: provider returned a response with no 'choices'. " + f"Raw keys: {list(response_object.keys())}" + ), + llm_provider="", + model="", + ) + for idx, choice in enumerate(response_object["choices"]): if ( choice["message"].get("tool_calls", None) is not None @@ -213,6 +226,20 @@ def convert_to_streaming_response(response_object: Optional[dict] = None): model_response_object = ModelResponseStream() choice_list: List[StreamingChoices] = [] + + if not response_object.get("choices"): + from litellm.exceptions import APIError + + raise APIError( + status_code=500, + message=( + "LiteLLM: provider returned a response with no 'choices'. " + f"Raw keys: {list(response_object.keys())}" + ), + llm_provider="", + model="", + ) + for idx, choice in enumerate(response_object["choices"]): delta = Delta(**choice["message"]) finish_reason = choice.get("finish_reason", None) @@ -536,9 +563,20 @@ def convert_to_model_response_object( # noqa: PLR0915 return convert_to_streaming_response(response_object=response_object) choice_list: List[Choices] = [] - assert response_object["choices"] is not None and isinstance( + if not response_object.get("choices") or not isinstance( response_object["choices"], Iterable - ) + ): + from litellm.exceptions import APIError + + raise APIError( + status_code=500, + message=( + "LiteLLM: provider returned a response with no 'choices'. " + f"Raw keys: {list(response_object.keys())}" + ), + llm_provider="", + model="", + ) for idx, choice in enumerate(response_object["choices"]): ## HANDLE JSON MODE - anthropic returns single function call] @@ -816,7 +854,12 @@ def convert_to_model_response_object( # noqa: PLR0915 model_response_object.results = response_object["results"] return model_response_object - except Exception: + except Exception as e: + from litellm.exceptions import APIError + + if isinstance(e, APIError): + raise + received_args = dict( response_object=response_object, model_response_object=model_response_object, diff --git a/litellm/litellm_core_utils/logging_worker.py b/litellm/litellm_core_utils/logging_worker.py index 3db3700ee0..294ba8e5de 100644 --- a/litellm/litellm_core_utils/logging_worker.py +++ b/litellm/litellm_core_utils/logging_worker.py @@ -3,6 +3,7 @@ import asyncio import contextvars +import logging from typing import Coroutine, Optional import atexit from typing_extensions import TypedDict @@ -494,31 +495,43 @@ class LoggingWorker: processed = 0 start_time = loop.time() - while not self._queue.empty() and processed < MAX_ITERATIONS_TO_CLEAR_QUEUE: - if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE: - self._safe_log( - "warning", - f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush", - ) - break + # logging.raiseExceptions is a process-wide global; scope the + # suppression to just the drain loop, where shutdown callbacks may + # log to already-closed handler streams, so other threads keep their + # logging error reporting for as little of the window as possible. + previous_raise_exceptions = logging.raiseExceptions + logging.raiseExceptions = False + try: + while ( + not self._queue.empty() + and processed < MAX_ITERATIONS_TO_CLEAR_QUEUE + ): + if loop.time() - start_time >= MAX_TIME_TO_CLEAR_QUEUE: + self._safe_log( + "warning", + f"[LoggingWorker] atexit: Reached time limit ({MAX_TIME_TO_CLEAR_QUEUE}s), stopping flush", + ) + break - try: - task = self._queue.get_nowait() - except asyncio.QueueEmpty: - break + try: + task = self._queue.get_nowait() + except asyncio.QueueEmpty: + break - # Run the coroutine synchronously in new loop - # Note: We run the coroutine directly, not via create_task, - # since we're in a new event loop context - try: - loop.run_until_complete(task["coroutine"]) - processed += 1 - except Exception: - # Silent failure to not break user's program - pass - finally: - # Clear reference to prevent memory leaks - task = None + # Run the coroutine synchronously in new loop + # Note: We run the coroutine directly, not via create_task, + # since we're in a new event loop context + try: + loop.run_until_complete(task["coroutine"]) + processed += 1 + except Exception: + # Silent failure to not break user's program + pass + finally: + # Clear reference to prevent memory leaks + task = None + finally: + logging.raiseExceptions = previous_raise_exceptions self._safe_log( "info", diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index b44d21368f..fe34731759 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -850,7 +850,47 @@ def extract_file_data(file_data: FileTypes) -> ExtractedFileData: # --------------------------------------------------------------------------- -def unpack_defs(schema: dict, defs: dict) -> None: +def _estimate_json_bytes(obj: Any) -> int: + """Estimate the JSON-serialised byte size of ``obj`` without materialising + JSON. Walks iteratively (no recursion stack risk). + + String length is read via ``len()`` (O(1) on Python ``str``) so a target + containing a 100MB description costs ~one walk step, not a 100MB + serialisation. Escape sequences are not counted exactly, so this is an + approximation -- but always within a small constant factor of the real + serialised size, which is what a schema-bomb budget needs. + """ + total = 0 + stack: list = [obj] + while stack: + x = stack.pop() + if isinstance(x, dict): + total += 2 # `{}` + for k, v in x.items(): + total += len(str(k)) + 4 # `"k":,` + stack.append(v) + elif isinstance(x, list): + total += 2 # `[]` + total += max(0, len(x) - 1) # commas between items + stack.extend(x) + elif isinstance(x, str): + total += len(x) + 2 + elif isinstance(x, bool): # bool subclasses int -- check first + total += 4 if x else 5 + elif x is None: + total += 4 + elif isinstance(x, (int, float)): + total += 24 # generous upper bound for stringified numbers + else: + total += 24 + return total + + +def unpack_defs( + schema: dict, + defs: dict, + max_inlined_bytes: Optional[int] = None, +) -> None: """Expand *all* ``$ref`` entries pointing into ``$defs`` / ``definitions``. This utility walks the entire schema tree (dicts and lists) so it naturally @@ -860,6 +900,15 @@ def unpack_defs(schema: dict, defs: dict) -> None: It mutates *schema* in-place and does **not** return anything. The helper keeps memory overhead low by resolving nodes as it encounters them rather than materialising a fully dereferenced copy first. + + ``max_inlined_bytes`` caps the cumulative JSON-byte size of every target + that has been inlined and is checked *before* each ``copy.deepcopy``, so + an oversized expansion is rejected without first materialising it. A byte + bound is the universal measure of expansion -- it simultaneously caps + ref-count fan-out, node-count amplification, and scalar-byte amplification + (a target containing a large string, ``const``, or ``enum`` entry). + Defaults to ``None`` (unbounded) so existing callers are unaffected; + raises ``ValueError`` on overflow. """ import copy @@ -879,6 +928,7 @@ def unpack_defs(schema: dict, defs: dict) -> None: queue: deque[ tuple[Any, Union[dict, list, None], Union[str, int, None], dict, set] ] = deque([(schema, None, None, root_defs, set())]) + inlined_bytes = 0 while queue: node, parent, key, active_defs, ref_chain = queue.popleft() @@ -899,6 +949,16 @@ def unpack_defs(schema: dict, defs: dict) -> None: if target_schema is None: continue + if max_inlined_bytes is not None: + inlined_bytes += _estimate_json_bytes(target_schema) + if inlined_bytes > max_inlined_bytes: + raise ValueError( + f"unpack_defs: inlined schema exceeded the " + f"{max_inlined_bytes:,}-byte budget. Refusing to " + f"deep-copy further to prevent schema-bomb " + f"resource exhaustion." + ) + # Merge defs from the target to capture nested definitions child_defs = { **active_defs, @@ -946,6 +1006,61 @@ def unpack_defs(schema: dict, defs: dict) -> None: queue.append((item, node, idx, active_defs, ref_chain)) +def _has_legacy_defs(schema: object) -> bool: + if not isinstance(schema, dict): + return False + components = schema.get("components") + return "definitions" in schema or ( + isinstance(components, dict) and isinstance(components.get("schemas"), dict) + ) + + +# Schema-bomb budget for ``unpack_legacy_defs``: cap the cumulative JSON-byte +# size of every inlined target. A byte cap is the universal measure of +# expansion -- it simultaneously bounds ref-count fan-out, node-count +# amplification, and scalar-byte amplification (large ``description`` / +# ``const`` / ``enum`` values). Real-world MCP / OpenAPI-derived tool schemas +# inline well under 1MB; 10MB sits two orders of magnitude above that, well +# below memory-pressure territory, and rejects request-supplied bombs before +# the proxy materialises them. +_LEGACY_DEFS_MAX_INLINED_BYTES = 10_000_000 + + +def unpack_legacy_defs( + schema: dict, + *, + copy: bool = False, + max_inlined_bytes: int = _LEGACY_DEFS_MAX_INLINED_BYTES, +) -> dict: + """Inline ``$ref``s backed by draft-04 ``definitions`` / OpenAPI + ``components.schemas``. ``$defs`` is left untouched. + + Anthropic and Fireworks tool-schema resolvers only recognise ``$defs``; + legacy / OpenAPI def blocks are otherwise silently dropped and leave + dangling pointers. See https://github.com/BerriAI/litellm/issues/26692. + + Mutates ``schema`` in place and returns it. Pass ``copy=True`` to deep-copy + first (only when there is actually work to do). ``max_inlined_bytes`` + bounds the cumulative JSON-byte size of inlined targets so request-supplied + schemas cannot expand into a schema-bomb before reaching the upstream + provider -- raises ``ValueError`` on overflow. + """ + if not _has_legacy_defs(schema): + return schema + if copy: + import copy as _copy + + schema = _copy.deepcopy(schema) + # On key collision, ``definitions`` wins over ``components.schemas`` -- + # ``unpack_defs`` keys refs by last path segment so a single name can only + # resolve to one body, and ``definitions`` is the JSON-Schema-native + # namespace. + defs = schema.pop("components", {}).get("schemas") or {} + defs.update(schema.pop("definitions", None) or {}) + unpack_defs(schema, defs, max_inlined_bytes=max_inlined_bytes) + return schema + + def _get_image_mime_type_from_url(url: str) -> Optional[str]: """ Get mime type for common image URLs diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 46e9b43a42..1460dbaf0a 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1670,15 +1670,15 @@ def convert_to_gemini_tool_call_result( # noqa: PLR0915 if gemini_call_id: _function_response["id"] = gemini_call_id - # Create part with function_response, and optionally inline_data for images (Computer Use) _part: VertexPartType = {"function_response": _function_response} - # For Computer Use, if we have images/files, we need separate parts: - # - One part with function_response - # - One part per inline_data item - # Gemini's PartType is a oneof, so we can't have both in the same part + # For multimodal function responses, Gemini expects media parts nested + # inside functionResponse.parts instead of sibling content parts. if inline_data_list: - return [_part] + [{"inline_data": d} for d in inline_data_list] + _function_response["parts"] = [ + {"inline_data": inline_data} for inline_data in inline_data_list + ] + return [_part] return _part diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 33bb6d7d2e..772f058d9b 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -92,8 +92,27 @@ class RealTimeStreaming: # Track whether we have already sent the guardrail turn-detection update # that disables provider auto-response for transcription guardrails. self._guardrail_turn_detection_update_sent: bool = False + # Deferred Gemini Live setup: Pipecat may stream audio before session.update. + # Buffer client audio until the backend acknowledges setup (setupComplete). + self._backend_setup_complete: bool = ( + provider_config is None or provider_config.requires_session_configuration() + ) + self._flushing_pending_messages_until_setup: bool = False + self._pending_messages_until_setup: List[str] = [] + self._pending_messages_byte_total: int = 0 + + # Per-connection caps for pre-setup audio frames (message count + total bytes). + _MAX_BUFFERED_MESSAGES: int = 200 + _MAX_BUFFERED_BYTES: int = 10 * 1024 * 1024 # 10 MB _SESSION_EVENT_TYPES = frozenset(["session.created", "session.updated"]) + _CLIENT_AUDIO_BUFFER_TYPES = frozenset( + [ + "input_audio_buffer.append", + "input_audio_buffer.commit", + "input_audio_buffer.clear", + ] + ) _AUDIO_FORMAT_MAP: Dict[str, Dict[str, Any]] = { "pcm16": {"type": "audio/pcm", "rate": 24000}, "g711_ulaw": {"type": "audio/G711-ulaw", "rate": 8000}, @@ -285,6 +304,86 @@ class RealTimeStreaming: await self.backend_ws.send(message) # type: ignore[union-attr, attr-defined] return True + def _uses_deferred_backend_setup(self) -> bool: + """True when setup is deferred until the client's first session.update.""" + if self.provider_config is None: + return False + return not self.provider_config.requires_session_configuration() + + def _should_buffer_client_message_until_setup(self, message: str) -> bool: + if not self._uses_deferred_backend_setup(): + return False + if ( + self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + ): + return False + try: + msg_obj = json.loads(message) + except (json.JSONDecodeError, TypeError): + return False + return msg_obj.get("type") in RealTimeStreaming._CLIENT_AUDIO_BUFFER_TYPES + + def _buffer_pending_message_until_setup(self, message: str) -> None: + msg_bytes = len(message.encode("utf-8")) + if ( + len(self._pending_messages_until_setup) + < RealTimeStreaming._MAX_BUFFERED_MESSAGES + and self._pending_messages_byte_total + msg_bytes + <= RealTimeStreaming._MAX_BUFFERED_BYTES + ): + self._pending_messages_until_setup.append(message) + self._pending_messages_byte_total += msg_bytes + else: + verbose_logger.warning( + "Pre-setup buffer full (%d messages / %d bytes); dropping frame", + len(self._pending_messages_until_setup), + self._pending_messages_byte_total, + ) + + async def _flush_pending_messages_until_setup(self) -> bool: + pending = self._pending_messages_until_setup + self._pending_messages_until_setup = [] + self._pending_messages_byte_total = 0 + for idx, message in enumerate(pending): + try: + await self._send_to_backend(message) + except Exception as e: + unsent = pending[idx:] + self._pending_messages_until_setup = ( + unsent + self._pending_messages_until_setup + ) + self._pending_messages_byte_total = sum( + len(msg.encode("utf-8")) + for msg in self._pending_messages_until_setup + ) + verbose_logger.debug( + "Failed to flush buffered client message after setup: %s " + "(%d buffered message(s) retained)", + e, + len(unsent), + ) + return False + return True + + async def _send_event_to_client(self, event: Any, event_str: str) -> bool: + if self._client_wants_beta and isinstance(event, dict): + try: + translated = self._translate_event_to_beta(event) + if translated is None: + return False + await self.websocket.send_text(json.dumps(translated)) + return True + except Exception as e: + verbose_logger.warning( + "Failed to translate %s to beta protocol, forwarding " + "untranslated event to client: %s", + event.get("type"), + e, + ) + await self.websocket.send_text(event_str) + return True + def _cache_session_configuration_request(self, transformed_message: str) -> None: """Store setup payload once sent to backend. @@ -547,6 +646,19 @@ class RealTimeStreaming: isinstance(event, dict) and event.get("type") == "session.created" ) if is_session_created_event: + if ( + self._uses_deferred_backend_setup() + and not self._backend_setup_complete + ): + self._backend_setup_complete = True + self._flushing_pending_messages_until_setup = True + try: + while self._pending_messages_until_setup: + flushed = await self._flush_pending_messages_until_setup() + if not flushed: + break + finally: + self._flushing_pending_messages_until_setup = False if self._session_created_sent_to_client: # A synthetic session.created (with placeholder defaults) was # already forwarded to the client when we connected. The @@ -569,7 +681,7 @@ class RealTimeStreaming: ## update if a prior attempt was dropped by the provider transform. if is_session_created_event and self._has_audio_transcription_guardrails(): self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) await self._maybe_send_guardrail_turn_detection_update() continue ## GUARDRAIL: run on transcription events in provider_config path too @@ -581,7 +693,7 @@ class RealTimeStreaming: transcript = event.get("transcript", "") self._collect_user_input_from_backend_event(cast(dict, event)) self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) blocked = await self.run_realtime_guardrails( cast(str, transcript), item_id=cast(Optional[str], event.get("item_id")), @@ -591,7 +703,7 @@ class RealTimeStreaming: continue ## LOGGING self.store_message(event_str) - await self.websocket.send_text(event_str) + await self._send_event_to_client(event, event_str) async def _handle_raw_backend_message(self, raw_response) -> bool: """Process a backend message without provider_config (raw path). @@ -880,6 +992,7 @@ class RealTimeStreaming: ## GUARDRAIL: intercept conversation.item.create for text-based injection. guardrail_turn_detection_injected = False + msg_type: Optional[str] = None try: msg_obj = json.loads(message) msg_type = msg_obj.get("type") @@ -1081,6 +1194,29 @@ class RealTimeStreaming: # actually forward to the backend. self.store_input(message=message) + if self._should_buffer_client_message_until_setup(message): + self._buffer_pending_message_until_setup(message) + continue + + if self._pending_messages_until_setup: + should_send_setup_before_buffered_messages = ( + not self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + and msg_type == "session.update" + ) + if not should_send_setup_before_buffered_messages: + self._buffer_pending_message_until_setup(message) + if ( + self._backend_setup_complete + and not self._flushing_pending_messages_until_setup + ): + await self._flush_pending_messages_until_setup() + continue + + if self._flushing_pending_messages_until_setup: + self._buffer_pending_message_until_setup(message) + continue + ## FORWARD TO BACKEND # Only mark the guardrail turn_detection update as sent after the # backend actually accepted the message. Setting the flag earlier diff --git a/litellm/litellm_core_utils/safe_json_dumps.py b/litellm/litellm_core_utils/safe_json_dumps.py index 051aa2f27a..154306d01b 100644 --- a/litellm/litellm_core_utils/safe_json_dumps.py +++ b/litellm/litellm_core_utils/safe_json_dumps.py @@ -6,10 +6,16 @@ from pydantic import BaseModel from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH +def strip_null_bytes(value: str) -> str: + """Strip NUL bytes, which PostgreSQL text/jsonb columns reject (error 22P05).""" + return value.replace("\x00", "") + + def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: """ Recursively serialize data while detecting circular references. If a circular reference is detected then a marker string is returned. + NUL bytes are stripped from strings to prevent PostgreSQL 22P05 errors. """ def _serialize(obj: Any, seen: set, depth: int) -> Any: @@ -17,7 +23,9 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: if depth > max_depth: return "MaxDepthExceeded" # Base-case: if it is a primitive, simply return it. - if isinstance(obj, (str, int, float, bool, type(None))): + if isinstance(obj, str): + return strip_null_bytes(obj) + if isinstance(obj, (int, float, bool, type(None))): return obj # Check for circular reference. if id(obj) in seen: @@ -28,7 +36,7 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: result = {} for k, v in obj.items(): if isinstance(k, (str)): - result[k] = _serialize(v, seen, depth + 1) + result[strip_null_bytes(k)] = _serialize(v, seen, depth + 1) seen.remove(id(obj)) return result elif isinstance(obj, list): @@ -51,7 +59,7 @@ def safe_dumps(data: Any, max_depth: int = DEFAULT_MAX_RECURSE_DEPTH) -> str: else: # Fall back to string conversion for non-serializable objects. try: - return str(obj) + return strip_null_bytes(str(obj)) except Exception: return "Unserializable Object" diff --git a/litellm/litellm_core_utils/secret_redaction.py b/litellm/litellm_core_utils/secret_redaction.py index 5c4e3e3dac..b526068589 100644 --- a/litellm/litellm_core_utils/secret_redaction.py +++ b/litellm/litellm_core_utils/secret_redaction.py @@ -50,13 +50,15 @@ def _build_secret_patterns() -> "re.Pattern[str]": r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)", # Databricks personal access tokens r"dapi[0-9a-f]{32}", + # Module-level provider keys logged as litellm._key= + r"litellm\.[A-Za-z0-9_]*_key['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+", # ── Key-name-based redaction ── # Catches secrets inside dicts/config dumps by matching on the KEY name # regardless of what the value looks like. # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..." # private_key with PEM-aware value capture r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""", - r"(?:master_key|database_url|db_url|connection_string|" + r"(?:master_key|xai_key|database_url|db_url|connection_string|" r"signing_key|encryption_key|" r"auth_token|access_token|refresh_token|" r"slack_webhook_url|webhook_url|" diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 55042a733e..f3274151e5 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -1149,6 +1149,32 @@ class CustomStreamWrapper: completion_obj: Dict[str, Any] = {"content": ""} from litellm.types.utils import GenericStreamingChunk as GChunk + if ( + isinstance(chunk, ModelResponseStream) + and self.custom_llm_provider is not None + and self.custom_llm_provider in litellm._custom_providers + ): + _has_content = bool( + chunk.choices + and chunk.choices[0].delta is not None + and ( + chunk.choices[0].delta.content + or chunk.choices[0].delta.tool_calls + ) + ) + if self.received_finish_reason is not None: + if not _has_content: + raise StopIteration + if chunk.choices and chunk.choices[0].finish_reason: + self.received_finish_reason = chunk.choices[0].finish_reason + if not _has_content: + return None + # Strip finish_reason from the content chunk so it appears + # only on the trailing empty-delta chunk (OpenAI spec). + # finish_reason_handler() will emit the proper terminal chunk. + chunk.choices[0].finish_reason = None # type: ignore[assignment] + return chunk + if ( isinstance(chunk, dict) and generic_chunk_has_all_required_fields( diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 57609cfcd2..7949e150c2 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -29,6 +29,7 @@ from litellm.constants import ( RESPONSE_FORMAT_TOOL_NAME, ) from litellm.litellm_core_utils.core_helpers import map_finish_reason +from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_legacy_defs from litellm.llms.base_llm.base_utils import type_to_response_format_param from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.types.llms.anthropic import ( @@ -80,7 +81,6 @@ from litellm.types.utils import ( from litellm.utils import ( ModelResponse, Usage, - _supports_factory, add_dummy_tool, any_assistant_message_has_thinking_blocks, get_max_tokens, @@ -336,50 +336,6 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): v in model_lower for v in ("opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7") ) - @staticmethod - def _supports_model_capability(model: str, key: str) -> bool: - """Check a boolean capability ``key`` in the model map. - - Strips bedrock/vertex prefixes so a provider-routed Claude still - resolves to the Anthropic model-map entry. - """ - try: - if _supports_factory( - model=model, - custom_llm_provider="anthropic", - key=key, - ): - return True - except Exception: - pass - candidates = [model] - for prefix in ( - "bedrock/converse/", - "bedrock/invoke/", - "bedrock/", - "vertex_ai/", - ): - if model.startswith(prefix): - candidates.append(model[len(prefix) :]) - try: - from litellm.llms.bedrock.common_utils import BedrockModelInfo - - base = BedrockModelInfo.get_base_model(model) - if base: - candidates.append(base) - candidates.append(f"bedrock/{base}") - except Exception: - pass - try: - for cand in candidates: - if cand in litellm.model_cost and ( - litellm.model_cost[cand].get(key) is True - ): - return True - except Exception: - pass - return False - @staticmethod def _supports_effort_level(model: str, level: str) -> bool: """Check ``supports_{level}_reasoning_effort`` in the model map.""" @@ -680,6 +636,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): if "properties" not in _input_schema: _input_schema["properties"] = {} + # Inline legacy / OpenAPI $refs before the allow-list filter strips + # their backing def blocks (https://github.com/BerriAI/litellm/issues/26692). + _input_schema = unpack_legacy_defs(_input_schema, copy=True) + _allowed_properties = set(AnthropicInputSchema.__annotations__.keys()) input_schema_filtered = { k: v for k, v in _input_schema.items() if k in _allowed_properties @@ -913,7 +873,39 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): anthropic_tools = [] mcp_servers = [] for tool in tools: - if "input_schema" in tool: # assume in anthropic format + if tool.get("type") == "namespace": + # Namespace is a grouping container (e.g. codex's multi_agent_v1). + # Extract its nested tools and map them individually. + for nested in tool.get("tools") or []: + if "input_schema" in nested: + # Already in Anthropic format. + anthropic_tools.append(nested) + elif "function" not in nested and "name" in nested: + # Flat format: {type, name, description, parameters, ...}. + # Normalize to OpenAI-wrapped format before mapping. + wrapped = cast( + ChatCompletionToolParam, + { + "type": nested.get("type", "function"), + "function": { + k: v for k, v in nested.items() if k != "type" + }, + }, + ) + nested_tool, nested_mcp = self._map_tool_helper(wrapped) + if nested_tool is not None: + anthropic_tools.append(nested_tool) + if nested_mcp is not None: + mcp_servers.append(nested_mcp) + elif "function" in nested: + nested_tool, nested_mcp = self._map_tool_helper( + cast(ChatCompletionToolParam, nested) + ) + if nested_tool is not None: + anthropic_tools.append(nested_tool) + if nested_mcp is not None: + mcp_servers.append(nested_mcp) + elif "input_schema" in tool: # assume in anthropic format anthropic_tools.append(tool) else: # assume openai tool call new_tool, mcp_server_tool = self._map_tool_helper(tool) @@ -1973,6 +1965,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): # Remove internal LiteLLM parameters that should not be sent to Anthropic API optional_params.pop("is_vertex_request", None) + optional_params.pop("client_metadata", None) data = { "model": model, diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 31131d722a..3f002d73cb 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -272,19 +272,63 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) @staticmethod - def _is_adaptive_thinking_model(model: str) -> bool: - """Claude 4.6+ models use adaptive thinking with ``output_config.effort``.""" + def _supports_model_capability(model: str, key: str) -> bool: + """Check a boolean capability ``key`` in the model map. + + Strips bedrock/vertex prefixes so a provider-routed Claude still + resolves to the Anthropic model-map entry. + """ from litellm.utils import _supports_factory try: if _supports_factory( model=model, - custom_llm_provider=None, - key="supports_adaptive_thinking", + custom_llm_provider="anthropic", + key=key, ): return True except Exception: pass + candidates = [model] + for prefix in ( + "bedrock/converse/", + "bedrock/invoke/", + "bedrock/", + "vertex_ai/", + ): + if model.startswith(prefix): + candidates.append(model[len(prefix) :]) + try: + from litellm.llms.bedrock.common_utils import BedrockModelInfo + + base = BedrockModelInfo.get_base_model(model) + if base: + candidates.append(base) + candidates.append(f"bedrock/{base}") + except Exception: + pass + try: + for cand in candidates: + if cand in litellm.model_cost and ( + litellm.model_cost[cand].get(key) is True + ): + return True + except Exception: + pass + return False + + @staticmethod + def _is_adaptive_thinking_model(model: str) -> bool: + """Claude 4.6+ models use adaptive thinking with ``output_config.effort``. + + Driven by the ``supports_adaptive_thinking`` flag in the model map; the + 4.6/4.7 name checks remain only as a fallback for provider-routed ids + whose map entries predate the flag. + """ + if AnthropicModelInfo._supports_model_capability( + model, "supports_adaptive_thinking" + ): + return True return AnthropicModelInfo._is_claude_4_6_model( model ) or AnthropicModelInfo._is_claude_4_7_model(model) diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 02e0c56265..150f056dc8 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -1510,6 +1510,17 @@ class LiteLLMAnthropicMessagesAdapter: return "thinking", ChatCompletionThinkingBlock( type="thinking", thinking=thinking, signature=signature ) + # OpenAI-compatible reasoning backends (e.g. vLLM/SGLang reasoning + # parsers) populate ``reasoning_content`` without ``thinking_blocks``. + # ``Delta`` deletes the ``thinking_blocks`` attribute when unset, so the + # branch above is skipped entirely; open a ``thinking`` block here so the + # matching ``thinking_delta`` stream is not emitted into a text block. + elif isinstance(choice, StreamingChoices) and getattr( + choice.delta, "reasoning_content", None + ): + return "thinking", ChatCompletionThinkingBlock( + type="thinking", thinking="", signature="" + ) return "text", TextBlock(type="text", text="") diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 62eced8e6f..a3ac465c46 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -229,9 +229,20 @@ async def anthropic_messages( **kwargs, ) - # Extract modified parameters + # Extract modified parameters. Pop every named param of `anthropic_messages` + # that we may forward explicitly downstream, so we (a) honor pre-request hook + # overrides and (b) avoid duplicate-keyword conflicts when splatting `kwargs` + # into call sites that already pass these as named arguments. tools = request_kwargs.pop("tools", tools) stream = request_kwargs.pop("stream", stream) + metadata = request_kwargs.pop("metadata", metadata) + stop_sequences = request_kwargs.pop("stop_sequences", stop_sequences) + system = request_kwargs.pop("system", system) + temperature = request_kwargs.pop("temperature", temperature) + thinking = request_kwargs.pop("thinking", thinking) + tool_choice = request_kwargs.pop("tool_choice", tool_choice) + top_k = request_kwargs.pop("top_k", top_k) + top_p = request_kwargs.pop("top_p", top_p) # Propagate the provider derived inside pre-request hooks, if not already set. # The litellm_params dict may have been overwritten by **kwargs in # _execute_pre_request_hooks, so fall back to get_llm_provider() if needed. @@ -265,8 +276,8 @@ async def anthropic_messages( return short_circuit_response # Run registered MessagesInterceptors (e.g. advisor orchestration loop). - # api_key and api_base are explicit params (not in **kwargs) so pass them - # explicitly so interceptor sub-calls can route to the same backend. + # Named params on `anthropic_messages` are bound to locals, not `**kwargs`, + # so forward them explicitly — otherwise interceptor sub-calls drop them. for interceptor in get_messages_interceptors(): if interceptor.can_handle(tools, custom_llm_provider): return await interceptor.handle( @@ -278,6 +289,14 @@ async def anthropic_messages( custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, + metadata=metadata, + stop_sequences=stop_sequences, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + top_k=top_k, + top_p=top_p, **kwargs, ) diff --git a/litellm/llms/apiserpent/__init__.py b/litellm/llms/apiserpent/__init__.py new file mode 100644 index 0000000000..2edf992adc --- /dev/null +++ b/litellm/llms/apiserpent/__init__.py @@ -0,0 +1 @@ +"""APISerpent integration for LiteLLM.""" diff --git a/litellm/llms/apiserpent/search/__init__.py b/litellm/llms/apiserpent/search/__init__.py new file mode 100644 index 0000000000..4e9f88a2f1 --- /dev/null +++ b/litellm/llms/apiserpent/search/__init__.py @@ -0,0 +1,8 @@ +""" +APISerpent Search API module. +""" + +from litellm.llms.apiserpent.search.defaults import APISerpentSearchParams +from litellm.llms.apiserpent.search.transformation import APISerpentSearchConfig + +__all__ = ["APISerpentSearchConfig", "APISerpentSearchParams"] diff --git a/litellm/llms/apiserpent/search/defaults.py b/litellm/llms/apiserpent/search/defaults.py new file mode 100644 index 0000000000..219178587d --- /dev/null +++ b/litellm/llms/apiserpent/search/defaults.py @@ -0,0 +1,70 @@ +""" +Default parameter values and shared constants for APISerpent search. + +Single source of truth for the supported request parameters and their +package-level defaults. See https://apiserpent.com/docs. +""" + +from dataclasses import asdict, dataclass +from typing import Dict, Literal, Optional + +SearchEngine = Literal["google", "bing", "yahoo", "ddg"] +SafeSearch = Literal["off", "moderate", "strict"] +Freshness = Literal["h", "1h", "d", "1d", "7d", "w", "m", "1m", "y", "1y"] +ResponseFormat = Literal["full", "simple"] + +NUM_MIN = 1 +NUM_MIN_DEEP = 10 +NUM_MAX = 100 +PAGES_MIN = 1 +PAGES_MAX = 10 + + +@dataclass(frozen=True) +class APISerpentSearchParams: + """ + Supported APISerpent search parameters with package defaults. + + Fields defaulting to ``None`` are only sent when the caller provides them; + the rest are always sent so behavior is deterministic regardless of any + server-side defaults. + """ + + engine: SearchEngine = "google" + country: str = "us" + num: int = 10 + format: ResponseFormat = "full" + pages: Optional[int] = None + freshness: Optional[Freshness] = None + safe: Optional[SafeSearch] = None + language: Optional[str] = None + pixel_position: Optional[bool] = None + + def __post_init__(self) -> None: + # num's deep-search floor (NUM_MIN_DEEP) is endpoint-specific and enforced + # in the transform layer; here we only bound the absolute range. + if not NUM_MIN <= self.num <= NUM_MAX: + raise ValueError( + f"num must be between {NUM_MIN} and {NUM_MAX}, got {self.num}" + ) + if self.pages is not None and not PAGES_MIN <= self.pages <= PAGES_MAX: + raise ValueError( + f"pages must be between {PAGES_MIN} and {PAGES_MAX}, got {self.pages}" + ) + + def to_request_params(self) -> Dict: + """Return non-None fields as request params, booleans lowercased.""" + params: Dict = {} + for key, value in asdict(self).items(): + if value is None: + continue + params[key] = str(value).lower() if isinstance(value, bool) else value + return params + + @classmethod + def field_names(cls) -> set: + return set(cls.__dataclass_fields__.keys()) + + +QUICK_SEARCH_PATH = "/api/search/quick" +DEEP_SEARCH_PATH = "/api/search" diff --git a/litellm/llms/apiserpent/search/transformation.py b/litellm/llms/apiserpent/search/transformation.py new file mode 100644 index 0000000000..1eb7d34c87 --- /dev/null +++ b/litellm/llms/apiserpent/search/transformation.py @@ -0,0 +1,182 @@ +""" +Calls APISerpent's search endpoints to search Google, Bing, Yahoo, or DuckDuckGo. + +Two endpoints under one provider, selected via the ``deep`` boolean param: +- ``deep=False`` (default) -> quick search (/api/search/quick) +- ``deep=True`` -> deep search (/api/search) + +APISerpent API Reference: https://apiserpent.com/docs +""" + +from typing import Dict, List, Literal, Optional, Union, cast +from urllib.parse import urlencode + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.apiserpent.search.defaults import ( + DEEP_SEARCH_PATH, + NUM_MAX, + NUM_MIN, + NUM_MIN_DEEP, + QUICK_SEARCH_PATH, + APISerpentSearchParams, +) +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + +DEEP_SEARCH_PARAM = "deep" +APISERPENT_BASE = "https://apiserpent.com" +APISERPENT_PARAMS_KEY = "_apiserpent_params" + + +class APISerpentSearchConfig(BaseSearchConfig): + @staticmethod + def ui_friendly_name() -> str: + return "APISerpent" + + def get_http_method(self) -> Literal["GET", "POST"]: + return "GET" + + @staticmethod + def _is_deep_search(optional_params: dict) -> bool: + return bool(optional_params.get(DEEP_SEARCH_PARAM)) + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + api_key = api_key or get_secret_str("APISERPENT_API_KEY") + if not api_key: + raise ValueError( + "APISERPENT_API_KEY is not set. Set `APISERPENT_API_KEY` environment variable." + ) + headers["X-API-Key"] = api_key + headers["Content-Type"] = "application/json" + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Build the search URL. APISerpent uses GET, so the transformed request is + serialized into the query string. The endpoint path (quick vs deep) is + always applied; an ``api_base`` / ``APISERPENT_API_BASE`` override only + changes the host. The ``endswith`` guard keeps this idempotent, since the + handler re-invokes this method with the already-resolved URL as api_base. + """ + base = ( + api_base or get_secret_str("APISERPENT_API_BASE") or APISERPENT_BASE + ).rstrip("/") + path = ( + DEEP_SEARCH_PATH + if self._is_deep_search(optional_params) + else QUICK_SEARCH_PATH + ) + if not base.endswith(path): + base = f"{base}{path}" + + if data and isinstance(data, dict) and APISERPENT_PARAMS_KEY in data: + query_string = urlencode(data[APISERPENT_PARAMS_KEY], doseq=True) + return f"{base}?{query_string}" + + return base + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform a unified search request into APISerpent query params. + + Unified spec mappings: + - query -> q + - max_results -> num (clamped to the endpoint's valid range) + - country -> country (lowercased) + - search_domain_filter -> site: clauses appended to q + + All other APISerpent params (engine, language, freshness, safe, pages, + format, pixel_position) pass through, defaulting via APISerpentSearchParams. + """ + if isinstance(query, list): + query = " ".join(query) + + is_deep = self._is_deep_search(optional_params) + + overrides: Dict = {} + if "max_results" in optional_params: + num_min = NUM_MIN_DEEP if is_deep else NUM_MIN + overrides["num"] = max( + num_min, min(optional_params["max_results"], NUM_MAX) + ) + if "country" in optional_params: + overrides["country"] = cast(str, optional_params["country"]).lower() + + for param, value in optional_params.items(): + if param in APISerpentSearchParams.field_names() and param not in overrides: + overrides[param] = value + + params = {**APISerpentSearchParams(**overrides).to_request_params(), "q": query} + + if "search_domain_filter" in optional_params: + domains = optional_params["search_domain_filter"] + if isinstance(domains, list) and len(domains) > 0: + params["q"] = self._append_domain_filters(str(params["q"]), domains) + + return {APISERPENT_PARAMS_KEY: params} + + @staticmethod + def _append_domain_filters(query: str, domains: List[str]) -> str: + domain_clauses = " OR ".join(f"site:{domain}" for domain in domains) + return f"({query}) ({domain_clauses})" + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: Optional[LiteLLMLoggingObj], + **kwargs, + ) -> SearchResponse: + """ + Transform APISerpent response to the unified SearchResponse format. + + Full format nests results under ``results.organic[]``; simple format + returns a flat ``results[]`` array. Both expose title/url/snippet. + """ + response_json = raw_response.json() + + raw_results = response_json.get("results") or {} + organic = ( + raw_results.get("organic", []) + if isinstance(raw_results, dict) + else raw_results + ) + + results: List[SearchResult] = [] + for result in organic: + results.append( + SearchResult( + title=result.get("title", ""), + url=result.get("url", ""), + snippet=result.get("snippet", ""), + date=result.get("date"), + last_updated=None, + ) + ) + + return SearchResponse( + results=results, + object="search", + ) diff --git a/litellm/llms/azure/image_edit/transformation.py b/litellm/llms/azure/image_edit/transformation.py index a450ee0b21..72f1eef36c 100644 --- a/litellm/llms/azure/image_edit/transformation.py +++ b/litellm/llms/azure/image_edit/transformation.py @@ -97,8 +97,15 @@ class AzureImageEditConfig(OpenAIImageEditConfig): ) original_url = httpx.URL(api_base) - # Extract api_version or use default - api_version = cast(Optional[str], litellm_params.get("api_version")) + # Resolve api_version: litellm_params > litellm.api_version > AZURE_API_VERSION env > default. + # Mirrors the fallback chain used by the Azure chat path in common_utils.py, + # so callers that set a global / env api_version don't get an unversioned URL. + api_version = ( + cast(Optional[str], litellm_params.get("api_version")) + or litellm.api_version + or get_secret_str("AZURE_API_VERSION") + or litellm.AZURE_DEFAULT_API_VERSION + ) # Create a new dictionary with existing params query_params = dict(original_url.params) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index 529ec71c53..008a8a766e 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -1,4 +1,5 @@ import enum +import re from typing import Any, List, Optional, Tuple, cast from urllib.parse import urlparse @@ -275,21 +276,25 @@ class AzureAIStudioConfig(OpenAIConfig): should_drop_params = litellm_params.get("drop_params") or litellm.drop_params error_text = e.response.text - if should_drop_params and "Extra inputs are not permitted" in error_text: + if "Extra inputs are not permitted" in error_text: + if should_drop_params or self._error_has_tool_level_extra_fields( + error_text + ): + return True + if "unknown field: parameter index is not a valid field" in error_text: return True - elif ( - "unknown field: parameter index is not a valid field" in error_text - ): # remove index from tool calls - return True - elif ( + if ( AzureFoundryErrorStrings.SET_EXTRA_PARAMETERS_TO_PASS_THROUGH.value in error_text - ): # remove extra-parameters from tool calls + ): return True return super().should_retry_llm_api_inside_llm_translation_on_http_error( e=e, litellm_params=litellm_params ) + def _error_has_tool_level_extra_fields(self, error_text: str) -> bool: + return bool(re.search(r"tools\[\d+\]\.", error_text)) + @property def max_retry_on_unprocessable_entity_error(self) -> int: return 2 @@ -297,9 +302,10 @@ class AzureAIStudioConfig(OpenAIConfig): def transform_request_on_unprocessable_entity_error( self, e: httpx.HTTPStatusError, request_data: dict ) -> dict: + error_text = e.response.text _messages = cast(Optional[List[AllMessageValues]], request_data.get("messages")) if ( - "unknown field: parameter index is not a valid field" in e.response.text + "unknown field: parameter index is not a valid field" in error_text and _messages is not None ): litellm.remove_index_from_tool_calls( @@ -307,14 +313,31 @@ class AzureAIStudioConfig(OpenAIConfig): ) elif ( AzureFoundryErrorStrings.SET_EXTRA_PARAMETERS_TO_PASS_THROUGH.value - in e.response.text + in error_text ): request_data = self._drop_extra_params_from_request_data( - request_data, e.response.text + request_data, error_text ) + if ( + "Extra inputs are not permitted" in error_text + and self._error_has_tool_level_extra_fields(error_text) + ): + request_data = self._drop_tool_level_extra_fields(request_data, error_text) data = drop_params_from_unprocessable_entity_error(e=e, data=request_data) return data + def _drop_tool_level_extra_fields( + self, request_data: dict, error_text: str + ) -> dict: + fields_to_drop = set(re.findall(r"tools\[\d+\]\.([\w-]+)", error_text)) + tools = request_data.get("tools") + if fields_to_drop and isinstance(tools, list): + for tool in tools: + if isinstance(tool, dict): + for field in fields_to_drop: + tool.pop(field, None) + return request_data + def _drop_extra_params_from_request_data( self, request_data: dict, error_text: str ) -> dict: @@ -332,9 +355,6 @@ class AzureAIStudioConfig(OpenAIConfig): Error text looks like this" "Extra parameters ['stream_options', 'extra-parameters'] are not allowed when extra-parameters is not set or set to be 'error'. """ - import re - - # Extract parameters within square brackets match = re.search(r"\[(.*?)\]", error_text) if not match: return [] diff --git a/litellm/llms/bedrock/base_aws_llm.py b/litellm/llms/bedrock/base_aws_llm.py index b659c1b0a0..b1b0682938 100644 --- a/litellm/llms/bedrock/base_aws_llm.py +++ b/litellm/llms/bedrock/base_aws_llm.py @@ -1534,10 +1534,9 @@ class BaseAWSLLM: ) sigv4 = SigV4Auth(credentials, service_name, aws_region_name) - if headers is not None: + headers = headers or {} + if not any(header_name.lower() == "content-type" for header_name in headers): headers = {"Content-Type": "application/json", **headers} - else: - headers = {"Content-Type": "application/json"} aws_signature_headers = self._filter_headers_for_aws_signature(headers) request = AWSRequest( diff --git a/litellm/llms/bedrock/files/transformation.py b/litellm/llms/bedrock/files/transformation.py index 6669363093..cec2e934af 100644 --- a/litellm/llms/bedrock/files/transformation.py +++ b/litellm/llms/bedrock/files/transformation.py @@ -233,6 +233,259 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # example; add others here as they adopt the same schema. CONVERSE_INVOKE_PROVIDERS = ("nova",) + # OpenAI batch URL that signals an embedding request. Per OpenAI Batch API + # spec, every JSONL record carries a `url` field; we use it as the + # authoritative signal to route the line to the embedding code path + # instead of inferring from the presence of `input` vs `messages`. + OPENAI_EMBEDDINGS_URL = "/v1/embeddings" + + @staticmethod + def _is_embedding_record(openai_jsonl_record: Dict[str, Any]) -> bool: + """ + Decide whether an OpenAI batch JSONL line is an embedding request. + + Precedence (strict - any explicit `url` short-circuits): + 1. `url == "/v1/embeddings"` -> embedding. Authoritative per the + OpenAI Batch API spec. + 2. Any other non-empty `url` (e.g. `/v1/chat/completions`) -> NOT + embedding. We trust the caller's explicit signal even if the + body would otherwise suggest embedding; misrouting a chat + record into the embedding transformer would corrupt the + modelInput, while a chat-shaped body sent to the chat path + either succeeds or fails cleanly inside that transformer. + 3. `url` missing/empty -> fall back to body shape. Requires + `input` present AND `messages` absent so a malformed record + carrying both keys routes to the chat path (safer default: + Anthropic transforms ignore unknown top-level keys, whereas + the embedding transformer would silently drop the messages). + """ + url = openai_jsonl_record.get("url") + if url == BedrockFilesConfig.OPENAI_EMBEDDINGS_URL: + return True + if url: + return False + body = openai_jsonl_record.get("body", {}) + if not isinstance(body, dict): + return False + return "input" in body and "messages" not in body + + # Identifier for the Bedrock Titan v2 InvokeModel body schema as stored + # in `model_prices_and_context_window.json`. Centralized so future + # embedding-schema variants can add their own value + # (e.g. `cohere_v3`, `titan_g1`, `titan_multimodal`) without touching + # the detection logic. + _TITAN_V2_INVOCATION_SCHEMA = "titan_v2" + + # Substring marker used as a fallback when the registry can't resolve + # the model id - notably cross-region inference profile prefixes + # (`us.amazon.titan-embed-text-v2:0`) and Bedrock ARN forms, which + # `get_model_info` doesn't normalize today. + _TITAN_V2_EMBED_MODEL_MARKER = "titan-embed-text-v2" + + # Nested field name under `provider_specific_entry` that identifies the + # Bedrock InvokeModel body schema for batch inference. + # `provider_specific_entry` is the registry's escape hatch for fields + # `get_model_info` doesn't promote to top-level - exactly what we need + # here. Documented in the `sample_spec` entry of + # `model_prices_and_context_window.json` and surfaced by + # `get_model_info` (see `ModelInfo.provider_specific_entry`). + _BEDROCK_INVOCATION_SCHEMA_FIELD = "bedrock_invocation_schema" + + @staticmethod + def _is_titan_v2_embed_model(model: str) -> bool: + """ + True iff `model` refers to Amazon Titan Text Embeddings V2. + + Resolution order: + 1. `model_prices_and_context_window.json` via `get_model_info`. + The Titan v2 registry entry carries an explicit + `provider_specific_entry.bedrock_invocation_schema` discriminator + (`"titan_v2"`). When the registry resolves the id we trust that + field as the source of truth - no hardcoded model-id comparison + needed. + 2. Substring fallback (`titan-embed-text-v2` followed by `:`, `/`, + or end-of-string) for ids the registry can't normalize. This + catches cross-region inference profile prefixes + (`us.amazon.titan-embed-text-v2:0`) and Bedrock ARN forms; the + marker boundary check rejects lookalikes like + `titan-embed-text-v20` or `titan-embed-text-v2-experimental`. + + Tolerant of common id shapes: + - "amazon.titan-embed-text-v2:0" + - "bedrock/amazon.titan-embed-text-v2:0" + - "us.amazon.titan-embed-text-v2:0" (cross-region inference profile) + - ARN forms ending in ".../amazon.titan-embed-text-v2:0" + """ + # Registry-driven path: when get_model_info resolves the id we trust + # the registry's discriminator. A resolved id with a different (or + # absent) schema value here is intentionally not given a substring + # second-chance - the registry is authoritative for ids it knows. + registry_schema = BedrockFilesConfig._lookup_provider_specific_field( + model, BedrockFilesConfig._BEDROCK_INVOCATION_SCHEMA_FIELD + ) + if registry_schema is not None: + return registry_schema == BedrockFilesConfig._TITAN_V2_INVOCATION_SCHEMA + + # Registry silence -> substring fallback for unmapped ids only. + normalized = model.lower() + if normalized.startswith("bedrock/"): + normalized = normalized[len("bedrock/") :] + marker = BedrockFilesConfig._TITAN_V2_EMBED_MODEL_MARKER + idx = normalized.find(marker) + if idx < 0: + return False + end = idx + len(marker) + return end == len(normalized) or normalized[end] in (":", "/") + + @staticmethod + def _lookup_provider_specific_field(model_id: str, field: str) -> Optional[str]: + """ + Read a nested string field from the registry entry's + `provider_specific_entry` dict via `litellm.get_model_info`. + + Returns the field's string value when: + - the registry resolves `model_id`, + - the entry exposes `provider_specific_entry` as a dict, and + - that dict has `field` mapped to a non-empty string. + Otherwise returns `None`. + + Isolating this means feature detectors (Titan v2 today, future + Cohere Embed / Nova Multimodal branches) share one defensive + try/except shape instead of duplicating it. The `None` return + covers every realistic failure mode: `get_model_info` raises + (cross-region profile prefixes, Bedrock ARN forms, unreleased + models), returns a non-dict, has no `provider_specific_entry`, or + the requested field is missing / non-string / empty. + """ + try: + from litellm import get_model_info + + info = get_model_info(model_id) + except Exception: + return None + if not isinstance(info, dict): + return None + provider_specific = info.get("provider_specific_entry") + if not isinstance(provider_specific, dict): + return None + value = provider_specific.get(field) + return value if isinstance(value, str) and value else None + + @staticmethod + def _coerce_embedding_input_to_string(raw_input: Any, model: str = "") -> str: + """ + Normalize an OpenAI /v1/embeddings `input` field into the single + string that Bedrock Titan v2 InvokeModel expects in `inputText`. + + Accepts: a string, or a single-element list containing one string. + Rejects (with actionable messages): + - None / missing -> ValueError + - Multi-element string lists -> ValueError, prompts caller to + emit one JSONL line per input + - Pre-tokenized inputs (List[int], List[List[int]]) -> NotImplementedError + - Any other type -> ValueError + + Extracted so the validation can be exercised in isolation and so + future embedding-provider branches (Titan G1, Cohere) can reuse it + without duplicating the type-shaping logic. + """ + if raw_input is None: + raise ValueError( + "Embedding batch record is missing required `input` field: " + f"model={model}" + ) + + # Bedrock InvokeModel for Titan v2 takes exactly one string `inputText` + # per call. Pre-tokenized inputs and multi-element string lists are + # explicitly unsupported so callers emit one JSONL line per embedding + # instead of relying on us to silently fan out or concatenate. + if isinstance(raw_input, list): + if len(raw_input) == 1: + candidate = raw_input[0] + else: + raise ValueError( + "Bedrock batch embedding requires one input per JSONL " + "record. Got a list with " + f"{len(raw_input)} items for model={model}; emit one " + "JSONL line per input string instead." + ) + else: + candidate = raw_input + + # Catches pre-tokenized inputs (List[int] from OpenAI spec, or a + # single int slipping past the list-unwrap above). + # NOTE: bool is a subclass of int but treating True/False as a token + # is meaningless either way, so the broad check is fine. + if isinstance(candidate, (list, int)): + raise NotImplementedError( + "Bedrock Titan v2 batch embedding does not support " + "pre-tokenized integer inputs. Pass `input` as a string " + f"(model={model})." + ) + if not isinstance(candidate, str): + raise ValueError( + "Bedrock batch embedding `input` must be a string (or a " + "single-element list of strings). Got type " + f"{type(candidate).__name__} for model={model}." + ) + return candidate + + def _map_openai_embedding_to_bedrock_params( + self, + openai_request_body: Dict[str, Any], + ) -> Dict[str, Any]: + """ + Transform an OpenAI /v1/embeddings request body into the + Bedrock InvokeModel `modelInput` for embedding models that AWS + supports via batch inference (CreateModelInvocationJob). + + Currently routes Amazon Titan Text Embeddings V2 only; other + embedding providers (Titan G1, Titan Multimodal, Cohere Embed, + Nova Multimodal Embeddings) raise NotImplementedError until they + get a dedicated branch. Splitting them keeps PR scope tight and + lets each model's request schema be exercised by its own tests. + + AWS docs (Titan v2 InvokeModel body): + https://docs.aws.amazon.com/bedrock/latest/userguide/model-parameters-titan-embed-text.html + """ + from litellm.llms.bedrock.embed.amazon_titan_v2_transformation import ( + AmazonTitanV2Config, + ) + + _model = openai_request_body.get("model", "") + if not self._is_titan_v2_embed_model(_model): + # Refuse early instead of silently shaping the body for the wrong + # provider. The synchronous /v1/embeddings path supports more + # models, but each has a different InvokeModel schema; mapping + # them here without dedicated tests would risk corrupt batches. + raise NotImplementedError( + "Bedrock batch embedding currently supports only Amazon " + "Titan Text Embeddings V2 (model id contains " + f"'titan-embed-text-v2'). Got model={_model!r}. Track other " + "embedding models in https://github.com/BerriAI/litellm/issues." + ) + + input_text = self._coerce_embedding_input_to_string( + openai_request_body.get("input"), model=_model + ) + + # Map OpenAI-style params (dimensions, encoding_format) onto the + # Titan v2 schema (dimensions, embeddingTypes) via the embed config + # so this stays in sync with the synchronous /v1/embeddings path. + non_default_params = { + k: v for k, v in openai_request_body.items() if k not in ("model", "input") + } + titan_config = AmazonTitanV2Config() + inference_params = titan_config.map_openai_params( + non_default_params=non_default_params, + optional_params={}, + ) + return dict( + titan_config._transform_request( + input=input_text, inference_params=inference_params + ) + ) + def _map_openai_to_bedrock_params( self, openai_request_body: Dict[str, Any], @@ -349,10 +602,19 @@ class BedrockFilesConfig(BaseAWSLLM, BaseFilesConfig): # Determine provider from model name provider = self.get_bedrock_invoke_provider(model) - # Transform to Bedrock modelInput format - model_input = self._map_openai_to_bedrock_params( - openai_request_body=openai_body, provider=provider - ) + # Route to the embedding transformer when the OpenAI batch line + # targets /v1/embeddings; otherwise fall back to the existing + # chat-completion path. We branch here (rather than inside + # `_map_openai_to_bedrock_params`) so the chat helper keeps its + # narrow contract and the embedding helper can evolve independently. + if self._is_embedding_record(_openai_jsonl_content): + model_input = self._map_openai_embedding_to_bedrock_params( + openai_request_body=openai_body + ) + else: + model_input = self._map_openai_to_bedrock_params( + openai_request_body=openai_body, provider=provider + ) # Create Bedrock batch record record_id = _openai_jsonl_content.get( diff --git a/litellm/llms/bedrock_mantle/responses/__init__.py b/litellm/llms/bedrock_mantle/responses/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/bedrock_mantle/responses/transformation.py b/litellm/llms/bedrock_mantle/responses/transformation.py new file mode 100644 index 0000000000..b63fd0ecdb --- /dev/null +++ b/litellm/llms/bedrock_mantle/responses/transformation.py @@ -0,0 +1,81 @@ +""" +Amazon Bedrock Mantle - Responses API backend. + +gpt-5.5 / gpt-5.4 on Mantle are exposed ONLY on the `/openai/v1/responses` +path (not the standard `/v1/responses`). Payloads and SSE follow the OpenAI +Responses spec, so this config inherits OpenAIResponsesAPIConfig and overrides +only the endpoint URL and Bearer authentication. + +Auth: AWS Bedrock API key as Bearer token (BEDROCK_MANTLE_API_KEY or the +standard AWS_BEARER_TOKEN_BEDROCK), NOT SigV4. +""" + +from typing import Optional + +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders + +BEDROCK_MANTLE_DEFAULT_REGION = "us-east-1" + +# Checked longest/most-specific first so a full endpoint URL collapses to host +# in one pass and the appended path never doubles. +_BASE_SUFFIXES_TO_STRIP = ( + "/openai/v1/responses", + "/v1/responses", + "/responses", + "/openai/v1", + "/v1", +) + + +class BedrockMantleResponsesAPIConfig(OpenAIResponsesAPIConfig): + @property + def custom_llm_provider(self) -> LlmProviders: + return LlmProviders.BEDROCK_MANTLE + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + region = ( + get_secret_str("BEDROCK_MANTLE_REGION") + or get_secret_str("AWS_REGION") + or BEDROCK_MANTLE_DEFAULT_REGION + ) + base = ( + api_base + or get_secret_str("BEDROCK_MANTLE_API_BASE") + or f"https://bedrock-mantle.{region}.api.aws" + ) + base = base.rstrip("/") + for suffix in _BASE_SUFFIXES_TO_STRIP: + if base.endswith(suffix): + base = base[: -len(suffix)] + break + return f"{base}/openai/v1/responses" + + def validate_environment( + self, headers: dict, model: str, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or get_secret_str("BEDROCK_MANTLE_API_KEY") + or get_secret_str("AWS_BEARER_TOKEN_BEDROCK") + ) + if not api_key: + raise ValueError( + "Bedrock Mantle API key is required. Set BEDROCK_MANTLE_API_KEY " + "(or AWS_BEARER_TOKEN_BEDROCK) or pass api_key." + ) + headers["Authorization"] = f"Bearer {api_key}" + return headers + + def supports_native_file_search(self) -> bool: + return False + + def supports_native_websocket(self) -> bool: + return False diff --git a/litellm/llms/black_forest_labs/common_utils.py b/litellm/llms/black_forest_labs/common_utils.py index 507ef17c50..237208693f 100644 --- a/litellm/llms/black_forest_labs/common_utils.py +++ b/litellm/llms/black_forest_labs/common_utils.py @@ -5,6 +5,7 @@ Common utilities, constants, and error handling for Black Forest Labs API. """ from typing import Dict +from urllib.parse import urlparse from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -18,6 +19,42 @@ class BlackForestLabsError(BaseLLMException): # API Constants DEFAULT_API_BASE = "https://api.bfl.ai" +# BFL uses regional subdomains (e.g. gateway.bfl.ai) for polling URLs that +# differ from the submission host (api.bfl.ai). We validate against the +# registered domain rather than doing a strict same-origin check. +_BFL_REGISTERED_DOMAIN = "bfl.ai" + + +def assert_bfl_polling_url(polling_url: str) -> None: + """Validate that a polling URL points to a BFL-controlled host. + + BFL returns polling URLs on subdomains like ``gateway.bfl.ai`` that differ + from the submission host ``api.bfl.ai``. A strict same-origin check would + reject these legitimate URLs. Instead we verify the host is ``bfl.ai`` or + any subdomain of it, which keeps the SSRF guarantee (credentials only go + to BFL-controlled infrastructure) without false-positives on regional hosts. + + Raises: + BlackForestLabsError: If the polling URL scheme or host is not trusted. + """ + parsed = urlparse(polling_url) + host = (parsed.hostname or "").lower() + + if parsed.scheme != "https": + raise BlackForestLabsError( + status_code=502, + message="Rejected polling URL: scheme must be https", + ) + + if host != _BFL_REGISTERED_DOMAIN and not host.endswith( + "." + _BFL_REGISTERED_DOMAIN + ): + raise BlackForestLabsError( + status_code=502, + message="Rejected polling URL: host is not within the bfl.ai domain", + ) + + # Polling configuration DEFAULT_POLLING_INTERVAL = 1.5 # seconds DEFAULT_MAX_POLLING_TIME = 300 # 5 minutes diff --git a/litellm/llms/black_forest_labs/image_edit/handler.py b/litellm/llms/black_forest_labs/image_edit/handler.py index f5784e0836..ab191c165f 100644 --- a/litellm/llms/black_forest_labs/image_edit/handler.py +++ b/litellm/llms/black_forest_labs/image_edit/handler.py @@ -15,7 +15,6 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -29,6 +28,7 @@ from ..common_utils import ( DEFAULT_MAX_POLLING_TIME, DEFAULT_POLLING_INTERVAL, BlackForestLabsError, + assert_bfl_polling_url, ) from .transformation import BlackForestLabsImageEditConfig @@ -332,16 +332,11 @@ class BlackForestLabsImageEdit: message="No polling_url in BFL response", ) - # Reject cross-origin polling URLs — the ``x-key`` auth header - # would otherwise leak to whatever URL the upstream returns. - # VERIA-51. - try: - assert_same_origin(polling_url, str(initial_response.request.url)) - except SSRFError as ssrf_err: - raise BlackForestLabsError( - status_code=502, - message=f"Rejected polling URL: {ssrf_err}", - ) + # Reject polling URLs that don't belong to BFL-controlled infrastructure. + # BFL uses regional subdomains (e.g. gateway.bfl.ai) that differ from the + # submission host (api.bfl.ai), so we validate against the registered + # domain rather than doing a strict same-origin check. VERIA-51. + assert_bfl_polling_url(polling_url) # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} @@ -428,16 +423,11 @@ class BlackForestLabsImageEdit: message="No polling_url in BFL response", ) - # Reject cross-origin polling URLs — the ``x-key`` auth header - # would otherwise leak to whatever URL the upstream returns. - # VERIA-51. - try: - assert_same_origin(polling_url, str(initial_response.request.url)) - except SSRFError as ssrf_err: - raise BlackForestLabsError( - status_code=502, - message=f"Rejected polling URL: {ssrf_err}", - ) + # Reject polling URLs that don't belong to BFL-controlled infrastructure. + # BFL uses regional subdomains (e.g. gateway.bfl.ai) that differ from the + # submission host (api.bfl.ai), so we validate against the registered + # domain rather than doing a strict same-origin check. VERIA-51. + assert_bfl_polling_url(polling_url) # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} diff --git a/litellm/llms/black_forest_labs/image_generation/handler.py b/litellm/llms/black_forest_labs/image_generation/handler.py index 8af4a236fd..f797fac419 100644 --- a/litellm/llms/black_forest_labs/image_generation/handler.py +++ b/litellm/llms/black_forest_labs/image_generation/handler.py @@ -15,7 +15,6 @@ import httpx import litellm from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin from litellm.llms.custom_httpx.http_handler import ( AsyncHTTPHandler, HTTPHandler, @@ -29,6 +28,7 @@ from ..common_utils import ( DEFAULT_MAX_POLLING_TIME, DEFAULT_POLLING_INTERVAL, BlackForestLabsError, + assert_bfl_polling_url, ) from .transformation import BlackForestLabsImageGenerationConfig @@ -172,6 +172,10 @@ class BlackForestLabsImageGeneration: raw_response=final_response, model_response=model_response, logging_obj=logging_obj, + request_data=data, + optional_params=optional_params, + litellm_params=litellm_params_dict, + encoding=None, ) async def async_image_generation( @@ -274,6 +278,10 @@ class BlackForestLabsImageGeneration: raw_response=final_response, model_response=model_response, logging_obj=logging_obj, + request_data=data, + optional_params=optional_params, + litellm_params=litellm_params_dict, + encoding=None, ) def _poll_for_result_sync( @@ -318,16 +326,11 @@ class BlackForestLabsImageGeneration: message="No polling_url in BFL response", ) - # Reject cross-origin polling URLs — the ``x-key`` auth header - # would otherwise leak to whatever URL the upstream returns. - # VERIA-51. - try: - assert_same_origin(polling_url, str(initial_response.request.url)) - except SSRFError as ssrf_err: - raise BlackForestLabsError( - status_code=502, - message=f"Rejected polling URL: {ssrf_err}", - ) + # Reject polling URLs that don't belong to BFL-controlled infrastructure. + # BFL uses regional subdomains (e.g. gateway.bfl.ai) that differ from the + # submission host (api.bfl.ai), so we validate against the registered + # domain rather than doing a strict same-origin check. VERIA-51. + assert_bfl_polling_url(polling_url) # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} @@ -414,16 +417,11 @@ class BlackForestLabsImageGeneration: message="No polling_url in BFL response", ) - # Reject cross-origin polling URLs — the ``x-key`` auth header - # would otherwise leak to whatever URL the upstream returns. - # VERIA-51. - try: - assert_same_origin(polling_url, str(initial_response.request.url)) - except SSRFError as ssrf_err: - raise BlackForestLabsError( - status_code=502, - message=f"Rejected polling URL: {ssrf_err}", - ) + # Reject polling URLs that don't belong to BFL-controlled infrastructure. + # BFL uses regional subdomains (e.g. gateway.bfl.ai) that differ from the + # submission host (api.bfl.ai), so we validate against the registered + # domain rather than doing a strict same-origin check. VERIA-51. + assert_bfl_polling_url(polling_url) # Get just the auth header for polling polling_headers = {"x-key": headers.get("x-key", "")} diff --git a/litellm/llms/cohere/chat/v2_transformation.py b/litellm/llms/cohere/chat/v2_transformation.py index 190491adfc..9aa8c11490 100644 --- a/litellm/llms/cohere/chat/v2_transformation.py +++ b/litellm/llms/cohere/chat/v2_transformation.py @@ -120,6 +120,7 @@ class CohereV2ChatConfig(OpenAIGPTConfig): "stream", "temperature", "max_tokens", + "max_completion_tokens", "top_p", "frequency_penalty", "presence_penalty", @@ -143,7 +144,12 @@ class CohereV2ChatConfig(OpenAIGPTConfig): optional_params["stream"] = value if param == "temperature": optional_params["temperature"] = value - if param == "max_tokens": + if ( + param == "max_tokens" + and "max_completion_tokens" not in non_default_params + ): + optional_params["max_tokens"] = value + if param == "max_completion_tokens": optional_params["max_tokens"] = value if param == "n": optional_params["num_generations"] = value diff --git a/litellm/llms/custom_httpx/aiohttp_transport.py b/litellm/llms/custom_httpx/aiohttp_transport.py index 132191c946..62f707b362 100644 --- a/litellm/llms/custom_httpx/aiohttp_transport.py +++ b/litellm/llms/custom_httpx/aiohttp_transport.py @@ -256,7 +256,10 @@ class LiteLLMAiohttpTransport(AiohttpTransport): from yarl import URL as YarlURL try: - data = request.content + # Coerce an empty body to None so aiohttp does not attach a + # `Content-Type: application/octet-stream` header for bodyless + # requests (e.g. DELETE /responses/{id}), which upstream APIs reject. + data = request.content or None except httpx.RequestNotRead: data = request.stream # type: ignore request.headers.pop("transfer-encoding", None) # handled by aiohttp diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index e11d8532db..01c9447643 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -589,6 +589,7 @@ class AsyncHTTPHandler: params: Optional[dict] = None, headers: Optional[dict] = None, follow_redirects: Optional[bool] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ): # Set follow_redirects to UseClientDefault if None _follow_redirects = ( @@ -599,7 +600,11 @@ class AsyncHTTPHandler: params.update(HTTPHandler.extract_query_params(url)) response = await self.client.get( - url, params=params, headers=headers, follow_redirects=_follow_redirects # type: ignore + url, + params=params, + headers=headers, # type: ignore + follow_redirects=_follow_redirects, # type: ignore + timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response @@ -1115,6 +1120,7 @@ class HTTPHandler: params: Optional[dict] = None, headers: Optional[dict] = None, follow_redirects: Optional[bool] = None, + timeout: Optional[Union[float, httpx.Timeout]] = None, ): # Set follow_redirects to UseClientDefault if None _follow_redirects = ( @@ -1128,6 +1134,7 @@ class HTTPHandler: params=params, headers=headers, follow_redirects=_follow_redirects, + timeout=timeout if timeout is not None else USE_CLIENT_DEFAULT, ) return response diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 941fe59e82..31c772510b 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1751,6 +1751,7 @@ class BaseLLMHTTPHandler: api_base=api_base, optional_params=optional_params, data=data, + api_key=api_key, ) ## LOGGING @@ -1833,6 +1834,7 @@ class BaseLLMHTTPHandler: api_base=api_base, optional_params=optional_params, data=data, + api_key=api_key, ) ## LOGGING @@ -2026,6 +2028,7 @@ class BaseLLMHTTPHandler: litellm_params={ "preset_cache_key": None, "stream_response": {}, + "model_info": kwargs.get("model_info"), **anthropic_messages_optional_request_params, }, custom_llm_provider=custom_llm_provider, @@ -2585,6 +2588,8 @@ class BaseLLMHTTPHandler: headers=headers, ) + headers.setdefault("Content-Type", "application/json") + ## LOGGING logging_obj.pre_call( input=input, @@ -2675,6 +2680,8 @@ class BaseLLMHTTPHandler: headers=headers, ) + headers.setdefault("Content-Type", "application/json") + ## LOGGING logging_obj.pre_call( input=input, @@ -5527,6 +5534,7 @@ class BaseLLMHTTPHandler: user_api_key_dict: Optional[Any] = None, litellm_metadata: Optional[Dict[str, Any]] = None, custom_llm_provider: Optional[str] = None, + first_message: Optional[str] = None, **kwargs: Any, ): """ @@ -5558,6 +5566,7 @@ class BaseLLMHTTPHandler: api_base=api_base, timeout=timeout, custom_llm_provider=custom_llm_provider, + first_message=first_message, **kwargs, ) await handler.run() @@ -5623,6 +5632,7 @@ class BaseLLMHTTPHandler: logging_obj=logging_obj, user_api_key_dict=user_api_key_dict, request_data=_request_data, + first_message=first_message, ) await streaming.bidirectional_forward() diff --git a/litellm/llms/fal_ai/image_generation/__init__.py b/litellm/llms/fal_ai/image_generation/__init__.py index 9deeb403c4..7f3358934a 100644 --- a/litellm/llms/fal_ai/image_generation/__init__.py +++ b/litellm/llms/fal_ai/image_generation/__init__.py @@ -7,6 +7,7 @@ from .flux_pro_v11_transformation import FalAIFluxProV11Config from .flux_pro_v11_ultra_transformation import FalAIFluxProV11UltraConfig from .flux_schnell_transformation import FalAIFluxSchnellConfig from .imagen4_transformation import FalAIImagen4Config +from .nano_banana_transformation import FalAINanoBananaConfig from .recraft_v3_transformation import FalAIRecraftV3Config from .ideogram_v3_transformation import FalAIIdeogramV3Config from .stable_diffusion_transformation import FalAIStableDiffusionConfig @@ -20,6 +21,7 @@ __all__ = [ "FalAIBaseConfig", "FalAIImageGenerationConfig", "FalAIImagen4Config", + "FalAINanoBananaConfig", "FalAIRecraftV3Config", "FalAIBriaConfig", "FalAIFluxProV11Config", @@ -45,7 +47,9 @@ def get_fal_ai_image_generation_config(model: str) -> BaseImageGenerationConfig: model_lower = model.lower() # Map model names to their corresponding configuration classes - if "imagen4" in model_lower or "imagen-4" in model_lower: + if "nano-banana" in model_lower or "gemini-25-flash-image" in model_lower: + return FalAINanoBananaConfig() + elif "imagen4" in model_lower or "imagen-4" in model_lower: return FalAIImagen4Config() elif "recraft" in model_lower: return FalAIRecraftV3Config() diff --git a/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py new file mode 100644 index 0000000000..dd4758055a --- /dev/null +++ b/litellm/llms/fal_ai/image_generation/nano_banana_transformation.py @@ -0,0 +1,105 @@ +from typing import List, Optional + +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import OpenAIImageGenerationOptionalParams + +from .transformation import FalAIBaseConfig + + +class FalAINanoBananaConfig(FalAIBaseConfig): + """ + Configuration for Fal AI's Nano Banana / Gemini 2.5 Flash Image models. + + Serves the imagen4 deprecation migration path. The same underlying model is + exposed under two endpoints that share an identical schema: + - fal-ai/nano-banana + - fal-ai/gemini-25-flash-image + + Documentation: https://fal.ai/models/fal-ai/nano-banana + """ + + SUPPORTED_ASPECT_RATIOS: List[str] = [ + "21:9", + "16:9", + "3:2", + "4:3", + "5:4", + "1:1", + "4:5", + "3:4", + "2:3", + "9:16", + ] + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + base_url: str = ( + api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL + ).rstrip("/") + endpoint = model if model.startswith("fal-ai/") else f"fal-ai/{model}" + return f"{base_url}/{endpoint}" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIImageGenerationOptionalParams]: + return ["n", "response_format", "size"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for key, value in non_default_params.items(): + if key == "response_format": + continue + elif key == "n": + if "num_images" not in optional_params: + optional_params["num_images"] = value + elif key == "size": + if "aspect_ratio" not in optional_params: + optional_params["aspect_ratio"] = self._map_aspect_ratio(value) + elif key not in optional_params and not drop_params: + raise ValueError( + f"Parameter {key} is not supported for model {model}. " + f"Supported parameters are {supported_params}. " + "Set drop_params=True to drop unsupported parameters." + ) + return optional_params + + def _map_aspect_ratio(self, size: str) -> str: + if not isinstance(size, str) or "x" not in size: + return "1:1" + try: + width, height = (int(part) for part in size.split("x")) + target = width / height + except (ValueError, ZeroDivisionError): + return "1:1" + + def ratio_of(aspect_ratio: str) -> float: + w, h = (int(part) for part in aspect_ratio.split(":")) + return w / h + + return min( + self.SUPPORTED_ASPECT_RATIOS, + key=lambda aspect_ratio: abs(ratio_of(aspect_ratio) - target), + ) + + def transform_image_generation_request( + self, + model: str, + prompt: str, + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + return {"prompt": prompt, **optional_params} diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index d39adf0b6f..cca3b3da37 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -8,6 +8,7 @@ from litellm._logging import verbose_logger from litellm._uuid import uuid from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_legacy_defs from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, ) @@ -169,11 +170,6 @@ class FireworksAIConfig(OpenAIGPTConfig): is_response_format_supported=False, enforce_tool_choice=False, # tools and response_format are both set, don't enforce tool_choice ) - elif "json_schema" in value: - optional_params["response_format"] = { - "type": "json_object", - "schema": value["json_schema"]["schema"], - } else: optional_params["response_format"] = value elif param == "max_completion_tokens": @@ -216,8 +212,13 @@ class FireworksAIConfig(OpenAIGPTConfig): self, tools: List[OpenAIChatCompletionToolParam] ) -> List[OpenAIChatCompletionToolParam]: for tool in tools: - if tool.get("type") == "function": - tool["function"].pop("strict", None) + if tool.get("type") != "function": + continue + function = tool["function"] + function.pop("strict", None) + params = function.get("parameters") + if isinstance(params, dict): + unpack_legacy_defs(params) return tools def _transform_messages_helper( diff --git a/litellm/llms/gemini/chat/transformation.py b/litellm/llms/gemini/chat/transformation.py index b69b7e1913..4e9764446c 100644 --- a/litellm/llms/gemini/chat/transformation.py +++ b/litellm/llms/gemini/chat/transformation.py @@ -93,6 +93,7 @@ class GoogleAIStudioGeminiConfig(VertexGeminiConfig): "modalities", "parallel_tool_calls", "web_search_options", + "include_server_side_tool_invocations", "service_tier", ] if supports_reasoning(model, custom_llm_provider="gemini"): diff --git a/litellm/llms/gemini/common_utils.py b/litellm/llms/gemini/common_utils.py index bc963d62b5..42a807983b 100644 --- a/litellm/llms/gemini/common_utils.py +++ b/litellm/llms/gemini/common_utils.py @@ -1,6 +1,8 @@ import base64 import datetime -from typing import Any, Dict, List, Optional, Union +import json +import math +from typing import Any, Dict, List, Optional, Sequence, Union import httpx @@ -12,6 +14,245 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import TokenCountResponse +GEMINI_IMAGE_ASPECT_RATIOS: Dict[str, float] = { + "1:1": 1 / 1, + "1:4": 1 / 4, + "1:8": 1 / 8, + "2:3": 2 / 3, + "3:2": 3 / 2, + "3:4": 3 / 4, + "4:1": 4 / 1, + "4:3": 4 / 3, + "4:5": 4 / 5, + "5:4": 5 / 4, + "8:1": 8 / 1, + "9:16": 9 / 16, + "16:9": 16 / 9, + "21:9": 21 / 9, +} + +# Supported aspect ratio dimensions from Google Gemini image generation docs: +# https://ai.google.dev/gemini-api/docs/image-generation#aspect_ratios_and_image_size +GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: Dict[tuple[int, int], str] = { + (512, 512): "1:1", + (1024, 1024): "1:1", + (2048, 2048): "1:1", + (4096, 4096): "1:1", + (256, 1024): "1:4", + (512, 2048): "1:4", + (1024, 4096): "1:4", + (2048, 8192): "1:4", + (192, 1536): "1:8", + (384, 3072): "1:8", + (768, 6144): "1:8", + (1536, 12288): "1:8", + (424, 632): "2:3", + (848, 1264): "2:3", + (1696, 2528): "2:3", + (3392, 5056): "2:3", + (632, 424): "3:2", + (1264, 848): "3:2", + (2528, 1696): "3:2", + (5056, 3392): "3:2", + (448, 600): "3:4", + (896, 1200): "3:4", + (1792, 2400): "3:4", + (3584, 4800): "3:4", + (1024, 256): "4:1", + (2048, 512): "4:1", + (4096, 1024): "4:1", + (8192, 2048): "4:1", + (600, 448): "4:3", + (1200, 896): "4:3", + (2400, 1792): "4:3", + (4800, 3584): "4:3", + (464, 576): "4:5", + (928, 1152): "4:5", + (1856, 2304): "4:5", + (3712, 4608): "4:5", + (576, 464): "5:4", + (1152, 928): "5:4", + (2304, 1856): "5:4", + (4608, 3712): "5:4", + (1536, 192): "8:1", + (3072, 384): "8:1", + (6144, 768): "8:1", + (12288, 1536): "8:1", + (384, 688): "9:16", + (768, 1376): "9:16", + (1536, 2752): "9:16", + (3072, 5504): "9:16", + (688, 384): "16:9", + (1376, 768): "16:9", + (2752, 1536): "16:9", + (5504, 3072): "16:9", + (792, 336): "21:9", + (1584, 672): "21:9", + (3168, 1344): "21:9", + (6336, 2688): "21:9", + (1280, 896): "4:3", + (896, 1280): "3:4", +} + + +def map_openai_size_to_gemini_image_config( + size: str, model: str +) -> Optional[Dict[str, str]]: + dimensions = _parse_openai_image_size(size) + if dimensions is None: + return None + + width, height = dimensions + image_config = { + "aspectRatio": _map_dimensions_to_gemini_aspect_ratio(width, height) + } + image_size = _map_dimensions_to_gemini_image_size(width, height) + if is_gemini_image_model(model): + if supports_gemini_image_size(model): + image_config["imageSize"] = image_size + else: + image_config["imageSize"] = image_size + return image_config + + +def supports_gemini_image_size(model: str) -> bool: + try: + model_info = litellm.get_model_info(model=model) + value = model_info.get("supports_image_size") + if value is not None: + return bool(value) + except Exception: + pass + return "2.5-flash" not in model + + +def is_gemini_image_model(model: str) -> bool: + base_model = model.split("/", 1)[-1] + return "gemini" in base_model + + +def map_openai_image_params_to_gemini( + params: Dict[str, Any], + model: str, + supported_params: Sequence[str], + optional_params: Optional[Dict[str, Any]] = None, + parse_image_config_string: bool = False, +) -> Dict[str, Any]: + optional_params = optional_params or {} + filtered_params = { + key: value for key, value in params.items() if key in supported_params + } + + mapped_params: Dict[str, Any] = {} + + if "n" in filtered_params and "n" not in optional_params: + mapped_params["sampleCount"] = filtered_params["n"] + + if "size" in filtered_params and "size" not in optional_params: + image_config = map_openai_size_to_gemini_image_config( + filtered_params["size"], + model, + ) + if image_config is not None: + if is_gemini_image_model(model): + mapped_params["imageConfig"] = image_config + else: + mapped_params["aspectRatio"] = image_config["aspectRatio"] + if "imageSize" in image_config: + mapped_params["imageSize"] = image_config["imageSize"] + + image_config_param = filtered_params.get("imageConfig") + if isinstance(image_config_param, str) and parse_image_config_string: + try: + image_config_param = json.loads(image_config_param) + except json.JSONDecodeError as exc: + raise litellm.UnsupportedParamsError( + model=model, + message="`imageConfig` must be valid JSON when provided as a string.", + ) from exc + if isinstance(image_config_param, dict): + mapped_params["imageConfig"] = image_config_param + + for key, value in filtered_params.items(): + if key not in ("n", "size", "imageConfig") and key not in optional_params: + mapped_params[key] = value + + return mapped_params + + +def get_gemini_image_generation_config( + model: str, + optional_params: Dict[str, Any], +) -> Dict[str, Any]: + generation_config: Dict[str, Any] = {"response_modalities": ["IMAGE", "TEXT"]} + + image_config: Dict[str, Any] = {} + if isinstance(optional_params.get("imageConfig"), dict): + image_config.update(optional_params["imageConfig"]) + + if not supports_gemini_image_size(model): + image_config.pop("imageSize", None) + + if image_config: + generation_config["imageConfig"] = image_config + + candidate_count = next( + ( + optional_params[key] + for key in ("candidateCount", "candidate_count", "sampleCount", "n") + if optional_params.get(key) is not None + ), + None, + ) + if candidate_count is not None: + generation_config["candidateCount"] = candidate_count + + return generation_config + + +def _parse_openai_image_size(size: str) -> Optional[tuple[int, int]]: + if size == "auto": + return None + + width_str, separator, height_str = size.lower().partition("x") + if not separator: + return None + + try: + width = int(width_str) + height = int(height_str) + except ValueError: + return None + + if width <= 0 or height <= 0: + return None + + return width, height + + +def _map_dimensions_to_gemini_aspect_ratio(width: int, height: int) -> str: + if (width, height) in GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO: + return GEMINI_IMAGE_SIZE_TO_ASPECT_RATIO[(width, height)] + + requested_ratio = width / height + return min( + GEMINI_IMAGE_ASPECT_RATIOS, + key=lambda aspect_ratio: abs( + math.log(GEMINI_IMAGE_ASPECT_RATIOS[aspect_ratio] / requested_ratio) + ), + ) + + +def _map_dimensions_to_gemini_image_size(width: int, height: int) -> str: + effective_square_side = math.sqrt(width * height) + if effective_square_side < 768: + return "512" + if effective_square_side < 1536: + return "1K" + if effective_square_side < 3072: + return "2K" + return "4K" + class GeminiError(BaseLLMException): pass diff --git a/litellm/llms/gemini/image_edit/cost_calculator.py b/litellm/llms/gemini/image_edit/cost_calculator.py index 2e332a7fc0..956edb849a 100644 --- a/litellm/llms/gemini/image_edit/cost_calculator.py +++ b/litellm/llms/gemini/image_edit/cost_calculator.py @@ -4,8 +4,9 @@ Gemini Image Edit Cost Calculator from typing import Any -import litellm -from litellm.types.utils import ImageResponse +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as image_generation_cost_calculator, +) def cost_calculator( @@ -15,20 +16,10 @@ def cost_calculator( """ Gemini image edit cost calculator. - Mirrors image generation pricing: charge per returned image based on - model metadata (`output_cost_per_image`). + Gemini image edits and generations share image response billing behavior: + use provider token usage when present, otherwise fall back to per-image pricing. """ - model_info = litellm.get_model_info( + return image_generation_cost_calculator( model=model, - custom_llm_provider="gemini", + image_response=image_response, ) - - output_cost_per_image: float = model_info.get("output_cost_per_image") or 0.0 - - if not isinstance(image_response, ImageResponse): - raise ValueError( - f"image_response must be of type ImageResponse got type={type(image_response)}" - ) - - num_images = len(image_response.data or []) - return output_cost_per_image * num_images diff --git a/litellm/llms/gemini/image_edit/transformation.py b/litellm/llms/gemini/image_edit/transformation.py index c8aaab0e14..2316361d6e 100644 --- a/litellm/llms/gemini/image_edit/transformation.py +++ b/litellm/llms/gemini/image_edit/transformation.py @@ -7,10 +7,22 @@ from httpx._types import RequestFiles from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.image_edit.transformation import BaseImageEditConfig +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.images.main import ImageEditOptionalRequestParams from litellm.types.router import GenericLiteLLMParams -from litellm.types.utils import FileTypes, ImageObject, ImageResponse, OpenAIImage +from litellm.types.utils import ( + FileTypes, + ImageObject, + ImageResponse, + OpenAIImage, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -22,7 +34,7 @@ else: class GeminiImageEditConfig(BaseImageEditConfig): DEFAULT_BASE_URL: str = "https://generativelanguage.googleapis.com/v1beta" - SUPPORTED_PARAMS: List[str] = ["size"] + SUPPORTED_PARAMS: List[str] = ["n", "size", "imageConfig"] def get_supported_openai_params(self, model: str) -> List[str]: return list(self.SUPPORTED_PARAMS) @@ -33,21 +45,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): model: str, drop_params: bool, ) -> Dict[str, Any]: - supported_params = self.get_supported_openai_params(model) - filtered_params = { - key: value - for key, value in image_edit_optional_params.items() - if key in supported_params - } - - mapped_params: Dict[str, Any] = {} - - if "size" in filtered_params: - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio( - filtered_params["size"] # type: ignore[arg-type] - ) - - return mapped_params + return map_openai_image_params_to_gemini( + params=image_edit_optional_params, # type: ignore[arg-type] + model=model, + supported_params=self.get_supported_openai_params(model), + parse_image_config_string=True, + ) def validate_environment( self, @@ -107,18 +110,10 @@ class GeminiImageEditConfig(BaseImageEditConfig): request_body: Dict[str, Any] = {"contents": contents} - generation_config: Dict[str, Any] = {} - - if "aspectRatio" in image_edit_optional_request_params: - # Move aspectRatio into imageConfig inside generationConfig - if "imageConfig" not in generation_config: - generation_config["imageConfig"] = {} - generation_config["imageConfig"]["aspectRatio"] = ( - image_edit_optional_request_params["aspectRatio"] - ) - - if generation_config: - request_body["generationConfig"] = generation_config + request_body["generationConfig"] = get_gemini_image_generation_config( + model=model, + optional_params=image_edit_optional_request_params, + ) empty_files = cast(RequestFiles, []) return request_body, empty_files @@ -156,18 +151,12 @@ class GeminiImageEditConfig(BaseImageEditConfig): ) model_response.data = cast(List[OpenAIImage], data_list) + if "usageMetadata" in response_json: + model_response.usage = transform_gemini_image_usage( + response_json["usageMetadata"] + ) return model_response - def _map_size_to_aspect_ratio(self, size: str) -> str: - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - def _prepare_inline_image_parts( self, image: Union[FileTypes, List[FileTypes]] ) -> List[Dict[str, Any]]: diff --git a/litellm/llms/gemini/image_generation/transformation.py b/litellm/llms/gemini/image_generation/transformation.py index 9c4cd008b8..e6770a76bc 100644 --- a/litellm/llms/gemini/image_generation/transformation.py +++ b/litellm/llms/gemini/image_generation/transformation.py @@ -5,18 +5,21 @@ import httpx from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) +from litellm.llms.gemini.common_utils import ( + get_gemini_image_generation_config, + is_gemini_image_model, + map_openai_image_params_to_gemini, +) +from litellm.llms.gemini.image_usage_transformation import ( + transform_gemini_image_usage, +) from litellm.secret_managers.main import get_secret_str from litellm.types.llms.gemini import GeminiImageGenerationRequest from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ( - ImageObject, - ImageResponse, - ImageUsage, - ImageUsageInputTokensDetails, -) +from litellm.types.utils import ImageObject, ImageResponse if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -36,7 +39,10 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): Google AI Imagen API supported parameters https://ai.google.dev/gemini-api/docs/imagen """ - return ["n", "size"] + supported_params = ["n", "size"] + if is_gemini_image_model(model): + supported_params.append("imageConfig") + return supported_params # type: ignore[return-value] def map_openai_params( self, @@ -45,64 +51,11 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model: str, drop_params: bool, ) -> dict: - supported_params = self.get_supported_openai_params(model) - mapped_params = {} - - for k, v in non_default_params.items(): - if k not in optional_params.keys(): - if k in supported_params: - # Map OpenAI parameters to Google format - if k == "n": - mapped_params["sampleCount"] = v - elif k == "size": - # Map OpenAI size format to Google aspectRatio - mapped_params["aspectRatio"] = self._map_size_to_aspect_ratio(v) - else: - mapped_params[k] = v - return mapped_params - - def _map_size_to_aspect_ratio(self, size: str) -> str: - """ - https://ai.google.dev/gemini-api/docs/image-generation - - """ - aspect_ratio_map = { - "1024x1024": "1:1", - "1792x1024": "16:9", - "1024x1792": "9:16", - "1280x896": "4:3", - "896x1280": "3:4", - } - return aspect_ratio_map.get(size, "1:1") - - def _transform_image_usage(self, usage_metadata: dict) -> ImageUsage: - """ - Transform Gemini usageMetadata to ImageUsage format - """ - input_tokens_details = ImageUsageInputTokensDetails( - image_tokens=0, - text_tokens=0, - ) - - # Extract detailed token counts from promptTokensDetails - tokens_details = usage_metadata.get("promptTokensDetails", []) - for details in tokens_details: - if isinstance(details, dict): - modality = str(details.get("modality", "")).upper() - raw_token_count = details.get( - "tokenCount", details.get("token_count", 0) - ) - token_count = raw_token_count if isinstance(raw_token_count, int) else 0 - if modality == "TEXT": - input_tokens_details.text_tokens += token_count - elif modality == "IMAGE": - input_tokens_details.image_tokens += token_count - - return ImageUsage( - input_tokens=usage_metadata.get("promptTokenCount", 0), - input_tokens_details=input_tokens_details, - output_tokens=usage_metadata.get("candidatesTokenCount", 0), - total_tokens=usage_metadata.get("totalTokenCount", 0), + return map_openai_image_params_to_gemini( + params=non_default_params, + model=model, + supported_params=self.get_supported_openai_params(model), + optional_params=optional_params, ) def get_complete_url( @@ -127,7 +80,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): complete_url = complete_url.rstrip("/") # Gemini Flash Image Preview models use generateContent endpoint - if "gemini" in model: + if is_gemini_image_model(model): complete_url = f"{complete_url}/models/{model}:generateContent" else: # All other Imagen models use predict endpoint @@ -179,10 +132,13 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): } """ # For Gemini Flash Image Preview models, use standard Gemini format - if "gemini" in model: + if is_gemini_image_model(model): request_body: dict = { "contents": [{"parts": [{"text": prompt}]}], - "generationConfig": {"response_modalities": ["IMAGE", "TEXT"]}, + "generationConfig": get_gemini_image_generation_config( + model=model, + optional_params=optional_params, + ), } return request_body else: @@ -200,6 +156,9 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): ) return request_body_obj.model_dump(exclude_none=True) + def _transform_image_usage(self, usage_metadata: dict): + return transform_gemini_image_usage(usage_metadata) + def transform_image_generation_response( self, model: str, @@ -229,7 +188,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): model_response.data = [] # Handle different response formats based on model - if "gemini" in model: + if is_gemini_image_model(model): # Gemini Flash Image Preview models return in candidates format candidates = response_data.get("candidates", []) for candidate in candidates: @@ -255,7 +214,7 @@ class GoogleImageGenConfig(BaseImageGenerationConfig): # Extract usage metadata for Gemini models if "usageMetadata" in response_data: - model_response.usage = self._transform_image_usage( + model_response.usage = transform_gemini_image_usage( response_data["usageMetadata"] ) else: diff --git a/litellm/llms/gemini/image_usage_transformation.py b/litellm/llms/gemini/image_usage_transformation.py new file mode 100644 index 0000000000..5a55bdeffb --- /dev/null +++ b/litellm/llms/gemini/image_usage_transformation.py @@ -0,0 +1,73 @@ +from typing import Any + +from litellm.types.utils import ImageUsage, ImageUsageInputTokensDetails + + +def _get_token_count(details: dict) -> int: + raw_token_count = details.get("tokenCount", details.get("token_count", 0)) + return raw_token_count if isinstance(raw_token_count, int) else 0 + + +def _get_modality_token_details(usage_metadata: dict, *details_keys: str) -> list: + for details_key in details_keys: + details = usage_metadata.get(details_key) + if isinstance(details, list): + return details + return [] + + +def _sum_modality_token_details( + usage_metadata: dict, *details_keys: str +) -> ImageUsageInputTokensDetails: + tokens_details = ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ) + + for details in _get_modality_token_details(usage_metadata, *details_keys): + if isinstance(details, dict): + modality = str(details.get("modality", "")).upper() + token_count = _get_token_count(details) + if modality == "TEXT": + tokens_details.text_tokens += token_count + elif modality == "IMAGE": + tokens_details.image_tokens += token_count + + return tokens_details + + +def transform_gemini_image_usage(usage_metadata: dict) -> ImageUsage: + """ + Transform Gemini usageMetadata to ImageUsage format. + """ + input_tokens_details = _sum_modality_token_details( + usage_metadata, "promptTokensDetails", "prompt_tokens_details" + ) + output_tokens = usage_metadata.get("candidatesTokenCount", 0) + output_tokens_details = _sum_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ) + + if not _get_modality_token_details( + usage_metadata, "candidatesTokensDetails", "candidates_tokens_details" + ): + output_tokens_details.image_tokens = output_tokens + else: + known_output_tokens = ( + output_tokens_details.text_tokens + output_tokens_details.image_tokens + ) + if output_tokens > known_output_tokens: + output_tokens_details.text_tokens += output_tokens - known_output_tokens + + usage_payload: dict[str, Any] = { + "input_tokens": usage_metadata.get("promptTokenCount", 0), + "input_tokens_details": input_tokens_details, + "output_tokens": output_tokens, + "total_tokens": usage_metadata.get("totalTokenCount", 0), + "prompt_tokens": usage_metadata.get("promptTokenCount", 0), + "prompt_tokens_details": input_tokens_details.model_dump(), + "completion_tokens": output_tokens, + "completion_tokens_details": output_tokens_details.model_dump(), + "output_tokens_details": output_tokens_details.model_dump(), + } + return ImageUsage(**usage_payload) diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index cf1fc75ef1..212287fb7f 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -27,7 +27,6 @@ from litellm.types.llms.gemini import ( ) from litellm.types.llms.openai import ( OpenAIRealtimeContentPartDone, - OpenAIRealtimeConversationItemCreated, OpenAIRealtimeDoneEvent, OpenAIRealtimeEvents, OpenAIRealtimeEventTypes, @@ -79,6 +78,12 @@ _KNOWN_GEMINI_TOP_LEVEL_KEYS: set = { map_key.split(".", 1)[0] for map_key in MAP_GEMINI_FIELD_TO_OPENAI_EVENT } +# Gemini Live native-audio model ids carry this marker (e.g. +# ``gemini-2.5-flash-native-audio-preview-09-2025``). These models reject a +# ``speechConfig`` on ``setup`` with a 1007 invalid-argument error, so it is +# stripped in ``_finalize_gemini_live_setup``. +_GEMINI_NATIVE_AUDIO_MODEL_MARKER = "native-audio" + class GeminiRealtimeConfig(BaseRealtimeConfig): # Cap the LRU of in-flight tool calls so long sessions with many tool @@ -98,6 +103,33 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): # bypassing spend and budget accounting. self._pending_usage_metadata: Optional[dict] = None + @staticmethod + def _usage_detail_alias(details: Any, defaults: Dict[str, int]) -> Dict[str, Any]: + if not isinstance(details, dict): + return dict(defaults) + return { + **defaults, + **{key: value for key, value in details.items() if value is not None}, + } + + @staticmethod + def _add_pipecat_usage_detail_aliases(usage_dict: Dict[str, Any]) -> Dict[str, Any]: + usage_dict.setdefault( + "input_token_details", + GeminiRealtimeConfig._usage_detail_alias( + usage_dict.get("input_tokens_details"), + {"cached_tokens": 0, "text_tokens": 0, "audio_tokens": 0}, + ), + ) + usage_dict.setdefault( + "output_token_details", + GeminiRealtimeConfig._usage_detail_alias( + usage_dict.get("output_tokens_details"), + {"text_tokens": 0, "audio_tokens": 0}, + ), + ) + return usage_dict + def validate_environment( self, headers: dict, model: str, api_key: Optional[str] = None ) -> dict: @@ -173,9 +205,25 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): def map_automatic_turn_detection( self, value: OpenAIRealtimeTurnDetection ) -> AutomaticActivityDetection: + """Map OpenAI ``server_vad`` to Gemini ``automaticActivityDetection``. + + OpenAI ``semantic_vad`` has no Gemini Live equivalent — return an empty + dict so callers omit ``realtimeInputConfig`` (mapping it with + ``disabled: true`` breaks native-audio sessions). + """ + if ( + isinstance(value, dict) + and value.get("type") == "semantic_vad" + and "create_response" not in value + ): + return AutomaticActivityDetection() + automatic_activity_dection = AutomaticActivityDetection() if "create_response" in value and isinstance(value["create_response"], bool): automatic_activity_dection["disabled"] = not value["create_response"] + elif isinstance(value, dict) and value.get("type") == "server_vad": + # OpenAI server VAD enables activity detection by default. + automatic_activity_dection["disabled"] = False else: automatic_activity_dection["disabled"] = True if "prefix_padding_ms" in value and isinstance(value["prefix_padding_ms"], int): @@ -197,6 +245,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "tools", "input_audio_transcription", "turn_detection", + "voice", ] def map_openai_params( @@ -231,17 +280,33 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): optional_params["inputAudioTranscription"] = {} elif key == "turn_detection": value_typed = cast(OpenAIRealtimeTurnDetection, value) + if ( + isinstance(value_typed, dict) + and value_typed.get("type") == "semantic_vad" + and "create_response" not in value_typed + ): + # Pipecat/OpenAI GA semantic VAD — skip; Gemini uses its own VAD. + # Only skip when there is no create_response override so that + # a guardrail-injected create_response:false is not dropped. + continue transformed_audio_activity_config = self.map_automatic_turn_detection( value_typed ) - if ( - len(transformed_audio_activity_config) > 0 - ): # if the config is not empty, add it to the optional params + if transformed_audio_activity_config: optional_params["realtimeInputConfig"] = ( BidiGenerateContentRealtimeInputConfig( automaticActivityDetection=transformed_audio_activity_config ) ) + elif key == "voice": + from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( + VertexGeminiConfig, + ) + + vertex_gemini_config = VertexGeminiConfig() + speech_config = vertex_gemini_config._map_audio_params({"voice": value}) + if speech_config: + optional_params["generationConfig"]["speechConfig"] = speech_config if len(optional_params["generationConfig"]) == 0: optional_params.pop("generationConfig") return optional_params @@ -297,6 +362,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): and "transcription" in input_cfg ): normalized["input_audio_transcription"] = input_cfg["transcription"] + output_cfg = audio.get("output") + if isinstance(output_cfg, dict) and output_cfg.get("voice"): + normalized["voice"] = output_cfg["voice"] extracted_turn_detection = GeminiRealtimeConfig._extract_turn_detection( normalized @@ -308,6 +376,18 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return normalized + @staticmethod + def _finalize_gemini_live_setup( + model: str, setup: Dict[str, Any] + ) -> Dict[str, Any]: + """Drop fields Gemini Live native-audio rejects on ``setup``.""" + if _GEMINI_NATIVE_AUDIO_MODEL_MARKER not in model.lower(): + return setup + generation_config = setup.get("generationConfig") + if isinstance(generation_config, dict): + generation_config.pop("speechConfig", None) + return setup + def _handle_session_update( self, json_message: dict, @@ -351,7 +431,11 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): verbose_logger.debug( "Gemini Realtime: Sending initial setup with tools to backend" ) - return [json.dumps({"setup": new_overrides})] + return [ + json.dumps( + {"setup": self._finalize_gemini_live_setup(model, new_overrides)} + ) + ] if not new_overrides: verbose_logger.debug( @@ -420,7 +504,15 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): verbose_logger.debug( "Gemini Realtime: Forwarding session.update as follow-up setup" ) - return [json.dumps({"setup": follow_up_setup})] + return [ + json.dumps( + { + "setup": self._finalize_gemini_live_setup( + model, cast(Dict[str, Any], follow_up_setup) + ) + } + ) + ] def _handle_conversation_item(self, json_message: dict) -> List[str]: """ @@ -666,6 +758,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "object": "realtime.response", "id": response_id, "status": "in_progress", + "status_details": None, "output": [], "conversation_id": conversation_id, "modalities": _modalities, @@ -675,9 +768,10 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) response_items.append(response_created) - ## - return response.output_item.added ← adds ‘item_id’ same for all subsequent events + ## - return response.output_item.added response_output_item_added = OpenAIRealtimeStreamResponseOutputItemAdded( type="response.output_item.added", + event_id="event_{}".format(uuid.uuid4()), response_id=response_id, output_index=0, item={ @@ -690,20 +784,28 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): }, ) response_items.append(response_output_item_added) - ## - return conversation.item.created - conversation_item_created = OpenAIRealtimeConversationItemCreated( - type="conversation.item.created", - event_id="event_{}".format(uuid.uuid4()), - item={ - "id": output_item_id, - "object": "realtime.item", - "type": "message", - "status": "in_progress", - "role": "assistant", - "content": [], - }, + ## - return conversation.item.added + # Pipecat 1.3.x handles "conversation.item.added" (not ".created"). + # Sending ".created" raises "Unimplemented server event type" which + # kills the receive task handler. + response_items.append( + cast( + OpenAIRealtimeEvents, + { + "type": "conversation.item.added", + "event_id": "event_{}".format(uuid.uuid4()), + "previous_item_id": None, + "item": { + "id": output_item_id, + "object": "realtime.item", + "type": "message", + "status": "in_progress", + "role": "assistant", + "content": [], + }, + }, + ) ) - response_items.append(conversation_item_created) ## - return response.content_part.added response_content_part_added = OpenAIRealtimeResponseContentPartAdded( type="response.content_part.added", @@ -749,9 +851,9 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): return OpenAIRealtimeResponseDelta( type=( - "response.text.delta" + "response.output_text.delta" if delta_type == "text" - else "response.audio.delta" + else "response.output_audio.delta" ), content_index=0, event_id="event_{}".format(uuid.uuid4()), @@ -778,7 +880,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): current_response_id = "resp_{}".format(uuid.uuid4()) if delta_type == "text": return OpenAIRealtimeResponseTextDone( - type="response.text.done", + type="response.output_text.done", content_index=0, event_id="event_{}".format(uuid.uuid4()), item_id=current_output_item_id, @@ -788,7 +890,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) elif delta_type == "audio": return OpenAIRealtimeResponseAudioDone( - type="response.audio.done", + type="response.output_audio.done", content_index=0, event_id="event_{}".format(uuid.uuid4()), item_id=current_output_item_id, @@ -914,7 +1016,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): events: List[OpenAIRealtimeFunctionCallArgumentsDone] = [] for idx, fc in enumerate(function_calls): - call_id = fc.get("id", "") + call_id = fc.get("id", "") or f"call_{uuid.uuid4().hex[:16]}" name = fc.get("name", "") # Store call_id → name mapping for round-trip. Use an LRU so @@ -962,7 +1064,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): current_delta_chunks = [] any_delta_chunk = False for event in transformed_message: - if event["type"] == "response.text.delta": + if event["type"] == "response.output_text.delta": current_delta_chunks.append( cast(OpenAIRealtimeResponseDelta, event) ) @@ -973,7 +1075,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) else: if ( - transformed_message["type"] == "response.text.delta" + transformed_message["type"] == "response.output_text.delta" ): # ONLY ACCUMULATE TEXT DELTA CHUNKS - AUDIO WILL CAUSE SERVER MEMORY ISSUES if current_delta_chunks is None: current_delta_chunks = [] @@ -1067,6 +1169,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( _chat_completion_usage, ) + _usage_dict = responses_api_usage.model_dump() + self._add_pipecat_usage_detail_aliases(_usage_dict) response_done_event = OpenAIRealtimeDoneEvent( type="response.done", event_id="event_{}".format(uuid.uuid4()), @@ -1074,6 +1178,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): object="realtime.response", id=current_response_id, status="completed", + status_details=None, # type: ignore[typeddict-item] output=( [output_item["item"] for output_item in output_items] if output_items @@ -1081,7 +1186,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ), conversation_id=current_conversation_id, modalities=_modalities, - usage=responses_api_usage.model_dump(), + usage=_usage_dict, ), ) if temperature is not None: @@ -1294,19 +1399,36 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): output_tx = server_content.get("outputTranscription") if isinstance(output_tx, dict) and output_tx.get("text"): + if current_response_id is None: + current_response_id = "resp_{}".format(uuid.uuid4()) + if current_output_item_id is None: + current_output_item_id = "item_{}".format(uuid.uuid4()) + current_conversation_id = ( + current_conversation_id or "conv_{}".format(uuid.uuid4()) + ) + returned_message.extend( + self.return_new_content_delta_events( + session_configuration_request=session_configuration_request, + response_id=current_response_id, + output_item_id=current_output_item_id, + conversation_id=current_conversation_id, + delta_type="audio", + ) + ) + # Emit as the GA event name; _GA_TO_BETA_EVENT_TYPES translates + # this back to response.audio_transcript.delta for beta clients. returned_message.append( cast( OpenAIRealtimeEvents, { - "type": "response.audio_transcript.delta", + "type": "response.output_audio_transcript.delta", "event_id": "event_{}".format(uuid.uuid4()), - "delta": output_tx["text"], - "item_id": current_output_item_id - or "item_{}".format(uuid.uuid4()), - "response_id": current_response_id - or "resp_{}".format(uuid.uuid4()), - "output_index": 0, + "transcript": output_tx["text"], + "item_id": current_output_item_id, "content_index": 0, + "output_index": 0, + "response_id": current_response_id, + "delta": output_tx["text"], }, ) ) @@ -1416,6 +1538,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): "object": "realtime.response", "id": current_response_id, "status": "in_progress", + "status_details": None, "output": [], "conversation_id": current_conversation_id, "modalities": tool_call_modalities, @@ -1460,6 +1583,29 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): }, ) ) + # conversation.item.added — Pipecat 1.3.x registers the + # call_id into _pending_function_calls inside + # _handle_evt_conversation_item_added, which is triggered + # by this event (NOT by response.output_item.added and NOT + # by the old conversation.item.created which Pipecat 1.3.x + # does not handle). Without this event the subsequent + # response.function_call_arguments.done finds an empty + # pending-calls dict and drops the tool invocation silently. + returned_message.append( + cast( + OpenAIRealtimeEvents, + { + "type": "conversation.item.added", + "event_id": f"event_{uuid.uuid4()}", + "previous_item_id": None, + "item": { + **function_call_item, + "status": "in_progress", + "arguments": "", + }, + }, + ) + ) # response.function_call_arguments.delta — Gemini delivers # the full arguments string in a single toolCall frame # rather than streaming partial chunks, so emit one delta @@ -1496,14 +1642,6 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): item={**function_call_item}, ) ) - # conversation.item.created - returned_message.append( - OpenAIRealtimeConversationItemCreated( - type="conversation.item.created", - event_id=f"event_{uuid.uuid4()}", - item={**function_call_item}, - ) - ) # response.done - close the response so clients can submit tool # results. Mirror the non-tool-call RESPONSE_DONE path: if Gemini @@ -1537,6 +1675,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): tool_call_responses_api_usage = LiteLLMCompletionResponsesConfig._transform_chat_completion_usage_to_responses_usage( _tool_call_chat_completion_usage, ) + _tool_usage_dict = tool_call_responses_api_usage.model_dump() + self._add_pipecat_usage_detail_aliases(_tool_usage_dict) tool_call_done_event = OpenAIRealtimeDoneEvent( type="response.done", event_id=f"event_{uuid.uuid4()}", @@ -1544,6 +1684,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): id=current_response_id, object="realtime.response", status="completed", + status_details=None, # type: ignore[typeddict-item] output=[ { "id": te["item_id"], @@ -1558,7 +1699,7 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ], conversation_id=current_conversation_id, modalities=tool_call_modalities, - usage=tool_call_responses_api_usage.model_dump(), + usage=_tool_usage_dict, ), ) tool_call_temperature = tool_call_generation_config.get("temperature") diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index 77a95bfa5a..644e96a7dd 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -265,7 +265,11 @@ class GeminiVideoConfig(BaseVideoConfig): { "instances": [ { - "prompt": "A cat playing with a ball of yarn" + "prompt": "A cat playing with a ball of yarn", + "image": { + "bytesBase64Encoded": "...", + "mimeType": "image/jpeg" + } } ], "parameters": { @@ -275,13 +279,18 @@ class GeminiVideoConfig(BaseVideoConfig): } } """ - instance = GeminiVideoGenerationInstance(prompt=prompt) + instance: GeminiVideoGenerationInstance = {"prompt": prompt} params_copy = video_create_optional_request_params.copy() - if "image" in params_copy and params_copy["image"] is not None: - image_data = _convert_image_to_gemini_format(params_copy["image"]) - params_copy["image"] = image_data + if "image" in params_copy: + image = params_copy.pop("image") + if image is not None: + if isinstance(image, dict): + image_data = image + else: + image_data = _convert_image_to_gemini_format(image) + instance["image"] = image_data parameters = GeminiVideoGenerationParameters(**params_copy) diff --git a/litellm/llms/github_copilot/chat/transformation.py b/litellm/llms/github_copilot/chat/transformation.py index 6651a3c60b..72dacb59f8 100644 --- a/litellm/llms/github_copilot/chat/transformation.py +++ b/litellm/llms/github_copilot/chat/transformation.py @@ -1,10 +1,15 @@ -from typing import List, Optional, Tuple +import json +from typing import Any, List, Optional, Tuple import os +import httpx + from litellm.exceptions import AuthenticationError +from litellm.llms.anthropic.chat.transformation import AnthropicConfig from litellm.llms.openai.openai import OpenAIConfig -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk +from litellm.types.utils import ModelResponse from ..authenticator import Authenticator from ..common_utils import ( @@ -164,3 +169,147 @@ class GithubCopilotConfig(OpenAIConfig): if content_type == "image_url": return True return False + + @staticmethod + def _parse_anthropic_native_content( + content_blocks: List[Any], + ) -> Tuple[str, List[ChatCompletionToolCallChunk], Optional[List[Any]]]: + """ + Parse Anthropic-native content blocks into OpenAI-compatible fields. + + Concatenates all text blocks, extracts tool_use blocks as tool_calls, and + preserves thinking blocks when present. + """ + ( + text_content, + _citations, + thinking_blocks, + _reasoning_content, + tool_calls, + _web_search_results, + _tool_results, + _compaction_blocks, + ) = AnthropicConfig().extract_response_content( + completion_response={"content": content_blocks} + ) + return text_content, tool_calls, thinking_blocks + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: "ModelResponse", + logging_obj: Any, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> "ModelResponse": + """ + Handle newer Copilot models (e.g. claude-opus-4.7, claude-opus-4.8) that + return Anthropic-native format responses without a `choices` array. + + Synthesizes the missing `choices` from Anthropic-native fields, then + delegates to the parent so all standard post-processing applies. + + See: https://github.com/BerriAI/litellm/issues/29391 + """ + try: + response_json = raw_response.json() + except Exception: + return super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) + + if not response_json.get("choices"): + content = "" + tool_calls: List[ChatCompletionToolCallChunk] = [] + thinking_blocks: Optional[List[Any]] = None + if "content" in response_json and isinstance( + response_json["content"], list + ): + content, tool_calls, thinking_blocks = ( + self._parse_anthropic_native_content(response_json["content"]) + ) + elif isinstance(response_json.get("content"), str): + content = response_json["content"] + + stop_reason = response_json.get("stop_reason") + finish_reason_map = { + "end_turn": "stop", + "max_tokens": "length", + "stop_sequence": "stop", + "tool_use": "tool_calls", + } + # Prefer tool_calls when blocks were extracted; otherwise map stop_reason. + if tool_calls: + finish_reason = "tool_calls" + elif stop_reason in finish_reason_map: + finish_reason = finish_reason_map[stop_reason] + elif content: + finish_reason = "stop" + else: + finish_reason = "length" + + message: dict = { + "role": "assistant", + "content": content if content or not tool_calls else None, + } + if tool_calls: + message["tool_calls"] = tool_calls + if thinking_blocks: + message["thinking_blocks"] = thinking_blocks + + response_json["choices"] = [ + { + "index": 0, + "message": message, + "finish_reason": finish_reason, + } + ] + + if "usage" in response_json: + usage = response_json["usage"] + if "input_tokens" in usage and "prompt_tokens" not in usage: + usage["prompt_tokens"] = usage["input_tokens"] + if "output_tokens" in usage and "completion_tokens" not in usage: + usage["completion_tokens"] = usage["output_tokens"] + if "total_tokens" not in usage: + usage["total_tokens"] = usage.get("prompt_tokens", 0) + usage.get( + "completion_tokens", 0 + ) + + # Build a patched response so super() sees valid JSON with choices + patched = httpx.Response( + status_code=raw_response.status_code, + headers=raw_response.headers, + content=json.dumps(response_json).encode(), + ) + raw_response = patched + + return super().transform_response( + model=model, + raw_response=raw_response, + model_response=model_response, + logging_obj=logging_obj, + request_data=request_data, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + encoding=encoding, + api_key=api_key, + json_mode=json_mode, + ) diff --git a/litellm/llms/huggingface/embedding/handler.py b/litellm/llms/huggingface/embedding/handler.py index 226f6b2eba..6be885b1f9 100644 --- a/litellm/llms/huggingface/embedding/handler.py +++ b/litellm/llms/huggingface/embedding/handler.py @@ -239,7 +239,7 @@ class HuggingFaceEmbedding(BaseLLM): model_response.model = model input_tokens = 0 for text in input: - input_tokens += len(encoding.encode(text)) + input_tokens += len(encoding.encode(text, disallowed_special=())) setattr( model_response, diff --git a/litellm/llms/inception/__init__.py b/litellm/llms/inception/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/inception/chat/__init__.py b/litellm/llms/inception/chat/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/inception/chat/transformation.py b/litellm/llms/inception/chat/transformation.py new file mode 100644 index 0000000000..d591f783a9 --- /dev/null +++ b/litellm/llms/inception/chat/transformation.py @@ -0,0 +1,54 @@ +""" +Translate from OpenAI's `/v1/chat/completions` to Inception's `/v1/chat/completions` + +Inception Labs (https://www.inceptionlabs.ai) serves the Mercury family of +diffusion LLMs through an OpenAI-compatible API, so we only need to point the +OpenAI-like handler at the Inception API base and pick up the Inception API key. +""" + +from typing import List, Optional, Tuple + +import litellm +from litellm.secret_managers.main import get_secret_str + +from ...openai_like.chat.transformation import OpenAILikeChatConfig + + +class InceptionChatConfig(OpenAILikeChatConfig): + """ + Inception is OpenAI-compatible with standard endpoints + """ + + @property + def custom_llm_provider(self) -> Optional[str]: + return "inception" + + def get_supported_openai_params(self, model: str) -> List: + return [ + "max_tokens", + "max_completion_tokens", + "temperature", + "stop", + "tools", + "tool_choice", + "stream", + "stream_options", + "response_format", + "reasoning_effort", + "reasoning_summary", + "reasoning_summary_wait", + "diffusing", + "realtime", + ] + + def _get_openai_compatible_provider_info( + self, api_base: Optional[str], api_key: Optional[str] + ) -> Tuple[Optional[str], Optional[str]]: + passed_api_base = api_base + api_base = api_base or get_secret_str("INCEPTION_API_BASE") or "https://api.inceptionlabs.ai/v1" # type: ignore + dynamic_api_key = api_key + if passed_api_base is None or api_key: + dynamic_api_key = ( + api_key or litellm.inception_key or get_secret_str("INCEPTION_API_KEY") + ) + return api_base, dynamic_api_key diff --git a/litellm/llms/inception/completion/__init__.py b/litellm/llms/inception/completion/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/inception/completion/transformation.py b/litellm/llms/inception/completion/transformation.py new file mode 100644 index 0000000000..1035042f6b --- /dev/null +++ b/litellm/llms/inception/completion/transformation.py @@ -0,0 +1,43 @@ +""" +Inception fill-in-the-middle (FIM) completions. + +Inception's FIM endpoint is OpenAI text-completion compatible: it takes a +`prompt` (prefix) plus an optional `suffix` and returns standard +`choices[].text`. It is served at `/v1/fim/completions` rather than +`/v1/completions`, so routing points the OpenAI client at the `/v1/fim` base +(see the `text-completion-inception` branch in `main.py`). +""" + +from typing import List + +from litellm.llms.openai.completion.transformation import OpenAITextCompletionConfig + + +class InceptionTextCompletionConfig(OpenAITextCompletionConfig): + def get_supported_openai_params(self, model: str) -> List: + return [ + "suffix", + "max_tokens", + "max_completion_tokens", + "top_p", + "frequency_penalty", + "presence_penalty", + "stop", + "stream", + "stream_options", + ] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + supported_params = self.get_supported_openai_params(model) + for param, value in non_default_params.items(): + if param == "max_completion_tokens": + optional_params["max_tokens"] = value + elif param in supported_params: + optional_params[param] = value + return optional_params diff --git a/litellm/llms/langflow/__init__.py b/litellm/llms/langflow/__init__.py new file mode 100644 index 0000000000..d1270fc91f --- /dev/null +++ b/litellm/llms/langflow/__init__.py @@ -0,0 +1 @@ +"""LangFlow LLM provider for LiteLLM.""" diff --git a/litellm/llms/langflow/a2a.py b/litellm/llms/langflow/a2a.py new file mode 100644 index 0000000000..dbe3e02401 --- /dev/null +++ b/litellm/llms/langflow/a2a.py @@ -0,0 +1,37 @@ +import hashlib +from typing import Any, Dict, Optional + + +def get_session_id_from_a2a_params(params: Dict[str, Any]) -> Optional[str]: + message = params.get("message", {}) + if isinstance(message, dict): + return message.get("contextId") + return getattr(message, "contextId", None) + + +def scope_session_to_principal(session_id: str, principal: Optional[str]) -> str: + """ + Bind a client-supplied A2A contextId to the authenticated principal. + + Without this, two distinct keys authorized for the same LangFlow agent could + set the same contextId and read/append to each other's LangFlow memory. The + principal is hashed (it is already a hashed token) so the raw value is never + sent to the LangFlow backend, while the original contextId is kept as a + suffix for operator-side correlation. + """ + if not principal: + return session_id + principal_prefix = hashlib.sha256(principal.encode("utf-8")).hexdigest()[:16] + return f"{principal_prefix}-{session_id}" + + +def merge_a2a_session_into_litellm_params( + litellm_params: Dict[str, Any], + params: Dict[str, Any], + principal: Optional[str] = None, +) -> Dict[str, Any]: + merged = dict(litellm_params) + session_id = get_session_id_from_a2a_params(params) + if session_id and "session_id" not in merged: + merged["session_id"] = scope_session_to_principal(session_id, principal) + return merged diff --git a/litellm/llms/langflow/chat/__init__.py b/litellm/llms/langflow/chat/__init__.py new file mode 100644 index 0000000000..286b12e31f --- /dev/null +++ b/litellm/llms/langflow/chat/__init__.py @@ -0,0 +1 @@ +"""LangFlow chat transformation.""" diff --git a/litellm/llms/langflow/chat/transformation.py b/litellm/llms/langflow/chat/transformation.py new file mode 100644 index 0000000000..f898163ad0 --- /dev/null +++ b/litellm/llms/langflow/chat/transformation.py @@ -0,0 +1,327 @@ +"""LangFlow run API: POST {api_base}/api/v1/run/{flow_id}""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union +from urllib.parse import quote + +import httpx + +from litellm._logging import verbose_logger +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.utils import CustomStreamWrapper + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + HTTPHandler = Any + AsyncHTTPHandler = Any + CustomStreamWrapper = Any + + +class LangFlowError(BaseLLMException): + """Exception class for LangFlow API errors.""" + + pass + + +class LangFlowConfig(BaseConfig): + """ + Configuration for the LangFlow API. + + LangFlow is a visual, low-code platform for building AI agents and pipelines. + Each flow has a unique flow_id and is invoked via a simple HTTP endpoint. + """ + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + def _get_openai_compatible_provider_info( + self, + api_base: Optional[str], + api_key: Optional[str], + ) -> Tuple[Optional[str], Optional[str]]: + from litellm.secret_managers.main import get_secret_str + + api_base = ( + api_base or get_secret_str("LANGFLOW_API_BASE") or "http://localhost:7860" + ) + api_key = api_key or get_secret_str("LANGFLOW_API_KEY") + return api_base, api_key + + def get_supported_openai_params(self, model: str) -> List[str]: + return ["stream"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + return optional_params + + def _get_flow_id(self, model: str, optional_params: dict) -> str: + """ + Extract flow_id from the authorized model name only. + + Model format: "langflow/{flow_id}". Request kwargs must not override + flow_id (would allow calling another flow with the same API key). + """ + if optional_params.get("flow_id") is not None: + raise LangFlowError( + status_code=400, + message=( + "flow_id cannot be set via request parameters; " + "use model langflow/{flow_id}" + ), + ) + + flow_id = (model.split("/", 1)[1] if "/" in model else model).strip() + if not flow_id: + raise LangFlowError( + status_code=400, + message="flow_id is required; use model langflow/{flow_id}", + ) + return flow_id + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + if api_base is None: + raise ValueError( + "api_base is required for LangFlow. Set it via LANGFLOW_API_BASE env var or api_base parameter." + ) + + api_base = api_base.rstrip("/") + flow_id = quote(self._get_flow_id(model, optional_params), safe="") + return f"{api_base}/api/v1/run/{flow_id}" + + def _get_last_user_message(self, messages: List[AllMessageValues]) -> str: + """Extract the text of the last user message to use as input_value.""" + for msg in reversed(messages): + if msg.get("role") == "user": + content = msg.get("content", "") + if isinstance(content, list): + content = convert_content_list_to_str(msg) + if not isinstance(content, str): + content = str(content) + return content + + # Fallback: use last message regardless of role + if messages: + content = messages[-1].get("content", "") + if isinstance(content, list): + content = convert_content_list_to_str(messages[-1]) + if not isinstance(content, str): + content = str(content) + return content + + return "" + + def _reject_caller_tweaks(self, params: dict) -> None: + if params.get("tweaks") is not None: + raise LangFlowError( + status_code=400, + message=( + "tweaks cannot be set via request parameters; they would " + "override the operator-configured LangFlow flow components" + ), + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to LangFlow format. + + LangFlow request format: + { + "input_value": "", + "input_type": "chat", + "output_type": "chat", + "session_id": "" + } + """ + self._reject_caller_tweaks(optional_params) + + input_value = self._get_last_user_message(messages) + + payload: Dict[str, Any] = { + "input_value": input_value, + "input_type": optional_params.get("input_type", "chat"), + "output_type": optional_params.get("output_type", "chat"), + } + + session_id = optional_params.get("session_id") + if session_id: + payload["session_id"] = session_id + + verbose_logger.debug(f"LangFlow request payload: {payload}") + return payload + + def _extract_content_from_response(self, response_json: dict) -> Optional[str]: + """ + Extract the assistant text from a LangFlow run response. + + Expected structure: + {"outputs": [{"outputs": [{"results": {"message": {"text": "..."}}}]}]} + + Returns None when no message text is present so the caller can surface an + explicit error instead of forwarding a raw JSON blob as the answer. + """ + outputs = response_json.get("outputs", []) + if not (isinstance(outputs, list) and outputs): + return None + + first_output = outputs[0] + if not isinstance(first_output, dict): + return None + + inner_outputs = first_output.get("outputs", []) + if not (isinstance(inner_outputs, list) and inner_outputs): + return None + + first_inner = inner_outputs[0] + if not isinstance(first_inner, dict): + return None + + results = first_inner.get("results", {}) + if isinstance(results, dict): + message = results.get("message", {}) + if isinstance(message, dict) and message.get("text"): + return message["text"] + + outputs_dict = first_inner.get("outputs", {}) + if isinstance(outputs_dict, dict): + for val in outputs_dict.values(): + if isinstance(val, dict): + msg = val.get("message", {}) + if isinstance(msg, dict) and msg.get("text"): + return msg["text"] + + return None + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + try: + response_json = raw_response.json() + except Exception as e: + raise LangFlowError( + message=f"LangFlow returned a non-JSON response: {e}", + status_code=raw_response.status_code, + ) + + verbose_logger.debug(f"LangFlow response: {response_json}") + + content = self._extract_content_from_response(response_json) + if content is None: + raise LangFlowError( + message=( + "Could not extract a message from the LangFlow response; " + "ensure the flow ends in a Chat Output component" + ), + status_code=500, + ) + + message = Message(content=content, role="assistant") + choice = Choices(finish_reason="stop", index=0, message=message) + + model_response.choices = [choice] + model_response.model = model + + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model=model, messages=messages) + completion_tokens = token_counter( + model=model, text=content, count_response_tokens=True + ) + usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=prompt_tokens + completion_tokens, + ) + setattr(model_response, "usage", usage) + except Exception as e: + verbose_logger.warning(f"Failed to calculate token usage: {e}") + + return model_response + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + self._reject_caller_tweaks(request_data) + return headers, None + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + headers["Content-Type"] = "application/json" + + if api_key: + headers["x-api-key"] = api_key + + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return LangFlowError(status_code=status_code, message=error_message) + + @property + def supports_stream_param_in_request_body(self) -> bool: + return False + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + return stream is True diff --git a/litellm/llms/lemonade/chat/transformation.py b/litellm/llms/lemonade/chat/transformation.py index 168d51a16d..fa546f9e14 100644 --- a/litellm/llms/lemonade/chat/transformation.py +++ b/litellm/llms/lemonade/chat/transformation.py @@ -3,10 +3,12 @@ Translate from OpenAI's `/v1/chat/completions` to Lemonade's `/v1/chat/completio """ from typing import Any, List, Optional, Tuple, Union +from urllib.parse import quote import httpx import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( @@ -18,6 +20,8 @@ from ...openai_like.chat.transformation import OpenAILikeChatConfig class LemonadeChatConfig(OpenAILikeChatConfig): + _DEFAULT_API_KEY = "lemonade" + repeat_penalty: Optional[float] = None functions: Optional[list] = None logit_bias: Optional[dict] = None @@ -68,7 +72,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): This method queries the Lemonade /models endpoint to retrieve the list of available models. Args: - api_key: Optional API key (Lemonade doesn't require authentication) + api_key: Optional API key for authenticated Lemonade servers api_base: Optional API base URL (defaults to LEMONADE_API_BASE env var or http://localhost:8000) Returns: @@ -87,6 +91,7 @@ class LemonadeChatConfig(OpenAILikeChatConfig): try: response = litellm.module_level_client.get( url=f"{api_base}/models", + headers=self._get_auth_headers(api_key), ) except Exception as e: raise ValueError( @@ -101,19 +106,131 @@ class LemonadeChatConfig(OpenAILikeChatConfig): model_list = response.json().get("data", []) return ["lemonade/" + model["id"] for model in model_list] + @staticmethod + def _get_positive_int(value: Any) -> Optional[int]: + if isinstance(value, bool): + return None + if isinstance(value, int) and value > 0: + return value + if isinstance(value, str): + try: + parsed = int(value) + except ValueError: + return None + if parsed > 0: + return parsed + return None + + @staticmethod + def _get_provider_specific_entry(model_info: dict) -> dict: + provider_specific_entry = model_info.get("provider_specific_entry") + if not isinstance(provider_specific_entry, dict): + provider_specific_entry = {} + else: + provider_specific_entry = provider_specific_entry.copy() + + for key in ("recipe_options", "context_window", "max_context_window"): + if key in model_info: + provider_specific_entry[key] = model_info[key] + + return provider_specific_entry + + def _get_context_window(self, model_info: dict) -> Optional[int]: + provider_specific_entry = self._get_provider_specific_entry(model_info) + recipe_options = provider_specific_entry.get("recipe_options") + if not isinstance(recipe_options, dict): + recipe_options = {} + + for value in ( + recipe_options.get("ctx_size"), + model_info.get("max_input_tokens"), + provider_specific_entry.get("context_window"), + provider_specific_entry.get("max_context_window"), + ): + parsed = self._get_positive_int(value) + if parsed is not None: + return parsed + return None + + def _get_default_model_info(self, model: str) -> dict: + return { + "key": "lemonade/" + model, + "litellm_provider": "lemonade", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + } + + def get_model_info( + self, + model: str, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + ) -> Any: + if model.startswith("lemonade/"): + model = model.split("/", 1)[1] + + api_base, api_key = self._get_openai_compatible_provider_info( + api_base=api_base, api_key=api_key + ) + encoded_model = quote(model, safe="") + + try: + response = litellm.module_level_client.get( + url=f"{api_base}/models/{encoded_model}", + headers=self._get_auth_headers(api_key), + ) + response.raise_for_status() + model_info = response.json() + except Exception: + verbose_logger.debug("LemonadeError: Could not get model info.") + return self._get_default_model_info(model) + + max_input_tokens = self._get_context_window(model_info) + max_output_tokens = self._get_positive_int(model_info.get("max_output_tokens")) + max_tokens = self._get_positive_int(model_info.get("max_tokens")) + provider_specific_entry = self._get_provider_specific_entry(model_info) + + model_info_response = self._get_default_model_info(model) + model_info_response.update( + { + "max_tokens": max_tokens or max_output_tokens, + "max_input_tokens": max_input_tokens, + "max_output_tokens": max_output_tokens, + } + ) + if provider_specific_entry: + model_info_response["provider_specific_entry"] = provider_specific_entry + return model_info_response + def _get_openai_compatible_provider_info( self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: # lemonade is openai compatible, we just need to set this to custom_openai and have the api_base be lemonade's endpoint + passed_api_base = api_base api_base = ( api_base or get_secret_str("LEMONADE_API_BASE") or "http://localhost:8000/api/v1" ) # type: ignore - # Lemonade doesn't check the key - key = "lemonade" + key = self._DEFAULT_API_KEY + if passed_api_base is None or api_key: + key = ( + api_key + or litellm.lemonade_key + or get_secret_str("LEMONADE_API_KEY") + or self._DEFAULT_API_KEY + ) return api_base, key + def _get_auth_headers(self, api_key: Optional[str]) -> dict: + if api_key is None or api_key == self._DEFAULT_API_KEY: + return {} + return {"Authorization": f"Bearer {api_key}"} + def transform_response( self, model: str, diff --git a/litellm/llms/moonshot/chat/transformation.py b/litellm/llms/moonshot/chat/transformation.py index 4eb00fd81d..da8687bce7 100644 --- a/litellm/llms/moonshot/chat/transformation.py +++ b/litellm/llms/moonshot/chat/transformation.py @@ -134,11 +134,15 @@ class MoonshotChatConfig(OpenAIGPTConfig): ########################################## # temperature limitations - # 1. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2] - # 2. If temperature < 0.3 and n > 1, KIMI will raise an exception. + # 1. reasoning models (kimi-k2.5, kimi-k2.6, ...) reject every temperature + # except 1, so the param is dropped and the model's default is used + # 2. `temperature` on KIMI API is [0, 1] but OpenAI is [0, 2] + # 3. If temperature < 0.3 and n > 1, KIMI will raise an exception. # If we enter this condition, we set the temperature to 0.3 as suggested by Moonshot AI ########################################## - if "temperature" in optional_params: + if supports_reasoning(model=model, custom_llm_provider="moonshot"): + optional_params.pop("temperature", None) + elif "temperature" in optional_params: if optional_params["temperature"] > 1: optional_params["temperature"] = 1 if optional_params["temperature"] < 0.3 and optional_params.get("n", 1) > 1: diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 8ca8b7d383..7d52ef14dd 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union +from typing import Any, List, Optional, Union import httpx @@ -65,7 +65,8 @@ class OllamaModelInfo(BaseLLMModelInfo): from litellm.secret_managers.main import get_secret_str return ( - os.environ.get("OLLAMA_API_KEY") + api_key + or os.environ.get("OLLAMA_API_KEY") or litellm.api_key or litellm.openai_key or get_secret_str("OLLAMA_API_KEY") @@ -78,13 +79,31 @@ class OllamaModelInfo(BaseLLMModelInfo): # env var OLLAMA_API_BASE or default return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" + @classmethod + def get_server_api_base(cls, api_base: Optional[str] = None) -> str: + api_base = cls.get_api_base(api_base).rstrip("/") + for suffix in ( + "/api/generate", + "/api/chat", + "/api/embed", + "/api/embeddings", + "/api/show", + "/api/tags", + ): + if api_base.endswith(suffix): + return api_base[: -len(suffix)] + return api_base + def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]: """ List all models available on the Ollama server via /api/tags endpoint. """ - base = self.get_api_base(api_base) - api_key = self.get_api_key() + passed_api_base = api_base + base = self.get_server_api_base(api_base) + api_key = ( + self.get_api_key(api_key) if passed_api_base is None or api_key else None + ) headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} names: set[str] = set() @@ -126,6 +145,103 @@ class OllamaModelInfo(BaseLLMModelInfo): result = sorted(names) return result + @staticmethod + def _strip_ollama_model_prefix(model: str) -> str: + if model.startswith("ollama/") or model.startswith("ollama_chat/"): + return model.split("/", 1)[1] + return model + + @staticmethod + def _is_static_ollama_model(model: str) -> bool: + from litellm import model_cost + + stripped_model = OllamaModelInfo._strip_ollama_model_prefix(model) + potential_model_names = { + model, + stripped_model, + "ollama/" + stripped_model, + "ollama_chat/" + stripped_model, + } + model_cost_keys = {key.lower() for key in model_cost} + return any(name.lower() in model_cost_keys for name in potential_model_names) + + @staticmethod + def _supports_function_calling(ollama_model_info: dict) -> bool: + _template: str = str(ollama_model_info.get("template", "") or "") + return "tools" in _template.lower() + + @staticmethod + def _get_max_tokens(ollama_model_info: dict) -> Optional[int]: + _model_info: dict = ollama_model_info.get("model_info", {}) + + for key, value in _model_info.items(): + if "context_length" in key: + return value + return None + + def get_runtime_model_info( + self, + model: str, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + ) -> dict[str, Any]: + from litellm import module_level_client + + model = self._strip_ollama_model_prefix(model) + passed_api_base = api_base + api_base = self.get_server_api_base(api_base) + api_key = ( + self.get_api_key(api_key) if passed_api_base is None or api_key else None + ) + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + + try: + response = module_level_client.post( + url=f"{api_base}/api/show", + json={"name": model}, + headers=headers, + ) + response.raise_for_status() + except Exception: + verbose_logger.debug("OllamaError: Could not get model info.") + return { + "key": model, + "litellm_provider": "ollama", + "mode": "chat", + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "max_tokens": None, + "max_input_tokens": None, + "max_output_tokens": None, + } + + model_info = response.json() + max_tokens = self._get_max_tokens(model_info) + + return { + "key": model, + "litellm_provider": "ollama", + "mode": "chat", + "supports_function_calling": self._supports_function_calling(model_info), + "input_cost_per_token": 0.0, + "output_cost_per_token": 0.0, + "max_tokens": max_tokens, + "max_input_tokens": max_tokens, + "max_output_tokens": max_tokens, + } + + def get_model_info( + self, + model: str, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + ) -> Optional[dict[str, Any]]: + if self._is_static_ollama_model(model): + return None + return self.get_runtime_model_info( + model=model, api_base=api_base, api_key=api_key + ) + def validate_environment( self, headers: dict, diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index 3298177675..7e34af43d4 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -6,7 +6,7 @@ from typing import TYPE_CHECKING, Any, AsyncIterator, Iterator, List, Optional, from httpx._models import Headers, Response import litellm -from litellm._logging import verbose_logger, verbose_proxy_logger +from litellm._logging import verbose_proxy_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages, ) @@ -17,19 +17,17 @@ from litellm.litellm_core_utils.prompt_templates.factory import ( ) from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues, ChatCompletionUsageBlock from litellm.types.utils import ( Delta, GenericStreamingChunk, - ModelInfoBase, ModelResponse, ModelResponseStream, ProviderField, StreamingChoices, ) -from ..common_utils import OllamaError, _convert_image +from ..common_utils import OllamaError, OllamaModelInfo, _convert_image if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -224,59 +222,18 @@ class OllamaConfig(BaseConfig): ) def get_model_info( - self, model: str, api_base: Optional[str] = None - ) -> ModelInfoBase: + self, + model: str, + api_base: Optional[str] = None, + api_key: Optional[str] = None, + ) -> Any: """ curl http://localhost:11434/api/show -d '{ "name": "mistral" }' """ - if model.startswith("ollama/") or model.startswith("ollama_chat/"): - model = model.split("/", 1)[1] - api_base = ( - api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" - ) - api_key = self.get_api_key() - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - - try: - response = litellm.module_level_client.post( - url=f"{api_base}/api/show", - json={"name": model}, - headers=headers, - ) - except Exception as e: - verbose_logger.debug( - "OllamaError: Could not get model info for %s from %s. Error: %s", - model, - api_base, - e, - ) - return ModelInfoBase( - key=model, - litellm_provider="ollama", - mode="chat", - input_cost_per_token=0.0, - output_cost_per_token=0.0, - max_tokens=None, - max_input_tokens=None, - max_output_tokens=None, - ) - - model_info = response.json() - - _max_tokens: Optional[int] = self._get_max_tokens(model_info) - - return ModelInfoBase( - key=model, - litellm_provider="ollama", - mode="chat", - supports_function_calling=self._supports_function_calling(model_info), - input_cost_per_token=0.0, - output_cost_per_token=0.0, - max_tokens=_max_tokens, - max_input_tokens=_max_tokens, - max_output_tokens=_max_tokens, + return OllamaModelInfo().get_model_info( + model=model, api_base=api_base, api_key=api_key ) def get_error_class( diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d413a24453..8c9a8228da 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -376,6 +376,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) guardrailed_texts = guardrailed_inputs.get("texts", []) + returned_tool_calls = guardrailed_inputs.get("tool_calls") + guardrailed_tool_calls: List[Dict[str, Any]] = ( + cast(List[Dict[str, Any]], returned_tool_calls) + if isinstance(returned_tool_calls, list) + and len(returned_tool_calls) == len(tool_calls_to_check) + else tool_calls_to_check + ) # Step 3: Map guardrail responses back to original response structure if guardrailed_texts and texts_to_check: @@ -386,10 +393,10 @@ class OpenAIChatCompletionsHandler(BaseTranslation): ) # Step 4: Apply guardrailed tool calls back to response - if tool_calls_to_check: + if guardrailed_tool_calls: await self._apply_guardrail_responses_to_output_tool_calls( response=response, - tool_calls=tool_calls_to_check, + tool_calls=guardrailed_tool_calls, task_mappings=tool_call_task_mappings, ) @@ -748,10 +755,11 @@ class OpenAIChatCompletionsHandler(BaseTranslation): task_mappings: List[Tuple[int, int]], ) -> None: """ - Apply guardrailed tool calls back to output response. + Apply guardrailed tool calls back to the output response. - The guardrail may have modified the tool_calls list in place, - so we apply the modified tool calls back to the original response. + The guardrail may return updated tool calls (either mutated in place or as + a new list), so we apply the provided tool calls back to the original + response. Override this method to customize how tool call responses are applied. """ diff --git a/litellm/llms/openai/responses/transformation.py b/litellm/llms/openai/responses/transformation.py index 5043d25ee3..c18f2216f6 100644 --- a/litellm/llms/openai/responses/transformation.py +++ b/litellm/llms/openai/responses/transformation.py @@ -299,11 +299,8 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig): or litellm.openai_key or get_secret_str("OPENAI_API_KEY") ) - headers.update( - { - "Authorization": f"Bearer {api_key}", - } - ) + headers.setdefault("Content-Type", "application/json") + headers["Authorization"] = f"Bearer {api_key}" return headers def get_complete_url( diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index b5e5aa4ea2..49b3801c82 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -114,5 +114,23 @@ "param_mappings": { "max_completion_tokens": "max_tokens" } + }, + "neosantara": { + "base_url": "https://api.neosantara.xyz/v1", + "api_key_env": "NEOSANTARA_API_KEY", + "api_base_env": "NEOSANTARA_API_BASE", + "param_mappings": { + "max_completion_tokens": "max_tokens" + }, + "supported_endpoints": ["/v1/chat/completions", "/v1/responses"] + }, + "tensormesh": { + "base_url": "https://serverless.tensormesh.ai/v1", + "api_key_env": "TENSORMESH_INFERENCE_API_KEY", + "api_base_env": "TENSORMESH_SERVERLESS_BASE_URL", + "base_class": "openai_gpt", + "param_mappings": { + "max_completion_tokens": "max_tokens" + } } } diff --git a/litellm/llms/snowflake/utils.py b/litellm/llms/snowflake/utils.py index d84efdd9fc..4f79006f6f 100644 --- a/litellm/llms/snowflake/utils.py +++ b/litellm/llms/snowflake/utils.py @@ -25,6 +25,7 @@ class SnowflakeBaseConfig: "temperature", "max_tokens", "top_p", + "stream", "response_format", "tools", "tool_choice", diff --git a/litellm/llms/soniox/__init__.py b/litellm/llms/soniox/__init__.py new file mode 100644 index 0000000000..778211a2a5 --- /dev/null +++ b/litellm/llms/soniox/__init__.py @@ -0,0 +1 @@ +"""Soniox LLM provider implementation.""" diff --git a/litellm/llms/soniox/audio_transcription/__init__.py b/litellm/llms/soniox/audio_transcription/__init__.py new file mode 100644 index 0000000000..3da6032ce6 --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/__init__.py @@ -0,0 +1 @@ +"""Soniox audio transcription implementation.""" diff --git a/litellm/llms/soniox/audio_transcription/handler.py b/litellm/llms/soniox/audio_transcription/handler.py new file mode 100644 index 0000000000..d4774fea46 --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/handler.py @@ -0,0 +1,802 @@ +""" +Handler for Soniox async speech-to-text transcription. + +Soniox's async transcription API requires multiple HTTP calls: + 1. (optional) POST /v1/files — upload a local audio file + 2. POST /v1/transcriptions — create a transcription job + 3. GET /v1/transcriptions/{id} — poll until status == "completed" + 4. GET /v1/transcriptions/{id}/transcript — fetch the transcript + 5. (optional) DELETE /v1/transcriptions/{id} — cleanup + 6. (optional) DELETE /v1/files/{id} — cleanup + +Because this does not fit the single-request shape of +`base_llm_http_handler.audio_transcriptions`, the dispatch in +`litellm.main.transcription()` routes Soniox requests directly to this +handler (analogous to the OpenAI / Azure transcription handlers). +""" + +import asyncio +import math +import time +from typing import ( + TYPE_CHECKING, + Any, + Coroutine, + Dict, + List, + Optional, + Tuple, + Union, +) + +import httpx + +from litellm.litellm_core_utils.audio_utils.utils import ( + get_audio_file_name, + process_audio_file, +) +from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + HTTPHandler, + _get_httpx_client, + get_async_httpx_client, +) +from litellm.llms.soniox.audio_transcription.transformation import ( + SonioxAudioTranscriptionConfig, +) +from litellm.llms.soniox.common_utils import ( + SONIOX_DEFAULT_CLEANUP, + SONIOX_DEFAULT_MAX_POLL_ATTEMPTS, + SONIOX_DEFAULT_POLL_INTERVAL, + SONIOX_MAX_POLL_ATTEMPTS, + SONIOX_MAX_POLL_INTERVAL, + SONIOX_MIN_POLL_INTERVAL, + SONIOX_SECRET_FIELDS, + SonioxException, + get_soniox_api_base, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObj, + ) +else: + LiteLLMLoggingObj = Any + + +class SonioxAudioTranscriptionHandler: + """Orchestrates the Soniox async transcription flow.""" + + # ------------------------------------------------------------------ + # Public entry points + # ------------------------------------------------------------------ + + def audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + max_retries: int, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, + atranscription: bool = False, + headers: Optional[Dict[str, Any]] = None, + provider_config: Optional[SonioxAudioTranscriptionConfig] = None, + ) -> Union[TranscriptionResponse, Coroutine[Any, Any, TranscriptionResponse]]: + """Sync/async dispatch for Soniox transcription requests. + + Note: ``max_retries`` is accepted for signature compatibility with + ``litellm.transcription`` but is **not yet implemented** for the Soniox + async pipeline. Transient HTTP failures during upload, create, poll, + or fetch will surface immediately. Wrap calls with the standard + ``litellm.Router`` / ``num_retries`` mechanism for retry behaviour. + """ + config = provider_config or SonioxAudioTranscriptionConfig() + + if atranscription is True: + return self._async_audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + client=client if isinstance(client, AsyncHTTPHandler) else None, + headers=headers or {}, + provider_config=config, + ) + + return self._sync_audio_transcriptions( + model=model, + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + model_response=model_response, + timeout=timeout, + logging_obj=logging_obj, + api_key=api_key, + api_base=api_base, + client=client if isinstance(client, HTTPHandler) else None, + headers=headers or {}, + provider_config=config, + ) + + # ------------------------------------------------------------------ + # Helpers shared between sync and async paths + # ------------------------------------------------------------------ + + def _prepare( + self, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str], + api_base: Optional[str], + provider_config: SonioxAudioTranscriptionConfig, + headers: Dict[str, Any], + ) -> Tuple[ + Dict[str, str], # auth headers + str, # api_base (no trailing slash) + Dict[str, Any], # body for POST /v1/transcriptions (without file_id/audio_url) + Dict[str, Any], # handler-only options (poll interval, cleanup, ...) + ]: + # Validate env -> auth headers. + auth_headers = provider_config.validate_environment( + headers=headers, + model="", # unused + messages=[], + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + ) + + base_url = get_soniox_api_base(api_base) + + # Operate on a local copy so we don't mutate the caller's dict + # (the caller may reuse `optional_params` for retries or logging). + params = dict(optional_params) + + # Pull handler-only kwargs out of params so they aren't sent + # to Soniox. + poll_interval = float( + params.pop("soniox_polling_interval", SONIOX_DEFAULT_POLL_INTERVAL) + ) + try: + max_attempts = int( + params.pop( + "soniox_max_polling_attempts", SONIOX_DEFAULT_MAX_POLL_ATTEMPTS + ) + ) + except (ValueError, OverflowError): + max_attempts = SONIOX_DEFAULT_MAX_POLL_ATTEMPTS + cleanup_raw = params.pop("soniox_cleanup", SONIOX_DEFAULT_CLEANUP) + if cleanup_raw is None: + cleanup: List[str] = [] + elif isinstance(cleanup_raw, str): + cleanup = [cleanup_raw] + else: + cleanup = list(cleanup_raw) + filename_override = params.pop("filename", None) + + # Server-side clamps. Caller-supplied poll settings (from request kwargs) + # are bounded so an authenticated caller cannot force a worker into a + # tight poll loop (zero interval) or pin it indefinitely (huge attempt + # count). Total polling time is bounded by + # SONIOX_MAX_POLL_ATTEMPTS * SONIOX_MAX_POLL_INTERVAL. + if not math.isfinite(poll_interval): + poll_interval = SONIOX_DEFAULT_POLL_INTERVAL + clamped_poll_interval = max( + SONIOX_MIN_POLL_INTERVAL, min(poll_interval, SONIOX_MAX_POLL_INTERVAL) + ) + clamped_max_attempts = max(1, min(max_attempts, SONIOX_MAX_POLL_ATTEMPTS)) + + handler_opts: Dict[str, Any] = { + "poll_interval": clamped_poll_interval, + "max_attempts": clamped_max_attempts, + "cleanup": cleanup, + "filename_override": filename_override, + "audio_url": params.pop("audio_url", None), + "file_id": params.pop("file_id", None), + } + + # Soniox does not accept `language` directly; map_openai_params should + # already have translated it, but drop any leftover to be safe. + params.pop("language", None) + + # response_format is handled by LiteLLM post-processing, not Soniox. + handler_opts["response_format"] = params.pop("response_format", None) + + return auth_headers, base_url, params, handler_opts + + def _build_create_body( + self, + model: str, + optional_params: dict, + handler_opts: Dict[str, Any], + file_id: Optional[str], + ) -> Dict[str, Any]: + body: Dict[str, Any] = {"model": model} + # Soniox-native passthrough fields + for key, value in optional_params.items(): + if value is None: + continue + body[key] = value + + if handler_opts.get("audio_url"): + body["audio_url"] = handler_opts["audio_url"] + if file_id: + body["file_id"] = file_id + + return body + + @staticmethod + def _redact_body_for_logging(body: Dict[str, Any]) -> Dict[str, Any]: + """Return a shallow copy of ``body`` with secret fields redacted. + + Soniox's create-transcription body can include + ``webhook_auth_header_value`` (a shared secret used to authenticate + webhook callbacks). Forwarding that value to logging callbacks would + let anyone with read access to those sinks forge webhook requests, so + we replace any value of a known secret-bearing field with the literal + ``"[REDACTED]"`` before logging. Non-secret fields are passed through + unchanged. + """ + if not body: + return body + redacted = dict(body) + for field in SONIOX_SECRET_FIELDS: + if field in redacted and redacted[field] is not None: + redacted[field] = "[REDACTED]" + return redacted + + @staticmethod + def _safe_log_pre_call( + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: str, + body: Dict[str, Any], + ) -> None: + try: + logging_obj.pre_call( + input=None, + api_key=api_key, + additional_args={ + "api_base": f"{api_base}/v1/transcriptions", + "atranscription": True, + "complete_input_dict": SonioxAudioTranscriptionHandler._redact_body_for_logging( + body + ), + }, + ) + except Exception: + # Logging hooks are best-effort: a misbehaving callback or third-party + # observability integration must never break a real Soniox call. + pass + + @staticmethod + def _safe_log_post_call( + logging_obj: LiteLLMLoggingObj, + audio_file: Optional[FileTypes], + api_key: Optional[str], + body: Dict[str, Any], + original_response: Any, + ) -> None: + try: + logging_obj.post_call( + input=get_audio_file_name(audio_file) if audio_file else None, + api_key=api_key, + additional_args={ + "complete_input_dict": SonioxAudioTranscriptionHandler._redact_body_for_logging( + body + ) + }, + original_response=original_response, + ) + except Exception: + # Logging hooks are best-effort: a misbehaving callback or third-party + # observability integration must never break a real Soniox call. + pass + + @staticmethod + def _raise_for_response( + response: httpx.Response, + provider_config: SonioxAudioTranscriptionConfig, + action: str, + ) -> None: + if response.status_code >= 400: + try: + payload = response.json() + message = ( + payload.get("error_message") + or payload.get("error") + or response.text + ) + except Exception: + message = response.text + raise provider_config.get_error_class( + error_message=f"Soniox {action} failed (HTTP {response.status_code}): {message}", + status_code=response.status_code, + headers=response.headers, + ) + + # ------------------------------------------------------------------ + # Sync flow + # ------------------------------------------------------------------ + + def _sync_audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[HTTPHandler], + headers: Dict[str, Any], + provider_config: SonioxAudioTranscriptionConfig, + ) -> TranscriptionResponse: + auth_headers, base_url, opt_params, handler_opts = self._prepare( + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + headers=headers, + ) + + http_client = ( + client + if isinstance(client, HTTPHandler) + else ( + _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + ) + ) + + file_id = handler_opts.get("file_id") + uploaded_file_id: Optional[str] = None + transcription_id: Optional[str] = None + + try: + if not file_id and not handler_opts.get("audio_url"): + if audio_file is None: + raise SonioxException( + message=( + "Soniox transcription requires one of: a file argument, " + "an `audio_url` kwarg, or a `file_id` kwarg." + ), + status_code=400, + headers=None, + ) + uploaded_file_id = self._sync_upload_file( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + audio_file=audio_file, + filename_override=handler_opts.get("filename_override"), + timeout=timeout, + provider_config=provider_config, + ) + file_id = uploaded_file_id + + body = self._build_create_body(model, opt_params, handler_opts, file_id) + self._safe_log_pre_call(logging_obj, api_key, base_url, body) + + create_resp = http_client.post( + url=f"{base_url}/v1/transcriptions", + headers=auth_headers, + json=body, + timeout=timeout, + ) + self._raise_for_response( + create_resp, provider_config, "create transcription" + ) + transcription_id = create_resp.json()["id"] + + transcription_meta = self._sync_poll_until_completed( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + transcription_id=transcription_id, + poll_interval=handler_opts["poll_interval"], + max_attempts=handler_opts["max_attempts"], + timeout=timeout, + provider_config=provider_config, + ) + + transcript_resp = http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response( + transcript_resp, provider_config, "fetch transcript" + ) + transcript = transcript_resp.json() + + payload = {"transcription": transcription_meta, "transcript": transcript} + response = provider_config._build_response_from_payload( + payload, + model_response=model_response, + response_format=handler_opts.get("response_format"), + ) + + self._safe_log_post_call(logging_obj, audio_file, api_key, body, payload) + + audio_duration_ms = transcription_meta.get("audio_duration_ms") + response._hidden_params.update( + { + "model": model, + "custom_llm_provider": "soniox", + "audio_transcription_duration": ( + float(audio_duration_ms) / 1000.0 + if audio_duration_ms is not None + else None + ), + } + ) + return response + finally: + self._sync_cleanup( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + cleanup=handler_opts["cleanup"], + file_id_to_cleanup=uploaded_file_id, + transcription_id=transcription_id, + timeout=timeout, + ) + + def _sync_upload_file( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + audio_file: FileTypes, + filename_override: Optional[str], + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> str: + processed = process_audio_file(audio_file) + filename = filename_override or processed.filename + files = { + "file": (filename, processed.file_content, processed.content_type), + } + # `Authorization` header is fine; httpx sets multipart Content-Type. + upload_headers = {"Authorization": auth_headers["Authorization"]} + resp = http_client.post( + url=f"{base_url}/v1/files", + headers=upload_headers, + files=files, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "upload file") + return resp.json()["id"] + + def _sync_poll_until_completed( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + transcription_id: str, + poll_interval: float, + max_attempts: int, + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> Dict[str, Any]: + for _ in range(max_attempts): + resp = http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "poll transcription") + data = resp.json() + status = data.get("status") + if status == "completed": + return data + if status == "error": + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} failed: " + f"{data.get('error_message') or data.get('error_type') or 'unknown error'}" + ), + status_code=500, + headers=resp.headers, + ) + time.sleep(poll_interval) + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} did not complete after " + f"{max_attempts} polling attempts (interval={poll_interval}s)." + ), + status_code=504, + headers={}, + ) + + def _sync_cleanup( + self, + http_client: HTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + cleanup: List[str], + file_id_to_cleanup: Optional[str], + transcription_id: Optional[str], + timeout: float, + ) -> None: + if not cleanup: + return + if "transcription" in cleanup and transcription_id: + try: + http_client.delete( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort: a failed delete leaves stale data on + # Soniox but must not mask the original transcription result + # (or, on the error path, the original error). + pass + if "file" in cleanup and file_id_to_cleanup: + try: + http_client.delete( + url=f"{base_url}/v1/files/{file_id_to_cleanup}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort; see comment above. + pass + + # ------------------------------------------------------------------ + # Async flow + # ------------------------------------------------------------------ + + async def _async_audio_transcriptions( + self, + model: str, + audio_file: Optional[FileTypes], + optional_params: dict, + litellm_params: dict, + model_response: TranscriptionResponse, + timeout: float, + logging_obj: LiteLLMLoggingObj, + api_key: Optional[str], + api_base: Optional[str], + client: Optional[AsyncHTTPHandler], + headers: Dict[str, Any], + provider_config: SonioxAudioTranscriptionConfig, + ) -> TranscriptionResponse: + import litellm + + auth_headers, base_url, opt_params, handler_opts = self._prepare( + audio_file=audio_file, + optional_params=optional_params, + litellm_params=litellm_params, + api_key=api_key, + api_base=api_base, + provider_config=provider_config, + headers=headers, + ) + + http_client = ( + client + if isinstance(client, AsyncHTTPHandler) + else ( + get_async_httpx_client( + llm_provider=litellm.LlmProviders.SONIOX, + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + ) + ) + + file_id = handler_opts.get("file_id") + uploaded_file_id: Optional[str] = None + transcription_id: Optional[str] = None + + try: + if not file_id and not handler_opts.get("audio_url"): + if audio_file is None: + raise SonioxException( + message=( + "Soniox transcription requires one of: a file argument, " + "an `audio_url` kwarg, or a `file_id` kwarg." + ), + status_code=400, + headers=None, + ) + uploaded_file_id = await self._async_upload_file( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + audio_file=audio_file, + filename_override=handler_opts.get("filename_override"), + timeout=timeout, + provider_config=provider_config, + ) + file_id = uploaded_file_id + + body = self._build_create_body(model, opt_params, handler_opts, file_id) + self._safe_log_pre_call(logging_obj, api_key, base_url, body) + + create_resp = await http_client.post( + url=f"{base_url}/v1/transcriptions", + headers=auth_headers, + json=body, + timeout=timeout, + ) + self._raise_for_response( + create_resp, provider_config, "create transcription" + ) + transcription_id = create_resp.json()["id"] + + transcription_meta = await self._async_poll_until_completed( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + transcription_id=transcription_id, + poll_interval=handler_opts["poll_interval"], + max_attempts=handler_opts["max_attempts"], + timeout=timeout, + provider_config=provider_config, + ) + + transcript_resp = await http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}/transcript", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response( + transcript_resp, provider_config, "fetch transcript" + ) + transcript = transcript_resp.json() + + payload = {"transcription": transcription_meta, "transcript": transcript} + response = provider_config._build_response_from_payload( + payload, + model_response=model_response, + response_format=handler_opts.get("response_format"), + ) + + self._safe_log_post_call(logging_obj, audio_file, api_key, body, payload) + + audio_duration_ms = transcription_meta.get("audio_duration_ms") + response._hidden_params.update( + { + "model": model, + "custom_llm_provider": "soniox", + "audio_transcription_duration": ( + float(audio_duration_ms) / 1000.0 + if audio_duration_ms is not None + else None + ), + } + ) + return response + finally: + await self._async_cleanup( + http_client=http_client, + base_url=base_url, + auth_headers=auth_headers, + cleanup=handler_opts["cleanup"], + file_id_to_cleanup=uploaded_file_id, + transcription_id=transcription_id, + timeout=timeout, + ) + + async def _async_upload_file( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + audio_file: FileTypes, + filename_override: Optional[str], + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> str: + processed = process_audio_file(audio_file) + filename = filename_override or processed.filename + files = { + "file": (filename, processed.file_content, processed.content_type), + } + upload_headers = {"Authorization": auth_headers["Authorization"]} + resp = await http_client.post( + url=f"{base_url}/v1/files", + headers=upload_headers, + files=files, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "upload file") + return resp.json()["id"] + + async def _async_poll_until_completed( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + transcription_id: str, + poll_interval: float, + max_attempts: int, + timeout: float, + provider_config: SonioxAudioTranscriptionConfig, + ) -> Dict[str, Any]: + for _ in range(max_attempts): + resp = await http_client.get( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + self._raise_for_response(resp, provider_config, "poll transcription") + data = resp.json() + status = data.get("status") + if status == "completed": + return data + if status == "error": + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} failed: " + f"{data.get('error_message') or data.get('error_type') or 'unknown error'}" + ), + status_code=500, + headers=resp.headers, + ) + await asyncio.sleep(poll_interval) + raise provider_config.get_error_class( + error_message=( + f"Soniox transcription {transcription_id} did not complete after " + f"{max_attempts} polling attempts (interval={poll_interval}s)." + ), + status_code=504, + headers={}, + ) + + async def _async_cleanup( + self, + http_client: AsyncHTTPHandler, + base_url: str, + auth_headers: Dict[str, str], + cleanup: List[str], + file_id_to_cleanup: Optional[str], + transcription_id: Optional[str], + timeout: float, + ) -> None: + if not cleanup: + return + if "transcription" in cleanup and transcription_id: + try: + await http_client.delete( + url=f"{base_url}/v1/transcriptions/{transcription_id}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort: a failed delete leaves stale data on + # Soniox but must not mask the original transcription result + # (or, on the error path, the original error). + pass + if "file" in cleanup and file_id_to_cleanup: + try: + await http_client.delete( + url=f"{base_url}/v1/files/{file_id_to_cleanup}", + headers=auth_headers, + timeout=timeout, + ) + except Exception: + # Cleanup is best-effort; see comment above. + pass diff --git a/litellm/llms/soniox/audio_transcription/transformation.py b/litellm/llms/soniox/audio_transcription/transformation.py new file mode 100644 index 0000000000..681d4352df --- /dev/null +++ b/litellm/llms/soniox/audio_transcription/transformation.py @@ -0,0 +1,281 @@ +""" +Translates between OpenAI's `/v1/audio/transcriptions` shape and Soniox's +async transcription API (https://soniox.com/docs/stt/async/async-transcription). + +This config covers parameter mapping, env validation and response shaping. +The actual orchestration (file upload -> create -> poll -> fetch -> cleanup) +lives in `litellm.llms.soniox.audio_transcription.handler`, because Soniox's +async API requires multiple HTTP calls and does not fit the single-request +contract of `base_llm_http_handler.audio_transcriptions`. +""" + +from typing import Any, Dict, List, Optional, Union + +from httpx import Headers, Response + +from litellm.llms.base_llm.audio_transcription.transformation import ( + AudioTranscriptionRequestData, + BaseAudioTranscriptionConfig, +) +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.soniox.common_utils import ( + SonioxException, + get_soniox_api_base, + get_soniox_api_key, + render_soniox_tokens, + render_soniox_tokens_as_srt, + render_soniox_tokens_as_vtt, +) +from litellm.types.llms.openai import ( + AllMessageValues, + OpenAIAudioTranscriptionOptionalParams, +) +from litellm.types.utils import FileTypes, TranscriptionResponse + +# Soniox-native kwargs the user can pass through `litellm.transcription(..., **kwargs)` +# in addition to the standard OpenAI params. +SONIOX_PASSTHROUGH_PARAMS: List[str] = [ + "language_hints", + "language_hints_strict", + "enable_language_identification", + "enable_speaker_diarization", + "context", + "translation", + "client_reference_id", + "webhook_url", + "webhook_auth_header_name", + "webhook_auth_header_value", + "audio_url", + "file_id", +] + +# Handler-only kwargs (consumed by the handler, not sent to Soniox). +SONIOX_HANDLER_ONLY_PARAMS: List[str] = [ + "soniox_polling_interval", + "soniox_max_polling_attempts", + "soniox_cleanup", + "filename", +] + + +class SonioxAudioTranscriptionConfig(BaseAudioTranscriptionConfig): + """Configuration for Soniox async speech-to-text transcription.""" + + def get_supported_openai_params( + self, model: str + ) -> List[OpenAIAudioTranscriptionOptionalParams]: + # `language` is mapped onto Soniox's `language_hints`. + # `response_format` is handled by LiteLLM (Soniox doesn't support + # SRT/VTT natively but we synthesize them from token timestamps). + return ["language", "response_format"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + # Translate the OpenAI `language` param into Soniox `language_hints`. + if "language" in non_default_params and non_default_params["language"]: + language = non_default_params["language"] + existing_hints = optional_params.get("language_hints") + if not existing_hints: + optional_params["language_hints"] = [language] + elif language not in existing_hints: + optional_params["language_hints"] = [language] + list(existing_hints) + + # Capture response_format for post-processing (not sent to Soniox API). + if "response_format" in non_default_params: + optional_params["response_format"] = non_default_params["response_format"] + + # Pass through Soniox-native kwargs unchanged. + for key in SONIOX_PASSTHROUGH_PARAMS + SONIOX_HANDLER_ONLY_PARAMS: + if key in non_default_params and non_default_params[key] is not None: + optional_params[key] = non_default_params[key] + + return optional_params + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, Headers] + ) -> BaseLLMException: + return SonioxException( + message=error_message, status_code=status_code, headers=headers + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + resolved_key = get_soniox_api_key(api_key) + if not resolved_key: + raise SonioxException( + message=( + "Missing Soniox API key. Set the SONIOX_API_KEY environment " + "variable or pass api_key=... to litellm.transcription()." + ), + status_code=401, + headers=None, + ) + + merged_headers: Dict[str, str] = { + "Authorization": f"Bearer {resolved_key}", + } + if headers: + merged_headers.update(headers) + return merged_headers + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + # The handler builds per-call URLs (uploads, create, poll, fetch, delete); + # we just return the resolved base. + return get_soniox_api_base(api_base) + + def transform_audio_transcription_request( + self, + model: str, + audio_file: FileTypes, + optional_params: dict, + litellm_params: dict, + ) -> AudioTranscriptionRequestData: + """ + Build the JSON body for `POST /v1/transcriptions`. + + The handler is responsible for the file upload (if `audio_file` is bytes) + and for filling in `file_id`/`audio_url`. This method exists so the + config can be exercised in isolation by unit tests. + """ + body: Dict[str, Any] = {"model": model} + + for key in SONIOX_PASSTHROUGH_PARAMS: + value = optional_params.get(key) + if value is not None: + body[key] = value + + return AudioTranscriptionRequestData( + data=body, files=None, content_type="application/json" + ) + + def transform_audio_transcription_response( + self, + raw_response: Response, + model_response: Optional[TranscriptionResponse] = None, + ) -> TranscriptionResponse: + """ + Build a TranscriptionResponse from a Soniox transcript payload. + + `raw_response.json()` may be either: + - a Soniox transcript object: `{"id": "...", "text": "...", "tokens": [...]}` + - or a merged envelope: `{"transcription": {...}, "transcript": {...}}` + produced by the handler so transcription metadata is also available. + """ + try: + payload = raw_response.json() + except Exception as exc: + raise SonioxException( + message=f"Failed to parse Soniox response: {exc}", + status_code=getattr(raw_response, "status_code", 500), + headers=getattr(raw_response, "headers", None), + ) + + return self._build_response_from_payload(payload, model_response=model_response) + + def _build_response_from_payload( + self, + payload: Dict[str, Any], + model_response: Optional[TranscriptionResponse] = None, + response_format: Optional[str] = None, + ) -> TranscriptionResponse: + """Shared response-building logic (also used by the handler).""" + transcription_meta: Dict[str, Any] = {} + transcript: Dict[str, Any] + + if isinstance(payload, dict) and "transcript" in payload: + transcription_meta = payload.get("transcription") or {} + transcript = payload.get("transcript") or {} + else: + transcript = payload if isinstance(payload, dict) else {} + + tokens: List[Dict[str, Any]] = transcript.get("tokens") or [] + + # Decide what to put in `text` based on response_format: + # - "srt": render tokens as SRT subtitles (synthesized from timestamps) + # - "vtt": render tokens as WebVTT subtitles (synthesized from timestamps) + # - "verbose_json": return JSON with word-level timing (handled below) + # - "text" / "json" / None: default plain text rendering + if response_format == "srt" and tokens: + text = render_soniox_tokens_as_srt(tokens) + elif response_format == "vtt" and tokens: + text = render_soniox_tokens_as_vtt(tokens) + else: + # Default text rendering (also used for "json", "text", + # "verbose_json") + has_speaker = any(t.get("speaker") is not None for t in tokens) + has_language = any(t.get("language") is not None for t in tokens) + + if (has_speaker or has_language) and tokens: + text = render_soniox_tokens(tokens) + elif transcript.get("text"): + text = transcript["text"] + elif tokens: + text = "".join(t.get("text", "") for t in tokens) + else: + text = "" + + response = model_response or TranscriptionResponse(text=text) + response.text = text + response["task"] = "transcribe" + + # Best-effort metadata fields matching OpenAI's verbose_json shape. + if transcription_meta.get("audio_duration_ms") is not None: + try: + response["duration"] = ( + float(transcription_meta["audio_duration_ms"]) / 1000.0 + ) + except (TypeError, ValueError): + pass + + # Surface a representative language if all tokens agree. + has_language = any(t.get("language") is not None for t in tokens) + if has_language: + languages = {t.get("language") for t in tokens if t.get("language")} + if len(languages) == 1: + response["language"] = next(iter(languages)) + + # For verbose_json, include word-level timing from tokens. + if response_format == "verbose_json" and tokens: + words: List[Dict[str, Any]] = [] + for token in tokens: + word_entry: Dict[str, Any] = {"word": token.get("text", "")} + if token.get("start_ms") is not None: + word_entry["start"] = float(token["start_ms"]) / 1000.0 + if token.get("end_ms") is not None: + word_entry["end"] = float(token["end_ms"]) / 1000.0 + words.append(word_entry) + if words: + response["words"] = words + + # Stash the raw Soniox payload so power-users can read tokens, segments, + # speaker/language data, etc. + response._hidden_params.update( + { + "soniox_raw": { + "transcription": transcription_meta, + "transcript": transcript, + } + } + ) + return response diff --git a/litellm/llms/soniox/common_utils.py b/litellm/llms/soniox/common_utils.py new file mode 100644 index 0000000000..01f8062fc9 --- /dev/null +++ b/litellm/llms/soniox/common_utils.py @@ -0,0 +1,274 @@ +""" +Shared utilities for the Soniox provider (https://soniox.com). +""" + +from typing import Any, Dict, List, Optional + +from litellm.llms.base_llm.chat.transformation import BaseLLMException + +# Soniox API base URL. +SONIOX_API_BASE: str = "https://api.soniox.com" + +# Default polling interval in seconds when waiting for an async transcription +# to finish. Mirrors the Soniox SDK default. +SONIOX_DEFAULT_POLL_INTERVAL: float = 1.0 + +# Minimum polling interval (in seconds) the server will accept from caller- +# supplied `soniox_polling_interval` kwargs. Prevents an authenticated caller +# from forcing a worker into a tight poll loop with a zero/near-zero interval. +SONIOX_MIN_POLL_INTERVAL: float = 0.5 + +# Maximum polling interval (in seconds). Prevents a caller from setting an +# excessively large or non-finite interval that would keep a worker sleeping +# far longer than necessary between status checks. +SONIOX_MAX_POLL_INTERVAL: float = 60.0 + +# Default maximum number of polling attempts (1800 attempts * 1s ~= 30 minutes). +SONIOX_DEFAULT_MAX_POLL_ATTEMPTS: int = 1800 + +# Hard upper bound on polling attempts. Combined with `SONIOX_MIN_POLL_INTERVAL` +# this caps total polling time per request at ~3000s (50 minutes), preventing a +# caller from pinning a worker indefinitely via a huge attempt count. +SONIOX_MAX_POLL_ATTEMPTS: int = 6000 + +# Default cleanup behaviour: delete both the uploaded file (if any) and the +# transcription record after the transcript has been fetched. +SONIOX_DEFAULT_CLEANUP: List[str] = ["file", "transcription"] + +# Body fields that may carry secrets and must be redacted before being +# forwarded to logging callbacks. Soniox accepts a webhook auth header value +# alongside the create-transcription request; that value lets the recipient +# authenticate webhook callbacks and must not leak into observability sinks. +SONIOX_SECRET_FIELDS: List[str] = ["webhook_auth_header_value"] + + +class SonioxException(BaseLLMException): + """Provider-specific exception class for Soniox.""" + + pass + + +def get_soniox_api_key(api_key: Optional[str] = None) -> Optional[str]: + """Resolve the Soniox API key from arg or env var.""" + # Local import to avoid a circular import: litellm.secret_managers.main + # imports from litellm at top-level. + from litellm.secret_managers.main import get_secret_str + + return api_key or get_secret_str("SONIOX_API_KEY") + + +def get_soniox_api_base(api_base: Optional[str] = None) -> str: + """Resolve the Soniox API base URL from arg or env var (defaults to public API).""" + from litellm.secret_managers.main import get_secret_str + + base = api_base or get_secret_str("SONIOX_API_BASE") or SONIOX_API_BASE + return base.rstrip("/") + + +def render_soniox_tokens(tokens: List[Dict[str, Any]]) -> str: + """ + Render a list of Soniox tokens to a readable transcript string. + + Mirrors the behaviour of the official Soniox SDK's `renderTokens` helper: + - When the speaker changes, a `Speaker N:` tag is inserted. + - When the language changes, a `[lang]` (or `[Translation][lang]`) tag is + inserted. + + If neither speaker nor language information is present on any token (i.e. + diarization and language identification are disabled), the function simply + concatenates the token texts. + """ + if not tokens: + return "" + + text_parts: List[str] = [] + current_speaker: Optional[Any] = None + current_language: Optional[Any] = None + + for token in tokens: + text = token.get("text", "") + speaker = token.get("speaker") + language = token.get("language") + is_translation = token.get("translation_status") == "translation" + + # Speaker changed -> emit a speaker tag. + if speaker is not None and speaker != current_speaker: + if current_speaker is not None: + text_parts.append("\n\n") + current_speaker = speaker + current_language = None # reset language whenever speaker changes + text_parts.append(f"Speaker {current_speaker}:") + + # Language changed -> emit a language (or translation) tag. + if language is not None and language != current_language: + current_language = language + prefix = "[Translation] " if is_translation else "" + text_parts.append(f"\n{prefix}[{current_language}] ") + text = text.lstrip() if isinstance(text, str) else text + + text_parts.append(text) + + return "".join(text_parts) + + +# --------------------------------------------------------------------------- +# SRT / VTT subtitle rendering +# --------------------------------------------------------------------------- + +# Maximum number of tokens to group into a single subtitle cue. +_CUE_MAX_TOKENS: int = 15 + +# Maximum duration (in ms) for a single cue before forcing a break. +_CUE_MAX_DURATION_MS: int = 5000 + + +def _format_timestamp_srt(ms: int) -> str: + """Format milliseconds as SRT timestamp: HH:MM:SS,mmm""" + if ms < 0: + ms = 0 + hours = ms // 3_600_000 + ms %= 3_600_000 + minutes = ms // 60_000 + ms %= 60_000 + seconds = ms // 1_000 + millis = ms % 1_000 + return f"{hours:02d}:{minutes:02d}:{seconds:02d},{millis:03d}" + + +def _format_timestamp_vtt(ms: int) -> str: + """Format milliseconds as VTT timestamp: HH:MM:SS.mmm""" + if ms < 0: + ms = 0 + hours = ms // 3_600_000 + ms %= 3_600_000 + minutes = ms // 60_000 + ms %= 60_000 + seconds = ms // 1_000 + millis = ms % 1_000 + return f"{hours:02d}:{minutes:02d}:{seconds:02d}.{millis:03d}" + + +def _group_tokens_into_cues( + tokens: List[Dict[str, Any]], +) -> List[Dict[str, Any]]: + """ + Group Soniox tokens into subtitle cues. + + Each cue has: + - start_ms: int + - end_ms: int + - text: str + + Grouping heuristics: + - A new cue starts when token count exceeds _CUE_MAX_TOKENS. + - A new cue starts when duration exceeds _CUE_MAX_DURATION_MS. + - A new cue starts when the speaker changes (if diarization is on). + - Tokens without timestamps are appended to the current cue. + """ + cues: List[Dict[str, Any]] = [] + current_tokens: List[str] = [] + current_start: Optional[int] = None + current_end: Optional[int] = None + current_speaker: Optional[Any] = None + + def _flush() -> None: + if current_tokens and current_start is not None: + text = "".join(current_tokens).strip() + if text: + cues.append( + { + "start_ms": current_start, + "end_ms": ( + current_end if current_end is not None else current_start + ), + "text": text, + } + ) + + for token in tokens: + start_ms = token.get("start_ms") + end_ms = token.get("end_ms") + text = token.get("text", "") + speaker = token.get("speaker") + + # Skip tokens with no timestamp data entirely if we have no cue started + if start_ms is None and current_start is None: + continue + + # Speaker change forces a new cue + if speaker is not None and speaker != current_speaker: + _flush() + current_tokens = [] + current_start = start_ms + current_end = end_ms + current_speaker = speaker + current_tokens.append(text) + continue + + # Duration or token count exceeded -> flush + should_break = False + if len(current_tokens) >= _CUE_MAX_TOKENS: + should_break = True + elif ( + current_start is not None + and start_ms is not None + and (start_ms - current_start) >= _CUE_MAX_DURATION_MS + ): + should_break = True + + if should_break: + _flush() + current_tokens = [] + current_start = start_ms + current_end = end_ms + current_tokens.append(text) + else: + if current_start is None: + current_start = start_ms + if end_ms is not None: + current_end = end_ms + current_tokens.append(text) + + _flush() + return cues + + +def render_soniox_tokens_as_srt(tokens: List[Dict[str, Any]]) -> str: + """ + Render Soniox tokens as SRT (SubRip) subtitle format. + + Returns an empty string if no tokens have timestamp data. + """ + cues = _group_tokens_into_cues(tokens) + if not cues: + return "" + + lines: List[str] = [] + for idx, cue in enumerate(cues, start=1): + start = _format_timestamp_srt(cue["start_ms"]) + end = _format_timestamp_srt(cue["end_ms"]) + lines.append(str(idx)) + lines.append(f"{start} --> {end}") + lines.append(cue["text"]) + lines.append("") # blank line between cues + + return "\n".join(lines) + + +def render_soniox_tokens_as_vtt(tokens: List[Dict[str, Any]]) -> str: + """ + Render Soniox tokens as WebVTT subtitle format. + + Returns the VTT header even if no cues are present. + """ + cues = _group_tokens_into_cues(tokens) + + lines: List[str] = ["WEBVTT", ""] + for cue in cues: + start = _format_timestamp_vtt(cue["start_ms"]) + end = _format_timestamp_vtt(cue["end_ms"]) + lines.append(f"{start} --> {end}") + lines.append(cue["text"]) + lines.append("") # blank line between cues + + return "\n".join(lines) diff --git a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py index 3f945adca0..e9f08f403f 100644 --- a/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py +++ b/litellm/llms/vertex_ai/context_caching/vertex_ai_context_caching.py @@ -337,6 +337,7 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None tools = optional_params.pop("tools", None) + tool_choice = optional_params.pop("tool_choice", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -371,7 +372,7 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) google_cache_name = self.check_cache( cache_key=generated_cache_key, @@ -402,6 +403,8 @@ class ContextCachingEndpoints(VertexBase): ) cached_content_request_body["tools"] = tools + if tool_choice is not None: + cached_content_request_body["toolConfig"] = tool_choice ## LOGGING logging_obj.pre_call( @@ -487,6 +490,7 @@ class ContextCachingEndpoints(VertexBase): return messages, optional_params, None tools = optional_params.pop("tools", None) + tool_choice = optional_params.pop("tool_choice", None) ## AUTHORIZATION ## token, url = self._get_token_and_url_context_caching( @@ -518,7 +522,7 @@ class ContextCachingEndpoints(VertexBase): ## CHECK IF CACHED ALREADY generated_cache_key = local_cache_obj.get_cache_key( - messages=cached_messages, tools=tools, model=model + messages=cached_messages, tools=tools, tool_choice=tool_choice, model=model ) google_cache_name = await self.async_check_cache( cache_key=generated_cache_key, @@ -550,6 +554,8 @@ class ContextCachingEndpoints(VertexBase): ) cached_content_request_body["tools"] = tools + if tool_choice is not None: + cached_content_request_body["toolConfig"] = tool_choice ## LOGGING logging_obj.pre_call( diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 4f5846cc5b..c578d6cd28 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -996,7 +996,19 @@ def _gemini_convert_messages_with_history( # noqa: PLR0915 excluded_keys=["thoughtSignature"], ): assistant_content.append(gemini_tool_call_part) - last_message_with_tool_calls = assistant_msg + # Only record this as the active tool-call message when it actually + # carries tool calls. The `if` guard above is also entered for a + # text-only assistant message (`assistant_msg.get("tool_calls", []) + # is not None` is True for an empty list), so without this check a + # later assistant message with no tool calls would clobber the + # reference. The following tool result would then be matched against + # an assistant message that has no tool_calls, raising "Missing + # corresponding tool call for tool response message". + if ( + assistant_msg.get("tool_calls") + or assistant_msg.get("function_call") is not None + ): + last_message_with_tool_calls = assistant_msg ## HANDLE SERVER-SIDE TOOL INVOCATIONS (context circulation) _psf = assistant_msg.get("provider_specific_fields") @@ -1109,6 +1121,61 @@ def _pop_and_merge_extra_body(data: RequestBody, optional_params: dict) -> None: data_dict[k] = v +def _has_google_maps_tool(tools: Optional[Any]) -> bool: + """Return True if any tool object in the list has a 'googleMaps' key.""" + if not isinstance(tools, list): + return False + return any( + isinstance(t, dict) and VertexToolName.GOOGLE_MAPS.value in t for t in tools + ) + + +def _rewrite_mime_type_to_response_format(generation_config: GenerationConfig) -> None: + """ + Convert response_mime_type + response_json_schema/response_schema to the newer + responseFormat structure when googleMaps is present in tools. + + The Gemini API rejects the combination of googleMaps + response_mime_type: + 'application/json' with the error: + "Google Maps tool with a response mime type: 'application/json' is unsupported" + + The newer responseFormat field supports this combination on both the Gemini API + (generativelanguage.googleapis.com) and Vertex AI endpoints. + + Before: + generationConfig: { + response_mime_type: "application/json", + response_json_schema: {...} + } + + After: + generationConfig: { + responseFormat: { + "text": {"mimeType": "APPLICATION_JSON", "schema": {...}} + } + } + """ + schema = generation_config.pop("response_json_schema", None) # type: ignore[misc] + if schema is None: + schema = generation_config.pop("response_schema", None) # type: ignore[misc] + generation_config.pop("response_mime_type", None) # type: ignore[misc] + + response_format: Dict[str, Any] = {"text": {"mimeType": "APPLICATION_JSON"}} + if schema is not None: + response_format["text"]["schema"] = schema + generation_config["responseFormat"] = response_format # type: ignore[typeddict-unknown-key] + + +def _rewrite_google_maps_response_format(data: RequestBody) -> None: + generation_config = cast(Optional[GenerationConfig], data.get("generationConfig")) + if ( + isinstance(generation_config, dict) + and _has_google_maps_tool(data.get("tools")) + and generation_config.get("response_mime_type") == "application/json" + ): + _rewrite_mime_type_to_response_format(generation_config) + + def _transform_request_body( # noqa: PLR0915 messages: List[AllMessageValues], model: str, @@ -1234,6 +1301,7 @@ def _transform_request_body( # noqa: PLR0915 if labels and custom_llm_provider != LlmProviders.GEMINI: data["labels"] = labels _pop_and_merge_extra_body(data, optional_params) + _rewrite_google_maps_response_format(data) except Exception as e: raise e diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 189ac7a7f6..5cd02293f1 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1147,6 +1147,26 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): return cast(dict, speech_config) + @staticmethod + def _apply_include_server_side_tool_invocations( + non_default_params: Dict, + optional_params: Dict, + ) -> None: + """ + Set include_server_side_tool_invocations before tools are mapped. + + map_openai_params iterates non_default_params in request order; if tools + appear before this flag, _resolve_search_tool_conflict would drop search + tools before the flag is applied. + """ + for key in ( + "include_server_side_tool_invocations", + "includeServerSideToolInvocations", + ): + if non_default_params.get(key) is True or optional_params.get(key) is True: + optional_params["include_server_side_tool_invocations"] = True + return + def map_openai_params( # noqa: PLR0915 self, non_default_params: Dict, @@ -1154,6 +1174,9 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): model: str, drop_params: bool, ) -> Dict: + self._apply_include_server_side_tool_invocations( + non_default_params, optional_params + ) gemini_sampling_params_warned: bool = False for param, value in non_default_params.items(): if param == "temperature": diff --git a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py index 61fb848b40..46dedb3d0a 100644 --- a/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py +++ b/litellm/llms/vertex_ai/vector_stores/search_api/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import httpx from litellm import get_model_info +from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.url_utils import encode_url_path_segment from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig from litellm.llms.vertex_ai.vertex_llm_base import VertexBase @@ -16,6 +17,8 @@ from litellm.types.vector_stores import ( VectorStoreSearchOptionalRequestParams, VectorStoreSearchResponse, VectorStoreSearchResult, + VertexSearchDataStoreExtraBody, + VertexSearchEngineExtraBody, ) if TYPE_CHECKING: @@ -26,6 +29,31 @@ else: LiteLLMLoggingObj = Any +# Fields that select which data store / serving config to search. These are +# always determined by the request URL path (vector_store_id / vertex_engine_id), +# so allowing them per request could silently redirect the search to a different +# target. Rejected in both data-store and engine/app modes. +VERTEX_SEARCH_TARGET_SELECTING_FIELDS = frozenset( + { + "branch", + "servingConfig", + "entity", + } +) + +# Allowlists of native Discovery Engine SearchRequest fields callers may forward +# via extra_body, derived from the TypedDicts so the type is the source of truth. +# Engine/app mode is a superset (adds dataStoreSpecs, numResultsPerDataStore), +# since an app fans out across multiple member data stores. +VERTEX_SEARCH_DATASTORE_EXTRA_BODY_FIELDS = frozenset( + VertexSearchDataStoreExtraBody.__annotations__ +) + +VERTEX_SEARCH_ENGINE_EXTRA_BODY_FIELDS = frozenset( + VertexSearchEngineExtraBody.__annotations__ +) + + class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): """ Configuration for Vertex AI Search API Vector Store @@ -36,6 +64,66 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): def __init__(self): super().__init__() + @staticmethod + def get_supported_extra_body_fields(is_engine: bool = False) -> frozenset: + """ + Native SearchRequest fields callers may forward via ``extra_body``. + + The set depends on which serving config the request targets: + - engine/app mode (``is_engine=True``): includes multi-store fields such + as ``dataStoreSpecs`` and ``numResultsPerDataStore``. + - data-store mode: the engine-only fields are excluded. + """ + if is_engine: + return VERTEX_SEARCH_ENGINE_EXTRA_BODY_FIELDS + return VERTEX_SEARCH_DATASTORE_EXTRA_BODY_FIELDS + + @classmethod + def _filter_extra_body( + cls, extra_body: Dict[str, Any], is_engine: bool = False + ) -> Dict[str, Any]: + """ + Validate ``extra_body`` against the supported-field allowlist for the + active serving config (engine/app vs data store). + + Raises ``BadRequestError`` (HTTP 400) if the caller includes a + target-selecting field (e.g. ``servingConfig``) or any field not + supported for the active mode, so the request fails loudly instead of + silently searching the wrong target. Engine-only fields + (``dataStoreSpecs``, ``numResultsPerDataStore``) are rejected in + data-store mode where they are meaningless. + """ + supported = cls.get_supported_extra_body_fields(is_engine=is_engine) + filtered = { + key: value for key, value in extra_body.items() if value is not None + } + + target_selecting = set(filtered) & VERTEX_SEARCH_TARGET_SELECTING_FIELDS + if target_selecting: + raise BadRequestError( + message=( + "Vertex AI Search extra_body may not set target-selecting fields " + f"{sorted(target_selecting)}: the data store is scoped by " + "vector_store_id / vertex_engine_id and cannot be overridden per request." + ), + model="vertex_ai/search_api", + llm_provider="vertex_ai", + ) + + unsupported = set(filtered) - supported + if unsupported: + mode = "engine/app" if is_engine else "data store" + raise BadRequestError( + message=( + f"Unsupported Vertex AI Search extra_body fields {sorted(unsupported)} " + f"for {mode} mode. Supported fields: {sorted(supported)}." + ), + model="vertex_ai/search_api", + llm_provider="vertex_ai", + ) + + return filtered + def get_auth_credentials( self, litellm_params: dict ) -> BaseVectorStoreAuthCredentials: @@ -80,31 +168,47 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): litellm_params: dict, ) -> str: """ - Get the Base endpoint for Vertex AI Search API + Get the Base endpoint for Vertex AI Search API. + + Branches on whether a `vertex_engine_id` is configured: + - Engine ID present: route through the search app (engine) — required for website, + healthcare, and connector-based data stores. Note the serving config name differs + (`default_serving_config` vs `default_config` for direct data store search). + - Engine ID absent: query the data store directly via `vector_store_id`. """ + if api_base: + return api_base.rstrip("/") + vertex_location = self.get_vertex_ai_location(litellm_params) vertex_project = self.get_vertex_ai_project(litellm_params) collection_id = ( litellm_params.get("vertex_collection_id") or "default_collection" ) - datastore_id = litellm_params.get("vector_store_id") - if not datastore_id: - raise ValueError("vector_store_id is required") - if api_base: - return api_base.rstrip("/") encoded_collection_id = encode_url_path_segment( collection_id, field_name="vertex_collection_id" ) + base = ( + f"https://discoveryengine.googleapis.com/v1/" + f"projects/{vertex_project}/locations/{vertex_location}/" + f"collections/{encoded_collection_id}" + ) + + engine_id = litellm_params.get("vertex_engine_id") + if engine_id: + encoded_engine_id = encode_url_path_segment( + engine_id, field_name="vertex_engine_id" + ) + return f"{base}/engines/{encoded_engine_id}/servingConfigs/default_serving_config" + + datastore_id = litellm_params.get("vector_store_id") + if not datastore_id: + raise ValueError( + "vector_store_id is required when vertex_engine_id is not set" + ) encoded_datastore_id = encode_url_path_segment( datastore_id, field_name="vector_store_id" ) - - # Vertex AI Search API endpoint for search - return ( - f"https://discoveryengine.googleapis.com/v1/" - f"projects/{vertex_project}/locations/{vertex_location}/" - f"collections/{encoded_collection_id}/dataStores/{encoded_datastore_id}/servingConfigs/default_config" - ) + return f"{base}/dataStores/{encoded_datastore_id}/servingConfigs/default_config" def transform_search_vector_store_request( self, @@ -117,23 +221,41 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase): extra_body: Optional[Dict[str, Any]] = None, ) -> Tuple[str, Dict[str, Any]]: """ - Transform search request for Vertex AI RAG API + Transform a search request for the Vertex AI Search (Discovery Engine) API. + + Per-request params pass through to the engine: max_num_results maps to + pageSize, and extra_body fields on the supported allowlist + (`get_supported_extra_body_fields`) are merged in with precedence, so + callers can send native Discovery Engine tuning fields such as filter, + boostSpec, or contentSearchSpec. + + The allowlist depends on the serving config: engine/app mode (when + `vertex_engine_id` is set) additionally accepts multi-store fields like + `dataStoreSpecs` and `numResultsPerDataStore`, while data-store mode + rejects them. Target-selecting fields (e.g. servingConfig, branch) are + rejected in both modes: the target is scoped by the URL path + (vector_store_id / vertex_engine_id) and must not be overridable per + request. """ - # Convert query to string if it's a list if isinstance(query, list): query = " ".join(query) - # Vertex AI RAG API endpoint for retrieving contexts url = f"{api_base}:search" - # Construct full rag corpus path - # Build the request body for Vertex AI Search API - request_body = {"query": query, "pageSize": 10} + is_engine = bool(litellm_params.get("vertex_engine_id")) - ######################################################### - # Update logging object with details of the request - ######################################################### - litellm_logging_obj.model_call_details["query"] = query + request_body: Dict[str, Any] = {"query": query, "pageSize": 10} + max_num_results = vector_store_search_optional_params.get("max_num_results") + if max_num_results is not None: + request_body["pageSize"] = max_num_results + if isinstance(extra_body, dict): + request_body.update( + self._filter_extra_body(extra_body, is_engine=is_engine) + ) + + litellm_logging_obj.model_call_details["query"] = request_body.get( + "query", query + ) return url, request_body diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 4be4c2d5e7..1e92754857 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -159,6 +159,6 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert "model", None ) # do not pass model in request body to vertex ai - sanitize_vertex_anthropic_output_params(anthropic_messages_request) + sanitize_vertex_anthropic_output_params(anthropic_messages_request, model) return anthropic_messages_request diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py index a33ad67778..280cc1c888 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py @@ -10,23 +10,38 @@ import; extracting the helper into a leaf module resolves the warning and keeps the parent module's import surface narrow. """ -# Keys inside ``output_config`` that Vertex AI Claude does not accept. -# Add an entry only when a 400 "Extra inputs are not permitted" is -# reproducible against the live Vertex endpoint. +# Keys inside ``output_config`` that Vertex AI Claude rejects regardless of +# the target model. Add an entry only when a 400 "Extra inputs are not +# permitted" is reproducible against the live Vertex endpoint for every model. VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS: frozenset = frozenset() -def sanitize_vertex_anthropic_output_params(data: dict) -> None: +def _model_accepts_output_config_effort(model: str) -> bool: + """Whether ``model`` accepts ``output_config.effort`` on Vertex. + + Opus/Sonnet 4.6+ advertise ``supports_output_config`` (or a reasoning + effort level) and accept it; Haiku 4.5 advertises neither and 400s on + ``output_config.effort: Extra inputs are not permitted``. Imported lazily + so this stays a leaf module (see module docstring). + """ + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + + return AnthropicConfig._model_supports_effort_param(model) + + +def sanitize_vertex_anthropic_output_params(data: dict, model: str) -> None: """ Strip Vertex-unsupported keys from ``output_config`` / ``output_format`` in-place; forward whatever remains. Behavior: - * ``output_config`` containing only unsupported keys (e.g. ``effort`` - alone) is removed entirely so the request body has no empty dict. - * ``output_config`` containing a mix of supported + unsupported keys - has the unsupported subset filtered out and the rest forwarded. - * ``output_config`` that is supported in full passes through unchanged. + * ``output_config.effort`` is dropped for models that don't accept it + (e.g. Haiku 4.5) and forwarded for those that do (Opus/Sonnet 4.6+). + Clients like Claude Code inject it into every Messages payload, so the + gate has to live here rather than rely on the caller. + * Keys in ``VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS`` are always filtered. + * ``output_config`` left empty after filtering is removed so the request + body has no empty dict. * ``output_format`` is forwarded as-is (Vertex AI Claude accepts it). * Non-dict values for ``output_config`` are dropped to avoid sending malformed payloads downstream. @@ -37,11 +52,19 @@ def sanitize_vertex_anthropic_output_params(data: dict) -> None: if not isinstance(output_config, dict): data.pop("output_config", None) return - sanitized = { - k: v - for k, v in output_config.items() - if k not in VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS - } + + drop_keys = set(VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS) + if "effort" in output_config and not _model_accepts_output_config_effort(model): + from litellm._logging import verbose_logger + + verbose_logger.debug( + "Dropping unsupported output_config.effort for vertex_ai model=%s " + "(no supports_output_config in the model map)", + model, + ) + drop_keys.add("effort") + + sanitized = {k: v for k, v in output_config.items() if k not in drop_keys} if sanitized: data["output_config"] = sanitized else: diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py index 4627d9f6df..c852909d47 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/transformation.py @@ -106,7 +106,7 @@ class VertexAIAnthropicConfig(AnthropicConfig): data.pop("model", None) # vertex anthropic doesn't accept 'model' parameter - sanitize_vertex_anthropic_output_params(data) + sanitize_vertex_anthropic_output_params(data, model) tools = optional_params.get("tools") tool_search_used = self.is_tool_search_used(tools) diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py index 13aa2a5350..960d348384 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/main.py @@ -41,6 +41,7 @@ class PartnerModelPrefixes(str, Enum): MINIMAX_PREFIX = "minimaxai/" MOONSHOT_PREFIX = "moonshotai/" ZAI_PREFIX = "zai-org/" + GEMMA_MAAS_PREFIX = "google/gemma-" class VertexAIPartnerModels(VertexBase): @@ -68,6 +69,7 @@ class VertexAIPartnerModels(VertexBase): or model.startswith(PartnerModelPrefixes.MINIMAX_PREFIX) or model.startswith(PartnerModelPrefixes.MOONSHOT_PREFIX) or model.startswith(PartnerModelPrefixes.ZAI_PREFIX) + or model.startswith(PartnerModelPrefixes.GEMMA_MAAS_PREFIX) ): return True return False @@ -82,6 +84,7 @@ class VertexAIPartnerModels(VertexBase): PartnerModelPrefixes.MINIMAX_PREFIX, PartnerModelPrefixes.MOONSHOT_PREFIX, PartnerModelPrefixes.ZAI_PREFIX, + PartnerModelPrefixes.GEMMA_MAAS_PREFIX, ] if any(provider in model for provider in OPENAI_LIKE_VERTEX_PROVIDERS): return True diff --git a/litellm/llms/vertex_ai/vertex_model_garden/main.py b/litellm/llms/vertex_ai/vertex_model_garden/main.py index 732d5f90dc..f54b8d9350 100644 --- a/litellm/llms/vertex_ai/vertex_model_garden/main.py +++ b/litellm/llms/vertex_ai/vertex_model_garden/main.py @@ -114,33 +114,18 @@ class VertexAIModelGardenModels(VertexBase): openai_like_chat_completions = OpenAILikeChatHandler() ## CONSTRUCT API BASE + # Skip _check_custom_proxy: its ":verb" URL construction corrupts a + # user-supplied api_base (e.g. Vertex MG dedicated endpoint), and + # OpenAILikeChatHandler already appends "/chat/completions". stream: bool = optional_params.get("stream", False) or False optional_params["stream"] = stream - default_api_base = create_vertex_url( - vertex_location=vertex_location or "us-central1", - vertex_project=vertex_project or project_id, - stream=stream, - model=model, - ) - - if len(default_api_base.split(":")) > 1: - endpoint = default_api_base.split(":")[-1] - else: - endpoint = "" - - _, api_base = self._check_custom_proxy( - api_base=api_base, - custom_llm_provider="vertex_ai", - gemini_api_key=None, - endpoint=endpoint, - stream=stream, - auth_header=None, - url=default_api_base, - model=model, - vertex_project=vertex_project or project_id, - vertex_location=vertex_location or "us-central1", - vertex_api_version="v1beta1", - ) + if api_base is None: + api_base = create_vertex_url( + vertex_location=vertex_location or "us-central1", + vertex_project=vertex_project or project_id, + stream=stream, + model=model, + ) # Publisher/catalog models: model id must be sent in the JSON body (OpenAPI route). # Single-segment endpoint ids: model is encoded in the URL path; body model stays empty. if not _vertex_model_garden_model_id_in_json_body(model): diff --git a/litellm/llms/watsonx/passthrough/__init__.py b/litellm/llms/watsonx/passthrough/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/watsonx/passthrough/transformation.py b/litellm/llms/watsonx/passthrough/transformation.py new file mode 100644 index 0000000000..9162eef0e0 --- /dev/null +++ b/litellm/llms/watsonx/passthrough/transformation.py @@ -0,0 +1,69 @@ +from typing import TYPE_CHECKING, List, Optional, Tuple + +from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig +from litellm.llms.watsonx.common_utils import IBMWatsonXMixin + +if TYPE_CHECKING: + from httpx import URL + + +class WatsonxPassthroughConfig(IBMWatsonXMixin, BasePassthroughConfig): + """ + Watsonx-specific passthrough configuration. + """ + + def is_streaming_request(self, endpoint: str, request_data: dict) -> bool: + """Check if request should be streamed""" + return request_data.get("stream", False) + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + endpoint: str, + request_query_params: Optional[dict], + litellm_params: dict, + ) -> Tuple["URL", str]: + """ + Construct complete Watsonx URL with version parameter. + + This ensures the version parameter is ALWAYS included in the URL, + solving the query parameter issue. + """ + base_target_url = str(self.get_api_base(api_base)) + + # Use the format_url helper to construct URL with query params + complete_url = self.format_url( + endpoint=endpoint, + base_target_url=base_target_url, + request_query_params=request_query_params, + ) + + return (complete_url, base_target_url) + + @staticmethod + def get_api_base( + api_base: Optional[str] = None, + ) -> Optional[str]: + return api_base or IBMWatsonXMixin()._get_base_url(api_base=api_base) + + @staticmethod + def get_api_key( + api_key: Optional[str] = None, + ) -> Optional[str]: + return ( + api_key + or IBMWatsonXMixin.get_watsonx_credentials( + optional_params=dict(), api_base=None, api_key=api_key + )["api_key"] + ) + + @staticmethod + def get_base_model(model: str) -> Optional[str]: + return model + + def get_models( + self, api_key: Optional[str] = None, api_base: Optional[str] = None + ) -> List[str]: + return super().get_models(api_key, api_base) diff --git a/litellm/llms/xai/chat/transformation.py b/litellm/llms/xai/chat/transformation.py index 7325c0596a..c06928516e 100644 --- a/litellm/llms/xai/chat/transformation.py +++ b/litellm/llms/xai/chat/transformation.py @@ -9,6 +9,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( filter_value_from_dict, strip_name_from_messages, ) +from litellm.llms.xai.common_utils import XAIModelInfo from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ( @@ -35,7 +36,7 @@ class XAIChatConfig(OpenAIGPTConfig): self, api_base: Optional[str], api_key: Optional[str] ) -> Tuple[Optional[str], Optional[str]]: api_base = api_base or get_secret_str("XAI_API_BASE") or XAI_API_BASE # type: ignore - dynamic_api_key = api_key or get_secret_str("XAI_API_KEY") + dynamic_api_key = XAIModelInfo.get_api_key(api_key) return api_base, dynamic_api_key def get_supported_openai_params(self, model: str) -> list: diff --git a/litellm/llms/xai/common_utils.py b/litellm/llms/xai/common_utils.py index df324cf3ee..adc857894c 100644 --- a/litellm/llms/xai/common_utils.py +++ b/litellm/llms/xai/common_utils.py @@ -45,8 +45,28 @@ class XAIModelInfo(BaseLLMModelInfo): return api_base or get_secret_str("XAI_API_BASE") or "https://api.x.ai" @staticmethod - def get_api_key(api_key: Optional[str] = None) -> Optional[str]: - return api_key or get_secret_str("XAI_API_KEY") + def get_api_key( + api_key: Optional[str] = None, + legacy_generic_before_env: bool = False, + ) -> Optional[str]: + """ + Resolve xAI API keys while preserving endpoint-specific legacy order. + + Chat uses xai_key before XAI_API_KEY without adding a generic + litellm.api_key fallback. Responses and realtime historically + preferred litellm.api_key over XAI_API_KEY, so those paths opt into + the legacy order with legacy_generic_before_env=True. In both modes, + the provider-specific litellm.xai_key takes precedence over fallbacks. + """ + if legacy_generic_before_env: + return ( + api_key + or litellm.xai_key + or litellm.api_key + or get_secret_str("XAI_API_KEY") + ) + + return api_key or litellm.xai_key or get_secret_str("XAI_API_KEY") @staticmethod def get_base_model(model: str) -> Optional[str]: @@ -59,7 +79,7 @@ class XAIModelInfo(BaseLLMModelInfo): api_key = self.get_api_key(api_key) if api_base is None or api_key is None: raise ValueError( - "XAI_API_BASE or XAI_API_KEY is not set. Please set the environment variable, to query XAI's `/models` endpoint." + "XAI API base or key is not set. Set XAI_API_BASE and provide an xAI API key via api_key, litellm.xai_key, or XAI_API_KEY." ) response = litellm.module_level_client.get( url=f"{api_base}/v1/models", diff --git a/litellm/llms/xai/responses/transformation.py b/litellm/llms/xai/responses/transformation.py index 23aee3a120..55805ddaed 100644 --- a/litellm/llms/xai/responses/transformation.py +++ b/litellm/llms/xai/responses/transformation.py @@ -4,6 +4,7 @@ import litellm from litellm._logging import verbose_logger from litellm.constants import XAI_API_BASE from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig +from litellm.llms.xai.common_utils import XAIModelInfo from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.llms.xai import XAIWebSearchTool, XAIXSearchTool @@ -212,16 +213,17 @@ class XAIResponsesAPIConfig(OpenAIResponsesAPIConfig): """ Validate environment and set up headers for XAI API. - Uses XAI_API_KEY from environment or litellm_params. + Uses the shared xAI key resolver with Responses API legacy precedence. """ litellm_params = litellm_params or GenericLiteLLMParams() - api_key = ( - litellm_params.api_key or litellm.api_key or get_secret_str("XAI_API_KEY") + api_key = XAIModelInfo.get_api_key( + litellm_params.api_key, legacy_generic_before_env=True ) if not api_key: raise ValueError( - "XAI API key is required. Set XAI_API_KEY environment variable or pass api_key parameter." + "XAI API key is required. Set api_key, litellm.xai_key, " + "litellm.api_key, or XAI_API_KEY." ) headers.update( diff --git a/litellm/llms/you_com/__init__.py b/litellm/llms/you_com/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/litellm/llms/you_com/search/__init__.py b/litellm/llms/you_com/search/__init__.py new file mode 100644 index 0000000000..41bd9ce6b1 --- /dev/null +++ b/litellm/llms/you_com/search/__init__.py @@ -0,0 +1,7 @@ +""" +You.com Search API module. +""" + +from litellm.llms.you_com.search.transformation import YouComSearchConfig + +__all__ = ["YouComSearchConfig"] diff --git a/litellm/llms/you_com/search/transformation.py b/litellm/llms/you_com/search/transformation.py new file mode 100644 index 0000000000..3c94b99173 --- /dev/null +++ b/litellm/llms/you_com/search/transformation.py @@ -0,0 +1,193 @@ +""" +Calls You.com's /v1/search endpoint to search the web. + +You.com API Reference: https://you.com/docs/api-reference/search/v1-search +OpenAPI spec: https://you.com/specs/openapi_search_v1.yaml +""" + +from typing import Dict, List, Optional, TypedDict, Union + +import httpx + +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.llms.base_llm.search.transformation import ( + BaseSearchConfig, + SearchResponse, + SearchResult, +) +from litellm.secret_managers.main import get_secret_str + + +class _YouComSearchRequestRequired(TypedDict): + """Required fields for You.com Search API request.""" + + query: str + + +class YouComSearchRequest(_YouComSearchRequestRequired, total=False): + """ + You.com Search API request format. + Based on: https://you.com/specs/openapi_search_v1.yaml + """ + + count: int + country: str + language: str + freshness: str + include_domains: List[str] + exclude_domains: List[str] + safesearch: str + + +class YouComSearchConfig(BaseSearchConfig): + # Keyed tier (higher rate limits): authenticate with X-API-Key. + YOU_COM_API_BASE = "https://ydc-index.io" + # Keyless free tier: IP-throttled (100 queries/day) and requires no auth. + # Used automatically when YOUCOM_API_KEY is not set. + YOU_COM_FREE_API_BASE = "https://api.you.com/v1/agents/search" + + @staticmethod + def ui_friendly_name() -> str: + return "You.com" + + def validate_environment( + self, + headers: Dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + **kwargs, + ) -> Dict: + """ + Set headers for the You.com Search API. + + If YOUCOM_API_KEY (or an explicit api_key) is present, use the keyed + endpoint with the `X-API-Key` header. Otherwise fall through to the + keyless free tier; no auth header is required. + """ + api_key = api_key or get_secret_str("YOUCOM_API_KEY") + headers["Content-Type"] = "application/json" + # Pin Accept-Encoding to identity: the keyless `api.you.com/v1/agents/search` + # endpoint advertises gzip content-encoding but returns body bytes the + # decoder rejects, which surfaces as httpx.DecodingError through litellm's + # http handler. Identity is harmless on the keyed endpoint. + headers.setdefault("Accept-Encoding", "identity") + if api_key: + headers["X-API-Key"] = api_key + return headers + + def get_complete_url( + self, + api_base: Optional[str], + optional_params: dict, + data: Optional[Union[Dict, List[Dict]]] = None, + **kwargs, + ) -> str: + """ + Pick the endpoint based on whether an API key is configured. + + - api_base explicit override -> use it as-is (normalized) + - YOUCOM_API_KEY set -> keyed endpoint (ydc-index.io/v1/search) + - no key -> keyless free tier (api.you.com/v1/agents/search) + """ + if api_base is None: + api_base = get_secret_str("YOUCOM_API_BASE") + + if api_base is None: + api_key = kwargs.get("api_key") or get_secret_str("YOUCOM_API_KEY") + if api_key: + api_base = self.YOU_COM_API_BASE + else: + # Keyless free tier already includes the full path. + return self.YOU_COM_FREE_API_BASE + + api_base = api_base.rstrip("/") + + if not api_base.endswith("/v1/search") and not api_base.endswith( + "/v1/agents/search" + ): + api_base = f"{api_base}/v1/search" + + return api_base + + def transform_search_request( + self, + query: Union[str, List[str]], + optional_params: dict, + **kwargs, + ) -> Dict: + """ + Transform Search request to You.com API format. + + Perplexity unified spec → You.com mappings: + - query → query + - max_results → count + - search_domain_filter → include_domains + - country → country + - max_tokens_per_page → (not applicable, ignored) + """ + if isinstance(query, list): + query = " ".join(query) + + request_data: YouComSearchRequest = { + "query": query, + } + + if "max_results" in optional_params: + request_data["count"] = optional_params["max_results"] + + if "search_domain_filter" in optional_params: + request_data["include_domains"] = optional_params["search_domain_filter"] + + if "country" in optional_params: + request_data["country"] = optional_params["country"].lower() + + result_data = dict(request_data) + + for param, value in optional_params.items(): + if ( + param not in self.get_supported_perplexity_optional_params() + and param not in result_data + ): + result_data[param] = value + + return result_data + + def transform_search_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + **kwargs, + ) -> SearchResponse: + """ + Transform You.com API response to LiteLLM unified SearchResponse format. + + You.com → LiteLLM mappings (for both `results.web[]` and `results.news[]`): + - title → SearchResult.title + - url → SearchResult.url + - snippets[0] → SearchResult.snippet (falls back to `description`) + - page_age → SearchResult.date + """ + response_json = raw_response.json() + raw_results = response_json.get("results") or {} + + web_results = raw_results.get("web") or [] + news_results = raw_results.get("news") or [] + + results: List[SearchResult] = [] + for item in list(web_results) + list(news_results): + snippets = item.get("snippets") or [] + snippet = snippets[0] if snippets else item.get("description", "") + results.append( + SearchResult( + title=item.get("title", ""), + url=item.get("url", ""), + snippet=snippet, + date=item.get("page_age"), + last_updated=None, + ) + ) + + return SearchResponse( + results=results, + object="search", + ) diff --git a/litellm/main.py b/litellm/main.py index 09c70998cf..64891e2def 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -437,6 +437,7 @@ async def acompletion( # noqa: PLR0915 # Optional liteLLM function params thinking: Optional[AnthropicThinkingParam] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + include_server_side_tool_invocations: Optional[bool] = None, # Session management shared_session: Optional["ClientSession"] = None, # Per-request JSON schema validation (overrides litellm.enable_json_schema_validation) @@ -584,6 +585,7 @@ async def acompletion( # noqa: PLR0915 "acompletion": True, # assuming this is a required parameter "thinking": thinking, "web_search_options": web_search_options, + "include_server_side_tool_invocations": include_server_side_tool_invocations, "shared_session": shared_session, "enable_json_schema_validation": enable_json_schema_validation, } @@ -641,6 +643,7 @@ async def acompletion( # noqa: PLR0915 if ( custom_llm_provider == "text-completion-openai" or custom_llm_provider == "text-completion-codestral" + or custom_llm_provider == "text-completion-inception" ) and isinstance(response, TextCompletionResponse): response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( response_object=response, @@ -1115,6 +1118,7 @@ def completion( # type: ignore # noqa: PLR0915 top_logprobs: Optional[int] = None, parallel_tool_calls: Optional[bool] = None, web_search_options: Optional[OpenAIWebSearchOptions] = None, + include_server_side_tool_invocations: Optional[bool] = None, deployment_id=None, extra_headers: Optional[dict] = None, safety_identifier: Optional[str] = None, @@ -1318,7 +1322,9 @@ def completion( # type: ignore # noqa: PLR0915 preset_cache_key = kwargs.get("preset_cache_key", None) hf_model_name = kwargs.get("hf_model_name", None) supports_system_message = kwargs.get("supports_system_message", None) - base_model = kwargs.get("base_model", None) + base_model = kwargs.get("base_model", None) or ( + model_info.get("base_model") if isinstance(model_info, dict) else None + ) ### DISABLE FLAGS ### disable_add_transform_inline_image_block = kwargs.get( "disable_add_transform_inline_image_block", None @@ -1530,11 +1536,7 @@ def completion( # type: ignore # noqa: PLR0915 "logit_bias": logit_bias, "user": user, # params to identify the model - "model": ( - model_info.get("base_model") - if isinstance(model_info, dict) and model_info.get("base_model") - else model - ), + "model": model, "custom_llm_provider": custom_llm_provider, "response_format": response_format, "seed": seed, @@ -1549,6 +1551,11 @@ def completion( # type: ignore # noqa: PLR0915 "reasoning_effort": reasoning_effort, "thinking": thinking, "web_search_options": web_search_options, + "include_server_side_tool_invocations": ( + include_server_side_tool_invocations + if include_server_side_tool_invocations is not None + else kwargs.get("include_server_side_tool_invocations") + ), "safety_identifier": safety_identifier, "service_tier": service_tier, "allowed_openai_params": kwargs.get("allowed_openai_params"), @@ -3803,6 +3810,67 @@ def completion( # type: ignore # noqa: PLR0915 ): return _model_response response = _model_response + elif custom_llm_provider == "text-completion-inception": + passed_api_base = ( + api_base + or optional_params.pop("api_base", None) + or optional_params.pop("base_url", None) + ) + api_base = ( + passed_api_base + or get_secret_str("INCEPTION_API_BASE") + or "https://api.inceptionlabs.ai/v1" + ) + # FIM is served at `/v1/fim/completions`; the OpenAI client appends + # `/completions`, so point it at the `/v1/fim` base. + api_base = api_base.rstrip("/") + if not api_base.endswith("/fim"): + api_base += "/fim" + + # Don't forward the server-managed Inception key to a caller-supplied + # api_base; only resolve it for the default/server base, or when the + # caller passes their own key. + if passed_api_base is None or api_key: + api_key = ( + api_key + or litellm.inception_key + or get_secret_str("INCEPTION_API_KEY") + ) + + _response = openai_text_completions.completion( + model=model, + messages=messages, + model_response=model_response, + print_verbose=print_verbose, + api_key=api_key, # type: ignore[arg-type] + custom_llm_provider="text-completion-inception", + api_base=api_base, + acompletion=acompletion, + client=client, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + logger_fn=logger_fn, + timeout=timeout, # type: ignore + ) + + if ( + optional_params.get("stream", False) is False + and acompletion is False + and text_completion is False + ): + _response = litellm.OpenAITextCompletionConfig().convert_to_chat_model_response_object( + response_object=_response, model_response_object=model_response + ) + + if optional_params.get("stream", False) or acompletion is True: + logging.post_call( + input=messages, + api_key=api_key, + original_response=_response, + additional_args={"headers": headers}, + ) + response = _response elif custom_llm_provider in ("sagemaker_chat", "sagemaker_nova"): # boto3 reads keys from .env # sagemaker_chat: HF Messages API endpoints @@ -4503,6 +4571,39 @@ def completion( # type: ignore # noqa: PLR0915 client=client, ) + elif custom_llm_provider == "langflow": + # LangFlow - Visual AI Agent Platform + from litellm.llms.langflow.chat.transformation import LangFlowConfig + + ( + api_base, + api_key, + ) = LangFlowConfig()._get_openai_compatible_provider_info( + api_base=api_base or litellm.api_base, + api_key=api_key or litellm.api_key, + ) + + headers = headers or litellm.headers + + response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + acompletion=acompletion, + api_base=api_base, + model_response=model_response, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + custom_llm_provider=custom_llm_provider, + timeout=timeout, + headers=headers, + encoding=_get_encoding(), + api_key=api_key, + logging_obj=logging, + client=client, + ) + else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider @@ -6554,7 +6655,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: @client -def transcription( +def transcription( # noqa: PLR0915 model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## @@ -6746,6 +6847,35 @@ def transcription( else None ), ) + elif custom_llm_provider == "soniox": + from litellm.llms.soniox.audio_transcription.handler import ( + SonioxAudioTranscriptionHandler, + ) + + response = SonioxAudioTranscriptionHandler().audio_transcriptions( + model=model, + audio_file=file, + optional_params=optional_params, + litellm_params=litellm_params_dict, + model_response=model_response, + atranscription=atranscription, + client=( + client + if client is not None + and ( + isinstance(client, HTTPHandler) + or isinstance(client, AsyncHTTPHandler) + ) + else None + ), + timeout=timeout, + max_retries=max_retries, + logging_obj=litellm_logging_obj, + api_base=api_base, + api_key=api_key, + headers=extra_headers, + provider_config=provider_config, # type: ignore[arg-type] + ) elif provider_config is not None: response = base_llm_http_handler.audio_transcriptions( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index ce6d4ac824..397f96fdb1 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -577,7 +577,10 @@ "max_tokens": 8192, "mode": "embedding", "output_cost_per_token": 0.0, - "output_vector_size": 1024 + "output_vector_size": 1024, + "provider_specific_entry": { + "bedrock_invocation_schema": "titan_v2" + } }, "amazon.titan-image-generator-v1": { "input_cost_per_image": 0.0, @@ -1072,6 +1075,7 @@ }, "eu.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1101,6 +1105,7 @@ }, "au.anthropic.claude-opus-4-6-v1": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1238,6 +1243,7 @@ }, "eu.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1268,6 +1274,7 @@ }, "au.anthropic.claude-opus-4-7": { "cache_creation_input_token_cost": 6.875e-06, + "cache_creation_input_token_cost_above_1hr": 1.1e-05, "cache_read_input_token_cost": 5.5e-07, "input_cost_per_token": 5.5e-06, "litellm_provider": "bedrock_converse", @@ -1312,6 +1319,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1343,6 +1351,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1374,6 +1383,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1405,6 +1415,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1436,6 +1447,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -1451,6 +1463,36 @@ "supports_output_config": true, "bedrock_output_config_effort_ceiling": "xhigh" }, + "jp.anthropic.claude-opus-4-7": { + "cache_creation_input_token_cost": 6.875e-06, + "cache_read_input_token_cost": 5.5e-07, + "input_cost_per_token": 5.5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.75e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": false, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_xhigh_reasoning_effort": true, + "tool_use_system_prompt_tokens": 346, + "supports_native_structured_output": true, + "supports_max_reasoning_effort": true, + "supports_minimal_reasoning_effort": true + }, "anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -1540,6 +1582,7 @@ }, "eu.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1568,6 +1611,7 @@ }, "au.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1596,6 +1640,7 @@ }, "jp.anthropic.claude-sonnet-4-6": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "litellm_provider": "bedrock_converse", @@ -1992,11 +2037,13 @@ }, "au.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -2182,6 +2229,7 @@ "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -7491,6 +7539,27 @@ "supports_video_input": true, "supports_vision": true }, + "azure_ai/kimi-k2.6": { + "input_cost_per_token": 9.5e-07, + "litellm_provider": "azure_ai", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 4e-06, + "source": "https://techcommunity.microsoft.com/blog/azure-ai-foundry-blog/introducing-kimi-k2-6-in-microsoft-foundry/4513125", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text" + ], + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true, + "supports_vision": true + }, "azure_ai/ministral-3b": { "input_cost_per_token": 4e-08, "litellm_provider": "azure_ai", @@ -8899,15 +8968,16 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-east-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 4.5e-06, + "cache_creation_input_token_cost_above_1hr": 7.2e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -8920,15 +8990,16 @@ "supports_native_structured_output": true }, "bedrock/us-gov-east-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 4.5e-06, + "cache_creation_input_token_cost_above_1hr": 7.2e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -9072,15 +9143,16 @@ "cache_creation_input_token_cost": 3.75e-07 }, "bedrock/us-gov-west-1/anthropic.claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 4.5e-06, + "cache_creation_input_token_cost_above_1hr": 7.2e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -9093,15 +9165,16 @@ "supports_native_structured_output": true }, "bedrock/us-gov-west-1/claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, + "cache_creation_input_token_cost": 4.5e-06, + "cache_creation_input_token_cost_above_1hr": 7.2e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, "litellm_provider": "bedrock", "max_input_tokens": 200000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -12675,7 +12748,8 @@ "litellm_provider": "deepinfra", "mode": "chat", "supports_tool_choice": true, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "deepinfra/google/gemini-2.5-pro": { "max_tokens": 1000000, @@ -13380,6 +13454,22 @@ "notes": "Serper Google Search API. Pricing: $1.00/1k queries (Starter), $0.75/1k (Standard), $0.50/1k (Scale), $0.30/1k (Ultimate)." } }, + "apiserpent/search": { + "input_cost_per_query": 0.0006, + "litellm_provider": "apiserpent", + "mode": "search", + "metadata": { + "notes": "APISerpent quick search (/api/search/quick), multi-engine (Google, Bing, Yahoo, DuckDuckGo). Pricing: $0.60/1k searches." + } + }, + "apiserpent/deep_search": { + "input_cost_per_query": 0.0006, + "litellm_provider": "apiserpent", + "mode": "search", + "metadata": { + "notes": "APISerpent deep search (/api/search), multi-engine (Google, Bing, Yahoo, DuckDuckGo). Pricing: $0.60/1k searches." + } + }, "elevenlabs/scribe_v1": { "input_cost_per_second": 6.11e-05, "litellm_provider": "elevenlabs", @@ -13560,6 +13650,7 @@ }, "eu.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "deprecation_date": "2026-10-15", @@ -13764,11 +13855,13 @@ }, "eu.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -13921,6 +14014,22 @@ "/v1/images/generations" ] }, + "fal_ai/fal-ai/nano-banana": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "supported_endpoints": [ + "/v1/images/generations" + ] + }, + "fal_ai/fal-ai/gemini-25-flash-image": { + "litellm_provider": "fal_ai", + "mode": "image_generation", + "output_cost_per_image": 0.039, + "supported_endpoints": [ + "/v1/images/generations" + ] + }, "featherless_ai/featherless-ai/Qwerky-72B": { "litellm_provider": "featherless_ai", "max_input_tokens": 32768, @@ -14983,7 +15092,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -15033,7 +15143,8 @@ "supports_vision": true, "supports_web_search": false, "tpm": 8000000, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -15173,10 +15284,16 @@ "supports_service_tier": true }, "gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -15188,9 +15305,12 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, - "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -15313,7 +15433,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -15363,7 +15484,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15413,7 +15535,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-live-2.5-flash-preview-native-audio-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -15564,7 +15687,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini-2.5-pro": { "cache_read_input_token_cost": 1.25e-07, @@ -16574,7 +16698,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, @@ -16630,7 +16755,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -16809,7 +16935,8 @@ "search_context_size_medium": 0.035, "search_context_size_high": 0.035 }, - "supports_service_tier": true + "supports_service_tier": true, + "supports_image_size": false }, "gemini/gemini-2.5-flash-lite-preview-09-2025": { "cache_read_input_token_cost": 1e-08, @@ -16861,7 +16988,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-09-2025": { "cache_read_input_token_cost": 7.5e-08, @@ -16913,7 +17041,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-flash-latest": { "cache_read_input_token_cost": 7.5e-08, @@ -17070,7 +17199,8 @@ "search_context_size_low": 0.035, "search_context_size_medium": 0.035, "search_context_size_high": 0.035 - } + }, + "supports_image_size": false }, "gemini/gemini-2.5-flash-preview-tts": { "input_cost_per_token": 3e-07, @@ -17295,10 +17425,16 @@ "supports_service_tier": true }, "gemini/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "gemini", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -17310,10 +17446,13 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "rpm": 15, - "source": "https://ai.google.dev/gemini-api/docs/pricing", + "source": "https://ai.google.dev/gemini-api/docs/pricing#gemini-3.1-flash-lite", "supported_endpoints": [ "/v1/chat/completions", "/v1/completions", @@ -18075,23 +18214,22 @@ }, "github_copilot/claude-haiku-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/claude-opus-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" @@ -18099,7 +18237,6 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_vision": true, - "supports_reasoning": true, "supports_output_config": true }, "github_copilot/claude-opus-4.6-fast": { @@ -18115,22 +18252,6 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/claude-opus-4.7": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, "github_copilot/claude-opus-41": { "litellm_provider": "github_copilot", "max_input_tokens": 80000, @@ -18157,33 +18278,16 @@ }, "github_copilot/claude-sonnet-4.5": { "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, + "max_input_tokens": 128000, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "supported_endpoints": [ "/v1/chat/completions" ], "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/claude-sonnet-4.6": { - "litellm_provider": "github_copilot", - "max_input_tokens": 200000, - "max_output_tokens": 32000, - "max_tokens": 32000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/messages" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-2.5-pro": { "litellm_provider": "github_copilot", @@ -18193,25 +18297,7 @@ "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_reasoning": true - }, - "github_copilot/gemini-3-flash-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gemini-3-pro-preview": { "litellm_provider": "github_copilot", @@ -18223,30 +18309,13 @@ "supports_parallel_function_calling": true, "supports_vision": true }, - "github_copilot/gemini-3.1-pro-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_reasoning": true - }, "github_copilot/gpt-3.5-turbo": { "litellm_provider": "github_copilot", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-3.5-turbo-0613": { "litellm_provider": "github_copilot", @@ -18254,10 +18323,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4": { "litellm_provider": "github_copilot", @@ -18265,22 +18331,7 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] - }, - "github_copilot/gpt-4-0125-preview": { - "litellm_provider": "github_copilot", - "max_input_tokens": 128000, - "max_output_tokens": 4096, - "max_tokens": 4096, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions" - ], - "supports_function_calling": true, - "supports_parallel_function_calling": true + "supports_function_calling": true }, "github_copilot/gpt-4-0613": { "litellm_provider": "github_copilot", @@ -18288,22 +18339,16 @@ "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", - "supports_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_function_calling": true }, "github_copilot/gpt-4-o-preview": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4.1": { "litellm_provider": "github_copilot", @@ -18314,10 +18359,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4.1-2025-04-14": { "litellm_provider": "github_copilot", @@ -18328,89 +18370,68 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-41-copilot": { "litellm_provider": "github_copilot", - "mode": "chat" + "mode": "completion" }, "github_copilot/gpt-4o": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-05-13": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-2024-08-06": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-2024-11-20": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_vision": true }, "github_copilot/gpt-4o-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-4o-mini-2024-07-18": { "litellm_provider": "github_copilot", - "max_input_tokens": 128000, + "max_input_tokens": 64000, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supported_endpoints": [ - "/v1/chat/completions" - ] + "supports_parallel_function_calling": true }, "github_copilot/gpt-5": { "litellm_provider": "github_copilot", @@ -18429,19 +18450,14 @@ }, "github_copilot/gpt-5-mini": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.1": { "litellm_provider": "github_copilot", @@ -18474,7 +18490,7 @@ }, "github_copilot/gpt-5.2": { "litellm_provider": "github_copilot", - "max_input_tokens": 264000, + "max_input_tokens": 128000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", @@ -18485,27 +18501,11 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.2-codex": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true + "supports_vision": true }, "github_copilot/gpt-5.3-codex": { "litellm_provider": "github_copilot", - "max_input_tokens": 400000, + "max_input_tokens": 128000, "max_output_tokens": 128000, "max_tokens": 128000, "mode": "responses", @@ -18515,96 +18515,25 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_response_schema": true, - "supports_vision": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.4-mini": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/gpt-5.5": { - "litellm_provider": "github_copilot", - "max_input_tokens": 400000, - "max_output_tokens": 128000, - "max_tokens": 128000, - "mode": "responses", - "supported_endpoints": [ - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true, - "supports_reasoning": true - }, - "github_copilot/oswe-vscode-prime": { - "litellm_provider": "github_copilot", - "max_input_tokens": 264000, - "max_output_tokens": 64000, - "max_tokens": 64000, - "mode": "chat", - "supported_endpoints": [ - "/v1/chat/completions", - "/v1/responses" - ], - "supports_vision": true, - "supports_function_calling": true, - "supports_parallel_function_calling": true, - "supports_response_schema": true + "supports_vision": true }, "github_copilot/text-embedding-3-small": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-3-small-inference": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "github_copilot/text-embedding-ada-002": { "litellm_provider": "github_copilot", "max_input_tokens": 8191, "max_tokens": 8191, - "mode": "embedding", - "supported_endpoints": [ - "/v1/embeddings" - ] + "mode": "embedding" }, "chatgpt/gpt-5.4": { "litellm_provider": "chatgpt", @@ -23255,11 +23184,13 @@ }, "jp.anthropic.claude-sonnet-4-5-20250929-v1:0": { "cache_creation_input_token_cost": 4.125e-06, + "cache_creation_input_token_cost_above_1hr": 6.6e-06, "cache_read_input_token_cost": 3.3e-07, "input_cost_per_token": 3.3e-06, "input_cost_per_token_above_200k_tokens": 6.6e-06, "output_cost_per_token_above_200k_tokens": 2.475e-05, "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.32e-05, "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, @@ -23285,6 +23216,7 @@ }, "jp.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -23397,6 +23329,31 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "inception/mercury-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "inception", + "max_input_tokens": 128000, + "max_output_tokens": 50000, + "max_tokens": 50000, + "mode": "chat", + "output_cost_per_token": 7.5e-07, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "text-completion-inception/mercury-edit-2": { + "cache_read_input_token_cost": 2.5e-08, + "input_cost_per_token": 2.5e-07, + "litellm_provider": "text-completion-inception", + "max_input_tokens": 32000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "completion", + "output_cost_per_token": 7.5e-07 + }, "lambda_ai/deepseek-llama3.3-70b": { "input_cost_per_token": 2e-07, "litellm_provider": "lambda_ai", @@ -24185,6 +24142,21 @@ "max_input_tokens": 200000, "max_output_tokens": 8192 }, + "minimax/MiniMax-M3": { + "input_cost_per_token": 6e-07, + "output_cost_per_token": 2.4e-06, + "cache_read_input_token_cost": 1.2e-07, + "litellm_provider": "minimax", + "mode": "chat", + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_system_messages": true, + "supports_vision": true, + "max_input_tokens": 512000, + "max_output_tokens": 128000 + }, "mistral.devstral-2-123b": { "input_cost_per_token": 4e-07, "litellm_provider": "bedrock_converse", @@ -24894,6 +24866,21 @@ "supports_tool_choice": true, "supports_vision": true }, + "mistral/ministral-8b-latest": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "mistral", + "max_input_tokens": 262144, + "max_output_tokens": 262144, + "max_tokens": 262144, + "mode": "chat", + "output_cost_per_token": 1.5e-07, + "source": "https://mistral.ai/pricing", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "mistral/mistral-tiny": { "input_cost_per_token": 2.5e-07, "litellm_provider": "mistral", @@ -25052,6 +25039,7 @@ }, "moonshot/kimi-k2-0711-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25066,6 +25054,7 @@ }, "moonshot/kimi-k2-0905-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25080,6 +25069,7 @@ }, "moonshot/kimi-k2-turbo-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 1.15e-06, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25104,6 +25094,7 @@ "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true @@ -25120,12 +25111,14 @@ "source": "https://platform.kimi.ai/docs/pricing/chat-k26", "supports_function_calling": true, "supports_reasoning": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_video_input": true, "supports_vision": true }, "moonshot/kimi-latest": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25140,6 +25133,7 @@ }, "moonshot/kimi-latest-128k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25154,6 +25148,7 @@ }, "moonshot/kimi-latest-32k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", "max_input_tokens": 32768, @@ -25168,6 +25163,7 @@ }, "moonshot/kimi-latest-8k": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-01-28", "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", "max_input_tokens": 8192, @@ -25182,6 +25178,7 @@ }, "moonshot/kimi-thinking-preview": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2025-11-11", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25194,6 +25191,7 @@ }, "moonshot/kimi-k2-thinking": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 6e-07, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25209,6 +25207,7 @@ }, "moonshot/kimi-k2-thinking-turbo": { "cache_read_input_token_cost": 1.5e-07, + "deprecation_date": "2026-05-25", "input_cost_per_token": 1.15e-06, "litellm_provider": "moonshot", "max_input_tokens": 262144, @@ -25232,9 +25231,11 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-128k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 2e-06, "litellm_provider": "moonshot", "max_input_tokens": 131072, @@ -25256,6 +25257,7 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25269,9 +25271,11 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-32k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 1e-06, "litellm_provider": "moonshot", "max_input_tokens": 32768, @@ -25293,6 +25297,7 @@ "output_cost_per_token": 3e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25306,9 +25311,11 @@ "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "moonshot/moonshot-v1-8k-0430": { + "deprecation_date": "2024-04-30", "input_cost_per_token": 2e-07, "litellm_provider": "moonshot", "max_input_tokens": 8192, @@ -25330,6 +25337,7 @@ "output_cost_per_token": 2e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true }, @@ -25343,6 +25351,7 @@ "output_cost_per_token": 5e-06, "source": "https://platform.moonshot.ai/docs/pricing", "supports_function_calling": true, + "supports_response_schema": true, "supports_tool_choice": true }, "morph/morph-v3-fast": { @@ -26394,6 +26403,32 @@ "supports_vision": true, "supports_web_search": true }, + "oci/meta.llama-3.1-8b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/meta.llama-3.1-70b-instruct": { + "input_cost_per_token": 7.2e-07, + "litellm_provider": "oci", + "max_input_tokens": 128000, + "max_output_tokens": 4000, + "max_tokens": 4000, + "mode": "chat", + "output_cost_per_token": 7.2e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true + }, "oci/meta.llama-3.1-405b-instruct": { "input_cost_per_token": 1.068e-05, "litellm_provider": "oci", @@ -26404,7 +26439,8 @@ "output_cost_per_token": 1.068e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-3.2-90b-vision-instruct": { "input_cost_per_token": 2e-06, @@ -26417,6 +26453,7 @@ "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, "supports_response_schema": false, + "supports_native_streaming": true, "supports_vision": true }, "oci/meta.llama-3.3-70b-instruct": { @@ -26429,31 +26466,35 @@ "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/meta.llama-4-maverick-17b-128e-instruct-fp8": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 512000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 1048576, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true }, "oci/meta.llama-4-scout-17b-16e-instruct": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", - "max_input_tokens": 192000, - "max_output_tokens": 4000, - "max_tokens": 4000, + "max_input_tokens": 10485760, + "max_output_tokens": 8192, + "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3": { "input_cost_per_token": 3e-06, @@ -26465,7 +26506,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-fast": { "input_cost_per_token": 5e-06, @@ -26477,7 +26519,8 @@ "output_cost_per_token": 2.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini": { "input_cost_per_token": 3e-07, @@ -26489,7 +26532,8 @@ "output_cost_per_token": 5e-07, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-3-mini-fast": { "input_cost_per_token": 6e-07, @@ -26501,7 +26545,8 @@ "output_cost_per_token": 4e-06, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/xai.grok-4": { "input_cost_per_token": 3e-06, @@ -26513,7 +26558,8 @@ "output_cost_per_token": 1.5e-05, "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-latest": { "input_cost_per_token": 1.56e-06, @@ -26525,7 +26571,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-a-03-2025": { "input_cost_per_token": 1.56e-06, @@ -26537,7 +26584,8 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true }, "oci/cohere.command-plus-latest": { "input_cost_per_token": 1.56e-06, @@ -26549,7 +26597,88 @@ "output_cost_per_token": 1.56e-06, "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", "supports_function_calling": true, - "supports_response_schema": false + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/google.gemini-2.5-pro": { + "input_cost_per_token": 1.25e-06, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 1e-05, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true + }, + "oci/google.gemini-2.5-flash-lite": { + "input_cost_per_token": 7.5e-08, + "litellm_provider": "oci", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 65536, + "mode": "chat", + "output_cost_per_token": 3e-07, + "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", + "supports_function_calling": true, + "supports_response_schema": true, + "supports_vision": true, + "supports_native_streaming": true, + "supports_image_size": false + }, + "oci/cohere.command-a-vision": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": true, + "supports_response_schema": false, + "supports_native_streaming": true, + "supports_vision": true + }, + "oci/cohere.command-a-reasoning": { + "input_cost_per_token": 1.56e-06, + "litellm_provider": "oci", + "max_input_tokens": 256000, + "max_output_tokens": 8192, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 1.56e-06, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_function_calling": false, + "supports_response_schema": false, + "supports_native_streaming": true + }, + "oci/cohere.embed-multilingual-image-v3.0": { + "input_cost_per_token": 1e-07, + "litellm_provider": "oci", + "max_input_tokens": 512, + "mode": "embedding", + "output_vector_size": 1024, + "source": "https://www.oracle.com/cloud/ai/generative-ai/pricing/", + "supports_vision": true }, "oci/cohere.command-a-reasoning-08-2025": { "input_cost_per_token": 1.56e-06, @@ -26625,18 +26754,6 @@ "supports_response_schema": false, "supports_vision": true }, - "oci/meta.llama-3.1-70b-instruct": { - "input_cost_per_token": 7.2e-07, - "litellm_provider": "oci", - "max_input_tokens": 128000, - "max_output_tokens": 4000, - "max_tokens": 4000, - "mode": "chat", - "output_cost_per_token": 7.2e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": false - }, "oci/meta.llama-3.3-70b-instruct-fp8-dynamic": { "input_cost_per_token": 7.2e-07, "litellm_provider": "oci", @@ -26754,45 +26871,6 @@ "supports_response_schema": true, "supports_vision": true }, - "oci/google.gemini-2.5-pro": { - "input_cost_per_token": 1.25e-06, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 1e-05, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash": { - "input_cost_per_token": 1.5e-07, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 6e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, - "oci/google.gemini-2.5-flash-lite": { - "input_cost_per_token": 7.5e-08, - "litellm_provider": "oci", - "max_input_tokens": 1048576, - "max_output_tokens": 65536, - "max_tokens": 65536, - "mode": "chat", - "output_cost_per_token": 3e-07, - "source": "https://www.oracle.com/artificial-intelligence/generative-ai/generative-ai-service/pricing", - "supports_function_calling": true, - "supports_response_schema": true, - "supports_vision": true - }, "oci/cohere.embed-english-v3.0": { "input_cost_per_token": 1e-07, "litellm_provider": "oci", @@ -27600,7 +27678,8 @@ "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_vision": true + "supports_vision": true, + "supports_image_size": false }, "openrouter/google/gemini-2.5-pro": { "input_cost_per_audio_token": 7e-07, @@ -29470,7 +29549,8 @@ "mode": "responses", "supports_web_search": true, "supports_reasoning": false, - "supports_function_calling": true + "supports_function_calling": true, + "supports_image_size": false }, "perplexity/xai/grok-4-1-fast-non-reasoning": { "litellm_provider": "perplexity", @@ -30052,7 +30132,8 @@ "supports_vision": true, "supports_system_messages": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "replicate/openai/gpt-oss-120b": { "input_cost_per_token": 1.8e-07, @@ -31843,19 +31924,21 @@ "supports_native_structured_output": true }, "us-gov.anthropic.claude-sonnet-4-5-20250929-v1:0": { - "cache_creation_input_token_cost": 4.125e-06, - "cache_read_input_token_cost": 3.3e-07, - "input_cost_per_token": 3.3e-06, - "input_cost_per_token_above_200k_tokens": 6.6e-06, - "output_cost_per_token_above_200k_tokens": 2.475e-05, - "cache_creation_input_token_cost_above_200k_tokens": 8.25e-06, - "cache_read_input_token_cost_above_200k_tokens": 6.6e-07, + "cache_creation_input_token_cost": 4.5e-06, + "cache_creation_input_token_cost_above_1hr": 7.2e-06, + "cache_read_input_token_cost": 3.6e-07, + "input_cost_per_token": 3.6e-06, + "input_cost_per_token_above_200k_tokens": 7.2e-06, + "output_cost_per_token_above_200k_tokens": 2.7e-05, + "cache_creation_input_token_cost_above_200k_tokens": 9.0e-06, + "cache_creation_input_token_cost_above_1hr_above_200k_tokens": 1.44e-05, + "cache_read_input_token_cost_above_200k_tokens": 7.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 200000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", - "output_cost_per_token": 1.65e-05, + "output_cost_per_token": 1.8e-05, "supports_assistant_prefill": true, "supports_computer_use": true, "supports_function_calling": true, @@ -31869,6 +31952,7 @@ }, "au.anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.375e-06, + "cache_creation_input_token_cost_above_1hr": 2.2e-06, "cache_read_input_token_cost": 1.1e-07, "input_cost_per_token": 1.1e-06, "litellm_provider": "bedrock_converse", @@ -32748,7 +32832,8 @@ "supports_vision": true, "supports_function_calling": true, "supports_tool_choice": true, - "supports_response_schema": true + "supports_response_schema": true, + "supports_image_size": false }, "vercel_ai_gateway/google/gemini-2.5-pro": { "input_cost_per_token": 2.5e-06, @@ -33515,6 +33600,7 @@ }, "vertex_ai/claude-haiku-4-5": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33536,6 +33622,7 @@ }, "vertex_ai/claude-haiku-4-5@20251001": { "cache_creation_input_token_cost": 1.25e-06, + "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 1e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33586,6 +33673,7 @@ }, "vertex_ai/claude-3-7-sonnet@20250219": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "deprecation_date": "2026-05-11", "input_cost_per_token": 3e-06, @@ -33685,6 +33773,7 @@ }, "vertex_ai/claude-opus-4": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -33710,6 +33799,7 @@ }, "vertex_ai/claude-opus-4-1": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33727,6 +33817,7 @@ }, "vertex_ai/claude-opus-4-1@20250805": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "input_cost_per_token_batches": 7.5e-06, @@ -33744,6 +33835,7 @@ }, "vertex_ai/claude-opus-4-5": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33770,6 +33862,7 @@ }, "vertex_ai/claude-opus-4-5@20251101": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33797,6 +33890,7 @@ }, "vertex_ai/claude-opus-4-6": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33824,6 +33918,7 @@ }, "vertex_ai/claude-opus-4-6@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33851,6 +33946,7 @@ }, "vertex_ai/claude-opus-4-7": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33878,6 +33974,7 @@ }, "vertex_ai/claude-opus-4-7@default": { "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -33919,6 +34016,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -33947,6 +34045,7 @@ "search_context_size_low": 0.01, "search_context_size_medium": 0.01 }, + "supports_adaptive_thinking": true, "supports_assistant_prefill": false, "supports_computer_use": true, "supports_function_calling": true, @@ -33961,6 +34060,7 @@ }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -33987,6 +34087,7 @@ }, "vertex_ai/claude-sonnet-4-6": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -34014,6 +34115,7 @@ }, "vertex_ai/claude-sonnet-4-5@20250929": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34041,6 +34143,7 @@ }, "vertex_ai/claude-opus-4@20250514": { "cache_creation_input_token_cost": 1.875e-05, + "cache_creation_input_token_cost_above_1hr": 3e-05, "cache_read_input_token_cost": 1.5e-06, "input_cost_per_token": 1.5e-05, "litellm_provider": "vertex_ai-anthropic_models", @@ -34066,6 +34169,7 @@ }, "vertex_ai/claude-sonnet-4": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34095,6 +34199,7 @@ }, "vertex_ai/claude-sonnet-4@20250514": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "input_cost_per_token_above_200k_tokens": 6e-06, @@ -34302,7 +34407,8 @@ "supports_url_context": true, "supports_vision": true, "supports_web_search": false, - "tpm": 8000000 + "tpm": 8000000, + "supports_image_size": false }, "vertex_ai/gemini-3-pro-image-preview": { "input_cost_per_image": 0.0011, @@ -34390,10 +34496,16 @@ "web_search_billing_unit": "per_query" }, "vertex_ai/gemini-3.1-flash-lite": { - "cache_read_input_token_cost": 4.5e-08, - "cache_read_input_token_cost_per_audio_token": 9e-08, - "input_cost_per_audio_token": 9e-07, - "input_cost_per_token": 4.5e-07, + "cache_read_input_token_cost": 2.5e-08, + "cache_read_input_token_cost_batches": 1.25e-08, + "cache_read_input_token_cost_flex": 1.25e-08, + "cache_read_input_token_cost_per_audio_token": 5e-08, + "cache_read_input_token_cost_priority": 4.5e-08, + "input_cost_per_audio_token": 5e-07, + "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, + "input_cost_per_token_flex": 1.25e-07, + "input_cost_per_token_priority": 4.5e-07, "litellm_provider": "vertex_ai-language-models", "max_audio_length_hours": 8.4, "max_audio_per_prompt": 1, @@ -34405,8 +34517,11 @@ "max_video_length": 1, "max_videos_per_prompt": 10, "mode": "chat", - "output_cost_per_reasoning_token": 2.7e-06, - "output_cost_per_token": 2.7e-06, + "output_cost_per_reasoning_token": 1.5e-06, + "output_cost_per_token": 1.5e-06, + "output_cost_per_token_batches": 7.5e-07, + "output_cost_per_token_flex": 7.5e-07, + "output_cost_per_token_priority": 2.7e-06, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", "supported_endpoints": [ "/v1/chat/completions", @@ -34944,6 +35059,22 @@ "us-central1" ] }, + "vertex_ai/google/gemma-4-26b-a4b-it-maas": { + "input_cost_per_token": 1.5e-07, + "litellm_provider": "vertex_ai-openai_models", + "max_input_tokens": 256000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://cloud.google.com/vertex-ai/generative-ai/docs/maas/google/gemma-4-26b-a4b-it", + "supported_regions": [ + "global" + ], + "supports_function_calling": true, + "supports_tool_choice": true, + "supports_vision": true + }, "vertex_ai/openai/gpt-oss-120b-maas": { "input_cost_per_token": 1.5e-07, "litellm_provider": "vertex_ai-openai_models", @@ -35964,7 +36095,8 @@ "supports_prompt_caching": true, "supports_response_schema": false, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-3-beta": { "cache_read_input_token_cost": 7.5e-07, @@ -36163,7 +36295,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -36180,7 +36313,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-0709": { "input_cost_per_token": 3e-06, @@ -36196,7 +36330,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_tool_choice": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-latest": { "input_cost_per_token": 3e-06, @@ -36254,7 +36389,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-reasoning-latest": { "cache_read_input_token_cost": 5e-08, @@ -36275,7 +36411,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-non-reasoning": { "cache_read_input_token_cost": 5e-08, @@ -36295,7 +36432,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4-1-fast-non-reasoning-latest": { "cache_read_input_token_cost": 5e-08, @@ -36315,7 +36453,8 @@ "supports_response_schema": true, "supports_tool_choice": true, "supports_vision": true, - "supports_web_search": true + "supports_web_search": true, + "deprecation_date": "2026-05-15" }, "xai/grok-4.20-multi-agent-beta-0309": { "cache_read_input_token_cost": 2e-07, @@ -36466,7 +36605,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "deprecation_date": "2026-05-15" }, "xai/grok-code-fast-1-0825": { "cache_read_input_token_cost": 2e-08, @@ -36481,7 +36621,8 @@ "supports_function_calling": true, "supports_prompt_caching": true, "supports_reasoning": true, - "supports_tool_choice": true + "supports_tool_choice": true, + "deprecation_date": "2026-05-15" }, "xai/grok-vision-beta": { "input_cost_per_image": 5e-06, @@ -41095,6 +41236,7 @@ }, "vertex_ai/claude-sonnet-4-6@default": { "cache_creation_input_token_cost": 3.75e-06, + "cache_creation_input_token_cost_above_1hr": 6e-06, "cache_read_input_token_cost": 3e-07, "input_cost_per_token": 3e-06, "litellm_provider": "vertex_ai-anthropic_models", @@ -41182,6 +41324,44 @@ "supports_response_schema": true, "supports_tool_choice": true }, + "bedrock_mantle/openai.gpt-5.5": { + "input_cost_per_token": 5.5e-06, + "cache_read_input_token_cost": 5.5e-07, + "output_cost_per_token": 3.3e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, + "bedrock_mantle/openai.gpt-5.4": { + "input_cost_per_token": 2.75e-06, + "cache_read_input_token_cost": 2.75e-07, + "output_cost_per_token": 1.65e-05, + "litellm_provider": "bedrock_mantle", + "max_input_tokens": 272000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "responses", + "supported_endpoints": ["/v1/responses"], + "supported_modalities": ["text", "image"], + "supported_output_modalities": ["text"], + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true + }, "volcengine/doubao-seed-2-0-pro-260215": { "litellm_provider": "volcengine", "max_input_tokens": 256000, @@ -41417,6 +41597,7 @@ }, "bedrock/us-gov-east-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, + "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -41439,6 +41620,7 @@ }, "bedrock/us-gov-west-1/anthropic.claude-haiku-4-5-20251001-v1:0": { "cache_creation_input_token_cost": 1.5e-06, + "cache_creation_input_token_cost_above_1hr": 2.4e-06, "cache_read_input_token_cost": 1.2e-07, "input_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", @@ -41458,5 +41640,18 @@ "supports_vision": true, "supports_native_structured_output": true, "supports_pdf_input": true + }, + "soniox/stt-async-v4": { + "litellm_provider": "soniox", + "max_output_tokens": 8000, + "max_tokens": 8000, + "input_cost_per_second": 0.0, + "output_cost_per_second": 0.0000277778, + "mode": "audio_transcription", + "source": "https://soniox.com/pricing", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supports_audio_input": true } -} +} \ No newline at end of file diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0562b41d2c..e0eeb014c5 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -1539,6 +1539,23 @@ "interactions": true } }, + "neosantara": { + "display_name": "Neosantara (`neosantara`)", + "url": "https://docs.litellm.ai/docs/providers/neosantara", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, "nvidia_nim": { "display_name": "Nvidia NIM (`nvidia_nim`)", "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", diff --git a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py index 2aacab80f5..863e6acd41 100644 --- a/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py +++ b/litellm/proxy/_experimental/mcp_server/auth/user_api_key_auth_mcp.py @@ -7,6 +7,7 @@ from starlette.requests import Request from starlette.types import Scope from litellm._logging import verbose_logger +from litellm.constants import DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL from litellm.proxy._types import ( LiteLLM_TeamTable, ProxyException, @@ -14,6 +15,88 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.auth.ip_address_utils import IPAddressUtils + + +def _parse_mcp_server_names_from_path( + path: str, mcp_servers_header: Optional[List[str]] = None +) -> Optional[List[str]]: + """Resolve the single MCP server name a cold-start passthrough bypass may + target. Delegates parsing to + :meth:`MCPRequestHandler._extract_target_server_names_from_path` so the + names used here always match the names downstream routing uses; returns + ``None`` whenever the bypass must not activate (aggregate ``/mcp``, + multi-server CSV paths, or any other unrecognized path). + + Also fails closed when the ``x-mcp-servers`` header introduces any server + not present in the path-derived target set. Downstream routing for + ``/mcp/...`` paths overrides the header with path-derived names, but a + header/path mismatch here is a sign of a confused or hostile caller — + refuse the cold-start bypass rather than admit anonymously based on the + path while the header advertises a stricter, non-passthrough target.""" + servers = MCPRequestHandler._extract_target_server_names_from_path(path) + if len(servers) != 1: + verbose_logger.debug( + "MCP cold-start: path %r resolved to %r; passthrough 401 bypass " + "requires exactly one target and will not activate", + path, + servers, + ) + return None + if mcp_servers_header is not None and (set(mcp_servers_header) - set(servers)): + verbose_logger.debug( + "MCP cold-start: x-mcp-servers header %r introduces target(s) not " + "in path-derived set %r; passthrough 401 bypass will not activate", + mcp_servers_header, + servers, + ) + return None + return servers + + +def _is_mcp_passthrough_cold_start( + mcp_servers: Optional[List[str]], client_ip: Optional[str] +) -> bool: + """True only when EVERY targeted server is a pass-through server with no + auth headers — the cold-start OAuth discovery case per RFC 9728 / MCP + Authorization spec. Lets the route handler's 401 emitter produce the + spec-compliant WWW-Authenticate challenge instead of surfacing a generic + admission error. + + Uses "all" semantics (mirrors :meth:`MCPRequestHandler._target_servers_use_oauth2`): + one non-passthrough target in a co-targeted set must not flip the bypass + open for the others. Fails closed when any target cannot be resolved.""" + if not mcp_servers: + return False + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + global_mcp_server_manager, + ) + + for name in mcp_servers: + server = global_mcp_server_manager.get_mcp_server_by_name( + name, client_ip=client_ip + ) + if server is None or not getattr(server, "is_oauth_passthrough", False): + return False + return True + + +def _is_litellm_auth_admission_error(exc: Exception) -> bool: + if isinstance(exc, HTTPException): + return exc.status_code == 401 + if isinstance(exc, ProxyException): + try: + return int(exc.code) == 401 + except (TypeError, ValueError): + return False + return False + + +def _has_client_supplied_mcp_auth( + mcp_auth_header: Optional[str], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], +) -> bool: + return bool(mcp_auth_header) or bool(mcp_server_auth_headers) class MCPRequestHandler: @@ -37,7 +120,7 @@ class MCPRequestHandler: LITELLM_MCP_ACCESS_GROUPS_HEADER_NAME = SpecialHeaders.mcp_access_groups.value @staticmethod - async def process_mcp_request( + async def process_mcp_request( # noqa: PLR0915 scope: Scope, ) -> Tuple[ UserAPIKeyAuth, @@ -130,7 +213,9 @@ class MCPRequestHandler: elif ( not litellm_api_key and MCPRequestHandler._target_servers_delegate_auth_to_upstream( # noqa: E501 - path=request_route, mcp_servers=mcp_servers + path=request_route, + mcp_servers=mcp_servers, + client_ip=IPAddressUtils.get_mcp_client_ip(request), ) ): # Operator opted this oauth2 server into upstream-delegated auth @@ -172,25 +257,87 @@ class MCPRequestHandler: # than coercing (``int("None")`` would raise ValueError and # rewrite the auth error as a 500). status = e.status_code if isinstance(e, HTTPException) else e.code - if status in ( - 401, - 403, - "401", - "403", - ) and MCPRequestHandler._target_servers_use_oauth2( - path=request_route, mcp_servers=mcp_servers + is_auth_error = status in (401, 403, "401", "403") + is_unauthenticated = status in (401, "401") + client_ip = IPAddressUtils.get_mcp_client_ip(request) + if is_auth_error and MCPRequestHandler._target_servers_use_oauth2( + path=request_route, + mcp_servers=mcp_servers, + client_ip=client_ip, ): verbose_logger.debug( "MCP OAuth2: target server is OAuth2-mode, treating " "Authorization as upstream OAuth2 token passthrough" ) validated_user_api_key_auth = UserAPIKeyAuth() + elif is_unauthenticated: + # Pass-through cold-start return: per RFC 9728 / MCP + # Authorization spec the client completes upstream OAuth + # discovery and returns with ``Authorization: Bearer + # ``. For ``auth_type=none`` passthrough + # servers that bearer is not a LiteLLM key (auth above + # failed) but is meant to be forwarded upstream + # unchanged. Fall back to anonymous admission so the + # caller is not rejected for following the discovery + # flow without also setting ``x-litellm-api-key``. + # Only trigger on 401 (token unrecognized); a 403 means + # the key WAS recognized but is forbidden (e.g. over + # budget / rate limited) and must propagate so those + # controls are not bypassed via anonymous admission. + mcp_servers_from_path = _parse_mcp_server_names_from_path( + request_route, mcp_servers + ) + if ( + mcp_servers_from_path is not None + and not _has_client_supplied_mcp_auth( + mcp_auth_header, + mcp_server_auth_headers, + ) + and _is_mcp_passthrough_cold_start( + mcp_servers_from_path, client_ip=client_ip + ) + ): + verbose_logger.debug( + "MCP pass-through return: target server is " + "passthrough, treating Authorization as " + "upstream OAuth token for delegated auth" + ) + validated_user_api_key_auth = UserAPIKeyAuth() + else: + raise else: raise else: - validated_user_api_key_auth = await user_api_key_auth( - api_key=litellm_api_key, request=request - ) + try: + validated_user_api_key_auth = await user_api_key_auth( + api_key=litellm_api_key, request=request + ) + except (HTTPException, ProxyException) as exc: + # Cold-start MCP OAuth discovery: RFC 9728 / MCP Authorization spec + # require unauthenticated requests to protected resources to receive + # 401 + WWW-Authenticate. Defer to _raise_preemptive_401_for_unauthenticated_servers + # for pass-through servers instead of surfacing a generic admission error. + mcp_servers_from_path = _parse_mcp_server_names_from_path( + request_route, mcp_servers + ) + client_ip = IPAddressUtils.get_mcp_client_ip(request) + if ( + mcp_servers_from_path is not None + and not _has_client_supplied_mcp_auth( + mcp_auth_header, + mcp_server_auth_headers, + ) + and _is_litellm_auth_admission_error(exc) + and _is_mcp_passthrough_cold_start( + mcp_servers_from_path, client_ip=client_ip + ) + ): + verbose_logger.debug( + "MCP pass-through cold start: deferring admission to route 401 emitter" + ) + validated_user_api_key_auth = UserAPIKeyAuth() + else: + raise return ( validated_user_api_key_auth, @@ -262,7 +409,9 @@ class MCPRequestHandler: return [servers_and_path] @staticmethod - def _target_servers_use_oauth2(path: str, mcp_servers: Optional[List[str]]) -> bool: + def _target_servers_use_oauth2( + path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str] + ) -> bool: """ True only when EVERY MCP server the request targets is configured for ``auth_type == oauth2``. If any target is non-OAuth2 — or if the target @@ -291,14 +440,16 @@ class MCPRequestHandler: return False for name in target_names: - server = global_mcp_server_manager.get_mcp_server_by_name(name) + server = global_mcp_server_manager.get_mcp_server_by_name( + name, client_ip=client_ip + ) if server is None or server.auth_type != MCPAuth.oauth2: return False return True @staticmethod def _target_servers_delegate_auth_to_upstream( - path: str, mcp_servers: Optional[List[str]] + path: str, mcp_servers: Optional[List[str]], client_ip: Optional[str] ) -> bool: """ True only when EVERY MCP server the request targets is configured for @@ -328,7 +479,9 @@ class MCPRequestHandler: return False for name in target_names: - server = global_mcp_server_manager.get_mcp_server_by_name(name) + server = global_mcp_server_manager.get_mcp_server_by_name( + name, client_ip=client_ip + ) if server is None or server.auth_type != MCPAuth.oauth2: return False # `is True` is intentional: opt-in must be an explicit boolean @@ -1090,22 +1243,21 @@ class MCPRequestHandler: ) return [] - # Sentinel stored in cache when an org has no object_permission, so we - # don't re-query the DB on every MCP request for that org. - _ORG_NO_PERMISSION_SENTINEL = "__org_no_mcp_permission__" - @staticmethod async def _get_org_object_permission( user_api_key_auth: Optional[UserAPIKeyAuth] = None, ): """ - Get org object_permission, using user_api_key_cache to avoid DB hits on every request. - - Caches both positive results and the absence of an object_permission so that orgs - with no MCP permissions configured (the common default) do not trigger a DB query - on every request. + Get org object_permission via the established ``get_org_object`` / + ``get_object_permission`` helpers so MCP requests share the same + ``user_api_key_cache`` entries as the rest of the proxy. """ - from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + from litellm.proxy.auth.auth_checks import get_object_permission, get_org_object + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if not user_api_key_auth or not user_api_key_auth.org_id: return None @@ -1114,45 +1266,25 @@ class MCPRequestHandler: verbose_logger.debug("prisma_client is None") return None - org_id = user_api_key_auth.org_id - cache_key = f"org_object_permission:{org_id}" - - from litellm.proxy._types import LiteLLM_ObjectPermissionTable - try: - cached = await user_api_key_cache.async_get_cache(key=cache_key) - if cached is not None: - # Sentinel means the DB confirmed no object_permission for this org - if cached == MCPRequestHandler._ORG_NO_PERMISSION_SENTINEL: - return None - # Redis deserialises to a plain dict; reconstruct the Pydantic model - # so callers can access .mcp_servers / .mcp_tool_permissions as attrs. - if isinstance(cached, dict): - return LiteLLM_ObjectPermissionTable(**cached) - return cached - - org_row = await prisma_client.db.litellm_organizationtable.find_unique( - where={"organization_id": org_id}, - include={"object_permission": True}, + org_obj = await get_org_object( + org_id=user_api_key_auth.org_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - if org_row is None or org_row.object_permission is None: - # Cache the negative result so subsequent calls skip the DB - await user_api_key_cache.async_set_cache( - key=cache_key, - value=MCPRequestHandler._ORG_NO_PERMISSION_SENTINEL, - ) + if org_obj is None or not org_obj.object_permission_id: return None - # Convert raw Prisma model → Pydantic before caching. Caching the - # Pydantic .dict() ensures the value survives a Redis JSON round-trip - # as a plain dict that we can reconstruct above (same pattern used by - # get_end_user_object / get_team_object in auth_checks.py). - obj_perm = LiteLLM_ObjectPermissionTable(**org_row.object_permission.dict()) - await user_api_key_cache.async_set_cache( - key=cache_key, value=obj_perm.dict() + return await get_object_permission( + object_permission_id=org_obj.object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, ) - return obj_perm except Exception as e: verbose_logger.warning(f"Failed to get org object permission: {str(e)}") return None @@ -1273,16 +1405,26 @@ class MCPRequestHandler: ) return [] + # Sentinel stored in cache when an agent has no object_permission, so we + # don't re-query the DB on every MCP request for that agent. + _AGENT_NO_PERMISSION_SENTINEL = "__agent_no_mcp_permission__" + @staticmethod async def _get_agent_object_permission( user_api_key_auth: Optional[UserAPIKeyAuth] = None, ): """ - Fetch the agent's object_permission from the DB (single query). - - Returns the object_permission object or None. + Get agent object_permission via the established ``get_object_permission`` + helper. Caches the ``agent_id -> object_permission_id`` mapping so we + avoid re-reading the agent row on every request, and reuses the shared + ``object_permission_id`` cache populated by the org / team / key paths. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.auth.auth_checks import get_object_permission + from litellm.proxy.proxy_server import ( + prisma_client, + proxy_logging_obj, + user_api_key_cache, + ) if not user_api_key_auth or not user_api_key_auth.agent_id: return None @@ -1291,15 +1433,42 @@ class MCPRequestHandler: verbose_logger.debug("prisma_client is None") return None + agent_id = user_api_key_auth.agent_id + cache_key = f"agent_object_permission_id:{agent_id}" + try: - agent_row = await prisma_client.db.litellm_agentstable.find_unique( - where={"agent_id": user_api_key_auth.agent_id}, - include={"object_permission": True}, + object_permission_id: Optional[str] = ( + await user_api_key_cache.async_get_cache(key=cache_key) ) - if agent_row is None or agent_row.object_permission is None: + + if object_permission_id == MCPRequestHandler._AGENT_NO_PERMISSION_SENTINEL: return None - return agent_row.object_permission + if object_permission_id is None: + agent_row = await prisma_client.db.litellm_agentstable.find_unique( + where={"agent_id": agent_id}, + ) + object_permission_id = ( + getattr(agent_row, "object_permission_id", None) + if agent_row is not None + else None + ) + await user_api_key_cache.async_set_cache( + key=cache_key, + value=object_permission_id + or MCPRequestHandler._AGENT_NO_PERMISSION_SENTINEL, + ttl=DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, + ) + if not object_permission_id: + return None + + return await get_object_permission( + object_permission_id=object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=user_api_key_auth.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) except Exception as e: verbose_logger.warning(f"Failed to get agent object permission: {str(e)}") return None diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index e30667776c..0ba0181200 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -1,16 +1,19 @@ import base64 import binascii +import hashlib import json from datetime import datetime, timedelta, timezone from typing import Any, Dict, Iterable, List, Optional, Set, Union, cast from litellm._logging import verbose_proxy_logger from litellm._uuid import uuid +from litellm.constants import MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS from litellm.proxy._types import ( LiteLLM_MCPServerTable, LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, MCPApprovalStatus, + MCPEnvVarScope, MCPSubmissionsSummary, NewMCPServerRequest, SpecialMCPServerName, @@ -28,6 +31,144 @@ from litellm.types.llms.custom_http import httpxSpecialProvider from litellm.types.mcp import MCPCredentials +def _is_global_env_var_scope(scope: Any) -> bool: + """``scope="user"`` entries are placeholders the user fills in; everything + else (including a missing scope) is an admin-supplied global value.""" + return scope != MCPEnvVarScope.user and scope != "user" + + +def _encrypt_global_env_var_values(env_vars: Iterable[Dict[str, Any]]) -> None: + """Encrypt ``scope="global"`` env var values in place before persisting. + + Global values hold admin-supplied secrets (API keys, passwords) that get + interpolated into headers, so they are encrypted at rest like credentials + and the per-user ``values_b64`` column. Per-user placeholders are not + secrets and are stored verbatim. + """ + for entry in env_vars: + if not _is_global_env_var_scope(entry.get("scope")): + continue + value = entry.get("value") + if value: + entry["value"] = encrypt_value_helper(value) + + +def decrypt_global_env_var_values(env_vars: Optional[Iterable[Any]]) -> None: + """Decrypt ``scope="global"`` env var values in place after reading the DB. + + Accepts ``MCPEnvVar`` models (``LiteLLM_MCPServerTable``) or plain dicts + (raw rows / deserialized JSON). Global values are always stored encrypted, + so a value that no longer decrypts (e.g. after a salt-key change) is dropped + and a warning is logged rather than forwarding the ciphertext into upstream + ``${NAME}`` headers, where it would silently fail. + """ + if not env_vars: + return + for entry in env_vars: + is_dict = isinstance(entry, dict) + scope = entry.get("scope") if is_dict else getattr(entry, "scope", None) + if not _is_global_env_var_scope(scope): + continue + value = entry.get("value") if is_dict else getattr(entry, "value", None) + if not value: + continue + decrypted = decrypt_value_helper( + value=value, + key="mcp_global_env_var", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + name = entry.get("name") if is_dict else getattr(entry, "name", None) + verbose_proxy_logger.warning( + "MCP global env var %s failed to decrypt (LITELLM_SALT_KEY " + "changed?); dropping it so ciphertext is not sent upstream", + name, + ) + decrypted = "" + if is_dict: + entry["value"] = decrypted + else: + entry.value = decrypted + + +def _decrypt_env_vars_on_returned_row(row: Any) -> None: + """Decrypt ``scope="global"`` env var values on a row returned by Prisma create/update. + + Prisma may hand back ``env_vars`` either as a parsed list (the common case for + JSONB columns) or as a raw JSON string (observed for some write paths). The + in-place decrypt helper only mutates iterables of dicts/models, so a string + payload would silently skip decryption and ciphertext would leak into the + registry via ``add_server``/``update_server`` (which trust the caller). + Parse the string back to a list so the in-place decrypt actually runs, and + write the decrypted list back onto the row so downstream consumers see plain + values. + """ + env_vars = getattr(row, "env_vars", None) + if env_vars is None: + return + if isinstance(env_vars, str): + try: + env_vars = json.loads(env_vars) + except (json.JSONDecodeError, TypeError): + return + if not isinstance(env_vars, list): + return + try: + setattr(row, "env_vars", env_vars) + except (AttributeError, TypeError): + pass + decrypt_global_env_var_values(env_vars) + + +def _reencrypt_global_env_var_values( + env_vars: Optional[Iterable[Any]], new_encryption_key: str +) -> Optional[List[Dict[str, Any]]]: + """Re-encrypt ``scope="global"`` env var values for master-key rotation. + + Each global value is decrypted with the current salt key and re-encrypted + under ``new_encryption_key``. Returns the rebuilt list when at least one + value was rotated, else ``None`` so the caller can skip the DB write. A + value that fails to decrypt is left untouched (and logged) so a corrupt + entry is preserved for recovery rather than overwritten. + """ + if not env_vars: + return None + if isinstance(env_vars, str): + try: + env_vars = json.loads(env_vars) + except (json.JSONDecodeError, TypeError): + return None + if not env_vars: + return None + rebuilt = [dict(v) for v in env_vars] + rotated = False + for entry in rebuilt: + if not _is_global_env_var_scope(entry.get("scope")): + continue + value = entry.get("value") + if not value: + continue + decrypted = decrypt_value_helper( + value=value, + key="mcp_global_env_var", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + verbose_proxy_logger.warning( + "rotate_mcp_server_credentials_master_key: could not decrypt " + "global env var %s, skipping", + entry.get("name"), + ) + continue + entry["value"] = encrypt_value_helper( + decrypted, new_encryption_key=new_encryption_key + ) + rotated = True + return rebuilt if rotated else None + + def _prepare_mcp_server_data( data: Union[NewMCPServerRequest, UpdateMCPServerRequest], exclude_unset: bool = False, @@ -67,6 +208,17 @@ def _prepare_mcp_server_data( # ``alias=None`` is a valid request to clear the stored alias. if data_dict.get("alias") is None and "alias" not in fields_set: data_dict.pop("alias", None) + # Prisma ``allowed_tools`` is a required String[]; ``null`` is invalid. + # The UI sends null to clear a whitelist — treat that as ``[]``. + if "allowed_tools" in data_dict and data_dict["allowed_tools"] is None: + data_dict["allowed_tools"] = [] + # Json map fields use ``@default("{}")``; explicit null means clear overrides. + for json_map_field in ( + "tool_name_to_display_name", + "tool_name_to_description", + ): + if json_map_field in data_dict and data_dict[json_map_field] is None: + data_dict[json_map_field] = {} else: data_dict = data.model_dump(exclude_none=True) # Ensure alias is always present in the dict (even if None) @@ -87,19 +239,29 @@ def _prepare_mcp_server_data( if data_dict.get("static_headers") is not None: data_dict["static_headers"] = safe_dumps(data_dict["static_headers"]) + # env_vars is read from ``data_dict`` (not ``data``) like every other JSON + # column so the exclude_unset filter is respected: a partial update that + # omits env_vars never overwrites the stored value. Global values are + # encrypted at rest before serialization. + env_vars = data_dict.get("env_vars") + if env_vars is not None: + serialized_env_vars = [dict(v) for v in env_vars] + _encrypt_global_env_var_values(serialized_env_vars) + data_dict["env_vars"] = safe_dumps(serialized_env_vars) + if data_dict.get("mcp_info") is not None: data_dict["mcp_info"] = safe_dumps(data_dict["mcp_info"]) if data_dict.get("env") is not None: data_dict["env"] = safe_dumps(data_dict["env"]) - if data_dict.get("tool_name_to_display_name") is not None: + if "tool_name_to_display_name" in data_dict: data_dict["tool_name_to_display_name"] = safe_dumps( - data_dict["tool_name_to_display_name"] + data_dict["tool_name_to_display_name"] or {} ) - if data_dict.get("tool_name_to_description") is not None: + if "tool_name_to_description" in data_dict: data_dict["tool_name_to_description"] = safe_dumps( - data_dict["tool_name_to_description"] + data_dict["tool_name_to_description"] or {} ) # mcp_access_groups is already List[str], no serialization needed @@ -196,10 +358,13 @@ async def get_all_mcp_servers( where=where if where else {} ) - return [ + tables = [ LiteLLM_MCPServerTable(**mcp_server.model_dump()) for mcp_server in mcp_servers ] + for table in tables: + decrypt_global_env_var_values(table.env_vars) + return tables except Exception as e: verbose_proxy_logger.debug( "litellm.proxy._experimental.mcp_server.db.py::get_all_mcp_servers - {}".format( @@ -215,14 +380,16 @@ async def get_mcp_server( """ Returns the matching mcp server from the db iff exists """ - mcp_server: Optional[LiteLLM_MCPServerTable] = ( - await prisma_client.db.litellm_mcpservertable.find_unique( - where={ - "server_id": server_id, - } - ) + mcp_server = await prisma_client.db.litellm_mcpservertable.find_unique( + where={ + "server_id": server_id, + } ) - return mcp_server + if mcp_server is None: + return None + table = LiteLLM_MCPServerTable(**mcp_server.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def get_mcp_servers( @@ -240,7 +407,9 @@ async def get_mcp_servers( ) final_mcp_servers: List[LiteLLM_MCPServerTable] = [] for _mcp_server in _mcp_servers: - final_mcp_servers.append(LiteLLM_MCPServerTable(**_mcp_server.model_dump())) + table = LiteLLM_MCPServerTable(**_mcp_server.model_dump()) + decrypt_global_env_var_values(table.env_vars) + final_mcp_servers.append(table) return final_mcp_servers @@ -388,6 +557,11 @@ async def delete_mcp_server( """ Delete the mcp server from the db by server_id + The server-row delete is the commit point. Per-user env var rows have no FK + cascade, so they are cleaned up afterwards on a best-effort basis: a transient + failure there leaves only orphaned rows pointing at a now-missing server and + must not turn a successful delete into a caller-visible error. + Returns the deleted mcp server record if it exists, otherwise None """ deleted_server = await prisma_client.db.litellm_mcpservertable.delete( @@ -395,6 +569,18 @@ async def delete_mcp_server( "server_id": server_id, }, ) + if deleted_server is not None: + try: + await prisma_client.db.litellm_mcpuserenvvars.delete_many( + where={"server_id": server_id} + ) + except Exception as e: + verbose_proxy_logger.warning( + "MCP server %s deleted but per-user env var cleanup failed; " + "orphaned rows can be removed on a later delete: %s", + server_id, + e, + ) return deleted_server @@ -418,6 +604,7 @@ async def create_mcp_server( data=data_dict # type: ignore ) + _decrypt_env_vars_on_returned_row(new_mcp_server) return new_mcp_server @@ -495,40 +682,52 @@ async def update_mcp_server( where={"server_id": data.server_id}, data=data_dict # type: ignore ) + _decrypt_env_vars_on_returned_row(updated_mcp_server) return updated_mcp_server async def rotate_mcp_server_credentials_master_key( prisma_client: PrismaClient, touched_by: str, new_master_key: str ): + from litellm.litellm_core_utils.safe_json_dumps import safe_dumps + mcp_servers = await prisma_client.db.litellm_mcpservertable.find_many() + updated = 0 for mcp_server in mcp_servers: + update_data: Dict[str, Any] = {} + credentials = mcp_server.credentials - if not credentials: + if credentials: + # Decrypt with current key first, then re-encrypt with new key + decrypted_credentials = decrypt_credentials( + credentials=cast(MCPCredentials, dict(credentials)), + ) + encrypted_credentials = encrypt_credentials( + credentials=decrypted_credentials, + encryption_key=new_master_key, + ) + update_data["credentials"] = safe_dumps(encrypted_credentials) + + rotated_env_vars = _reencrypt_global_env_var_values( + mcp_server.env_vars, new_master_key + ) + if rotated_env_vars is not None: + update_data["env_vars"] = safe_dumps(rotated_env_vars) + + if not update_data: continue - credentials_copy = dict(credentials) - # Decrypt with current key first, then re-encrypt with new key - decrypted_credentials = decrypt_credentials( - credentials=cast(MCPCredentials, credentials_copy), - ) - encrypted_credentials = encrypt_credentials( - credentials=decrypted_credentials, - encryption_key=new_master_key, - ) - - from litellm.litellm_core_utils.safe_json_dumps import safe_dumps - - serialized_credentials = safe_dumps(encrypted_credentials) - + update_data["updated_by"] = touched_by await prisma_client.db.litellm_mcpservertable.update( where={"server_id": mcp_server.server_id}, - data={ - "credentials": serialized_credentials, - "updated_by": touched_by, - }, + data=update_data, ) + updated += 1 + verbose_proxy_logger.info( + "rotate_mcp_server_credentials_master_key: rotated %d MCP server row(s)", + updated, + ) def _decode_user_credential(stored: str) -> Optional[str]: @@ -583,6 +782,8 @@ async def rotate_mcp_user_credentials_master_key( are logged and skipped so one corrupt row does not abort the rotation. """ rows = await prisma_client.db.litellm_mcpusercredentials.find_many() + rotated = 0 + skipped = 0 for row in rows: plaintext = _decode_user_credential(row.credential_b64) if plaintext is None: @@ -592,6 +793,7 @@ async def rotate_mcp_user_credentials_master_key( row.user_id, row.server_id, ) + skipped += 1 continue re_encrypted = encrypt_value_helper( plaintext, new_encryption_key=new_master_key @@ -605,6 +807,61 @@ async def rotate_mcp_user_credentials_master_key( }, data={"credential_b64": re_encrypted}, ) + rotated += 1 + verbose_proxy_logger.info( + "rotate_mcp_user_credentials_master_key: rotated %d row(s), skipped %d", + rotated, + skipped, + ) + + +async def rotate_mcp_user_env_vars_master_key( + prisma_client: PrismaClient, new_master_key: str +): + """Re-encrypt every ``LiteLLM_MCPUserEnvVars`` row with ``new_master_key``. + + Reads each ``values_b64`` blob with the current salt key and writes it back + encrypted under the new master key. Rows that fail to decrypt are logged and + skipped so one corrupt row does not abort the rotation nor overwrite values + that may still be recoverable. + """ + rows = await prisma_client.db.litellm_mcpuserenvvars.find_many() + rotated = 0 + skipped = 0 + for row in rows: + plaintext = decrypt_value_helper( + value=row.values_b64, + key="mcp_user_env_vars", + exception_type="debug", + return_original_value=False, + ) + if plaintext is None: + verbose_proxy_logger.warning( + "rotate_mcp_user_env_vars_master_key: could not decrypt env vars " + "for user_id=%s server_id=%s, skipping", + row.user_id, + row.server_id, + ) + skipped += 1 + continue + re_encrypted = encrypt_value_helper( + plaintext, new_encryption_key=new_master_key + ) + await prisma_client.db.litellm_mcpuserenvvars.update( + where={ + "user_id_server_id": { + "user_id": row.user_id, + "server_id": row.server_id, + } + }, + data={"values_b64": re_encrypted}, + ) + rotated += 1 + verbose_proxy_logger.info( + "rotate_mcp_user_env_vars_master_key: rotated %d row(s), skipped %d", + rotated, + skipped, + ) async def store_user_credential( @@ -740,11 +997,14 @@ async def store_user_oauth_credential( ) -def is_oauth_credential_expired(cred: Dict[str, Any]) -> bool: +def is_oauth_credential_expired(cred: Dict[str, Any], buffer_seconds: int = 0) -> bool: """Return True if the OAuth2 credential's access_token has expired. Checks the ``expires_at`` ISO-format string stored in the credential payload. Returns False when ``expires_at`` is absent or unparseable (treat as non-expired). + With ``buffer_seconds`` > 0, a token that is still valid but expires within the + buffer is also treated as expired, so callers can refresh proactively instead of + handing back a token that may lapse mid-request. """ expires_at = cred.get("expires_at") if not expires_at: @@ -753,7 +1013,7 @@ def is_oauth_credential_expired(cred: Dict[str, Any]) -> bool: exp_dt = datetime.fromisoformat(expires_at) if exp_dt.tzinfo is None: exp_dt = exp_dt.replace(tzinfo=timezone.utc) - return datetime.now(timezone.utc) > exp_dt + return datetime.now(timezone.utc) + timedelta(seconds=buffer_seconds) > exp_dt except (ValueError, TypeError): return False @@ -901,6 +1161,50 @@ async def refresh_user_oauth_token( return await get_user_oauth_credential(prisma_client, user_id, server_id) +async def resolve_valid_user_oauth_token( + user_id: str, + server: Any, + cred: Optional[Dict[str, Any]], + prisma_client: Optional[PrismaClient] = None, +) -> Optional[Dict[str, Any]]: + """Return an OAuth2 credential whose access_token is good for the next request. + + Returns the credential unchanged while its token is valid for at least + ``MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS``. Only when the token is expired (or + expiring within that buffer) and a refresh_token is stored does it mint a new one + via ``refresh_user_oauth_token``. Returns None when there is no usable token + (missing token, expired with no refresh_token, or a failed refresh). + + The refresh_token is only ever sent to the server's token_url inside + ``refresh_user_oauth_token``; it is never exposed to the caller beyond the cred + dict it already holds. ``prisma_client`` is fetched lazily and only when a refresh + actually happens, so the valid-token path never requires a DB handle. + """ + if not cred or not cred.get("access_token"): + return None + if not is_oauth_credential_expired( + cred, buffer_seconds=MCP_PER_USER_TOKEN_EXPIRY_BUFFER_SECONDS + ): + return cred + if not cred.get("refresh_token"): + return None + if prisma_client is None: + from litellm.proxy.utils import get_prisma_client_or_throw + + prisma_client = get_prisma_client_or_throw( + "Database not connected. Cannot refresh OAuth token." + ) + refreshed = await refresh_user_oauth_token( + prisma_client=prisma_client, + user_id=user_id, + server=server, + cred=cred, + ) + if not refreshed or not refreshed.get("access_token"): + return None + return refreshed + + async def approve_mcp_server( prisma_client: PrismaClient, server_id: str, @@ -916,7 +1220,9 @@ async def approve_mcp_server( "updated_by": touched_by, }, ) - return LiteLLM_MCPServerTable(**updated.model_dump()) + table = LiteLLM_MCPServerTable(**updated.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def reject_mcp_server( @@ -938,7 +1244,9 @@ async def reject_mcp_server( where={"server_id": server_id}, data=data, ) - return LiteLLM_MCPServerTable(**updated.model_dump()) + table = LiteLLM_MCPServerTable(**updated.model_dump()) + decrypt_global_env_var_values(table.env_vars) + return table async def get_mcp_submissions( @@ -955,6 +1263,8 @@ async def get_mcp_submissions( take=500, # safety cap; paginate if needed in a future iteration ) items = [LiteLLM_MCPServerTable(**r.model_dump()) for r in rows] + for item in items: + decrypt_global_env_var_values(item.env_vars) pending = sum( 1 for i in items if i.approval_status == MCPApprovalStatus.pending_review @@ -969,3 +1279,121 @@ async def get_mcp_submissions( rejected=rejected, items=items, ) + + +# ── Per-user MCP environment variables ──────────────────────────────────── + + +def _decode_user_env_vars(stored: str) -> Dict[str, str]: + """Decrypt a ``values_b64`` blob and parse it as a flat ``{name: value}`` dict.""" + decrypted = decrypt_value_helper( + value=stored, + key="mcp_user_env_vars", + exception_type="debug", + return_original_value=False, + ) + if decrypted is None: + if stored: + verbose_proxy_logger.warning( + "MCP per-user env vars failed to decrypt (LITELLM_SALT_KEY " + "changed?); treating as unset so the user is prompted to " + "re-enter them rather than silently forwarding ciphertext" + ) + return {} + try: + parsed = json.loads(decrypted) + except (ValueError, TypeError): + return {} + if not isinstance(parsed, dict): + return {} + return {str(k): str(v) for k, v in parsed.items()} + + +async def get_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, +) -> Dict[str, str]: + """Return the calling user's env var dict for ``server_id`` (empty if none).""" + row = await prisma_client.db.litellm_mcpuserenvvars.find_unique( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} + ) + if row is None: + return {} + return _decode_user_env_vars(row.values_b64) + + +async def get_user_env_vars_bulk( + prisma_client: PrismaClient, + user_id: str, + server_ids: Iterable[str], +) -> Dict[str, Dict[str, str]]: + """Return ``{server_id: {var_name: value}}`` for one user across many servers. + + Servers with no stored row are simply absent from the result. + """ + ids = list(server_ids) + if not ids: + return {} + rows = await prisma_client.db.litellm_mcpuserenvvars.find_many( + where={"user_id": user_id, "server_id": {"in": ids}} + ) + return {row.server_id: _decode_user_env_vars(row.values_b64) for row in rows} + + +async def merge_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, + updates: Dict[str, str], + allowed_names: Iterable[str], +) -> Dict[str, str]: + """Merge ``updates`` into the user's stored env vars for ``server_id`` and + return the resulting set. + + The read-modify-write runs inside a transaction guarded by a + ``(user_id, server_id)`` advisory lock so two concurrent writes from the + same user can't drop one update. Names outside ``allowed_names`` are pruned, + so an admin retiring a user-scoped variable also clears its stored value. + """ + allowed = set(allowed_names) + lock_key = int.from_bytes( + hashlib.blake2b(f"{user_id}:{server_id}".encode(), digest_size=8).digest(), + "big", + signed=True, + ) + async with prisma_client.db.tx() as tx: + await tx.execute_raw("SELECT pg_advisory_xact_lock($1::bigint)", lock_key) + row = await tx.litellm_mcpuserenvvars.find_unique( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}} + ) + existing = _decode_user_env_vars(row.values_b64) if row is not None else {} + merged = {k: v for k, v in {**existing, **updates}.items() if k in allowed} + encoded = encrypt_value_helper(json.dumps(merged)) + await tx.litellm_mcpuserenvvars.upsert( + where={"user_id_server_id": {"user_id": user_id, "server_id": server_id}}, + data={ + "create": { + "user_id": user_id, + "server_id": server_id, + "values_b64": encoded, + }, + "update": {"values_b64": encoded}, + }, + ) + return merged + + +async def delete_user_env_vars( + prisma_client: PrismaClient, + user_id: str, + server_id: str, +) -> None: + """Remove the calling user's env var values for ``server_id``. + + Uses ``delete_many`` so a missing row is a no-op; real DB errors still + propagate to the caller instead of being silently swallowed. + """ + await prisma_client.db.litellm_mcpuserenvvars.delete_many( + where={"user_id": user_id, "server_id": server_id} + ) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index 8324ba641a..ed374635fe 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -1,8 +1,11 @@ +import asyncio import html as _html import json -from typing import Any, Dict, Optional +import time +from typing import Any, Dict, Optional, Tuple from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse +import httpx from fastapi import APIRouter, Form, HTTPException, Request from fastapi.responses import HTMLResponse, JSONResponse, RedirectResponse @@ -26,11 +29,54 @@ from litellm.proxy.utils import get_server_root_path from litellm.types.mcp import MCPAuth from litellm.types.mcp_server.mcp_server_manager import MCPServer +# TTL cache for upstream OAuth metadata fetched from pass-through MCP servers. +# Keeps us from hammering the upstream IdP on each discovery request. +# Keyed by (server_id, resource_url) → (expires_at_epoch, payload). +# A payload of ``None`` is a negative-result entry that prevents repeated +# upstream fetches when the IdP consistently has no metadata to serve. +_OAUTH_METADATA_CACHE: Dict[Tuple[str, str], Tuple[float, Optional[dict]]] = {} +_OAUTH_METADATA_CACHE_TTL_SECONDS = 300 +_OAUTH_METADATA_NEGATIVE_CACHE_TTL_SECONDS = 60 +_OAUTH_METADATA_CACHE_MAX_SIZE = 128 +# Per-(server_id, resource_url) async locks so concurrent discovery requests +# coalesce onto a single upstream fetch instead of issuing N parallel calls. +_OAUTH_METADATA_FETCH_LOCKS: Dict[Tuple[str, str], asyncio.Lock] = {} + router = APIRouter( tags=["mcp"], ) +def _prune_oauth_metadata_cache(now: Optional[float] = None) -> None: + now = now if now is not None else time.time() + expired_cache_keys = [ + cache_key + for cache_key, (expires_at, _payload) in _OAUTH_METADATA_CACHE.items() + if expires_at <= now + ] + for cache_key in expired_cache_keys: + _OAUTH_METADATA_CACHE.pop(cache_key, None) + + if len(_OAUTH_METADATA_CACHE) > _OAUTH_METADATA_CACHE_MAX_SIZE: + overflow = len(_OAUTH_METADATA_CACHE) - _OAUTH_METADATA_CACHE_MAX_SIZE + cache_keys_by_expiry = sorted( + _OAUTH_METADATA_CACHE, + key=lambda cache_key: _OAUTH_METADATA_CACHE[cache_key][0], + ) + for cache_key in cache_keys_by_expiry[:overflow]: + _OAUTH_METADATA_CACHE.pop(cache_key, None) + + # Drop locks whose cache entry has been evicted and that aren't currently + # held; held locks stay so in-flight callers continue to coalesce. + for cache_key in list(_OAUTH_METADATA_FETCH_LOCKS): + if cache_key in _OAUTH_METADATA_CACHE: + continue + lock = _OAUTH_METADATA_FETCH_LOCKS.get(cache_key) + if lock is None or lock.locked(): + continue + _OAUTH_METADATA_FETCH_LOCKS.pop(cache_key, None) + + def encode_state_with_base_url( base_url: str, original_state: str, @@ -125,6 +171,17 @@ def _resolve_oauth2_server_for_root_endpoints( return None +def _normalize_for_token_comparison(value: Any) -> str: + """Stringify ``value`` for token-rule comparison. + + Booleans are lower-cased so Python's ``True`` / ``False`` line up with + JSON-style ``"true"`` / ``"false"`` rules from admin config. + """ + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + def _validate_token_response( token_response: Dict[str, Any], validation_rules: Dict[str, Any], @@ -136,7 +193,9 @@ def _validate_token_response( ``token_response["team"]["enterprise_id"]``). Top-level keys are tried first, then dot-split traversal. All comparisons are string-coerced so that numeric values in the response (e.g. ``"org_id": 12345``) match string rules - (``"org_id": "12345"``). + (``"org_id": "12345"``). Booleans are normalised to JSON-style ``"true"`` / + ``"false"`` so admin rules written as ``{"verified": "true"}`` match upstream + responses of ``{"verified": true}``. """ for key, expected in validation_rules.items(): actual: Any = token_response.get(key) @@ -163,7 +222,9 @@ def _validate_token_response( ), }, ) - if str(actual) != str(expected): + if _normalize_for_token_comparison(actual) != _normalize_for_token_comparison( + expected + ): raise HTTPException( status_code=403, detail={ @@ -400,6 +461,11 @@ async def exchange_token_with_server( headers={"Accept": "application/json"}, data=token_data, ) + if response is None: + raise HTTPException( + status_code=502, + detail="MCP upstream token endpoint returned no response", + ) response.raise_for_status() token_response = response.json() @@ -505,6 +571,11 @@ async def register_client_with_server( headers=headers, json=register_data, ) + if response is None: + raise HTTPException( + status_code=502, + detail="MCP upstream registration endpoint returned no response", + ) response.raise_for_status() token_response = response.json() @@ -766,7 +837,119 @@ async def callback( """ -def _build_oauth_protected_resource_response( +async def fetch_upstream_oauth_protected_resource( + mcp_server: MCPServer, +) -> Optional[dict]: + """Fetch the upstream MCP server's ``.well-known/oauth-protected-resource`` + metadata for a pass-through server. + + Tries host-only first, then falls back to the RFC 9728 §3.1 path-suffix + form (e.g. ``https://host/.well-known/oauth-protected-resource/mcp``) to + cover upstreams that scope metadata per resource path. + + Responses are cached in-process for ~5 minutes keyed on + ``(server_id, resource_url)`` so we do not hammer the IdP. + + Returns the parsed JSON dict on success, or ``None`` if neither form + responds with a 2xx JSON payload. Raises on network/connection errors so + the caller can emit HTTP 502 rather than fabricate a gateway response. + """ + if not mcp_server.url: + return None + + upstream = urlparse(mcp_server.url) + if not upstream.scheme or not upstream.netloc: + return None + + cache_key = (mcp_server.server_id, mcp_server.url) + now = time.time() + _prune_oauth_metadata_cache(now) + cached = _OAUTH_METADATA_CACHE.get(cache_key) + if cached is not None and cached[0] > now: + return cached[1] + + lock = _OAUTH_METADATA_FETCH_LOCKS.setdefault(cache_key, asyncio.Lock()) + async with lock: + now = time.time() + cached = _OAUTH_METADATA_CACHE.get(cache_key) + if cached is not None and cached[0] > now: + return cached[1] + + host_base = f"{upstream.scheme}://{upstream.netloc}" + candidates = [f"{host_base}/.well-known/oauth-protected-resource"] + # RFC 9728 §3.1 path fallback + if upstream.path and upstream.path not in ("", "/"): + candidates.append( + f"{host_base}/.well-known/oauth-protected-resource" + f"{upstream.path.rstrip('/')}" + ) + + async_client = get_async_httpx_client( + llm_provider=httpxSpecialProvider.Oauth2Check + ) + + network_errors: list[Exception] = [] + for candidate in candidates: + try: + response = await async_client.get( + candidate, + headers={"Accept": "application/json"}, + ) + except Exception as exc: + if is_network_error(exc): + network_errors.append(exc) + else: + verbose_logger.warning( + "MCP OAuth metadata fetch for %s raised non-transport " + "%s: %s — treating as no metadata for this candidate", + candidate, + type(exc).__name__, + exc, + ) + continue + if response.status_code == 200: + try: + payload = response.json() + except Exception as exc: + verbose_logger.warning( + "MCP OAuth metadata at %s returned 200 but JSON " + "decode failed (%s: %s) — treating as no metadata", + candidate, + type(exc).__name__, + exc, + ) + continue + if isinstance(payload, dict): + now = time.time() + _OAUTH_METADATA_CACHE[cache_key] = ( + now + _OAUTH_METADATA_CACHE_TTL_SECONDS, + payload, + ) + _prune_oauth_metadata_cache(now) + return payload + + if len(network_errors) == len(candidates): + raise network_errors[-1] + + # Negative-result caching: when no candidate yielded a usable payload, + # remember that for a shorter TTL so we don't re-fetch on every + # subsequent discovery request (and so the per-key lock can be pruned). + now = time.time() + _OAUTH_METADATA_CACHE[cache_key] = ( + now + _OAUTH_METADATA_NEGATIVE_CACHE_TTL_SECONDS, + None, + ) + _prune_oauth_metadata_cache(now) + return None + + +def is_network_error(exc: Exception) -> bool: + """True for transport-layer failures (connection refused, DNS, TLS, timeout) + as opposed to HTTP protocol errors (4xx/5xx with a valid response).""" + return isinstance(exc, httpx.TransportError) + + +async def _build_oauth_protected_resource_response( request: Request, mcp_server_name: Optional[str], use_standard_pattern: bool, @@ -774,6 +957,12 @@ def _build_oauth_protected_resource_response( """ Build OAuth protected resource response with the appropriate URL pattern. + For pass-through MCP servers (``MCPServer.is_oauth_passthrough``), the + gateway proxies the upstream's own ``oauth-protected-resource`` metadata + so that standards-compliant MCP clients discover the **upstream** IdP + instead of the gateway. The ``resource`` field is rewritten to the + gateway's own URL so clients present the bearer token back to the gateway. + Args: request: FastAPI Request object mcp_server_name: Name of the MCP server @@ -813,6 +1002,46 @@ def _build_oauth_protected_resource_response( else: resource_url = f"{request_base_url}/mcp" + # Pass-through branch: proxy the upstream's own metadata so discovery + # directs the client at the real IdP (Okta, Keycloak, …) instead of us. + if mcp_server is not None and mcp_server.is_oauth_passthrough: + try: + upstream_metadata = await fetch_upstream_oauth_protected_resource( + mcp_server + ) + except Exception as exc: + verbose_logger.warning( + "Failed to fetch upstream oauth-protected-resource metadata " + f"for pass-through MCP server {mcp_server.name!r}: {exc}" + ) + raise HTTPException( + status_code=502, + detail=( + "Failed to fetch upstream oauth-protected-resource " + f"metadata for MCP server {mcp_server.name!r}" + ), + ) + + if upstream_metadata is not None: + response = {**upstream_metadata, "resource": resource_url} + return response + + # Upstream responded but with non-200 or non-dict payload. For + # pass-through servers the gateway is NOT the authorization server, + # so we must not fall through to the default gateway metadata — + # that would point clients at the wrong IdP. + verbose_logger.warning( + "Upstream oauth-protected-resource metadata unavailable for " + f"pass-through MCP server {mcp_server.name!r}" + ) + raise HTTPException( + status_code=502, + detail=( + "Upstream oauth-protected-resource metadata unavailable " + f"for MCP server {mcp_server.name!r}" + ), + ) + return { "authorization_servers": [ ( @@ -843,7 +1072,7 @@ async def oauth_protected_resource_mcp_standard(request: Request, mcp_server_nam This endpoint is compliant with MCP specification and works with standard MCP clients like mcp-inspector and VSCode Copilot. """ - return _build_oauth_protected_resource_response( + return await _build_oauth_protected_resource_response( request=request, mcp_server_name=mcp_server_name, use_standard_pattern=True, @@ -868,36 +1097,22 @@ async def oauth_protected_resource_mcp( This endpoint is kept for backward compatibility. New integrations should use the standard MCP pattern (/mcp/{server_name}) instead. """ - return _build_oauth_protected_resource_response( + return await _build_oauth_protected_resource_response( request=request, mcp_server_name=mcp_server_name, use_standard_pattern=False, ) -""" - https://datatracker.ietf.org/doc/html/rfc8414#section-3.1 - RFC 8414: Path-aware OAuth discovery - If the issuer identifier value contains a path component, any - terminating "/" MUST be removed before inserting "/.well-known/" and - the well-known URI suffix between the host component and the path(include root path) - component. -""" - - def _build_oauth_authorization_server_response( request: Request, mcp_server_name: Optional[str], ) -> dict: - """ - Build OAuth authorization server metadata response. + """Build OAuth authorization server metadata response (gateway-as-AS shape). - Args: - request: FastAPI Request object - mcp_server_name: Name of the MCP server - - Returns: - OAuth authorization server metadata dict + Synchronous because the body only does dict construction and synchronous + registry lookups; unlike :func:`_build_oauth_protected_resource_response` + it does not need to await any upstream IO. """ from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, diff --git a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py new file mode 100644 index 0000000000..e42270bf10 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py @@ -0,0 +1,163 @@ +""" +MCP Elicitation Handler +Handles `elicitation/create` requests from upstream MCP servers by either: +1. Relaying them to the connected downstream MCP client (if it supports elicitation) +2. Returning a decline/error response (if no downstream client or unsupported) +Supports both Form mode (structured data collection) and URL mode (external URL +navigation for sensitive interactions like OAuth). +MCP Spec Reference: + https://modelcontextprotocol.io/specification/2025-11-25/client/elicitation +""" + +from typing import Any, Optional, Union +from litellm._logging import verbose_logger + +# Guard imports that require the mcp package +try: + from mcp.types import ( + ElicitRequestFormParams, + ElicitRequestParams, + ElicitRequestURLParams, + ElicitResult, + ErrorData, + ) + + MCP_ELICITATION_AVAILABLE = True +except ImportError: + MCP_ELICITATION_AVAILABLE = False + + +async def handle_elicitation_request( + context: Any, + params: "ElicitRequestParams", + downstream_session: Optional[Any] = None, + downstream_capabilities: Optional[Any] = None, +) -> Union["ElicitResult", "ErrorData"]: + """ + Handle an MCP elicitation/create request from an upstream MCP server. + In Gateway mode (Mode A), we relay the elicitation request to the + connected downstream client if they declared elicitation capabilities. + In Tool Bridge mode (Mode B), there's no persistent downstream MCP + client, so we return a decline response. + Args: + context: MCP RequestContext from the upstream server connection. + params: The ElicitRequestParams (either form or URL mode). + downstream_session: The ServerSession to the downstream client, + if available (for relaying). + downstream_capabilities: The downstream client's declared + capabilities, used to check elicitation support. + Returns: + ElicitResult with the user's response, or ErrorData on failure. + """ + if not MCP_ELICITATION_AVAILABLE: + return ErrorData( + code=-1, + message="MCP elicitation is not available (mcp package not installed)", + ) + try: + mode = getattr(params, "mode", "form") + verbose_logger.info( + "MCP elicitation: received request mode=%s, message=%s", + mode, + getattr(params, "message", ""), + ) + # Check if we have a downstream session to relay to + if downstream_session is not None: + return await _relay_elicitation_to_downstream( + params=params, + downstream_session=downstream_session, + downstream_capabilities=downstream_capabilities, + ) + # No downstream session — we're in Tool Bridge mode + # or the client doesn't support elicitation + verbose_logger.info( + "MCP elicitation: no downstream session available, declining" + ) + return ElicitResult( + action="decline", + ) + except Exception as e: + verbose_logger.exception("MCP elicitation handler failed: %s", e) + return ErrorData( + code=-1, + message=f"Elicitation failed: {str(e)}", + ) + + +async def _relay_elicitation_to_downstream( + params: "ElicitRequestParams", + downstream_session: Any, + downstream_capabilities: Optional[Any] = None, +) -> Union["ElicitResult", "ErrorData"]: + """ + Relay an elicitation request to the downstream MCP client. + Uses the ServerSession's elicit_form() or elicit_url() methods to + send the elicitation request back to the connected client. + Args: + params: The elicitation request parameters. + downstream_session: The ServerSession connected to the downstream client. + downstream_capabilities: Client capabilities to check support. + Returns: + ElicitResult from the downstream client. + """ + mode = getattr(params, "mode", "form") + # Check if the downstream client supports the requested mode + if downstream_capabilities is not None: + elicit_caps = getattr(downstream_capabilities, "elicitation", None) + if elicit_caps is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support elicitation" + ) + return ElicitResult(action="decline") + if mode == "url": + url_cap = getattr(elicit_caps, "url", None) + if url_cap is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support URL mode" + ) + return ElicitResult(action="decline") + if mode == "form": + form_cap = getattr(elicit_caps, "form", None) + if form_cap is None: + verbose_logger.info( + "MCP elicitation: downstream client does not support form mode" + ) + return ElicitResult(action="decline") + try: + if mode == "url" and isinstance(params, ElicitRequestURLParams): + # URL mode: relay URL to client for external navigation + verbose_logger.info( + "MCP elicitation: relaying URL mode to downstream, url=%s", + getattr(params, "url", ""), + ) + result = await downstream_session.elicit_url( + message=params.message, + url=params.url, + elicitation_id=getattr(params, "elicitationId", None), + ) + elif isinstance(params, ElicitRequestFormParams): + # Form mode: relay structured form to client + verbose_logger.info("MCP elicitation: relaying form mode to downstream") + result = await downstream_session.elicit_form( + message=params.message, + requestedSchema=getattr(params, "requestedSchema", None), + ) + else: + # Fallback for generic ElicitRequestParams — pass an empty schema + # since elicit() requires requestedSchema as a positional arg. + verbose_logger.info( + "MCP elicitation: relaying generic elicitation to downstream" + ) + result = await downstream_session.elicit( + message=getattr(params, "message", ""), + requestedSchema=getattr(params, "requestedSchema", {}), + ) + verbose_logger.info( + "MCP elicitation: downstream responded with action=%s", + getattr(result, "action", "unknown"), + ) + return result + except Exception as e: + verbose_logger.warning("MCP elicitation: failed to relay to downstream: %s", e) + # If relay fails, decline gracefully + return ElicitResult(action="decline") diff --git a/litellm/proxy/_experimental/mcp_server/exceptions.py b/litellm/proxy/_experimental/mcp_server/exceptions.py new file mode 100644 index 0000000000..fd8fc3d5e5 --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/exceptions.py @@ -0,0 +1,80 @@ +"""Exceptions raised by the LiteLLM MCP proxy.""" + +from typing import Optional + +from fastapi import HTTPException + + +class MCPUpstreamAuthError(Exception): + """Raised when an upstream MCP server returns an authentication failure + (typically HTTP 401) and the gateway should surface it transparently to + the client instead of swallowing it. + + Only relevant for pass-through MCP servers (see + ``MCPServer.is_oauth_passthrough``). The gateway converts this exception + into an HTTP 401 response on single-server routes, preserving any + ``WWW-Authenticate`` challenge emitted by the upstream so standards- + compliant MCP clients can trigger the upstream OAuth flow. + """ + + def __init__( + self, + status_code: int, + www_authenticate: Optional[str], + server_name: str, + ) -> None: + self.status_code = status_code + self.www_authenticate = www_authenticate + self.server_name = server_name + super().__init__(f"Upstream MCP server {server_name!r} returned {status_code}") + + def to_http_exception( + self, + base_url: Optional[str] = None, + request_path: Optional[str] = None, + ) -> HTTPException: + """Convert this upstream-auth error into an ``HTTPException`` that + preserves the upstream status code and any ``WWW-Authenticate`` + challenge, so standards-compliant MCP clients can trigger the + upstream OAuth flow. + + When the upstream 401 omits ``WWW-Authenticate`` (non-compliant per + RFC 7235 §3.1) we fabricate a ``Bearer resource_metadata=`` challenge + that points at the gateway's well-known endpoint for this server, so + MCP clients can still initiate RFC 9728 discovery against the upstream + IdP via the gateway's proxied metadata. Callers must pass ``base_url`` + (the gateway origin, no trailing slash) so the fabricated URI is + absolute as RFC 9728 §3.2 requires; if ``base_url`` is missing we + skip fabrication entirely rather than emit a relative URI that strict + clients reject in the Bearer challenge. + + When ``request_path`` is supplied and matches the legacy + ``/{server_name}/mcp`` MCP transport route, the fabricated URI uses + the matching legacy well-known form + ``/.well-known/oauth-protected-resource/{server_name}/mcp``. Otherwise + we default to the standard form + ``/.well-known/oauth-protected-resource/mcp/{server_name}``. This + keeps the ``resource_metadata`` URI aligned with the resource pattern + the client originally targeted, matching the path-aware behaviour of + ``_get_passthrough_resource_metadata_url`` in ``server.py``. + """ + challenge: Optional[str] = self.www_authenticate + if challenge is None and self.status_code == 401 and base_url: + prefix = base_url.rstrip("/") + if request_path and request_path.startswith(f"/{self.server_name}/mcp"): + resource_metadata_url = ( + f"{prefix}/.well-known/oauth-protected-resource/" + f"{self.server_name}/mcp" + ) + else: + resource_metadata_url = ( + f"{prefix}/.well-known/oauth-protected-resource/" + f"mcp/{self.server_name}" + ) + challenge = f'Bearer resource_metadata="{resource_metadata_url}"' + detail = "Forbidden" if self.status_code == 403 else "Unauthorized" + return HTTPException( + status_code=self.status_code, + detail=detail, + headers={"www-authenticate": challenge} if challenge else None, + ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index f35aa30a7c..7048f5bf7c 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -48,23 +48,36 @@ from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) +from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError +from litellm.proxy._experimental.mcp_server.elicitation_handler import ( + MCP_ELICITATION_AVAILABLE, +) +from litellm.proxy._experimental.mcp_server.sampling_handler import ( + MCP_SAMPLING_AVAILABLE, +) from litellm.proxy._experimental.mcp_server.oauth2_token_cache import resolve_mcp_auth from litellm.proxy._experimental.mcp_server.utils import ( MCP_TOOL_PREFIX_SEPARATOR, + MCPMissingUserEnvVarsError, add_server_prefix_to_name, + build_env_var_setup_url, + collect_env_var_references, compute_short_server_prefix, get_server_prefix, + interpolate_headers, is_short_mcp_tool_prefix_enabled, is_tool_name_prefixed, iter_known_server_prefixes, merge_mcp_headers, normalize_server_name, + parse_admin_env_vars, split_server_prefix_from_name, validate_mcp_server_name, ) from litellm.proxy._types import ( LiteLLM_MCPServerTable, MCPAuthType, + MCPEnvVar, MCPTransport, MCPTransportType, UserAPIKeyAuth, @@ -117,6 +130,130 @@ _AZURE_ENTRA_HOSTS = { "login.chinacloudapi.cn", # China } +# Short-lived in-memory cache for per-user MCP env var values, mirroring the +# BYOK credential cache. Keyed by (user_id, server_id); value is +# (values_dict, monotonic_timestamp). Keeps the tool-call and tool-listing +# paths off the DB on every request within the TTL window. +_user_env_vars_cache: Dict[Tuple[str, str], Tuple[Dict[str, str], float]] = {} +_USER_ENV_VARS_CACHE_TTL = 60 # seconds +_USER_ENV_VARS_CACHE_MAX_SIZE = 4096 # cap to prevent unbounded growth + + +def invalidate_user_env_vars_cache(user_id: str, server_id: str) -> None: + """Drop a cached entry after the user stores or clears their env var values + so the next request reads the fresh value instead of a stale one.""" + _user_env_vars_cache.pop((user_id, server_id), None) + + +def _write_user_env_vars_cache( + user_id: str, server_id: str, values: Dict[str, str] +) -> None: + cache_key = (user_id, server_id) + # Re-insert at the tail so eviction drops the oldest-written entry, not a + # freshly refreshed one, and only sheds a single entry instead of wiping the + # whole cache (which would stampede the DB). + _user_env_vars_cache.pop(cache_key, None) + if len(_user_env_vars_cache) >= _USER_ENV_VARS_CACHE_MAX_SIZE: + _user_env_vars_cache.pop(next(iter(_user_env_vars_cache)), None) + _user_env_vars_cache[cache_key] = (values, time.monotonic()) + + +def _should_strip_caller_authorization( + mcp_server: MCPServer, + raw_headers: Optional[Dict[str, str]], + user_api_key_auth: Optional[UserAPIKeyAuth], +) -> bool: + """Decide whether the caller's ``Authorization`` header must NOT be + forwarded upstream when populating ``extra_headers`` for an MCP server. + + Centralized so ``_call_regular_mcp_tool`` (this module) and + ``_prepare_mcp_server_headers`` (``server.py``) cannot drift apart on + this security-sensitive decision. + + Strip rules: + - **M2M (client_credentials) servers**: never forward the caller's + ``Authorization`` — the proxy fetches its own upstream token. + - **OAuth pass-through servers**: strip when the ``Authorization`` + header is actually the LiteLLM API key — either because admission + validated it (``user_api_key_auth.api_key`` is set) and the caller + did NOT also supply ``x-litellm-api-key`` to disambiguate, or + because the legacy ``user_api_key_auth is None`` call sites did + not supply an explicit admission header. In the anonymous / + pass-through cold-start case (RFC 9728) the bearer in + ``Authorization`` is the upstream OAuth token and must be + forwarded, so we keep it. + """ + if mcp_server.has_client_credentials: + return True + if not mcp_server.is_oauth_passthrough: + return False + + normalized_raw_headers = { + str(k).lower(): v for k, v in (raw_headers or {}).items() if isinstance(k, str) + } + has_explicit_litellm_admission_header = ( + normalized_raw_headers.get("x-litellm-api-key") is not None + ) + admission_consumed_authorization_as_litellm_key = ( + user_api_key_auth is not None + and bool(getattr(user_api_key_auth, "api_key", None)) + and not has_explicit_litellm_admission_header + ) + return admission_consumed_authorization_as_litellm_key or ( + user_api_key_auth is None and not has_explicit_litellm_admission_header + ) + + +def _extract_upstream_auth_failure( + exc: BaseException, +) -> Optional[Tuple[int, Optional[str]]]: + """Walk the exception tree looking for an HTTP 401/403 response from the + upstream MCP server. + + The MCP SDK wraps transport errors in anyio ``ExceptionGroup`` objects and + may chain through ``__cause__`` / ``__context__``. We inspect all of those + layers for an ``httpx.Response``-bearing exception (typically + ``httpx.HTTPStatusError``) and extract the status code and any upstream + ``WWW-Authenticate`` header. + + Returns ``(status_code, www_authenticate)`` on match, else ``None``. + """ + seen: Set[int] = set() + stack: List[BaseException] = [exc] + while stack: + current = stack.pop() + if id(current) in seen: + continue + seen.add(id(current)) + + response = getattr(current, "response", None) + if response is not None: + status_code = getattr(response, "status_code", None) + if isinstance(status_code, int) and status_code in (401, 403): + www_authenticate: Optional[str] = None + headers = getattr(response, "headers", None) + if headers is not None: + try: + www_authenticate = headers.get("www-authenticate") + except Exception: + www_authenticate = None + return status_code, www_authenticate + + # anyio / PEP 654 ExceptionGroup + sub_exceptions = getattr(current, "exceptions", None) + if sub_exceptions: + stack.extend(sub_exceptions) + + if current.__cause__ is not None: + stack.append(current.__cause__) + if ( + current.__context__ is not None + and current.__context__ is not current.__cause__ + ): + stack.append(current.__context__) + + return None + def _warn_on_server_name_fields( *, @@ -191,6 +328,107 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]: return data +def _deserialize_json_list(data: Any) -> Optional[List[Dict[str, Any]]]: + """Deserialize a JSON array stored in the DB (``env_vars`` and friends). + + Returns ``None`` for empty / null / unparseable input. Accepts strings + (raw JSON), already-materialized lists of dicts, and lists of Pydantic + models (Prisma may hydrate a JSON column such as ``env_vars`` into + ``MCPEnvVar`` objects); model entries are normalized to plain dicts so + downstream consumers expecting ``List[Dict[str, Any]]`` validate. + """ + if data is None or data == "" or data == []: + return None + if isinstance(data, str): + try: + parsed = json.loads(data) + except (json.JSONDecodeError, TypeError): + return None + data = parsed + if not isinstance(data, list): + return None + return [ + item.model_dump(mode="json") if hasattr(item, "model_dump") else item + for item in data + ] + + +def _create_sampling_callback(user_api_key_auth: Optional[Any] = None): + """ + Create a sampling callback for MCP ClientSession. + Returns a callable that handles sampling/createMessage requests from + upstream MCP servers by routing them through litellm.acompletion(). + """ + if not MCP_SAMPLING_AVAILABLE: + return None + + async def _sampling_callback(context, params): + from litellm.proxy._experimental.mcp_server.sampling_handler import ( + handle_sampling_create_message, + ) + import litellm + from litellm.proxy._experimental.mcp_server.server import ( + get_active_auth_context, + ) + + auth_context = get_active_auth_context() + resolved_auth = user_api_key_auth or ( + auth_context.user_api_key_auth if auth_context else None + ) + # Forward original HTTP headers and client IP so that + # header-dependent guardrails, tag-based routing, trace + # correlation, and forward_llm_provider_auth_headers work + # correctly for sampling sub-calls. + _raw_headers = getattr(auth_context, "raw_headers", None) + _client_ip = getattr(auth_context, "client_ip", None) + + return await handle_sampling_create_message( + context=context, + params=params, + default_model=getattr(litellm, "default_mcp_sampling_model", None), + user_api_key_auth=resolved_auth, + raw_headers=_raw_headers, + client_ip=_client_ip, + ) + + return _sampling_callback + + +def _create_elicitation_callback(): + """ + Create an elicitation callback for MCP ClientSession. + Returns a callable that handles elicitation/create requests from + upstream MCP servers. In gateway mode, this relays to the downstream + client; in tool bridge mode, it returns a decline response. + """ + if not MCP_ELICITATION_AVAILABLE: + return None + + async def _elicitation_callback(context, params): + from litellm.proxy._experimental.mcp_server.elicitation_handler import ( + handle_elicitation_request, + ) + from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session + + # In Gateway mode, we relay the elicitation request to the downstream client + # that triggered the current operation. + downstream_session = get_active_mcp_session() + downstream_capabilities = ( + getattr(downstream_session, "capabilities", None) + if downstream_session + else None + ) + + return await handle_elicitation_request( + context=context, + params=params, + downstream_session=downstream_session, + downstream_capabilities=downstream_capabilities, + ) + + return _elicitation_callback + + class MCPServerManager: _STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$") @@ -276,7 +514,8 @@ class MCPServerManager: - server is OpenAPI (spec_path), - non-empty upstream instructions are already cached, - auth preconditions match health_check_server's skip rules - (per-user auth / missing static auth token), + (per-user auth / missing static auth token / static headers that + reference a per-user env var), - a prior probe attempt for this server is within MCP_HEALTH_CHECK_TIMEOUT seconds (the probe is a health-check-shaped op and already uses this knob for its inner call timeout; reusing it @@ -291,6 +530,8 @@ class MCPServerManager: return if server.requires_per_user_auth: return + if self._references_per_user_env_var(server): + return if ( server.auth_type and server.auth_type != MCPAuth.none @@ -315,8 +556,13 @@ class MCPServerManager: ) try: + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server=server, + user_api_key_auth=None, + raise_on_missing=False, + ) extra_headers: Optional[Dict[str, str]] = ( - dict(server.static_headers) if server.static_headers else None + dict(resolved_static_headers) if resolved_static_headers else None ) client = await self._create_mcp_client( server=server, @@ -476,6 +722,7 @@ class MCPServerManager: allowed_params=server_config.get("allowed_params", None), access_groups=server_config.get("access_groups", None), static_headers=server_config.get("static_headers", None), + env_vars=server_config.get("env_vars", None), allow_all_keys=bool(server_config.get("allow_all_keys", False)), available_on_public_internet=bool( server_config.get("available_on_public_internet", True) @@ -483,6 +730,7 @@ class MCPServerManager: delegate_auth_to_upstream=bool( server_config.get("delegate_auth_to_upstream", False) ), + oauth_passthrough=bool(server_config.get("oauth_passthrough", False)), # AWS SigV4 fields aws_access_key_id=server_config.get("aws_access_key_id", None), aws_secret_access_key=server_config.get("aws_secret_access_key", None), @@ -501,6 +749,9 @@ class MCPServerManager: "subject_token_type", "urn:ietf:params:oauth:token-type:access_token", ), + allow_sampling=bool(server_config.get("allow_sampling", False)), + allow_elicitation=bool(server_config.get("allow_elicitation", False)), + timeout=server_config.get("timeout", None), ) self._assign_unique_short_prefix(new_server) _warn_internal_delegate_pkce_if_applicable(new_server, source="config") @@ -600,8 +851,7 @@ class MCPServerManager: ) verbose_logger.debug( - f"Using headers for OpenAPI tools (excluding sensitive values): " - f"{list(headers.keys())}" + f"Using headers for OpenAPI tools (excluding sensitive values): {list(headers.keys())}" ) # Extract and register tools from OpenAPI paths @@ -737,17 +987,41 @@ class MCPServerManager: f"Server ID {mcp_server.server_id} not found in registry" ) + def _resolve_env_vars_list( + self, + mcp_server: LiteLLM_MCPServerTable, + *, + env_vars_are_encrypted: bool, + ) -> Optional[List[Dict[str, Any]]]: + env_vars_list = _deserialize_json_list(getattr(mcp_server, "env_vars", None)) + if env_vars_are_encrypted: + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + decrypt_global_env_var_values, + ) + + decrypt_global_env_var_values(env_vars_list) + return env_vars_list + async def build_mcp_server_from_table( self, mcp_server: LiteLLM_MCPServerTable, *, credentials_are_encrypted: bool = True, + env_vars_are_encrypted: Optional[bool] = None, ) -> MCPServer: _mcp_info: MCPInfo = mcp_server.mcp_info or {} env_dict = _deserialize_json_dict(getattr(mcp_server, "env", None)) static_headers_dict = _deserialize_json_dict( getattr(mcp_server, "static_headers", None) ) + env_vars_list = self._resolve_env_vars_list( + mcp_server, + env_vars_are_encrypted=( + credentials_are_encrypted + if env_vars_are_encrypted is None + else env_vars_are_encrypted + ), + ) credentials_dict = _deserialize_json_dict( getattr(mcp_server, "credentials", None) ) @@ -847,6 +1121,7 @@ class MCPServerManager: mcp_info=mcp_info, extra_headers=getattr(mcp_server, "extra_headers", None), static_headers=static_headers_dict, + env_vars=env_vars_list, client_id=client_id_value or getattr(mcp_server, "client_id", None), client_secret=client_secret_value or getattr(mcp_server, "client_secret", None), @@ -881,6 +1156,7 @@ class MCPServerManager: delegate_auth_to_upstream=bool( getattr(mcp_server, "delegate_auth_to_upstream", False) ), + oauth_passthrough=bool(getattr(mcp_server, "oauth_passthrough", False)), created_at=getattr(mcp_server, "created_at", None), updated_at=getattr(mcp_server, "updated_at", None), tool_name_to_display_name=_deserialize_json_dict( @@ -913,6 +1189,7 @@ class MCPServerManager: credentials_dict.get("subject_token_type") if credentials_dict else None ) or "urn:ietf:params:oauth:token-type:access_token", + timeout=getattr(mcp_server, "timeout", None), ) _warn_internal_delegate_pkce_if_applicable(new_server, source="database") return new_server @@ -943,7 +1220,14 @@ class MCPServerManager: return try: if mcp_server.server_id not in self.registry: - new_server = await self.build_mcp_server_from_table(mcp_server) + # Callers hand us a record returned by the db.py read/write + # helpers, which already decrypt global env var values (the + # `credentials` field is the only one still encrypted here). + # Re-decrypting plaintext would zero the values, so build with + # env_vars_are_encrypted=False. + new_server = await self.build_mcp_server_from_table( + mcp_server, env_vars_are_encrypted=False + ) self._assign_unique_short_prefix(new_server) self.registry[mcp_server.server_id] = new_server await self._maybe_register_openapi_tools(new_server) @@ -966,7 +1250,11 @@ class MCPServerManager: return try: if mcp_server.server_id in self.registry: - new_server = await self.build_mcp_server_from_table(mcp_server) + # See add_server: db.py helpers already decrypted env var + # values, so don't decrypt them a second time here. + new_server = await self.build_mcp_server_from_table( + mcp_server, env_vars_are_encrypted=False + ) # Carry the previously-resolved short prefix across so the # tool names stay stable for clients holding cached lists. existing_prefix = self.registry[mcp_server.server_id].short_prefix @@ -1387,6 +1675,180 @@ class MCPServerManager: return resolved_env + def _references_per_user_env_var(self, server: MCPServer) -> bool: + """True when ``server.static_headers`` reference a per-user ``${NAME}`` env var. + + Such placeholders can only be filled from a calling user's stored values, + so a userless probe (health check / instructions prefetch) would forward + the literal ``${NAME}`` upstream and get rejected. Callers skip the probe + and report ``unknown`` instead of a misleading ``unhealthy``. + """ + static_headers = server.static_headers + env_vars = getattr(server, "env_vars", None) + if not static_headers or not env_vars: + return False + _global_values, user_specs = parse_admin_env_vars(env_vars) + user_var_names = {spec["name"] for spec in user_specs} + if not user_var_names: + return False + referenced = collect_env_var_references(strings=static_headers.values()) + return bool(referenced & user_var_names) + + async def _resolve_static_headers_with_env_vars( + self, + server: MCPServer, + user_api_key_auth: Optional[UserAPIKeyAuth], + *, + raise_on_missing: bool = True, + ) -> Optional[Dict[str, str]]: + """Return server.static_headers with ``${NAME}`` interpolated. + + Globals come from ``server.env_vars`` entries with ``scope=="global"``. + Per-user values come from the ``LiteLLM_MCPUserEnvVars`` row for the + calling user. + + When ``raise_on_missing`` is ``True`` (the tool-*call* path), raises + ``MCPMissingUserEnvVarsError`` if ``static_headers`` reference a per-user + variable the calling user has not yet supplied — converted into a + user-facing 412 by the REST layer. + + When ``raise_on_missing`` is ``False`` (the tool-*list* path), missing + per-user vars are non-blocking: we interpolate whatever is available and + leave unfilled ``${NAME}`` references untouched, so the server's tools + still appear in the listing. The user only hits the friendly error when + they actually invoke a tool that needs the missing value. + """ + static_headers = server.static_headers + env_vars = getattr(server, "env_vars", None) + if not static_headers and not env_vars: + return static_headers + + global_values, user_specs = parse_admin_env_vars(env_vars) + # An empty-valued global is treated as unset: it must not mask a per-user + # var the user still has to supply, nor override a value the user did + # supply. The unresolved ${NAME} is then left untouched, like any other + # undefined reference. + global_values = {name: value for name, value in global_values.items() if value} + user_var_names = {spec["name"] for spec in user_specs} + + # If no env vars are configured, return static_headers as-is. + if not global_values and not user_specs: + return static_headers + + # Figure out which user-scoped vars are actually referenced. A var that + # also carries a global value is always covered by that global (globals + # win in the merge below), so it can never be genuinely "missing" even if + # the user hasn't filled it in -- only vars without a global fallback do. + referenced = collect_env_var_references(strings=(static_headers or {}).values()) + referenced_user_vars = referenced & user_var_names + required_user_vars = { + name for name in referenced_user_vars if name not in global_values + } + + user_values: Dict[str, str] = {} + if required_user_vars: + try: + user_values = await self._load_user_env_vars(server, user_api_key_auth) + except Exception as exc: + # On the tool-call path a DB failure must surface as a real + # server error, not a misleading "set up your credentials" 412. + # On the listing path we stay best-effort and leave the + # unfilled ${NAME} references untouched so tools still appear. + if raise_on_missing: + raise + verbose_logger.warning( + "MCPServerManager: best-effort user env var load failed for " + "server=%s: %s", + server.server_id, + exc, + ) + + if raise_on_missing: + missing = sorted( + name for name in required_user_vars if not user_values.get(name) + ) + if missing: + # A cached negative must never produce a 412: cache + # invalidation is process-local, so a user who just stored + # values on another worker would otherwise be told their + # credentials are missing until the entry expires. Confirm + # against the DB before raising. + user_values = await self._load_user_env_vars( + server, user_api_key_auth, force_refresh=True + ) + missing = sorted( + name for name in required_user_vars if not user_values.get(name) + ) + if missing: + raise MCPMissingUserEnvVarsError( + server_id=server.server_id, + server_name=server.server_name or server.name, + missing=missing, + setup_url=build_env_var_setup_url(server.server_id), + ) + + # Only honor stored user values for currently user-scoped vars, and let + # admin globals win, so a stale row from when a var was user-scoped can + # never override the global value the admin set after switching it. + scoped_user_values = { + name: value for name, value in user_values.items() if name in user_var_names + } + merged_vars: Dict[str, str] = {**scoped_user_values, **global_values} + if not static_headers: + return static_headers + return interpolate_headers(static_headers, merged_vars) + + async def _load_user_env_vars( + self, + server: MCPServer, + user_api_key_auth: Optional[UserAPIKeyAuth], + *, + force_refresh: bool = False, + ) -> Dict[str, str]: + """Look up the calling user's env var values for ``server``. + + Returns an empty dict when no user is available. Results are cached in a + short-lived in-memory map keyed by (user_id, server_id) so the tool-call + and tool-listing paths avoid a DB round-trip per request within the TTL + window; the cache is invalidated when the user stores or clears values. + Pass ``force_refresh`` to bypass the cache read and re-fetch from the DB + (used before raising a "missing credentials" error so a process-local + stale entry cannot mask values stored on another worker). A missing DB + connection and any other DB error propagate so the caller can decide + between failing the request (tool-call path) and staying best-effort + (listing path); they must never be mistaken for "user has no values", + which would send the user a misleading "set up your credentials" 412. + """ + if user_api_key_auth is None: + return {} + user_id = getattr(user_api_key_auth, "user_id", None) + if not user_id: + return {} + + cache_key = (user_id, server.server_id) + if not force_refresh: + cached = _user_env_vars_cache.get(cache_key) + if cached is not None: + values, ts = cached + if time.monotonic() - ts < _USER_ENV_VARS_CACHE_TTL: + return values + + from litellm.proxy.proxy_server import prisma_client # noqa: PLC0415 + + if prisma_client is None: + raise RuntimeError( + "MCP per-user env vars require a database connection, but none " + "is configured. Connect a database to your proxy to use per-user " + "MCP env vars." + ) + from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 + get_user_env_vars, + ) + + values = await get_user_env_vars(prisma_client, user_id, server.server_id) + _write_user_env_vars_cache(user_id, server.server_id, values) + return values + async def _create_mcp_client( self, server: MCPServer, @@ -1394,6 +1856,7 @@ class MCPServerManager: extra_headers: Optional[Dict[str, str]] = None, stdio_env: Optional[Dict[str, str]] = None, subject_token: Optional[str] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> MCPClient: """ Create an MCPClient instance for the given server. @@ -1410,6 +1873,7 @@ class MCPServerManager: extra_headers: Additional headers to forward. stdio_env: Environment variables for stdio transport. subject_token: Optional user JWT for token exchange (OBO) flow. + user_api_key_auth: Optional auth context for sampling callbacks. Returns: Configured MCP client instance. @@ -1420,23 +1884,44 @@ class MCPServerManager: transport = server.transport or MCPTransport.sse + # Create sampling and elicitation callbacks for this client + sampling_cb = ( + _create_sampling_callback(user_api_key_auth=user_api_key_auth) + if server.allow_sampling + else None + ) + elicitation_cb = ( + _create_elicitation_callback() if server.allow_elicitation else None + ) + # Handle stdio transport if transport == MCPTransport.stdio: resolved_env = ( - stdio_env if stdio_env is not None else dict(server.env or {}) + stdio_env + if stdio_env is not None + else (dict(server.env) if server.env is not None else None) ) # Ensure npm-based STDIO MCP servers have a writable cache dir. # In containers the default (~/.npm or /app/.npm) may not exist # or be read-only, causing npx to fail with ENOENT. - if "NPM_CONFIG_CACHE" not in resolved_env: + if resolved_env is not None and "NPM_CONFIG_CACHE" not in resolved_env: resolved_env["NPM_CONFIG_CACHE"] = MCP_NPM_CACHE_DIR # Defense-in-depth: block commands not in the allowlist. # The Pydantic validator blocks new servers; this catches legacy # config/DB records predating the allowlist. if server.command: base_command = os.path.basename(server.command) - if base_command not in MCP_STDIO_ALLOWED_COMMANDS: + # Strip .exe/.cmd/.bat/.com suffix for Windows compatibility + base_command_no_ext = base_command.lower() + for ext in [".exe", ".cmd", ".bat", ".com"]: + if base_command.lower().endswith(ext): + base_command_no_ext = base_command[: -len(ext)].lower() + break + if ( + base_command.lower() not in MCP_STDIO_ALLOWED_COMMANDS + and base_command_no_ext not in MCP_STDIO_ALLOWED_COMMANDS + ): raise HTTPException( status_code=403, detail=f"MCP stdio command '{server.command}' is not in the allowlist ({sorted(MCP_STDIO_ALLOWED_COMMANDS)}). " @@ -1456,9 +1941,13 @@ class MCPServerManager: transport_type=transport, auth_type=server.auth_type, auth_value=auth_value, - timeout=MCP_CLIENT_TIMEOUT, + timeout=( + server.timeout if server.timeout is not None else MCP_CLIENT_TIMEOUT + ), stdio_config=stdio_config, extra_headers=extra_headers, + sampling_callback=sampling_cb, + elicitation_callback=elicitation_cb, ) else: # For HTTP/SSE transports @@ -1482,9 +1971,13 @@ class MCPServerManager: transport_type=transport, auth_type=server.auth_type, auth_value=auth_value, - timeout=MCP_CLIENT_TIMEOUT, + timeout=( + server.timeout if server.timeout is not None else MCP_CLIENT_TIMEOUT + ), extra_headers=extra_headers, aws_auth=aws_auth, + sampling_callback=sampling_cb, + elicitation_callback=elicitation_cb, ) async def _get_tools_from_server( @@ -1516,10 +2009,17 @@ class MCPServerManager: client = None try: - if server.static_headers: + # Tool *listing* must not be blocked by missing per-user env vars — + # the server's tools should still appear so the client connects. The + # friendly "missing vars" error is raised only on the tool-*call* + # path (see _call_regular_mcp_tool). + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server, user_api_key_auth, raise_on_missing=False + ) + if resolved_static_headers: if extra_headers is None: extra_headers = {} - extra_headers.update(server.static_headers) + extra_headers.update(resolved_static_headers) # MCPJWTSigner: inject signed JWT for tools/list (list path skips pre_call_hook). # Skip entirely when the signer is not configured (avoid an unnecessary @@ -1568,6 +2068,7 @@ class MCPServerManager: mcp_auth_header=mcp_auth_header, extra_headers=extra_headers, stdio_env=stdio_env, + user_api_key_auth=user_api_key_auth, ) ## HANDLE OPENAPI TOOLS @@ -1599,7 +2100,9 @@ class MCPServerManager: ] return tools else: - tools = await self._fetch_tools_with_timeout(client, server.name) + tools = await self._fetch_tools_with_timeout( + client, server.name, server=server + ) self._remember_upstream_initialize_instructions(server, client) prefixed_or_original_tools = self._create_prefixed_tools( @@ -1608,6 +2111,11 @@ class MCPServerManager: return prefixed_or_original_tools + except MCPUpstreamAuthError: + # Pass-through 401 must surface to single-server routes so the + # client triggers the upstream OAuth flow. The multi-server + # aggregator catches this explicitly to keep absorbing. + raise except Exception as e: verbose_logger.warning( f"Failed to get tools from server {server.name}: {str(e)}" @@ -2209,7 +2717,10 @@ class MCPServerManager: return None async def _fetch_tools_with_timeout( - self, client: MCPClient, server_name: str + self, + client: MCPClient, + server_name: str, + server: Optional[MCPServer] = None, ) -> List[MCPTool]: """ Fetch tools from MCP client with timeout and error handling. @@ -2217,16 +2728,28 @@ class MCPServerManager: Uses anyio.fail_after() instead of asyncio.wait_for() to avoid conflicts with the MCP SDK's anyio TaskGroup. See GitHub issue #20715 for details. + For pass-through MCP servers (``MCPServer.is_oauth_passthrough``) an + upstream HTTP 401 is converted into :class:`MCPUpstreamAuthError` + instead of being swallowed to an empty tool list. That lets the + single-server HTTP routes surface a proper 401 + ``WWW-Authenticate`` + challenge so standards-compliant MCP clients trigger the upstream + OAuth flow. Non-pass-through servers keep today's swallow-and-log + behaviour so the multi-server ``/mcp`` aggregator doesn't get + tainted by a single bad server. + Args: client: MCP client instance server_name: Name of the server for logging + server: Optional MCPServer; when pass-through, auth errors are + re-raised as :class:`MCPUpstreamAuthError`. Returns: List of tools from the server """ + is_passthrough = bool(server is not None and server.is_oauth_passthrough) try: with anyio.fail_after(MCP_TOOL_LISTING_TIMEOUT): - tools = await client.list_tools() + tools = await client.list_tools(raise_on_error=is_passthrough) verbose_logger.debug(f"Tools from {server_name}: {tools}") return tools except TimeoutError: @@ -2243,6 +2766,19 @@ class MCPServerManager: ) return [] except Exception as e: + if is_passthrough: + auth_info = _extract_upstream_auth_failure(e) + if auth_info is not None: + status_code, www_authenticate = auth_info + verbose_logger.info( + f"Upstream auth failure from pass-through MCP server " + f"{server_name}: HTTP {status_code}" + ) + raise MCPUpstreamAuthError( + status_code=status_code, + www_authenticate=www_authenticate, + server_name=server_name, + ) from e verbose_logger.warning(f"Error listing tools from {server_name}: {str(e)}") return [] @@ -2429,7 +2965,13 @@ class MCPServerManager: """ Check if the tool is allowed or banned for the given server """ - if server.allowed_tools: + from litellm.proxy._experimental.mcp_server.utils import ( + server_applies_tool_allowlist, + ) + + if server_applies_tool_allowlist(server): + if not server.allowed_tools: + return False return ( tool_name in server.allowed_tools or f"{server.name}-{tool_name}" in server.allowed_tools @@ -2644,6 +3186,9 @@ class MCPServerManager: "name": name, "arguments": arguments, "server_name": server_name, + "mcp_rate_limit_server_name": server.alias + or server.server_name + or server.name, "user_api_key_auth": user_api_key_auth, "user_api_key_user_id": ( getattr(user_api_key_auth, "user_id", None) @@ -2762,6 +3307,7 @@ class MCPServerManager: proxy_logging_obj: Optional[ProxyLogging], host_progress_callback: Optional[Callable] = None, hook_extra_headers: Optional[Dict[str, str]] = None, + user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> CallToolResult: """ Call a regular MCP tool using the MCP client. @@ -2827,23 +3373,33 @@ class MCPServerManager: normalized_raw_headers = { str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str) } + strip_caller_authorization = _should_strip_caller_authorization( + mcp_server=mcp_server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + for header in mcp_server.extra_headers: if not isinstance(header, str): continue - if ( - mcp_server.has_client_credentials - and header.lower() == "authorization" - ): + if header.lower() == "authorization" and strip_caller_authorization: continue header_value = normalized_raw_headers.get(header.lower()) if header_value is None: continue extra_headers[header] = header_value - if mcp_server.static_headers: + # Interpolate env vars into static_headers. Raises + # MCPMissingUserEnvVarsError when the calling user has not filled in + # a required per-user variable — the REST layer converts that into + # a friendly 412 with a setup URL. + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + mcp_server, user_api_key_auth + ) + if resolved_static_headers: if extra_headers is None: extra_headers = {} - extra_headers.update(mcp_server.static_headers) + extra_headers.update(resolved_static_headers) if hook_extra_headers: if extra_headers is None: @@ -2882,6 +3438,7 @@ class MCPServerManager: extra_headers=extra_headers, stdio_env=stdio_env, subject_token=subject_token, + user_api_key_auth=user_api_key_auth, ) call_tool_params = MCPCallToolRequestParams( @@ -2898,14 +3455,26 @@ class MCPServerManager: asyncio.create_task(_call_tool_via_client(client, call_tool_params)) ) + _timeout = ( + mcp_server.timeout if mcp_server.timeout is not None else MCP_CLIENT_TIMEOUT + ) try: - mcp_responses = await asyncio.gather(*tasks) + mcp_responses = await asyncio.wait_for( + asyncio.gather(*tasks), timeout=_timeout + ) + except asyncio.TimeoutError: + raise HTTPException( + status_code=504, + detail={ + "error": "timeout", + "message": f"MCP tool call timed out after {_timeout}s", + }, + ) except ( BlockedPiiEntityError, GuardrailRaisedException, HTTPException, ) as e: - # Re-raise guardrail exceptions to properly fail the MCP call verbose_logger.error( f"Guardrail blocked MCP tool call during result check: {str(e)}" ) @@ -3112,7 +3681,6 @@ class MCPServerManager: ) ) else: - # For regular MCP servers, use the MCP client return await self._call_regular_mcp_tool( mcp_server=mcp_server, original_tool_name=name, @@ -3125,6 +3693,7 @@ class MCPServerManager: proxy_logging_obj=proxy_logging_obj, host_progress_callback=host_progress_callback, hook_extra_headers=hook_result.get("extra_headers"), + user_api_key_auth=user_api_key_auth, ) return await self._gather_openapi_tool_tasks(tasks, proxy_logging_obj) @@ -3156,7 +3725,23 @@ class MCPServerManager: if server.needs_user_oauth_token: # Skip OAuth2 servers that rely on user-provided tokens continue - tools = await self._get_tools_from_server(server) + try: + tools = await self._get_tools_from_server(server) + except MCPUpstreamAuthError as e: + # Pass-through servers expect a user-supplied bearer token; + # at startup we have none, so an upstream 401 is normal. + # Swallow it so we keep mapping the remaining servers. + verbose_logger.debug( + f"Skipping tool name mapping for server {server.name} " + f"due to upstream auth error: {str(e)}" + ) + continue + except Exception as e: + verbose_logger.warning( + f"Failed to get tools from server {server.name} during " + f"tool name mapping initialization: {str(e)}" + ) + continue for tool in tools: # The tool.name here is already prefixed from _get_tools_from_server # Extract original name for mapping @@ -3272,7 +3857,13 @@ class MCPServerManager: verbose_logger.debug( f"Building server from DB: {server.server_id} ({server.server_name})" ) - new_server = await self.build_mcp_server_from_table(server) + # raw_rows come straight from the DB, so their global env var + # values (like credentials) are still encrypted here, unlike the + # already-decrypted records add_server/update_server are handed. + # Decrypt them while building the registry entry. + new_server = await self.build_mcp_server_from_table( + server, env_vars_are_encrypted=True + ) # Carry the cached short_prefix from the previous registry entry # (if any) so the prefix is stable across reloads. if existing_server is not None and existing_server.short_prefix: @@ -3375,15 +3966,37 @@ class MCPServerManager: def get_public_mcp_servers(self) -> List[MCPServer]: """ - Get the public MCP servers (available_on_public_internet=True flag on server). - Also includes servers from litellm.public_mcp_servers for backwards compat. + Return the MCP servers published to the AI Hub via /v1/mcp/make_public. + + Default (litellm.public_mcp_hub_strict_whitelist=True): mirrors + /public/model_hub and /public/agent_hub — gates strictly on the + litellm.public_mcp_servers whitelist. Returns an empty list when no + servers have been published. The per-server available_on_public_internet + flag is unrelated — it governs IP-based access in + _is_server_accessible_from_ip, not hub visibility. + + Legacy (litellm.public_mcp_hub_strict_whitelist=False): preserves the + pre-fix behavior where any server with available_on_public_internet=True + is also included. Intended as a one-release migration window for + deployments that relied on the OR-with-default semantics; will be + removed in a future release. """ - servers: List[MCPServer] = [] + if litellm.public_mcp_hub_strict_whitelist: + if litellm.public_mcp_servers is None: + return [] + public_ids = set(litellm.public_mcp_servers) + return [ + server + for server in self.get_registry().values() + if server.server_id in public_ids + ] + public_ids = set(litellm.public_mcp_servers or []) - for server in self.get_registry().values(): - if server.available_on_public_internet or server.server_id in public_ids: - servers.append(server) - return servers + return [ + server + for server in self.get_registry().values() + if server.available_on_public_internet or server.server_id in public_ids + ] def expand_permission_list(self, identifiers: List[str]) -> List[str]: """ @@ -3590,11 +4203,21 @@ class MCPServerManager: and not server.authentication_token ): should_skip_health_check = True + # Skip if static_headers reference a per-user env var: a userless probe + # can't fill ${NAME} and would forward the literal placeholder upstream, + # flipping the server to unhealthy even though real user calls succeed. + elif self._references_per_user_env_var(server): + should_skip_health_check = True if not should_skip_health_check: - extra_headers = {} - if server.static_headers: - extra_headers.update(server.static_headers) + resolved_static_headers = await self._resolve_static_headers_with_env_vars( + server=server, + user_api_key_auth=None, + raise_on_missing=False, + ) + extra_headers = ( + dict(resolved_static_headers) if resolved_static_headers else {} + ) client = await self._create_mcp_client( server=server, @@ -3644,6 +4267,7 @@ class MCPServerManager: extra_headers=server.extra_headers or [], mcp_info=server.mcp_info, static_headers=server.static_headers, + env_vars=self._env_vars_to_models(server.env_vars), status=status, last_health_check=datetime.now(), health_check_error=health_check_error, @@ -3655,6 +4279,7 @@ class MCPServerManager: registration_url=server.registration_url, allow_all_keys=server.allow_all_keys, instructions=server.instructions, + timeout=server.timeout, ) async def get_all_mcp_servers_with_health_and_teams( @@ -3716,6 +4341,14 @@ class MCPServerManager: return list_mcp_servers + @staticmethod + def _env_vars_to_models( + env_vars: Optional[List[Dict[str, Any]]], + ) -> Optional[List[MCPEnvVar]]: + if env_vars is None: + return None + return [MCPEnvVar.model_validate(env_var) for env_var in env_vars] + def _build_mcp_server_table(self, server: MCPServer) -> LiteLLM_MCPServerTable: return LiteLLM_MCPServerTable( server_id=server.server_id, @@ -3736,6 +4369,7 @@ class MCPServerManager: extra_headers=server.extra_headers or [], mcp_info=server.mcp_info, static_headers=server.static_headers, + env_vars=self._env_vars_to_models(server.env_vars), status=None, # No health check performed last_health_check=None, # No health check performed health_check_error=None, @@ -3748,11 +4382,13 @@ class MCPServerManager: allow_all_keys=server.allow_all_keys, available_on_public_internet=server.available_on_public_internet, delegate_auth_to_upstream=server.delegate_auth_to_upstream, + oauth_passthrough=getattr(server, "oauth_passthrough", False), is_byok=server.is_byok, byok_description=server.byok_description, byok_api_key_help_url=server.byok_api_key_help_url, source_url=server.source_url, instructions=server.instructions, + timeout=server.timeout, ) async def get_all_mcp_servers_unfiltered(self) -> List[LiteLLM_MCPServerTable]: diff --git a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py index cec5224e18..725f7a335b 100644 --- a/litellm/proxy/_experimental/mcp_server/rest_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/rest_endpoints.py @@ -1,3 +1,4 @@ +import asyncio import importlib from datetime import datetime from typing import ( @@ -13,13 +14,18 @@ from typing import ( Union, ) +import httpx from fastapi import APIRouter, Depends, HTTPException, Query, Request, status from litellm._logging import verbose_logger +from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.ui_session_utils import ( build_effective_auth_contexts, ) -from litellm.proxy._experimental.mcp_server.utils import merge_mcp_headers +from litellm.proxy._experimental.mcp_server.utils import ( + MCPMissingUserEnvVarsError, + merge_mcp_headers, +) from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.ip_address_utils import IPAddressUtils from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -40,12 +46,37 @@ router = APIRouter( tags=["mcp"], ) + +def _connection_error_message(exc: BaseException) -> str: + if isinstance(exc, httpx.LocalProtocolError): + return ( + "Failed to connect to MCP server: a request header is malformed. " + "Check static headers for leading/trailing spaces or illegal characters." + ) + if isinstance(exc, (httpx.ConnectError, httpx.ConnectTimeout)): + return ( + "Failed to connect to MCP server: the server is unreachable. " + "Check the URL and that the server is running." + ) + if isinstance(exc, httpx.TimeoutException): + return "Failed to connect to MCP server: the connection timed out." + if isinstance(exc, httpx.HTTPStatusError): + return ( + f"Failed to connect to MCP server: it returned HTTP " + f"{exc.response.status_code}." + ) + return "Failed to connect to MCP server. Check proxy logs for details." + + if MCP_AVAILABLE: from mcp.types import Tool as MCPTool from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( global_mcp_server_manager, ) + from litellm.proxy._experimental.mcp_server.oauth_utils import ( + get_request_base_url, + ) from litellm.proxy._experimental.mcp_server.server import ( ListMCPToolsRestAPIResponseObject, MCPServer, @@ -115,9 +146,10 @@ if MCP_AVAILABLE: try: from litellm.proxy._experimental.mcp_server.db import ( get_user_oauth_credential, - is_oauth_credential_expired, + resolve_valid_user_oauth_token, ) + prisma_client = None if prefetched_creds is not None: cred = prefetched_creds.get(server_id) else: @@ -129,13 +161,13 @@ if MCP_AVAILABLE: cred = await get_user_oauth_credential( prisma_client, user_id, server_id ) + cred = await resolve_valid_user_oauth_token( + user_id=user_id, + server=server, + cred=cred, + prisma_client=prisma_client, + ) if cred and cred.get("access_token"): - if is_oauth_credential_expired(cred): - verbose_logger.debug( - f"_get_user_oauth_extra_headers: token expired for " - f"user={user_id} server={server_id}" - ) - return None return {"Authorization": f"Bearer {cred['access_token']}"} except Exception as e: verbose_logger.warning( @@ -365,10 +397,9 @@ if MCP_AVAILABLE: user_api_key_auth=user_api_key_auth, ) - # Filter tools based on allowed_tools configuration - # Only filter if allowed_tools is explicitly configured (not None and not empty) - if server.allowed_tools is not None and len(server.allowed_tools) > 0: - tools = filter_tools_by_allowed_tools(tools, server) + # Always apply allowed_tools/disallowed_tools so the blacklist is + # enforced even when no allowlist is set (matches the SSE/HTTP path). + tools = filter_tools_by_allowed_tools(tools, server) # Filter tools based on user_api_key_auth.object_permission.mcp_tool_permissions # This provides per-key/team/org control over which tools can be accessed @@ -424,101 +455,6 @@ if MCP_AVAILABLE: allowed_mcp_servers.append(server) return allowed_mcp_servers - async def _list_tools_for_single_server( - server_id: str, - allowed_server_ids: List[str], - rest_client_ip: Optional[str], - mcp_server_auth_headers: dict, - mcp_auth_header: Optional[str], - raw_headers_from_request: dict, - user_api_key_dict: "UserAPIKeyAuth", - ) -> dict: - """ - Resolve and fetch tools for a single specified MCP server. - - Returns the full REST response dict (tools / error / message). - Raises HTTPException on access / IP-filter errors. - """ - # Resolve a server name to its UUID if needed - _name_resolved = None - if server_id not in allowed_server_ids: - _name_resolved = global_mcp_server_manager.get_mcp_server_by_name(server_id) - if _name_resolved is not None and _name_resolved.server_id in set( - allowed_server_ids - ): - server_id = _name_resolved.server_id - - if server_id not in allowed_server_ids: - _server = ( - global_mcp_server_manager.get_mcp_server_by_id(server_id) - or _name_resolved - ) - if ( - _server is not None - and rest_client_ip is not None - and not global_mcp_server_manager._is_server_accessible_from_ip( - _server, rest_client_ip - ) - ): - raise HTTPException( - status_code=403, - detail={ - "error": "ip_filtering", - "message": ( - f"MCP server '{server_id}' is not accessible from your IP address " - f"({rest_client_ip}). This server is restricted to internal " - "networks only. To make it externally accessible, set " - "'available_on_public_internet: true' in the server configuration." - ), - }, - ) - raise HTTPException( - status_code=403, - detail={ - "error": "access_denied", - "message": f"The key is not allowed to access server {server_id}", - }, - ) - - server = global_mcp_server_manager.get_mcp_server_by_id(server_id) - if server is None: - return { - "tools": [], - "error": "server_not_found", - "message": f"Server with id {server_id} not found", - } - - server_auth_header = _get_server_auth_header( - server, mcp_server_auth_headers, mcp_auth_header - ) - user_oauth_extra_headers = await _get_user_oauth_extra_headers( - server, user_api_key_dict - ) - - try: - tools = await _get_tools_for_single_server( - server, - server_auth_header, - raw_headers_from_request, - user_api_key_dict, - extra_headers=user_oauth_extra_headers, - ) - except Exception as e: - verbose_logger.exception(f"Error getting tools from {server.name}: {e}") - return { - "tools": [], - "error": "server_error", - "message": f"Failed to get tools from server {server.name}: {str(e)}", - } - - return { - "tools": tools, - "error": None, - "message": "Successfully retrieved tools", - } - - ######################################################## - async def _list_tools_for_single_server( server_id: str, allowed_server_ids: List[str], @@ -592,6 +528,11 @@ if MCP_AVAILABLE: user_api_key_dict, extra_headers=user_oauth_extra_headers, ) + except MCPUpstreamAuthError: + # Surface the upstream 401/403 to the caller so it can emit the + # matching status code and WWW-Authenticate challenge; that is what + # lets standards-compliant MCP clients run the upstream OAuth flow. + raise except Exception as e: verbose_logger.exception(f"Error getting tools from {server.name}: {e}") return { @@ -758,6 +699,24 @@ if MCP_AVAILABLE: ), } + except MCPUpstreamAuthError as e: + # Surface upstream pass-through 401/403 challenges to the client so + # standards-compliant MCP clients can run the upstream OAuth flow. + raise e.to_http_exception( + base_url=get_request_base_url(request), + request_path=request.scope.get("_original_path") or request.url.path, + ) + except HTTPException as http_exc: + # Internal access/IP 403s keep the legacy error-dict response shape + # so the existing contract stays intact. + verbose_logger.exception( + "HTTPException in list_tool_rest_api: %s", str(http_exc) + ) + return { + "tools": [], + "error": "unexpected_error", + "message": (f"An unexpected error occurred: {http_exc.detail}"), + } except Exception as e: verbose_logger.exception( "Unexpected error in list_tool_rest_api: %s", str(e) @@ -881,6 +840,23 @@ if MCP_AVAILABLE: requested_server_id=canonical_server_id, ) return result + except MCPMissingUserEnvVarsError as e: + verbose_logger.info( + "MCP tool call missing per-user env vars: server_id=%s missing=%s", + e.server_id, + e.missing, + ) + raise HTTPException( + status_code=412, + detail={ + "error": "missing_user_env_vars", + "message": str(e), + "server_id": e.server_id, + "server_name": e.server_name, + "missing": e.missing, + "setup_url": e.setup_url, + }, + ) except BlockedPiiEntityError as e: verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}") raise HTTPException( @@ -1030,14 +1006,14 @@ if MCP_AVAILABLE: return await operation(client) - except (KeyboardInterrupt, SystemExit): + except (KeyboardInterrupt, SystemExit, asyncio.CancelledError): raise except BaseException as e: verbose_logger.error("Error in MCP operation: %s", e, exc_info=True) return { "status": "error", "error": True, - "message": "Failed to connect to MCP server. Check proxy logs for details.", + "message": _connection_error_message(e), } async def _preview_openapi_tools(spec_path: str) -> dict: diff --git a/litellm/proxy/_experimental/mcp_server/sampling_handler.py b/litellm/proxy/_experimental/mcp_server/sampling_handler.py new file mode 100644 index 0000000000..1637c9eb0b --- /dev/null +++ b/litellm/proxy/_experimental/mcp_server/sampling_handler.py @@ -0,0 +1,1279 @@ +""" +MCP Sampling Handler +Handles `sampling/createMessage` requests from upstream MCP servers by +routing them through LiteLLM's internal completion infrastructure. +This allows MCP servers to perform agentic reasoning (e.g., multi-step +tool calling, chain-of-thought) without needing their own LLM API keys — +LiteLLM acts as the LLM provider using its existing 100+ provider support, +cost tracking, rate limiting, and model routing. +MCP Spec Reference: + https://modelcontextprotocol.io/specification/2025-11-25/client/sampling +""" + +from typing import Any, Dict, List, Optional, Union +import typing + +if typing.TYPE_CHECKING: + from litellm.proxy.utils import ProxyLogging + +from litellm._logging import verbose_logger + +from fastapi import HTTPException + +# Guard imports that require the mcp package +try: + from mcp.types import ( + CreateMessageRequestParams, + CreateMessageResult, + CreateMessageResultWithTools, + ErrorData, + ModelPreferences, + SamplingMessage, + TextContent, + Tool, + ToolChoice, + ToolUseContent, + ) + + MCP_SAMPLING_AVAILABLE = True +except ImportError as _sampling_import_err: + MCP_SAMPLING_AVAILABLE = False + verbose_logger.warning( + "MCP sampling disabled: failed to import required types from mcp.types — %s. " + "This usually means the 'mcp' package is not installed or is an older version " + "that does not support sampling. Install/upgrade with: pip install 'mcp>=1.1'", + _sampling_import_err, + ) + + +def _resolve_model_from_preferences( + model_preferences: Optional["ModelPreferences"], + default_model: Optional[str] = None, +) -> str: + """ + Resolve an LLM model name from MCP ModelPreferences. + Strategy: + 1. Check hints for substring matches against known model names. + 2. Fall back to priority-based selection (cost/speed/intelligence). + 3. Fall back to the configured default model. + Args: + model_preferences: MCP ModelPreferences with hints and priorities. + default_model: Fallback model if no hint matches. + Returns: + A model string suitable for litellm.acompletion(). + """ + import litellm + + # Build list of available model names from proxy Router or litellm.model_list + available_model_names: list = [] + try: + from litellm.proxy.proxy_server import llm_router + + if llm_router is not None: + available_model_names = llm_router.get_model_names() + except Exception: + pass + if not available_model_names and litellm.model_list: + for entry in litellm.model_list: + if isinstance(entry, dict): + name = entry.get("model_name") + if name: + available_model_names.append(name) + elif isinstance(entry, str): + available_model_names.append(entry) + if model_preferences and model_preferences.hints: + for hint in model_preferences.hints: + hint_name = getattr(hint, "name", None) + if not hint_name: + continue + # Try direct match first + if hint_name in available_model_names: + verbose_logger.debug( + "MCP sampling model resolution: direct hint match '%s'", + hint_name, + ) + return hint_name + # Try substring match against known models + for model_name in available_model_names: + if hint_name.lower() in model_name.lower(): + verbose_logger.debug( + "MCP sampling model resolution: substring hint match " + "'%s' -> '%s'", + hint_name, + model_name, + ) + return model_name + verbose_logger.debug( + "MCP sampling model resolution: no hint matched from %s " + "against %d available models", + [getattr(h, "name", None) for h in model_preferences.hints], + len(available_model_names), + ) + + # 2. Priority-based selection (cost/speed/intelligence) + if ( + model_preferences + and available_model_names + and _has_priorities(model_preferences) + ): + best = _select_model_by_priority(available_model_names, model_preferences) + if best is not None: + verbose_logger.debug( + "MCP sampling model resolution: priority-based selection chose '%s'", + best, + ) + return best + + # 3. Use default model from caller + if default_model: + verbose_logger.debug( + "MCP sampling model resolution: using caller-provided default '%s'", + default_model, + ) + return default_model + # Fall back to first available model + if available_model_names: + verbose_logger.debug( + "MCP sampling model resolution: no default configured, " + "falling back to first available model '%s'", + available_model_names[0], + ) + return available_model_names[0] + # Last resort - use LiteLLM default or raise error + default_sampling_model = getattr(litellm, "default_mcp_sampling_model", None) + if default_sampling_model: + verbose_logger.debug( + "MCP sampling model resolution: using litellm.default_mcp_sampling_model='%s'", + default_sampling_model, + ) + return default_sampling_model + raise ValueError( + "No model could be resolved for MCP sampling. Please configure 'default_mcp_sampling_model' in your LiteLLM configuration." + ) + + +def _has_priorities(model_preferences: "ModelPreferences") -> bool: + """Return True if any priority weight is set (non-None and > 0).""" + return any( + (getattr(model_preferences, attr, None) or 0) > 0 + for attr in ("costPriority", "speedPriority", "intelligencePriority") + ) + + +def _select_model_by_priority( + model_names: List[str], + model_preferences: "ModelPreferences", +) -> Optional[str]: + """Score available models by MCP priority weights and return the best. + + Scoring strategy (per the MCP spec, priorities are 0-1 floats): + + * **costPriority** — higher means "prefer cheaper models". + Metric: combined (input + output) cost per token from + ``model_prices_and_context_window.json``. Lower cost → higher score. + + * **speedPriority** — higher means "prefer faster models". + Metric: ``output_tokens_per_second`` from model info when available; + otherwise a neutral score for every candidate, since no reliable + latency proxy exists (context-window size does not track speed). + + * **intelligencePriority** — higher means "prefer smarter models". + Metric: ``max_output_tokens`` is used as a rough capability proxy + (frontier models expose larger context windows). + + Each metric is min-max normalised across the candidate set so that + every model gets a 0-1 score per dimension. The final score is the + weighted sum of the three normalised dimensions. + + Returns the highest-scoring model name, or None if scoring fails for + all candidates (e.g. no model_info available). + """ + import litellm as _litellm + + cost_weight = getattr(model_preferences, "costPriority", None) or 0.0 + speed_weight = getattr(model_preferences, "speedPriority", None) or 0.0 + intel_weight = getattr(model_preferences, "intelligencePriority", None) or 0.0 + + # Gather raw metrics for each model + scored: List[Dict[str, Any]] = [] + for name in model_names: + try: + info = _litellm.get_model_info(name) + except Exception: + continue + input_cost = info.get("input_cost_per_token") or 0.0 + output_cost = info.get("output_cost_per_token") or 0.0 + total_cost = input_cost + output_cost + max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0 + output_tps = info.get("output_tokens_per_second") or 0.0 + scored.append( + { + "name": name, + "cost": total_cost, + "max_output": max_output, + "output_tps": output_tps, + } + ) + + if not scored: + return None + + # Min-max normalisation helpers + def _normalise(values: List[float], invert: bool = False) -> List[float]: + """Normalise to [0, 1]. If *invert*, lower raw → higher score.""" + lo, hi = min(values), max(values) + if hi == lo: + return [0.5] * len(values) # all equal → neutral score + normed = [(v - lo) / (hi - lo) for v in values] + if invert: + normed = [1.0 - n for n in normed] + return normed + + costs = [s["cost"] for s in scored] + max_outputs = [float(s["max_output"]) for s in scored] + output_tps_values = [s["output_tps"] for s in scored] + + # costPriority: lower cost → higher score (invert) + cost_scores = _normalise(costs, invert=True) + # speedPriority: use output_tokens_per_second if any model has it, + # otherwise a neutral score (no reliable latency proxy is available). + if any(v > 0 for v in output_tps_values): + speed_scores = _normalise(output_tps_values, invert=False) + else: + speed_scores = [0.5] * len(scored) + # intelligencePriority: higher max_output → smarter + intel_scores = _normalise(max_outputs, invert=False) + + best_name = None + best_score = -1.0 + for i, entry in enumerate(scored): + score = ( + cost_weight * cost_scores[i] + + speed_weight * speed_scores[i] + + intel_weight * intel_scores[i] + ) + verbose_logger.debug( + "MCP priority scoring: model=%s cost_score=%.3f speed_score=%.3f " + "intel_score=%.3f → weighted=%.3f", + entry["name"], + cost_scores[i], + speed_scores[i], + intel_scores[i], + score, + ) + if score > best_score: + best_score = score + best_name = entry["name"] + + return best_name + + +def _convert_mcp_content_to_openai( + content: Any, +) -> Union[str, Dict[str, Any], List[Dict[str, Any]]]: + """ + Convert MCP SamplingMessage content to OpenAI message content format. + Handles: + - TextContent → string or {"type": "text", "text": ...} + - ImageContent → {"type": "image_url", "image_url": {"url": "data:..."}} + - AudioContent → {"type": "input_audio", "input_audio": {...}} + - ToolUseContent → function call representation + - ToolResultContent → tool result representation + - List of mixed content → list of content parts + """ + if isinstance(content, list): + parts = [] + for item in content: + converted = _convert_single_content(item) + if isinstance(converted, list): + parts.extend(converted) + else: + parts.append(converted) + return parts + return _convert_single_content(content) + + +def _convert_single_content( + content: Any, +) -> Union[Dict[str, Any], List[Dict[str, Any]]]: + """Convert a single MCP content item to OpenAI format. + + For text/image/audio content, returns a single content-part dict. + For tool_use/tool_result, returns a dict with a ``_marker_type`` key + so the caller (``_convert_mcp_messages_to_openai``) can hoist it to + the correct message-level position (``tool_calls`` array or a + separate ``role: "tool"`` message). + """ + import json + + content_type = getattr(content, "type", None) + if content_type == "text": + return {"type": "text", "text": content.text} + elif content_type == "image": + data = getattr(content, "data", "") + mime_type = getattr(content, "mimeType", "image/png") + return { + "type": "image_url", + "image_url": {"url": f"data:{mime_type};base64,{data}"}, + } + elif content_type == "audio": + data = getattr(content, "data", "") + mime_type = getattr(content, "mimeType", "audio/wav") + # Map MIME type to OpenAI audio format + format_map = { + "audio/wav": "wav", + "audio/mp3": "mp3", + "audio/mpeg": "mp3", + "audio/flac": "flac", + "audio/ogg": "ogg", + } + audio_format = format_map.get(mime_type, "wav") + return { + "type": "input_audio", + "input_audio": {"data": data, "format": audio_format}, + } + elif content_type == "tool_use": + # ToolUseContent → proper OpenAI function-call representation. + # The ``_marker_type`` key lets the message-level converter + # hoist this into the ``tool_calls`` array on the assistant + # message instead of embedding it inline as a content part. + return { + "_marker_type": "tool_use", + "id": getattr(content, "id", f"call_{id(content)}"), + "type": "function", + "function": { + "name": getattr(content, "name", ""), + "arguments": json.dumps(getattr(content, "input", {}), default=str), + }, + } + elif content_type == "tool_result": + # ToolResultContent → proper OpenAI tool-role message. + # Marked so the message-level converter can emit it as a + # separate ``{"role": "tool", ...}`` message. + tool_use_id = getattr(content, "toolUseId", "") + nested_content = getattr(content, "content", []) + if isinstance(nested_content, list): + text_parts = [ + getattr(c, "text", str(c)) + for c in nested_content + if getattr(c, "type", None) == "text" + ] + result_text = "\n".join(text_parts) if text_parts else "" + else: + result_text = str(nested_content) + return { + "_marker_type": "tool_result", + "role": "tool", + "tool_call_id": tool_use_id, + "content": result_text, + } + # Fallback: treat as text + return {"type": "text", "text": str(content)} + + +def _convert_mcp_messages_to_openai( + messages: List["SamplingMessage"], + system_prompt: Optional[str] = None, +) -> List[Dict[str, Any]]: + """ + Convert MCP SamplingMessage list to OpenAI messages format. + MCP messages use: + - role: "user" | "assistant" + - content: TextContent | ImageContent | AudioContent | ToolUseContent + | ToolResultContent | list[...] + OpenAI messages use: + - role: "system" | "user" | "assistant" | "tool" + - content: str | list[content_part] + """ + openai_messages: List[Dict[str, Any]] = [] + # Add system prompt if provided + if system_prompt: + openai_messages.append({"role": "system", "content": system_prompt}) + for msg in messages: + role = msg.role + content = msg.content + # Handle tool use content from assistant + if role == "assistant" and _has_tool_use(content): + tool_calls = _extract_tool_calls(content) + if tool_calls: + openai_msg: Dict[str, Any] = { + "role": "assistant", + "tool_calls": tool_calls, + } + # Also include any text content alongside tool calls + text_parts = _extract_text_parts(content) + if text_parts: + openai_msg["content"] = text_parts + openai_messages.append(openai_msg) + continue + # Handle tool result content from user + if role == "user" and _has_tool_result(content): + tool_results = _extract_tool_results(content) + for tool_result in tool_results: + openai_messages.append(tool_result) + continue + # Standard text/image/audio message — also handles any stray + # tool_use / tool_result that slipped past the fast-path checks + # above (e.g. unexpected role, single non-list content). + converted = _convert_mcp_content_to_openai(content) + converted_parts = ( + converted + if isinstance(converted, list) + else ([converted] if isinstance(converted, dict) else []) + ) + + # Separate marker items from regular content parts + tool_call_markers = [] + tool_result_markers = [] + regular_parts = [] + for part in converted_parts: + marker = part.get("_marker_type") if isinstance(part, dict) else None + if marker == "tool_use": + # Strip the internal marker before emitting + tc = {k: v for k, v in part.items() if k != "_marker_type"} + tool_call_markers.append(tc) + elif marker == "tool_result": + tr = {k: v for k, v in part.items() if k != "_marker_type"} + tool_result_markers.append(tr) + else: + regular_parts.append(part) + + # Emit assistant message with tool_calls if any were found + if tool_call_markers: + openai_msg_tc: Dict[str, Any] = { + "role": "assistant", + "tool_calls": tool_call_markers, + } + if regular_parts: + openai_msg_tc["content"] = regular_parts + openai_messages.append(openai_msg_tc) + elif regular_parts: + if isinstance(converted, str): + openai_messages.append({"role": role, "content": converted}) + else: + openai_messages.append({"role": role, "content": regular_parts}) + + # Emit separate tool-result messages + for tr in tool_result_markers: + openai_messages.append(tr) + + return openai_messages + + +def _has_tool_use(content: Any) -> bool: + """Check if content contains ToolUseContent.""" + if isinstance(content, list): + return any(getattr(c, "type", None) == "tool_use" for c in content) + return getattr(content, "type", None) == "tool_use" + + +def _has_tool_result(content: Any) -> bool: + """Check if content contains ToolResultContent.""" + if isinstance(content, list): + return any(getattr(c, "type", None) == "tool_result" for c in content) + return getattr(content, "type", None) == "tool_result" + + +def _extract_tool_calls(content: Any) -> List[Dict[str, Any]]: + """Extract OpenAI-format tool_calls from MCP ToolUseContent.""" + import json + + items = content if isinstance(content, list) else [content] + tool_calls = [] + for item in items: + if getattr(item, "type", None) == "tool_use": + tool_calls.append( + { + "id": getattr(item, "id", f"call_{id(item)}"), + "type": "function", + "function": { + "name": getattr(item, "name", ""), + "arguments": json.dumps( + getattr(item, "input", {}), default=str + ), + }, + } + ) + return tool_calls + + +def _extract_text_parts(content: Any) -> Optional[str]: + """Extract text parts from mixed content.""" + items = content if isinstance(content, list) else [content] + texts = [] + for item in items: + if getattr(item, "type", None) == "text": + texts.append(getattr(item, "text", "")) + return "\n".join(texts) if texts else None + + +def _extract_tool_results(content: Any) -> List[Dict[str, Any]]: + """Extract OpenAI-format tool messages from MCP ToolResultContent.""" + items = content if isinstance(content, list) else [content] + results = [] + for item in items: + if getattr(item, "type", None) == "tool_result": + tool_use_id = getattr(item, "toolUseId", "") + # Extract text from nested content + nested_content = getattr(item, "content", []) + if isinstance(nested_content, list): + text_parts = [ + getattr(c, "text", str(c)) + for c in nested_content + if getattr(c, "type", None) == "text" + ] + result_text = "\n".join(text_parts) if text_parts else "" + else: + result_text = str(nested_content) + results.append( + { + "role": "tool", + "tool_call_id": tool_use_id, + "content": result_text, + } + ) + return results + + +def _convert_mcp_tools_to_openai( + tools: Optional[List["Tool"]], +) -> Optional[List[Dict[str, Any]]]: + """ + Convert MCP Tool definitions to OpenAI function calling format. + MCP Tool: {name, description, inputSchema} + OpenAI Tool: {type: "function", function: {name, description, parameters}} + """ + if not tools: + return None + openai_tools = [] + for tool in tools: + openai_tool = { + "type": "function", + "function": { + "name": tool.name, + "description": tool.description or "", + "parameters": tool.inputSchema + or { + "type": "object", + "properties": {}, + }, + }, + } + openai_tools.append(openai_tool) + return openai_tools + + +def _convert_mcp_tool_choice_to_openai( + tool_choice: Optional["ToolChoice"], +) -> Optional[Union[str, Dict[str, Any]]]: + """ + Convert MCP ToolChoice to OpenAI tool_choice format. + MCP: {mode: "auto"} | {mode: "required"} | {mode: "none"} + OpenAI: "auto" | "required" | "none" + """ + if not tool_choice: + return None + mode = getattr(tool_choice, "mode", "auto") + if mode == "auto": + return "auto" + elif mode == "required": + return "required" + elif mode == "none": + return "none" + return "auto" + + +def _convert_openai_response_to_mcp_result( + response: Any, + model_name: str, +) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: + """ + Convert a litellm completion response to MCP CreateMessageResult. + Args: + response: The litellm ModelResponse. + model_name: The model that was used. + Returns: + MCP CreateMessageResult or CreateMessageResultWithTools. + """ + if not response.choices: + verbose_logger.warning( + "MCP sampling: LLM returned empty choices list for model=%s " + "(possible content filter or provider error)", + model_name, + ) + return ErrorData( + code=-1, + message=( + f"LLM returned no choices for model '{model_name}'. " + "This may indicate content filtering or a provider-side error." + ), + ) + choice = response.choices[0] + message = choice.message + # Determine stop reason + finish_reason = getattr(choice, "finish_reason", "stop") + if finish_reason == "tool_calls": + stop_reason = "toolUse" + elif finish_reason == "length": + stop_reason = "maxTokens" + else: + stop_reason = "endTurn" + actual_model = getattr(response, "model", model_name) or model_name + # Check if response has tool calls + tool_calls = getattr(message, "tool_calls", None) + if tool_calls: + # Build ToolUseContent items + content_parts: "List[Any]" = [] + # Include text content if present + if message.content: + content_parts.append(TextContent(type="text", text=message.content)) + # Convert tool calls to MCP ToolUseContent + for tc in tool_calls: + import json + + tool_input = tc.function.arguments + if isinstance(tool_input, str): + try: + tool_input = json.loads(tool_input) + except (json.JSONDecodeError, TypeError): + tool_input = {"raw": tool_input} + content_parts.append( + ToolUseContent( + type="tool_use", + id=tc.id, + name=tc.function.name, + input=tool_input, + ) + ) + return CreateMessageResultWithTools( + role="assistant", + content=content_parts, + model=actual_model, + stopReason=stop_reason, + ) + # Simple text response + text = message.content or "" + return CreateMessageResult( + role="assistant", + content=TextContent(type="text", text=text), + model=actual_model, + stopReason=stop_reason, + ) + + +async def _check_model_access( # noqa: PLR0915 + model: str, user_api_key_auth: Any +) -> Optional["ErrorData"]: + """Enforce model-permission checks for MCP sampling requests. + + Runs the same authorization checks as ``/chat/completions``: + key-level, team-level, per-member, user-level, and project-level + model restrictions. The model name comes from the upstream MCP + server (untrusted input). + + Returns None if authorized, or an ErrorData describing the denial. + """ + if user_api_key_auth is None: + return None + + _api_key = getattr(user_api_key_auth, "api_key", None) + _token = getattr(user_api_key_auth, "token", None) + _user_role = getattr(user_api_key_auth, "user_role", None) + + _has_real_credential = bool(_api_key) or bool(_token) + _is_admin = ( + _user_role in ("proxy_admin", "proxy_admin_viewer") if _user_role else False + ) + + if not _has_real_credential and not _is_admin: + verbose_logger.warning( + "MCP sampling: denying model access for model=%s — " + "auth context has no real LiteLLM credential (possible " + "OAuth passthrough placeholder). api_key=%s, token=%s, role=%s", + model, + bool(_api_key), + bool(_token), + _user_role, + ) + return ErrorData( + code=-1, + message=( + "Model access denied: sampling requires a valid LiteLLM " + "API key or admin credential. OAuth-only sessions cannot " + "trigger proxy model calls without explicit authorization." + ), + ) + + try: + import litellm + from litellm.proxy.auth.auth_checks import ( + can_key_call_model, + can_team_access_model, + can_user_call_model, + can_project_access_model, + _check_team_member_model_access, + get_team_object, + get_user_object, + get_project_object, + ) + + try: + from litellm.proxy.proxy_server import llm_router as _llm_router + except ImportError: + _llm_router = None + + await can_key_call_model( + model=model, + llm_model_list=getattr(litellm, "model_list", None), + valid_token=user_api_key_auth, + llm_router=_llm_router, + ) + + _team_id = getattr(user_api_key_auth, "team_id", None) + _user_id = getattr(user_api_key_auth, "user_id", None) + _project_id = getattr(user_api_key_auth, "project_id", None) + + try: + from litellm.proxy.proxy_server import ( + prisma_client as _prisma_client, + user_api_key_cache as _user_api_key_cache, + proxy_logging_obj as _proxy_logging_obj, + ) + except ImportError: + _prisma_client = None + _user_api_key_cache = None # type: ignore[assignment] + _proxy_logging_obj = None # type: ignore[assignment] + + if _team_id and _prisma_client and _user_api_key_cache: + try: + team_obj = await get_team_object( + team_id=_team_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + team_obj = None + + if team_obj: + await can_team_access_model( + model=model, + team_object=team_obj, + llm_router=_llm_router, + team_model_aliases=getattr( + user_api_key_auth, "team_model_aliases", None + ), + ) + if _user_id and _proxy_logging_obj: + await _check_team_member_model_access( + model=model, + team_object=team_obj, + valid_token=user_api_key_auth, + llm_router=_llm_router, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + elif not _team_id and _user_id and _prisma_client and _user_api_key_cache: + try: + user_obj = await get_user_object( + user_id=_user_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + user_obj = None + + if user_obj: + await can_user_call_model( + model=model, + llm_router=_llm_router, + user_object=user_obj, + ) + + if _project_id and _prisma_client and _user_api_key_cache: + try: + project_obj = await get_project_object( + project_id=_project_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + project_obj = None + + if project_obj: + can_project_access_model( + model=model, + project_object=project_obj, + llm_router=_llm_router, + ) + + verbose_logger.debug( + "MCP sampling: model access check passed for model=%s", + model, + ) + return None + except Exception as access_err: + verbose_logger.warning( + "MCP sampling: model access denied for model=%s: %s", + model, + access_err, + ) + return ErrorData( + code=-1, + message=( + f"Model access denied: the API key is not authorized " + f"to use model '{model}'. {access_err}" + ), + ) + + +async def _run_budget_checks( + model: str, + user_api_key_auth: Any, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Optional["ErrorData"]: + """Enforce key/team/user/org/global budget checks for sampling requests. + + Runs the same ``common_checks`` path that ``/chat/completions`` uses, + so sampling cannot bypass budget limits. + + Returns None if all checks pass, or an ErrorData describing the denial. + """ + try: + from litellm.proxy.auth.auth_checks import common_checks + from litellm.proxy.proxy_server import ( + general_settings, + llm_router as _llm_router, + prisma_client as _prisma_client, + proxy_logging_obj as _proxy_logging_obj, + user_api_key_cache as _user_api_key_cache, + ) + from litellm.proxy.auth.auth_checks import ( + get_team_object, + get_user_object, + ) + import litellm + except ImportError as import_err: + verbose_logger.warning( + "MCP sampling: budget check imports unavailable: %s", import_err + ) + return None # Can't enforce budgets without the modules + + _team_id = getattr(user_api_key_auth, "team_id", None) + _user_id = getattr(user_api_key_auth, "user_id", None) + + team_obj = None + if _team_id and _prisma_client and _user_api_key_cache: + try: + team_obj = await get_team_object( + team_id=_team_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + pass + + user_obj = None + if _user_id and _prisma_client and _user_api_key_cache: + try: + user_obj = await get_user_object( + user_id=_user_id, + prisma_client=_prisma_client, + user_api_key_cache=_user_api_key_cache, + user_id_upsert=False, + proxy_logging_obj=_proxy_logging_obj, + ) + except Exception: + pass + + dummy_request = _build_sampling_request( + raw_headers=raw_headers, + client_ip=client_ip, + ) + + # Enforce virtual-key route restrictions: a key limited to MCP routes + # must not be able to trigger a /chat/completions call via sampling. + # This mirrors the RouteChecks.should_call_route gate that runs in + # user_api_key_auth before common_checks for regular requests. + try: + from litellm.proxy.auth.route_checks import RouteChecks + + RouteChecks.should_call_route( + route="/chat/completions", + valid_token=user_api_key_auth, + request=dummy_request, + ) + except HTTPException as route_err: + verbose_logger.warning( + "MCP sampling: route check denied /chat/completions for key: %s", + route_err.detail, + ) + return ErrorData( + code=-1, + message=f"Sampling denied: virtual key is not allowed to call /chat/completions. {route_err.detail}", + ) + + global_proxy_spend = getattr(litellm, "_global_proxy_spend", None) + + # Build request body and merge x-litellm-tags from MCP headers BEFORE + # common_checks runs. _tag_max_budget_check inside common_checks only + # inspects request_body; without this pre-merge, header-supplied tags + # bypass per-tag budget enforcement (mirroring the regular auth path). + request_body: Dict[str, Any] = {"model": model} + try: + from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup + + LiteLLMProxyRequestSetup.apply_client_tag_policy_pre_auth( + request=dummy_request, + request_data=request_body, + user_api_key_dict=user_api_key_auth, + ) + except Exception: + # Non-fatal: tag merge is defense-in-depth; don't block sampling + # if the merge utility is unavailable or fails. + pass + + try: + await common_checks( + request_body=request_body, + team_object=team_obj, + user_object=user_obj, + end_user_object=None, + global_proxy_spend=global_proxy_spend, + general_settings=general_settings or {}, + route="/chat/completions", + llm_router=_llm_router, + proxy_logging_obj=typing.cast("ProxyLogging", _proxy_logging_obj), + valid_token=user_api_key_auth, + request=dummy_request, + ) + except Exception as budget_err: + verbose_logger.warning( + "MCP sampling: budget check failed for model=%s: %s", + model, + budget_err, + ) + return ErrorData( + code=-1, + message=f"Sampling denied: {budget_err}", + ) + + verbose_logger.debug("MCP sampling: budget checks passed for model=%s", model) + return None + + +def _build_sampling_request( + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Any: + """Build a synthetic FastAPI Request for sampling sub-calls. + + Converts the original MCP connection's HTTP headers into ASGI + scope format so that ``add_litellm_data_to_request`` can apply + header-dependent guardrails, tag-based routing, trace correlation, + and ``forward_llm_provider_auth_headers``. + + Key fields populated: + - **headers**: All original HTTP headers are forwarded (except + hop-by-hop: content-length, transfer-encoding). This ensures + ``traceparent``, ``authorization``, ``user-agent``, and + ``x-litellm-api-key`` are visible to pre-call utils. + - **client**: The ASGI ``(host, port)`` tuple so that + ``request.client.host`` returns the real client IP for + IP-based routing and guardrails. + - **server**: Derived from the running proxy's ``server_host`` + / ``server_port`` when available, avoiding the misleading + ``127.0.0.1:0`` placeholder. + - **x-forwarded-for**: Injected from ``client_ip`` if the + original headers don't already carry it, as a fallback for + IP attribution. + """ + from fastapi import Request + + # --- Build ASGI headers --- + _scope_headers: list = [(b"content-type", b"application/json")] + # Hop-by-hop headers that must NOT be forwarded into the + # synthetic request (they describe the original HTTP framing, + # not the logical request). + _HOP_BY_HOP = frozenset( + { + "content-length", + "transfer-encoding", + "connection", + "keep-alive", + "upgrade", + "te", + "trailer", + } + ) + if raw_headers: + for hdr_name, hdr_value in raw_headers.items(): + _key = hdr_name.lower() + # Skip content-type (already set), x-forwarded-for (use resolved + # client_ip instead to prevent spoofing), and hop-by-hop headers + if _key in {"content-type", "x-forwarded-for"} or _key in _HOP_BY_HOP: + continue + _scope_headers.append( + ( + _key.encode("latin-1", errors="replace"), + hdr_value.encode("utf-8"), + ) + ) + + # Inject x-forwarded-for from captured client_ip if the + # original headers don't already carry it + if client_ip and not any(h[0] == b"x-forwarded-for" for h in _scope_headers): + _scope_headers.append((b"x-forwarded-for", client_ip.encode("utf-8"))) + + # --- Derive server (host, port) from the running proxy --- + _server_host = "127.0.0.1" + _server_port = 4000 # LiteLLM default + try: + import litellm.proxy.proxy_server as proxy_server + + _proxy_host = getattr(proxy_server, "server_host", None) + _proxy_port = getattr(proxy_server, "server_port", None) + + if _proxy_host: + _server_host = str(_proxy_host) + if _proxy_port: + _server_port = int(_proxy_port) + except (ImportError, AttributeError, TypeError, ValueError): + pass + + # --- Build ASGI client tuple for request.client.host --- + _client_tuple = None + if client_ip: + _client_tuple = (client_ip, 0) + + scope: Dict[str, Any] = { + "type": "http", + "method": "POST", + "path": "/mcp/sampling/createMessage", + "scheme": "http", + "server": (_server_host, _server_port), + "query_string": b"", + "root_path": "", + "headers": _scope_headers, + } + if _client_tuple is not None: + scope["client"] = _client_tuple + + return Request(scope=scope) + + +async def _build_completion_kwargs( + params: "CreateMessageRequestParams", + model: str, + user_api_key_auth: Any, + raw_headers: Optional[Dict[str, str]], + client_ip: Optional[str], +) -> Dict[str, Any]: + openai_messages = _convert_mcp_messages_to_openai( + messages=params.messages, + system_prompt=params.systemPrompt, + ) + completion_kwargs: Dict[str, Any] = { + "model": model, + "messages": openai_messages, + "max_tokens": params.maxTokens, + } + if params.temperature is not None: + completion_kwargs["temperature"] = params.temperature + if params.stopSequences: + completion_kwargs["stop"] = params.stopSequences + openai_tools = _convert_mcp_tools_to_openai(params.tools) + if openai_tools: + completion_kwargs["tools"] = openai_tools + openai_tool_choice = _convert_mcp_tool_choice_to_openai(params.toolChoice) + if openai_tool_choice is not None: + completion_kwargs["tool_choice"] = openai_tool_choice + completion_kwargs["metadata"] = {} + if params.metadata: + completion_kwargs["metadata"]["mcp_metadata"] = params.metadata + + from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request + from litellm.proxy.proxy_server import proxy_config + + completion_kwargs["user"] = getattr(user_api_key_auth, "user_id", None) + _dummy_request = _build_sampling_request( + raw_headers=raw_headers, client_ip=client_ip + ) + completion_kwargs = await add_litellm_data_to_request( + data=completion_kwargs, + request=_dummy_request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, + ) + return completion_kwargs + + +async def _run_guardrails_and_call_llm( + completion_kwargs: Dict[str, Any], + user_api_key_auth: Any, +) -> Any: + try: + from litellm.proxy.proxy_server import proxy_logging_obj as _plo + + if _plo is not None: + completion_kwargs = await typing.cast("ProxyLogging", _plo).pre_call_hook( + user_api_key_dict=user_api_key_auth, + data=completion_kwargs, + call_type="acompletion", + ) + except ImportError: + pass + except Exception as guardrail_err: + verbose_logger.warning( + "MCP sampling: pre-call guardrail rejected request: %s", + guardrail_err, + ) + raise + + import litellm + + try: + from litellm.proxy.proxy_server import llm_router + + if llm_router is not None: + return await llm_router.acompletion(**completion_kwargs) + return await litellm.acompletion(**completion_kwargs) + except ImportError: + return await litellm.acompletion(**completion_kwargs) + + +async def handle_sampling_create_message( + context: Any, + params: "CreateMessageRequestParams", + default_model: Optional[str] = None, + user_api_key_auth: Optional[Any] = None, + raw_headers: Optional[Dict[str, str]] = None, + client_ip: Optional[str] = None, +) -> Union["CreateMessageResult", "CreateMessageResultWithTools", "ErrorData"]: + """ + Handle an MCP sampling/createMessage request by routing through LiteLLM. + This is the main entry point called by the MCP client session when an + upstream MCP server requests LLM inference. + Args: + context: MCP RequestContext (contains session info). + params: The CreateMessageRequestParams from the MCP server. + default_model: Default model to use if no preferences match. + user_api_key_auth: Auth context for the requesting user. + raw_headers: Original HTTP headers from the MCP connection. + Forwarded into the internal acompletion call so that + header-dependent guardrails, IP-routing, trace-id + correlation, and forward_llm_provider_auth_headers + work correctly for sampling sub-calls. + client_ip: Original client IP address for IP-based guardrails. + Returns: + CreateMessageResult with the LLM's response, or ErrorData on failure. + """ + if not MCP_SAMPLING_AVAILABLE: + return ErrorData( + code=-1, + message="MCP sampling is not available (mcp package not installed)", + ) + + if user_api_key_auth is None: + return ErrorData( + code=-1, + message=( + "Sampling requires an authenticated user context. " + "Internal or unauthenticated sessions cannot trigger " + "upstream-initiated model calls." + ), + ) + + try: + model = _resolve_model_from_preferences( + model_preferences=params.modelPreferences, + default_model=default_model, + ) + verbose_logger.info( + "MCP sampling: resolved model=%s from preferences=%s", + model, + params.modelPreferences, + ) + + access_denial = await _check_model_access(model, user_api_key_auth) + if access_denial is not None: + return access_denial + + budget_denial = await _run_budget_checks( + model=model, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + if budget_denial is not None: + return budget_denial + + completion_kwargs = await _build_completion_kwargs( + params=params, + model=model, + user_api_key_auth=user_api_key_auth, + raw_headers=raw_headers, + client_ip=client_ip, + ) + + openai_messages = completion_kwargs["messages"] + openai_tools = completion_kwargs.get("tools") + verbose_logger.debug( + "MCP sampling: calling litellm.acompletion with model=%s, num_messages=%d, has_tools=%s", + model, + len(openai_messages), + bool(openai_tools), + ) + + response = await _run_guardrails_and_call_llm( + completion_kwargs=completion_kwargs, + user_api_key_auth=user_api_key_auth, + ) + + result = _convert_openai_response_to_mcp_result( + response=response, model_name=model + ) + verbose_logger.info( + "MCP sampling: completed successfully, model=%s, stopReason=%s", + getattr(result, "model", "unknown"), + getattr(result, "stopReason", "unknown"), + ) + return result + except Exception as e: + from litellm.exceptions import ( + AuthenticationError, + BudgetExceededError, + ContextWindowExceededError, + PermissionDeniedError, + RateLimitError, + ServiceUnavailableError, + ) + + from litellm.proxy._types import ProxyException + + if isinstance( + e, + ( + HTTPException, + BudgetExceededError, + RateLimitError, + AuthenticationError, + PermissionDeniedError, + ContextWindowExceededError, + ServiceUnavailableError, + ProxyException, + ), + ): + raise + + verbose_logger.exception("MCP sampling handler failed: %s", e) + return ErrorData( + code=-1, + message=f"Sampling failed: {str(e)}", + ) diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index a05ce3f741..0477a5d324 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -6,6 +6,7 @@ LiteLLM MCP Server Routes import asyncio import contextlib +import contextvars import hashlib import json import time @@ -20,6 +21,7 @@ from typing import ( Dict, List, Optional, + Set, Tuple, Union, cast, @@ -38,6 +40,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.proxy._experimental.mcp_server.auth.user_api_key_auth_mcp import ( MCPRequestHandler, ) +from litellm.proxy._experimental.mcp_server.exceptions import MCPUpstreamAuthError from litellm.proxy._experimental.mcp_server.discoverable_endpoints import ( get_request_base_url, ) @@ -50,6 +53,7 @@ from litellm.proxy._experimental.mcp_server.utils import ( LITELLM_MCP_SERVER_DESCRIPTION, LITELLM_MCP_SERVER_NAME, LITELLM_MCP_SERVER_VERSION, + MCPMissingUserEnvVarsError, add_server_prefix_to_name, get_server_prefix, iter_known_server_prefixes, @@ -123,6 +127,18 @@ try: GetPromptResult, ResourceTemplate, TextResourceContents, + Tool, + ) + from mcp.server.session import ServerSession as _McpServerSession + import weakref + + # Robust auth lookup keyed by session_object. + _session_obj_auth_storage: ( + "weakref.WeakKeyDictionary[Any, MCPAuthenticatedUser]" + ) = weakref.WeakKeyDictionary() + + active_mcp_session_var: contextvars.ContextVar[Optional[_McpServerSession]] = ( + contextvars.ContextVar("active_mcp_session", default=None) ) except ImportError as e: verbose_logger.debug(f"MCP module not found: {e}") @@ -145,6 +161,73 @@ _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK = asyncio.Lock() +def _mcp_session_id_from_headers( + raw_headers: Optional[Dict[str, str]], +) -> Optional[str]: + """The ``mcp-session-id`` of a stateful MCP session, read case-insensitively + from the request headers. ``None`` for stateless calls (no such header).""" + if not raw_headers: + return None + for key, value in raw_headers.items(): + if isinstance(key, str) and key.lower() == "mcp-session-id": + return value or None + return None + + +def _jsonrpc_text_has_top_level_method(text: str) -> bool: + """Whether a (possibly truncated) JSON-RPC envelope has a ``method`` key at + the root object's top level. + + Used to tell a request/notification (carries ``method``) apart from a + response (carries ``result``/``error`` and no top-level ``method``). A + response payload can itself nest a ``method`` field, so only keys at the + root object's depth are inspected rather than searching the whole string. + Returns ``True`` only when a top-level ``method`` key is positively found; + truncation that hides it yields ``False``. + """ + depth = 0 + in_string = False + escaped = False + in_object: List[bool] = [] + reading_key = False + expect_key = False + key_chars: List[str] = [] + for ch in text: + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == '"': + in_string = False + if reading_key and depth == 1 and "".join(key_chars) == "method": + return True + elif reading_key: + key_chars.append(ch) + continue + if ch == '"': + in_string = True + reading_key = expect_key and depth >= 1 and in_object[-1] + key_chars = [] + expect_key = False + elif ch == "{" or ch == "[": + depth += 1 + in_object.append(ch == "{") + expect_key = ch == "{" + elif ch == "}" or ch == "]": + if in_object: + in_object.pop() + depth -= 1 + if depth <= 0: + break + expect_key = False + elif ch == ",": + expect_key = bool(in_object) and in_object[-1] + elif ch == ":": + expect_key = False + return False + + if MCP_AVAILABLE: from mcp.server import Server from mcp.server.lowlevel.server import NotificationOptions @@ -174,6 +257,7 @@ if MCP_AVAILABLE: ) from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( MCPServerManager, + _should_strip_caller_authorization, global_mcp_server_manager, ) from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import ( @@ -467,10 +551,18 @@ if MCP_AVAILABLE: ######################################################## @server.list_tools() - async def list_tools() -> List[MCPTool]: + async def handle_list_tools() -> List[Tool]: """ - List all available tools + List all available tools. + Also captures the active session for propagation to callbacks. """ + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: # Get user authentication from context variable ( @@ -481,7 +573,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_tools - User API Key Auth from context: {user_api_key_auth}" ) @@ -512,152 +604,188 @@ if MCP_AVAILABLE: # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.call_tool() - async def mcp_server_tool_call( - name: str, arguments: Optional[Dict[str, Any]] + async def mcp_server_tool_call( # noqa: PLR0915 + name: str, arguments: Dict[str, Any] | None ) -> CallToolResult: """ Call a specific tool with the provided arguments - Args: name (str): Name of the tool to call arguments (Dict[str, Any] | None): Arguments to pass to the tool - Returns: List[Union[MCPTextContent, MCPImageContent, MCPEmbeddedResource]]: Tool execution results - Raises: HTTPException: If tool not found or arguments missing """ from fastapi import Request - from litellm.exceptions import BlockedPiiEntityError, GuardrailRaisedException from litellm.proxy.litellm_pre_call_utils import add_litellm_data_to_request from litellm.proxy.proxy_server import proxy_config + from mcp.types import CallToolResult + from mcp.server.lowlevel.server import request_ctx - # Validate arguments - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - verbose_logger.debug( - f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" - ) - host_progress_callback = None try: - host_ctx = server.request_context - if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta: - host_token = getattr(host_ctx.meta, "progressToken", None) - if host_token and hasattr(host_ctx, "session") and host_ctx.session: - host_session = host_ctx.session - - async def forward_progress(progress: float, total: Optional[float]): - """Forward progress notifications from external MCP to Host""" - try: - await host_session.send_progress_notification( - progress_token=host_token, - progress=progress, - total=total, - ) - verbose_logger.debug( - f"Forwarded progress {progress}/{total} to Host" - ) - except Exception as e: - verbose_logger.error( - f"Failed to forward progress to Host: {e}" - ) - - host_progress_callback = forward_progress - verbose_logger.debug( - f"Host progressToken captured: {host_token[:8]}..." - ) - except Exception as e: - verbose_logger.warning(f"Could not capture host progress context: {e}") - try: - # Create a body date for logging - body_data = {"name": name, "arguments": arguments} - # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) - chain_id = get_chain_id_from_headers(raw_headers) - if chain_id: - body_data["litellm_trace_id"] = chain_id - body_data["litellm_session_id"] = chain_id - - request = Request( - scope={ - "type": "http", - "method": "POST", - "path": "/mcp/tools/call", - "headers": [(b"content-type", b"application/json")], - } + # Validate arguments + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() + verbose_logger.debug( + f"MCP mcp_server_tool_call - user_api_key_auth={user_api_key_auth}, user_role={getattr(user_api_key_auth, 'user_role', 'N/A')}" ) - if user_api_key_auth is not None: - data = await add_litellm_data_to_request( - data=body_data, - request=request, - user_api_key_dict=user_api_key_auth, - proxy_config=proxy_config, + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + host_progress_callback = None + try: + host_ctx = server.request_context + if host_ctx and hasattr(host_ctx, "meta") and host_ctx.meta: + host_token = getattr(host_ctx.meta, "progressToken", None) + if host_token and hasattr(host_ctx, "session") and host_ctx.session: + host_session = host_ctx.session + + async def forward_progress( + progress: float, total: Optional[float] + ): + """Forward progress notifications from external MCP to Host""" + try: + await host_session.send_progress_notification( + progress_token=host_token, + progress=progress, + total=total, + ) + verbose_logger.debug( + f"Forwarded progress {progress}/{total} to Host" + ) + except Exception as e: + verbose_logger.error( + f"Failed to forward progress to Host: {e}" + ) + + host_progress_callback = forward_progress + verbose_logger.debug( + f"Host progressToken captured: {host_token[:8]}..." + ) + except Exception as e: + verbose_logger.warning(f"Could not capture host progress context: {e}") + try: + # Create a body date for logging + body_data = {"name": name, "arguments": arguments} + # Set trace/session id from raw_headers so spend logs and logging_obj stay consistent (same as A2A) + chain_id = get_chain_id_from_headers(raw_headers) + if chain_id: + body_data["litellm_trace_id"] = chain_id + body_data["litellm_session_id"] = chain_id + + request = Request( + scope={ + "type": "http", + "method": "POST", + "path": "/mcp/tools/call", + "headers": [(b"content-type", b"application/json")], + } ) - else: - data = body_data - - response = await call_mcp_tool( - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - host_progress_callback=host_progress_callback, - **data, # for logging - ) - except BlockedPiiEntityError as e: - verbose_logger.error(f"BlockedPiiEntityError in MCP tool call: {str(e)}") - return CallToolResult( - content=[ - TextContent( - text=f"Error: Blocked PII entity detected - {str(e)}", - type="text", + if user_api_key_auth is not None: + data = await add_litellm_data_to_request( + data=body_data, + request=request, + user_api_key_dict=user_api_key_auth, + proxy_config=proxy_config, ) - ], - isError=True, - ) - except GuardrailRaisedException as e: - verbose_logger.error(f"GuardrailRaisedException in MCP tool call: {str(e)}") - return CallToolResult( - content=[ - TextContent( - text=f"Error: Guardrail violation - {str(e)}", type="text" - ) - ], - isError=True, - ) - except HTTPException as e: - verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}") - return CallToolResult( - content=[TextContent(text=f"Error: {str(e.detail)}", type="text")], - isError=True, - ) - except Exception as e: - verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") - return CallToolResult( - content=[TextContent(text=f"Error: {str(e)}", type="text")], - isError=True, - ) + else: + data = body_data - return response + response = await call_mcp_tool( + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + host_progress_callback=host_progress_callback, + **data, # for logging + ) + except MCPMissingUserEnvVarsError as e: + verbose_logger.info( + "MCP mcp_server_tool_call missing per-user env vars: server_id=%s missing=%s", + e.server_id, + e.missing, + ) + return CallToolResult( + content=[TextContent(text=str(e), type="text")], + isError=True, + ) + except BlockedPiiEntityError as e: + verbose_logger.error( + f"BlockedPiiEntityError in MCP tool call: {str(e)}" + ) + return CallToolResult( + content=[ + TextContent( + text=f"Error: Blocked PII entity detected - {str(e)}", + type="text", + ) + ], + isError=True, + ) + except GuardrailRaisedException as e: + verbose_logger.error( + f"GuardrailRaisedException in MCP tool call: {str(e)}" + ) + return CallToolResult( + content=[ + TextContent( + text=f"Error: Guardrail violation - {str(e)}", type="text" + ) + ], + isError=True, + ) + except HTTPException as e: + verbose_logger.error(f"HTTPException in MCP tool call: {str(e)}") + return CallToolResult( + content=[TextContent(text=f"Error: {str(e.detail)}", type="text")], + isError=True, + ) + except Exception as e: + verbose_logger.exception(f"MCP mcp_server_tool_call - error: {e}") + return CallToolResult( + content=[TextContent(text=f"Error: {str(e)}", type="text")], + isError=True, + ) + + return response + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.list_prompts() async def list_prompts() -> List[Prompt]: """ List all available prompts """ + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: # Get user authentication from context variable ( @@ -668,7 +796,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_prompts - User API Key Auth from context: {user_api_key_auth}" ) @@ -697,6 +825,9 @@ if MCP_AVAILABLE: # Return empty list instead of failing completely # This prevents the HTTP stream from failing and allows the client to get a response return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.get_prompt() async def get_prompt( @@ -714,33 +845,13 @@ if MCP_AVAILABLE: """ # Validate arguments - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + from mcp.server.lowlevel.server import request_ctx - verbose_logger.debug( - f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" - ) - return await mcp_get_prompt( - name=name, - arguments=arguments, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - @server.list_resources() - async def list_resources() -> List[Resource]: - """List all available resources.""" try: ( user_api_key_auth, @@ -750,7 +861,45 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() + + verbose_logger.debug( + f"MCP mcp_server_tool_call - User API Key Auth from context: {user_api_key_auth}" + ) + return await mcp_get_prompt( + name=name, + arguments=arguments, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) + + @server.list_resources() + async def list_resources() -> List[Resource]: + """List all available resources.""" + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_resources - User API Key Auth from context: {user_api_key_auth}" ) @@ -776,10 +925,20 @@ if MCP_AVAILABLE: except Exception as e: verbose_logger.exception(f"Error in list_resources endpoint: {str(e)}") return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.list_resource_templates() async def list_resource_templates() -> List[ResourceTemplate]: """List all available resource templates.""" + from mcp.server.lowlevel.server import request_ctx + + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) + try: ( user_api_key_auth, @@ -789,7 +948,7 @@ if MCP_AVAILABLE: oauth2_headers, raw_headers, _client_ip, - ) = get_auth_context() + ) = await get_or_extract_auth_context() verbose_logger.debug( f"MCP list_resource_templates - User API Key Auth from context: {user_api_key_auth}" ) @@ -809,8 +968,7 @@ if MCP_AVAILABLE: raw_headers=raw_headers, ) verbose_logger.info( - "MCP list_resource_templates - Successfully returned " - f"{len(resource_templates)} resource templates" + f"MCP list_resource_templates - Successfully returned {len(resource_templates)} resource templates" ) return resource_templates except Exception as e: @@ -818,30 +976,44 @@ if MCP_AVAILABLE: f"Error in list_resource_templates endpoint: {str(e)}" ) return [] + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) @server.read_resource() async def read_resource(url: AnyUrl) -> list[ReadResourceContents]: - ( - user_api_key_auth, - mcp_auth_header, - mcp_servers, - mcp_server_auth_headers, - oauth2_headers, - raw_headers, - _client_ip, - ) = get_auth_context() + from mcp.server.lowlevel.server import request_ctx - read_resource_result = await mcp_read_resource( - url=url, - user_api_key_auth=user_api_key_auth, - mcp_auth_header=mcp_auth_header, - mcp_servers=mcp_servers, - mcp_server_auth_headers=mcp_server_auth_headers, - oauth2_headers=oauth2_headers, - raw_headers=raw_headers, - ) + req_ctx = request_ctx.get(None) + _session_reset_token = None + if req_ctx: + _session_reset_token = active_mcp_session_var.set(req_ctx.session) - return _normalize_resource_contents(read_resource_result.contents) + try: + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = await get_or_extract_auth_context() + + read_resource_result = await mcp_read_resource( + url=url, + user_api_key_auth=user_api_key_auth, + mcp_auth_header=mcp_auth_header, + mcp_servers=mcp_servers, + mcp_server_auth_headers=mcp_server_auth_headers, + oauth2_headers=oauth2_headers, + raw_headers=raw_headers, + ) + + return _normalize_resource_contents(read_resource_result.contents) + finally: + if _session_reset_token is not None: + active_mcp_session_var.reset(_session_reset_token) ######################################################## ############ End of MCP Server Routes ################## @@ -945,10 +1117,16 @@ if MCP_AVAILABLE: Returns: Filtered list of tools """ + from litellm.proxy._experimental.mcp_server.utils import ( + server_applies_tool_allowlist, + ) + tools_to_return = tools # Filter by allowed_tools (whitelist) - if mcp_server.allowed_tools: + if server_applies_tool_allowlist(mcp_server): + if not mcp_server.allowed_tools: + return [] tools_to_return = [ tool for tool in tools @@ -1080,6 +1258,42 @@ if MCP_AVAILABLE: return allowed_mcp_servers + def _client_has_passthrough_authorization( + server: MCPServer, + oauth2_headers: Optional[Dict[str, str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + ) -> bool: + """True if the incoming request already carries an ``Authorization`` + header the gateway will forward to this pass-through server. + + The client may supply the bearer as either the top-level + ``Authorization`` header (surfaced via ``oauth2_headers``) or a + per-server ``x-mcp-auth-`` style header (surfaced via + ``mcp_server_auth_headers``). Either form skips the pre-emptive 401. + """ + if oauth2_headers: + for k in oauth2_headers.keys(): + if k.lower() == "authorization": + return True + if mcp_server_auth_headers: + for key in (server.alias, server.server_name, server.name): + if not key: + continue + server_headers = None + for k, v in mcp_server_auth_headers.items(): + if k.lower() == key.lower(): + server_headers = v + break + if server_headers is None: + continue + if isinstance(server_headers, str) and server_headers.strip(): + return True + if isinstance(server_headers, dict): + for hk in server_headers.keys(): + if hk.lower() == "authorization": + return True + return False + async def _get_user_oauth_extra_headers_from_db( server: MCPServer, user_api_key_auth: Optional[UserAPIKeyAuth], @@ -1108,8 +1322,7 @@ if MCP_AVAILABLE: try: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 get_user_oauth_credential, - is_oauth_credential_expired, - refresh_user_oauth_token, + resolve_valid_user_oauth_token, ) from litellm.proxy._experimental.mcp_server.oauth2_token_cache import ( # noqa: PLC0415 _compute_per_user_token_ttl, @@ -1122,14 +1335,14 @@ if MCP_AVAILABLE: cached_token = await mcp_per_user_token_cache.get(user_id, server_id) if cached_token is not None: verbose_logger.debug( - "_get_user_oauth_extra_headers_from_db: Redis hit for " - "user=%s server=%s", + "_get_user_oauth_extra_headers_from_db: Redis hit for user=%s server=%s", user_id, server_id, ) return {"Authorization": f"Bearer {cached_token}"} # ── Slow path: DB lookup ────────────────────────────────────────── + prisma_client = None if prefetched_creds is not None: cred = prefetched_creds.get(server_id) else: @@ -1147,45 +1360,17 @@ if MCP_AVAILABLE: if not cred or not cred.get("access_token"): return None - if is_oauth_credential_expired(cred): - verbose_logger.debug( - "_get_user_oauth_extra_headers_from_db: token expired for " - "user=%s server=%s — attempting refresh", - user_id, - server_id, - ) - # Attempt token refresh; requires a DB client (not available from prefetch) - if cred.get("refresh_token"): - try: - from litellm.proxy.utils import ( # noqa: PLC0415 - get_prisma_client_or_throw, - ) - - prisma_client = get_prisma_client_or_throw( - "Database not connected. Cannot refresh OAuth token." - ) - cred = await refresh_user_oauth_token( - prisma_client=prisma_client, - user_id=user_id, - server=server, - cred=cred, - ) - except Exception as refresh_exc: - verbose_logger.warning( - "_get_user_oauth_extra_headers_from_db: refresh failed " - "for user=%s server=%s: %s", - user_id, - server_id, - refresh_exc, - ) - cred = None - - if not cred or not cred.get("access_token"): - # Clear stale Redis/cache entry so we don't serve it again. - # Do this for both the individual and prefetch paths so the - # next request doesn't get a stale cache hit. - await mcp_per_user_token_cache.delete(user_id, server_id) - return None + cred = await resolve_valid_user_oauth_token( + user_id=user_id, + server=server, + cred=cred, + prisma_client=prisma_client, + ) + if cred is None: + # Refresh failed or token expired with no usable refresh_token — + # clear the stale Redis entry so the next request doesn't reuse it. + await mcp_per_user_token_cache.delete(user_id, server_id) + return None access_token: str = cred["access_token"] @@ -1217,8 +1402,7 @@ if MCP_AVAILABLE: return {"Authorization": f"Bearer {access_token}"} except Exception as e: verbose_logger.warning( - "_get_user_oauth_extra_headers_from_db: failed to retrieve credential for " - "user=%s server=%s: %s", + "_get_user_oauth_extra_headers_from_db: failed to retrieve credential for user=%s server=%s: %s", user_id, server_id, e, @@ -1260,6 +1444,7 @@ if MCP_AVAILABLE: mcp_auth_header: Optional[str], oauth2_headers: Optional[Dict[str, str]], raw_headers: Optional[Dict[str, str]], + user_api_key_auth: Optional[UserAPIKeyAuth] = None, ) -> Tuple[Optional[Union[Dict[str, str], str]], Optional[Dict[str, str]]]: """Build auth and extra headers for a server.""" server_auth_header: Optional[Union[Dict[str, str], str]] = None @@ -1292,10 +1477,20 @@ if MCP_AVAILABLE: str(k).lower(): v for k, v in raw_headers.items() if isinstance(k, str) } + # Centralized strip decision shared with + # ``MCPServerManager._call_regular_mcp_tool`` so the two + # code paths cannot drift on this security-sensitive choice. + # See ``_should_strip_caller_authorization`` for the rules. + strip_caller_authorization = _should_strip_caller_authorization( + mcp_server=server, + raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, + ) + for header in server.extra_headers: if not isinstance(header, str): continue - if server.has_client_credentials and header.lower() == "authorization": + if header.lower() == "authorization" and strip_caller_authorization: continue header_value = normalized_raw_headers.get(header.lower()) if header_value is None: @@ -1504,6 +1699,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) # Prefer server-stored per-user OAuth when configured, so a stale @@ -1555,6 +1751,13 @@ if MCP_AVAILABLE: f"Successfully fetched {len(tools)} tools from server {server.name}, {len(filtered_tools)} after filtering" ) return filtered_tools + except MCPUpstreamAuthError: + # Surface upstream 401/403 to the outer handler so the + # client receives a proper WWW-Authenticate challenge + # instead of a silently empty tool list. Without this + # re-raise the broad ``except Exception`` below would + # swallow the auth error. + raise except Exception as e: verbose_logger.exception( f"Error getting tools from server {server.name}: {str(e)}" @@ -1678,6 +1881,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) try: @@ -1735,6 +1939,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) try: @@ -1790,6 +1995,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) try: @@ -2318,6 +2524,7 @@ if MCP_AVAILABLE: name=original_tool_name, # Use original name for logging arguments=arguments, server_name=server_name, + session_id=_mcp_session_id_from_headers(raw_headers), ) ) litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get( @@ -2404,7 +2611,7 @@ if MCP_AVAILABLE: arguments=arguments or {}, server_name=server_name or mcp_server.name, user_api_key_auth=user_api_key_auth, - proxy_logging_obj=proxy_logging_obj, + proxy_logging_obj=proxy_logging_obj, # type: ignore[arg-type] server=mcp_server, raw_headers=raw_headers, ) @@ -2625,6 +2832,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) return await global_mcp_server_manager.get_prompt_from_server( @@ -2662,8 +2870,7 @@ if MCP_AVAILABLE: raise HTTPException( status_code=400, detail=( - "Multiple MCP servers configured; read_resource currently " - "supports exactly one allowed server." + "Multiple MCP servers configured; read_resource currently supports exactly one allowed server." ), ) @@ -2675,6 +2882,7 @@ if MCP_AVAILABLE: mcp_auth_header=mcp_auth_header, oauth2_headers=oauth2_headers, raw_headers=raw_headers, + user_api_key_auth=user_api_key_auth, ) return await global_mcp_server_manager.read_resource_from_server( @@ -2689,8 +2897,10 @@ if MCP_AVAILABLE: name: str, arguments: Dict[str, Any], server_name: Optional[str], + session_id: Optional[str] = None, ) -> StandardLoggingMCPToolCall: mcp_server = global_mcp_server_manager._get_mcp_server_from_tool_name(name) + namespaced_tool_name = f"{server_name}/{name}" if server_name else name if mcp_server: mcp_info = mcp_server.mcp_info or {} return StandardLoggingMCPToolCall( @@ -2698,13 +2908,15 @@ if MCP_AVAILABLE: arguments=arguments, mcp_server_name=mcp_info.get("server_name"), mcp_server_logo_url=mcp_info.get("logo_url"), - namespaced_tool_name=f"{server_name}/{name}" if server_name else name, + namespaced_tool_name=namespaced_tool_name, + mcp_session_id=session_id, ) else: return StandardLoggingMCPToolCall( name=name, arguments=arguments, - namespaced_tool_name=f"{server_name}/{name}" if server_name else name, + namespaced_tool_name=namespaced_tool_name, + mcp_session_id=session_id, ) async def _handle_managed_mcp_tool( @@ -3037,8 +3249,7 @@ if MCP_AVAILABLE: return False except Exception: verbose_logger.debug( - "Unable to inspect active MCP sessions for '%s'. " - "Deferring to session manager.", + "Unable to inspect active MCP sessions for '%s'. Deferring to session manager.", _session_id, ) return False @@ -3049,8 +3260,7 @@ if MCP_AVAILABLE: if method == "DELETE": _remove_stateful_session_tracking(_session_id) verbose_logger.info( - "DELETE request for non-existent MCP session '%s'. " - "Returning success (idempotent DELETE).", + "DELETE request for non-existent MCP session '%s'. Returning success (idempotent DELETE).", _session_id, ) success_response = JSONResponse( @@ -3130,6 +3340,117 @@ if MCP_AVAILABLE: ) return user_api_key_auth.model_copy(update={"object_permission": updated_op}) + def _get_passthrough_resource_metadata_url(scope: Scope, server_name: str) -> str: + request = StarletteRequest(scope) + base_url = get_request_base_url(request) + _path = scope.get("_original_path") or scope.get("path", "") or "" + + if _path.startswith(f"/{server_name}/mcp"): + return f"{base_url}/.well-known/oauth-protected-resource/{server_name}/mcp" + return f"{base_url}/.well-known/oauth-protected-resource/mcp/{server_name}" + + def _get_passthrough_www_authenticate( + scope: Scope, + server_name: str, + invalid_token: bool = False, + ) -> str: + resource_metadata_url = _get_passthrough_resource_metadata_url( + scope=scope, + server_name=server_name, + ) + params = [] + if invalid_token: + params.append('error="invalid_token"') + params.append(f'resource_metadata="{resource_metadata_url}"') + return "Bearer " + ", ".join(params) + + async def _raise_preemptive_401_for_unauthenticated_servers( + scope: Scope, + mcp_servers: Optional[List[str]], + oauth2_headers: Optional[Dict[str, str]], + mcp_server_auth_headers: Optional[Dict[str, Dict[str, str]]], + user_api_key_auth: Optional[UserAPIKeyAuth], + client_ip: Optional[str], + allowed_server_ids: Optional[Set[str]] = None, + ) -> None: + """Fail fast with HTTP 401 for MCP servers that need user auth but + didn't receive it on this request. Covers both gateway-managed OAuth2 + (points clients at the gateway AS metadata) and pass-through OAuth + (points clients at the upstream resource-metadata via our well-known). + + ``allowed_server_ids`` may be passed by callers that have already + narrowed the authorized server set (e.g. toolset scoping); servers + not in that set are skipped so a client targeting a toolset that + excludes a passthrough server is not pushed into an OAuth flow for + a server it will be 403'd on immediately after authentication. + """ + for server_name in mcp_servers or []: + server = global_mcp_server_manager.get_mcp_server_by_name( + server_name, client_ip=client_ip + ) + if ( + server is not None + and allowed_server_ids is not None + and server.server_id not in allowed_server_ids + ): + # Caller's narrowed scope excludes this server — skip the + # preemptive challenge and let downstream authorization + # return 403. + continue + if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers: + # For per-user OAuth servers, only skip the pre-emptive 401 when + # a stored token actually exists for this user+server pair. + # If no stored token exists, fail fast with 401 so clients can + # kick off PKCE/interactive OAuth flow immediately. + if server.needs_user_oauth_token: + stored_oauth_headers = await _get_user_oauth_extra_headers_from_db( + server=server, + user_api_key_auth=user_api_key_auth, + ) + if stored_oauth_headers: + continue + + request = StarletteRequest(scope) + base_url = get_request_base_url(request) + _path = scope.get("_original_path") or scope.get("path", "") or "" + + # Pick the well-known AS-metadata form that matches the inbound route + # so strict RFC 9728 §3.2 clients can resolve it correctly. + if _path.startswith(f"/mcp/{server_name}"): + _as_url = f"{base_url}/.well-known/oauth-authorization-server/mcp/{server_name}" + else: + _as_url = f"{base_url}/.well-known/oauth-authorization-server/{server_name}" + authorization_uri = f'Bearer authorization_uri="{_as_url}"' + + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"www-authenticate": authorization_uri}, + ) + + # Pass-through OAuth: when the admin has opted a server into + # forwarding the client's bearer token (is_oauth_passthrough) and + # the client hasn't supplied one, fail fast with 401 and point + # them at the gateway's oauth-protected-resource well-known URL. + # That endpoint proxies the upstream's metadata so the client + # kicks off OAuth against the real upstream IdP, not the gateway. + if ( + server + and server.is_oauth_passthrough + and not _client_has_passthrough_authorization( + server, oauth2_headers, mcp_server_auth_headers + ) + ): + www_authenticate = _get_passthrough_www_authenticate( + scope=scope, + server_name=server_name, + ) + raise HTTPException( + status_code=401, + detail="Unauthorized", + headers={"www-authenticate": www_authenticate}, + ) + def _get_forwarded_auth_from_scope(scope: Scope) -> Optional[str]: """Return the upstream-bound ``Authorization`` header value, or None. @@ -3242,12 +3563,15 @@ if MCP_AVAILABLE: passthrough_servers = [ srv for srv in allowed_servers - if srv.extra_headers - and any(h.lower() == "authorization" for h in srv.extra_headers) - # Exclude M2M servers: _prepare_mcp_server_headers skips caller - # Authorization when has_client_credentials is set, so probing - # those with the caller's token would send the wrong credential. - and not srv.has_client_credentials + # Restrict to genuine OAuth pass-through servers (auth_type none + + # Authorization in extra_headers). Gateway-managed OAuth2 servers + # must not receive the ``resource_metadata=`` challenge emitted + # below — they require ``authorization_uri=`` pointing at the + # gateway AS metadata. ``is_oauth_passthrough`` already requires + # ``auth_type in (None, MCPAuth.none)``, which is mutually + # exclusive with ``has_client_credentials`` (oauth2 + M2M flow), + # so M2M servers are implicitly excluded here. + if srv.is_oauth_passthrough ] if not passthrough_servers: return @@ -3258,19 +3582,20 @@ if MCP_AVAILABLE: for srv in passthrough_servers ] ) - request = StarletteRequest(scope) - base_url = get_request_base_url(request) for srv, (probe_status, _) in zip(passthrough_servers, probe_results): if probe_status == 401: - # Token is missing or expired — direct the client to re-authorize. - authorization_uri = ( - f"Bearer authorization_uri=" - f"{base_url}/.well-known/oauth-authorization-server/{srv.name}" + # Token is missing or expired: keep pass-through clients on the + # protected-resource discovery flow so they re-authorize against + # the upstream IdP metadata proxied by LiteLLM. + www_authenticate = _get_passthrough_www_authenticate( + scope=scope, + server_name=srv.name, + invalid_token=True, ) raise HTTPException( status_code=401, detail="Unauthorized", - headers={"WWW-Authenticate": authorization_uri}, + headers={"www-authenticate": www_authenticate}, ) if probe_status == 403: # Token is valid but the caller lacks permission — do not hint @@ -3305,39 +3630,6 @@ if MCP_AVAILABLE: verbose_logger.debug( f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" ) - # https://datatracker.ietf.org/doc/html/rfc9728#name-www-authenticate-response - for server_name in mcp_servers or []: - server = global_mcp_server_manager.get_mcp_server_by_name( - server_name, client_ip=_client_ip - ) - if server and server.auth_type == MCPAuth.oauth2 and not oauth2_headers: - # For per-user OAuth servers, only skip the pre-emptive 401 when - # a stored token actually exists for this user+server pair. - # If no stored token exists, fail fast with 401 so clients can - # kick off PKCE/interactive OAuth flow immediately. - if server.needs_user_oauth_token: - stored_oauth_headers = ( - await _get_user_oauth_extra_headers_from_db( - server=server, - user_api_key_auth=user_api_key_auth, - ) - ) - if stored_oauth_headers: - continue - - request = StarletteRequest(scope) - base_url = get_request_base_url(request) - - authorization_uri = ( - f"Bearer authorization_uri=" - f"{base_url}/.well-known/oauth-authorization-server/{server_name}" - ) - - raise HTTPException( - status_code=401, - detail="Unauthorized", - headers={"www-authenticate": authorization_uri}, - ) # Strip any client-supplied x-mcp-toolset-id to prevent forgery. scope["headers"] = [ @@ -3349,10 +3641,28 @@ if MCP_AVAILABLE: # Apply toolset scope if set server-side via ContextVar (set by # /toolset/{name}/mcp and /{name}/mcp route handlers in proxy_server.py). active_toolset_id = _mcp_active_toolset_id.get() + toolset_allowed_server_ids: Optional[Set[str]] = None if active_toolset_id and user_api_key_auth is not None: user_api_key_auth = await _apply_toolset_scope( user_api_key_auth, active_toolset_id ) + op = user_api_key_auth.object_permission + toolset_allowed_server_ids = set(op.mcp_servers or []) if op else set() + + # https://datatracker.ietf.org/doc/html/rfc9728#name-www-authenticate-response + # Must run after toolset scoping so the challenge set is derived + # from the fully-authorized server set: a passthrough server that + # the active toolset excludes should not trigger an OAuth flow + # for a server the caller will be 403'd on after authentication. + await _raise_preemptive_401_for_unauthenticated_servers( + scope=scope, + mcp_servers=mcp_servers, + oauth2_headers=oauth2_headers, + mcp_server_auth_headers=mcp_server_auth_headers, + user_api_key_auth=user_api_key_auth, + client_ip=_client_ip, + allowed_server_ids=toolset_allowed_server_ids, + ) # Pre-flight auth check for pass-through servers. Must run after # toolset scoping so the probe list is derived from the fully-authorized @@ -3428,6 +3738,7 @@ if MCP_AVAILABLE: return session_id = _get_session_id_from_scope(scope) + body = b"" if scope.get("method") == "POST": consumed_messages, body = await _read_request_body_for_routing(receive) is_initialize = _is_initialize_request(body) @@ -3452,8 +3763,7 @@ if MCP_AVAILABLE: ) if not await _enforce_stateful_session_cap_for_owner(request_owner): verbose_logger.warning( - "Rejecting MCP initialize: caller already holds the maximum " - "number of active stateful sessions." + "Rejecting MCP initialize: caller already holds the maximum number of active stateful sessions." ) too_many_response = JSONResponse( status_code=429, @@ -3485,9 +3795,56 @@ if MCP_AVAILABLE: # POST/DELETE are the methods that actually mutate the shared # auth context, so serializing those is sufficient for the # clobbering race between concurrent JSON-RPC calls. - session_lock: Optional[asyncio.Lock] = None + # + # Also skip the lock for JSON-RPC *responses* (POSTs that carry + # a ``result`` or ``error`` but no ``method``). These are replies + # to server-initiated requests such as ``elicitation/create`` or + # ``sampling/createMessage``. The in-flight tool-call POST that + # triggered the server request already holds the session lock, so + # trying to acquire it again for the response POST would deadlock. + is_jsonrpc_response = False request_method = (scope.get("method") or "").upper() - if use_stateful and session_id and request_method in ("POST", "DELETE"): + if body and request_method == "POST": + try: + _peeked = json.loads(body) + if ( + isinstance(_peeked, dict) + and _peeked.get("jsonrpc") == "2.0" + and "id" in _peeked + and "method" not in _peeked + and ("result" in _peeked or "error" in _peeked) + ): + is_jsonrpc_response = True + verbose_logger.debug( + "MCP: detected JSON-RPC response POST (id=%s), skipping session lock to avoid deadlock", + _peeked.get("id"), + ) + except (json.JSONDecodeError, TypeError): + # Peek cap truncated the body, so it can't be fully parsed. + # Scan the top-level keys (depth-aware) instead of a flat + # substring search: a response's result payload may nest a + # "method" field, and misreading that would acquire the lock + # and deadlock the in-flight tool call awaiting this + # response. A false skip is harmless; a false acquire is not. + _body_str = body.decode("utf-8", errors="replace") + if ( + '"jsonrpc"' in _body_str + and ('"result"' in _body_str or '"error"' in _body_str) + and not _jsonrpc_text_has_top_level_method(_body_str) + ): + is_jsonrpc_response = True + verbose_logger.debug( + "MCP: detected truncated JSON-RPC response POST via " + "top-level key scan, skipping session lock to avoid deadlock" + ) + + session_lock: Optional[asyncio.Lock] = None + if ( + use_stateful + and session_id + and request_method in ("POST", "DELETE") + and not is_jsonrpc_response + ): session_lock = _stateful_session_locks.setdefault( session_id, asyncio.Lock() ) @@ -3583,6 +3940,13 @@ if MCP_AVAILABLE: not in _stateful_session_auth_contexts ): _stateful_session_locks.pop(active_request_session_id, None) + except MCPUpstreamAuthError as e: + # Pass-through server returned 401 — surface it to the client so + # standards-compliant MCP clients trigger the upstream OAuth flow. + raise e.to_http_exception( + base_url=get_request_base_url(StarletteRequest(scope)), + request_path=scope.get("_original_path") or scope.get("path"), + ) except HTTPException: # Re-raise HTTP exceptions to preserve status codes and details raise @@ -3626,6 +3990,50 @@ if MCP_AVAILABLE: verbose_logger.debug( f"MCP server auth headers: {list(mcp_server_auth_headers.keys()) if mcp_server_auth_headers else None}" ) + + # Strip any client-supplied x-mcp-toolset-id to prevent forgery. + scope["headers"] = [ + (k, v) + for k, v in scope.get("headers", []) + if k.lower() != b"x-mcp-toolset-id" + ] + + # Apply toolset scope if set server-side via ContextVar so the + # downstream probe list matches the fully-authorized server set + # (mirrors the streamable HTTP handler). + active_toolset_id = _mcp_active_toolset_id.get() + toolset_allowed_server_ids: Optional[Set[str]] = None + if active_toolset_id and user_api_key_auth is not None: + user_api_key_auth = await _apply_toolset_scope( + user_api_key_auth, active_toolset_id + ) + op = user_api_key_auth.object_permission + toolset_allowed_server_ids = set(op.mcp_servers or []) if op else set() + + # https://datatracker.ietf.org/doc/html/rfc9728#name-www-authenticate-response + # Must run after toolset scoping so the challenge set is derived + # from the fully-authorized server set: a passthrough server that + # the active toolset excludes should not trigger an OAuth flow + # for a server the caller will be 403'd on after authentication. + await _raise_preemptive_401_for_unauthenticated_servers( + scope=scope, + mcp_servers=mcp_servers, + oauth2_headers=oauth2_headers, + mcp_server_auth_headers=mcp_server_auth_headers, + user_api_key_auth=user_api_key_auth, + client_ip=_sse_client_ip, + allowed_server_ids=toolset_allowed_server_ids, + ) + + # Pre-flight auth check for pass-through servers: surface upstream + # 401/403 as a proper challenge before the SSE session commits 200 + # headers, so clients can refresh their OAuth token instead of + # being stuck with a silently empty tool list. Must run after + # toolset scoping so the probe list is derived from the fully- + # authorized server set, not the raw user-supplied names. + await _check_passthrough_upstream_auth( + scope, user_api_key_auth, mcp_servers, _sse_client_ip + ) set_auth_context( user_api_key_auth=user_api_key_auth, mcp_auth_header=mcp_auth_header, @@ -3646,9 +4054,20 @@ if MCP_AVAILABLE: _sse_client_ip, ): await sse_session_manager.handle_request(scope, receive, send) + except MCPUpstreamAuthError as e: + # Pass-through server returned 401 — surface it to the client so + # standards-compliant MCP clients trigger the upstream OAuth flow. + raise e.to_http_exception( + base_url=get_request_base_url(StarletteRequest(scope)), + request_path=scope.get("_original_path") or scope.get("path"), + ) + except HTTPException: + # Re-raise HTTP exceptions to preserve status codes and details + # (e.g. 401 + WWW-Authenticate challenges from OAuth pass-through). + raise except Exception as e: verbose_logger.exception(f"Error handling MCP request: {e}") - # Instead of re-raising, try to send a graceful error response + # Try to send a graceful error response for non-HTTP exceptions try: # Send a proper HTTP error response instead of letting the exception bubble up from starlette.responses import JSONResponse @@ -3850,6 +4269,119 @@ if MCP_AVAILABLE: ) return None, None, None, None, None, None, None + def _get_current_session(): + try: + from mcp.server.lowlevel.server import request_ctx + + return request_ctx.get().session + except (LookupError, ImportError): + return None + + def _cache_auth_context_lazily(): + session = _get_current_session() + if session is None: + return + try: + if session in _session_obj_auth_storage: + return + except TypeError: + verbose_logger.debug( + "_cache_auth_context_lazily: session object is unhashable (type=%s), cannot cache auth context", + type(session).__name__, + ) + return + + auth = auth_context_var.get() + if auth and isinstance(auth, MCPAuthenticatedUser): + try: + _session_obj_auth_storage[session] = auth + except TypeError: + verbose_logger.debug( + "_cache_auth_context_lazily: could not store auth via " + "session identity — session object is unhashable" + ) + + def _recover_auth_from_session() -> Optional[MCPAuthenticatedUser]: + session = _get_current_session() + if session is None: + return None + + stored: Optional[MCPAuthenticatedUser] = None + try: + stored = _session_obj_auth_storage.get(session) + except TypeError: + verbose_logger.debug( + "_recover_auth_from_session: session object is unhashable " + "(type=%s), skipping _session_obj_auth_storage lookup", + type(session).__name__, + ) + + return stored + + async def get_or_extract_auth_context() -> Tuple[ + Optional[UserAPIKeyAuth], + Optional[str], + Optional[List[str]], + Optional[Dict[str, Dict[str, str]]], + Optional[Dict[str, str]], + Optional[Dict[str, str]], + Optional[str], + ]: + """ + Get auth context from ContextVar first, then fall back to session + storage (which survives cross-task boundaries in the MCP SDK). + """ + ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) = get_auth_context() + + if user_api_key_auth is not None: + _cache_auth_context_lazily() + else: + stored = _recover_auth_from_session() + + if stored: + user_api_key_auth = stored.user_api_key_auth + mcp_auth_header = stored.mcp_auth_header + mcp_servers = stored.mcp_servers + mcp_server_auth_headers = stored.mcp_server_auth_headers + oauth2_headers = stored.oauth2_headers + raw_headers = stored.raw_headers + _client_ip = stored.client_ip + return ( + user_api_key_auth, + mcp_auth_header, + mcp_servers, + mcp_server_auth_headers, + oauth2_headers, + raw_headers, + _client_ip, + ) + + def get_active_mcp_session() -> Optional[_McpServerSession]: + """Return the active MCP session captured during handler execution.""" + session = active_mcp_session_var.get() + if session is not None: + return session + return _get_current_session() + + def get_active_auth_context() -> Optional[MCPAuthenticatedUser]: + """Return auth context from ContextVar or session storage.""" + auth = auth_context_var.get() + if auth and isinstance(auth, MCPAuthenticatedUser): + return auth + + stored = _recover_auth_from_session() + if stored is not None: + return stored + return None + ######################################################## ############ End of Auth Context Functions ############# ######################################################## diff --git a/litellm/proxy/_experimental/mcp_server/utils.py b/litellm/proxy/_experimental/mcp_server/utils.py index b8b9207555..97cfa74ea4 100644 --- a/litellm/proxy/_experimental/mcp_server/utils.py +++ b/litellm/proxy/_experimental/mcp_server/utils.py @@ -2,12 +2,25 @@ MCP Server Utilities """ +import json import re -from typing import Any, Dict, Iterator, Mapping, Optional, Tuple, Union +from typing import ( + Any, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Set, + Tuple, + Union, +) import hashlib import importlib import os +from urllib.parse import quote # Constants LITELLM_MCP_SERVER_NAME = "litellm-mcp-server" @@ -162,6 +175,36 @@ def lookup_mcp_server_auth_in_headers( return None +MCP_TOOL_ALLOWLIST_ENFORCED_KEY = "tool_allowlist_enforced" + + +def _parse_mcp_info_dict(mcp_info: Any) -> Optional[Dict[str, Any]]: + if mcp_info is None: + return None + if isinstance(mcp_info, dict): + return mcp_info + if isinstance(mcp_info, str): + try: + parsed = json.loads(mcp_info) + except (ValueError, TypeError): + return None + return parsed if isinstance(parsed, dict) else None + return None + + +def is_server_tool_allowlist_enforced(mcp_server: Any) -> bool: + mcp_info = _parse_mcp_info_dict(getattr(mcp_server, "mcp_info", None)) + if not mcp_info: + return False + return bool(mcp_info.get(MCP_TOOL_ALLOWLIST_ENFORCED_KEY)) + + +def server_applies_tool_allowlist(mcp_server: Any) -> bool: + """Whether server-level allowed_tools whitelist filtering is active.""" + allowed_tools = getattr(mcp_server, "allowed_tools", None) or [] + return is_server_tool_allowlist_enforced(mcp_server) or bool(allowed_tools) + + def validate_and_normalize_mcp_server_payload(payload: Any) -> None: """ Validate and normalize MCP server payload fields (server_name and alias). @@ -339,6 +382,130 @@ def validate_mcp_server_name( raise Exception(error_message) +class MCPMissingUserEnvVarsError(Exception): + """Raised when an MCP request can't be built because the calling user has + not supplied one or more required per-user environment variables. + + The error message is user-facing and includes a URL the user can visit + to fill them in. + """ + + def __init__( + self, + *, + server_id: str, + server_name: Optional[str], + missing: List[str], + setup_url: str, + ) -> None: + self.server_id = server_id + self.server_name = server_name + self.missing = missing + self.setup_url = setup_url + label = server_name or server_id + bullet_list = "\n".join(f"- {name}" for name in missing) + message = ( + f'Cannot connect to MCP server "{label}".\n\n' + f"Your administrator configured this server to require per-user " + f"variables, but you haven't set the following yet:\n" + f"{bullet_list}\n\n" + f"Set your credentials here:\n" + f"{setup_url}" + ) + super().__init__(message) + + +# Pattern for ``${NAME}`` substitution. Matches the standard env-var +# identifier rules — letters, digits, underscores, can't start with a digit. +_ENV_VAR_PATTERN = re.compile(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}") + + +def parse_admin_env_vars( + env_vars: Optional[Iterable[Any]], +) -> Tuple[Dict[str, str], List[Dict[str, Any]]]: + """Split admin-configured env var entries into globals and per-user specs. + + Accepts the raw value of ``MCPServer.env_vars`` (list of dicts or Pydantic + models). Returns: + + - ``global_values``: ``{name: value}`` for entries with ``scope=="global"``. + - ``user_specs``: list of ``{name, description}`` for entries with + ``scope=="user"`` — these are the names the user must fill in. + + Unknown / malformed entries are skipped silently. + """ + global_values: Dict[str, str] = {} + user_specs: List[Dict[str, Any]] = [] + if not env_vars: + return global_values, user_specs + for raw in env_vars: + if raw is None: + continue + if hasattr(raw, "model_dump"): + entry = raw.model_dump() + elif isinstance(raw, dict): + entry = raw + else: + continue + name = entry.get("name") + if not isinstance(name, str) or not name: + continue + scope = entry.get("scope") or "global" + if scope == "user": + user_specs.append({"name": name, "description": entry.get("description")}) + else: + value = entry.get("value") + global_values[name] = "" if value is None else str(value) + return global_values, user_specs + + +def find_env_var_references(value: str) -> Set[str]: + """Return the set of ``${NAME}`` identifiers referenced inside ``value``.""" + if not value: + return set() + return set(_ENV_VAR_PATTERN.findall(value)) + + +def collect_env_var_references(*, strings: Iterable[str]) -> Set[str]: + """Union of every ``${NAME}`` reference across a collection of strings.""" + refs: Set[str] = set() + for s in strings: + if isinstance(s, str): + refs |= find_env_var_references(s) + return refs + + +def interpolate_env_vars(value: str, variables: Mapping[str, str]) -> str: + """Replace ``${NAME}`` references in ``value`` with the matching mapping + entry. Unknown names are left untouched so callers can detect them via + ``find_env_var_references`` on the result if needed. + """ + if not value: + return value + + def _sub(match: "re.Match[str]") -> str: + name = match.group(1) + if name in variables: + return variables[name] + return match.group(0) + + return _ENV_VAR_PATTERN.sub(_sub, value) + + +def interpolate_headers( + headers: Mapping[str, str], variables: Mapping[str, str] +) -> Dict[str, str]: + """Return a copy of ``headers`` with every value passed through ``interpolate_env_vars``.""" + return {k: interpolate_env_vars(v, variables) for k, v in headers.items()} + + +def build_env_var_setup_url(server_id: str) -> str: + """The frontend URL where a user can fill in their per-user env vars.""" + base = os.environ.get("PROXY_BASE_URL", "").rstrip("/") + path = f"/ui/?page=mcp-servers&fill_env_vars={quote(server_id, safe='')}" + return f"{base}{path}" if base else path + + def merge_mcp_headers( *, extra_headers: Optional[Mapping[str, str]] = None, diff --git a/litellm/proxy/_experimental/out/404.html b/litellm/proxy/_experimental/out/404.html index f27612ff54..45de348c4d 100644 --- a/litellm/proxy/_experimental/out/404.html +++ b/litellm/proxy/_experimental/out/404.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/404/index.html b/litellm/proxy/_experimental/out/404/index.html index f27612ff54..45de348c4d 100644 --- a/litellm/proxy/_experimental/out/404/index.html +++ b/litellm/proxy/_experimental/out/404/index.html @@ -1 +1 @@ -404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file +404: This page could not be found.LiteLLM Dashboard

404

This page could not be found.

\ No newline at end of file diff --git a/litellm/proxy/_experimental/out/__next.__PAGE__.txt b/litellm/proxy/_experimental/out/__next.__PAGE__.txt index c024136e8d..095c8f4339 100644 --- a/litellm/proxy/_experimental/out/__next.__PAGE__.txt +++ b/litellm/proxy/_experimental/out/__next.__PAGE__.txt @@ -1,10 +1,10 @@ 1:"$Sreact.fragment" 2:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -3:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +3:I[952683,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js"],"default"] 6:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 7:"$Sreact.suspense" :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","async":true}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","async":true}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","async":true}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","async":true}],["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true}],["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","async":true}],["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true}],["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","async":true}],["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","async":true}],["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","async":true}],["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","async":true}],["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true}],["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true}],["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true}],["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true}],["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","async":true}],["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"loading":null,"isPartial":false} 4:{} 5:"$0:rsc:props:children:0:props:serverProvidedParams:params" 8:null diff --git a/litellm/proxy/_experimental/out/__next._full.txt b/litellm/proxy/_experimental/out/__next._full.txt index 0c119086b9..2b2b385020 100644 --- a/litellm/proxy/_experimental/out/__next._full.txt +++ b/litellm/proxy/_experimental/out/__next._full.txt @@ -1,34 +1,34 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"AuthProvider"] 5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 7:I[347257,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ClientPageRoot"] -8:I[952683,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js"],"default"] +8:I[952683,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js"],"default"] 1a:I[168027,[],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"P":null,"b":"TrcGiQpTupSbDYFFfkFHY","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L7",null,{"Component":"$8","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@9","$@a"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/52c4ecc57f72065e.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/4bb663ff806dc32f.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/ee8f89c672745c59.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/9710770c6333a72f.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/d734cb3d5659b0da.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/09c1f51da7e82268.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/d028f8c28935d281.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/61aa637257592262.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/bf962cd5264be987.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/37f229ef9335f8c3.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/36d1c027ba991a4f.js","async":true,"nonce":"$undefined"}],"$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17"],"$L18"]}],{},null,false,false]},null,false,false],"$L19",false]],"m":"$undefined","G":["$1a",[]],"S":true} +0:{"P":null,"b":"LpqGBJeKQM0vUG-9uVaiY","c":["",""],"q":"","i":false,"f":[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],[["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","async":true,"nonce":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L6",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]],"forbidden":"$undefined","unauthorized":"$undefined"}]}]}]}]}]}]]}],{"children":[["$","$1","c",{"children":[["$","$L7",null,{"Component":"$8","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@9","$@a"]}}],[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","precedence":"next","crossOrigin":"$undefined","nonce":"$undefined"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/bee4095c26818f05.js","async":true,"nonce":"$undefined"}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/81937424fe90f746.js","async":true,"nonce":"$undefined"}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/e2257d8308d35cf4.js","async":true,"nonce":"$undefined"}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/2954392b7a60a6a1.js","async":true,"nonce":"$undefined"}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/04711b0f8ffa7bbd.js","async":true,"nonce":"$undefined"}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/eb1ba04e211a533f.js","async":true,"nonce":"$undefined"}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/88c74f8b4b20d25a.js","async":true,"nonce":"$undefined"}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/4cb93eefa53f21a3.js","async":true,"nonce":"$undefined"}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/a1ef280b7ad5ae6a.js","async":true,"nonce":"$undefined"}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/40a2744137b1aec2.js","async":true,"nonce":"$undefined"}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/542a1a209eb732c6.js","async":true,"nonce":"$undefined"}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/84a27349dda457cd.js","async":true,"nonce":"$undefined"}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/8ddf82e7e0b331fc.js","async":true,"nonce":"$undefined"}],["$","script","script-13",{"src":"/litellm-asset-prefix/_next/static/chunks/1d7b3500478e93ae.js","async":true,"nonce":"$undefined"}],["$","script","script-14",{"src":"/litellm-asset-prefix/_next/static/chunks/f0e079183e7bb90c.js","async":true,"nonce":"$undefined"}],["$","script","script-15",{"src":"/litellm-asset-prefix/_next/static/chunks/10757c2146f43db4.js","async":true,"nonce":"$undefined"}],"$Lb","$Lc","$Ld","$Le","$Lf","$L10","$L11","$L12","$L13","$L14","$L15","$L16","$L17"],"$L18"]}],{},null,false,false]},null,false,false],"$L19",false]],"m":"$undefined","G":["$1a",[]],"S":true} 1b:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"OutletBoundary"] 1c:"$Sreact.suspense" 1e:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"ViewportBoundary"] 20:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] b:["$","script","script-16",{"src":"/litellm-asset-prefix/_next/static/chunks/786e88f4abdd5c58.js","async":true,"nonce":"$undefined"}] -c:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/43c3db1352241a8b.js","async":true,"nonce":"$undefined"}] +c:["$","script","script-17",{"src":"/litellm-asset-prefix/_next/static/chunks/ffa46de7b8384155.js","async":true,"nonce":"$undefined"}] d:["$","script","script-18",{"src":"/litellm-asset-prefix/_next/static/chunks/31275eb5c6f6332f.js","async":true,"nonce":"$undefined"}] -e:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/0ac09b227f50edb4.js","async":true,"nonce":"$undefined"}] -f:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/4c848b12d4ecda3d.js","async":true,"nonce":"$undefined"}] -10:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/91c828abd7c0aff5.js","async":true,"nonce":"$undefined"}] -11:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/08c348f8e09a5cb0.js","async":true,"nonce":"$undefined"}] +e:["$","script","script-19",{"src":"/litellm-asset-prefix/_next/static/chunks/80f4410629229bf9.js","async":true,"nonce":"$undefined"}] +f:["$","script","script-20",{"src":"/litellm-asset-prefix/_next/static/chunks/75ee9aba04c74e23.js","async":true,"nonce":"$undefined"}] +10:["$","script","script-21",{"src":"/litellm-asset-prefix/_next/static/chunks/193886179a5779b5.js","async":true,"nonce":"$undefined"}] +11:["$","script","script-22",{"src":"/litellm-asset-prefix/_next/static/chunks/2063ca6435a47940.js","async":true,"nonce":"$undefined"}] 12:["$","script","script-23",{"src":"/litellm-asset-prefix/_next/static/chunks/7e417dd24c8becd0.js","async":true,"nonce":"$undefined"}] 13:["$","script","script-24",{"src":"/litellm-asset-prefix/_next/static/chunks/b323e0ef008e6348.js","async":true,"nonce":"$undefined"}] 14:["$","script","script-25",{"src":"/litellm-asset-prefix/_next/static/chunks/4ac3235460262f36.js","async":true,"nonce":"$undefined"}] 15:["$","script","script-26",{"src":"/litellm-asset-prefix/_next/static/chunks/51494a4a4b6fc437.js","async":true,"nonce":"$undefined"}] -16:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/878832edb30e99a4.js","async":true,"nonce":"$undefined"}] -17:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/9f1486622270556b.js","async":true,"nonce":"$undefined"}] +16:["$","script","script-27",{"src":"/litellm-asset-prefix/_next/static/chunks/d7c18aec4a87a237.js","async":true,"nonce":"$undefined"}] +17:["$","script","script-28",{"src":"/litellm-asset-prefix/_next/static/chunks/dac86522fa98e760.js","async":true,"nonce":"$undefined"}] 18:["$","$L1b",null,{"children":["$","$1c",null,{"name":"Next.MetadataOutlet","children":"$@1d"}]}] 19:["$","$1","h",{"children":[null,["$","$L1e",null,{"children":"$L1f"}],["$","div",null,{"hidden":true,"children":["$","$L20",null,{"children":["$","$1c",null,{"name":"Next.Metadata","children":"$L21"}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}] 9:{} @@ -36,4 +36,4 @@ a:"$0:f:0:1:1:children:0:props:children:0:props:serverProvidedParams:params" 1f:[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]] 22:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] 1d:null -21:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L22","4",{}]] +21:[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L22","4",{}]] diff --git a/litellm/proxy/_experimental/out/__next._head.txt b/litellm/proxy/_experimental/out/__next._head.txt index ea6e509545..870c89c7e1 100644 --- a/litellm/proxy/_experimental/out/__next._head.txt +++ b/litellm/proxy/_experimental/out/__next._head.txt @@ -3,4 +3,4 @@ 3:I[897367,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"MetadataBoundary"] 4:"$Sreact.suspense" 5:I[27201,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"IconMark"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"./favicon.ico"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.1d32c690.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._index.txt b/litellm/proxy/_experimental/out/__next._index.txt index ebf6d8fec0..67c452e8c2 100644 --- a/litellm/proxy/_experimental/out/__next._index.txt +++ b/litellm/proxy/_experimental/out/__next._index.txt @@ -1,9 +1,9 @@ 1:"$Sreact.fragment" -2:I[867271,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -3:I[71195,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"default"] -4:I[557951,["/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js"],"AuthProvider"] +2:I[867271,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +3:I[71195,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"default"] +4:I[557951,["/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js"],"AuthProvider"] 5:I[339756,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] 6:I[837457,["/litellm-asset-prefix/_next/static/chunks/d96012bcfc98706a.js","/litellm-asset-prefix/_next/static/chunks/dbca964212122d58.js"],"default"] :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/aefca6f40ea185cd.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/632b4c8e836bd956.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"loading":null,"isPartial":false} +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/59002382e3e0d318.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/25c705f79a0254af.js","async":true}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"children":["$","$L3",null,{"children":["$","$L4",null,{"children":["$","$L5",null,{"parallelRouterKey":"children","template":["$","$L6",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}]}]}]}]}]]}],"loading":null,"isPartial":false} diff --git a/litellm/proxy/_experimental/out/__next._tree.txt b/litellm/proxy/_experimental/out/__next._tree.txt index 4a08f4f9e1..86dc121c5f 100644 --- a/litellm/proxy/_experimental/out/__next._tree.txt +++ b/litellm/proxy/_experimental/out/__next._tree.txt @@ -1,5 +1,5 @@ :HL["/litellm-asset-prefix/_next/static/chunks/4e20891f2fd03463.css","style"] -:HL["/litellm-asset-prefix/_next/static/chunks/1bcca3c38c9deb02.css","style"] +:HL["/litellm-asset-prefix/_next/static/chunks/47150bfa067220d3.css","style"] :HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.3a6ba036.woff2","font",{"crossOrigin":"","type":"font/woff2"}] :HL["/litellm-asset-prefix/_next/static/chunks/3f3fa56b5786d58c.css","style"] -0:{"buildId":"TrcGiQpTupSbDYFFfkFHY","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} +0:{"buildId":"LpqGBJeKQM0vUG-9uVaiY","tree":{"name":"","paramType":null,"paramKey":"","hasRuntimePrefetch":false,"slots":{"children":{"name":"__PAGE__","paramType":null,"paramKey":"__PAGE__","hasRuntimePrefetch":false,"slots":null,"isRootLayout":false}},"isRootLayout":true},"staleTime":300} diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_buildManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_buildManifest.js rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_buildManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_clientMiddlewareManifest.json similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_clientMiddlewareManifest.json rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_clientMiddlewareManifest.json diff --git a/litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js b/litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_ssgManifest.js similarity index 100% rename from litellm/proxy/_experimental/out/_next/static/TrcGiQpTupSbDYFFfkFHY/_ssgManifest.js rename to litellm/proxy/_experimental/out/_next/static/LpqGBJeKQM0vUG-9uVaiY/_ssgManifest.js diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js b/litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js similarity index 98% rename from litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js rename to litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js index 825fe327e9..55ce00c27b 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/adfb3758f3e2c464.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/10376d0955336027.js @@ -1,4 +1,4 @@ -(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,295320,e=>{"use strict";e.i(247167);var t=e.i(931067),i=e.i(271645);let r={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M704 446H320c-4.4 0-8 3.6-8 8v402c0 4.4 3.6 8 8 8h384c4.4 0 8-3.6 8-8V454c0-4.4-3.6-8-8-8zm-328 64h272v117H376V510zm272 290H376V683h272v117z"}},{tag:"path",attrs:{d:"M424 748a32 32 0 1064 0 32 32 0 10-64 0zm0-178a32 32 0 1064 0 32 32 0 10-64 0z"}},{tag:"path",attrs:{d:"M811.4 368.9C765.6 248 648.9 162 512.2 162S258.8 247.9 213 368.8C126.9 391.5 63.5 470.2 64 563.6 64.6 668 145.6 752.9 247.6 762c4.7.4 8.7-3.3 8.7-8v-60.4c0-4-3-7.4-7-7.9-27-3.4-52.5-15.2-72.1-34.5-24-23.5-37.2-55.1-37.2-88.6 0-28 9.1-54.4 26.2-76.4 16.7-21.4 40.2-36.9 66.1-43.7l37.9-10 13.9-36.7c8.6-22.8 20.6-44.2 35.7-63.5 14.9-19.2 32.6-36 52.4-50 41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.3c19.9 14 37.5 30.8 52.4 50 15.1 19.3 27.1 40.7 35.7 63.5l13.8 36.6 37.8 10c54.2 14.4 92.1 63.7 92.1 120 0 33.6-13.2 65.1-37.2 88.6-19.5 19.2-44.9 31.1-71.9 34.5-4 .5-6.9 3.9-6.9 7.9V754c0 4.7 4.1 8.4 8.8 8 101.7-9.2 182.5-94 183.2-198.2.6-93.4-62.7-172.1-148.6-194.9z"}}]},name:"cloud-server",theme:"outlined"};var n=e.i(9583),a=i.forwardRef(function(e,a){return i.createElement(n.default,(0,t.default)({},e,{ref:a,icon:r}))});e.s(["CloudServerOutlined",0,a],295320)},283713,e=>{"use strict";var t=e.i(271645),i=e.i(764205),r=e.i(612256);let n="litellm_selected_worker_id";e.s(["useWorker",0,()=>{let{data:e}=(0,r.useUIConfig)(),a=e?.is_control_plane??!1,o=e?.workers??[],[l,s]=(0,t.useState)(()=>localStorage.getItem(n));(0,t.useEffect)(()=>{if(!l||0===o.length)return;let e=o.find(e=>e.worker_id===l);e&&(0,i.switchToWorkerUrl)(e.url)},[l,o]);let c=o.find(e=>e.worker_id===l)??null,d=(0,t.useCallback)(e=>{let t=o.find(t=>t.worker_id===e);t&&(s(e),localStorage.setItem(n,e),(0,i.switchToWorkerUrl)(t.url))},[o]);return{isControlPlane:a,workers:o,selectedWorkerId:l,selectedWorker:c,selectWorker:d,disconnectFromWorker:(0,t.useCallback)(()=>{s(null),localStorage.removeItem(n),(0,i.switchToWorkerUrl)(null)},[])}}])},954616,e=>{"use strict";var t=e.i(271645),i=e.i(114272),r=e.i(540143),n=e.i(915823),a=e.i(619273),o=class extends n.Subscribable{#e;#t=void 0;#i;#r;constructor(e,t){super(),this.#e=e,this.setOptions(t),this.bindMethods(),this.#n()}bindMethods(){this.mutate=this.mutate.bind(this),this.reset=this.reset.bind(this)}setOptions(e){let t=this.options;this.options=this.#e.defaultMutationOptions(e),(0,a.shallowEqualObjects)(this.options,t)||this.#e.getMutationCache().notify({type:"observerOptionsUpdated",mutation:this.#i,observer:this}),t?.mutationKey&&this.options.mutationKey&&(0,a.hashKey)(t.mutationKey)!==(0,a.hashKey)(this.options.mutationKey)?this.reset():this.#i?.state.status==="pending"&&this.#i.setOptions(this.options)}onUnsubscribe(){this.hasListeners()||this.#i?.removeObserver(this)}onMutationUpdate(e){this.#n(),this.#a(e)}getCurrentResult(){return this.#t}reset(){this.#i?.removeObserver(this),this.#i=void 0,this.#n(),this.#a()}mutate(e,t){return this.#r=t,this.#i?.removeObserver(this),this.#i=this.#e.getMutationCache().build(this.#e,this.options),this.#i.addObserver(this),this.#i.execute(e)}#n(){let e=this.#i?.state??(0,i.getDefaultState)();this.#t={...e,isPending:"pending"===e.status,isSuccess:"success"===e.status,isError:"error"===e.status,isIdle:"idle"===e.status,mutate:this.mutate,reset:this.reset}}#a(e){r.notifyManager.batch(()=>{if(this.#r&&this.hasListeners()){let t=this.#t.variables,i=this.#t.context,r={client:this.#e,meta:this.options.meta,mutationKey:this.options.mutationKey};if(e?.type==="success"){try{this.#r.onSuccess?.(e.data,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(e.data,null,t,i,r)}catch(e){Promise.reject(e)}}else if(e?.type==="error"){try{this.#r.onError?.(e.error,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(void 0,e.error,t,i,r)}catch(e){Promise.reject(e)}}}this.listeners.forEach(e=>{e(this.#t)})})}},l=e.i(912598);function s(e,i){let n=(0,l.useQueryClient)(i),[s]=t.useState(()=>new o(n,e));t.useEffect(()=>{s.setOptions(e)},[s,e]);let c=t.useSyncExternalStore(t.useCallback(e=>s.subscribe(r.notifyManager.batchCalls(e)),[s]),()=>s.getCurrentResult(),()=>s.getCurrentResult()),d=t.useCallback((e,t)=>{s.mutate(e,t).catch(a.noop)},[s]);if(c.error&&(0,a.shouldThrowError)(s.options.throwOnError,[c.error]))throw c.error;return{...c,mutate:d,mutateAsync:c.mutate}}e.s(["useMutation",()=>s],954616)},175712,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(529681),n=e.i(242064),a=e.i(517455),o=e.i(185793),l=e.i(721369),s=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let c=e=>{var{prefixCls:r,className:a,hoverable:o=!0}=e,l=s(e,["prefixCls","className","hoverable"]);let{getPrefixCls:c}=t.useContext(n.ConfigContext),d=c("card",r),u=(0,i.default)(`${d}-grid`,a,{[`${d}-grid-hoverable`]:o});return t.createElement("div",Object.assign({},l,{className:u}))};e.i(296059);var d=e.i(915654),u=e.i(183293),m=e.i(246422),p=e.i(838378);let g=(0,m.genStyleHooks)("Card",e=>{let t=(0,p.mergeToken)(e,{cardShadow:e.boxShadowCard,cardHeadPadding:e.padding,cardPaddingBase:e.paddingLG,cardActionsIconSize:e.fontSize});return[(e=>{let{componentCls:t,cardShadow:i,cardHeadPadding:r,colorBorderSecondary:n,boxShadowTertiary:a,bodyPadding:o,extraColor:l}=e;return{[t]:Object.assign(Object.assign({},(0,u.resetComponent)(e)),{position:"relative",background:e.colorBgContainer,borderRadius:e.borderRadiusLG,[`&:not(${t}-bordered)`]:{boxShadow:a},[`${t}-head`]:(e=>{let{antCls:t,componentCls:i,headerHeight:r,headerPadding:n,tabsMarginBottom:a}=e;return Object.assign(Object.assign({display:"flex",justifyContent:"center",flexDirection:"column",minHeight:r,marginBottom:-1,padding:`0 ${(0,d.unit)(n)}`,color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.headerFontSize,background:e.headerBg,borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`,borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`},(0,u.clearFix)()),{"&-wrapper":{width:"100%",display:"flex",alignItems:"center"},"&-title":Object.assign(Object.assign({display:"inline-block",flex:1},u.textEllipsis),{[` +(globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,295320,e=>{"use strict";e.i(247167);var t=e.i(931067),i=e.i(271645);let r={icon:{tag:"svg",attrs:{viewBox:"64 64 896 896",focusable:"false"},children:[{tag:"path",attrs:{d:"M704 446H320c-4.4 0-8 3.6-8 8v402c0 4.4 3.6 8 8 8h384c4.4 0 8-3.6 8-8V454c0-4.4-3.6-8-8-8zm-328 64h272v117H376V510zm272 290H376V683h272v117z"}},{tag:"path",attrs:{d:"M424 748a32 32 0 1064 0 32 32 0 10-64 0zm0-178a32 32 0 1064 0 32 32 0 10-64 0z"}},{tag:"path",attrs:{d:"M811.4 368.9C765.6 248 648.9 162 512.2 162S258.8 247.9 213 368.8C126.9 391.5 63.5 470.2 64 563.6 64.6 668 145.6 752.9 247.6 762c4.7.4 8.7-3.3 8.7-8v-60.4c0-4-3-7.4-7-7.9-27-3.4-52.5-15.2-72.1-34.5-24-23.5-37.2-55.1-37.2-88.6 0-28 9.1-54.4 26.2-76.4 16.7-21.4 40.2-36.9 66.1-43.7l37.9-10 13.9-36.7c8.6-22.8 20.6-44.2 35.7-63.5 14.9-19.2 32.6-36 52.4-50 41.1-28.9 89.5-44.2 140-44.2s98.9 15.3 140 44.3c19.9 14 37.5 30.8 52.4 50 15.1 19.3 27.1 40.7 35.7 63.5l13.8 36.6 37.8 10c54.2 14.4 92.1 63.7 92.1 120 0 33.6-13.2 65.1-37.2 88.6-19.5 19.2-44.9 31.1-71.9 34.5-4 .5-6.9 3.9-6.9 7.9V754c0 4.7 4.1 8.4 8.8 8 101.7-9.2 182.5-94 183.2-198.2.6-93.4-62.7-172.1-148.6-194.9z"}}]},name:"cloud-server",theme:"outlined"};var n=e.i(9583),a=i.forwardRef(function(e,a){return i.createElement(n.default,(0,t.default)({},e,{ref:a,icon:r}))});e.s(["CloudServerOutlined",0,a],295320)},283713,e=>{"use strict";var t=e.i(271645),i=e.i(602869),r=e.i(612256);let n="litellm_selected_worker_id";e.s(["useWorker",0,()=>{let{data:e}=(0,r.useUIConfig)(),a=e?.is_control_plane??!1,o=e?.workers??[],[l,s]=(0,t.useState)(()=>localStorage.getItem(n));(0,t.useEffect)(()=>{if(!l||0===o.length)return;let e=o.find(e=>e.worker_id===l);e&&(0,i.switchToWorkerUrl)(e.url)},[l,o]);let c=o.find(e=>e.worker_id===l)??null,d=(0,t.useCallback)(e=>{let t=o.find(t=>t.worker_id===e);t&&(s(e),localStorage.setItem(n,e),(0,i.switchToWorkerUrl)(t.url))},[o]);return{isControlPlane:a,workers:o,selectedWorkerId:l,selectedWorker:c,selectWorker:d,disconnectFromWorker:(0,t.useCallback)(()=>{s(null),localStorage.removeItem(n),(0,i.switchToWorkerUrl)(null)},[])}}])},954616,e=>{"use strict";var t=e.i(271645),i=e.i(114272),r=e.i(540143),n=e.i(915823),a=e.i(619273),o=class extends n.Subscribable{#e;#t=void 0;#i;#r;constructor(e,t){super(),this.#e=e,this.setOptions(t),this.bindMethods(),this.#n()}bindMethods(){this.mutate=this.mutate.bind(this),this.reset=this.reset.bind(this)}setOptions(e){let t=this.options;this.options=this.#e.defaultMutationOptions(e),(0,a.shallowEqualObjects)(this.options,t)||this.#e.getMutationCache().notify({type:"observerOptionsUpdated",mutation:this.#i,observer:this}),t?.mutationKey&&this.options.mutationKey&&(0,a.hashKey)(t.mutationKey)!==(0,a.hashKey)(this.options.mutationKey)?this.reset():this.#i?.state.status==="pending"&&this.#i.setOptions(this.options)}onUnsubscribe(){this.hasListeners()||this.#i?.removeObserver(this)}onMutationUpdate(e){this.#n(),this.#a(e)}getCurrentResult(){return this.#t}reset(){this.#i?.removeObserver(this),this.#i=void 0,this.#n(),this.#a()}mutate(e,t){return this.#r=t,this.#i?.removeObserver(this),this.#i=this.#e.getMutationCache().build(this.#e,this.options),this.#i.addObserver(this),this.#i.execute(e)}#n(){let e=this.#i?.state??(0,i.getDefaultState)();this.#t={...e,isPending:"pending"===e.status,isSuccess:"success"===e.status,isError:"error"===e.status,isIdle:"idle"===e.status,mutate:this.mutate,reset:this.reset}}#a(e){r.notifyManager.batch(()=>{if(this.#r&&this.hasListeners()){let t=this.#t.variables,i=this.#t.context,r={client:this.#e,meta:this.options.meta,mutationKey:this.options.mutationKey};if(e?.type==="success"){try{this.#r.onSuccess?.(e.data,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(e.data,null,t,i,r)}catch(e){Promise.reject(e)}}else if(e?.type==="error"){try{this.#r.onError?.(e.error,t,i,r)}catch(e){Promise.reject(e)}try{this.#r.onSettled?.(void 0,e.error,t,i,r)}catch(e){Promise.reject(e)}}}this.listeners.forEach(e=>{e(this.#t)})})}},l=e.i(912598);function s(e,i){let n=(0,l.useQueryClient)(i),[s]=t.useState(()=>new o(n,e));t.useEffect(()=>{s.setOptions(e)},[s,e]);let c=t.useSyncExternalStore(t.useCallback(e=>s.subscribe(r.notifyManager.batchCalls(e)),[s]),()=>s.getCurrentResult(),()=>s.getCurrentResult()),d=t.useCallback((e,t)=>{s.mutate(e,t).catch(a.noop)},[s]);if(c.error&&(0,a.shouldThrowError)(s.options.throwOnError,[c.error]))throw c.error;return{...c,mutate:d,mutateAsync:c.mutate}}e.s(["useMutation",()=>s],954616)},175712,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(529681),n=e.i(242064),a=e.i(517455),o=e.i(185793),l=e.i(721369),s=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let c=e=>{var{prefixCls:r,className:a,hoverable:o=!0}=e,l=s(e,["prefixCls","className","hoverable"]);let{getPrefixCls:c}=t.useContext(n.ConfigContext),d=c("card",r),u=(0,i.default)(`${d}-grid`,a,{[`${d}-grid-hoverable`]:o});return t.createElement("div",Object.assign({},l,{className:u}))};e.i(296059);var d=e.i(915654),u=e.i(183293),m=e.i(246422),p=e.i(838378);let g=(0,m.genStyleHooks)("Card",e=>{let t=(0,p.mergeToken)(e,{cardShadow:e.boxShadowCard,cardHeadPadding:e.padding,cardPaddingBase:e.paddingLG,cardActionsIconSize:e.fontSize});return[(e=>{let{componentCls:t,cardShadow:i,cardHeadPadding:r,colorBorderSecondary:n,boxShadowTertiary:a,bodyPadding:o,extraColor:l}=e;return{[t]:Object.assign(Object.assign({},(0,u.resetComponent)(e)),{position:"relative",background:e.colorBgContainer,borderRadius:e.borderRadiusLG,[`&:not(${t}-bordered)`]:{boxShadow:a},[`${t}-head`]:(e=>{let{antCls:t,componentCls:i,headerHeight:r,headerPadding:n,tabsMarginBottom:a}=e;return Object.assign(Object.assign({display:"flex",justifyContent:"center",flexDirection:"column",minHeight:r,marginBottom:-1,padding:`0 ${(0,d.unit)(n)}`,color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.headerFontSize,background:e.headerBg,borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`,borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`},(0,u.clearFix)()),{"&-wrapper":{width:"100%",display:"flex",alignItems:"center"},"&-title":Object.assign(Object.assign({display:"inline-block",flex:1},u.textEllipsis),{[` > ${i}-typography, > ${i}-typography-edit-content `]:{insetInlineStart:0,marginTop:0,marginBottom:0}}),[`${t}-tabs-top`]:{clear:"both",marginBottom:a,color:e.colorText,fontWeight:"normal",fontSize:e.fontSize,"&-bar":{borderBottom:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${e.colorBorderSecondary}`}}})})(e),[`${t}-extra`]:{marginInlineStart:"auto",color:l,fontWeight:"normal",fontSize:e.fontSize},[`${t}-body`]:{padding:o,borderRadius:`0 0 ${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)}`},[`${t}-grid`]:(e=>{let{cardPaddingBase:t,colorBorderSecondary:i,cardShadow:r,lineWidth:n}=e;return{width:"33.33%",padding:t,border:0,borderRadius:0,boxShadow:` @@ -9,4 +9,4 @@ 0 ${(0,d.unit)(n)} 0 0 ${i} inset; `,transition:`all ${e.motionDurationMid}`,"&-hoverable:hover":{position:"relative",zIndex:1,boxShadow:r}}})(e),[`${t}-cover`]:{"> *":{display:"block",width:"100%",borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0`}},[`${t}-actions`]:(e=>{let{componentCls:t,iconCls:i,actionsLiMargin:r,cardActionsIconSize:n,colorBorderSecondary:a,actionsBg:o}=e;return Object.assign(Object.assign({margin:0,padding:0,listStyle:"none",background:o,borderTop:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${a}`,display:"flex",borderRadius:`0 0 ${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)}`},(0,u.clearFix)()),{"& > li":{margin:r,color:e.colorTextDescription,textAlign:"center","> span":{position:"relative",display:"block",minWidth:e.calc(e.cardActionsIconSize).mul(2).equal(),fontSize:e.fontSize,lineHeight:e.lineHeight,cursor:"pointer","&:hover":{color:e.colorPrimary,transition:`color ${e.motionDurationMid}`},[`a:not(${t}-btn), > ${i}`]:{display:"inline-block",width:"100%",color:e.colorIcon,lineHeight:(0,d.unit)(e.fontHeight),transition:`color ${e.motionDurationMid}`,"&:hover":{color:e.colorPrimary}},[`> ${i}`]:{fontSize:n,lineHeight:(0,d.unit)(e.calc(n).mul(e.lineHeight).equal())}},"&:not(:last-child)":{borderInlineEnd:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${a}`}}})})(e),[`${t}-meta`]:Object.assign(Object.assign({margin:`${(0,d.unit)(e.calc(e.marginXXS).mul(-1).equal())} 0`,display:"flex"},(0,u.clearFix)()),{"&-avatar":{paddingInlineEnd:e.padding},"&-detail":{overflow:"hidden",flex:1,"> div:not(:last-child)":{marginBottom:e.marginXS}},"&-title":Object.assign({color:e.colorTextHeading,fontWeight:e.fontWeightStrong,fontSize:e.fontSizeLG},u.textEllipsis),"&-description":{color:e.colorTextDescription}})}),[`${t}-bordered`]:{border:`${(0,d.unit)(e.lineWidth)} ${e.lineType} ${n}`,[`${t}-cover`]:{marginTop:-1,marginInlineStart:-1,marginInlineEnd:-1}},[`${t}-hoverable`]:{cursor:"pointer",transition:`box-shadow ${e.motionDurationMid}, border-color ${e.motionDurationMid}`,"&:hover":{borderColor:"transparent",boxShadow:i}},[`${t}-contain-grid`]:{borderRadius:`${(0,d.unit)(e.borderRadiusLG)} ${(0,d.unit)(e.borderRadiusLG)} 0 0 `,[`${t}-body`]:{display:"flex",flexWrap:"wrap"},[`&:not(${t}-loading) ${t}-body`]:{marginBlockStart:e.calc(e.lineWidth).mul(-1).equal(),marginInlineStart:e.calc(e.lineWidth).mul(-1).equal(),padding:0}},[`${t}-contain-tabs`]:{[`> div${t}-head`]:{minHeight:0,[`${t}-head-title, ${t}-extra`]:{paddingTop:r}}},[`${t}-type-inner`]:(e=>{let{componentCls:t,colorFillAlter:i,headerPadding:r,bodyPadding:n}=e;return{[`${t}-head`]:{padding:`0 ${(0,d.unit)(r)}`,background:i,"&-title":{fontSize:e.fontSize}},[`${t}-body`]:{padding:`${(0,d.unit)(e.padding)} ${(0,d.unit)(n)}`}}})(e),[`${t}-loading`]:(e=>{let{componentCls:t}=e;return{overflow:"hidden",[`${t}-body`]:{userSelect:"none"}}})(e),[`${t}-rtl`]:{direction:"rtl"}}})(t),(e=>{let{componentCls:t,bodyPaddingSM:i,headerPaddingSM:r,headerHeightSM:n,headerFontSizeSM:a}=e;return{[`${t}-small`]:{[`> ${t}-head`]:{minHeight:n,padding:`0 ${(0,d.unit)(r)}`,fontSize:a,[`> ${t}-head-wrapper`]:{[`> ${t}-extra`]:{fontSize:e.fontSize}}},[`> ${t}-body`]:{padding:i}},[`${t}-small${t}-contain-tabs`]:{[`> ${t}-head`]:{[`${t}-head-title, ${t}-extra`]:{paddingTop:0,display:"flex",alignItems:"center"}}}}})(t)]},e=>{var t,i;return{headerBg:"transparent",headerFontSize:e.fontSizeLG,headerFontSizeSM:e.fontSize,headerHeight:e.fontSizeLG*e.lineHeightLG+2*e.padding,headerHeightSM:e.fontSize*e.lineHeight+2*e.paddingXS,actionsBg:e.colorBgContainer,actionsLiMargin:`${e.paddingSM}px 0`,tabsMarginBottom:-e.padding-e.lineWidth,extraColor:e.colorText,bodyPaddingSM:12,headerPaddingSM:12,bodyPadding:null!=(t=e.bodyPadding)?t:e.paddingLG,headerPadding:null!=(i=e.headerPadding)?i:e.paddingLG}});var h=e.i(792812),f=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let b=e=>{let{actionClasses:i,actions:r=[],actionStyle:n}=e;return t.createElement("ul",{className:i,style:n},r.map((e,i)=>{let n=`action-${i}`;return t.createElement("li",{style:{width:`${100/r.length}%`},key:n},t.createElement("span",null,e))}))},y=t.forwardRef((e,s)=>{let d,{prefixCls:u,className:m,rootClassName:p,style:y,extra:v,headStyle:x={},bodyStyle:$={},title:S,loading:j,bordered:w,variant:O,size:C,type:E,cover:I,actions:N,tabList:k,children:z,activeTabKey:L,defaultActiveTabKey:M,tabBarExtraContent:R,hoverable:P,tabProps:T={},classNames:_,styles:G}=e,B=f(e,["prefixCls","className","rootClassName","style","extra","headStyle","bodyStyle","title","loading","bordered","variant","size","type","cover","actions","tabList","children","activeTabKey","defaultActiveTabKey","tabBarExtraContent","hoverable","tabProps","classNames","styles"]),{getPrefixCls:A,direction:H,card:U}=t.useContext(n.ConfigContext),[W]=(0,h.default)("card",O,w),D=e=>{var t;return(0,i.default)(null==(t=null==U?void 0:U.classNames)?void 0:t[e],null==_?void 0:_[e])},F=e=>{var t;return Object.assign(Object.assign({},null==(t=null==U?void 0:U.styles)?void 0:t[e]),null==G?void 0:G[e])},K=t.useMemo(()=>{let e=!1;return t.Children.forEach(z,t=>{(null==t?void 0:t.type)===c&&(e=!0)}),e},[z]),q=A("card",u),[V,X,J]=g(q),Q=t.createElement(o.default,{loading:!0,active:!0,paragraph:{rows:4},title:!1},z),Y=void 0!==L,Z=Object.assign(Object.assign({},T),{[Y?"activeKey":"defaultActiveKey"]:Y?L:M,tabBarExtraContent:R}),ee=(0,a.default)(C),et=ee&&"default"!==ee?ee:"large",ei=k?t.createElement(l.default,Object.assign({size:et},Z,{className:`${q}-head-tabs`,onChange:t=>{var i;null==(i=e.onTabChange)||i.call(e,t)},items:k.map(e=>{var{tab:t}=e;return Object.assign({label:t},f(e,["tab"]))})})):null;if(S||v||ei){let e=(0,i.default)(`${q}-head`,D("header")),r=(0,i.default)(`${q}-head-title`,D("title")),n=(0,i.default)(`${q}-extra`,D("extra")),a=Object.assign(Object.assign({},x),F("header"));d=t.createElement("div",{className:e,style:a},t.createElement("div",{className:`${q}-head-wrapper`},S&&t.createElement("div",{className:r,style:F("title")},S),v&&t.createElement("div",{className:n,style:F("extra")},v)),ei)}let er=(0,i.default)(`${q}-cover`,D("cover")),en=I?t.createElement("div",{className:er,style:F("cover")},I):null,ea=(0,i.default)(`${q}-body`,D("body")),eo=Object.assign(Object.assign({},$),F("body")),el=t.createElement("div",{className:ea,style:eo},j?Q:z),es=(0,i.default)(`${q}-actions`,D("actions")),ec=(null==N?void 0:N.length)?t.createElement(b,{actionClasses:es,actionStyle:F("actions"),actions:N}):null,ed=(0,r.default)(B,["onTabChange"]),eu=(0,i.default)(q,null==U?void 0:U.className,{[`${q}-loading`]:j,[`${q}-bordered`]:"borderless"!==W,[`${q}-hoverable`]:P,[`${q}-contain-grid`]:K,[`${q}-contain-tabs`]:null==k?void 0:k.length,[`${q}-${ee}`]:ee,[`${q}-type-${E}`]:!!E,[`${q}-rtl`]:"rtl"===H},m,p,X,J),em=Object.assign(Object.assign({},null==U?void 0:U.style),y);return V(t.createElement("div",Object.assign({ref:s},ed,{className:eu,style:em}),d,en,el,ec))});var v=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};y.Grid=c,y.Meta=e=>{let{prefixCls:r,className:a,avatar:o,title:l,description:s}=e,c=v(e,["prefixCls","className","avatar","title","description"]),{getPrefixCls:d}=t.useContext(n.ConfigContext),u=d("card",r),m=(0,i.default)(`${u}-meta`,a),p=o?t.createElement("div",{className:`${u}-meta-avatar`},o):null,g=l?t.createElement("div",{className:`${u}-meta-title`},l):null,h=s?t.createElement("div",{className:`${u}-meta-description`},s):null,f=g||h?t.createElement("div",{className:`${u}-meta-detail`},g,h):null;return t.createElement("div",Object.assign({},c,{className:m}),p,f)},e.s(["Card",0,y],175712)},770914,908286,38243,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(343794),r=e.i(876556);function n(e){return["small","middle","large"].includes(e)}function a(e){return!!e&&"number"==typeof e&&!Number.isNaN(e)}e.s(["isPresetSize",()=>n,"isValidGapNumber",()=>a],908286);var o=e.i(242064),l=e.i(249616),s=e.i(372409),c=e.i(246422);let d=(0,c.genStyleHooks)(["Space","Addon"],e=>[(e=>{let{componentCls:t,borderRadius:i,paddingSM:r,colorBorder:n,paddingXS:a,fontSizeLG:o,fontSizeSM:l,borderRadiusLG:c,borderRadiusSM:d,colorBgContainerDisabled:u,lineWidth:m}=e;return{[t]:[{display:"inline-flex",alignItems:"center",gap:0,paddingInline:r,margin:0,background:u,borderWidth:m,borderStyle:"solid",borderColor:n,borderRadius:i,"&-large":{fontSize:o,borderRadius:c},"&-small":{paddingInline:a,borderRadius:d,fontSize:l},"&-compact-last-item":{borderEndStartRadius:0,borderStartStartRadius:0},"&-compact-first-item":{borderEndEndRadius:0,borderStartEndRadius:0},"&-compact-item:not(:first-child):not(:last-child)":{borderRadius:0},"&-compact-item:not(:last-child)":{borderInlineEndWidth:0}},(0,s.genCompactItemStyle)(e,{focus:!1})]}})(e)]);var u=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let m=t.default.forwardRef((e,r)=>{let{className:n,children:a,style:s,prefixCls:c}=e,m=u(e,["className","children","style","prefixCls"]),{getPrefixCls:p,direction:g}=t.default.useContext(o.ConfigContext),h=p("space-addon",c),[f,b,y]=d(h),{compactItemClassnames:v,compactSize:x}=(0,l.useCompactItemContext)(h,g),$=(0,i.default)(h,b,v,y,{[`${h}-${x}`]:x},n);return f(t.default.createElement("div",Object.assign({ref:r,className:$,style:s},m),a))}),p=t.default.createContext({latestIndex:0}),g=p.Provider,h=({className:e,index:i,children:r,split:n,style:a})=>{let{latestIndex:o}=t.useContext(p);return null==r?null:t.createElement(t.Fragment,null,t.createElement("div",{className:e,style:a},r),i{let t=(0,f.mergeToken)(e,{spaceGapSmallSize:e.paddingXS,spaceGapMiddleSize:e.padding,spaceGapLargeSize:e.paddingLG});return[(e=>{let{componentCls:t,antCls:i}=e;return{[t]:{display:"inline-flex","&-rtl":{direction:"rtl"},"&-vertical":{flexDirection:"column"},"&-align":{flexDirection:"column","&-center":{alignItems:"center"},"&-start":{alignItems:"flex-start"},"&-end":{alignItems:"flex-end"},"&-baseline":{alignItems:"baseline"}},[`${t}-item:empty`]:{display:"none"},[`${t}-item > ${i}-badge-not-a-wrapper:only-child`]:{display:"block"}}}})(t),(e=>{let{componentCls:t}=e;return{[t]:{"&-gap-row-small":{rowGap:e.spaceGapSmallSize},"&-gap-row-middle":{rowGap:e.spaceGapMiddleSize},"&-gap-row-large":{rowGap:e.spaceGapLargeSize},"&-gap-col-small":{columnGap:e.spaceGapSmallSize},"&-gap-col-middle":{columnGap:e.spaceGapMiddleSize},"&-gap-col-large":{columnGap:e.spaceGapLargeSize}}}})(t)]},()=>({}),{resetStyle:!1});var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v=t.forwardRef((e,l)=>{var s;let{getPrefixCls:c,direction:d,size:u,className:m,style:p,classNames:f,styles:v}=(0,o.useComponentConfig)("space"),{size:x=null!=u?u:"small",align:$,className:S,rootClassName:j,children:w,direction:O="horizontal",prefixCls:C,split:E,style:I,wrap:N=!1,classNames:k,styles:z}=e,L=y(e,["size","align","className","rootClassName","children","direction","prefixCls","split","style","wrap","classNames","styles"]),[M,R]=Array.isArray(x)?x:[x,x],P=n(R),T=n(M),_=a(R),G=a(M),B=(0,r.default)(w,{keepEmpty:!0}),A=void 0===$&&"horizontal"===O?"center":$,H=c("space",C),[U,W,D]=b(H),F=(0,i.default)(H,m,W,`${H}-${O}`,{[`${H}-rtl`]:"rtl"===d,[`${H}-align-${A}`]:A,[`${H}-gap-row-${R}`]:P,[`${H}-gap-col-${M}`]:T},S,j,D),K=(0,i.default)(`${H}-item`,null!=(s=null==k?void 0:k.item)?s:f.item),q=Object.assign(Object.assign({},v.item),null==z?void 0:z.item),V=B.map((e,i)=>{let r=(null==e?void 0:e.key)||`${K}-${i}`;return t.createElement(h,{className:K,key:r,index:i,split:E,style:q},e)}),X=t.useMemo(()=>({latestIndex:B.reduce((e,t,i)=>null!=t?i:e,0)}),[B]);if(0===B.length)return null;let J={};return N&&(J.flexWrap="wrap"),!T&&G&&(J.columnGap=M),!P&&_&&(J.rowGap=R),U(t.createElement("div",Object.assign({ref:l,className:F,style:Object.assign(Object.assign(Object.assign({},J),p),I)},L),t.createElement(g,{value:X},V)))});v.Compact=l.default,v.Addon=m,e.s(["default",0,v],38243),e.s(["Space",0,v],770914)},560445,e=>{"use strict";e.i(247167);var t=e.i(271645),i=e.i(201072),r=e.i(726289),n=e.i(864517),a=e.i(562901),o=e.i(779573),l=e.i(343794),s=e.i(361275),c=e.i(244009),d=e.i(611935),u=e.i(763731),m=e.i(242064);e.i(296059);var p=e.i(915654),g=e.i(183293),h=e.i(246422);let f=(e,t,i,r,n)=>({background:e,border:`${(0,p.unit)(r.lineWidth)} ${r.lineType} ${t}`,[`${n}-icon`]:{color:i}}),b=(0,h.genStyleHooks)("Alert",e=>[(e=>{let{componentCls:t,motionDurationSlow:i,marginXS:r,marginSM:n,fontSize:a,fontSizeLG:o,lineHeight:l,borderRadiusLG:s,motionEaseInOutCirc:c,withDescriptionIconSize:d,colorText:u,colorTextHeading:m,withDescriptionPadding:p,defaultPadding:h}=e;return{[t]:Object.assign(Object.assign({},(0,g.resetComponent)(e)),{position:"relative",display:"flex",alignItems:"center",padding:h,wordWrap:"break-word",borderRadius:s,[`&${t}-rtl`]:{direction:"rtl"},[`${t}-content`]:{flex:1,minWidth:0},[`${t}-icon`]:{marginInlineEnd:r,lineHeight:0},"&-description":{display:"none",fontSize:a,lineHeight:l},"&-message":{color:m},[`&${t}-motion-leave`]:{overflow:"hidden",opacity:1,transition:`max-height ${i} ${c}, opacity ${i} ${c}, padding-top ${i} ${c}, padding-bottom ${i} ${c}, - margin-bottom ${i} ${c}`},[`&${t}-motion-leave-active`]:{maxHeight:0,marginBottom:"0 !important",paddingTop:0,paddingBottom:0,opacity:0}}),[`${t}-with-description`]:{alignItems:"flex-start",padding:p,[`${t}-icon`]:{marginInlineEnd:n,fontSize:d,lineHeight:0},[`${t}-message`]:{display:"block",marginBottom:r,color:m,fontSize:o},[`${t}-description`]:{display:"block",color:u}},[`${t}-banner`]:{marginBottom:0,border:"0 !important",borderRadius:0}}})(e),(e=>{let{componentCls:t,colorSuccess:i,colorSuccessBorder:r,colorSuccessBg:n,colorWarning:a,colorWarningBorder:o,colorWarningBg:l,colorError:s,colorErrorBorder:c,colorErrorBg:d,colorInfo:u,colorInfoBorder:m,colorInfoBg:p}=e;return{[t]:{"&-success":f(n,r,i,e,t),"&-info":f(p,m,u,e,t),"&-warning":f(l,o,a,e,t),"&-error":Object.assign(Object.assign({},f(d,c,s,e,t)),{[`${t}-description > pre`]:{margin:0,padding:0}})}}})(e),(e=>{let{componentCls:t,iconCls:i,motionDurationMid:r,marginXS:n,fontSizeIcon:a,colorIcon:o,colorIconHover:l}=e;return{[t]:{"&-action":{marginInlineStart:n},[`${t}-close-icon`]:{marginInlineStart:n,padding:0,overflow:"hidden",fontSize:a,lineHeight:(0,p.unit)(a),backgroundColor:"transparent",border:"none",outline:"none",cursor:"pointer",[`${i}-close`]:{color:o,transition:`color ${r}`,"&:hover":{color:l}}},"&-close-text":{color:o,transition:`color ${r}`,"&:hover":{color:l}}}}})(e)],e=>({withDescriptionIconSize:e.fontSizeHeading3,defaultPadding:`${e.paddingContentVerticalSM}px 12px`,withDescriptionPadding:`${e.paddingMD}px ${e.paddingContentHorizontalLG}px`}));var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v={success:i.default,info:o.default,error:r.default,warning:a.default},x=e=>{let{icon:i,prefixCls:r,type:n}=e,a=v[n]||null;return i?(0,u.replaceElement)(i,t.createElement("span",{className:`${r}-icon`},i),()=>({className:(0,l.default)(`${r}-icon`,i.props.className)})):t.createElement(a,{className:`${r}-icon`})},$=e=>{let{isClosable:i,prefixCls:r,closeIcon:a,handleClose:o,ariaProps:l}=e,s=!0===a||void 0===a?t.createElement(n.default,null):a;return i?t.createElement("button",Object.assign({type:"button",onClick:o,className:`${r}-close-icon`,tabIndex:0},l),s):null},S=t.forwardRef((e,i)=>{let{description:r,prefixCls:n,message:a,banner:o,className:u,rootClassName:p,style:g,onMouseEnter:h,onMouseLeave:f,onClick:v,afterClose:S,showIcon:j,closable:w,closeText:O,closeIcon:C,action:E,id:I}=e,N=y(e,["description","prefixCls","message","banner","className","rootClassName","style","onMouseEnter","onMouseLeave","onClick","afterClose","showIcon","closable","closeText","closeIcon","action","id"]),[k,z]=t.useState(!1),L=t.useRef(null);t.useImperativeHandle(i,()=>({nativeElement:L.current}));let{getPrefixCls:M,direction:R,closable:P,closeIcon:T,className:_,style:G}=(0,m.useComponentConfig)("alert"),B=M("alert",n),[A,H,U]=b(B),W=t=>{var i;z(!0),null==(i=e.onClose)||i.call(e,t)},D=t.useMemo(()=>void 0!==e.type?e.type:o?"warning":"info",[e.type,o]),F=t.useMemo(()=>"object"==typeof w&&!!w.closeIcon||!!O||("boolean"==typeof w?w:!1!==C&&null!=C||!!P),[O,C,w,P]),K=!!o&&void 0===j||j,q=(0,l.default)(B,`${B}-${D}`,{[`${B}-with-description`]:!!r,[`${B}-no-icon`]:!K,[`${B}-banner`]:!!o,[`${B}-rtl`]:"rtl"===R},_,u,p,U,H),V=(0,c.default)(N,{aria:!0,data:!0}),X=t.useMemo(()=>"object"==typeof w&&w.closeIcon?w.closeIcon:O||(void 0!==C?C:"object"==typeof P&&P.closeIcon?P.closeIcon:T),[C,w,P,O,T]),J=t.useMemo(()=>{let e=null!=w?w:P;if("object"==typeof e){let{closeIcon:t}=e;return y(e,["closeIcon"])}return{}},[w,P]);return A(t.createElement(s.default,{visible:!k,motionName:`${B}-motion`,motionAppear:!1,motionEnter:!1,onLeaveStart:e=>({maxHeight:e.offsetHeight}),onLeaveEnd:S},({className:i,style:n},o)=>t.createElement("div",Object.assign({id:I,ref:(0,d.composeRef)(L,o),"data-show":!k,className:(0,l.default)(q,i),style:Object.assign(Object.assign(Object.assign({},G),g),n),onMouseEnter:h,onMouseLeave:f,onClick:v,role:"alert"},V),K?t.createElement(x,{description:r,icon:e.icon,prefixCls:B,type:D}):null,t.createElement("div",{className:`${B}-content`},a?t.createElement("div",{className:`${B}-message`},a):null,r?t.createElement("div",{className:`${B}-description`},r):null),E?t.createElement("div",{className:`${B}-action`},E):null,t.createElement($,{isClosable:F,prefixCls:B,closeIcon:X,handleClose:W,ariaProps:J}))))});var j=e.i(278409),w=e.i(233848),O=e.i(487806),C=e.i(479671),E=e.i(480002),I=e.i(868917);let N=function(e){function i(){var e,t,r;return(0,j.default)(this,i),t=i,r=arguments,t=(0,O.default)(t),(e=(0,E.default)(this,(0,C.default)()?Reflect.construct(t,r||[],(0,O.default)(this).constructor):t.apply(this,r))).state={error:void 0,info:{componentStack:""}},e}return(0,I.default)(i,e),(0,w.default)(i,[{key:"componentDidCatch",value:function(e,t){this.setState({error:e,info:t})}},{key:"render",value:function(){let{message:e,description:i,id:r,children:n}=this.props,{error:a,info:o}=this.state,l=(null==o?void 0:o.componentStack)||null,s=void 0===e?(a||"").toString():e;return a?t.createElement(S,{id:r,type:"error",message:s,description:t.createElement("pre",{style:{fontSize:"0.9em",overflowX:"auto"}},void 0===i?l:i)}):n}}])}(t.Component);S.ErrorBoundary=N,e.s(["Alert",0,S],560445)},936578,571303,e=>{"use strict";var t=e.i(843476),i=e.i(115504),r=e.i(271645);function n({className:e="",...n}){var a,o;let l=(0,r.useId)();return a=()=>{let e=document.getAnimations().filter(e=>e instanceof CSSAnimation&&"spin"===e.animationName),t=e.find(e=>e.effect.target?.getAttribute("data-spinner-id")===l),i=e.find(e=>e.effect instanceof KeyframeEffect&&e.effect.target?.getAttribute("data-spinner-id")!==l);t&&i&&(t.currentTime=i.currentTime)},o=[l],(0,r.useLayoutEffect)(a,o),(0,t.jsxs)("svg",{"data-spinner-id":l,className:(0,i.cx)("pointer-events-none size-12 animate-spin text-current",e),fill:"none",viewBox:"0 0 24 24",...n,children:[(0,t.jsx)("circle",{className:"opacity-25",cx:"12",cy:"12",r:"10",stroke:"currentColor",strokeWidth:"4"}),(0,t.jsx)("path",{className:"opacity-75",fill:"currentColor",d:"M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"})]})}function a(){return(0,t.jsxs)("div",{className:(0,i.cx)("h-screen","flex items-center justify-center gap-4"),children:[(0,t.jsx)("div",{className:"text-lg font-medium py-2 pr-4 border-r border-r-gray-200",children:"🚅 LiteLLM"}),(0,t.jsxs)("div",{className:"flex items-center justify-center gap-2",children:[(0,t.jsx)(n,{className:"size-4"}),(0,t.jsx)("span",{className:"text-gray-600 text-sm",children:"Loading..."})]})]})}e.s(["UiLoadingSpinner",()=>n],571303),e.s(["default",()=>a],936578)},594542,e=>{"use strict";var t=e.i(843476),i=e.i(954616),r=e.i(764205),n=e.i(612256),a=e.i(936578),o=e.i(268004),l=e.i(161281),s=e.i(321836),c=e.i(827252),d=e.i(295320),u=e.i(560445),m=e.i(464571),p=e.i(175712),g=e.i(808613),h=e.i(311451),f=e.i(282786),b=e.i(199133),y=e.i(770914),v=e.i(898586),x=e.i(618566),$=e.i(271645),S=e.i(283713);function j(){let[e,j]=(0,$.useState)(""),[w,O]=(0,$.useState)(""),[C,E]=(0,$.useState)(!0),{data:I,isLoading:N}=(0,n.useUIConfig)(),k=(0,i.useMutation)({mutationFn:async({username:e,password:t,useV3:i})=>await (0,r.loginCall)(e,t,i)}),z=(0,x.useRouter)(),{workers:L,selectWorker:M}=(0,S.useWorker)(),[R,P]=(0,$.useState)(null);(0,$.useEffect)(()=>{let e=new URLSearchParams(window.location.search).get("worker");e&&P(e)},[]),(0,$.useEffect)(()=>{if(N)return;if(I&&I.admin_ui_disabled)return void E(!1);let e=new URLSearchParams(window.location.search),t=e.get("code"),i=t&&/^[a-zA-Z0-9._~+/=-]+$/.test(t)?t:null;if(i){let t=localStorage.getItem("litellm_worker_url"),n=t&&/^https?:\/\/.+/.test(t)?t:null;(0,r.exchangeLoginCode)(i,n).then(()=>{e.delete("code");let t=e.toString();window.history.replaceState(null,"",window.location.pathname+(t?`?${t}`:"")),z.replace("/ui/?login=success")});return}if(e.has("worker")&&I?.is_control_plane){(0,o.clearTokenCookies)(),E(!1);return}let n=(0,o.getCookieFromDocument)("token");if(n&&!(0,l.isJwtExpired)(n)){let e=(0,s.consumeReturnUrl)();e?z.replace(e):z.replace("/ui");return}if(I&&I.auto_redirect_to_sso){let e=(0,s.getReturnUrl)(),t=`${(0,r.getProxyBaseUrl)()}/sso/key/generate`;e&&(0,s.isValidReturnUrl)(e)&&(t+=`?redirect_to=${encodeURIComponent(e)}`),z.push(t);return}E(!1)},[N,z,I]);let T=k.error instanceof Error?k.error.message:null,_=k.isPending,{Title:G,Text:B,Paragraph:A}=v.Typography;return N||C?(0,t.jsx)(a.default,{}):I&&I.admin_ui_disabled?(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsx)(p.Card,{className:"w-full max-w-lg shadow-md",children:(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsx)(u.Alert,{message:"Admin UI Disabled",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(A,{className:"text-sm",children:"The Admin UI has been disabled by the administrator. To re-enable it, please update the following environment variable:"}),(0,t.jsx)(A,{className:"text-sm",children:(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"DISABLE_ADMIN_UI=False"})})]}),type:"warning",showIcon:!0})]})})}):(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsxs)(p.Card,{className:"w-full max-w-lg shadow-md",children:[(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsxs)("div",{className:"text-center",children:[(0,t.jsx)(G,{level:3,children:"Login"}),(0,t.jsx)(B,{type:"secondary",children:"Access your LiteLLM Admin UI."})]}),(0,t.jsx)(u.Alert,{message:"Default Credentials",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(A,{className:"text-sm",children:["By default, Username is ",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"admin"})," and Password is your set LiteLLM Proxy",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"MASTER_KEY"}),"."]}),(0,t.jsxs)(A,{className:"text-sm",children:["Need to set UI credentials or SSO?"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/ui",target:"_blank",rel:"noopener noreferrer",children:"Check the documentation"}),"."]})]}),type:"info",icon:(0,t.jsx)(c.InfoCircleOutlined,{}),showIcon:!0}),T&&(0,t.jsx)(u.Alert,{message:T,type:"error",showIcon:!0}),(0,t.jsxs)(g.Form,{onFinish:()=>{let t=L.find(e=>e.worker_id===R);t&&(0,r.switchToWorkerUrl)(t.url),k.mutate({username:e,password:w,useV3:!!t},{onSuccess:e=>{if(t)M(t.worker_id),z.push("/ui/?login=success");else{let t=(0,s.consumeReturnUrl)();t?z.push(t):z.push(e.redirect_url)}},onError:()=>{t&&(0,r.switchToWorkerUrl)(null)}})},layout:"vertical",requiredMark:!1,children:[I?.is_control_plane&&L.length>0&&(0,t.jsx)(g.Form.Item,{label:"Worker",style:{marginBottom:16},children:(0,t.jsx)(b.Select,{value:R||void 0,onChange:e=>P(e),placeholder:"Choose a worker to connect to",size:"large",suffixIcon:(0,t.jsx)(d.CloudServerOutlined,{}),options:L.map(e=>({label:e.name,value:e.worker_id}))})}),(0,t.jsx)(g.Form.Item,{label:"Username",name:"username",rules:[{required:!0,message:"Please enter your username"}],children:(0,t.jsx)(h.Input,{placeholder:"Enter your username",autoComplete:"username",value:e,onChange:e=>j(e.target.value),disabled:_,size:"large",className:"rounded-md border-gray-300"})}),(0,t.jsx)(g.Form.Item,{label:"Password",name:"password",rules:[{required:!0,message:"Please enter your password"}],children:(0,t.jsx)(h.Input.Password,{placeholder:"Enter your password",autoComplete:"current-password",value:w,onChange:e=>O(e.target.value),disabled:_,size:"large"})}),(0,t.jsx)(g.Form.Item,{children:(0,t.jsx)(m.Button,{type:"primary",htmlType:"submit",loading:_,disabled:_,block:!0,size:"large",children:_?"Logging in...":"Login"})}),(0,t.jsx)(g.Form.Item,{children:I?.sso_configured?(0,t.jsx)(m.Button,{disabled:_||!!R&&0===L.length,onClick:()=>{let e=L.find(e=>e.worker_id===R);e&&(localStorage.setItem("litellm_selected_worker_id",R),(0,r.switchToWorkerUrl)(e.url));let t=e?.url??(0,r.getProxyBaseUrl)(),i=encodeURIComponent(window.location.origin+"/ui/login");z.push(`${t}/sso/key/generate?return_to=${i}`)},block:!0,size:"large",children:"Login with SSO"}):(0,t.jsx)(f.Popover,{content:"Please configure SSO to log in with SSO.",trigger:"hover",children:(0,t.jsx)(m.Button,{disabled:!0,block:!0,size:"large",children:"Login with SSO"})})})]})]}),I?.sso_configured&&(0,t.jsx)(u.Alert,{type:"info",showIcon:!0,closable:!0,message:(0,t.jsxs)(B,{children:["Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading this page. To re-enable auto-redirect-to-SSO, set ",(0,t.jsx)(B,{code:!0,children:"AUTO_REDIRECT_UI_LOGIN_TO_SSO=true"})," in your environment configuration."]})})]})})}e.s(["default",0,function(){return(0,t.jsx)(j,{})}],594542)}]); \ No newline at end of file + margin-bottom ${i} ${c}`},[`&${t}-motion-leave-active`]:{maxHeight:0,marginBottom:"0 !important",paddingTop:0,paddingBottom:0,opacity:0}}),[`${t}-with-description`]:{alignItems:"flex-start",padding:p,[`${t}-icon`]:{marginInlineEnd:n,fontSize:d,lineHeight:0},[`${t}-message`]:{display:"block",marginBottom:r,color:m,fontSize:o},[`${t}-description`]:{display:"block",color:u}},[`${t}-banner`]:{marginBottom:0,border:"0 !important",borderRadius:0}}})(e),(e=>{let{componentCls:t,colorSuccess:i,colorSuccessBorder:r,colorSuccessBg:n,colorWarning:a,colorWarningBorder:o,colorWarningBg:l,colorError:s,colorErrorBorder:c,colorErrorBg:d,colorInfo:u,colorInfoBorder:m,colorInfoBg:p}=e;return{[t]:{"&-success":f(n,r,i,e,t),"&-info":f(p,m,u,e,t),"&-warning":f(l,o,a,e,t),"&-error":Object.assign(Object.assign({},f(d,c,s,e,t)),{[`${t}-description > pre`]:{margin:0,padding:0}})}}})(e),(e=>{let{componentCls:t,iconCls:i,motionDurationMid:r,marginXS:n,fontSizeIcon:a,colorIcon:o,colorIconHover:l}=e;return{[t]:{"&-action":{marginInlineStart:n},[`${t}-close-icon`]:{marginInlineStart:n,padding:0,overflow:"hidden",fontSize:a,lineHeight:(0,p.unit)(a),backgroundColor:"transparent",border:"none",outline:"none",cursor:"pointer",[`${i}-close`]:{color:o,transition:`color ${r}`,"&:hover":{color:l}}},"&-close-text":{color:o,transition:`color ${r}`,"&:hover":{color:l}}}}})(e)],e=>({withDescriptionIconSize:e.fontSizeHeading3,defaultPadding:`${e.paddingContentVerticalSM}px 12px`,withDescriptionPadding:`${e.paddingMD}px ${e.paddingContentHorizontalLG}px`}));var y=function(e,t){var i={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(i[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var n=0,r=Object.getOwnPropertySymbols(e);nt.indexOf(r[n])&&Object.prototype.propertyIsEnumerable.call(e,r[n])&&(i[r[n]]=e[r[n]]);return i};let v={success:i.default,info:o.default,error:r.default,warning:a.default},x=e=>{let{icon:i,prefixCls:r,type:n}=e,a=v[n]||null;return i?(0,u.replaceElement)(i,t.createElement("span",{className:`${r}-icon`},i),()=>({className:(0,l.default)(`${r}-icon`,i.props.className)})):t.createElement(a,{className:`${r}-icon`})},$=e=>{let{isClosable:i,prefixCls:r,closeIcon:a,handleClose:o,ariaProps:l}=e,s=!0===a||void 0===a?t.createElement(n.default,null):a;return i?t.createElement("button",Object.assign({type:"button",onClick:o,className:`${r}-close-icon`,tabIndex:0},l),s):null},S=t.forwardRef((e,i)=>{let{description:r,prefixCls:n,message:a,banner:o,className:u,rootClassName:p,style:g,onMouseEnter:h,onMouseLeave:f,onClick:v,afterClose:S,showIcon:j,closable:w,closeText:O,closeIcon:C,action:E,id:I}=e,N=y(e,["description","prefixCls","message","banner","className","rootClassName","style","onMouseEnter","onMouseLeave","onClick","afterClose","showIcon","closable","closeText","closeIcon","action","id"]),[k,z]=t.useState(!1),L=t.useRef(null);t.useImperativeHandle(i,()=>({nativeElement:L.current}));let{getPrefixCls:M,direction:R,closable:P,closeIcon:T,className:_,style:G}=(0,m.useComponentConfig)("alert"),B=M("alert",n),[A,H,U]=b(B),W=t=>{var i;z(!0),null==(i=e.onClose)||i.call(e,t)},D=t.useMemo(()=>void 0!==e.type?e.type:o?"warning":"info",[e.type,o]),F=t.useMemo(()=>"object"==typeof w&&!!w.closeIcon||!!O||("boolean"==typeof w?w:!1!==C&&null!=C||!!P),[O,C,w,P]),K=!!o&&void 0===j||j,q=(0,l.default)(B,`${B}-${D}`,{[`${B}-with-description`]:!!r,[`${B}-no-icon`]:!K,[`${B}-banner`]:!!o,[`${B}-rtl`]:"rtl"===R},_,u,p,U,H),V=(0,c.default)(N,{aria:!0,data:!0}),X=t.useMemo(()=>"object"==typeof w&&w.closeIcon?w.closeIcon:O||(void 0!==C?C:"object"==typeof P&&P.closeIcon?P.closeIcon:T),[C,w,P,O,T]),J=t.useMemo(()=>{let e=null!=w?w:P;if("object"==typeof e){let{closeIcon:t}=e;return y(e,["closeIcon"])}return{}},[w,P]);return A(t.createElement(s.default,{visible:!k,motionName:`${B}-motion`,motionAppear:!1,motionEnter:!1,onLeaveStart:e=>({maxHeight:e.offsetHeight}),onLeaveEnd:S},({className:i,style:n},o)=>t.createElement("div",Object.assign({id:I,ref:(0,d.composeRef)(L,o),"data-show":!k,className:(0,l.default)(q,i),style:Object.assign(Object.assign(Object.assign({},G),g),n),onMouseEnter:h,onMouseLeave:f,onClick:v,role:"alert"},V),K?t.createElement(x,{description:r,icon:e.icon,prefixCls:B,type:D}):null,t.createElement("div",{className:`${B}-content`},a?t.createElement("div",{className:`${B}-message`},a):null,r?t.createElement("div",{className:`${B}-description`},r):null),E?t.createElement("div",{className:`${B}-action`},E):null,t.createElement($,{isClosable:F,prefixCls:B,closeIcon:X,handleClose:W,ariaProps:J}))))});var j=e.i(278409),w=e.i(233848),O=e.i(487806),C=e.i(479671),E=e.i(480002),I=e.i(868917);let N=function(e){function i(){var e,t,r;return(0,j.default)(this,i),t=i,r=arguments,t=(0,O.default)(t),(e=(0,E.default)(this,(0,C.default)()?Reflect.construct(t,r||[],(0,O.default)(this).constructor):t.apply(this,r))).state={error:void 0,info:{componentStack:""}},e}return(0,I.default)(i,e),(0,w.default)(i,[{key:"componentDidCatch",value:function(e,t){this.setState({error:e,info:t})}},{key:"render",value:function(){let{message:e,description:i,id:r,children:n}=this.props,{error:a,info:o}=this.state,l=(null==o?void 0:o.componentStack)||null,s=void 0===e?(a||"").toString():e;return a?t.createElement(S,{id:r,type:"error",message:s,description:t.createElement("pre",{style:{fontSize:"0.9em",overflowX:"auto"}},void 0===i?l:i)}):n}}])}(t.Component);S.ErrorBoundary=N,e.s(["Alert",0,S],560445)},936578,571303,e=>{"use strict";var t=e.i(843476),i=e.i(115504),r=e.i(271645);function n({className:e="",...n}){var a,o;let l=(0,r.useId)();return a=()=>{let e=document.getAnimations().filter(e=>e instanceof CSSAnimation&&"spin"===e.animationName),t=e.find(e=>e.effect.target?.getAttribute("data-spinner-id")===l),i=e.find(e=>e.effect instanceof KeyframeEffect&&e.effect.target?.getAttribute("data-spinner-id")!==l);t&&i&&(t.currentTime=i.currentTime)},o=[l],(0,r.useLayoutEffect)(a,o),(0,t.jsxs)("svg",{"data-spinner-id":l,className:(0,i.cx)("pointer-events-none size-12 animate-spin text-current",e),fill:"none",viewBox:"0 0 24 24",...n,children:[(0,t.jsx)("circle",{className:"opacity-25",cx:"12",cy:"12",r:"10",stroke:"currentColor",strokeWidth:"4"}),(0,t.jsx)("path",{className:"opacity-75",fill:"currentColor",d:"M4 12a8 8 0 018-8V0C5.373 0 0 5.373 0 12h4zm2 5.291A7.962 7.962 0 014 12H0c0 3.042 1.135 5.824 3 7.938l3-2.647z"})]})}function a(){return(0,t.jsxs)("div",{className:(0,i.cx)("h-screen","flex items-center justify-center gap-4"),children:[(0,t.jsx)("div",{className:"text-lg font-medium py-2 pr-4 border-r border-r-gray-200",children:"🚅 LiteLLM"}),(0,t.jsxs)("div",{className:"flex items-center justify-center gap-2",children:[(0,t.jsx)(n,{className:"size-4"}),(0,t.jsx)("span",{className:"text-gray-600 text-sm",children:"Loading..."})]})]})}e.s(["UiLoadingSpinner",()=>n],571303),e.s(["default",()=>a],936578)},594542,e=>{"use strict";var t=e.i(843476),i=e.i(954616),r=e.i(602869),n=e.i(612256),a=e.i(936578),o=e.i(268004),l=e.i(161281),s=e.i(321836),c=e.i(827252),d=e.i(295320),u=e.i(560445),m=e.i(464571),p=e.i(175712),g=e.i(808613),h=e.i(311451),f=e.i(282786),b=e.i(199133),y=e.i(770914),v=e.i(898586),x=e.i(618566),$=e.i(271645),S=e.i(283713);function j(){let[e,j]=(0,$.useState)(""),[w,O]=(0,$.useState)(""),[C,E]=(0,$.useState)(!0),{data:I,isLoading:N}=(0,n.useUIConfig)(),k=(0,i.useMutation)({mutationFn:async({username:e,password:t,useV3:i})=>await (0,r.loginCall)(e,t,i)}),z=(0,x.useRouter)(),{workers:L,selectWorker:M}=(0,S.useWorker)(),[R,P]=(0,$.useState)(null);(0,$.useEffect)(()=>{let e=new URLSearchParams(window.location.search).get("worker");e&&P(e)},[]),(0,$.useEffect)(()=>{if(N)return;if(I&&I.admin_ui_disabled)return void E(!1);let e=new URLSearchParams(window.location.search),t=e.get("code"),i=t&&/^[a-zA-Z0-9._~+/=-]+$/.test(t)?t:null;if(i){let t=localStorage.getItem("litellm_worker_url"),n=t&&/^https?:\/\/.+/.test(t)?t:null;(0,r.exchangeLoginCode)(i,n).then(()=>{e.delete("code");let t=e.toString();window.history.replaceState(null,"",window.location.pathname+(t?`?${t}`:"")),z.replace("/ui/?login=success")});return}if(e.has("worker")&&I?.is_control_plane){(0,o.clearTokenCookies)(),E(!1);return}let n=(0,o.getCookieFromDocument)("token");if(n&&!(0,l.isJwtExpired)(n)){let e=(0,s.consumeReturnUrl)();e?z.replace(e):z.replace("/ui");return}if(I&&I.auto_redirect_to_sso){let e=(0,s.getReturnUrl)(),t=`${(0,r.getProxyBaseUrl)()}/sso/key/generate`;e&&(0,s.isValidReturnUrl)(e)&&(t+=`?redirect_to=${encodeURIComponent(e)}`),z.push(t);return}E(!1)},[N,z,I]);let T=k.error instanceof Error?k.error.message:null,_=k.isPending,{Title:G,Text:B,Paragraph:A}=v.Typography;return N||C?(0,t.jsx)(a.default,{}):I&&I.admin_ui_disabled?(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsx)(p.Card,{className:"w-full max-w-lg shadow-md",children:(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsx)(u.Alert,{message:"Admin UI Disabled",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsx)(A,{className:"text-sm",children:"The Admin UI has been disabled by the administrator. To re-enable it, please update the following environment variable:"}),(0,t.jsx)(A,{className:"text-sm",children:(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"DISABLE_ADMIN_UI=False"})})]}),type:"warning",showIcon:!0})]})})}):(0,t.jsx)("div",{className:"min-h-screen flex items-center justify-center bg-gray-50",children:(0,t.jsxs)(p.Card,{className:"w-full max-w-lg shadow-md",children:[(0,t.jsxs)(y.Space,{direction:"vertical",size:"middle",className:"w-full",children:[(0,t.jsx)("div",{className:"text-center",children:(0,t.jsx)(G,{level:2,children:"🚅 LiteLLM"})}),(0,t.jsxs)("div",{className:"text-center",children:[(0,t.jsx)(G,{level:3,children:"Login"}),(0,t.jsx)(B,{type:"secondary",children:"Access your LiteLLM Admin UI."})]}),(0,t.jsx)(u.Alert,{message:"Default Credentials",description:(0,t.jsxs)(t.Fragment,{children:[(0,t.jsxs)(A,{className:"text-sm",children:["By default, Username is ",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"admin"})," and Password is your set LiteLLM Proxy",(0,t.jsx)("code",{className:"bg-gray-100 px-1 py-0.5 rounded text-xs",children:"MASTER_KEY"}),"."]}),(0,t.jsxs)(A,{className:"text-sm",children:["Need to set UI credentials or SSO?"," ",(0,t.jsx)("a",{href:"https://docs.litellm.ai/docs/proxy/ui",target:"_blank",rel:"noopener noreferrer",children:"Check the documentation"}),"."]})]}),type:"info",icon:(0,t.jsx)(c.InfoCircleOutlined,{}),showIcon:!0}),T&&(0,t.jsx)(u.Alert,{message:T,type:"error",showIcon:!0}),(0,t.jsxs)(g.Form,{onFinish:()=>{let t=L.find(e=>e.worker_id===R);t&&(0,r.switchToWorkerUrl)(t.url),k.mutate({username:e,password:w,useV3:!!t},{onSuccess:e=>{if(t)M(t.worker_id),z.push("/ui/?login=success");else{let t=(0,s.consumeReturnUrl)();t?z.push(t):z.push(e.redirect_url)}},onError:()=>{t&&(0,r.switchToWorkerUrl)(null)}})},layout:"vertical",requiredMark:!1,children:[I?.is_control_plane&&L.length>0&&(0,t.jsx)(g.Form.Item,{label:"Worker",style:{marginBottom:16},children:(0,t.jsx)(b.Select,{value:R||void 0,onChange:e=>P(e),placeholder:"Choose a worker to connect to",size:"large",suffixIcon:(0,t.jsx)(d.CloudServerOutlined,{}),options:L.map(e=>({label:e.name,value:e.worker_id}))})}),(0,t.jsx)(g.Form.Item,{label:"Username",name:"username",rules:[{required:!0,message:"Please enter your username"}],children:(0,t.jsx)(h.Input,{placeholder:"Enter your username",autoComplete:"username",value:e,onChange:e=>j(e.target.value),disabled:_,size:"large",className:"rounded-md border-gray-300"})}),(0,t.jsx)(g.Form.Item,{label:"Password",name:"password",rules:[{required:!0,message:"Please enter your password"}],children:(0,t.jsx)(h.Input.Password,{placeholder:"Enter your password",autoComplete:"current-password",value:w,onChange:e=>O(e.target.value),disabled:_,size:"large"})}),(0,t.jsx)(g.Form.Item,{children:(0,t.jsx)(m.Button,{type:"primary",htmlType:"submit",loading:_,disabled:_,block:!0,size:"large",children:_?"Logging in...":"Login"})}),(0,t.jsx)(g.Form.Item,{children:I?.sso_configured?(0,t.jsx)(m.Button,{disabled:_||!!R&&0===L.length,onClick:()=>{let e=L.find(e=>e.worker_id===R);e&&(localStorage.setItem("litellm_selected_worker_id",R),(0,r.switchToWorkerUrl)(e.url));let t=e?.url??(0,r.getProxyBaseUrl)(),i=encodeURIComponent(window.location.origin+"/ui/login");z.push(`${t}/sso/key/generate?return_to=${i}`)},block:!0,size:"large",children:"Login with SSO"}):(0,t.jsx)(f.Popover,{content:"Please configure SSO to log in with SSO.",trigger:"hover",children:(0,t.jsx)(m.Button,{disabled:!0,block:!0,size:"large",children:"Login with SSO"})})})]})]}),I?.sso_configured&&(0,t.jsx)(u.Alert,{type:"info",showIcon:!0,closable:!0,message:(0,t.jsxs)(B,{children:["Single Sign-On (SSO) is enabled. LiteLLM no longer automatically redirects to the SSO login flow upon loading this page. To re-enable auto-redirect-to-SSO, set"," ",(0,t.jsx)(B,{code:!0,children:"AUTO_REDIRECT_UI_LOGIN_TO_SSO=true"})," in your environment configuration."]})})]})})}e.s(["default",0,function(){return(0,t.jsx)(j,{})}],594542)}]); \ No newline at end of file diff --git a/litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js b/litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js similarity index 66% rename from litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js rename to litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js index 091c3b7011..3b6538f90e 100644 --- a/litellm/proxy/_experimental/out/_next/static/chunks/36d1c027ba991a4f.js +++ b/litellm/proxy/_experimental/out/_next/static/chunks/10757c2146f43db4.js @@ -1,6 +1,6 @@ (globalThis.TURBOPACK||(globalThis.TURBOPACK=[])).push(["object"==typeof document?document.currentScript:void 0,127952,869216,368869,e=>{"use strict";var t=e.i(843476),n=e.i(560445),r=e.i(175712);e.i(247167);var l=e.i(271645),a=e.i(343794),o=e.i(908206),i=e.i(242064),s=e.i(517455),d=e.i(150073);let c={xxl:3,xl:3,lg:3,md:3,sm:2,xs:1},u=l.default.createContext({});var f=e.i(876556),m=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n},p=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let g=e=>{let{itemPrefixCls:t,component:n,span:r,className:o,style:i,labelStyle:s,contentStyle:d,bordered:c,label:f,content:m,colon:p,type:g,styles:h}=e,{classNames:x}=l.useContext(u),v=Object.assign(Object.assign({},s),null==h?void 0:h.label),b=Object.assign(Object.assign({},d),null==h?void 0:h.content);if(c)return l.createElement(n,{colSpan:r,style:i,className:(0,a.default)(o,{[`${t}-item-${g}`]:"label"===g||"content"===g,[null==x?void 0:x.label]:(null==x?void 0:x.label)&&"label"===g,[null==x?void 0:x.content]:(null==x?void 0:x.content)&&"content"===g})},null!=f&&l.createElement("span",{style:v},f),null!=m&&l.createElement("span",{style:b},m));return l.createElement(n,{colSpan:r,style:i,className:(0,a.default)(`${t}-item`,o)},l.createElement("div",{className:`${t}-item-container`},null!=f&&l.createElement("span",{style:v,className:(0,a.default)(`${t}-item-label`,null==x?void 0:x.label,{[`${t}-item-no-colon`]:!p})},f),null!=m&&l.createElement("span",{style:b,className:(0,a.default)(`${t}-item-content`,null==x?void 0:x.content)},m)))};function h(e,{colon:t,prefixCls:n,bordered:r},{component:a,type:o,showLabel:i,showContent:s,labelStyle:d,contentStyle:c,styles:u}){return e.map(({label:e,children:f,prefixCls:m=n,className:p,style:h,labelStyle:x,contentStyle:v,span:b=1,key:y,styles:w},j)=>"string"==typeof a?l.createElement(g,{key:`${o}-${y||j}`,className:p,style:h,styles:{label:Object.assign(Object.assign(Object.assign(Object.assign({},d),null==u?void 0:u.label),x),null==w?void 0:w.label),content:Object.assign(Object.assign(Object.assign(Object.assign({},c),null==u?void 0:u.content),v),null==w?void 0:w.content)},span:b,colon:t,component:a,itemPrefixCls:m,bordered:r,label:i?e:null,content:s?f:null,type:o}):[l.createElement(g,{key:`label-${y||j}`,className:p,style:Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},d),null==u?void 0:u.label),h),x),null==w?void 0:w.label),span:1,colon:t,component:a[0],itemPrefixCls:m,bordered:r,label:e,type:"label"}),l.createElement(g,{key:`content-${y||j}`,className:p,style:Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},c),null==u?void 0:u.content),h),v),null==w?void 0:w.content),span:2*b-1,component:a[1],itemPrefixCls:m,bordered:r,content:f,type:"content"})])}let x=e=>{let t=l.useContext(u),{prefixCls:n,vertical:r,row:a,index:o,bordered:i}=e;return r?l.createElement(l.Fragment,null,l.createElement("tr",{key:`label-${o}`,className:`${n}-row`},h(a,e,Object.assign({component:"th",type:"label",showLabel:!0},t))),l.createElement("tr",{key:`content-${o}`,className:`${n}-row`},h(a,e,Object.assign({component:"td",type:"content",showContent:!0},t)))):l.createElement("tr",{key:o,className:`${n}-row`},h(a,e,Object.assign({component:i?["th","td"]:"td",type:"item",showLabel:!0,showContent:!0},t)))};e.i(296059);var v=e.i(915654),b=e.i(183293),y=e.i(246422),w=e.i(838378);let j=(0,y.genStyleHooks)("Descriptions",e=>(e=>{let{componentCls:t,extraColor:n,itemPaddingBottom:r,itemPaddingEnd:l,colonMarginRight:a,colonMarginLeft:o,titleMarginBottom:i}=e;return{[t]:Object.assign(Object.assign(Object.assign({},(0,b.resetComponent)(e)),(e=>{let{componentCls:t,labelBg:n}=e;return{[`&${t}-bordered`]:{[`> ${t}-view`]:{border:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"> table":{tableLayout:"auto"},[`${t}-row`]:{borderBottom:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"&:first-child":{"> th:first-child, > td:first-child":{borderStartStartRadius:e.borderRadiusLG}},"&:last-child":{borderBottom:"none","> th:first-child, > td:first-child":{borderEndStartRadius:e.borderRadiusLG}},[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.padding)} ${(0,v.unit)(e.paddingLG)}`,borderInlineEnd:`${(0,v.unit)(e.lineWidth)} ${e.lineType} ${e.colorSplit}`,"&:last-child":{borderInlineEnd:"none"}},[`> ${t}-item-label`]:{color:e.colorTextSecondary,backgroundColor:n,"&::after":{display:"none"}}}},[`&${t}-middle`]:{[`${t}-row`]:{[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.paddingSM)} ${(0,v.unit)(e.paddingLG)}`}}},[`&${t}-small`]:{[`${t}-row`]:{[`> ${t}-item-label, > ${t}-item-content`]:{padding:`${(0,v.unit)(e.paddingXS)} ${(0,v.unit)(e.padding)}`}}}}}})(e)),{"&-rtl":{direction:"rtl"},[`${t}-header`]:{display:"flex",alignItems:"center",marginBottom:i},[`${t}-title`]:Object.assign(Object.assign({},b.textEllipsis),{flex:"auto",color:e.titleColor,fontWeight:e.fontWeightStrong,fontSize:e.fontSizeLG,lineHeight:e.lineHeightLG}),[`${t}-extra`]:{marginInlineStart:"auto",color:n,fontSize:e.fontSize},[`${t}-view`]:{width:"100%",borderRadius:e.borderRadiusLG,table:{width:"100%",tableLayout:"fixed",borderCollapse:"collapse"}},[`${t}-row`]:{"> th, > td":{paddingBottom:r,paddingInlineEnd:l},"> th:last-child, > td:last-child":{paddingInlineEnd:0},"&:last-child":{borderBottom:"none","> th, > td":{paddingBottom:0}}},[`${t}-item-label`]:{color:e.labelColor,fontWeight:"normal",fontSize:e.fontSize,lineHeight:e.lineHeight,textAlign:"start","&::after":{content:'":"',position:"relative",top:-.5,marginInline:`${(0,v.unit)(o)} ${(0,v.unit)(a)}`},[`&${t}-item-no-colon::after`]:{content:'""'}},[`${t}-item-no-label`]:{"&::after":{margin:0,content:'""'}},[`${t}-item-content`]:{display:"table-cell",flex:1,color:e.contentColor,fontSize:e.fontSize,lineHeight:e.lineHeight,wordBreak:"break-word",overflowWrap:"break-word"},[`${t}-item`]:{paddingBottom:0,verticalAlign:"top","&-container":{display:"flex",[`${t}-item-label`]:{display:"inline-flex",alignItems:"baseline"},[`${t}-item-content`]:{display:"inline-flex",alignItems:"baseline",minWidth:"1em"}}},"&-middle":{[`${t}-row`]:{"> th, > td":{paddingBottom:e.paddingSM}}},"&-small":{[`${t}-row`]:{"> th, > td":{paddingBottom:e.paddingXS}}}})}})((0,w.mergeToken)(e,{})),e=>({labelBg:e.colorFillAlter,labelColor:e.colorTextTertiary,titleColor:e.colorText,titleMarginBottom:e.fontSizeSM*e.lineHeightSM,itemPaddingBottom:e.padding,itemPaddingEnd:e.padding,colonMarginRight:e.marginXS,colonMarginLeft:e.marginXXS/2,contentColor:e.colorText,extraColor:e.colorText}));var k=function(e,t){var n={};for(var r in e)Object.prototype.hasOwnProperty.call(e,r)&&0>t.indexOf(r)&&(n[r]=e[r]);if(null!=e&&"function"==typeof Object.getOwnPropertySymbols)for(var l=0,r=Object.getOwnPropertySymbols(e);lt.indexOf(r[l])&&Object.prototype.propertyIsEnumerable.call(e,r[l])&&(n[r[l]]=e[r[l]]);return n};let C=e=>{let t,{prefixCls:n,title:r,extra:g,column:h,colon:v=!0,bordered:b,layout:y,children:w,className:C,rootClassName:S,style:N,size:E,labelStyle:_,contentStyle:O,styles:$,items:T,classNames:I}=e,P=k(e,["prefixCls","title","extra","column","colon","bordered","layout","children","className","rootClassName","style","size","labelStyle","contentStyle","styles","items","classNames"]),{getPrefixCls:M,direction:R,className:L,style:D,classNames:A,styles:K}=(0,i.useComponentConfig)("descriptions"),B=M("descriptions",n),F=(0,d.default)(),z=l.useMemo(()=>{var e;return"number"==typeof h?h:null!=(e=(0,o.matchScreen)(F,Object.assign(Object.assign({},c),h)))?e:3},[F,h]),H=(t=l.useMemo(()=>T||(0,f.default)(w).map(e=>Object.assign(Object.assign({},null==e?void 0:e.props),{key:e.key})),[T,w]),l.useMemo(()=>t.map(e=>{var{span:t}=e,n=m(e,["span"]);return"filled"===t?Object.assign(Object.assign({},n),{filled:!0}):Object.assign(Object.assign({},n),{span:"number"==typeof t?t:(0,o.matchScreen)(F,t)})}),[t,F])),V=(0,s.default)(E),W=((e,t)=>{let[n,r]=(0,l.useMemo)(()=>{let n,r,l,a;return n=[],r=[],l=!1,a=0,t.filter(e=>e).forEach(t=>{let{filled:o}=t,i=p(t,["filled"]);if(o){r.push(i),n.push(r),r=[],a=0;return}let s=e-a;(a+=t.span||1)>=e?(a>e?(l=!0,r.push(Object.assign(Object.assign({},i),{span:s}))):r.push(i),n.push(r),r=[],a=0):r.push(i)}),r.length>0&&n.push(r),[n=n.map(t=>{let n=t.reduce((e,t)=>e+(t.span||1),0);if(n({labelStyle:_,contentStyle:O,styles:{content:Object.assign(Object.assign({},K.content),null==$?void 0:$.content),label:Object.assign(Object.assign({},K.label),null==$?void 0:$.label)},classNames:{label:(0,a.default)(A.label,null==I?void 0:I.label),content:(0,a.default)(A.content,null==I?void 0:I.content)}}),[_,O,$,I,A,K]);return U(l.createElement(u.Provider,{value:X},l.createElement("div",Object.assign({className:(0,a.default)(B,L,A.root,null==I?void 0:I.root,{[`${B}-${V}`]:V&&"default"!==V,[`${B}-bordered`]:!!b,[`${B}-rtl`]:"rtl"===R},C,S,q,G),style:Object.assign(Object.assign(Object.assign(Object.assign({},D),K.root),null==$?void 0:$.root),N)},P),(r||g)&&l.createElement("div",{className:(0,a.default)(`${B}-header`,A.header,null==I?void 0:I.header),style:Object.assign(Object.assign({},K.header),null==$?void 0:$.header)},r&&l.createElement("div",{className:(0,a.default)(`${B}-title`,A.title,null==I?void 0:I.title),style:Object.assign(Object.assign({},K.title),null==$?void 0:$.title)},r),g&&l.createElement("div",{className:(0,a.default)(`${B}-extra`,A.extra,null==I?void 0:I.extra),style:Object.assign(Object.assign({},K.extra),null==$?void 0:$.extra)},g)),l.createElement("div",{className:`${B}-view`},l.createElement("table",null,l.createElement("tbody",null,W.map((e,t)=>l.createElement(x,{key:t,index:t,colon:v,prefixCls:B,vertical:"vertical"===y,bordered:b,row:e}))))))))};C.Item=({children:e})=>e,e.s(["Descriptions",0,C],869216);var S=e.i(311451),N=e.i(212931),E=e.i(898586),_=e.i(868297),O=e.i(732961),$=e.i(289882),T=e.i(170517),I=e.i(628882),P=e.i(320890),M=e.i(104458),R=e.i(722319),L=e.i(8398),D=e.i(279728);e.i(765846);var A=e.i(602716),K=e.i(328052);e.i(262370);var B=e.i(135551);let F=(e,t)=>new B.FastColor(e).setA(t).toRgbString(),z=(e,t)=>new B.FastColor(e).lighten(t).toHexString(),H=e=>{let t=(0,A.generate)(e,{theme:"dark"});return{1:t[0],2:t[1],3:t[2],4:t[3],5:t[6],6:t[5],7:t[4],8:t[6],9:t[5],10:t[4]}},V=(e,t)=>{let n=e||"#000",r=t||"#fff";return{colorBgBase:n,colorTextBase:r,colorText:F(r,.85),colorTextSecondary:F(r,.65),colorTextTertiary:F(r,.45),colorTextQuaternary:F(r,.25),colorFill:F(r,.18),colorFillSecondary:F(r,.12),colorFillTertiary:F(r,.08),colorFillQuaternary:F(r,.04),colorBgSolid:F(r,.95),colorBgSolidHover:F(r,1),colorBgSolidActive:F(r,.9),colorBgElevated:z(n,12),colorBgContainer:z(n,8),colorBgLayout:z(n,0),colorBgSpotlight:z(n,26),colorBgBlur:F(r,.04),colorBorder:z(n,26),colorBorderSecondary:z(n,19)}},W={defaultSeed:P.defaultConfig.token,useToken:function(){let[e,t,n]=(0,M.useToken)();return{theme:e,token:t,hashId:n}},defaultAlgorithm:R.default,darkAlgorithm:(e,t)=>{let n=Object.keys(T.defaultPresetColors).map(t=>{let n=(0,A.generate)(e[t],{theme:"dark"});return Array.from({length:10},()=>1).reduce((e,r,l)=>(e[`${t}-${l+1}`]=n[l],e[`${t}${l+1}`]=n[l],e),{})}).reduce((e,t)=>e=Object.assign(Object.assign({},e),t),{}),r=null!=t?t:(0,R.default)(e),l=(0,K.default)(e,{generateColorPalettes:H,generateNeutralColorPalettes:V});return Object.assign(Object.assign(Object.assign(Object.assign({},r),n),l),{colorPrimaryBg:l.colorPrimaryBorder,colorPrimaryBgHover:l.colorPrimaryBorderHover})},compactAlgorithm:(e,t)=>{let n=null!=t?t:(0,R.default)(e),r=n.fontSizeSM,l=n.controlHeight-4;return Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({},n),function(e){let{sizeUnit:t,sizeStep:n}=e,r=n-2;return{sizeXXL:t*(r+10),sizeXL:t*(r+6),sizeLG:t*(r+2),sizeMD:t*(r+2),sizeMS:t*(r+1),size:t*r,sizeSM:t*r,sizeXS:t*(r-1),sizeXXS:t*(r-1)}}(null!=t?t:e)),(0,D.default)(r)),{controlHeight:l}),(0,L.default)(Object.assign(Object.assign({},n),{controlHeight:l})))},getDesignToken:e=>{let t=(null==e?void 0:e.algorithm)?(0,_.createTheme)(e.algorithm):$.default,n=Object.assign(Object.assign({},T.default),null==e?void 0:e.token);return(0,O.getComputedToken)(n,{override:null==e?void 0:e.token},t,I.default)},defaultConfig:P.defaultConfig,_internalContext:P.DesignTokenContext};e.s(["theme",0,W],368869);var U=e.i(270377);function q({isOpen:e,title:a,alertMessage:o,message:i,resourceInformationTitle:s,resourceInformation:d,onCancel:c,onOk:u,confirmLoading:f,requiredConfirmation:m}){let{Title:p,Text:g}=E.Typography,{token:h}=W.useToken(),[x,v]=(0,l.useState)("");return(0,l.useEffect)(()=>{e&&v("")},[e]),(0,t.jsx)(N.Modal,{title:a,open:e,onOk:u,onCancel:c,confirmLoading:f,okText:f?"Deleting...":"Delete",cancelText:"Cancel",okButtonProps:{danger:!0,disabled:!!m&&x!==m||f},cancelButtonProps:{disabled:f},children:(0,t.jsxs)("div",{className:"space-y-4",children:[o&&(0,t.jsx)(n.Alert,{message:o,type:"warning"}),(0,t.jsx)(r.Card,{title:s,className:"mt-4",styles:{body:{padding:"16px"},header:{backgroundColor:h.colorErrorBg,borderColor:h.colorErrorBorder}},style:{backgroundColor:h.colorErrorBg,borderColor:h.colorErrorBorder},children:(0,t.jsx)(C,{column:1,size:"small",children:d&&d.map(({label:e,value:n,...r})=>(0,t.jsx)(C.Item,{label:(0,t.jsx)("span",{className:"font-semibold",children:e}),children:(0,t.jsx)(g,{...r,children:n??"-"})},e))})}),(0,t.jsx)("div",{children:(0,t.jsx)(g,{children:i})}),m&&(0,t.jsxs)("div",{className:"mb-6 mt-4 pt-4 border-t border-gray-200 dark:border-gray-700",children:[(0,t.jsxs)(g,{className:"block text-base font-medium text-gray-700 dark:text-gray-300 mb-2",children:[(0,t.jsx)(g,{children:"Type "}),(0,t.jsx)(g,{strong:!0,type:"danger",children:m}),(0,t.jsx)(g,{children:" to confirm deletion:"})]}),(0,t.jsx)(S.Input,{value:x,onChange:e=>v(e.target.value),placeholder:m,className:"rounded-md",prefix:(0,t.jsx)(U.ExclamationCircleOutlined,{style:{color:h.colorError}}),autoFocus:!0})]})]})})}e.s(["default",()=>q],127952)},950724,(e,t,n)=>{t.exports=function(e){var t=typeof e;return null!=e&&("object"==t||"function"==t)}},100236,(e,t,n)=>{t.exports=e.g&&e.g.Object===Object&&e.g},139088,(e,t,n)=>{var r=e.r(100236),l="object"==typeof self&&self&&self.Object===Object&&self;t.exports=r||l||Function("return this")()},631926,(e,t,n)=>{var r=e.r(139088);t.exports=function(){return r.Date.now()}},748891,(e,t,n)=>{var r=/\s/;t.exports=function(e){for(var t=e.length;t--&&r.test(e.charAt(t)););return t}},830364,(e,t,n)=>{var r=e.r(748891),l=/^\s+/;t.exports=function(e){return e?e.slice(0,r(e)+1).replace(l,""):e}},630353,(e,t,n)=>{t.exports=e.r(139088).Symbol},243436,(e,t,n)=>{var r=e.r(630353),l=Object.prototype,a=l.hasOwnProperty,o=l.toString,i=r?r.toStringTag:void 0;t.exports=function(e){var t=a.call(e,i),n=e[i];try{e[i]=void 0;var r=!0}catch(e){}var l=o.call(e);return r&&(t?e[i]=n:delete e[i]),l}},223243,(e,t,n)=>{var r=Object.prototype.toString;t.exports=function(e){return r.call(e)}},377684,(e,t,n)=>{var r=e.r(630353),l=e.r(243436),a=e.r(223243),o=r?r.toStringTag:void 0;t.exports=function(e){return null==e?void 0===e?"[object Undefined]":"[object Null]":o&&o in Object(e)?l(e):a(e)}},877289,(e,t,n)=>{t.exports=function(e){return null!=e&&"object"==typeof e}},361884,(e,t,n)=>{var r=e.r(377684),l=e.r(877289);t.exports=function(e){return"symbol"==typeof e||l(e)&&"[object Symbol]"==r(e)}},773759,(e,t,n)=>{var r=e.r(830364),l=e.r(950724),a=e.r(361884),o=0/0,i=/^[-+]0x[0-9a-f]+$/i,s=/^0b[01]+$/i,d=/^0o[0-7]+$/i,c=parseInt;t.exports=function(e){if("number"==typeof e)return e;if(a(e))return o;if(l(e)){var t="function"==typeof e.valueOf?e.valueOf():e;e=l(t)?t+"":t}if("string"!=typeof e)return 0===e?e:+e;e=r(e);var n=s.test(e);return n||d.test(e)?c(e.slice(2),n?2:8):i.test(e)?o:+e}},374009,(e,t,n)=>{var r=e.r(950724),l=e.r(631926),a=e.r(773759),o=Math.max,i=Math.min;t.exports=function(e,t,n){var s,d,c,u,f,m,p=0,g=!1,h=!1,x=!0;if("function"!=typeof e)throw TypeError("Expected a function");function v(t){var n=s,r=d;return s=d=void 0,p=t,u=e.apply(r,n)}function b(e){var n=e-m,r=e-p;return void 0===m||n>=t||n<0||h&&r>=c}function y(){var e,n,r,a=l();if(b(a))return w(a);f=setTimeout(y,(e=a-m,n=a-p,r=t-e,h?i(r,c-n):r))}function w(e){return(f=void 0,x&&s)?v(e):(s=d=void 0,u)}function j(){var e,n=l(),r=b(n);if(s=arguments,d=this,m=n,r){if(void 0===f)return p=e=m,f=setTimeout(y,t),g?v(e):u;if(h)return clearTimeout(f),f=setTimeout(y,t),v(m)}return void 0===f&&(f=setTimeout(y,t)),u}return t=a(t)||0,r(n)&&(g=!!n.leading,c=(h="maxWait"in n)?o(a(n.maxWait)||0,t):c,x="trailing"in n?!!n.trailing:x),j.cancel=function(){void 0!==f&&clearTimeout(f),p=0,s=m=d=f=void 0},j.flush=function(){return void 0===f?u:w(l())},j}},436289,503269,214520,814379,992704,684653,877891,401141,952744,605083,101852,249578,571616,e=>{"use strict";var t=e.i(271645);function n(e,t){return null!==e&&null!==t&&"object"==typeof e&&"object"==typeof t&&"id"in e&&"id"in t?e.id===t.id:e===t}function r(e=n){return(0,t.useCallback)((t,n)=>"string"==typeof e?(null==t?void 0:t[e])===(null==n?void 0:n[e]):e(t,n),[e])}e.s(["useByComparator",()=>r],436289);var l=e.i(914189);function a(e,n,r){let[a,o]=(0,t.useState)(r),i=void 0!==e,s=(0,t.useRef)(i),d=(0,t.useRef)(!1),c=(0,t.useRef)(!1);return!i||s.current||d.current?i||!s.current||c.current||(c.current=!0,s.current=i,console.error("A component is changing from controlled to uncontrolled. This may be caused by the value changing from a defined value to undefined, which should not happen.")):(d.current=!0,s.current=i,console.error("A component is changing from uncontrolled to controlled. This may be caused by the value changing from undefined to a defined value, which should not happen.")),[i?e:a,(0,l.useEvent)(e=>(i||o(e),null==n?void 0:n(e)))]}function o(e){let[n]=(0,t.useState)(e);return n}e.s(["useControllable",()=>a],503269),e.s(["useDefaultValue",()=>o],214520);var i=e.i(835696);function s(e,n){let r=(0,t.useRef)({left:0,top:0});if((0,i.useIsoMorphicEffect)(()=>{if(!n)return;let e=n.getBoundingClientRect();e&&(r.current=e)},[e,n]),null==n||!e||n===document.activeElement)return!1;let l=n.getBoundingClientRect();return l.top!==r.current.top||l.left!==r.current.left}function d(e,n=!1){let[r,l]=(0,t.useReducer)(()=>({}),{}),a=(0,t.useMemo)(()=>(function(e){if(null===e)return{width:0,height:0};let{width:t,height:n}=e.getBoundingClientRect();return{width:t,height:n}})(e),[e,r]);return(0,i.useIsoMorphicEffect)(()=>{if(!e)return;let t=new ResizeObserver(l);return t.observe(e),()=>{t.disconnect()}},[e]),n?{width:`${a.width}px`,height:`${a.height}px`}:a}e.s(["useDidElementMove",()=>s],814379),e.s(["useElementSize",()=>d],992704);var c=e.i(544508),u=e.i(402155);class f extends Map{constructor(e){super(),this.factory=e}get(e){let t=super.get(e);return void 0===t&&(t=this.factory(e),this.set(e,t)),t}}function m(e,t){let n=e(),r=new Set;return{getSnapshot:()=>n,subscribe:e=>(r.add(e),()=>r.delete(e)),dispatch(e,...l){let a=t[e].call(n,...l);a&&(n=a,r.forEach(e=>e()))}}}function p(e){return(0,t.useSyncExternalStore)(e.subscribe,e.getSnapshot,e.getSnapshot)}let g=new f(()=>m(()=>[],{ADD(e){return this.includes(e)?this:[...this,e]},REMOVE(e){let t=this.indexOf(e);if(-1===t)return this;let n=this.slice();return n.splice(t,1),n}}));function h(e,n){let r=g.get(n),l=(0,t.useId)(),a=p(r);if((0,i.useIsoMorphicEffect)(()=>{if(e)return r.dispatch("ADD",l),()=>r.dispatch("REMOVE",l)},[r,e]),!e)return!1;let o=a.indexOf(l),s=a.length;return -1===o&&(o=s,s+=1),o===s-1}let x=new Map,v=new Map;function b(e){var t;let n=null!=(t=v.get(e))?t:0;return v.set(e,n+1),0!==n||(x.set(e,{"aria-hidden":e.getAttribute("aria-hidden"),inert:e.inert}),e.setAttribute("aria-hidden","true"),e.inert=!0),()=>(function(e){var t;let n=null!=(t=v.get(e))?t:1;if(1===n?v.delete(e):v.set(e,n-1),1!==n)return;let r=x.get(e);r&&(null===r["aria-hidden"]?e.removeAttribute("aria-hidden"):e.setAttribute("aria-hidden",r["aria-hidden"]),e.inert=r.inert,x.delete(e))})(e)}function y(e,{allowed:t,disallowed:n}={}){let r=h(e,"inert-others");(0,i.useIsoMorphicEffect)(()=>{var e,l;if(!r)return;let a=(0,c.disposables)();for(let t of null!=(e=null==n?void 0:n())?e:[])t&&a.add(b(t));let o=null!=(l=null==t?void 0:t())?l:[];for(let e of o){if(!e)continue;let t=(0,u.getOwnerDocument)(e);if(!t)continue;let n=e.parentElement;for(;n&&n!==t.body;){for(let e of n.children)o.some(t=>e.contains(t))||a.add(b(e));n=n.parentElement}}return a.dispose},[r,t,n])}e.s(["useInertOthers",()=>y],684653);var w=e.i(941444);function j(e,n,r){let l=(0,w.useLatestValue)(e=>{let t=e.getBoundingClientRect();0===t.x&&0===t.y&&0===t.width&&0===t.height&&r()});(0,t.useEffect)(()=>{if(!e)return;let t=null===n?null:n instanceof HTMLElement?n:n.current;if(!t)return;let r=(0,c.disposables)();if("u">typeof ResizeObserver){let e=new ResizeObserver(()=>l.current(t));e.observe(t),r.add(()=>e.disconnect())}if("u">typeof IntersectionObserver){let e=new IntersectionObserver(()=>l.current(t));e.observe(t),r.add(()=>e.disconnect())}return()=>r.dispose()},[n,l,e])}e.s(["useOnDisappear",()=>j],877891);var k=e.i(652265);function C(){return/iPhone/gi.test(window.navigator.platform)||/Mac/gi.test(window.navigator.platform)&&window.navigator.maxTouchPoints>0}function S(e,n,r,l){let a=(0,w.useLatestValue)(r);(0,t.useEffect)(()=>{if(e)return document.addEventListener(n,t,l),()=>document.removeEventListener(n,t,l);function t(e){a.current(e)}},[e,n,l])}function N(e,n,r,l){let a=(0,w.useLatestValue)(r);(0,t.useEffect)(()=>{if(e)return window.addEventListener(n,t,l),()=>window.removeEventListener(n,t,l);function t(e){a.current(e)}},[e,n,l])}function E(e,n,r){let l=h(e,"outside-click"),a=(0,w.useLatestValue)(r),o=(0,t.useCallback)(function(e,t){if(e.defaultPrevented)return;let r=t(e);if(null!==r&&r.getRootNode().contains(r)&&r.isConnected){for(let t of function e(t){return"function"==typeof t?e(t()):Array.isArray(t)||t instanceof Set?t:[t]}(n))if(null!==t&&(t.contains(r)||e.composed&&e.composedPath().includes(t)))return;return(0,k.isFocusableElement)(r,k.FocusableMode.Loose)||-1===r.tabIndex||e.preventDefault(),a.current(e,r)}},[a,n]),i=(0,t.useRef)(null);S(l,"pointerdown",e=>{var t,n;i.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target},!0),S(l,"mousedown",e=>{var t,n;i.current=(null==(n=null==(t=e.composedPath)?void 0:t.call(e))?void 0:n[0])||e.target},!0),S(l,"click",e=>{C()||/Android/gi.test(window.navigator.userAgent)||i.current&&(o(e,()=>i.current),i.current=null)},!0);let s=(0,t.useRef)({x:0,y:0});S(l,"touchstart",e=>{s.current.x=e.touches[0].clientX,s.current.y=e.touches[0].clientY},!0),S(l,"touchend",e=>{let t={x:e.changedTouches[0].clientX,y:e.changedTouches[0].clientY};if(!(Math.abs(t.x-s.current.x)>=30||Math.abs(t.y-s.current.y)>=30))return o(e,()=>e.target instanceof HTMLElement?e.target:null)},!0),N(l,"blur",e=>o(e,()=>window.document.activeElement instanceof HTMLIFrameElement?window.document.activeElement:null),!0)}function _(...e){return(0,t.useMemo)(()=>(0,u.getOwnerDocument)(...e),[...e])}e.s(["useWindowEvent",()=>N],401141),e.s(["useOutsideClick",()=>E],952744),e.s(["useOwnerDocument",()=>_],605083);let O=m(()=>new Map,{PUSH(e,t){var n;let r=null!=(n=this.get(e))?n:{doc:e,count:0,d:(0,c.disposables)(),meta:new Set};return r.count++,r.meta.add(t),this.set(e,r),this},POP(e,t){let n=this.get(e);return n&&(n.count--,n.meta.delete(t)),this},SCROLL_PREVENT({doc:e,d:t,meta:n}){let r,l={doc:e,d:t,meta:function(e){let t={};for(let n of e)Object.assign(t,n(t));return t}(n)},a=[C()?{before({doc:e,d:t,meta:n}){function r(e){return n.containers.flatMap(e=>e()).some(t=>t.contains(e))}t.microTask(()=>{var n;if("auto"!==window.getComputedStyle(e.documentElement).scrollBehavior){let n=(0,c.disposables)();n.style(e.documentElement,"scrollBehavior","auto"),t.add(()=>t.microTask(()=>n.dispose()))}let l=null!=(n=window.scrollY)?n:window.pageYOffset,a=null;t.addEventListener(e,"click",t=>{if(t.target instanceof HTMLElement)try{let n=t.target.closest("a");if(!n)return;let{hash:l}=new URL(n.href),o=e.querySelector(l);o&&!r(o)&&(a=o)}catch{}},!0),t.addEventListener(e,"touchstart",e=>{if(e.target instanceof HTMLElement)if(r(e.target)){let n=e.target;for(;n.parentElement&&r(n.parentElement);)n=n.parentElement;t.style(n,"overscrollBehavior","contain")}else t.style(e.target,"touchAction","none")}),t.addEventListener(e,"touchmove",e=>{if(e.target instanceof HTMLElement&&"INPUT"!==e.target.tagName)if(r(e.target)){let t=e.target;for(;t.parentElement&&""!==t.dataset.headlessuiPortal&&!(t.scrollHeight>t.clientHeight||t.scrollWidth>t.clientWidth);)t=t.parentElement;""===t.dataset.headlessuiPortal&&e.preventDefault()}else e.preventDefault()},{passive:!1}),t.add(()=>{var e;l!==(null!=(e=window.scrollY)?e:window.pageYOffset)&&window.scrollTo(0,l),a&&a.isConnected&&(a.scrollIntoView({block:"nearest"}),a=null)})})}}:{},{before({doc:e}){var t;let n=e.documentElement;r=Math.max(0,(null!=(t=e.defaultView)?t:window).innerWidth-n.clientWidth)},after({doc:e,d:t}){let n=e.documentElement,l=Math.max(0,n.clientWidth-n.offsetWidth),a=Math.max(0,r-l);t.style(n,"paddingRight",`${a}px`)}},{before({doc:e,d:t}){t.style(e.documentElement,"overflow","hidden")}}];a.forEach(({before:e})=>null==e?void 0:e(l)),a.forEach(({after:e})=>null==e?void 0:e(l))},SCROLL_ALLOW({d:e}){e.dispose()},TEARDOWN({doc:e}){this.delete(e)}});function $(e,t,n=()=>[document.body]){!function(e,t,n=()=>({containers:[]})){let r=p(O),l=t?r.get(t):void 0;l&&l.count,(0,i.useIsoMorphicEffect)(()=>{if(!(!t||!e))return O.dispatch("PUSH",t,n),()=>O.dispatch("POP",t,n)},[e,t])}(h(e,"scroll-lock"),t,e=>{var t;return{containers:[...null!=(t=e.containers)?t:[],n]}})}O.subscribe(()=>{let e=O.getSnapshot(),t=new Map;for(let[n]of e)t.set(n,n.documentElement.style.overflow);for(let n of e.values()){let e="hidden"===t.get(n.doc),r=0!==n.count;(r&&!e||!r&&e)&&O.dispatch(n.count>0?"SCROLL_PREVENT":"SCROLL_ALLOW",n),0===n.count&&O.dispatch("TEARDOWN",n)}}),e.s(["useScrollLock",()=>$],101852);let T=/([\u2700-\u27BF]|[\uE000-\uF8FF]|\uD83C[\uDC00-\uDFFF]|\uD83D[\uDC00-\uDFFF]|[\u2011-\u26FF]|\uD83E[\uDD10-\uDDFF])/g;function I(e){var t,n;let r=null!=(t=e.innerText)?t:"",l=e.cloneNode(!0);if(!(l instanceof HTMLElement))return r;let a=!1;for(let e of l.querySelectorAll('[hidden],[aria-hidden],[role="img"]'))e.remove(),a=!0;let o=a?null!=(n=l.innerText)?n:"":r;return T.test(o)&&(o=o.replace(T,"")),o}function P(e){let n=(0,t.useRef)(""),r=(0,t.useRef)("");return(0,l.useEvent)(()=>{let t=e.current;if(!t)return"";let l=t.innerText;if(n.current===l)return r.current;let a=(function(e){let t=e.getAttribute("aria-label");if("string"==typeof t)return t.trim();let n=e.getAttribute("aria-labelledby");if(n){let e=n.split(" ").map(e=>{let t=document.getElementById(e);if(t){let e=t.getAttribute("aria-label");return"string"==typeof e?e.trim():I(t).trim()}return null}).filter(Boolean);if(e.length>0)return e.join(", ")}return I(e).trim()})(t).trim().toLowerCase();return n.current=l,r.current=a,a})}function M(e){return[e.screenX,e.screenY]}function R(){let e=(0,t.useRef)([-1,-1]);return{wasMoved(t){let n=M(t);return(e.current[0]!==n[0]||e.current[1]!==n[1])&&(e.current=n,!0)},update(t){e.current=M(t)}}}e.s(["useTextValue",()=>P],249578),e.s(["useTrackedPointer",()=>R],571616)},83733,e=>{"use strict";let t;var n,r,l=e.i(247167),a=e.i(271645),o=e.i(544508),i=e.i(746725),s=e.i(835696);void 0!==l.default&&"u">typeof globalThis&&"u">typeof Element&&(null==(n=null==l.default?void 0:l.default.env)?void 0:n.NODE_ENV)==="test"&&void 0===(null==(r=null==Element?void 0:Element.prototype)?void 0:r.getAnimations)&&(Element.prototype.getAnimations=function(){return console.warn(["Headless UI has polyfilled `Element.prototype.getAnimations` for your tests.","Please install a proper polyfill e.g. `jsdom-testing-mocks`, to silence these warnings.","","Example usage:","```js","import { mockAnimationsApi } from 'jsdom-testing-mocks'","mockAnimationsApi()","```"].join(` `)),[]});var d=((t=d||{})[t.None=0]="None",t[t.Closed=1]="Closed",t[t.Enter=2]="Enter",t[t.Leave=4]="Leave",t);function c(e){let t={};for(let n in e)!0===e[n]&&(t[`data-${n}`]="");return t}function u(e,t,n,r){let[l,d]=(0,a.useState)(n),{hasFlag:c,addFlag:u,removeFlag:f}=function(e=0){let[t,n]=(0,a.useState)(e),r=(0,a.useCallback)(e=>n(e),[t]),l=(0,a.useCallback)(e=>n(t=>t|e),[t]),o=(0,a.useCallback)(e=>(t&e)===e,[t]);return{flags:t,setFlag:r,addFlag:l,hasFlag:o,removeFlag:(0,a.useCallback)(e=>n(t=>t&~e),[n]),toggleFlag:(0,a.useCallback)(e=>n(t=>t^e),[n])}}(e&&l?3:0),m=(0,a.useRef)(!1),p=(0,a.useRef)(!1),g=(0,i.useDisposables)();return(0,s.useIsoMorphicEffect)(()=>{var l;if(e){if(n&&d(!0),!t){n&&u(3);return}return null==(l=null==r?void 0:r.start)||l.call(r,n),function(e,{prepare:t,run:n,done:r,inFlight:l}){let a=(0,o.disposables)();return function(e,{inFlight:t,prepare:n}){if(null!=t&&t.current)return n();let r=e.style.transition;e.style.transition="none",n(),e.offsetHeight,e.style.transition=r}(e,{prepare:t,inFlight:l}),a.nextFrame(()=>{n(),a.requestAnimationFrame(()=>{a.add(function(e,t){var n,r;let l=(0,o.disposables)();if(!e)return l.dispose;let a=!1;l.add(()=>{a=!0});let i=null!=(r=null==(n=e.getAnimations)?void 0:n.call(e).filter(e=>e instanceof CSSTransition))?r:[];return 0===i.length?t():Promise.allSettled(i.map(e=>e.finished)).then(()=>{a||t()}),l.dispose}(e,r))})}),a.dispose}(t,{inFlight:m,prepare(){p.current?p.current=!1:p.current=m.current,m.current=!0,p.current||(n?(u(3),f(4)):(u(4),f(2)))},run(){p.current?n?(f(3),u(4)):(f(4),u(3)):n?f(1):u(1)},done(){var e;p.current&&"function"==typeof t.getAnimations&&t.getAnimations().length>0||(m.current=!1,f(7),n||d(!1),null==(e=null==r?void 0:r.end)||e.call(r,n))}})}},[e,n,t,g]),e?[l,{closed:c(1),enter:c(2),leave:c(4),transition:c(2)||c(4)}]:[n,{closed:void 0,enter:void 0,leave:void 0,transition:void 0}]}e.s(["transitionDataAttributes",()=>c,"useTransition",()=>u],83733)},601893,919751,694421,140721,904016,942803,e=>{"use strict";var t=e.i(271645);let n=(0,t.createContext)(void 0);function r(){return(0,t.useContext)(n)}e.s(["useDisabled",()=>r],601893);var l=e.i(953760),a=e.i(174080),o="u">typeof document?t.useLayoutEffect:function(){};function i(e,t){let n,r,l;if(e===t)return!0;if(typeof e!=typeof t)return!1;if("function"==typeof e&&e.toString()===t.toString())return!0;if(e&&t&&"object"==typeof e){if(Array.isArray(e)){if((n=e.length)!==t.length)return!1;for(r=n;0!=r--;)if(!i(e[r],t[r]))return!1;return!0}if((n=(l=Object.keys(e)).length)!==Object.keys(t).length)return!1;for(r=n;0!=r--;)if(!({}).hasOwnProperty.call(t,l[r]))return!1;for(r=n;0!=r--;){let n=l[r];if(("_owner"!==n||!e.$$typeof)&&!i(e[n],t[n]))return!1}return!0}return e!=e&&t!=t}function s(e){return"u"{n.current=e}),n}let u=(e,t)=>({...(0,l.offset)(e),options:[e,t]});e.i(247167);var f=e.i(229315),m=e.i(343084);e.i(397126);let p={...t},g=p.useInsertionEffect||(e=>e());function h(e){let n=t.useRef(()=>{});return g(()=>{n.current=e}),t.useCallback(function(){for(var e=arguments.length,t=Array(e),r=0;rtypeof document?t.useLayoutEffect:t.useEffect;let v=!1,b=0,y=()=>"floating-ui-"+Math.random().toString(36).slice(2,6)+b++,w=p.useId||function(){let[e,n]=t.useState(()=>v?y():void 0);return x(()=>{null==e&&n(y())},[]),t.useEffect(()=>{v=!0},[]),e},j=t.createContext(null),k=t.createContext(null),C="active",S="selected";function N(e,t,n){let r=new Map,l="item"===n,a=e;if(l&&e){let{[C]:t,[S]:n,...r}=e;a=r}return{..."floating"===n&&{tabIndex:-1,"data-floating-ui-focusable":""},...a,...t.map(t=>{let r=t?t[n]:null;return"function"==typeof r?e?r(e):null:r}).concat(e).reduce((e,t)=>(t&&Object.entries(t).forEach(t=>{let[n,a]=t;if(!(l&&[C,S].includes(n)))if(0===n.indexOf("on")){if(r.has(n)||r.set(n,[]),"function"==typeof a){var o;null==(o=r.get(n))||o.push(a),e[n]=function(){for(var e,t=arguments.length,l=Array(t),a=0;ae(...l)).find(e=>void 0!==e)}}}else e[n]=a}),e),{})}}function E(e,t){return{...e,rects:{...e.rects,floating:{...e.rects.floating,height:t}}}}var _=e.i(746725),O=e.i(914189),$=e.i(835696);let T=(0,t.createContext)({styles:void 0,setReference:()=>{},setFloating:()=>{},getReferenceProps:()=>({}),getFloatingProps:()=>({}),slot:{}});T.displayName="FloatingContext";let I=(0,t.createContext)(null);function P(e){return(0,t.useMemo)(()=>e?"string"==typeof e?{to:e}:e:null,[e])}function M(){return(0,t.useContext)(T).setReference}function R(){return(0,t.useContext)(T).getReferenceProps}function L(){let{getFloatingProps:e,slot:n}=(0,t.useContext)(T);return(0,t.useCallback)((...t)=>Object.assign({},e(...t),{"data-anchor":n.anchor}),[e,n])}function D(e=null){!1===e&&(e=null),"string"==typeof e&&(e={to:e});let n=(0,t.useContext)(I),r=(0,t.useMemo)(()=>e,[JSON.stringify(e,(e,t)=>{var n;return null!=(n=null==t?void 0:t.outerHTML)?n:t})]);(0,$.useIsoMorphicEffect)(()=>{null==n||n(null!=r?r:null)},[n,r]);let l=(0,t.useContext)(T);return(0,t.useMemo)(()=>[l.setFloating,e?l.styles:{}],[l.setFloating,e,l.styles])}function A({children:e,enabled:n=!0}){var r,p,g,v,b,y,C;let S,_,P,M,R,L,D,A,B,F,z,H,V,W,U,q,[G,X]=(0,t.useState)(null),[Q,Y]=(0,t.useState)(0),J=(0,t.useRef)(null),[Z,ee]=(0,t.useState)(null);p=Z,(0,$.useIsoMorphicEffect)(()=>{if(!p)return;let e=new MutationObserver(()=>{let e=window.getComputedStyle(p).maxHeight,t=parseFloat(e);if(isNaN(t))return;let n=parseInt(e);isNaN(n)||t!==n&&(p.style.maxHeight=`${Math.ceil(t)}px`)});return e.observe(p,{attributes:!0,attributeFilter:["style"]}),()=>{e.disconnect()}},[p]);let et=n&&null!==G&&null!==Z,{to:en="bottom",gap:er=0,offset:el=0,padding:ea=0,inner:eo}=(g=G,v=Z,S=K(null!=(b=null==g?void 0:g.gap)?b:"var(--anchor-gap, 0)",v),_=K(null!=(y=null==g?void 0:g.offset)?y:"var(--anchor-offset, 0)",v),P=K(null!=(C=null==g?void 0:g.padding)?C:"var(--anchor-padding, 0)",v),{...g,gap:S,offset:_,padding:P}),[ei,es="center"]=en.split(" ");(0,$.useIsoMorphicEffect)(()=>{et&&Y(0)},[et]);let{refs:ed,floatingStyles:ec,context:eu}=function(e){void 0===e&&(e={});let{nodeId:n}=e,r=function(e){var n;let{open:r=!1,onOpenChange:l,elements:a}=e,o=w(),i=t.useRef({}),[s]=t.useState(()=>{let e;return e=new Map,{emit(t,n){var r;null==(r=e.get(t))||r.forEach(e=>e(n))},on(t,n){e.set(t,[...e.get(t)||[],n])},off(t,n){var r;e.set(t,(null==(r=e.get(t))?void 0:r.filter(e=>e!==n))||[])}}}),d=null!=((null==(n=t.useContext(j))?void 0:n.id)||null),[c,u]=t.useState(a.reference),f=h((e,t,n)=>{i.current.openEvent=e?t:void 0,s.emit("openchange",{open:e,event:t,reason:n,nested:d}),null==l||l(e,t,n)}),m=t.useMemo(()=>({setPositionReference:u}),[]),p=t.useMemo(()=>({reference:c||a.reference||null,floating:a.floating||null,domReference:a.reference}),[c,a.reference,a.floating]);return t.useMemo(()=>({dataRef:i,open:r,onOpenChange:f,elements:p,events:s,floatingId:o,refs:m}),[r,f,p,s,o,m])}({...e,elements:{reference:null,floating:null,...e.elements}}),u=e.rootContext||r,m=u.elements,[p,g]=t.useState(null),[v,b]=t.useState(null),y=(null==m?void 0:m.domReference)||p,C=t.useRef(null),S=t.useContext(k);x(()=>{y&&(C.current=y)},[y]);let N=function(e){void 0===e&&(e={});let{placement:n="bottom",strategy:r="absolute",middleware:u=[],platform:f,elements:{reference:m,floating:p}={},transform:g=!0,whileElementsMounted:h,open:x}=e,[v,b]=t.useState({x:0,y:0,strategy:r,placement:n,middlewareData:{},isPositioned:!1}),[y,w]=t.useState(u);i(y,u)||w(u);let[j,k]=t.useState(null),[C,S]=t.useState(null),N=t.useCallback(e=>{e!==$.current&&($.current=e,k(e))},[]),E=t.useCallback(e=>{e!==T.current&&(T.current=e,S(e))},[]),_=m||j,O=p||C,$=t.useRef(null),T=t.useRef(null),I=t.useRef(v),P=null!=h,M=c(h),R=c(f),L=c(x),D=t.useCallback(()=>{if(!$.current||!T.current)return;let e={placement:n,strategy:r,middleware:y};R.current&&(e.platform=R.current),(0,l.computePosition)($.current,T.current,e).then(e=>{let t={...e,isPositioned:!1!==L.current};A.current&&!i(I.current,t)&&(I.current=t,a.flushSync(()=>{b(t)}))})},[y,n,r,R,L]);o(()=>{!1===x&&I.current.isPositioned&&(I.current.isPositioned=!1,b(e=>({...e,isPositioned:!1})))},[x]);let A=t.useRef(!1);o(()=>(A.current=!0,()=>{A.current=!1}),[]),o(()=>{if(_&&($.current=_),O&&(T.current=O),_&&O){if(M.current)return M.current(_,O,D);D()}},[_,O,D,M,P]);let K=t.useMemo(()=>({reference:$,floating:T,setReference:N,setFloating:E}),[N,E]),B=t.useMemo(()=>({reference:_,floating:O}),[_,O]),F=t.useMemo(()=>{let e={position:r,left:0,top:0};if(!B.floating)return e;let t=d(B.floating,v.x),n=d(B.floating,v.y);return g?{...e,transform:"translate("+t+"px, "+n+"px)",...s(B.floating)>=1.5&&{willChange:"transform"}}:{position:r,left:t,top:n}},[r,g,B.floating,v.x,v.y]);return t.useMemo(()=>({...v,update:D,refs:K,elements:B,floatingStyles:F}),[v,D,K,B,F])}({...e,elements:{...m,...v&&{reference:v}}}),E=t.useCallback(e=>{let t=(0,f.isElement)(e)?{getBoundingClientRect:()=>e.getBoundingClientRect(),contextElement:e}:e;b(t),N.refs.setReference(t)},[N.refs]),_=t.useCallback(e=>{((0,f.isElement)(e)||null===e)&&(C.current=e,g(e)),((0,f.isElement)(N.refs.reference.current)||null===N.refs.reference.current||null!==e&&!(0,f.isElement)(e))&&N.refs.setReference(e)},[N.refs]),O=t.useMemo(()=>({...N.refs,setReference:_,setPositionReference:E,domReference:C}),[N.refs,_,E]),$=t.useMemo(()=>({...N.elements,domReference:y}),[N.elements,y]),T=t.useMemo(()=>({...N,...u,refs:O,elements:$,nodeId:n}),[N,O,$,n,u]);return x(()=>{u.dataRef.current.floatingContext=T;let e=null==S?void 0:S.nodesRef.current.find(e=>e.id===n);e&&(e.context=T)}),t.useMemo(()=>({...N,context:T,refs:O,elements:$}),[N,O,$,T])}({open:et,placement:"selection"===ei?"center"===es?"bottom":`bottom-${es}`:"center"===es?`${ei}`:`${ei}-${es}`,strategy:"absolute",transform:!1,middleware:[u({mainAxis:"selection"===ei?0:er,crossAxis:el}),(M={padding:ea},{...(0,l.shift)(M),options:[M,R]}),"selection"!==ei&&(L={padding:ea},{...(0,l.flip)(L),options:[L,D]}),"selection"===ei&&eo?{name:"inner",options:A={...eo,padding:ea,overflowRef:J,offset:Q,minItemsVisible:4,referenceOverflowThreshold:ea,onFallbackChange(e){var t,n;if(!e)return;let r=eu.elements.floating;if(!r)return;let l=parseFloat(getComputedStyle(r).scrollPaddingBottom)||0,a=Math.min(4,r.childElementCount),o=0,i=0;for(let e of null!=(n=null==(t=eu.elements.floating)?void 0:t.childNodes)?n:[])if(e instanceof HTMLElement){let t=e.offsetTop,n=t+e.clientHeight+l,s=r.scrollTop,d=s+r.clientHeight;if(t>=s&&n<=d)a--;else{i=Math.max(0,Math.min(n,d)-Math.max(t,s)),o=e.clientHeight;break}}a>=1&&Y(e=>{let t=o*a-i+l;return e>=t?e:t})}},async fn(e){let{listRef:t,overflowRef:n,onFallbackChange:r,offset:o=0,index:i=0,minItemsVisible:s=4,referenceOverflowThreshold:d=0,scrollRef:c,...f}=(0,m.evaluate)(A,e),{rects:p,elements:{floating:g}}=e,h=t.current[i],x=(null==c?void 0:c.current)||g,v=g.clientTop||x.clientTop,b=0!==g.clientTop,y=0!==x.clientTop,w=g===x;if(!h)return{};let j={...e,...await u(-h.offsetTop-g.clientTop-p.reference.height/2-h.offsetHeight/2-o).fn(e)},k=await (0,l.detectOverflow)(E(j,x.scrollHeight+v+g.clientTop),f),C=await (0,l.detectOverflow)(j,{...f,elementContext:"reference"}),S=(0,m.max)(0,k.top),N=j.y+S,_=(x.scrollHeight>x.clientHeight?e=>e:m.round)((0,m.max)(0,x.scrollHeight+(b&&w||y?2*v:0)-S-(0,m.max)(0,k.bottom)));if(x.style.maxHeight=_+"px",x.scrollTop=S,r){let e=x.offsetHeight=-d||C.bottom>=-d;a.flushSync(()=>r(e))}return n&&(n.current=await (0,l.detectOverflow)(E({...j,y:N},x.offsetHeight+v+g.clientTop),f)),{y:N}}}:null,(B={padding:ea,apply({availableWidth:e,availableHeight:t,elements:n}){Object.assign(n.floating.style,{overflow:"auto",maxWidth:`${e}px`,maxHeight:`min(var(--anchor-max-height, 100vh), ${t}px)`})}},{...(0,l.size)(B),options:[B,F]})].filter(Boolean),whileElementsMounted:l.autoUpdate}),[ef=ei,em=es]=eu.placement.split("-");"selection"===ei&&(ef="selection");let ep=(0,t.useMemo)(()=>({anchor:[ef,em].filter(Boolean).join(" ")}),[ef,em]),{getReferenceProps:eg,getFloatingProps:eh}=(z=(r=[function(e,n){let{open:r,elements:l}=e,{enabled:o=!0,overflowRef:i,scrollRef:s,onChange:d}=n,c=h(d),u=t.useRef(!1),f=t.useRef(null),m=t.useRef(null);t.useEffect(()=>{if(!o)return;function e(e){if(e.ctrlKey||!t||null==i.current)return;let n=e.deltaY,r=i.current.top>=-.5,l=i.current.bottom>=-.5,o=t.scrollHeight-t.clientHeight,s=n<0?-1:1,d=n<0?"max":"min";if(!(t.scrollHeight<=t.clientHeight))if(!r&&n>0||!l&&n<0)e.preventDefault(),a.flushSync(()=>{c(e=>e+Math[d](n,o*s))});else{let e;/firefox/i.test((e=navigator.userAgentData)&&Array.isArray(e.brands)?e.brands.map(e=>{let{brand:t,version:n}=e;return t+"/"+n}).join(" "):navigator.userAgent)&&(t.scrollTop+=n)}}let t=(null==s?void 0:s.current)||l.floating;if(r&&t)return t.addEventListener("wheel",e),requestAnimationFrame(()=>{f.current=t.scrollTop,null!=i.current&&(m.current={...i.current})}),()=>{f.current=null,m.current=null,t.removeEventListener("wheel",e)}},[o,r,l.floating,i,s,c]);let p=t.useMemo(()=>({onKeyDown(){u.current=!0},onWheel(){u.current=!1},onPointerMove(){u.current=!1},onScroll(){let e=(null==s?void 0:s.current)||l.floating;if(i.current&&e&&u.current){if(null!==f.current){let t=e.scrollTop-f.current;(i.current.bottom<-.5&&t<-1||i.current.top<-.5&&t>1)&&a.flushSync(()=>c(e=>e+t))}requestAnimationFrame(()=>{f.current=e.scrollTop})}}}),[l.floating,c,i,s]);return t.useMemo(()=>o?{floating:p}:{},[o,p])}(eu,{overflowRef:J,onChange:Y})]).map(e=>null==e?void 0:e.reference),H=r.map(e=>null==e?void 0:e.floating),V=r.map(e=>null==e?void 0:e.item),W=t.useCallback(e=>N(e,r,"reference"),z),U=t.useCallback(e=>N(e,r,"floating"),H),q=t.useCallback(e=>N(e,r,"item"),V),t.useMemo(()=>({getReferenceProps:W,getFloatingProps:U,getItemProps:q}),[W,U,q])),ex=(0,O.useEvent)(e=>{ee(e),ed.setFloating(e)});return t.createElement(I.Provider,{value:X},t.createElement(T.Provider,{value:{setFloating:ex,setReference:ed.setReference,styles:ec,getReferenceProps:eg,getFloatingProps:eh,slot:ep}},e))}function K(e,n,r){let l=(0,_.useDisposables)(),a=(0,O.useEvent)((e,t)=>{if(null==e)return[r,null];if("number"==typeof e)return[e,null];if("string"==typeof e){if(!t)return[r,null];let n=B(e,t);return[n,r=>{let a=function e(t){let n=/var\((.*)\)/.exec(t);if(n){let t=n[1].indexOf(",");if(-1===t)return[n[1]];let r=n[1].slice(0,t).trim(),l=n[1].slice(t+1).trim();return l?[r,...e(l)]:[r]}return[]}(e);{let o=a.map(e=>window.getComputedStyle(t).getPropertyValue(e));l.requestAnimationFrame(function i(){l.nextFrame(i);let s=!1;for(let[e,n]of a.entries()){let r=window.getComputedStyle(t).getPropertyValue(n);if(o[e]!==r){o[e]=r,s=!0;break}}if(!s)return;let d=B(e,t);n!==d&&(r(d),n=d)})}return l.dispose}]}return[r,null]}),o=(0,t.useMemo)(()=>a(e,n)[0],[e,n]),[i=o,s]=(0,t.useState)();return(0,$.useIsoMorphicEffect)(()=>{let[t,r]=a(e,n);if(s(t),r)return r(s)},[e,n]),i}function B(e,t){let n=document.createElement("div");t.appendChild(n),n.style.setProperty("margin-top","0px","important"),n.style.setProperty("margin-top",e,"important");let r=parseFloat(window.getComputedStyle(n).marginTop)||0;return t.removeChild(n),r}function F(e={},t=null,n=[]){for(let[r,l]of Object.entries(e))!function e(t,n,r){if(Array.isArray(r))for(let[l,a]of r.entries())e(t,z(n,l.toString()),a);else r instanceof Date?t.push([n,r.toISOString()]):"boolean"==typeof r?t.push([n,r?"1":"0"]):"string"==typeof r?t.push([n,r]):"number"==typeof r?t.push([n,`${r}`]):null==r?t.push([n,""]):F(r,n,t)}(n,z(t,r),l);return n}function z(e,t){return e?e+"["+t+"]":t}function H(e){var t,n;let r=null!=(t=null==e?void 0:e.form)?t:e.closest("form");if(r){for(let t of r.elements)if(t!==e&&("INPUT"===t.tagName&&"submit"===t.type||"BUTTON"===t.tagName&&"submit"===t.type||"INPUT"===t.nodeName&&"image"===t.type))return void t.click();null==(n=r.requestSubmit)||n.call(r)}}I.displayName="PlacementContext",e.s(["FloatingProvider",()=>A,"useFloatingPanel",()=>D,"useFloatingPanelProps",()=>L,"useFloatingReference",()=>M,"useFloatingReferenceProps",()=>R,"useResolvedAnchor",()=>P],919751),e.s(["attemptSubmit",()=>H,"objectToFormEntries",()=>F],694421);var V=e.i(700020),W=e.i(2788);let U=(0,t.createContext)(null);function q({children:e}){let n=(0,t.useContext)(U);if(!n)return t.default.createElement(t.default.Fragment,null,e);let{target:r}=n;return r?(0,a.createPortal)(t.default.createElement(t.default.Fragment,null,e),r):null}function G({data:e,form:n,disabled:r,onReset:l,overrides:a}){let[o,i]=(0,t.useState)(null),s=(0,_.useDisposables)();return(0,t.useEffect)(()=>{if(l&&o)return s.addEventListener(o,"reset",l)},[o,n,l]),t.default.createElement(q,null,t.default.createElement(X,{setForm:i,formId:n}),F(e).map(([e,l])=>t.default.createElement(W.Hidden,{features:W.HiddenFeatures.Hidden,...(0,V.compact)({key:e,as:"input",type:"hidden",hidden:!0,readOnly:!0,form:n,disabled:r,name:e,value:l,...a})})))}function X({setForm:e,formId:n}){return(0,t.useEffect)(()=>{if(n){let t=document.getElementById(n);t&&e(t)}},[e,n]),n?null:t.default.createElement(W.Hidden,{features:W.HiddenFeatures.Hidden,as:"input",type:"hidden",hidden:!0,readOnly:!0,ref:t=>{if(!t)return;let n=t.closest("form");n&&e(n)}})}function Q(e,n){let[r,l]=(0,t.useState)(n);return e||r===n||l(n),e?r:n}e.s(["FormFields",()=>G],140721),e.s(["useFrozenData",()=>Q],904016);let Y=(0,t.createContext)(void 0);function J(){return(0,t.useContext)(Y)}e.s(["useProvidedId",()=>J],942803)},233137,233538,e=>{"use strict";let t;var n=e.i(271645);let r=(0,n.createContext)(null);r.displayName="OpenClosedContext";var l=((t=l||{})[t.Open=1]="Open",t[t.Closed=2]="Closed",t[t.Closing=4]="Closing",t[t.Opening=8]="Opening",t);function a(){return(0,n.useContext)(r)}function o({value:e,children:t}){return n.default.createElement(r.Provider,{value:e},t)}function i({children:e}){return n.default.createElement(r.Provider,{value:null},e)}function s(e){let t=e.parentElement,n=null;for(;t&&!(t instanceof HTMLFieldSetElement);)t instanceof HTMLLegendElement&&(n=t),t=t.parentElement;let r=(null==t?void 0:t.getAttribute("disabled"))==="";return!(r&&function(e){if(!e)return!1;let t=e.previousElementSibling;for(;null!==t;){if(t instanceof HTMLLegendElement)return!1;t=t.previousElementSibling}return!0}(n))&&r}e.s(["OpenClosedProvider",()=>o,"ResetOpenClosedProvider",()=>i,"State",()=>l,"useOpenClosed",()=>a],233137),e.s(["isDisabledReactIssue7711",()=>s],233538)},35983,35889,722678,178677,635307,495470,333771,e=>{"use strict";let t,n,r,l,a;var o=e.i(290571),i=e.i(271645),s=e.i(429427),d=e.i(371330),c=e.i(174080),u=e.i(394487),f=e.i(436289),m=e.i(503269),p=e.i(214520),g=e.i(814379),h=e.i(746725),x=e.i(992704),v=e.i(914189),b=e.i(684653),y=e.i(835696),w=e.i(941444),j=e.i(877891),k=e.i(952744),C=e.i(605083),S=e.i(144279),N=e.i(101852),E=e.i(294316),_=e.i(249578),O=e.i(571616),$=e.i(83733),T=e.i(601893),I=e.i(919751),P=e.i(140721),M=e.i(904016),R=e.i(942803),L=e.i(233137),D=e.i(233538),A=((t=A||{})[t.First=0]="First",t[t.Previous=1]="Previous",t[t.Next=2]="Next",t[t.Last=3]="Last",t[t.Specific=4]="Specific",t[t.Nothing=5]="Nothing",t);function K(e,t){let n=t.resolveItems();if(n.length<=0)return null;let r=t.resolveActiveIndex(),l=null!=r?r:-1;switch(e.focus){case 0:for(let e=0;e=0;--e)if(!t.resolveDisabled(n[e],e,n))return e;return r;case 2:for(let e=l+1;e=0;--e)if(!t.resolveDisabled(n[e],e,n))return e;return r;case 4:for(let r=0;r0?e.join(" "):void 0,(0,i.useMemo)(()=>function(e){let n=(0,v.useEvent)(e=>(t(t=>[...t,e]),()=>t(t=>{let n=t.slice(),r=n.indexOf(e);return -1!==r&&n.splice(r,1),n}))),r=(0,i.useMemo)(()=>({register:n,slot:e.slot,name:e.name,props:e.props,value:e.value}),[n,e.slot,e.name,e.props,e.value]);return i.default.createElement(U.Provider,{value:r},e.children)},[t])]}U.displayName="DescriptionContext";let X=Object.assign((0,W.forwardRefWithAs)(function(e,t){let n=(0,i.useId)(),r=(0,T.useDisabled)(),{id:l=`headlessui-description-${n}`,...a}=e,o=function e(){let t=(0,i.useContext)(U);if(null===t){let t=Error("You used a component, but it is not inside a relevant parent.");throw Error.captureStackTrace&&Error.captureStackTrace(t,e),t}return t}(),s=(0,E.useSyncRefs)(t);(0,y.useIsoMorphicEffect)(()=>o.register(l),[l,o.register]);let d=r||!1,c=(0,i.useMemo)(()=>({...o.slot,disabled:d}),[o.slot,d]),u={ref:s,...o.props,id:l};return(0,W.useRender)()({ourProps:u,theirProps:a,slot:c,defaultTag:"p",name:o.name||"Description"})}),{});e.s(["Description",()=>X,"useDescribedBy",()=>q,"useDescriptions",()=>G],35889);var Q=e.i(998348);let Y=(0,i.createContext)(null);function J(e){var t,n,r;let l=null!=(n=null==(t=(0,i.useContext)(Y))?void 0:t.value)?n:void 0;return(null!=(r=null==e?void 0:e.length)?r:0)>0?[l,...e].filter(Boolean).join(" "):l}function Z({inherit:e=!1}={}){let t=J(),[n,r]=(0,i.useState)([]),l=e?[t,...n].filter(Boolean):n;return[l.length>0?l.join(" "):void 0,(0,i.useMemo)(()=>function(e){let t=(0,v.useEvent)(e=>(r(t=>[...t,e]),()=>r(t=>{let n=t.slice(),r=n.indexOf(e);return -1!==r&&n.splice(r,1),n}))),n=(0,i.useMemo)(()=>({register:t,slot:e.slot,name:e.name,props:e.props,value:e.value}),[t,e.slot,e.name,e.props,e.value]);return i.default.createElement(Y.Provider,{value:n},e.children)},[r])]}Y.displayName="LabelContext";let ee=Object.assign((0,W.forwardRefWithAs)(function(e,t){var n;let r=(0,i.useId)(),l=function e(){let t=(0,i.useContext)(Y);if(null===t){let t=Error("You used a