tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-14 12:26:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/litellm_internal_staging' into codex/auth-sensitive-routes

Browse Source 
# Conflicts:
#	litellm/proxy/health_endpoints/_health_endpoints.py
#	tests/test_litellm/proxy/auth/test_user_api_key_auth.py
This commit is contained in:
user
2026-05-01 19:08:06 -07:00
parent bef28aa789 0ff9d65f8d
commit 3fd0a2d761
282 changed files with 19871 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/check-lazy-openapi-snapshot.yml
-75
View File
@@ -1,75 +0,0 @@
name: Check Lazy OpenAPI Snapshot
on:
pull_request:
branches:
- main
- litellm_internal_staging
- "litellm_**"
permissions:
contents: read
checks: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
- name: Regenerate snapshot to /tmp
id: regen
run: |
cp litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.committed.json
uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
mv litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.fresh.json
mv /tmp/snapshot.committed.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Compare
id: diff
continue-on-error: true
run: |
diff -q /tmp/snapshot.fresh.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Mark neutral if drift
if: steps.diff.outcome == 'failure'
uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
name: lazy-openapi-snapshot
conclusion: neutral
output: |
{
"title": "Lazy openapi snapshot is stale",
"summary": "Run `python -m litellm.proxy._lazy_openapi_snapshot` and commit the regenerated `litellm/proxy/_lazy_openapi_snapshot.json`. Not blocking — the snapshot will regenerate at release if not committed."
}
Makefile
+3
View File
@@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
$(UV_RUN) pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
test-llm-translation-flush-vcr-cache:
$(UV_RUN) python tests/_flush_vcr_cache.py
README.md
+1 -1
View File
@@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>
docs/my-website/docs/providers/crusoe.md
+196
View File
@@ -0,0 +1,196 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
# Crusoe
## Overview
| Property | Details |
|-------|-------|
| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
| Provider Route on LiteLLM | `crusoe/` |
| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
| Supported Operations | [`/chat/completions`](#sample-usage) |
<br />
<br />
**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
## Available Models
| Model | Description | Context Window |
|-------|-------------|----------------|
| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
## Required Variables
```python showLineNumbers title="Environment Variables"
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
```
## Usage - LiteLLM Python SDK
### Non-streaming
```python showLineNumbers title="Crusoe Non-streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Hello, how are you?", "role": "user"}]
# Crusoe call
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages
)
print(response)
```
### Streaming
```python showLineNumbers title="Crusoe Streaming Completion"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
messages = [{"content": "Write a short story about AI", "role": "user"}]
# Crusoe call with streaming
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
stream=True
)
for chunk in response:
print(chunk)
```
### Function Calling
```python showLineNumbers title="Crusoe Function Calling"
import os
import litellm
from litellm import completion
os.environ["CRUSOE_API_KEY"] = "" # your Crusoe API key
tools = [{
"type": "function",
"function": {
"name": "get_weather",
"description": "Get the current weather in a location",
"parameters": {
"type": "object",
"properties": {
"location": {
"type": "string",
"description": "The city and state, e.g. San Francisco, CA"
}
},
"required": ["location"]
}
}
}]
messages = [{"role": "user", "content": "What's the weather in Boston?"}]
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=messages,
tools=tools,
tool_choice="auto"
)
print(response)
```
## Usage - LiteLLM Proxy Server
```yaml showLineNumbers title="config.yaml"
model_list:
- model_name: llama-3.3-70b
litellm_params:
model: crusoe/meta-llama/Llama-3.3-70B-Instruct
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-r1
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-R1-0528
api_key: os.environ/CRUSOE_API_KEY
- model_name: deepseek-v3
litellm_params:
model: crusoe/deepseek-ai/DeepSeek-V3-0324
api_key: os.environ/CRUSOE_API_KEY
- model_name: qwen3-235b
litellm_params:
model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
api_key: os.environ/CRUSOE_API_KEY
- model_name: kimi-k2
litellm_params:
model: crusoe/moonshotai/Kimi-K2-Thinking
api_key: os.environ/CRUSOE_API_KEY
```
## Custom API Base
**Option 1: Environment variable**
```python showLineNumbers title="Custom API Base via env var"
import os
from litellm import completion
os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
os.environ["CRUSOE_API_KEY"] = "" # your API key
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
)
```
**Option 2: Pass directly**
```python showLineNumbers title="Custom API Base via parameter"
from litellm import completion
response = completion(
model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
messages=[{"content": "Hello!", "role": "user"}],
api_base="https://custom.crusoecloud.com/v1",
api_key="your-api-key",
)
```
## Supported OpenAI Parameters
- `temperature`
- `max_tokens`
- `max_completion_tokens`
- `top_p`
- `frequency_penalty`
- `presence_penalty`
- `stop`
- `n`
- `stream`
- `tools`
- `tool_choice`
- `response_format`
- `seed`
- `user`
- `logit_bias`
- `logprobs`
- `top_logprobs`
enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py
+32 -22
View File
@@ -10,28 +10,21 @@ has already authenticated the user) and you need to extract user information fro
custom headers or other request attributes.
"""
from typing import TYPE_CHECKING, Dict, Optional, Union, cast
from typing import cast
from fastapi import Request
from fastapi.responses import RedirectResponse
if TYPE_CHECKING:
from fastapi_sso.sso.base import OpenID
else:
from typing import Any as OpenID
from litellm.proxy.management_endpoints.types import CustomOpenID
class EnterpriseCustomSSOHandler:
"""
Enterprise Custom SSO Handler for LiteLLM Proxy
This class provides methods for handling custom SSO authentication flows
where users can implement their own authentication logic by processing
request headers and returning user information in OpenID format.
"""
@staticmethod
async def handle_custom_ui_sso_sign_in(
request: Request,
@@ -40,16 +33,16 @@ class EnterpriseCustomSSOHandler:
Allow a user to execute their custom code to parse incoming request headers and return a OpenID object
Use this when you have an OAuth proxy in front of LiteLLM (where the OAuth proxy has already authenticated the user)
Args:
request: The FastAPI request object containing headers and other request data
Returns:
RedirectResponse: Redirect response that sends the user to the LiteLLM UI with authentication token
Raises:
ValueError: If custom_ui_sso_sign_in_handler is not configured
Example:
This method is typically called when a user has already been authenticated by an
external OAuth proxy and the proxy has added custom headers containing user information.
@@ -60,27 +53,44 @@ class EnterpriseCustomSSOHandler:
from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
from litellm.proxy.proxy_server import (
CommonProxyErrors,
general_settings,
premium_user,
user_custom_ui_sso_sign_in_handler,
)
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
if premium_user is not True:
raise ValueError(CommonProxyErrors.not_premium_user.value)
if user_custom_ui_sso_sign_in_handler is None:
raise ValueError("custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings.")
custom_sso_login_handler = cast(CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler)
openid_response: OpenID = await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
raise ValueError(
"custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings."
)
require_trusted_proxy_request(
request=request,
general_settings=general_settings,
feature_name="Custom UI SSO",
)
custom_sso_login_handler = cast(
CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler
)
openid_response: OpenID = (
await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
request=request,
)
)
# Import here to avoid circular imports
from litellm.proxy.management_endpoints.ui_sso import SSOAuthenticationHandler
return await SSOAuthenticationHandler.get_redirect_response_from_openid(
result=openid_response,
request=request,
received_response=None,
generic_client_id=None,
ui_access_mode=None,
)
)
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
+21 -15
View File
@@ -857,10 +857,16 @@ async def project_info(
where={"team_id": project.team_id}
)
if team:
is_team_member = (
user_api_key_dict.user_id in team.admins
or user_api_key_dict.user_id in team.members
)
caller_user_id = user_api_key_dict.user_id
for m in team.members_with_roles or []:
m_user_id = (
m.get("user_id")
if isinstance(m, dict)
else getattr(m, "user_id", None)
)
if m_user_id == caller_user_id:
is_team_member = True
break
if not (is_admin or is_team_member):
raise HTTPException(
@@ -911,20 +917,20 @@ async def list_projects(
include={"litellm_budget_table": True, "object_permission": True}
)
else:
# Get projects for teams the user belongs to
user_teams = await prisma_client.db.litellm_teamtable.find_many(
where={
"OR": [
{"members": {"has": user_api_key_dict.user_id}},
{"admins": {"has": user_api_key_dict.user_id}},
]
}
# Look up the user's team memberships via the reverse-index on
# LiteLLM_UserTable.teams (maintained by team_member_add alongside
# members_with_roles). This avoids a full scan of all team rows.
user_record = await prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_api_key_dict.user_id},
)
user_team_ids = (
user_record.teams
if user_record is not None and user_record.teams
else []
)
team_ids = [team.team_id for team in user_teams]
projects = await prisma_client.db.litellm_projecttable.find_many(
where={"team_id": {"in": team_ids}},
where={"team_id": {"in": user_team_ids}},
include={"litellm_budget_table": True, "object_permission": True},
)
litellm/_logging.py
+3 -66
View File
@@ -1,12 +1,12 @@
import ast
import logging
import os
import re
import sys
from datetime import datetime
from logging import Formatter
from typing import Any, Dict, List, Optional
from typing import Any, Dict, Optional
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
@@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = (
os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
)
_REDACTED = "REDACTED"
def _build_secret_patterns() -> re.Pattern:
patterns: List[str] = [
# ── PEM private key / certificate blocks ──
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# ── GCP OAuth2 access tokens (ya29.*) ──
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# ── Credential %s formatting (space separator, no key= prefix) ──
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
# Bearer tokens (OAuth, JWT, etc.)
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
# Basic auth headers
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
# OpenAI / Anthropic sk- prefixed keys
r"sk-[A-Za-z0-9\-_]{20,}",
# Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
# x-api-key / api-key header values (handles 'key': 'value' dict repr)
r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Anthropic internal header keys
r"x-ak-[A-Za-z0-9\-_]{20,}",
# Google API keys
r"AIza[0-9A-Za-z\-_]{35}",
# Password / secret params (handles key=value and 'key': 'value')
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
# Databricks personal access tokens
r"dapi[0-9a-f]{32}",
# ── Key-name-based redaction ──
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# ── Raw JWTs (without Bearer prefix) ──
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# ── Azure SAS tokens in URLs ──
r"[?&]sig=[A-Za-z0-9%+/=]+",
# ── Full JSON service-account blobs (single-line and multi-line) ──
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
_SECRET_RE = _build_secret_patterns()
def _redact_string(value: str) -> str:
if not _ENABLE_SECRET_REDACTION:
return value
return _SECRET_RE.sub(_REDACTED, value)
return redact_string(value)
def redact_secrets(value: str) -> str:
litellm/batches/batch_utils.py
+75 -3
View File
@@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
)
def _get_models_from_batch_input_file_content(
file_content_dictionary: List[dict],
) -> List[str]:
"""Extract the distinct ``body.model`` values from a batch *input* file.
Used by the proxy's batch pre-call hook to enforce that the caller is
authorized for every model named inside the JSONL — not just the one
on the outer request — so the proxy's per-key model allowlist isn't
bypassed by smuggling expensive models into the batch file.
"""
models: List[str] = []
seen: set = set()
for _item in file_content_dictionary:
body = _item.get("body") or {}
model = body.get("model")
if model and model not in seen:
seen.add(model)
models.append(model)
return models
def _get_batch_job_input_file_usage(
file_content_dictionary: List[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@@ -403,11 +424,25 @@ def _get_batch_job_input_file_usage(
for _item in file_content_dictionary:
body = _item.get("body", {})
model = body.get("model", model_name or "")
messages = body.get("messages", [])
# Chat completion payloads.
messages = body.get("messages")
if messages:
item_tokens = token_counter(model=model, messages=messages)
prompt_tokens += item_tokens
prompt_tokens += token_counter(model=model, messages=messages)
continue
# Text completion payloads (`prompt`).
prompt = body.get("prompt")
if prompt:
prompt_tokens += _count_prompt_or_input_tokens(model=model, value=prompt)
continue
# Embedding payloads (`input`).
input_data = body.get("input")
if input_data:
prompt_tokens += _count_prompt_or_input_tokens(
model=model, value=input_data
)
return Usage(
total_tokens=prompt_tokens + completion_tokens,
@@ -416,6 +451,43 @@ def _get_batch_job_input_file_usage(
)
def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
"""Token-count a ``prompt`` / ``input`` field that the OpenAI batch
schema allows in four shapes:
- ``str``: a single text prompt.
- ``list[str]``: multiple text prompts.
- ``list[int]``: a pre-tokenized prompt (each int counts as 1 token).
- ``list[list[int]]``: multiple pre-tokenized prompts.
Pre-fix only the string shapes were counted, so a caller could send
a large ``list[list[int]]`` payload and slip past TPM rate limits
with a recorded cost of zero tokens.
"""
if isinstance(value, str):
return token_counter(model=model, text=value)
if isinstance(value, list):
total = 0
for chunk in value:
if isinstance(chunk, str):
total += token_counter(model=model, text=chunk)
elif isinstance(chunk, int):
# Single pre-tokenized prompt at the top level: each
# int counts as one token.
total += 1
elif isinstance(chunk, list):
# Nested pre-tokenized prompt: every int contributes a
# token. Mixed string/int items still count.
total += sum(1 if isinstance(t, int) else 0 for t in chunk)
total += sum(
token_counter(model=model, text=t)
for t in chunk
if isinstance(t, str)
)
return total
return 0
def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
"""
Get the tokens of a batch job from the response body
litellm/cost_calculator.py
+5 -1
View File
@@ -513,7 +513,10 @@ def cost_per_token( # noqa: PLR0915
return fireworks_ai_cost_per_token(model=model, usage=usage_block)
elif custom_llm_provider == "azure":
return azure_openai_cost_per_token(
model=model, usage=usage_block, response_time_ms=response_time_ms
model=model,
usage=usage_block,
response_time_ms=response_time_ms,
service_tier=service_tier,
)
elif custom_llm_provider == "gemini":
return gemini_cost_per_token(
@@ -539,6 +542,7 @@ def cost_per_token( # noqa: PLR0915
usage=usage_block,
response_time_ms=response_time_ms,
request_model=request_model,
service_tier=service_tier,
)
else:
model_info = _cached_get_model_info_helper(
litellm/integrations/arize/arize_phoenix_client.py
+14 -1
View File
@@ -2,11 +2,23 @@
Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
"""
import urllib.parse
from typing import Any, Dict, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_id(identifier: str) -> str:
"""Reject path traversal characters and URL-encode the identifier."""
if any(c in identifier for c in ("/", "\\", "#", "?")):
raise ValueError(
f"Invalid identifier {identifier!r}: contains disallowed characters"
)
if ".." in identifier:
raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
return urllib.parse.quote(identifier, safe="")
class ArizePhoenixClient:
"""
Client for interacting with Arize Phoenix API to fetch prompt versions.
@@ -53,7 +65,8 @@ class ArizePhoenixClient:
Returns:
Dictionary containing prompt version data, or None if not found
"""
url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
safe_id = _sanitize_id(prompt_version_id)
url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
try:
# Use the underlying httpx client directly to avoid query param extraction
litellm/integrations/bitbucket/bitbucket_client.py
+22 -3
View File
@@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
"""
import base64
import urllib.parse
from typing import Any, Dict, List, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_file_path(file_path: str) -> str:
"""Reject path traversal and URL-encode each path segment."""
if "#" in file_path or "?" in file_path:
raise ValueError(
f"Invalid file path {file_path!r}: contains URL special characters"
)
parts = file_path.split("/")
for part in parts:
if part == "..":
raise ValueError(
f"Invalid file path {file_path!r}: path traversal detected"
)
return "/".join(urllib.parse.quote(part, safe="") for part in parts)
class BitBucketClient:
"""
Client for interacting with BitBucket API to fetch .prompt files.
@@ -72,7 +88,8 @@ class BitBucketClient:
Returns:
File content as string, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
response = self.http_handler.get(url, headers=self.headers)
@@ -119,7 +136,8 @@ class BitBucketClient:
Returns:
List of file paths
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
try:
response = self.http_handler.get(url, headers=self.headers)
@@ -211,7 +229,8 @@ class BitBucketClient:
Returns:
Dictionary containing file metadata, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
# Use GET with Range header to get just the headers (HEAD equivalent)
litellm/integrations/custom_sso_handler.py
+11
View File
@@ -18,6 +18,17 @@ class CustomSSOLoginHandler(CustomLogger):
self,
request: Request,
) -> OpenID:
from litellm.proxy.auth.trusted_proxy_utils import (
require_trusted_proxy_request,
)
from litellm.proxy.proxy_server import general_settings
require_trusted_proxy_request(
request=request,
general_settings=general_settings,
feature_name="Custom UI SSO",
)
request_headers_dict = dict(request.headers)
return OpenID(
id=request_headers_dict.get("x-litellm-user-id"),
litellm/integrations/langfuse/langfuse.py
+35 -8
View File
@@ -90,6 +90,29 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
return cache_read_input_tokens
def resolve_langfuse_credentials(
langfuse_public_key=None,
langfuse_secret=None,
langfuse_secret_key=None,
langfuse_host=None,
allow_env_credentials: bool = True,
):
if allow_env_credentials is False and langfuse_host is not None:
secret_key = langfuse_secret or langfuse_secret_key
public_key = langfuse_public_key
else:
secret_key = (
langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
)
public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
resolved_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
)
return public_key, secret_key, resolved_host
class LangFuseLogger:
# Class variables or attributes
def __init__(
@@ -98,6 +121,7 @@ class LangFuseLogger:
langfuse_secret=None,
langfuse_host=None,
flush_interval=1,
allow_env_credentials: bool = True,
):
try:
import langfuse
@@ -106,11 +130,13 @@ class LangFuseLogger:
raise Exception(
f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n{traceback.format_exc()}\033[0m"
)
# Instance variables
self.secret_key = langfuse_secret or os.getenv("LANGFUSE_SECRET_KEY")
self.public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
self.langfuse_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
self.public_key, self.secret_key, self.langfuse_host = (
resolve_langfuse_credentials(
langfuse_public_key=langfuse_public_key,
langfuse_secret=langfuse_secret,
langfuse_host=langfuse_host,
allow_env_credentials=allow_env_credentials,
)
)
if not (
self.langfuse_host.startswith("http://")
@@ -160,9 +186,10 @@ class LangFuseLogger:
project_id = None
if os.getenv("UPSTREAM_LANGFUSE_SECRET_KEY") is not None:
upstream_langfuse_debug_env = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
upstream_langfuse_debug = (
str_to_bool(self.upstream_langfuse_debug)
if self.upstream_langfuse_debug is not None
str_to_bool(upstream_langfuse_debug_env)
if upstream_langfuse_debug_env is not None
else None
)
self.upstream_langfuse_secret_key = os.getenv(
@@ -173,7 +200,7 @@ class LangFuseLogger:
)
self.upstream_langfuse_host = os.getenv("UPSTREAM_LANGFUSE_HOST")
self.upstream_langfuse_release = os.getenv("UPSTREAM_LANGFUSE_RELEASE")
self.upstream_langfuse_debug = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
self.upstream_langfuse_debug = upstream_langfuse_debug_env
self.upstream_langfuse = Langfuse(
public_key=self.upstream_langfuse_public_key,
secret_key=self.upstream_langfuse_secret_key,
litellm/integrations/langfuse/langfuse_handler.py
+3 -1
View File
@@ -115,8 +115,10 @@ class LangFuseHandler:
langfuse_logger = LangFuseLogger(
langfuse_public_key=credentials.get("langfuse_public_key"),
langfuse_secret=credentials.get("langfuse_secret"),
langfuse_secret=credentials.get("langfuse_secret")
or credentials.get("langfuse_secret_key"),
langfuse_host=credentials.get("langfuse_host"),
allow_env_credentials=credentials.get("langfuse_host") is None,
)
in_memory_dynamic_logger_cache.set_cache(
credentials=credentials,
litellm/integrations/langfuse/langfuse_prompt_management.py
+10 -9
View File
@@ -20,7 +20,7 @@ from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import (
DynamicLoggingCache,
)
from ..prompt_management_base import PromptManagementBase
from .langfuse import LangFuseLogger
from .langfuse import LangFuseLogger, resolve_langfuse_credentials
from .langfuse_handler import LangFuseHandler
if TYPE_CHECKING:
@@ -46,6 +46,7 @@ def langfuse_client_init(
langfuse_secret_key=None,
langfuse_host=None,
flush_interval=1,
allow_env_credentials: bool = True,
) -> LangfuseClass:
"""
Initialize Langfuse client with caching to prevent multiple initializations.
@@ -70,14 +71,12 @@ def langfuse_client_init(
f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n\033[0m"
)
# Instance variables
secret_key = (
langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
)
public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
langfuse_host = langfuse_host or os.getenv(
"LANGFUSE_HOST", "https://cloud.langfuse.com"
public_key, secret_key, langfuse_host = resolve_langfuse_credentials(
langfuse_public_key=langfuse_public_key,
langfuse_secret=langfuse_secret,
langfuse_secret_key=langfuse_secret_key,
langfuse_host=langfuse_host,
allow_env_credentials=allow_env_credentials,
)
if not (
@@ -222,6 +221,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
langfuse_host=dynamic_callback_params.get("langfuse_host"),
allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id,
@@ -246,6 +246,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
langfuse_host=dynamic_callback_params.get("langfuse_host"),
allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
)
langfuse_prompt_client = self._get_prompt_from_id(
langfuse_prompt_id=prompt_id,
litellm/integrations/langsmith.py
+25 -10
View File
@@ -112,17 +112,28 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_project: Optional[str] = None,
langsmith_base_url: Optional[str] = None,
langsmith_tenant_id: Optional[str] = None,
allow_env_credentials: bool = True,
) -> LangsmithCredentialsObject:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
_credentials_project = (
langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
_credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
if allow_env_credentials is False and langsmith_base_url is not None:
_credentials_api_key = langsmith_api_key
_credentials_project = langsmith_project or "litellm-completion"
_credentials_base_url = langsmith_base_url
_credentials_tenant_id = langsmith_tenant_id
else:
_credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
_credentials_project = (
langsmith_project
or os.getenv("LANGSMITH_PROJECT")
or "litellm-completion"
)
_credentials_base_url = (
langsmith_base_url
or os.getenv("LANGSMITH_BASE_URL")
or "https://api.smith.langchain.com"
)
_credentials_tenant_id = langsmith_tenant_id or os.getenv(
"LANGSMITH_TENANT_ID"
)
return LangsmithCredentialsObject(
LANGSMITH_API_KEY=_credentials_api_key,
@@ -540,6 +551,10 @@ class LangsmithLogger(CustomBatchLogger):
langsmith_tenant_id=standard_callback_dynamic_params.get(
"langsmith_tenant_id", None
),
allow_env_credentials=standard_callback_dynamic_params.get(
"langsmith_base_url", None
)
is None,
)
else:
credentials = self.default_credentials
litellm/integrations/opentelemetry.py
+35 -18
View File
@@ -69,6 +69,8 @@ class OpenTelemetryConfig:
deployment_environment: Optional[str] = None
model_id: Optional[str] = None
ignore_context_propagation: Optional[bool] = None
# When True, create a private TracerProvider instead of reusing or setting the global one.
skip_set_global: bool = False
def __post_init__(self) -> None:
# If endpoint is specified but exporter is still the default "console",
@@ -259,16 +261,21 @@ class OpenTelemetry(CustomLogger):
try:
existing_provider = get_existing_provider_fn()
# If a real SDK provider exists (set by another SDK like Langfuse), use it
# This uses a positive check for SDK providers instead of a negative check for proxy providers
if isinstance(existing_provider, sdk_provider_class):
verbose_logger.debug(
"OpenTelemetry: Using existing %s: %s",
provider_name,
type(existing_provider).__name__,
)
provider = existing_provider
# Don't call set_provider to preserve existing context
if skip_set_global:
verbose_logger.debug(
"OpenTelemetry: existing %s found but skip_set_global=True; creating private %s for isolation",
provider_name,
provider_name,
)
provider = create_new_provider_fn()
else:
verbose_logger.debug(
"OpenTelemetry: Using existing %s: %s",
provider_name,
type(existing_provider).__name__,
)
provider = existing_provider
else:
# Default proxy provider or unknown type, create our own
verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
@@ -293,6 +300,12 @@ class OpenTelemetry(CustomLogger):
return provider
def _skip_set_global(self) -> bool:
# langfuse_otel relies on the Langfuse SDK's providers; don't overwrite them.
return self.config.skip_set_global or (
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
def _init_tracing(self, tracer_provider):
from opentelemetry import trace
from opentelemetry.sdk.trace import TracerProvider
@@ -303,11 +316,6 @@ class OpenTelemetry(CustomLogger):
provider.add_span_processor(self._get_span_processor())
return provider
# CRITICAL FIX: For Langfuse OTEL, skip setting global provider to prevent interference
skip_global = (
hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
)
tracer_provider = self._get_or_create_provider(
provider=tracer_provider,
provider_name="TracerProvider",
@@ -315,16 +323,18 @@ class OpenTelemetry(CustomLogger):
sdk_provider_class=TracerProvider,
create_new_provider_fn=create_tracer_provider,
set_provider_fn=trace.set_tracer_provider,
skip_set_global=skip_global,
skip_set_global=self._skip_set_global(),
)
# Grab our tracer from the TracerProvider (not from global context)
# This ensures we use the provided TracerProvider (e.g., for testing)
self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
self._tracer_provider = tracer_provider
self.span_kind = SpanKind
def _init_metrics(self, meter_provider):
if not self.config.enable_metrics:
self._meter_provider = None
self._operation_duration_histogram = None
self._token_usage_histogram = None
self._cost_histogram = None
@@ -350,7 +360,9 @@ class OpenTelemetry(CustomLogger):
sdk_provider_class=MeterProvider,
create_new_provider_fn=create_meter_provider,
set_provider_fn=metrics.set_meter_provider,
skip_set_global=self._skip_set_global(),
)
self._meter_provider = meter_provider
meter = meter_provider.get_meter(__name__)
@@ -388,6 +400,7 @@ class OpenTelemetry(CustomLogger):
def _init_logs(self, logger_provider):
# nothing to do if events disabled
if not self.config.enable_events:
self._logger_provider = None
return
from opentelemetry._logs import get_logger_provider, set_logger_provider
@@ -404,13 +417,14 @@ class OpenTelemetry(CustomLogger):
)
return provider
self._get_or_create_provider(
self._logger_provider = self._get_or_create_provider(
provider=logger_provider,
provider_name="LoggerProvider",
get_existing_provider_fn=get_logger_provider,
sdk_provider_class=OTLoggerProvider,
create_new_provider_fn=create_logger_provider,
set_provider_fn=set_logger_provider,
skip_set_global=self._skip_set_global(),
)
def log_success_event(self, kwargs, response_obj, start_time, end_time):
@@ -1073,7 +1087,7 @@ class OpenTelemetry(CustomLogger):
# See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
# TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
from opentelemetry._logs import SeverityNumber, get_logger
from opentelemetry._logs import SeverityNumber
try:
from opentelemetry.sdk._logs import ( # type: ignore[attr-defined] # OTEL < 1.39.0
@@ -1084,7 +1098,10 @@ class OpenTelemetry(CustomLogger):
LogRecord as SdkLogRecord, # type: ignore[attr-defined] # OTEL >= 1.39.0
)
otel_logger = get_logger(LITELLM_LOGGER_NAME)
# Resolve through the handler's own LoggerProvider (which may be a
# private one when skip_set_global=True) rather than the module-level
# get_logger() which always goes through the global provider.
otel_logger = self._logger_provider.get_logger(LITELLM_LOGGER_NAME)
parent_ctx = span.get_span_context()
provider = (kwargs.get("litellm_params") or {}).get(
litellm/integrations/prometheus_helpers/prometheus_api.py
+23 -1
View File
@@ -2,6 +2,7 @@
Helper functions to query prometheus API
"""
import json
import time
from datetime import datetime, timedelta
from typing import Optional
@@ -81,6 +82,24 @@ def is_prometheus_connected() -> bool:
return False
def _quote_promql_string_literal(value: str) -> str:
"""Render ``value`` as a PromQL double-quoted string literal.
PromQL string literals follow Go's escape rules
(https://prometheus.io/docs/prometheus/latest/querying/basics/): a
backslash begins an escape sequence and a bare ``"`` ends the literal.
Without escaping, callers that accept arbitrary user-supplied values
(like the ``api_key`` filter on ``/global/spend/logs``) can inject extra
label matchers or selectors and read cross-tenant metrics.
JSON's quoting rules are a strict subset of Go's, so ``json.dumps`` of
a Python string produces a literal Prometheus accepts: ``\\``, ``\\"``,
and the standard ``\\n`` / ``\\t`` / ``\\uNNNN`` control-character
escapes. The returned value already includes the surrounding quotes.
"""
return json.dumps(value, ensure_ascii=False)
async def get_daily_spend_from_prometheus(api_key: Optional[str]):
"""
Expected Response Format:
@@ -109,8 +128,11 @@ async def get_daily_spend_from_prometheus(api_key: Optional[str]):
if api_key is None:
query = "sum(delta(litellm_spend_metric_total[1d]))"
else:
quoted_api_key = _quote_promql_string_literal(api_key)
query = (
f'sum(delta(litellm_spend_metric_total{{hashed_api_key="{api_key}"}}[1d]))'
"sum(delta(litellm_spend_metric_total{"
f"hashed_api_key={quoted_api_key}"
"}[1d]))"
)
params = {
litellm/litellm_core_utils/cli_token_utils.py
+17 -5
View File
@@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]:
return None
def get_litellm_gateway_api_key() -> Optional[str]:
def get_litellm_gateway_api_key(
expected_base_url: Optional[str] = None,
) -> Optional[str]:
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
and returns the API key for use in Python scripts.
Args:
expected_base_url: When provided, the key is only returned if it was
originally issued for this URL. Pass the target server URL to
prevent credential leakage when the client is pointed at a
different (possibly malicious) server.
Returns:
str: The API key if found, None otherwise
str: The API key if found (and origin matches), None otherwise
Example:
>>> import litellm
@@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]:
>>> )
"""
token_data = load_cli_token()
if token_data and "key" in token_data:
return token_data["key"]
return None
if not token_data or "key" not in token_data:
return None
if expected_base_url is not None:
stored_url = token_data.get("base_url")
if stored_url != expected_base_url.rstrip("/"):
return None
return token_data["key"]
litellm/litellm_core_utils/exception_mapping_utils.py
+16 -5
View File
@@ -6,7 +6,8 @@ from typing import Any, Optional
import httpx
import litellm
from litellm._logging import _redact_string, verbose_logger
from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
from litellm.litellm_core_utils.secret_redaction import redact_string
from litellm.types.utils import LlmProviders
from ..exceptions import (
@@ -261,10 +262,18 @@ def exception_type( # type: ignore # noqa: PLR0915
original_exception=original_exception
)
try:
error_str = str(original_exception)
error_str = (
redact_string(str(original_exception))
if _ENABLE_SECRET_REDACTION
else str(original_exception)
)
if model:
if hasattr(original_exception, "message"):
error_str = str(original_exception.message)
error_str = (
redact_string(str(original_exception.message))
if _ENABLE_SECRET_REDACTION
else str(original_exception.message)
)
if isinstance(original_exception, BaseException):
exception_type = type(original_exception).__name__
else:
@@ -2431,7 +2440,8 @@ def exception_type( # type: ignore # noqa: PLR0915
else:
raise APIConnectionError(
message="{}\n{}".format(
str(original_exception), _redact_string(traceback.format_exc())
str(original_exception),
_redact_string(traceback.format_exc()),
),
llm_provider=custom_llm_provider,
model=model,
@@ -2461,7 +2471,8 @@ def exception_type( # type: ignore # noqa: PLR0915
raise e # it's already mapped
raised_exc = APIConnectionError(
message="{}\n{}".format(
original_exception, _redact_string(traceback.format_exc())
original_exception,
_redact_string(traceback.format_exc()),
),
llm_provider="",
model="",
litellm/litellm_core_utils/litellm_logging.py
+7 -2
View File
@@ -3242,10 +3242,15 @@ class Logging(LiteLLMLoggingBaseClass):
),
langfuse_secret=self.standard_callback_dynamic_params.get(
"langfuse_secret"
),
)
or self.standard_callback_dynamic_params.get("langfuse_secret_key"),
langfuse_host=self.standard_callback_dynamic_params.get(
"langfuse_host"
),
allow_env_credentials=self.standard_callback_dynamic_params.get(
"langfuse_host"
)
is None,
)
return langFuseLogger
@@ -4720,7 +4725,7 @@ class StandardLoggingPayloadSetup:
):
for key, value in litellm_params["metadata"].items():
# Skip non-serializable objects like UserAPIKeyAuth
if key == "user_api_key_auth":
if key in {"user_api_key_auth", "user_api_key_budget_reservation"}:
continue
merged_metadata[key] = value
litellm/litellm_core_utils/llm_request_utils.py
+3 -3
View File
@@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
if litellm_params is None:
return {}
proxy_request_headers = (
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
)
proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get(
"headers"
) or {}
return proxy_request_headers
litellm/litellm_core_utils/secret_redaction.py
+81
View File
@@ -0,0 +1,81 @@
"""
Credential/secret redaction utilities.
This module owns the compiled regex and the public `redact_string` helper so
that any part of the codebase (logging, exception mapping, etc.) can scrub
secrets from strings without depending on the logging-configuration module.
"""
import re
from typing import List
_REDACTED = "REDACTED"
def _build_secret_patterns() -> "re.Pattern[str]":
patterns: List[str] = [
# PEM private key / certificate blocks
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
# GCP OAuth2 access tokens (ya29.*)
r"\bya29\.[A-Za-z0-9_.~+/-]+",
# Credential %s formatting (space separator, no key= prefix)
r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
# AWS access key IDs
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
# AWS secrets / session tokens / access key IDs (key=value)
r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
# Bearer tokens (OAuth, JWT, etc.)
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
# Basic auth headers
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
# OpenAI / Anthropic sk- prefixed keys
r"sk-[A-Za-z0-9\-_]{20,}",
# Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
# x-api-key / api-key header values (handles 'key': 'value' dict repr)
r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Anthropic internal header keys
r"x-ak-[A-Za-z0-9\-_]{20,}",
# Google API keys (bare key value)
r"AIza[0-9A-Za-z\-_]{35}",
# URL query-param key=VALUE (e.g. ?key=AIza... or &key=...) — catches the
# full "key=<secret>" fragment so the value is redacted regardless of format.
r"(?<=[?&])key=[^\s&'\"]{8,}",
# Password / secret params (handles key=value and 'key': 'value')
# Word boundary prevents O(n^2) backtracking on long word-char runs.
r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
# Database connection string credentials (scheme://user:pass@host)
r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
# Databricks personal access tokens
r"dapi[0-9a-f]{32}",
# ── Key-name-based redaction ──
# Catches secrets inside dicts/config dumps by matching on the KEY name
# regardless of what the value looks like.
# e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
# private_key with PEM-aware value capture
r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
r"(?:master_key|database_url|db_url|connection_string|"
r"signing_key|encryption_key|"
r"auth_token|access_token|refresh_token|"
r"slack_webhook_url|webhook_url|"
r"database_connection_string|"
r"huggingface_token|jwt_secret)"
r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
# Raw JWTs (without Bearer prefix)
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
# Azure SAS tokens in URLs
r"[?&]sig=[A-Za-z0-9%+/=]+",
# Full JSON service-account blobs (single-line and multi-line)
r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
]
return re.compile("|".join(patterns), re.IGNORECASE)
_SECRET_RE = _build_secret_patterns()
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(_REDACTED, value)
litellm/litellm_core_utils/streaming_handler.py
+1 -1
View File
@@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception)
)
raise e
self._handle_stream_fallback_error(e)
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:
litellm/litellm_core_utils/url_utils.py
+82 -1
View File
@@ -22,7 +22,7 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
import socket
from ipaddress import ip_address, ip_network
from typing import Any, List, Set, Tuple
from urllib.parse import urlparse, urlunparse
from urllib.parse import quote, urlparse, urlunparse
import httpx
@@ -46,6 +46,46 @@ class SSRFError(ValueError):
pass
def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str:
"""Percent-encode one user-controlled URL path segment.
``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986
unreserved characters such as ``.`` unescaped, so reject standalone dot
segments before they can be appended to an upstream URL and normalized by
the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
if value_str in {".", ".."}:
raise ValueError(f"{field_name} cannot be a dot path segment")
return quote(value_str, safe="")
def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str:
"""Percent-encode a user-controlled URL path made of multiple segments.
Empty segments are rejected, so leading, trailing, or consecutive slashes
fail closed instead of being normalized by the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
encoded_segments = []
for segment in value_str.split("/"):
encoded_segments.append(encode_url_path_segment(segment, field_name=field_name))
return "/".join(encoded_segments)
def _is_blocked_ip(addr: str) -> bool:
"""Return True for any IP not safe to reach from a user-supplied URL.
@@ -199,6 +239,47 @@ def validate_url(url: str) -> Tuple[str, str]:
return rewritten, host_header
def assert_same_origin(candidate_url: str, expected_url: str) -> None:
"""Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``.
Use when an upstream API returns a URL meant for follow-up requests
(e.g. an async-job polling URL that will be hit with the operator's
API key in the headers). The upstream is trusted because the operator
configured ``api_base``, but the URL it hands back must actually point
back at the same origin or we'd be blindly forwarding credentials
wherever the upstream told us to.
Hostnames are compared case-insensitively. Default ports are made
explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...``
and ``https://api.example.com/...`` are treated as the same origin.
Error messages identify *which* component mismatched but never echo
the operator's ``expected`` host or the candidate's hostname back to
the caller — in the SSRF threat model the caller is the attacker,
and reflecting host info would be a secondary leak of operator
infrastructure details.
"""
candidate = urlparse(candidate_url)
expected = urlparse(expected_url)
if candidate.scheme not in _ALLOWED_SCHEMES:
raise SSRFError("URL scheme is not allowed")
if candidate.scheme != expected.scheme:
raise SSRFError("Origin mismatch on scheme")
candidate_host = _normalize_host(candidate.hostname or "")
expected_host = _normalize_host(expected.hostname or "")
if not candidate_host or candidate_host != expected_host:
raise SSRFError("Origin mismatch on host")
default_port = 443 if candidate.scheme == "https" else 80
candidate_port = candidate.port if candidate.port is not None else default_port
expected_port = expected.port if expected.port is not None else default_port
if candidate_port != expected_port:
raise SSRFError("Origin mismatch on port")
_MAX_REDIRECTS = 10
litellm/llms/anthropic/batches/transformation.py
+3 -1
View File
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cas
import httpx
from httpx import Headers, Response
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
@@ -122,7 +123,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id}
"""
api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
return f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}"
def transform_retrieve_batch_request(
self,
litellm/llms/anthropic/files/handler.py
+5 -1
View File
@@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import (
FileContentRequest,
@@ -89,7 +90,10 @@ class AnthropicFilesHandler:
raise ValueError("Missing Anthropic API Key")
# Construct the Anthropic batch results URL
results_url = f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}/results"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
results_url = (
f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results"
)
# Prepare headers
headers = {
litellm/llms/anthropic/files/transformation.py
+7 -3
View File
@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union, cast
import httpx
from openai.types.file_deleted import FileDeleted
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
@@ -185,7 +186,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_retrieve_file_response(
self,
@@ -206,7 +208,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_delete_file_response(
self,
@@ -268,7 +271,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
def transform_file_content_response(
self,
litellm/llms/anthropic/skills/transformation.py
+3 -1
View File
@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.skills.transformation import (
BaseSkillsAPIConfig,
LiteLLMLoggingObj,
@@ -81,7 +82,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
api_base = AnthropicModelInfo.get_api_base()
if skill_id:
return f"{api_base}/v1/skills/{skill_id}"
encoded_skill_id = encode_url_path_segment(skill_id, field_name="skill_id")
return f"{api_base}/v1/skills/{encoded_skill_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_skill_request(
litellm/llms/azure/azure.py
+30 -4
View File
@@ -16,6 +16,7 @@ import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@@ -792,6 +793,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
client=client,
litellm_params=litellm_params,
api_base=api_base,
api_version=api_version,
)
azure_client = self.get_azure_openai_client(
api_version=api_version,
@@ -898,6 +900,17 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
# Reject polling URLs that don't share an origin with ``api_base``.
# Without this an upstream-controlled or attacker-controlled
# value would receive the operator's Azure API key in the
# request headers below. VERIA-51.
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = await async_handler.get(
url=operation_location_url,
headers=headers,
@@ -908,8 +921,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
# Don't reflect the raw response body — when the polling
# URL points at an internal JSON API (cloud metadata
# service etc.) reflecting it here turns Blind SSRF into
# Full-Read SSRF. VERIA-51.
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
@@ -1009,6 +1027,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = sync_handler.get(
url=operation_location_url,
headers=headers,
@@ -1019,8 +1044,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
litellm/llms/azure/cost_calculation.py
+5 -1
View File
@@ -12,7 +12,10 @@ from litellm.utils import get_model_info
def cost_per_token(
model: str, usage: Usage, response_time_ms: Optional[float] = 0.0
model: str,
usage: Usage,
response_time_ms: Optional[float] = 0.0,
service_tier: Optional[str] = None,
) -> Tuple[float, float]:
"""
Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -47,4 +50,5 @@ def cost_per_token(
model=model,
usage=usage,
custom_llm_provider="azure",
service_tier=service_tier,
)
litellm/llms/azure/responses/transformation.py
+9 -2
View File
@@ -5,6 +5,7 @@ import httpx
from openai.types.responses import ResponseReasoningItem
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
@@ -201,7 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}"
# Reconstruct the URL with all original components but with the modified path
constructed_url = urlunparse(
@@ -322,7 +326,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id and /cancel at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}/cancel"
# Reconstruct the URL with all original components but with the modified path
cancel_url = urlunparse(
litellm/llms/azure_ai/agents/handler.py
+14 -4
View File
@@ -36,6 +36,7 @@ from typing import (
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure_ai.agents.transformation import (
AzureAIAgentsConfig,
AzureAIAgentsError,
@@ -75,20 +76,29 @@ class AzureAIAgentsHandler:
def _build_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
return f"{api_base}/threads/{thread_id}/runs?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}"
def _build_run_status_url(
self, api_base: str, thread_id: str, run_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/runs/{run_id}?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}"
def _build_list_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str:
"""URL for the create-thread-and-run endpoint (supports streaming)."""
litellm/llms/azure_ai/cost_calculator.py
+2
View File
@@ -65,6 +65,7 @@ def cost_per_token(
usage: Usage,
response_time_ms: Optional[float] = 0.0,
request_model: Optional[str] = None,
service_tier: Optional[str] = None,
) -> Tuple[float, float]:
"""
Calculate the cost per token for Azure AI models.
@@ -102,6 +103,7 @@ def cost_per_token(
model=model,
usage=usage,
custom_llm_provider="azure_ai",
service_tier=service_tier,
)
except Exception as e:
# For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
+22 -1
View File
@@ -17,11 +17,13 @@ from urllib.parse import quote
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.constants import (
AZURE_DOCUMENT_INTELLIGENCE_API_VERSION,
AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI,
AZURE_OPERATION_POLLING_TIMEOUT,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
@@ -217,11 +219,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
if "/" in model:
# Extract the last part after the last slash
model_id = model.split("/")[-1]
encoded_model_id = encode_url_path_segment(model_id, field_name="model_id")
# Azure Document Intelligence analyze endpoint
# Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/)
url = (
f"{api_base}/documentintelligence/documentModels/{model_id}:analyze"
f"{api_base}/documentintelligence/documentModels/{encoded_model_id}:analyze"
f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
)
@@ -599,6 +602,16 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"Azure Document Intelligence returned 202 but no Operation-Location header found"
)
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(
f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
)
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get(
@@ -711,6 +724,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"Azure Document Intelligence returned 202 but no Operation-Location header found"
)
# Reject cross-origin polling URLs (see sync path). VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(
f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
)
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get(
litellm/llms/bedrock/chat/invoke_agent/transformation.py
+9 -1
View File
@@ -12,6 +12,7 @@ import httpx
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@@ -97,8 +98,15 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model)
session_id = self._get_session_id(optional_params)
encoded_agent_id = encode_url_path_segment(agent_id, field_name="agent_id")
encoded_agent_alias_id = encode_url_path_segment(
agent_alias_id, field_name="agent_alias_id"
)
encoded_session_id = encode_url_path_segment(
session_id, field_name="session_id"
)
endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text"
endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text"
return endpoint_url
litellm/llms/bedrock/count_tokens/transformation.py
+2 -1
View File
@@ -201,13 +201,14 @@ class BedrockCountTokensConfig(BaseAWSLLM):
# Remove bedrock/ prefix if present
if model_id.startswith("bedrock/"):
model_id = model_id[8:] # Remove "bedrock/" prefix
encoded_model_id = self.encode_model_id(model_id=model_id)
base_url, _ = self.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_region_name=aws_region_name,
)
endpoint = f"{base_url}/model/{model_id}/count-tokens"
endpoint = f"{base_url}/model/{encoded_model_id}/count-tokens"
return endpoint
litellm/llms/bedrock/vector_stores/transformation.py
+5 -1
View File
@@ -5,6 +5,7 @@ from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.types.integrations.rag.bedrock_knowledgebase import (
@@ -209,7 +210,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
if isinstance(query, list):
query = " ".join(query)
url = f"{api_base}/{vector_store_id}/retrieve"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/retrieve"
request_body: Dict[str, Any] = {
"retrievalQuery": BedrockKBRetrievalQuery(text=query),
litellm/llms/black_forest_labs/image_edit/handler.py
+23
View File
@@ -15,6 +15,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@@ -331,6 +332,17 @@ class BlackForestLabsImageEdit:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
@@ -416,6 +428,17 @@ class BlackForestLabsImageEdit:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
litellm/llms/black_forest_labs/image_generation/handler.py
+23
View File
@@ -15,6 +15,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@@ -317,6 +318,17 @@ class BlackForestLabsImageGeneration:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
@@ -402,6 +414,17 @@ class BlackForestLabsImageGeneration:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
litellm/llms/bytez/chat/transformation.py
+3 -1
View File
@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -149,7 +150,8 @@ class BytezChatConfig(BaseConfig):
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
return f"{API_BASE}/{model}"
encoded_model = encode_url_path_segments(model, field_name="model")
return f"{API_BASE}/{encoded_model}"
def transform_request(
self,
litellm/llms/cloudflare/chat/transformation.py
+3 -1
View File
@@ -5,6 +5,7 @@ from typing import AsyncIterator, Iterator, List, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import (
BaseConfig,
@@ -89,7 +90,8 @@ class CloudflareChatConfig(BaseConfig):
api_base = (
f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
)
return api_base + model
encoded_model = encode_url_path_segments(model, field_name="model")
return api_base + encoded_model
def get_supported_openai_params(self, model: str) -> List[str]:
return [
litellm/llms/custom_httpx/container_handler.py
+3 -1
View File
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@@ -72,7 +73,8 @@ def _build_url(
# Substitute path parameters
for param, value in path_params.items():
path_template = path_template.replace(f"{{{param}}}", value)
encoded_value = encode_url_path_segment(value, field_name=param)
path_template = path_template.replace(f"{{{param}}}", encoded_value)
# Parse the api_base to extract existing query params
parsed_base = httpx.URL(api_base)
litellm/llms/custom_httpx/llm_http_handler.py
+25 -6
View File
@@ -26,6 +26,7 @@ from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@@ -8948,7 +8949,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@@ -9015,7 +9019,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@@ -9214,7 +9221,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
@@ -9297,7 +9307,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
@@ -9363,7 +9376,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@@ -9428,7 +9444,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
litellm/llms/elevenlabs/text_to_speech/transformation.py
+4 -2
View File
@@ -11,13 +11,14 @@ import httpx
from httpx import Headers
import litellm
from litellm.types.utils import all_litellm_params
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import all_litellm_params
from ..common_utils import ElevenLabsException
@@ -321,7 +322,8 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
"ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`."
)
url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}"
encoded_voice_id = encode_url_path_segment(voice_id, field_name="voice_id")
url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}"
query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {})
if query_params:
litellm/llms/gemini/files/transformation.py
+10 -10
View File
@@ -12,6 +12,7 @@ import httpx
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
@@ -258,10 +259,14 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
normalized_file_id = file_id
normalized_file_id = normalized_file_id.strip("/")
if not normalized_file_id.startswith("files/"):
normalized_file_id = f"files/{normalized_file_id}"
if normalized_file_id.startswith("files/"):
normalized_file_id = normalized_file_id.removeprefix("files/")
return normalized_file_id
encoded_file_id = encode_url_path_segment(
normalized_file_id, field_name="file_id"
)
return f"files/{encoded_file_id}"
def transform_retrieve_file_response(
self,
@@ -337,13 +342,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
if not api_key:
raise ValueError("api_key is required")
# Extract file name from URI if full URI is provided
# file_id could be "files/abc123" or "https://generativelanguage.googleapis.com/v1beta/files/abc123"
if file_id.startswith("http"):
# Extract the file path from full URI
file_name = file_id.split("/v1beta/")[-1]
else:
file_name = file_id if file_id.startswith("files/") else f"files/{file_id}"
# Normalize and encode the file name before interpolating it into the URL.
file_name = self._normalize_gemini_file_id(file_id)
# Construct the delete URL
url = f"{api_base}/v1beta/{file_name}"
litellm/llms/gemini/interactions/transformation.py
+13 -3
View File
@@ -15,6 +15,7 @@ import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
from litellm.types.interactions import (
@@ -205,8 +206,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
{},
)
@@ -238,8 +242,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
{},
)
@@ -268,8 +275,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}:cancel",
{},
)
litellm/llms/hosted_vllm/embedding/README.md
+12 -1
View File
@@ -2,4 +2,15 @@ No transformation is required for hosted_vllm embedding.
VLLM is a superset of OpenAI's `embedding` endpoint.
To pass provider-specific parameters, see [this](https://docs.litellm.ai/docs/completion/provider_specific_params)
## `encoding_format`
For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request:
1. Explicit value on the embedding call (`encoding_format=...`).
2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry).
3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env).
4. Default **`float`**.
That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly.
To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params).
litellm/llms/manus/files/transformation.py
+7 -3
View File
@@ -18,6 +18,7 @@ from openai.types.file_deleted import FileDeleted
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
@@ -306,7 +307,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_retrieve_file_response(
self,
@@ -336,7 +338,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_delete_file_response(
self,
@@ -422,7 +425,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}/content", {}
def transform_file_content_response(
self,
litellm/llms/manus/responses/transformation.py
+5 -1
View File
@@ -6,6 +6,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@@ -270,7 +271,10 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
Reference: https://open.manus.im/docs/openai-compatibility
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
litellm/llms/openai/containers/transformation.py
+18 -4
View File
@@ -6,6 +6,7 @@ import litellm
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.secret_managers.main import get_secret_str
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
@@ -198,7 +199,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
) -> Tuple[str, Dict]:
"""Transform the OpenAI container retrieve request."""
# For container retrieve, we just need to construct the URL
url = join_container_api_base_path(api_base, f"/{container_id}")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No additional data needed for GET request
data: Dict[str, Any] = {}
@@ -230,7 +234,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
- DELETE /v1/containers/{container_id}
"""
# Construct the URL for container delete
url = join_container_api_base_path(api_base, f"/{container_id}")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No data needed for DELETE request
data: Dict[str, Any] = {}
@@ -267,7 +274,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files
"""
# Construct the URL for container files
url = join_container_api_base_path(api_base, f"/{container_id}/files")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}/files")
# Prepare query parameters
params: Dict[str, Any] = {}
@@ -311,8 +321,12 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files/{file_id}/content
"""
# Construct the URL for container file content
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
url = join_container_api_base_path(
api_base, f"/{container_id}/files/{file_id}/content"
api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content"
)
# No query parameters needed
litellm/llms/openai/evals/transformation.py
+16 -6
View File
@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.evals.transformation import (
BaseEvalsAPIConfig,
LiteLLMLoggingObj,
@@ -76,7 +77,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
api_base = "https://api.openai.com"
if eval_id:
return f"{api_base}/v1/evals/{eval_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
return f"{api_base}/v1/evals/{encoded_eval_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_eval_request(
@@ -276,7 +278,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
if litellm_params and litellm_params.api_base:
api_base = litellm_params.api_base
url = f"{api_base}/v1/evals/{eval_id}/runs"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
# Build request body
request_body = {k: v for k, v in create_request.items() if v is not None}
@@ -310,7 +313,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
if litellm_params and litellm_params.api_base:
api_base = litellm_params.api_base
url = f"{api_base}/v1/evals/{eval_id}/runs"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
# Build query parameters
query_params: Dict[str, Any] = {}
@@ -350,7 +354,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict]:
"""Transform get run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
verbose_logger.debug("Get run request - URL: %s", url)
@@ -376,7 +382,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict, Dict]:
"""Transform cancel run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel"
# Empty body for cancel request
request_body: Dict[str, Any] = {}
@@ -405,7 +413,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict, Dict]:
"""Transform delete run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
# Empty body for delete request
request_body: Dict[str, Any] = {}
litellm/llms/openai/responses/transformation.py
+17 -4
View File
@@ -7,6 +7,7 @@ from pydantic import BaseModel, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@@ -421,7 +422,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- DELETE /v1/responses/{response_id}
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@@ -457,7 +461,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- GET /v1/responses/{response_id}
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@@ -498,7 +505,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/input_items"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/input_items"
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
@@ -540,7 +550,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- POST /v1/responses/{response_id}/cancel
"""
url = f"{api_base}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/cancel"
data: Dict = {}
return url, data
litellm/llms/openai/vector_store_files/transformation.py
+13 -5
View File
@@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, Tuple, cast
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store_files.transformation import (
BaseVectorStoreFilesConfig,
)
@@ -98,7 +99,10 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
or "https://api.openai.com/v1"
)
base_url = base_url.rstrip("/")
return f"{base_url}/vector_stores/{vector_store_id}/files"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
return f"{base_url}/vector_stores/{encoded_vector_store_id}/files"
def transform_create_vector_store_file_request(
self,
@@ -163,7 +167,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_retrieve_vector_store_file_response(
self,
@@ -186,7 +191,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}/content", {}
def transform_retrieve_vector_store_file_content_response(
self,
@@ -218,7 +224,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
payload["attributes"] = filtered_attributes
else:
payload.pop("attributes", None)
return f"{api_base}/{file_id}", payload
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", payload
def transform_update_vector_store_file_response(
self,
@@ -241,7 +248,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_delete_vector_store_file_response(
self,
litellm/llms/openai/vector_stores/transformation.py
+5 -1
View File
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@@ -108,7 +109,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
litellm_params: dict,
extra_body: Optional[Dict[str, Any]] = None,
) -> Tuple[str, Dict]:
url = f"{api_base}/{vector_store_id}/search"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/search"
typed_request_body = VectorStoreSearchRequest(
query=query,
filters=vector_store_search_optional_params.get("filters", None),
litellm/llms/openai/videos/transformation.py
+28 -6
View File
@@ -1,11 +1,13 @@
import mimetypes
from io import BufferedReader, BytesIO
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from urllib.parse import quote
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
from litellm.secret_managers.main import get_secret_str
@@ -220,11 +222,18 @@ class OpenAIVideoConfig(BaseVideoConfig):
- GET /v1/videos/{video_id}/content?variant=thumbnail
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video content download
url = f"{api_base.rstrip('/')}/{original_video_id}/content"
url = f"{api_base.rstrip('/')}/{encoded_video_id}/content"
if variant is not None:
url = f"{url}?variant={variant}"
# Encode the user-controlled ``variant`` so a value like
# ``thumbnail&extra=1`` cannot inject additional query params
# into the upstream request — same hardening rationale as the
# path-segment encoding above.
url = f"{url}?variant={quote(variant, safe='')}"
# No additional data needed for GET content request
data: Dict[str, Any] = {}
@@ -247,9 +256,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
- POST /v1/videos/{video_id}/remix
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video remix
url = f"{api_base.rstrip('/')}/{original_video_id}/remix"
url = f"{api_base.rstrip('/')}/{encoded_video_id}/remix"
# Prepare the request data
data = {"prompt": prompt}
@@ -391,9 +403,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
- DELETE /v1/videos/{video_id}
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video delete
url = f"{api_base.rstrip('/')}/{original_video_id}"
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No data needed for DELETE request
data: Dict[str, Any] = {}
@@ -427,9 +442,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
"""
# Extract the original video_id (remove provider encoding if present)
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# For video retrieve, we just need to construct the URL
url = f"{api_base.rstrip('/')}/{original_video_id}"
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No additional data needed for GET request
data: Dict[str, Any] = {}
@@ -494,7 +512,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base.rstrip('/')}/characters/{character_id}"
original_character_id = extract_original_character_id(character_id)
encoded_character_id = encode_url_path_segment(
original_character_id, field_name="character_id"
)
url = f"{api_base.rstrip('/')}/characters/{encoded_character_id}"
return url, {}
def transform_video_get_character_response(
litellm/llms/openai_like/providers.json
+8
View File
@@ -106,5 +106,13 @@
"base_url": "https://aihubmix.com/v1",
"api_key_env": "AIHUBMIX_API_KEY",
"api_base_env": "AIHUBMIX_API_BASE"
},
"crusoe": {
"base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1",
"api_key_env": "CRUSOE_API_KEY",
"api_base_env": "CRUSOE_API_BASE",
"param_mappings": {
"max_completion_tokens": "max_tokens"
}
}
}
litellm/llms/pg_vector/vector_stores/transformation.py
+5 -1
View File
@@ -1,5 +1,6 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@@ -82,7 +83,10 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
litellm_params: dict,
extra_body: Optional[Dict[str, Any]] = None,
) -> Tuple[str, Dict]:
url = f"{api_base}/{vector_store_id}/search"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/search"
_, request_body = super().transform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,
litellm/llms/ragflow/chat/transformation.py
+4 -2
View File
@@ -13,6 +13,7 @@ Model name format:
from typing import List, Optional, Tuple
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.openai.openai import OpenAIConfig
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.llms.openai import AllMessageValues
@@ -126,10 +127,11 @@ class RAGFlowConfig(OpenAIConfig):
api_base = api_base[:-3] # Remove /v1
# Construct the RAGFlow-specific path
encoded_entity_id = encode_url_path_segment(entity_id, field_name="entity_id")
if endpoint_type == "chat":
path = f"/api/v1/chats_openai/{entity_id}/chat/completions"
path = f"/api/v1/chats_openai/{encoded_entity_id}/chat/completions"
else: # agent
path = f"/api/v1/agents_openai/{entity_id}/chat/completions"
path = f"/api/v1/agents_openai/{encoded_entity_id}/chat/completions"
# Ensure path starts with /
if not path.startswith("/"):
litellm/llms/runwayml/videos/transformation.py
+13 -3
View File
@@ -6,6 +6,7 @@ from httpx._types import RequestFiles
import litellm
from litellm.constants import RUNWAYML_DEFAULT_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.custom_httpx.http_handler import (
@@ -334,9 +335,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
We'll retrieve the task and extract the video URL.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Get task status to retrieve video URL
url = f"{api_base}/tasks/{original_video_id}"
url = f"{api_base}/tasks/{encoded_video_id}"
params: Dict[str, Any] = {}
@@ -495,9 +499,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
RunwayML uses task cancellation.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for task cancellation
url = f"{api_base}/tasks/{original_video_id}/cancel"
url = f"{api_base}/tasks/{encoded_video_id}/cancel"
data: Dict[str, Any] = {}
@@ -533,9 +540,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
RunwayML uses GET /v1/tasks/{task_id} to retrieve task status.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the full URL for task status retrieval
url = f"{api_base}/tasks/{original_video_id}"
url = f"{api_base}/tasks/{encoded_video_id}"
# Empty dict for GET request (no body)
data: Dict[str, Any] = {}
litellm/llms/vertex_ai/batches/handler.py
+9 -3
View File
@@ -4,7 +4,11 @@ from typing import Any, Coroutine, Dict, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.litellm_core_utils.url_utils import (
async_safe_get,
encode_url_path_segment,
safe_get,
)
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
@@ -170,7 +174,8 @@ class VertexAIBatchPrediction(VertexLLM):
)
# Append batch_id to the URL
default_api_base = f"{default_api_base}/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
default_api_base = f"{default_api_base}/{encoded_batch_id}"
if len(default_api_base.split(":")) > 1:
endpoint = default_api_base.split(":")[-1]
@@ -413,7 +418,8 @@ class VertexAIBatchPrediction(VertexLLM):
vertex_project=vertex_project or project_id,
)
retrieve_api_base_default = f"{default_api_base}/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
retrieve_api_base_default = f"{default_api_base}/{encoded_batch_id}"
cancel_api_base_default = f"{retrieve_api_base_default}:cancel"
_, api_base = self._check_custom_proxy(
litellm/llms/vertex_ai/gemini/transformation.py
+32 -10
View File
@@ -212,6 +212,22 @@ def _process_gemini_media(
return _apply_gemini_metadata(
part, model, media_resolution_enum, video_metadata
)
elif image_url.startswith(
"https://generativelanguage.googleapis.com/v1beta/files/"
):
# Gemini Files API URIs — the file is already uploaded to Google's
# servers; pass the URI through as file_data without fetching it.
# These URLs return 403 when accessed directly, so we must not try
# to resolve their MIME type via HTTP.
if format:
file_data = FileDataType(mime_type=format, file_uri=image_url)
else:
# Gemini Files API references can be passed through as URI-only.
file_data = cast(FileDataType, {"file_uri": image_url})
part = {"file_data": file_data}
return _apply_gemini_metadata(
part, model, media_resolution_enum, video_metadata
)
elif (
"https://" in image_url
and (image_type := format or _get_image_mime_type_from_url(image_url))
@@ -743,16 +759,22 @@ def _transform_request_body( # noqa: PLR0915
]
data = RequestBody(contents=content)
if system_instructions is not None:
data["system_instruction"] = system_instructions
if tools is not None:
data["tools"] = tools
if tool_choice is not None:
data["toolConfig"] = tool_choice
if include_server_side_tool_invocations:
if "toolConfig" not in data:
data["toolConfig"] = {}
data["toolConfig"]["includeServerSideToolInvocations"] = True
# Vertex rejects system_instruction/tools/toolConfig alongside cachedContent.
# Treat dropping these fields as a request mutation guarded by modify_params.
can_send_cache_incompatible_fields = (
cached_content is None or litellm.modify_params is False
)
if can_send_cache_incompatible_fields:
if system_instructions is not None:
data["system_instruction"] = system_instructions
if tools is not None:
data["tools"] = tools
if tool_choice is not None:
data["toolConfig"] = tool_choice
if include_server_side_tool_invocations:
if "toolConfig" not in data:
data["toolConfig"] = {}
data["toolConfig"]["includeServerSideToolInvocations"] = True
if safety_settings is not None:
data["safetySettings"] = safety_settings
if generation_config is not None and len(generation_config) > 0:
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py
+55 -14
View File
@@ -3,7 +3,7 @@ Google AI Studio /batchEmbedContents Embeddings Endpoint
"""
import json
from typing import Any, Dict, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
import httpx
@@ -13,8 +13,8 @@ from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
)
from litellm.types.llms.openai import EmbeddingInput
from litellm.types.llms.vertex_ai import (
GeminiEmbeddingInput,
VertexAIBatchEmbeddingsRequestBody,
VertexAIBatchEmbeddingsResponseObject,
)
@@ -23,7 +23,6 @@ from litellm.types.utils import EmbeddingResponse
from ..gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from .batch_embed_content_transformation import (
_is_file_reference,
_is_multimodal_input,
process_embed_content_response,
process_response,
transform_openai_input_gemini_content,
@@ -32,9 +31,24 @@ from .batch_embed_content_transformation import (
class GoogleBatchEmbeddings(VertexLLM):
@staticmethod
def _flatten_and_detect_file_refs(
input: GeminiEmbeddingInput,
) -> Tuple[List[str], bool]:
"""Flatten nested input lists and detect file references."""
input_list = [input] if isinstance(input, str) else input
flat_elements = [
e
for item in input_list
for e in (item if isinstance(item, list) else [item])
if isinstance(e, str)
]
has_file_refs = any(_is_file_reference(e) for e in flat_elements)
return flat_elements, has_file_refs
def _resolve_file_references(
self,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
api_key: str,
sync_handler: HTTPHandler,
) -> Dict[str, Dict[str, str]]:
@@ -42,7 +56,7 @@ class GoogleBatchEmbeddings(VertexLLM):
Resolve Gemini file references (files/...) to get mime_type and uri.
Args:
input: EmbeddingInput that may contain file references
input: GeminiEmbeddingInput that may contain file references
api_key: Gemini API key
sync_handler: HTTP client
@@ -73,7 +87,7 @@ class GoogleBatchEmbeddings(VertexLLM):
async def _async_resolve_file_references(
self,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
api_key: str,
async_handler: AsyncHTTPHandler,
) -> Dict[str, Dict[str, str]]:
@@ -81,7 +95,7 @@ class GoogleBatchEmbeddings(VertexLLM):
Async version of _resolve_file_references.
Args:
input: EmbeddingInput that may contain file references
input: GeminiEmbeddingInput that may contain file references
api_key: Gemini API key
async_handler: Async HTTP client
@@ -110,10 +124,10 @@ class GoogleBatchEmbeddings(VertexLLM):
return resolved_files
def batch_embeddings(
def batch_embeddings( # noqa: PLR0915
self,
model: str,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
print_verbose,
model_response: EmbeddingResponse,
custom_llm_provider: Literal["gemini", "vertex_ai"],
@@ -151,8 +165,7 @@ class GoogleBatchEmbeddings(VertexLLM):
optional_params = optional_params or {}
is_multimodal = _is_multimodal_input(input)
use_embed_content = is_multimodal or (custom_llm_provider == "vertex_ai")
use_embed_content = custom_llm_provider == "vertex_ai"
mode: Literal["embedding", "batch_embedding"]
if use_embed_content:
mode = "embedding"
@@ -215,8 +228,22 @@ class GoogleBatchEmbeddings(VertexLLM):
resolved_files=resolved_files,
)
else:
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
if has_file_refs and not api_key:
raise ValueError(
"An API key is required to resolve Gemini file references (files/...). "
"Pass api_key= or set GEMINI_API_KEY."
)
resolved_files = {}
if api_key and has_file_refs:
resolved_files = self._resolve_file_references(
input=flat_elements, api_key=api_key, sync_handler=sync_handler
)
request_data = transform_openai_input_gemini_content(
input=input, model=model, optional_params=optional_params
input=input,
model=model,
optional_params=optional_params,
resolved_files=resolved_files,
)
## LOGGING
@@ -264,7 +291,7 @@ class GoogleBatchEmbeddings(VertexLLM):
url: str,
data: Optional[Union[VertexAIBatchEmbeddingsRequestBody, dict]],
model_response: EmbeddingResponse,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
timeout: Optional[Union[float, httpx.Timeout]],
headers={},
client: Optional[AsyncHTTPHandler] = None,
@@ -303,8 +330,22 @@ class GoogleBatchEmbeddings(VertexLLM):
resolved_files=resolved_files,
)
else:
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
if has_file_refs and not api_key:
raise ValueError(
"An API key is required to resolve Gemini file references (files/...). "
"Pass api_key= or set GEMINI_API_KEY."
)
resolved_files = {}
if api_key and has_file_refs:
resolved_files = await self._async_resolve_file_references(
input=flat_elements, api_key=api_key, async_handler=async_handler
)
data = transform_openai_input_gemini_content(
input=input, model=model, optional_params=optional_params or {}
input=input,
model=model,
optional_params=optional_params or {},
resolved_files=resolved_files,
)
## LOGGING
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py
+127 -56
View File
@@ -6,12 +6,12 @@ Why separate file? Make it easy to see how transformation works
from typing import Dict, List, Optional, Tuple
from litellm.types.llms.openai import EmbeddingInput
from litellm.types.llms.vertex_ai import (
BlobType,
ContentType,
EmbedContentRequest,
FileDataType,
GeminiEmbeddingInput,
PartType,
VertexAIBatchEmbeddingsRequestBody,
VertexAIBatchEmbeddingsResponseObject,
@@ -114,33 +114,77 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
return media_type, base64_data
def _is_multimodal_input(input: EmbeddingInput) -> bool:
def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool:
"""
Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
Check if the input contains multimodal data (data URIs, file references,
GCS URLs, or nested lists for combined embeddings).
Args:
input: EmbeddingInput (str or List[str])
input: GeminiEmbeddingInput — str, List[str], or List[List[str]] for combined embeddings
Returns:
bool: True if any element is a data URI, file reference, or GCS URL
bool: True if any element is multimodal or a nested list
"""
if isinstance(input, str):
input_list = [input]
else:
input_list = input
return _is_multimodal_element(input)
for element in input_list:
if isinstance(element, str):
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
for element in input:
if isinstance(element, list):
if any(
_is_multimodal_element(sub) for sub in element if isinstance(sub, str)
):
return True
elif isinstance(element, str) and _is_multimodal_element(element):
return True
return False
def _is_multimodal_element(element: str) -> bool:
"""Check if a single string element is multimodal."""
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
return True
return False
def _build_part_for_input(
element: str,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
) -> PartType:
"""
Build a single PartType for an input element, handling text, data URIs,
file references, and GCS URLs.
"""
resolved_files = resolved_files or {}
if element.startswith("data:") and ";base64," in element:
mime_type, base64_data = _parse_data_url(element)
blob: BlobType = {"mime_type": mime_type, "data": base64_data}
return PartType(inline_data=blob)
elif _is_gcs_url(element):
mime_type = _infer_mime_type_from_gcs_url(element)
file_data: FileDataType = {
"mime_type": mime_type,
"file_uri": element,
}
return PartType(file_data=file_data)
elif _is_file_reference(element):
if element not in resolved_files:
raise ValueError(f"File reference {element} not resolved")
file_info = resolved_files[element]
file_data_ref: FileDataType = {
"mime_type": file_info["mime_type"],
"file_uri": file_info["uri"],
}
return PartType(file_data=file_data_ref)
else:
return PartType(text=element)
_SUPPORTED_EMBED_PARAMS = {"outputDimensionality", "taskType", "title"}
@@ -155,37 +199,60 @@ def _filter_embed_params(optional_params: dict) -> dict:
def transform_openai_input_gemini_content(
input: EmbeddingInput, model: str, optional_params: dict
input: GeminiEmbeddingInput,
model: str,
optional_params: dict,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
) -> VertexAIBatchEmbeddingsRequestBody:
"""
The content to embed. Only the parts.text fields will be counted.
Transform OpenAI embedding input to Gemini batchEmbedContents format.
Each input element becomes a separate EmbedContentRequest, supporting
text, data URIs, file references, and GCS URLs.
If an element is a list (nested input), all sub-elements are combined
into a single content with multiple parts, producing one combined
embedding for the group.
Examples:
input=["text", "image"] → 2 separate embeddings
input=[["text", "image"]] → 1 combined embedding
input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate)
"""
gemini_model_name = "models/{}".format(model)
gemini_params = _filter_embed_params(optional_params)
input_list = [input] if isinstance(input, str) else input
requests: List[EmbedContentRequest] = []
if isinstance(input, str):
for element in input_list:
if isinstance(element, list):
if not element:
raise ValueError("Nested input list must not be empty")
for sub in element:
if not isinstance(sub, str):
raise ValueError(
f"Elements inside a nested input list must be strings, got {type(sub)}"
)
parts = [
_build_part_for_input(sub, resolved_files=resolved_files)
for sub in element
]
else:
parts = [_build_part_for_input(element, resolved_files=resolved_files)]
request = EmbedContentRequest(
model=gemini_model_name,
content=ContentType(parts=[PartType(text=input)]),
content=ContentType(parts=parts),
**gemini_params,
)
requests.append(request)
else:
for i in input:
request = EmbedContentRequest(
model=gemini_model_name,
content=ContentType(parts=[PartType(text=i)]),
**gemini_params,
)
requests.append(request)
return VertexAIBatchEmbeddingsRequestBody(requests=requests)
def transform_openai_input_gemini_embed_content(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model: str,
optional_params: dict,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
@@ -194,7 +261,7 @@ def transform_openai_input_gemini_embed_content(
Transform OpenAI embedding input to Gemini embedContent format (multimodal).
Args:
input: EmbeddingInput (str or List[str]) with text, data URIs, or file references
input: GeminiEmbeddingInput with text, data URIs, or file references
model: Model name
optional_params: Additional parameters (taskType, outputDimensionality, etc.)
resolved_files: Dict mapping file names (files/abc) to {mime_type, uri}
@@ -210,31 +277,14 @@ def transform_openai_input_gemini_embed_content(
parts: List[PartType] = []
for element in input_list:
if isinstance(element, list):
raise ValueError(
"Nested (combined) embeddings are not supported on the embedContent path. "
"Use the batchEmbedContents path or pass a flat list instead."
)
if not isinstance(element, str):
raise ValueError(f"Unsupported input type: {type(element)}")
if element.startswith("data:") and ";base64," in element:
mime_type, base64_data = _parse_data_url(element)
blob: BlobType = {"mime_type": mime_type, "data": base64_data}
parts.append(PartType(inline_data=blob))
elif _is_gcs_url(element):
mime_type = _infer_mime_type_from_gcs_url(element)
file_data: FileDataType = {
"mime_type": mime_type,
"file_uri": element,
}
parts.append(PartType(file_data=file_data))
elif _is_file_reference(element):
if element not in resolved_files:
raise ValueError(f"File reference {element} not resolved")
file_info = resolved_files[element]
file_data_ref: FileDataType = {
"mime_type": file_info["mime_type"],
"file_uri": file_info["uri"],
}
parts.append(PartType(file_data=file_data_ref))
else:
parts.append(PartType(text=element))
parts.append(_build_part_for_input(element, resolved_files=resolved_files))
request_body: dict = {
"content": ContentType(parts=parts),
@@ -245,7 +295,7 @@ def transform_openai_input_gemini_embed_content(
def process_embed_content_response(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model_response: EmbeddingResponse,
model: str,
response_json: dict,
@@ -291,7 +341,7 @@ def process_embed_content_response(
def process_response(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model_response: EmbeddingResponse,
model: str,
_predictions: VertexAIBatchEmbeddingsResponseObject,
@@ -308,8 +358,29 @@ def process_response(
model_response.data = openai_embeddings
model_response.model = model
input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
has_nested = isinstance(input, list) and any(isinstance(e, list) for e in input)
if _is_multimodal_input(input) or has_nested:
input_list = input if isinstance(input, list) else [input]
text_elements: List[str] = []
for e in input_list:
if isinstance(e, list):
text_elements.extend(
sub
for sub in e
if isinstance(sub, str) and not _is_multimodal_element(sub)
)
elif isinstance(e, str) and not _is_multimodal_element(e):
text_elements.append(e)
if text_elements:
input_text = get_formatted_prompt(
data={"input": text_elements}, call_type="embedding"
)
prompt_tokens = token_counter(model=model, text=input_text)
else:
prompt_tokens = 0
else:
input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
model_response.usage = Usage(
prompt_tokens=prompt_tokens, total_tokens=prompt_tokens
)
litellm/llms/vertex_ai/vector_stores/search_api/transformation.py
+8 -1
View File
@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm import get_model_info
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.router import GenericLiteLLMParams
@@ -91,12 +92,18 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
raise ValueError("vector_store_id is required")
if api_base:
return api_base.rstrip("/")
encoded_collection_id = encode_url_path_segment(
collection_id, field_name="vertex_collection_id"
)
encoded_datastore_id = encode_url_path_segment(
datastore_id, field_name="vector_store_id"
)
# Vertex AI Search API endpoint for search
return (
f"https://discoveryengine.googleapis.com/v1/"
f"projects/{vertex_project}/locations/{vertex_location}/"
f"collections/{collection_id}/dataStores/{datastore_id}/servingConfigs/default_config"
f"collections/{encoded_collection_id}/dataStores/{encoded_datastore_id}/servingConfigs/default_config"
)
def transform_search_vector_store_request(
litellm/llms/volcengine/responses/transformation.py
+17 -4
View File
@@ -17,6 +17,7 @@ from pydantic import fields as pyd_fields
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@@ -300,7 +301,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@@ -333,7 +337,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@@ -372,7 +379,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/input_items"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/input_items"
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
@@ -408,7 +418,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/cancel"
data: Dict = {}
return url, data
litellm/main.py
+11 -2
View File
@@ -4923,8 +4923,17 @@ def embedding( # noqa: PLR0915
if encoding_format is not None:
optional_params["encoding_format"] = encoding_format
else:
# Omiting causes openai sdk to add default value of "float"
optional_params["encoding_format"] = None
env_fmt = get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT")
if env_fmt is not None and env_fmt.strip().lower() == "none":
optional_params.pop("encoding_format", None)
else:
_default_fmt = (
optional_params.get("encoding_format") or env_fmt or "float"
)
if _default_fmt.strip().lower() == "none":
optional_params.pop("encoding_format", None)
else:
optional_params["encoding_format"] = _default_fmt
api_version = None
litellm/model_prices_and_context_window_backup.json
+92
View File
@@ -22061,6 +22061,98 @@
"tool_use_system_prompt_tokens": 346,
"supports_native_structured_output": true
},
"crusoe/deepseek-ai/DeepSeek-R1-0528": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 7e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/deepseek-ai/DeepSeek-V3-0324": {
"input_cost_per_token": 1.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 163840,
"max_output_tokens": 163840,
"max_tokens": 163840,
"mode": "chat",
"output_cost_per_token": 1.5e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/google/gemma-3-12b-it": {
"input_cost_per_token": 1e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 1e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true,
"supports_vision": true
},
"crusoe/meta-llama/Llama-3.3-70B-Instruct": {
"input_cost_per_token": 2e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 2e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/moonshotai/Kimi-K2-Thinking": {
"input_cost_per_token": 2.5e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 2.5e-06,
"supports_function_calling": false,
"supports_reasoning": true,
"supports_system_messages": true,
"supports_tool_choice": false
},
"crusoe/openai/gpt-oss-120b": {
"input_cost_per_token": 8e-07,
"litellm_provider": "crusoe",
"max_input_tokens": 131072,
"max_output_tokens": 131072,
"max_tokens": 131072,
"mode": "chat",
"output_cost_per_token": 8e-07,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507": {
"input_cost_per_token": 3e-06,
"litellm_provider": "crusoe",
"max_input_tokens": 262144,
"max_output_tokens": 262144,
"max_tokens": 262144,
"mode": "chat",
"output_cost_per_token": 3e-06,
"supports_function_calling": true,
"supports_parallel_function_calling": true,
"supports_system_messages": true,
"supports_tool_choice": true
},
"lambda_ai/deepseek-llama3.3-70b": {
"input_cost_per_token": 2e-07,
"litellm_provider": "lambda_ai",
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
+9 -13
View File
@@ -33,10 +33,12 @@ def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
When behind a proxy (like nginx), the proxy may set:
- X-Forwarded-Proto: The original protocol (http/https)
- X-Forwarded-Host: The original host (may include port)
- X-Forwarded-Port: The original port (if not in Host header)
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
@@ -47,34 +49,28 @@ def get_request_base_url(request: Request) -> str:
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
# Get forwarded headers
if not IPAddressUtils.is_request_from_trusted_proxy(request):
return base_url
x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
x_forwarded_host = request.headers.get("X-Forwarded-Host")
x_forwarded_port = request.headers.get("X-Forwarded-Port")
# Start with the original scheme
scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
# Handle host and port
if x_forwarded_host:
# X-Forwarded-Host may already include port (e.g., "example.com:8080")
if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
# Host includes port
netloc = x_forwarded_host
elif x_forwarded_port:
# Port is separate
netloc = f"{x_forwarded_host}:{x_forwarded_port}"
else:
# Just host, no explicit port
netloc = x_forwarded_host
else:
# No X-Forwarded-Host, use original netloc
netloc = parsed.netloc
if x_forwarded_port and ":" not in netloc:
# Add forwarded port if not already in netloc
netloc = f"{netloc}:{x_forwarded_port}"
# Reconstruct the URL
return urlunparse((scheme, netloc, parsed.path, "", "", ""))
litellm/proxy/_experimental/mcp_server/server.py
+41
View File
@@ -2138,6 +2138,47 @@ if MCP_AVAILABLE:
#########################################################
local_tool = global_mcp_tool_registry.get_tool(name)
if local_tool:
# OpenAPI-backed tools used to bypass `pre_call_tool_check` —
# only the managed path ran allowed/banned-tool checks, key/team
# tool permissions, and parameter validation. Run the same checks
# before dispatching to the local registry. Refuse the call if
# we cannot resolve a server: tools registered via
# openapi_to_mcp_generator are always tied to a server, so a
# missing mcp_server here means the tool->server mapping has
# not finished initializing or the registry entry is orphaned.
# Skipping the check would re-open the same authorization gap.
if mcp_server is None:
raise HTTPException(
status_code=503,
detail=(
f"MCP server for tool '{name}' is not available; "
"refusing to dispatch without authorization checks. "
"Retry once the server is registered."
),
)
# `pre_call_tool_check` calls into `proxy_logging_obj` for the
# pre-call guardrail hooks, so source it from the canonical
# `proxy_server` module the same way `_handle_managed_mcp_tool`
# does. `kwargs.get("proxy_logging_obj")` is None on the MCP
# entry path and would crash with AttributeError after the
# security checks pass.
from litellm.proxy.proxy_server import proxy_logging_obj
hook_result = await global_mcp_server_manager.pre_call_tool_check(
name=original_tool_name,
arguments=arguments or {},
server_name=server_name or mcp_server.name,
user_api_key_auth=user_api_key_auth,
proxy_logging_obj=proxy_logging_obj,
server=mcp_server,
raw_headers=raw_headers,
)
# `pre_call_tool_check` may return guardrail-modified
# arguments; honor them on the local path too.
if isinstance(hook_result, dict) and "arguments" in hook_result:
arguments = hook_result["arguments"]
verbose_logger.debug(f"Executing local registry tool: {name}")
# For BYOK servers the credential must be injected via a ContextVar
# because the tool function has headers baked into its closure.
litellm/proxy/_experimental/out/404.html → litellm/proxy/_experimental/out/404/index.html
View File
litellm/proxy/_experimental/out/_not-found.html → litellm/proxy/_experimental/out/_not-found/index.html
View File
litellm/proxy/_experimental/out/api-reference.html → litellm/proxy/_experimental/out/api-reference/index.html
View File
litellm/proxy/_experimental/out/experimental/api-playground.html → litellm/proxy/_experimental/out/experimental/api-playground/index.html
View File
litellm/proxy/_experimental/out/experimental/budgets.html → litellm/proxy/_experimental/out/experimental/budgets/index.html
View File
litellm/proxy/_experimental/out/experimental/caching.html → litellm/proxy/_experimental/out/experimental/caching/index.html
View File
litellm/proxy/_experimental/out/experimental/claude-code-plugins.html → litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
View File
litellm/proxy/_experimental/out/experimental/old-usage.html → litellm/proxy/_experimental/out/experimental/old-usage/index.html
View File
litellm/proxy/_experimental/out/experimental/prompts.html → litellm/proxy/_experimental/out/experimental/prompts/index.html
View File
litellm/proxy/_experimental/out/experimental/tag-management.html → litellm/proxy/_experimental/out/experimental/tag-management/index.html
View File
litellm/proxy/_experimental/out/guardrails.html → litellm/proxy/_experimental/out/guardrails/index.html
View File
litellm/proxy/_experimental/out/login.html → litellm/proxy/_experimental/out/login/index.html
View File
litellm/proxy/_experimental/out/logs.html → litellm/proxy/_experimental/out/logs/index.html
View File
litellm/proxy/_experimental/out/mcp/oauth/callback.html → litellm/proxy/_experimental/out/mcp/oauth/callback/index.html
View File
litellm/proxy/_experimental/out/model-hub.html → litellm/proxy/_experimental/out/model-hub/index.html
View File
litellm/proxy/_experimental/out/model_hub.html → litellm/proxy/_experimental/out/model_hub/index.html
View File
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View File
litellm/proxy/_experimental/out/models-and-endpoints.html → litellm/proxy/_experimental/out/models-and-endpoints/index.html
View File
litellm/proxy/_experimental/out/onboarding.html → litellm/proxy/_experimental/out/onboarding/index.html
View File
litellm/proxy/_experimental/out/organizations.html → litellm/proxy/_experimental/out/organizations/index.html
View File
litellm/proxy/_experimental/out/playground.html → litellm/proxy/_experimental/out/playground/index.html
View File
litellm/proxy/_experimental/out/policies.html → litellm/proxy/_experimental/out/policies/index.html
View File
litellm/proxy/_experimental/out/settings/admin-settings.html → litellm/proxy/_experimental/out/settings/admin-settings/index.html
View File
litellm/proxy/_experimental/out/settings/logging-and-alerts.html → litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html
View File
litellm/proxy/_experimental/out/settings/router-settings.html → litellm/proxy/_experimental/out/settings/router-settings/index.html
View File
litellm/proxy/_experimental/out/settings/ui-theme.html → litellm/proxy/_experimental/out/settings/ui-theme/index.html
View File
litellm/proxy/_experimental/out/skills.html → litellm/proxy/_experimental/out/skills/index.html
View File
litellm/proxy/_experimental/out/teams.html → litellm/proxy/_experimental/out/teams/index.html
View File
litellm/proxy/_experimental/out/test-key.html → litellm/proxy/_experimental/out/test-key/index.html
View File
litellm/proxy/_experimental/out/tools/mcp-servers.html → litellm/proxy/_experimental/out/tools/mcp-servers/index.html
View File

Some files were not shown because too many files have changed in this diff Show More