tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-07-09 07:06:51 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'litellm_internal_staging' into litellm_metrics_auth

Browse Source
This commit is contained in:
Shivam Rawat
2026-04-17 16:53:21 -07:00
committed by GitHub
parent 49ddb0e5ba 1c128a86b8
commit d3f81fa799
2005 changed files with 64297 additions and 40908 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+2 -154
View File
@@ -638,7 +638,7 @@ jobs:
username: ${DOCKERHUB_USERNAME}
password: ${DOCKERHUB_PASSWORD}
working_directory: ~/project
resource_class: large
resource_class: xlarge
steps:
- checkout
@@ -682,7 +682,7 @@ jobs:
for dir in "${IGNORE_DIRS[@]}"; do
IGNORE_ARGS="$IGNORE_ARGS --ignore=$dir"
done
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 8 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5
uv run --no-sync python -m pytest -v tests/llm_translation $IGNORE_ARGS --junitxml=test-results/junit.xml --durations=20 -n 4 --timeout=120 --timeout_method=thread --retries 2 --retry-delay 5 --max-worker-restart=5
no_output_timeout: 15m
# Store test results
@@ -2916,90 +2916,6 @@ jobs:
- codecov/upload:
file: ./coverage.xml
publish_proxy_extras:
docker:
- image: cimg/python:3.12
working_directory: ~/project/litellm-proxy-extras
environment:
TWINE_USERNAME: __token__
steps:
- checkout:
path: ~/project
- run:
name: Check if litellm-proxy-extras dir or pyproject.toml was modified
command: |
curl -LsSf -o /tmp/uv-install.sh https://astral.sh/uv/0.10.9/install.sh
echo "7fc46e39cb97290b57169c0c813a17970585ac519139f19006453c99b5f2f45f /tmp/uv-install.sh" | sha256sum -c -
env UV_NO_MODIFY_PATH=1 sh /tmp/uv-install.sh
rm -f /tmp/uv-install.sh
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
# Get current version from pyproject.toml
CURRENT_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
# Get last published version from PyPI
LAST_VERSION=$(curl -s https://pypi.org/pypi/litellm-proxy-extras/json | python -c "import json, sys; print(json.load(sys.stdin)['info']['version'])")
echo "Current version: $CURRENT_VERSION"
echo "Last published version: $LAST_VERSION"
# Compare versions using Python's packaging.version
VERSION_COMPARE=$(uv run --with 'packaging==25.0' python -c "from packaging import version; print(1 if version.parse('$CURRENT_VERSION') < version.parse('$LAST_VERSION') else 0)")
echo "Version compare: $VERSION_COMPARE"
if [ "$VERSION_COMPARE" = "1" ]; then
echo "Error: Current version ($CURRENT_VERSION) is less than last published version ($LAST_VERSION)"
exit 1
fi
# If versions are equal or current is greater, compare against the published package contents.
EXTRACTED_DIR=$(uv run --with "litellm-proxy-extras==$LAST_VERSION" python -c 'import importlib.util; from pathlib import Path; spec = importlib.util.find_spec("litellm_proxy_extras"); assert spec is not None and spec.origin is not None, "litellm_proxy_extras not found in uv-run environment"; print(Path(spec.origin).resolve().parent)')
# Compare contents
if ! diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras; then
if [ "$CURRENT_VERSION" = "$LAST_VERSION" ]; then
echo "Error: Changes detected in litellm-proxy-extras but version was not bumped"
echo "Current version: $CURRENT_VERSION"
echo "Last published version: $LAST_VERSION"
echo "Changes:"
diff -r "$EXTRACTED_DIR" ./litellm_proxy_extras
exit 1
fi
else
echo "No changes detected in litellm-proxy-extras. Skipping PyPI publish."
circleci step halt
fi
- run:
name: Get new version
command: |
NEW_VERSION=$(python -c 'import tomllib; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); print(data["project"]["version"])')
echo "export NEW_VERSION=$NEW_VERSION" >> $BASH_ENV
- run:
name: Check if versions match
command: |
cd ~/project
# Check pyproject.toml
CURRENT_VERSION=$(uv run --with 'packaging==25.0' python -c 'import tomllib; from packaging.requirements import Requirement; from pathlib import Path; data = tomllib.loads(Path("pyproject.toml").read_text()); matches = [spec.version for requirement in data["project"]["optional-dependencies"]["proxy"] for parsed in [Requirement(requirement)] if parsed.name == "litellm-proxy-extras" and parsed.specifier for spec in parsed.specifier if spec.operator == "=="]; print(matches[0] if matches else (_ for _ in ()).throw(SystemExit("Could not find exact litellm-proxy-extras pin in project.optional-dependencies.proxy")))')
if [ "$CURRENT_VERSION" != "$NEW_VERSION" ]; then
echo "Error: Version in pyproject.toml ($CURRENT_VERSION) doesn't match new version ($NEW_VERSION)"
exit 1
fi
- run:
name: Publish to PyPI
command: |
echo -e "[pypi]\nusername = $PYPI_PUBLISH_USERNAME\npassword = $PYPI_PUBLISH_PASSWORD" > ~/.pypirc
echo 'export PATH="$HOME/.local/bin:$PATH"' >> "$BASH_ENV"
export PATH="$HOME/.local/bin:$PATH"
rm -rf build dist
uv build
uv tool run --from 'twine==6.2.0' twine upload --verbose dist/*
ui_build:
docker:
- image: cimg/node:20.19
@@ -3214,60 +3130,6 @@ jobs:
- litellm-docker-database.tar.zst
prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
resource_class: medium
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=litellm_schema_sync \
-p 5432:5432 \
postgres:14
- wait_for_service:
url: tcp://localhost:5432
timeout: "60"
- run:
name: Load Docker Database Image
command: |
zstd -d litellm-docker-database.tar.zst --stdout | docker load
docker images | grep litellm-docker-database
- run:
name: Run schema sync via prisma db push
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL="postgresql://postgres:postgres@host.docker.internal:5432/litellm_schema_sync" \
-e LITELLM_MASTER_KEY="sk-1234" \
--name schema-sync \
--add-host=host.docker.internal:host-gateway \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--use_prisma_db_push
- run:
name: Start outputting logs
command: docker logs -f schema-sync
background: true
- wait_for_service:
url: http://localhost:4000
timeout: "300"
- run:
name: Stop schema sync container
command: docker stop schema-sync
test_bad_database_url:
machine:
image: ubuntu-2204:2023.10.1
@@ -3421,14 +3283,6 @@ workflows:
only:
- main
- /litellm_.*/
- prisma_schema_sync:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
filters:
branches:
@@ -3688,9 +3542,3 @@ workflows:
only:
- main
- /litellm_.*/
- publish_proxy_extras:
filters:
branches:
only:
- main
- /litellm_release_day_.*/
.github/scripts/close_duplicate_issues.py
+31 -9
View File
@@ -42,7 +42,9 @@ def gh(*args: str) -> str:
def fetch_open_issues(repo: str | None) -> list[dict]:
"""Fetch all open issues (excluding PRs) via gh api --paginate."""
if repo:
endpoint = f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
endpoint = (
f"repos/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
)
else:
endpoint = "repos/{owner}/{repo}/issues?state=open&per_page=100&sort=created&direction=asc"
cmd = ["api", "--paginate", endpoint]
@@ -71,7 +73,9 @@ def close_as_duplicate(
repo_args = ["--repo", repo] if repo else []
if dry_run:
print(f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}")
print(
f" [DRY RUN] Would close #{issue_number} as duplicate of #{duplicate_of}"
)
return
# Add comment
@@ -115,7 +119,9 @@ def find_duplicate(
return None
def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bool) -> int:
def scan_all(
issues: list[dict], threshold: float, repo: str | None, dry_run: bool
) -> int:
"""Compare every issue against all older issues. Returns count of duplicates found."""
# Sort oldest first
issues.sort(key=lambda i: i["number"])
@@ -144,7 +150,11 @@ def scan_all(issues: list[dict], threshold: float, repo: str | None, dry_run: bo
def check_single(
issue_number: int, issues: list[dict], threshold: float, repo: str | None, dry_run: bool
issue_number: int,
issues: list[dict],
threshold: float,
repo: str | None,
dry_run: bool,
) -> bool:
"""Check a single issue against all older open issues. Returns True if duplicate found."""
target = None
@@ -178,13 +188,23 @@ def check_single(
def main() -> None:
parser = argparse.ArgumentParser(description="Detect and close duplicate GitHub issues")
parser = argparse.ArgumentParser(
description="Detect and close duplicate GitHub issues"
)
mode = parser.add_mutually_exclusive_group(required=True)
mode.add_argument("--scan", action="store_true", help="Scan all open issues")
mode.add_argument("--issue-number", type=int, help="Check a single issue number")
parser.add_argument("--threshold", type=float, default=0.85, help="Similarity threshold (0-1)")
parser.add_argument("--close", action="store_true", help="Actually close duplicates (default is dry-run)")
parser.add_argument("--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted.")
parser.add_argument(
"--threshold", type=float, default=0.85, help="Similarity threshold (0-1)"
)
parser.add_argument(
"--close",
action="store_true",
help="Actually close duplicates (default is dry-run)",
)
parser.add_argument(
"--repo", type=str, help="Repository (owner/repo). Auto-detected if omitted."
)
args = parser.parse_args()
dry_run = not args.close
@@ -200,7 +220,9 @@ def main() -> None:
count = scan_all(issues, args.threshold, args.repo, dry_run)
print(f"\nTotal duplicates {'found' if dry_run else 'closed'}: {count}")
else:
found = check_single(args.issue_number, issues, args.threshold, args.repo, dry_run)
found = check_single(
args.issue_number, issues, args.threshold, args.repo, dry_run
)
sys.exit(0 if found else 0) # Always exit 0; finding no dup is not an error
.github/scripts/scan_keywords.py
+14 -6
View File
@@ -67,14 +67,13 @@ def send_webhook(webhook_url: str, payload: dict) -> None:
def _excerpt(text: str, max_len: int = 400) -> str:
if not text:
return ""
# Keep original formatting
if len(text) <= max_len:
return text
return text[: max_len - 1] + "…"
def main() -> int:
event = read_event_payload()
if not event:
@@ -87,8 +86,19 @@ def main() -> int:
# Keywords from env or defaults
keywords_env = os.environ.get("KEYWORDS", "")
default_keywords = ["azure", "openai", "bedrock", "vertexai", "vertex ai", "anthropic"]
keywords = [k.strip() for k in keywords_env.split(",")] if keywords_env else default_keywords
default_keywords = [
"azure",
"openai",
"bedrock",
"vertexai",
"vertex ai",
"anthropic",
]
keywords = (
[k.strip() for k in keywords_env.split(",")]
if keywords_env
else default_keywords
)
matches = detect_keywords(combined_text, keywords)
found = bool(matches)
@@ -129,5 +139,3 @@ def main() -> int:
if __name__ == "__main__":
raise SystemExit(main())
.github/workflows/guard-main-branch.yml
+42
View File
@@ -0,0 +1,42 @@
name: Guard main branch
on:
pull_request:
branches:
- main
merge_group:
permissions: {}
# DO NOT RENAME the job's `name:` — it is referenced by GitHub branch
# protection as a required status check on `main`. Renaming silently
# breaks the gate.
jobs:
guard:
name: Verify PR source branch
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- name: Reject merge_group events
if: github.event_name == 'merge_group'
run: |
echo "::error::Merge queue is not supported for main. Disable merge queue or update this guard."
exit 1
- name: Check head branch name
env:
HEAD_REF: ${{ github.head_ref }}
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
BASE_REPO: ${{ github.repository }}
run: |
echo "PR head repo: $HEAD_REPO"
echo "PR head branch: $HEAD_REF"
if [ "$HEAD_REPO" != "$BASE_REPO" ]; then
echo "::error::PRs to main must originate from the canonical repository ($BASE_REPO), not a fork ($HEAD_REPO). External contributors should open PRs against the 'litellm_oss_branch' branch instead."
exit 1
fi
if [ "$HEAD_REF" = "litellm_internal_staging" ] || [[ "$HEAD_REF" == litellm_hotfix_?* ]]; then
echo "Allowed source branch."
exit 0
fi
echo "::error::PRs to main must originate from 'litellm_internal_staging' or a 'litellm_hotfix_*' branch. Got: '$HEAD_REF'. If this is a contribution, retarget the PR against 'litellm_oss_branch' instead."
exit 1
.github/workflows/test-linting.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: LiteLLM Linting
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-litellm-ui-build.yml
+5 -1
View File
@@ -4,7 +4,11 @@ permissions:
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
jobs:
build-ui:
.github/workflows/test-mcp.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: LiteLLM MCP Tests (folder - tests/mcp_tests)
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-model-map.yaml
+5 -1
View File
@@ -2,7 +2,11 @@ name: Validate model_prices_and_context_window.json
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-core-utils.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Core Utilities"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-documentation.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Documentation Validation"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-enterprise-routing.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Enterprise, Google GenAI & Routing"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-integrations.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Integrations (Callbacks & Logging)"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-llm-providers.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: LLM Provider Transformations"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-misc.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: MCP, Secrets, Containers & Misc"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-proxy-auth.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Auth & Key Management"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-proxy-db.yml
+1 -1
View File
@@ -3,7 +3,7 @@ name: "Unit Tests: Proxy DB Operations"
# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
on:
push:
branches: [main, "litellm_*"]
branches: [main, "litellm_**"]
permissions:
contents: read
.github/workflows/test-unit-proxy-endpoints.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy API Endpoints"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-proxy-infra.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Infrastructure"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-proxy-legacy.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Proxy Legacy Tests"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-responses-caching-types.yml
+5 -1
View File
@@ -2,7 +2,11 @@ name: "Unit Tests: Responses, Caching & Types"
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
permissions:
contents: read
.github/workflows/test-unit-security.yml
+1 -1
View File
@@ -3,7 +3,7 @@ name: "Unit Tests: Security"
# Uses DATABASE_URL secret — only runs on trusted branches, not PRs.
on:
push:
branches: [main, "litellm_*"]
branches: [main, "litellm_**"]
permissions:
contents: read
.github/workflows/test_server_root_path.yml
+5 -1
View File
@@ -4,7 +4,11 @@ permissions:
on:
pull_request:
branches: [main]
branches:
- main
- litellm_internal_staging
- litellm_oss_branch
- "litellm_**"
jobs:
test-server-root-path:
ci_cd/run_migration.py
+3 -1
View File
@@ -17,7 +17,9 @@ def create_migration(migration_name: str = None):
try:
# Get paths
root_dir = Path(__file__).parent.parent
migrations_dir = root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
migrations_dir = (
root_dir / "litellm-proxy-extras" / "litellm_proxy_extras" / "migrations"
)
schema_path = root_dir / "schema.prisma"
# Create temporary PostgreSQL database
cookbook/anthropic_agent_sdk/agent_with_mcp.py
+27 -23
View File
@@ -24,24 +24,26 @@ async def interactive_chat_with_mcp():
Interactive CLI chat with the agent and MCP server
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
available_models = await fetch_available_models(
litellm_base_url, config.LITELLM_API_KEY
)
current_model = config.LITELLM_MODEL
# MCP server configuration
mcp_server_url = f"{litellm_base_url}/mcp/deepwiki2"
use_mcp = os.getenv("USE_MCP", "true").lower() == "true"
if not use_mcp:
print("⚠️ MCP disabled via USE_MCP=false")
print_header(litellm_base_url, current_model, has_mcp=use_mcp)
while True:
# Configure agent options
if use_mcp:
@@ -58,7 +60,7 @@ async def interactive_chat_with_mcp():
"url": mcp_server_url,
"headers": {
"Authorization": f"Bearer {config.LITELLM_API_KEY}"
}
},
}
},
)
@@ -78,12 +80,12 @@ async def interactive_chat_with_mcp():
model=current_model,
max_turns=50,
)
# Create agent client
try:
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
@@ -91,34 +93,36 @@ async def interactive_chat_with_mcp():
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
if user_input.lower() in ["quit", "exit"]:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
if user_input.lower() == "clear":
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
if user_input.lower() == "models":
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if user_input.lower() == "model":
new_model, should_restart = handle_model_switch(
available_models, current_model
)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
except Exception as e:
print(f"\n❌ Error creating agent client: {e}")
print("This might be an MCP configuration issue. Try running without MCP:")
cookbook/anthropic_agent_sdk/common.py
+33 -29
View File
@@ -8,13 +8,13 @@ import httpx
class Config:
"""Configuration for LiteLLM Gateway connection"""
# LiteLLM proxy URL (default to local instance)
LITELLM_PROXY_URL = os.getenv("LITELLM_PROXY_URL", "http://localhost:4000")
# LiteLLM API key (master key or virtual key)
LITELLM_API_KEY = os.getenv("LITELLM_API_KEY", "sk-1234")
# Model name as configured in LiteLLM (e.g., "bedrock-claude-sonnet-4", "gpt-4", etc.)
LITELLM_MODEL = os.getenv("LITELLM_MODEL", "bedrock-claude-sonnet-4.5")
@@ -28,7 +28,7 @@ async def fetch_available_models(base_url: str, api_key: str) -> list[str]:
response = await client.get(
f"{base_url}/models",
headers={"Authorization": f"Bearer {api_key}"},
timeout=10.0
timeout=10.0,
)
response.raise_for_status()
data = response.json()
@@ -50,7 +50,7 @@ def setup_litellm_env(config: Config):
"""
Configure environment variables to point Agent SDK to LiteLLM
"""
litellm_base_url = config.LITELLM_PROXY_URL.rstrip('/')
litellm_base_url = config.LITELLM_PROXY_URL.rstrip("/")
os.environ["ANTHROPIC_BASE_URL"] = litellm_base_url
os.environ["ANTHROPIC_API_KEY"] = config.LITELLM_API_KEY
return litellm_base_url
@@ -87,10 +87,12 @@ def handle_model_list(available_models: list[str], current_model: str):
print(f" {marker} {i}. {model}")
def handle_model_switch(available_models: list[str], current_model: str) -> tuple[str, bool]:
def handle_model_switch(
available_models: list[str], current_model: str
) -> tuple[str, bool]:
"""
Handle model switching
Returns:
tuple: (new_model, should_restart_conversation)
"""
@@ -98,7 +100,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
for i, model in enumerate(available_models, 1):
marker = "✓" if model == current_model else " "
print(f" {marker} {i}. {model}")
try:
choice = input("\nEnter number (or press Enter to cancel): ").strip()
if choice:
@@ -112,7 +114,7 @@ def handle_model_switch(available_models: list[str], current_model: str) -> tupl
print("❌ Invalid choice")
except (ValueError, IndexError):
print("❌ Invalid input")
return current_model, False
@@ -120,41 +122,43 @@ async def stream_response(client, user_input: str):
"""
Stream response from the agent
"""
print("\n🤖 Assistant: ", end='', flush=True)
print("\n🤖 Assistant: ", end="", flush=True)
try:
await client.query(user_input)
# Show loading indicator
print("⏳ thinking...", end='', flush=True)
print("⏳ thinking...", end="", flush=True)
# Stream the response
first_chunk = True
async for msg in client.receive_response():
# Clear loading indicator on first message
if first_chunk:
print("\r🤖 Assistant: ", end='', flush=True)
print("\r🤖 Assistant: ", end="", flush=True)
first_chunk = False
# Handle different message types
if hasattr(msg, 'type'):
if msg.type == 'content_block_delta':
if hasattr(msg, "type"):
if msg.type == "content_block_delta":
# Streaming text delta
if hasattr(msg, 'delta') and hasattr(msg.delta, 'text'):
print(msg.delta.text, end='', flush=True)
elif msg.type == 'content_block_start':
if hasattr(msg, "delta") and hasattr(msg.delta, "text"):
print(msg.delta.text, end="", flush=True)
elif msg.type == "content_block_start":
# Start of content block
if hasattr(msg, 'content_block') and hasattr(msg.content_block, 'text'):
print(msg.content_block.text, end='', flush=True)
if hasattr(msg, "content_block") and hasattr(
msg.content_block, "text"
):
print(msg.content_block.text, end="", flush=True)
# Fallback to original content handling
if hasattr(msg, 'content'):
if hasattr(msg, "content"):
for content_block in msg.content:
if hasattr(content_block, 'text'):
print(content_block.text, end='', flush=True)
if hasattr(content_block, "text"):
print(content_block.text, end="", flush=True)
print() # New line after response
except Exception as e:
print(f"\r\n❌ Error: {e}")
print("Please check your LiteLLM gateway is running and configured correctly.")
cookbook/anthropic_agent_sdk/main.py
+23 -19
View File
@@ -24,17 +24,19 @@ async def interactive_chat():
Interactive CLI chat with the agent
"""
config = Config()
# Configure Anthropic SDK to point to LiteLLM gateway
litellm_base_url = setup_litellm_env(config)
# Fetch available models from proxy
available_models = await fetch_available_models(litellm_base_url, config.LITELLM_API_KEY)
available_models = await fetch_available_models(
litellm_base_url, config.LITELLM_API_KEY
)
current_model = config.LITELLM_MODEL
print_header(litellm_base_url, current_model)
while True:
# Configure agent options for each conversation
options = ClaudeAgentOptions(
@@ -42,11 +44,11 @@ async def interactive_chat():
model=current_model,
max_turns=50,
)
# Create agent client
async with ClaudeSDKClient(options=options) as client:
conversation_active = True
while conversation_active:
# Get user input
try:
@@ -54,31 +56,33 @@ async def interactive_chat():
except (EOFError, KeyboardInterrupt):
print("\n\n👋 Goodbye!")
return
# Handle commands
if user_input.lower() in ['quit', 'exit']:
if user_input.lower() in ["quit", "exit"]:
print("\n👋 Goodbye!")
return
if user_input.lower() == 'clear':
if user_input.lower() == "clear":
print("\n🔄 Starting new conversation...\n")
conversation_active = False
continue
if user_input.lower() == 'models':
if user_input.lower() == "models":
handle_model_list(available_models, current_model)
continue
if user_input.lower() == 'model':
new_model, should_restart = handle_model_switch(available_models, current_model)
if user_input.lower() == "model":
new_model, should_restart = handle_model_switch(
available_models, current_model
)
if should_restart:
current_model = new_model
conversation_active = False
continue
if not user_input:
continue
# Stream response from agent
await stream_response(client, user_input)
cookbook/litellm_proxy_server/batch_api/bedrock/bedrock.py
+3 -3
View File
@@ -11,15 +11,15 @@ BEDROCK_BATCH_MODEL = "bedrock/batch-anthropic.claude-3-5-sonnet-20240620-v1:0"
batch_input_file = client.files.create(
file=open("./bedrock_batch_completions.jsonl", "rb"),
purpose="batch",
extra_body={"target_model_names": BEDROCK_BATCH_MODEL}
extra_body={"target_model_names": BEDROCK_BATCH_MODEL},
)
print(batch_input_file)
# Create batch
batch = client.batches.create(
batch = client.batches.create(
input_file_id=batch_input_file.id,
endpoint="/v1/chat/completions",
completion_window="24h",
metadata={"description": "Test batch job"},
)
print(batch)
print(batch)
cookbook/litellm_proxy_server/cli_token_usage.py
+10 -9
View File
@@ -8,6 +8,7 @@ in your Python scripts after running `litellm-proxy login`.
from textwrap import indent
import litellm
LITELLM_BASE_URL = "http://localhost:4000/"
@@ -15,38 +16,38 @@ def main():
"""Using CLI token with LiteLLM SDK"""
print("🚀 Using CLI Token with LiteLLM SDK")
print("=" * 40)
#litellm._turn_on_debug()
# litellm._turn_on_debug()
# Get the CLI token
api_key = litellm.get_litellm_gateway_api_key()
if not api_key:
print("❌ No CLI token found. Please run 'litellm-proxy login' first.")
return
print("✅ Found CLI token.")
available_models = litellm.get_valid_models(
check_provider_endpoint=True,
custom_llm_provider="litellm_proxy",
api_key=api_key,
api_base=LITELLM_BASE_URL
api_base=LITELLM_BASE_URL,
)
print("✅ Available models:")
if available_models:
for i, model in enumerate(available_models, 1):
print(f" {i:2d}. {model}")
else:
print(" No models available")
# Use with LiteLLM
try:
response = litellm.completion(
model="litellm_proxy/gemini/gemini-2.5-flash",
messages=[{"role": "user", "content": "Hello from CLI token!"}],
api_key=api_key,
base_url=LITELLM_BASE_URL
base_url=LITELLM_BASE_URL,
)
print(f"✅ LLM Response: {response.model_dump_json(indent=4)}")
except Exception as e:
@@ -55,7 +56,7 @@ def main():
if __name__ == "__main__":
main()
print("\n💡 Tips:")
print("1. Run 'litellm-proxy login' to authenticate first")
print("2. Replace 'https://your-proxy.com' with your actual proxy URL")
cookbook/litellm_proxy_server/mcp/mcp_with_litellm_proxy.py
+6 -5
View File
@@ -3,11 +3,12 @@ Use LiteLLM Proxy MCP Gateway to call MCP tools.
When using LiteLLM Proxy, you can use the same MCP tools across all your LLM providers.
"""
import openai
client = openai.OpenAI(
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000" # paste your litellm proxy base url here
api_key="sk-1234", # paste your litellm proxy api key here
base_url="http://localhost:4000", # paste your litellm proxy base url here
)
print("Making API request to Responses API with MCP tools")
@@ -17,7 +18,7 @@ response = client.responses.create(
{
"role": "user",
"content": "give me TLDR of what BerriAI/litellm repo is about",
"type": "message"
"type": "message",
}
],
tools=[
@@ -25,11 +26,11 @@ response = client.responses.create(
"type": "mcp",
"server_label": "litellm",
"server_url": "litellm_proxy",
"require_approval": "never"
"require_approval": "never",
}
],
stream=True,
tool_choice="required"
tool_choice="required",
)
for chunk in response:
cookbook/litellm_proxy_server/secret_manager/my_secret_manager.py
+4 -3
View File
@@ -40,8 +40,10 @@ class InMemorySecretManager(CustomSecretManager):
) -> Optional[str]:
"""Read secret synchronously"""
from litellm._logging import verbose_proxy_logger
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}")
verbose_proxy_logger.info(
f"CUSTOM SECRET MANAGER: LOOKING FOR SECRET: {secret_name}"
)
value = self.secrets.get(secret_name)
verbose_proxy_logger.info(f"CUSTOM SECRET MANAGER: READ SECRET: {value}")
return value
@@ -76,4 +78,3 @@ class InMemorySecretManager(CustomSecretManager):
del self.secrets[secret_name]
return {"status": "deleted", "secret_name": secret_name}
return {"status": "not_found", "secret_name": secret_name}
cookbook/livekit_agent_sdk/main.py
+42 -33
View File
@@ -5,6 +5,7 @@ This example shows how to use LiveKit's xAI realtime plugin through LiteLLM prox
LiteLLM acts as a unified interface, allowing you to switch between xAI, OpenAI,
and Azure realtime APIs without changing your agent code.
"""
import asyncio
import json
import os
@@ -23,71 +24,79 @@ async def run_voice_agent():
2. Sends a user message
3. Streams back the response
"""
url = f"ws://{PROXY_URL.replace('http://', '').replace('https://', '')}/v1/realtime?model={MODEL}"
headers = {"Authorization": f"Bearer {API_KEY}"}
print(f"🎙️ Connecting to voice agent...")
print(f" Model: {MODEL}")
print(f" Proxy: {PROXY_URL}")
print()
async with websockets.connect(url, additional_headers=headers) as ws:
# Receive initial connection event
initial = json.loads(await ws.recv())
print(f"✅ Connected! Event: {initial['type']}\n")
# Get user input
user_message = input("💬 Your message: ").strip()
if not user_message:
user_message = "Tell me a fun fact about AI!"
print(f"\n🤖 Sending to {MODEL}...\n")
# Send user message
await ws.send(json.dumps({
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}]
}
}))
await ws.send(
json.dumps(
{
"type": "conversation.item.create",
"item": {
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": user_message}],
},
}
)
)
# Request response
await ws.send(json.dumps({
"type": "response.create",
"response": {"modalities": ["text", "audio"]}
}))
await ws.send(
json.dumps(
{
"type": "response.create",
"response": {"modalities": ["text", "audio"]},
}
)
)
# Stream response
print("🎤 Response: ", end='', flush=True)
print("🎤 Response: ", end="", flush=True)
transcript = []
try:
while True:
msg = await asyncio.wait_for(ws.recv(), timeout=15.0)
event = json.loads(msg)
# Capture transcript deltas
if event['type'] == 'response.output_audio_transcript.delta':
delta = event.get('delta', '')
if event["type"] == "response.output_audio_transcript.delta":
delta = event.get("delta", "")
if delta:
print(delta, end='', flush=True)
print(delta, end="", flush=True)
transcript.append(delta)
# Done when response completes
elif event['type'] == 'response.done':
elif event["type"] == "response.done":
break
except asyncio.TimeoutError:
pass
print("\n")
if transcript:
print(f"✅ Complete response: {''.join(transcript)}")
await ws.close()
@@ -97,7 +106,7 @@ def main():
print("LiveKit xAI Voice Agent via LiteLLM Proxy")
print("=" * 70)
print()
try:
asyncio.run(run_voice_agent())
except KeyboardInterrupt:
cookbook/misc/test_responses_api.py
+5 -9
View File
@@ -1,10 +1,9 @@
import base64
from openai import OpenAI
import time
client = OpenAI(
base_url="http://0.0.0.0:4001",
api_key="sk-1234"
)
client = OpenAI(base_url="http://0.0.0.0:4001", api_key="sk-1234")
# Function to encode the image
def encode_image(image_path):
@@ -25,7 +24,7 @@ response = client.responses.create(
{
"role": "user",
"content": [
{ "type": "input_text", "text": "what color is the image"},
{"type": "input_text", "text": "what color is the image"},
{
"type": "input_image",
"image_url": f"data:image/jpeg;base64,{base64_image}",
@@ -36,7 +35,6 @@ response = client.responses.create(
)
print(response.output_text)
print("response1 id===", response.id)
print("sleeping for 20 seconds...")
@@ -45,9 +43,7 @@ print("making follow up request for existing id")
response2 = client.responses.create(
model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
previous_response_id=response.id,
input="ok, and what objects are in the image?"
input="ok, and what objects are in the image?",
)
print(response2.output_text)
cookbook/nova_sonic_realtime.py
+7 -4
View File
@@ -52,11 +52,11 @@ class RealtimeClient:
async def connect(self):
"""Connect to LiteLLM proxy realtime endpoint."""
print(f"Connecting to {self.url}...")
headers = {}
if self.api_key:
headers["Authorization"] = f"Bearer {self.api_key}"
self.ws = await websockets.connect(
self.url,
additional_headers=headers,
@@ -175,7 +175,9 @@ class RealtimeClient:
try:
while self.is_active:
audio_data = self.input_stream.read(CHUNK_SIZE, exception_on_overflow=False)
audio_data = self.input_stream.read(
CHUNK_SIZE, exception_on_overflow=False
)
await self.send_audio_chunk(audio_data)
await asyncio.sleep(0.01) # Small delay to prevent overwhelming
except Exception as e:
@@ -270,6 +272,7 @@ async def main():
except Exception as e:
print(f"\n❌ Error: {e}")
import traceback
traceback.print_exc()
finally:
await client.close()
@@ -281,7 +284,7 @@ if __name__ == "__main__":
print("2. Bedrock is configured in proxy_server_config.yaml")
print("3. AWS credentials are set")
print()
try:
asyncio.run(main())
except KeyboardInterrupt:
cookbook/veo_video_generation.py
+87 -77
View File
@@ -21,49 +21,45 @@ from typing import Optional
class VeoVideoGenerator:
"""Complete Veo video generation client using LiteLLM proxy."""
def __init__(self, base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234"):
def __init__(
self,
base_url: str = "http://localhost:4000/gemini/v1beta",
api_key: str = "sk-1234",
):
"""
Initialize the Veo video generator.
Args:
base_url: Base URL for the LiteLLM proxy with Gemini pass-through
api_key: API key for LiteLLM proxy authentication
"""
self.base_url = base_url
self.api_key = api_key
self.headers = {
"x-goog-api-key": api_key,
"Content-Type": "application/json"
}
self.headers = {"x-goog-api-key": api_key, "Content-Type": "application/json"}
def generate_video(self, prompt: str) -> Optional[str]:
"""
Initiate video generation with Veo.
Args:
prompt: Text description of the video to generate
Returns:
Operation name if successful, None otherwise
"""
print(f"🎬 Generating video with prompt: '{prompt}'")
url = f"{self.base_url}/models/veo-3.0-generate-preview:predictLongRunning"
payload = {
"instances": [{
"prompt": prompt
}]
}
payload = {"instances": [{"prompt": prompt}]}
try:
response = requests.post(url, headers=self.headers, json=payload)
response.raise_for_status()
data = response.json()
operation_name = data.get("name")
if operation_name:
print(f"✅ Video generation started: {operation_name}")
return operation_name
@@ -71,58 +67,64 @@ class VeoVideoGenerator:
print("❌ No operation name returned")
print(f"Response: {json.dumps(data, indent=2)}")
return None
except requests.RequestException as e:
print(f"❌ Failed to start video generation: {e}")
if hasattr(e, 'response') and e.response is not None:
if hasattr(e, "response") and e.response is not None:
try:
error_data = e.response.json()
print(f"Error details: {json.dumps(error_data, indent=2)}")
except:
print(f"Error response: {e.response.text}")
return None
def wait_for_completion(self, operation_name: str, max_wait_time: int = 600) -> Optional[str]:
def wait_for_completion(
self, operation_name: str, max_wait_time: int = 600
) -> Optional[str]:
"""
Poll operation status until video generation is complete.
Args:
operation_name: Name of the operation to monitor
max_wait_time: Maximum time to wait in seconds (default: 10 minutes)
Returns:
Video URI if successful, None otherwise
"""
print("⏳ Waiting for video generation to complete...")
operation_url = f"{self.base_url}/{operation_name}"
start_time = time.time()
poll_interval = 10 # Start with 10 seconds
while time.time() - start_time < max_wait_time:
try:
print(f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)")
print(
f"🔍 Polling status... ({int(time.time() - start_time)}s elapsed)"
)
response = requests.get(operation_url, headers=self.headers)
response.raise_for_status()
data = response.json()
# Check for errors
if "error" in data:
print("❌ Error in video generation:")
print(json.dumps(data["error"], indent=2))
return None
# Check if operation is complete
is_done = data.get("done", False)
if is_done:
print("🎉 Video generation complete!")
try:
# Extract video URI from nested response
video_uri = data["response"]["generateVideoResponse"]["generatedSamples"][0]["video"]["uri"]
video_uri = data["response"]["generateVideoResponse"][
"generatedSamples"
][0]["video"]["uri"]
print(f"📹 Video URI: {video_uri}")
return video_uri
except KeyError as e:
@@ -130,64 +132,68 @@ class VeoVideoGenerator:
print("Full response:")
print(json.dumps(data, indent=2))
return None
# Wait before next poll, with exponential backoff
time.sleep(poll_interval)
poll_interval = min(poll_interval * 1.2, 30) # Cap at 30 seconds
except requests.RequestException as e:
print(f"❌ Error polling operation status: {e}")
time.sleep(poll_interval)
print(f"⏰ Timeout after {max_wait_time} seconds")
return None
def download_video(self, video_uri: str, output_filename: str = "generated_video.mp4") -> bool:
def download_video(
self, video_uri: str, output_filename: str = "generated_video.mp4"
) -> bool:
"""
Download the generated video file.
Args:
video_uri: URI of the video to download (from Google's response)
output_filename: Local filename to save the video
Returns:
True if download successful, False otherwise
"""
print(f"⬇️ Downloading video...")
print(f"Original URI: {video_uri}")
# Convert Google URI to LiteLLM proxy URI
# Example: files/abc123 -> /gemini/v1beta/files/abc123:download?alt=media
if video_uri.startswith("files/"):
download_path = f"{video_uri}:download?alt=media"
else:
download_path = video_uri
litellm_download_url = f"{self.base_url}/{download_path}"
print(f"Download URL: {litellm_download_url}")
try:
# Download with streaming and redirect handling
response = requests.get(
litellm_download_url,
headers=self.headers,
litellm_download_url,
headers=self.headers,
stream=True,
allow_redirects=True # Handle redirects automatically
allow_redirects=True, # Handle redirects automatically
)
response.raise_for_status()
# Save video file
with open(output_filename, 'wb') as f:
with open(output_filename, "wb") as f:
downloaded_size = 0
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded_size += len(chunk)
# Progress indicator for large files
if downloaded_size % (1024 * 1024) == 0: # Every MB
print(f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB...")
print(
f"📦 Downloaded {downloaded_size / (1024*1024):.1f} MB..."
)
# Verify file was created and has content
if os.path.exists(output_filename):
file_size = os.path.getsize(output_filename)
@@ -203,48 +209,52 @@ class VeoVideoGenerator:
else:
print("❌ File was not created")
return False
except requests.RequestException as e:
print(f"❌ Download failed: {e}")
if hasattr(e, 'response') and e.response is not None:
if hasattr(e, "response") and e.response is not None:
print(f"Status code: {e.response.status_code}")
print(f"Response headers: {dict(e.response.headers)}")
return False
def generate_and_download(self, prompt: str, output_filename: str = None) -> bool:
"""
Complete workflow: generate video and download it.
Args:
prompt: Text description for video generation
output_filename: Output filename (auto-generated if None)
Returns:
True if successful, False otherwise
"""
# Auto-generate filename if not provided
if output_filename is None:
timestamp = int(time.time())
safe_prompt = "".join(c for c in prompt[:30] if c.isalnum() or c in (' ', '-', '_')).rstrip()
output_filename = f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
safe_prompt = "".join(
c for c in prompt[:30] if c.isalnum() or c in (" ", "-", "_")
).rstrip()
output_filename = (
f"veo_video_{safe_prompt.replace(' ', '_')}_{timestamp}.mp4"
)
print("=" * 60)
print("🎬 VEO VIDEO GENERATION WORKFLOW")
print("=" * 60)
# Step 1: Generate video
operation_name = self.generate_video(prompt)
if not operation_name:
return False
# Step 2: Wait for completion
video_uri = self.wait_for_completion(operation_name)
if not video_uri:
return False
# Step 3: Download video
success = self.download_video(video_uri, output_filename)
if success:
print("=" * 60)
print("🎉 SUCCESS! Video generation complete!")
@@ -254,51 +264,51 @@ class VeoVideoGenerator:
print("=" * 60)
print("❌ FAILED! Video generation or download failed")
print("=" * 60)
return success
def main():
"""
Example usage of the VeoVideoGenerator.
Configure these environment variables:
- LITELLM_BASE_URL: Your LiteLLM proxy URL (default: http://localhost:4000/gemini/v1beta)
- LITELLM_API_KEY: Your LiteLLM API key (default: sk-1234)
"""
# Configuration from environment or defaults
base_url = os.getenv("LITELLM_BASE_URL", "http://localhost:4000/gemini/v1beta")
api_key = os.getenv("LITELLM_API_KEY", "sk-1234")
print("🚀 Starting Veo Video Generation Example")
print(f"📡 Using LiteLLM proxy at: {base_url}")
# Initialize generator
generator = VeoVideoGenerator(base_url=base_url, api_key=api_key)
# Example prompts - try different ones!
example_prompts = [
"A cat playing with a ball of yarn in a sunny garden",
"Ocean waves crashing against rocky cliffs at sunset",
"A bustling city street with people walking and cars passing by",
"A peaceful forest with sunlight filtering through the trees"
"A peaceful forest with sunlight filtering through the trees",
]
# Use first example or get from user
prompt = example_prompts[0]
print(f"🎬 Using prompt: '{prompt}'")
# Generate and download video
success = generator.generate_and_download(prompt)
if success:
print("\n✅ Example completed successfully!")
print("💡 Try modifying the prompt in the script for different videos!")
else:
print("\n❌ Example failed!")
print("🔧 Check your LiteLLM proxy configuration and Google AI Studio API key")
# Troubleshooting tips
print("\n🔍 Troubleshooting:")
print("1. Ensure LiteLLM proxy is running with Google AI Studio pass-through")
docs/my-website/blog/claude_opus_4_7/index.md
+366
View File
@@ -0,0 +1,366 @@
---
slug: claude_opus_4_7
title: "Day 0 Support: Claude Opus 4.7"
date: 2026-04-16T10:00:00
authors:
- sameer
- ishaan-alt
- krrish
description: "Day 0 support for Claude Opus 4.7 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock."
tags: [anthropic, claude, opus 4.7]
hide_table_of_contents: false
---
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
LiteLLM now supports [Claude Opus 4.7](https://www.anthropic.com/news/claude-opus-4-7) on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway.
{/* truncate */}
## Docker Image
```bash
docker pull ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7
```
## Usage - Anthropic
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-7
litellm_params:
model: anthropic/claude-opus-4-7
api_key: os.environ/ANTHROPIC_API_KEY
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Azure
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-7
litellm_params:
model: azure_ai/claude-opus-4-7
api_key: os.environ/AZURE_AI_API_KEY
api_base: os.environ/AZURE_AI_API_BASE # https://<resource>.services.ai.azure.com
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AZURE_AI_API_KEY=$AZURE_AI_API_KEY \
-e AZURE_AI_API_BASE=$AZURE_AI_API_BASE \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Vertex AI
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-7
litellm_params:
model: vertex_ai/claude-opus-4-7
vertex_project: os.environ/VERTEX_PROJECT
vertex_location: us-east5
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e VERTEX_PROJECT=$VERTEX_PROJECT \
-e GOOGLE_APPLICATION_CREDENTIALS=/app/credentials.json \
-v $(pwd)/config.yaml:/app/config.yaml \
-v $(pwd)/credentials.json:/app/credentials.json \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Usage - Bedrock
<Tabs>
<TabItem value="proxy" label="LiteLLM Proxy">
**1. Setup config.yaml**
```yaml
model_list:
- model_name: claude-opus-4-7
litellm_params:
model: bedrock/anthropic.claude-opus-4-7
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: us-east-1
```
**2. Start the proxy**
```bash
docker run -d \
-p 4000:4000 \
-e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \
-e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \
-v $(pwd)/config.yaml:/app/config.yaml \
ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.83.3-stable.opus-4.7 \
--config /app/config.yaml
```
**3. Test it!**
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "what llm are you"
}
]
}'
```
</TabItem>
</Tabs>
## Advanced Features
### Adaptive Thinking
:::note
When using `reasoning_effort` with Claude Opus 4.7, all values (`low`, `medium`, `high`, `xhigh`) are mapped to `thinking: {type: "adaptive"}`. To use explicit thinking budgets with `type: "enabled"`, pass the native `thinking` parameter directly.
:::
<Tabs>
<TabItem value="completions" label="/chat/completions">
LiteLLM supports adaptive thinking through the `reasoning_effort` parameter:
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "Solve this complex problem: What is the optimal strategy for..."
}
],
"reasoning_effort": "high"
}'
```
</TabItem>
<TabItem value="messages" label="/v1/messages">
Use the `thinking` parameter with `type: "adaptive"` to enable adaptive thinking mode:
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-7",
"max_tokens": 16000,
"thinking": {
"type": "adaptive"
},
"messages": [
{
"role": "user",
"content": "Explain why the sum of two even numbers is always even."
}
]
}'
```
</TabItem>
</Tabs>
### Effort Levels
Claude Opus 4.7 supports four effort levels: `low`, `medium`, `high` (default), and `xhigh`. These give you finer-grained control over how much reasoning the model applies to a task. Pass the effort level via the `output_config` parameter.
`xhigh` is a new effort level introduced with Opus 4.7 that sits above `high`. The `max` effort level is Claude Opus 4.6 only and is not available on 4.7.
<Tabs>
<TabItem value="completions" label="/chat/completions">
```bash
curl --location 'http://0.0.0.0:4000/chat/completions' \
--header 'Content-Type: application/json' \
--header 'Authorization: Bearer $LITELLM_KEY' \
--data '{
"model": "claude-opus-4-7",
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "xhigh"
}
}'
```
**Using OpenAI SDK:**
```python
import openai
client = openai.OpenAI(
api_key="your-litellm-key",
base_url="http://0.0.0.0:4000"
)
response = client.chat.completions.create(
model="claude-opus-4-7",
messages=[{"role": "user", "content": "Explain quantum computing"}],
extra_body={"output_config": {"effort": "xhigh"}}
)
```
**Using LiteLLM SDK:**
```python
from litellm import completion
response = completion(
model="anthropic/claude-opus-4-7",
messages=[{"role": "user", "content": "Explain quantum computing"}],
output_config={"effort": "xhigh"},
)
```
You can combine `reasoning_effort` with `output_config` for even more fine-grained control over the model's behavior.
</TabItem>
<TabItem value="messages" label="/v1/messages">
```bash
curl --location 'http://0.0.0.0:4000/v1/messages' \
--header 'x-api-key: sk-12345' \
--header 'content-type: application/json' \
--data '{
"model": "claude-opus-4-7",
"max_tokens": 4096,
"messages": [
{
"role": "user",
"content": "Explain quantum computing"
}
],
"output_config": {
"effort": "xhigh"
}
}'
```
</TabItem>
</Tabs>
**Effort level guide:**
| Effort | When to use |
|--------|-------------|
| `low` | Short, fast responses — simple lookups, formatting, classification |
| `medium` | Balanced tradeoff for everyday Q&A and light reasoning |
| `high` (default) | Complex reasoning, code generation, analysis |
| `xhigh` | Hardest problems — multi-step math, deep research, agentic planning |
docs/my-website/docs/index.md
+4
View File
@@ -9,6 +9,10 @@ import TabItem from '@theme/TabItem';
import NavigationCards from '@site/src/components/NavigationCards';
import Image from '@theme/IdealImage';
:::note Security Update
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
:::
<Image style={{padding: '10px', margin: '0 0 2.5rem'}} img={require('../img/hero.png')} />
**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format.
docs/my-website/docs/providers/github_copilot.md
+7
View File
@@ -192,6 +192,13 @@ export GITHUB_COPILOT_ACCESS_TOKEN_FILE="access-token"
# Optional: Custom API key file name
export GITHUB_COPILOT_API_KEY_FILE="api-key.json"
# Optional: Custom Copilot endpoints for authentication and usage
# (needed when using GitHub Enterprise subscriptions with custom endpoints or self-hosted GitHub servers
export GITHUB_COPILOT_API_BASE="https://copilot-api.my-company.ghe.com"
export GITHUB_COPILOT_DEVICE_CODE_URL="https://my-company.ghe.com/login/device/code"
export GITHUB_COPILOT_ACCESS_TOKEN_URL="https://my-company.ghe.com/login/oauth/access_token"
export GITHUB_COPILOT_API_KEY_URL="https://my-company.ghe.com/api/v3/copilot_internal/v2/token"
```
### Headers
docs/my-website/docs/proxy/config_settings.md
+9
View File
@@ -487,6 +487,7 @@ router_settings:
| AZURE_STORAGE_CLIENT_ID | The Application Client ID to use for Authentication to Azure Blob Storage logging
| AZURE_STORAGE_CLIENT_SECRET | The Application Client Secret to use for Authentication to Azure Blob Storage logging
| AZURE_VECTOR_STORE_COST_PER_GB_PER_DAY | Cost per GB per day for Azure Vector Store service
| BACKGROUND_HEALTH_CHECK_MAX_TOKENS | Optional global default for `max_tokens` on proxy background health checks when a model has no `health_check_max_tokens`. If unset, non-wildcard models default to 1. Applies to wildcard routes when set. Default is unset
| BATCH_STATUS_POLL_INTERVAL_SECONDS | Interval in seconds for polling batch status. Default is 3600 (1 hour)
| BATCH_STATUS_POLL_MAX_ATTEMPTS | Maximum number of attempts for polling batch status. Default is 24 (for 24 hours)
| BEDROCK_MAX_POLICY_SIZE | Maximum size for Bedrock policy. Default is 75
@@ -719,6 +720,11 @@ router_settings:
| GITHUB_COPILOT_TOKEN_DIR | Directory to store GitHub Copilot token for `github_copilot` llm provider
| GITHUB_COPILOT_API_KEY_FILE | File to store GitHub Copilot API key for `github_copilot` llm provider
| GITHUB_COPILOT_ACCESS_TOKEN_FILE | File to store GitHub Copilot access token for `github_copilot` llm provider
| GITHUB_COPILOT_API_BASE | Base URL for GitHub Copilot API. For GitHub Enterprise subscriptions with custom host, it is similar to https://copilot-api.my-company.ghe.com. Default is https://api.githubcopilot.com
| GITHUB_COPILOT_DEVICE_CODE_URL | URL for GitHub Copilot device code authentication. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/login/device/code. Default is https://github.com/login/device/code
| GITHUB_COPILOT_ACCESS_TOKEN_URL | URL for GitHub Copilot access token retrieval. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/login/oauth/access_token. Default is https://github.com/login/oauth/access_token
| GITHUB_COPILOT_API_KEY_URL | URL for GitHub Copilot API key retrieval. For GitHub Enterprise subscriptions with custom host, it is similar to https://my-company.ghe.com/api/v3/copilot_internal/v2/token. Default is https://api.github.com/copilot_internal/v2/token
| GITHUB_COPILOT_CLIENT_ID | Client ID for GitHub Copilot device flow authentication. This is used by the `github_copilot` provider for device code authentication. Default is "Iv1.b507a08c87ecfe98"
| GREENSCALE_API_KEY | API key for Greenscale service
| GREENSCALE_ENDPOINT | Endpoint URL for Greenscale service
| GRAYSWAN_API_BASE | Base URL for GraySwan API. Default is https://api.grayswan.ai
@@ -804,6 +810,8 @@ router_settings:
| LITELLM_ASSETS_PATH | Path to directory for UI assets and logos. Used when running with read-only filesystem (e.g., Kubernetes). Default is `/var/lib/litellm/assets` in Docker.
| LITELLM_BLOG_POSTS_URL | Custom URL for fetching LiteLLM blog posts JSON. Default is the GitHub main branch URL
| LITELLM_CLI_JWT_EXPIRATION_HOURS | Expiration time in hours for CLI-generated JWT tokens. Default is 24 hours
| LITELLM_CORS_ALLOW_CREDENTIALS | Set to `true` to explicitly allow credentials in CORS responses. When not set, credentials are disabled automatically if `LITELLM_CORS_ORIGINS` is `*` (wildcard) to prevent the browser security misconfiguration of reflecting any origin with credentials
| LITELLM_CORS_ORIGINS | Comma-separated list of allowed CORS origins (e.g. `https://app.example.com,https://admin.example.com`). Defaults to `*` (all origins) when not set
| LITELLM_DD_AGENT_HOST | Hostname or IP of DataDog agent for LiteLLM-specific logging. When set, logs are sent to agent instead of direct API
| LITELLM_DEPLOYMENT_ENVIRONMENT | Environment name for the deployment (e.g., "production", "staging"). Used as a fallback when OTEL_ENVIRONMENT_NAME is not set. Sets the `environment` tag in telemetry data
| LITELLM_DETAILED_TIMING | When true, adds detailed per-phase timing headers to responses (`x-litellm-timing-{pre-processing,llm-api,post-processing,message-copy}-ms`). Default is false. See [latency overhead docs](../troubleshoot/latency_overhead.md)
@@ -925,6 +933,7 @@ router_settings:
| OPENAI_CHATGPT_API_BASE | Alternative to CHATGPT_API_BASE. Base URL for ChatGPT API
| OPENAI_FILE_SEARCH_COST_PER_1K_CALLS | Cost per 1000 calls for OpenAI file search. Default is 0.0025
| OPENAI_ORGANIZATION | Organization identifier for OpenAI
| OPENAPI_URL | The path to the OpenAPI JSON endpoint. **By default this is "/openapi.json"**
| OPENID_BASE_URL | Base URL for OpenID Connect services
| OPENID_CLIENT_ID | Client ID for OpenID Connect authentication
| OPENID_CLIENT_SECRET | Client secret for OpenID Connect authentication
docs/my-website/docs/proxy/cost_tracking.md
+4
View File
@@ -14,6 +14,10 @@ Provider-specific cost tracking (e.g., [Vertex AI PayGo / priority pricing](../p
[Sync model pricing data from GitHub](./sync_models_github.md) to ensure accurate cost tracking.
:::
:::info Cost does not match your provider bill?
Use the step-by-step workflow in [Debugging a cost discrepancy](../troubleshoot/cost_discrepancy): align time ranges, compare token categories (including cache), then decide whether the gap is ingestion, formula, or model-map pricing.
:::
### How to Track Spend with LiteLLM
**Step 1**
docs/my-website/docs/proxy/ui_team_soft_budget_alerts.md
+10
View File
@@ -2,6 +2,16 @@ import Image from '@theme/IdealImage';
# Team Soft Budget Alerts
:::info
✨ This is an Enterprise feature. Email budget alerts require an enterprise license.
[Enterprise Pricing](https://www.litellm.ai/#pricing)
[Get free 7-day trial key](https://www.litellm.ai/enterprise#trial)
:::
Set a soft budget on a team and get email alerts when spending crosses the threshold — without blocking any requests.
## Overview
docs/my-website/docs/proxy/users.md
+61
View File
@@ -333,6 +333,67 @@ curl 'http://0.0.0.0:4000/key/generate' \
}'
```
#### **Set multiple budget windows on a key**
Apply multiple concurrent budget limits at different time scales on the same key — for example, cap a key at **$10/day** AND **$100/month**.
**When is this useful?**
A single `budget_duration` window can't prevent a bad day from burning your entire month. Multiple budget windows let you:
- Block a runaway usage spike within the day while still allowing normal monthly spend.
- Give Claude Code rollouts a daily guardrail (`24h`) and a monthly ceiling (`30d`) so a single heavy session doesn't exhaust the whole month.
- Layer fine-grained hourly limits for bursty workloads on top of a weekly cap.
:::info
See [User Budget docs](https://docs.litellm.ai/docs/proxy/users) for more on how budgets work across keys, teams, and users.
:::
**Via API**
Pass `budget_limits` as a list of `{budget_duration, max_budget}` objects:
```bash
curl 'http://0.0.0.0:4000/key/generate' \
--header 'Authorization: Bearer <your-master-key>' \
--header 'Content-Type: application/json' \
--data-raw '{
"budget_limits": [
{"budget_duration": "24h", "max_budget": 10},
{"budget_duration": "30d", "max_budget": 100}
]
}'
```
Each window is tracked independently and resets on its own schedule:
| `budget_duration` | Resets |
|---|---|
| `1h` | Every hour |
| `24h` | Daily at midnight UTC |
| `7d` | Every Sunday at midnight UTC |
| `30d` | 1st of every month at midnight UTC |
**Via Dashboard**
Open **Virtual Keys → Create Key → Optional Settings → Budget Windows**.
![Step 1 - open key settings](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/18930ba5-67c0-4031-afc0-57f37b4e59e4/ascreenshot_ef79d8a000bb41cdacf1bd9827732ee8_text_export.jpeg)
Click **+ Add Budget Window** to add a row, choose the period from the dropdown, and enter the spend cap.
![Step 2 - add a window](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/5ae8c0b3-2d03-41ad-a63c-47b20c350dfe/ascreenshot_1a7dc6c7d65544f38fd8a65604674f22_text_export.jpeg)
Add a second row for a different time period (e.g. monthly $100 on top of a daily $10).
![Step 3 - add second window](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/cbded3a7-1086-4e20-8f0f-de154b76146c/ascreenshot_c51c18752c3b4f8b976d28799b2638b6_text_export.jpeg)
Each window shows the reset schedule below the input so it's always clear when spend resets.
![Step 4 - reset hints](https://colony-recorder.s3.amazonaws.com/files/2026-04-01/8754f121-1640-4892-9dd0-fd4a870418bf/ascreenshot_8079eb0df2194e8f99e5258ba4b3c082_text_export.jpeg)
### ✨ Virtual Key (Model Specific)
docs/my-website/docs/skills_gateway.md
+111
View File
@@ -0,0 +1,111 @@
# Skills Gateway
<iframe width="840" height="500" src="https://www.loom.com/embed/cb74eb79df3e4c2b83a6efae54a589f9" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
LiteLLM acts as a **Skills Registry** — a central place to register, manage, and discover Claude Code skills across your organization. Teams can publish skills once and have agents and developers find them through a single hub.
## How it works
```mermaid
graph TD
Dev["👨‍💻 Developer<br/>registers a skill<br/>(GitHub URL or subdir)"] -->|POST /claude-code/plugins| Proxy["LiteLLM Proxy<br/>(Skills Registry)"]
Admin["🔑 Admin<br/>publishes skill<br/>(marks as public)"] -->|enable via UI or API| Proxy
Proxy -->|GET /public/skill_hub| SkillHub["🗂️ Skill Hub<br/>(AI Hub → Skill Hub tab)"]
Proxy -->|GET /claude-code/marketplace.json| Marketplace["📦 Claude Code<br/>Marketplace endpoint"]
SkillHub --> Human["🧑 Human<br/>browses & discovers skills<br/>in AI Hub UI"]
Marketplace --> Agent["🤖 Agent / Claude Code<br/>installs skill with<br/>/plugin marketplace add &lt;name&gt;"]
style Proxy fill:#1a73e8,color:#fff
style SkillHub fill:#e8f0fe,color:#1a73e8
style Marketplace fill:#e8f0fe,color:#1a73e8
```
## Quick start
### 1. Register a skill
Paste any GitHub URL into the Skills UI — LiteLLM auto-detects the source type and skill name.
```bash
curl -X POST https://your-proxy/claude-code/plugins \
-H "Authorization: Bearer $LITELLM_KEY" \
-H "Content-Type: application/json" \
-d '{
"name": "grill-me",
"source": {
"source": "git-subdir",
"url": "https://github.com/mattpocock/skills",
"path": "grill-me"
},
"description": "Interview skill for relentless questioning",
"domain": "Productivity",
"namespace": "interviews"
}'
```
Skills nested in subdirectories (e.g. `github.com/org/repo/tree/main/skill-name`) are supported — LiteLLM parses the URL automatically in the UI.
### 2. Publish to hub
In the Admin UI: **AI Hub → Skill Hub → Select Skills to Make Public**.
Or via API:
```bash
curl -X POST https://your-proxy/claude-code/plugins/grill-me/enable \
-H "Authorization: Bearer $LITELLM_KEY"
```
### 3. Browse the hub
Public skills appear at:
- **Admin UI**: AI Hub → Skill Hub tab
- **Public page**: `/ui/model_hub` → Skill Hub tab (no login required)
- **API**: `GET /public/skill_hub`
### 4. Install in Claude Code
Point Claude Code at your proxy marketplace once:
```json title="~/.claude/settings.json"
{
"extraKnownMarketplaces": {
"my-org": {
"source": "url",
"url": "https://your-proxy/claude-code/marketplace.json"
}
}
}
```
Then install any skill:
```
/plugin marketplace add grill-me
```
## Skill fields
| Field | Description |
|-------|-------------|
| `name` | Unique skill identifier (used in `/plugin marketplace add`) |
| `source` | Git source — `github`, `url`, or `git-subdir` |
| `description` | Short description shown in the hub |
| `domain` | Category for grouping (e.g. `Engineering`, `Productivity`) |
| `namespace` | Subcategory within a domain (e.g. `quality`, `meetings`) |
| `keywords` | Tags for search and filtering |
| `version` | Semver string |
## API reference
| Endpoint | Auth | Description |
|----------|------|-------------|
| `POST /claude-code/plugins` | Required | Register a skill |
| `GET /claude-code/plugins` | Required | List all skills (admin) |
| `POST /claude-code/plugins/{name}/enable` | Required | Publish a skill |
| `POST /claude-code/plugins/{name}/disable` | Required | Unpublish a skill |
| `GET /public/skill_hub` | None | List public skills |
| `GET /claude-code/marketplace.json` | None | Claude Code marketplace manifest |
docs/my-website/docs/troubleshoot/cost_discrepancy.md
+205
View File
@@ -0,0 +1,205 @@
# Debugging a cost discrepancy
Cost discrepancies between LiteLLM and your provider bill usually come from one of three areas: token ingestion, the cost formula LiteLLM applies, or stale or incorrect pricing in the model map. This page walks through how to tell which case you are in.
## Step 1: Pick a time range
Lock down a specific window where the discrepancy is visible.
- Use at least 7 days of data when you can.
- Prefer a window with stable usage so one-off spikes do not dominate the comparison.
- Set the **same start and end time** on both your provider dashboard and the LiteLLM UI.
![LiteLLM dashboard date range picker](/img/cost-discrepancy-debug/date-range-picker.png)
## Step 2: Confirm traffic only goes through LiteLLM
If any requests hit the provider directly (bypassing LiteLLM), the provider will show higher usage. That is expected, not a LiteLLM bug.
Before continuing, confirm:
- All clients use your LiteLLM proxy base URL.
- No SDK or script uses provider API keys against the provider directly for the models you are comparing.
- During the selected period, the models in question are only called via LiteLLM.
If you are unsure, filter the provider dashboard by the API key or IAM principal LiteLLM uses, rather than comparing to your whole account.
## Step 3: Compare token categories
In the LiteLLM UI, open **Model activity** (under Usage analytics) so you can inspect spend and tokens per model.
![Navigate to Model activity in the LiteLLM UI](/img/cost-discrepancy-debug/go-to-model-activity.png)
Scroll the **Model** list and select the model you are reconciling with your provider bill.
![Scroll to your model in the Model activity table](/img/cost-discrepancy-debug/scroll-to-model.png)
With the same time range on both sides, fill in:
| Category | LiteLLM | Provider | Delta |
| --- | --- | --- | --- |
| Total requests | — | — | — |
| Input tokens | — | — | — |
| Output tokens | — | — | — |
| Cache read tokens | — | — | — |
| Cache write tokens | — | — | — |
LiteLLM surfaces per-category token usage for the selected model—for example prompt, completion, and cache-related tokens.
![LiteLLM usage breakdown by token category](/img/cost-discrepancy-debug/token-categories.png)
Compare these figures with your provider’s usage view (for example AWS billing tools, Azure Monitor, or the OpenAI usage dashboard) for the same period.
### Cache token reporting
- **OpenAI:** Cache read tokens are typically included inside the reported input token count.
- **Anthropic:** Cache read tokens are often reported separately from non-cached input tokens.
Compare the correct columns on each side so you are not treating “input” differently between dashboards.
### Why use a 10% threshold?
Provider dashboards and LiteLLM do not bucket requests on identical timestamps. A call at 11:59 PM can land in different daily totals on each side. Token counts can also differ slightly due to rounding across SDKs and APIs. A delta **under ~10%** is often explained by boundary effects and rounding. A delta **over ~10%** usually means something is miscounted, dropped, or categorized differently.
## Step 4: Follow the right path
<svg width="100%" viewBox="0 0 680 482" role="img" xmlns="http://www.w3.org/2000/svg" style={{ maxWidth: '100%', fontFamily: 'system-ui, sans-serif' }} aria-labelledby="cost-disc-flow-title">
<title id="cost-disc-flow-title">Cost discrepancy debugging flowchart</title>
<desc>Flowchart branching into Path A (token ingestion) or Path B which splits further into B1 (formula issue) and B2 (model map issue).</desc>
<defs>
<marker id="cd-arrow" viewBox="0 0 10 10" refX="8" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
<path d="M2 1L8 5L2 9" fill="none" stroke="#888780" strokeWidth="1.5" strokeLinecap="round" strokeLinejoin="round" />
</marker>
</defs>
<rect x="215" y="24" width="250" height="44" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
<text x="340" y="47" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Compare provider vs LiteLLM</text>
<line x1="340" y1="68" x2="340" y2="104" stroke="#888780" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<rect x="175" y="104" width="330" height="56" rx="8" fill="#F1EFE8" stroke="#5F5E5A" strokeWidth="0.5" />
<text x="340" y="126" textAnchor="middle" dominantBaseline="central" fill="#444441" fontSize="14" fontWeight="500">Any category off by &gt; 10%?</text>
<text x="340" y="148" textAnchor="middle" dominantBaseline="central" fill="#5F5E5A" fontSize="12">requests, input, output, cache tokens</text>
<path d="M220 132 L100 132 L100 250" fill="none" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<text x="157" y="122" textAnchor="middle" fill="#0F6E56" fontSize="12">YES</text>
<path d="M505 132 L580 132 L580 250" fill="none" stroke="#993C1D" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<text x="543" y="122" textAnchor="middle" fill="#993C1D" fontSize="12">NO</text>
<rect x="40" y="250" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
<text x="150" y="271" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Path A</text>
<text x="150" y="291" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">Token ingestion issue</text>
<rect x="420" y="250" width="220" height="56" rx="8" fill="#FAECE7" stroke="#993C1D" strokeWidth="0.5" />
<text x="530" y="271" textAnchor="middle" dominantBaseline="central" fill="#712B13" fontSize="14" fontWeight="500">Path B</text>
<text x="530" y="291" textAnchor="middle" dominantBaseline="central" fill="#993C1D" fontSize="12">Quantities match, cost differs</text>
<line x1="150" y1="306" x2="150" y2="370" stroke="#0F6E56" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<line x1="530" y1="306" x2="530" y2="318" stroke="#854F0B" strokeWidth="1.5" />
<line x1="435" y1="318" x2="575" y2="318" stroke="#854F0B" strokeWidth="1.5" />
<line x1="435" y1="318" x2="435" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<line x1="575" y1="318" x2="575" y2="370" stroke="#854F0B" strokeWidth="1.5" markerEnd="url(#cd-arrow)" />
<text x="448" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B1</text>
<text x="562" y="312" textAnchor="middle" fill="#854F0B" fontSize="11">B2</text>
<rect x="40" y="370" width="220" height="56" rx="8" fill="#E1F5EE" stroke="#0F6E56" strokeWidth="0.5" />
<text x="150" y="391" textAnchor="middle" dominantBaseline="central" fill="#085041" fontSize="14" fontWeight="500">Report to LiteLLM team</text>
<text x="150" y="411" textAnchor="middle" dominantBaseline="central" fill="#0F6E56" fontSize="12">endpoints + model + screenshots</text>
<rect x="380" y="370" width="110" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
<text x="435" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B1</text>
<text x="435" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix formula</text>
<rect x="510" y="370" width="130" height="56" rx="8" fill="#FAEEDA" stroke="#854F0B" strokeWidth="0.5" />
<text x="575" y="391" textAnchor="middle" dominantBaseline="central" fill="#633806" fontSize="14" fontWeight="500">B2</text>
<text x="575" y="411" textAnchor="middle" dominantBaseline="central" fill="#854F0B" fontSize="12">Fix model map</text>
<path d="M150 426 L150 442 L340 442" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
<path d="M340 442 L435 442 L435 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
<path d="M340 442 L575 442 L575 428" fill="none" stroke="#888780" strokeWidth="0.5" strokeDasharray="4 3" />
<text x="340" y="454" textAnchor="middle" fill="#5F5E5A" fontSize="11">if neither path resolves it,</text>
<text x="340" y="470" textAnchor="middle" fill="#5F5E5A" fontSize="11">Open a github issue backing up with all your data</text>
</svg>
## Path A: Token quantity mismatch
If any category is off by more than about 10%, LiteLLM may not be ingesting that category correctly (or the provider dashboard is categorizing tokens differently—recheck Step 3 first).
**What to send the LiteLLM team:**
1. Screenshots of both dashboards with the date range visible.
2. Which category is off (input, output, cache reads, cache writes, or request count).
3. Endpoints used (for example `/chat/completions`, `/responses`, `/embeddings`).
4. Model names as sent in the request (for example `anthropic.claude-opus-4-5`, `gpt-4o`).
### For maintainers debugging ingestion
1. Start the proxy with verbose logging, for example:
```bash
litellm --config config.yaml --detailed_debug
```
2. Reproduce a single request with the reported endpoint and model.
3. Inspect the raw `usage` object in each streamed chunk (if streaming) or in the final response body.
4. Compare that to the standard logging object (or the UI request log for that call).
5. Any gap between raw provider usage and what LiteLLM logs or aggregates is where ingestion may be wrong.
## Path B: Quantities match but cost is wrong
If token and request counts agree within ~10% but dollar amounts differ, focus on how cost is computed.
### B1: Formula issue
Manually compute expected cost using the provider’s token breakdown and published rates (per million tokens or per token).
Add other billed dimensions your provider applies (for example cache creation, audio, or tier surcharges). If your hand calculation matches the provider bill but not LiteLLM, the implementation in LiteLLM for that provider or modality may be wrong.
### B2: Model map issue
If the formula structure matches how the provider bills, the values in LiteLLM’s model map may be stale or incorrect. Cross-check:
- [`model_prices_and_context_window.json`](https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)
- The provider’s current public pricing
Inspect `input_cost_per_token`, `output_cost_per_token`, and any cache-related pricing fields for your exact model id (including provider prefix).
### For maintainers
1. Take authoritative token quantities from the user’s provider report.
2. Derive the formula that reproduces the provider’s line item.
3. Diff that against LiteLLM’s cost path for the same provider and response shape.
4. If the formula matches but numbers differ, update pricing in `model_prices_and_context_window.json` (and follow the project’s sync / backup rules for that file).
5. If the formula in code is wrong, fix the calculation and add a regression test using the user’s token breakdown.
## Still stuck?
1. Open a GitHub issue on [BerriAI/litellm](https://github.com/BerriAI/litellm) with your Step 3 comparison table, endpoints, and model names.
On the issue, it helps to clarify:
- Reproducible on demand or intermittent?
- Single model or many?
- Steady over time, or starting from a specific release date or config change?
### For LiteLLM maintainers
If Path A and Path B do not close the case after triage, **you** should reach out and **schedule a call with the customer** (support or engineering), with the Step 3 table and screenshots—before treating the issue.
## Checklist
```
□ Same time range on both dashboards
□ Confirmed no direct-to-provider traffic for those models
□ Compared: requests, input tokens, output tokens, cache tokens
□ Noted cache reporting differences (OpenAI vs Anthropic, and so on)
□ If > ~10% delta on quantities → Path A: report with screenshots, endpoints, model names
□ If quantities match → Path B: verify formula (B1) and model map pricing (B2)
□ If neither path fits → open a GitHub issue.
```
## See also
- [Spend tracking](../proxy/cost_tracking)
- [Sync model pricing from GitHub](../proxy/sync_models_github)
docs/my-website/docusaurus.config.js
+27 -1
View File
@@ -187,6 +187,32 @@ const config = {
},
],
[
'@signalwire/docusaurus-plugin-llms-txt',
{
markdown: {
enableFiles: true,
includeDocs: true,
},
llmsTxt: {
enableLlmsFullTxt: true,
includeDocs: true,
},
ui: {
copyPageContent: {
buttonLabel: 'Copy Page',
actions: {
viewMarkdown: true,
ai: {
chatGPT: true,
claude: true,
},
},
},
},
},
],
() => ({
name: 'cripchat',
injectHtmlTags() {
@@ -239,7 +265,7 @@ const config = {
],
],
themes: ['@docusaurus/theme-mermaid'],
themes: ['@docusaurus/theme-mermaid', '@signalwire/docusaurus-theme-llms-txt'],
markdown: {
mermaid: true,
},
docs/my-website/package-lock.json
Generated
+396
View File
@@ -15,6 +15,8 @@
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7",
"@signalwire/docusaurus-theme-llms-txt": "1.0.0-alpha.9",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
@@ -7140,6 +7142,72 @@
"integrity": "sha512-RNiOoTPkptFtSVzQevY/yWtZwf/RxyVnPy/OcA9HBM3MlGDnBEYL5B41H0MTn0Uec8Hi+2qUtTfG2WWZBmMejQ==",
"license": "BSD-3-Clause"
},
"node_modules/@signalwire/docusaurus-plugin-llms-txt": {
"version": "2.0.0-alpha.7",
"resolved": "https://registry.npmjs.org/@signalwire/docusaurus-plugin-llms-txt/-/docusaurus-plugin-llms-txt-2.0.0-alpha.7.tgz",
"integrity": "sha512-v9EcYXVNvMydIWVIzI1H2iC4/BNdystE0jJAQIFu68SHy1a13dESz9hn5YJE9Izx18QPny1jhXym/3wEP9+8LA==",
"license": "MIT",
"dependencies": {
"fs-extra": "^11.0.0",
"hast-util-select": "^6.0.4",
"hast-util-to-html": "^9.0.5",
"hast-util-to-string": "^3.0.1",
"p-map": "^7.0.2",
"rehype-parse": "^9",
"rehype-remark": "^10",
"remark-gfm": "^4",
"remark-stringify": "^11",
"string-width": "^5.0.0",
"unified": "^11",
"unist-util-visit": "^5"
},
"engines": {
"node": ">=18.0.0"
},
"peerDependencies": {
"@docusaurus/core": "^3.0.0"
}
},
"node_modules/@signalwire/docusaurus-plugin-llms-txt/node_modules/p-map": {
"version": "7.0.4",
"resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.4.tgz",
"integrity": "sha512-tkAQEw8ysMzmkhgw8k+1U/iPhWNhykKnSk4Rd5zLoPJCuJaGRPo6YposrZgaxHKzDHdDWWZvE/Sk7hsL2X/CpQ==",
"license": "MIT",
"engines": {
"node": ">=18"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/@signalwire/docusaurus-theme-llms-txt": {
"version": "1.0.0-alpha.9",
"resolved": "https://registry.npmjs.org/@signalwire/docusaurus-theme-llms-txt/-/docusaurus-theme-llms-txt-1.0.0-alpha.9.tgz",
"integrity": "sha512-ULCKEKkAUZVnLr8+ocR4tl7ogiiW13Hqtoo8SfNbgOyX1l4LN3a6j3/vxgc6qRYbgWlnqY3EPC7S3tenRsDjgQ==",
"license": "MIT",
"dependencies": {
"@docusaurus/core": "^3.0.0",
"@docusaurus/theme-common": "^3.0.0",
"clsx": "^2.0.0",
"react-icons": "^5.5.0"
},
"engines": {
"node": ">=18.0.0"
},
"peerDependencies": {
"react": "^18.0.0",
"react-dom": "^18.0.0"
}
},
"node_modules/@signalwire/docusaurus-theme-llms-txt/node_modules/clsx": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/clsx/-/clsx-2.1.1.tgz",
"integrity": "sha512-eYm0QWBtUrBWZWG0d386OGAw16Z995PiOVo2B7bjWSbHedGl5e0ZWaq65kOGgUSNesEIDkB9ISbTg/JK9dhCZA==",
"license": "MIT",
"engines": {
"node": ">=6"
}
},
"node_modules/@sinclair/typebox": {
"version": "0.27.10",
"resolved": "https://registry.npmjs.org/@sinclair/typebox/-/typebox-0.27.10.tgz",
@@ -8972,6 +9040,16 @@
"integrity": "sha512-x+VAiMRL6UPkx+kudNvxTl6hB2XNNCG2r+7wixVfIYwu/2HKRXimwQyaumLjMveWvT2Hkd/cAJw+QBMfJ/EKVw==",
"license": "MIT"
},
"node_modules/bcp-47-match": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/bcp-47-match/-/bcp-47-match-2.0.3.tgz",
"integrity": "sha512-JtTezzbAibu8G0R9op9zb3vcWZd9JF6M0xOYGPn0fNCd7wOpRB1mU2mH9T8gaBGbAAyIIVgB2G7xG0GP98zMAQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/big.js": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/big.js/-/big.js-5.2.2.tgz",
@@ -10330,6 +10408,22 @@
"url": "https://github.com/sponsors/fb55"
}
},
"node_modules/css-selector-parser": {
"version": "3.3.0",
"resolved": "https://registry.npmjs.org/css-selector-parser/-/css-selector-parser-3.3.0.tgz",
"integrity": "sha512-Y2asgMGFqJKF4fq4xHDSlFYIkeVfRsm69lQC1q9kbEsH5XtnINTMrweLkjYMeaUgiXBy/uvKeO/a1JHTNnmB2g==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/mdevils"
},
{
"type": "patreon",
"url": "https://patreon.com/mdevils"
}
],
"license": "MIT"
},
"node_modules/css-tree": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/css-tree/-/css-tree-3.2.1.tgz",
@@ -11291,6 +11385,19 @@
"node": ">=8"
}
},
"node_modules/direction": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/direction/-/direction-2.0.1.tgz",
"integrity": "sha512-9S6m9Sukh1cZNknO1CWAr2QAWsbKLafQiyM5gZ7VgXHeuaoUwffKN4q6NC4A/Mf9iiPlOXQEKW/Mv/mh9/3YFA==",
"license": "MIT",
"bin": {
"direction": "cli.js"
},
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/dns-packet": {
"version": "5.6.1",
"resolved": "https://registry.npmjs.org/dns-packet/-/dns-packet-5.6.1.tgz",
@@ -12812,6 +12919,38 @@
"node": ">= 0.4"
}
},
"node_modules/hast-util-embedded": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-embedded/-/hast-util-embedded-3.0.0.tgz",
"integrity": "sha512-naH8sld4Pe2ep03qqULEtvYr7EjrLK2QHY8KJR6RJkTUjPGObe1vnx585uzem2hGra+s1q08DZZpfgDVYRbaXA==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-is-element": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-from-html": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/hast-util-from-html/-/hast-util-from-html-2.0.3.tgz",
"integrity": "sha512-CUSRHXyKjzHov8yKsQjGOElXy/3EKpyX56ELnkHH34vDVw1N1XSQ1ZcAvTyAPtGqLTuKP/uxM+aLkSPqF/EtMw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"devlop": "^1.1.0",
"hast-util-from-parse5": "^8.0.0",
"parse5": "^7.0.0",
"vfile": "^6.0.0",
"vfile-message": "^4.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-from-parse5": {
"version": "8.0.3",
"resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz",
@@ -12832,6 +12971,62 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-has-property": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-has-property/-/hast-util-has-property-3.0.0.tgz",
"integrity": "sha512-MNilsvEKLFpV604hwfhVStK0usFY/QmM5zX16bo7EjnAEGofr5YyI37kzopBlZJkHD4t887i+q/C8/tr5Q94cA==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-is-body-ok-link": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-is-body-ok-link/-/hast-util-is-body-ok-link-3.0.1.tgz",
"integrity": "sha512-0qpnzOBLztXHbHQenVB8uNuxTnm/QBFUOmdOSsEn7GnBtyY07+ENTWVFBAnXd/zEgd9/SUG3lRY7hSIBWRgGpQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-is-element": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-is-element/-/hast-util-is-element-3.0.0.tgz",
"integrity": "sha512-Val9mnv2IWpLbNPqc/pUem+a7Ipj2aHacCwgNfTiK0vJKl0LF+4Ba4+v1oPHFpf3bLYmreq0/l3Gud9S5OH42g==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-minify-whitespace": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/hast-util-minify-whitespace/-/hast-util-minify-whitespace-1.0.1.tgz",
"integrity": "sha512-L96fPOVpnclQE0xzdWb/D12VT5FabA7SnZOUMtL1DbXmYiHJMXZvFkIZfiMmTCNJHUeO2K9UYNXoVyfz+QHuOw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-embedded": "^3.0.0",
"hast-util-is-element": "^3.0.0",
"hast-util-whitespace": "^3.0.0",
"unist-util-is": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-parse-selector": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz",
@@ -12845,6 +13040,23 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-phrasing": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-phrasing/-/hast-util-phrasing-3.0.1.tgz",
"integrity": "sha512-6h60VfI3uBQUxHqTyMymMZnEbNl1XmEGtOxxKYL7stY2o601COo62AWAYBQR9lZbYXYSBoxag8UpPRXK+9fqSQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-embedded": "^3.0.0",
"hast-util-has-property": "^3.0.0",
"hast-util-is-body-ok-link": "^3.0.0",
"hast-util-is-element": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-raw": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz",
@@ -12870,6 +13082,33 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-select": {
"version": "6.0.4",
"resolved": "https://registry.npmjs.org/hast-util-select/-/hast-util-select-6.0.4.tgz",
"integrity": "sha512-RqGS1ZgI0MwxLaKLDxjprynNzINEkRHY2i8ln4DDjgv9ZhcYVIHN9rlpiYsqtFwrgpYU361SyWDQcGNIBVu3lw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"bcp-47-match": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"css-selector-parser": "^3.0.0",
"devlop": "^1.0.0",
"direction": "^2.0.0",
"hast-util-has-property": "^3.0.0",
"hast-util-to-string": "^3.0.0",
"hast-util-whitespace": "^3.0.0",
"nth-check": "^2.0.0",
"property-information": "^7.0.0",
"space-separated-tokens": "^2.0.0",
"unist-util-visit": "^5.0.0",
"zwitch": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-estree": {
"version": "3.1.3",
"resolved": "https://registry.npmjs.org/hast-util-to-estree/-/hast-util-to-estree-3.1.3.tgz",
@@ -12898,6 +13137,29 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-html": {
"version": "9.0.5",
"resolved": "https://registry.npmjs.org/hast-util-to-html/-/hast-util-to-html-9.0.5.tgz",
"integrity": "sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"ccount": "^2.0.0",
"comma-separated-tokens": "^2.0.0",
"hast-util-whitespace": "^3.0.0",
"html-void-elements": "^3.0.0",
"mdast-util-to-hast": "^13.0.0",
"property-information": "^7.0.0",
"space-separated-tokens": "^2.0.0",
"stringify-entities": "^4.0.0",
"zwitch": "^2.0.4"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-jsx-runtime": {
"version": "2.3.6",
"resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz",
@@ -12925,6 +13187,32 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-mdast": {
"version": "10.1.2",
"resolved": "https://registry.npmjs.org/hast-util-to-mdast/-/hast-util-to-mdast-10.1.2.tgz",
"integrity": "sha512-FiCRI7NmOvM4y+f5w32jPRzcxDIz+PUqDwEqn1A+1q2cdp3B8Gx7aVrXORdOKjMNDQsD1ogOr896+0jJHW1EFQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/mdast": "^4.0.0",
"@ungap/structured-clone": "^1.0.0",
"hast-util-phrasing": "^3.0.0",
"hast-util-to-html": "^9.0.0",
"hast-util-to-text": "^4.0.0",
"hast-util-whitespace": "^3.0.0",
"mdast-util-phrasing": "^4.0.0",
"mdast-util-to-hast": "^13.0.0",
"mdast-util-to-string": "^4.0.0",
"rehype-minify-whitespace": "^6.0.0",
"trim-trailing-lines": "^2.0.0",
"unist-util-position": "^5.0.0",
"unist-util-visit": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-parse5": {
"version": "8.0.0",
"resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.0.tgz",
@@ -12954,6 +13242,35 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/hast-util-to-string": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/hast-util-to-string/-/hast-util-to-string-3.0.1.tgz",
"integrity": "sha512-XelQVTDWvqcl3axRfI0xSeoVKzyIFPwsAGSLIsKdJKQMXDYJS4WYrBNF/8J7RdhIcFI2BOHgAifggsvsxp/3+A==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-text": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz",
"integrity": "sha512-KK6y/BN8lbaq654j7JgBydev7wuNMcID54lkRav1P0CaE1e47P72AWWPiGKXTJU271ooYzcvTAn/Zt0REnvc7A==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"hast-util-is-element": "^3.0.0",
"unist-util-find-after": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-whitespace": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz",
@@ -19478,6 +19795,15 @@
"react": "^16.8.0 || ^17 || ^18 || ^19"
}
},
"node_modules/react-icons": {
"version": "5.5.0",
"resolved": "https://registry.npmjs.org/react-icons/-/react-icons-5.5.0.tgz",
"integrity": "sha512-MEFcXdkP3dLo8uumGI5xN3lDFNsRtrjbOEKDLD7yv76v4wpnEq2Lt2qeHaQOr34I/wPN3s3+N08WkQ+CW37Xiw==",
"license": "MIT",
"peerDependencies": {
"react": "*"
}
},
"node_modules/react-is": {
"version": "16.13.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-16.13.1.tgz",
@@ -19883,6 +20209,35 @@
"regjsparser": "bin/parser"
}
},
"node_modules/rehype-minify-whitespace": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/rehype-minify-whitespace/-/rehype-minify-whitespace-6.0.2.tgz",
"integrity": "sha512-Zk0pyQ06A3Lyxhe9vGtOtzz3Z0+qZ5+7icZ/PL/2x1SHPbKao5oB/g/rlc6BCTajqBb33JcOe71Ye1oFsuYbnw==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-minify-whitespace": "^1.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-parse": {
"version": "9.0.1",
"resolved": "https://registry.npmjs.org/rehype-parse/-/rehype-parse-9.0.1.tgz",
"integrity": "sha512-ksCzCD0Fgfh7trPDxr2rSylbwq9iYDkSn8TCDmEJ49ljEUBxDVCzCHv7QNzZOfODanX4+bWQ4WZqLCRWYLfhag==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-from-html": "^2.0.0",
"unified": "^11.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-raw": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz",
@@ -19913,6 +20268,23 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-remark": {
"version": "10.0.1",
"resolved": "https://registry.npmjs.org/rehype-remark/-/rehype-remark-10.0.1.tgz",
"integrity": "sha512-EmDndlb5NVwXGfUa4c9GPK+lXeItTilLhE6ADSaQuHr4JUlKw9MidzGzx4HpqZrNCt6vnHmEifXQiiA+CEnjYQ==",
"license": "MIT",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/mdast": "^4.0.0",
"hast-util-to-mdast": "^10.0.0",
"unified": "^11.0.0",
"vfile": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/relateurl": {
"version": "0.2.7",
"resolved": "https://registry.npmjs.org/relateurl/-/relateurl-0.2.7.tgz",
@@ -21641,6 +22013,16 @@
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/trim-trailing-lines": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/trim-trailing-lines/-/trim-trailing-lines-2.1.0.tgz",
"integrity": "sha512-5UR5Biq4VlVOtzqkm2AZlgvSlDJtME46uV0br0gENbwN4l5+mMKT4b9gJKqWtuL2zAIqajGJGuvbCbcAJUZqBg==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/trough": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/trough/-/trough-2.2.0.tgz",
@@ -21825,6 +22207,20 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/unist-util-find-after": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/unist-util-find-after/-/unist-util-find-after-5.0.0.tgz",
"integrity": "sha512-amQa0Ep2m6hE2g72AugUItjbuM8X8cGQnFoHk0pGfrFeT9GZhzN5SW8nRsiGKK7Aif4CrACPENkA6P/Lw6fHGQ==",
"license": "MIT",
"dependencies": {
"@types/unist": "^3.0.0",
"unist-util-is": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/unist-util-is": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/unist-util-is/-/unist-util-is-6.0.1.tgz",
docs/my-website/package.json
+2
View File
@@ -21,6 +21,8 @@
"@docusaurus/theme-mermaid": "3.8.1",
"@inkeep/cxkit-docusaurus": "0.5.107",
"@mdx-js/react": "3.1.1",
"@signalwire/docusaurus-plugin-llms-txt": "2.0.0-alpha.7",
"@signalwire/docusaurus-theme-llms-txt": "1.0.0-alpha.9",
"clsx": "1.2.1",
"prism-react-renderer": "1.3.5",
"react": "18.3.1",
docs/my-website/sidebars.js
+8
View File
@@ -339,6 +339,13 @@ const sidebars = {
},
],
},
{
type: "category",
label: "Skills Gateway",
items: [
"skills_gateway",
],
},
],
},
{
@@ -1149,6 +1156,7 @@ const sidebars = {
label: "Troubleshooting",
items: [
"troubleshoot/ui_issues",
"troubleshoot/cost_discrepancy",
"mcp_troubleshoot",
{
type: "category",
docs/my-website/src/pages/index.md
+4
View File
@@ -1,6 +1,10 @@
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
:::note Security Update
The Trivy supply-chain compromise has been contained :tada: . All affected packages have been deleted and current releases are free of the compromised code/component. Please refer to our [Security Townhall](/blog/security-townhall-updates) for a deeper understanding of the problem, and [CI/CD v2](/blog/ci-cd-v2-improvements) for how we're improving moving forward.
:::
# LiteLLM - Getting Started
https://github.com/BerriAI/litellm
docs/my-website/static/img/cost-discrepancy-debug/date-range-picker.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 509 KiB

docs/my-website/static/img/cost-discrepancy-debug/go-to-model-activity.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 445 KiB

docs/my-website/static/img/cost-discrepancy-debug/scroll-to-model.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 296 KiB

docs/my-website/static/img/cost-discrepancy-debug/token-categories.png
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 281 KiB

litellm-proxy-extras/litellm_proxy_extras/_logging.py
+5 -2
View File
@@ -23,7 +23,8 @@ class JsonFormatter(logging.Formatter):
def _is_json_enabled():
try:
import litellm
return getattr(litellm, 'json_logs', False)
return getattr(litellm, "json_logs", False)
except (ImportError, AttributeError):
return os.getenv("JSON_LOGS", "false").lower() == "true"
@@ -35,6 +36,8 @@ if not logger.handlers:
if _is_json_enabled():
handler.setFormatter(JsonFormatter())
else:
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
handler.setFormatter(
logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
)
logger.addHandler(handler)
logger.setLevel(logging.INFO)
litellm-proxy-extras/litellm_proxy_extras/migrations/20260401000000_add_budget_limits/migration.sql
+5
View File
@@ -0,0 +1,5 @@
-- AlterTable: add budget_limits column to LiteLLM_VerificationToken
ALTER TABLE "LiteLLM_VerificationToken" ADD COLUMN IF NOT EXISTS "budget_limits" JSONB;
-- AlterTable: add budget_limits column to LiteLLM_TeamTable
ALTER TABLE "LiteLLM_TeamTable" ADD COLUMN IF NOT EXISTS "budget_limits" JSONB;
litellm-proxy-extras/litellm_proxy_extras/migrations/20260401000000_add_team_member_model_scope/migration.sql
+9
View File
@@ -0,0 +1,9 @@
-- Add per-member model scope to LiteLLM_BudgetTable
-- allowed_models: empty array = inherit team models; non-empty = enforce member-level restriction
ALTER TABLE "LiteLLM_BudgetTable"
ADD COLUMN IF NOT EXISTS "allowed_models" TEXT[] DEFAULT ARRAY[]::TEXT[];
-- Add default_team_member_models to LiteLLM_TeamTable
-- Seeds allowed_models for newly added team members; empty = no per-member restriction
ALTER TABLE "LiteLLM_TeamTable"
ADD COLUMN IF NOT EXISTS "default_team_member_models" TEXT[] DEFAULT ARRAY[]::TEXT[];
litellm-proxy-extras/litellm_proxy_extras/migrations/20260414140000_add_mcp_server_instructions/migration.sql
+2
View File
@@ -0,0 +1,2 @@
-- AlterTable
ALTER TABLE "LiteLLM_MCPServerTable" ADD COLUMN IF NOT EXISTS "instructions" TEXT;
litellm-proxy-extras/litellm_proxy_extras/migrations/20260415120000_health_check_latest_per_model_index/migration.sql
+12
View File
@@ -0,0 +1,12 @@
-- CreateIndex (CONCURRENTLY)
--
-- Disclaimer:
-- - CREATE INDEX CONCURRENTLY cannot run inside a transaction. This migration must stay a
-- single statement so Prisma Migrate on PostgreSQL can apply it outside a transaction.
-- - Builds are slower and use more I/O than a blocking CREATE INDEX; if the build is
-- interrupted, Postgres may leave an INVALID index that must be dropped and recreated.
-- - Do not edit this file after it has been applied to any database: Prisma checksums
-- migrations; add a new migration instead.
-- - Requires PostgreSQL that supports CONCURRENTLY with IF NOT EXISTS (use a new migration
-- without IF NOT EXISTS if you must support older versions).
CREATE INDEX CONCURRENTLY IF NOT EXISTS "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx" ON "LiteLLM_HealthCheckTable"("model_id", "model_name", "checked_at" DESC);
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+7 -1
View File
@@ -17,8 +17,9 @@ model LiteLLM_BudgetTable {
tpm_limit BigInt?
rpm_limit BigInt?
model_max_budget Json?
budget_duration String?
budget_duration String?
budget_reset_at DateTime?
allowed_models String[] @default([]) // per-member model scope; empty = inherit team models
created_at DateTime @default(now()) @map("created_at")
created_by String
updated_at DateTime @default(now()) @updatedAt @map("updated_at")
@@ -140,6 +141,8 @@ model LiteLLM_TeamTable {
team_member_permissions String[] @default([])
access_group_ids String[] @default([])
policies String[] @default([])
default_team_member_models String[] @default([]) // default allowed_models for newly added team members; empty = no per-member restriction
budget_limits Json? // per-model budget limits for the team
model_id Int? @unique // id for LiteLLM_ModelTable -> stores team-level model aliases
allow_team_guardrail_config Boolean @default(false) // if true, team admin can configure guardrails for this team
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
@@ -289,6 +292,7 @@ model LiteLLM_MCPServerTable {
server_name String?
alias String?
description String?
instructions String?
url String?
spec_path String?
transport String @default("sse")
@@ -400,6 +404,7 @@ model LiteLLM_VerificationToken {
rotation_interval String? // How often to rotate (e.g., "30d", "90d")
last_rotation_at DateTime? // When this key was last rotated
key_rotation_at DateTime? // When this key should next be rotated
budget_limits Json? // per-model budget limits for the key
litellm_budget_table LiteLLM_BudgetTable? @relation(fields: [budget_id], references: [budget_id])
litellm_organization_table LiteLLM_OrganizationTable? @relation(fields: [organization_id], references: [organization_id])
litellm_project_table LiteLLM_ProjectTable? @relation(fields: [project_id], references: [project_id])
@@ -1045,6 +1050,7 @@ model LiteLLM_HealthCheckTable {
@@index([model_name])
@@index([checked_at])
@@index([status])
@@index([model_id, model_name, checked_at(sort: Desc)], map: "LiteLLM_HealthCheckTable_model_id_model_name_checked_at_idx")
}
// Search Tools table for storing search tool configurations
litellm-proxy-extras/litellm_proxy_extras/utils.py
+76 -21
View File
@@ -4,8 +4,8 @@ import random
import re
import shutil
import subprocess
import tempfile
import time
from datetime import datetime
from pathlib import Path
from typing import Optional
@@ -256,21 +256,11 @@ class ProxyExtrasDBManager:
if not database_url:
logger.error("DATABASE_URL not set")
return
# Prefer DIRECT_URL for schema introspection — pooler URLs (e.g. neon -pooler)
# do not support the extended query protocol required by prisma migrate diff.
diff_url = os.getenv("DIRECT_URL") or database_url
diff_dir = (
Path(migrations_dir)
/ "migrations"
/ f"{datetime.now().strftime('%Y%m%d%H%M%S')}_baseline_diff"
)
try:
diff_dir.mkdir(parents=True, exist_ok=True)
except Exception as e:
if "Permission denied" in str(e):
logger.warning(
f"Permission denied - {e}\nunable to baseline db. Set LITELLM_MIGRATION_DIR environment variable to a writable directory to enable migrations."
)
return
raise e
diff_dir = Path(tempfile.mkdtemp(prefix="litellm_migration_diff_"))
diff_sql_path = diff_dir / "migration.sql"
# 1. Generate migration SQL for the diff between DB and schema
@@ -283,7 +273,7 @@ class ProxyExtrasDBManager:
"migrate",
"diff",
"--from-url",
database_url,
diff_url,
"--to-schema-datamodel",
schema_path,
"--script",
@@ -300,7 +290,40 @@ class ProxyExtrasDBManager:
# check if the migration was created
if not diff_sql_path.exists():
logger.warning("Migration diff was not created")
logger.warning(
"Migration diff was not created (prisma migrate diff failed — "
"likely a pooler URL). Falling back to direct SQL execution of "
"each migration file."
)
# Fall back: run each migration SQL file directly via prisma db execute.
# This works with pooler URLs (no schema introspection needed) and is
# safe to re-run because migrations use IF NOT EXISTS / IF EXISTS guards.
migration_files = sorted(Path(migrations_dir).glob("*/migration.sql"))
for mig_file in migration_files:
try:
subprocess.run(
[
_get_prisma_command(),
"db",
"execute",
"--file",
str(mig_file),
"--schema",
schema_path,
],
timeout=60,
check=True,
capture_output=True,
text=True,
env=_get_prisma_env(),
)
logger.info(f"Applied migration: {mig_file.parent.name}")
except subprocess.CalledProcessError as e:
logger.warning(
f"Failed to apply migration {mig_file.parent.name}: {e.stderr}"
)
except subprocess.TimeoutExpired:
logger.warning(f"Migration {mig_file.parent.name} timed out.")
return
logger.info(f"Migration diff created at {diff_sql_path}")
@@ -395,6 +418,14 @@ class ProxyExtrasDBManager:
logger.info("prisma migrate deploy completed")
# Skip sanity check when deploy reports no pending migrations —
# DB already matches schema, no drift to correct.
if "No pending migrations to apply" in result.stdout:
logger.info(
"No pending migrations — skipping post-migration sanity check"
)
return True
# Run sanity check to ensure DB matches schema
logger.info("Running post-migration sanity check...")
ProxyExtrasDBManager._resolve_all_migrations(
@@ -419,7 +450,10 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._roll_back_migration(
failed_migration
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as rollback_err:
logger.warning(
f"Failed to roll back migration {failed_migration}: {rollback_err}. "
f"It may already be in a rolled-back state."
@@ -431,10 +465,19 @@ class ProxyExtrasDBManager:
logger.info(
f"✅ Migration {failed_migration} resolved, retrying to apply remaining migrations"
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as resolve_err:
logger.warning(
f"Failed to resolve migration {failed_migration}: {resolve_err}"
)
# Apply any schema drift not covered by the marked-as-applied migration
ProxyExtrasDBManager._resolve_all_migrations(
migrations_dir,
schema_path,
mark_all_applied=False,
)
else:
logger.info(
f"Found failed migration: {failed_migration}, marking as rolled back"
@@ -531,7 +574,10 @@ class ProxyExtrasDBManager:
ProxyExtrasDBManager._roll_back_migration(
migration_name
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as rollback_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as rollback_err:
logger.warning(
f"Failed to roll back migration {migration_name}: {rollback_err}. "
f"It may already be in a rolled-back state."
@@ -548,10 +594,19 @@ class ProxyExtrasDBManager:
f"✅ Migration {migration_name} resolved, "
f"retrying to apply remaining migrations"
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as resolve_err:
except (
subprocess.CalledProcessError,
subprocess.TimeoutExpired,
) as resolve_err:
logger.warning(
f"Failed to resolve migration {migration_name}: {resolve_err}"
)
# Apply any schema drift not covered by the marked-as-applied migration
ProxyExtrasDBManager._resolve_all_migrations(
migrations_dir,
schema_path,
mark_all_applied=False,
)
else:
# Unknown P3018 error - log and re-raise for safety
logger.warning(
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.65"
version = "0.4.66"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@@ -25,7 +25,7 @@ required-version = "==0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.65"
version = "0.4.66"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",
litellm/__init__.py
+66 -54
View File
@@ -168,12 +168,12 @@ prometheus_latency_buckets: Optional[List[float]] = None
require_auth_for_metrics_endpoint: Optional[bool] = False
argilla_batch_size: Optional[int] = None
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
gcs_pub_sub_use_v1: Optional[
bool
] = False # if you want to use v1 gcs pubsub logged payload
generic_api_use_v1: Optional[
bool
] = False # if you want to use v1 generic api logged payload
gcs_pub_sub_use_v1: Optional[bool] = (
False # if you want to use v1 gcs pubsub logged payload
)
generic_api_use_v1: Optional[bool] = (
False # if you want to use v1 generic api logged payload
)
argilla_transformation_object: Optional[Dict[str, Any]] = None
_async_input_callback: List[
Union[str, Callable, "CustomLogger"]
@@ -193,26 +193,26 @@ _async_failure_callback: List[
pre_call_rules: List[Callable] = []
post_call_rules: List[Callable] = []
turn_off_message_logging: Optional[bool] = False
standard_logging_payload_excluded_fields: Optional[
List[str]
] = None # Fields to exclude from StandardLoggingPayload before callbacks receive it
standard_logging_payload_excluded_fields: Optional[List[str]] = (
None # Fields to exclude from StandardLoggingPayload before callbacks receive it
)
log_raw_request_response: bool = False
redact_messages_in_exceptions: Optional[bool] = False
redact_user_api_key_info: Optional[bool] = False
filter_invalid_headers: Optional[bool] = False
add_user_information_to_llm_headers: Optional[
bool
] = None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
add_user_information_to_llm_headers: Optional[bool] = (
None # adds user_id, team_id, token hash (params from StandardLoggingMetadata) to request headers
)
store_audit_logs = False # Enterprise feature, allow users to see audit logs
skip_system_message_in_guardrail: bool = False
### end of callbacks #############
email: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
token: Optional[
str
] = None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
email: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
token: Optional[str] = (
None # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
telemetry = True
max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults
drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False))
@@ -274,9 +274,9 @@ use_client: bool = False
ssl_verify: Union[str, bool] = True
ssl_security_level: Optional[str] = None
ssl_certificate: Optional[str] = None
ssl_ecdh_curve: Optional[
str
] = None # Set to 'X25519' to disable PQC and improve performance
ssl_ecdh_curve: Optional[str] = (
None # Set to 'X25519' to disable PQC and improve performance
)
disable_streaming_logging: bool = False
disable_token_counter: bool = False
disable_add_transform_inline_image_block: bool = False
@@ -330,20 +330,24 @@ enable_loadbalancing_on_batch_endpoints: Optional[bool] = None
enable_caching_on_provider_specific_optional_params: bool = (
False # feature-flag for caching on optional params - e.g. 'top_k'
)
caching: bool = False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
caching_with_models: bool = False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
cache: Optional[
"Cache"
] = None # cache object <- use this - https://docs.litellm.ai/docs/caching
caching: bool = (
False # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
caching_with_models: bool = (
False # # Not used anymore, will be removed in next MAJOR release - https://github.com/BerriAI/litellm/discussions/648
)
cache: Optional["Cache"] = (
None # cache object <- use this - https://docs.litellm.ai/docs/caching
)
default_in_memory_ttl: Optional[float] = None
default_redis_ttl: Optional[float] = None
default_redis_batch_cache_expiry: Optional[float] = None
model_alias_map: Dict[str, str] = {}
model_group_settings: Optional["ModelGroupSettings"] = None
max_budget: float = 0.0 # set the max budget across all providers
budget_duration: Optional[
str
] = None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
budget_duration: Optional[str] = (
None # proxy only - resets budget after fixed duration. You can set duration as seconds ("30s"), minutes ("30m"), hours ("30h"), days ("30d").
)
default_soft_budget: float = (
DEFAULT_SOFT_BUDGET # by default all litellm proxy keys have a soft budget of 50.0
)
@@ -352,7 +356,9 @@ forward_traceparent_to_llm_provider: bool = False
_current_cost = 0.0 # private variable, used if max budget is set
error_logs: Dict = {}
add_function_to_prompt: bool = False # if function calling not supported by api, append function call details to system prompt
add_function_to_prompt: bool = (
False # if function calling not supported by api, append function call details to system prompt
)
client_session: Optional[httpx.Client] = None
aclient_session: Optional[httpx.AsyncClient] = None
model_fallbacks: Optional[List] = None # Deprecated for 'litellm.fallbacks'
@@ -399,7 +405,9 @@ prometheus_emit_stream_label: bool = False
disable_add_prefix_to_prompt: bool = (
False # used by anthropic, to disable adding prefix to prompt
)
disable_copilot_system_to_assistant: bool = False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
disable_copilot_system_to_assistant: bool = (
False # If false (default), converts all 'system' role messages to 'assistant' for GitHub Copilot compatibility. Set to true to disable this behavior.
)
public_mcp_servers: Optional[List[str]] = None
public_model_groups: Optional[List[str]] = None
public_agent_groups: Optional[List[str]] = None
@@ -408,9 +416,9 @@ public_agent_groups: Optional[List[str]] = None
# Old format: { "displayName": "url" } (for backward compatibility)
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
#### REQUEST PRIORITIZATION #######
priority_reservation: Optional[
Dict[str, Union[float, "PriorityReservationDict"]]
] = None
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = (
None
)
# priority_reservation_settings is lazy-loaded via __getattr__
# Only declare for type checking - at runtime __getattr__ handles it
if TYPE_CHECKING:
@@ -418,13 +426,17 @@ if TYPE_CHECKING:
######## Networking Settings ########
use_aiohttp_transport: bool = True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
use_aiohttp_transport: bool = (
True # Older variable, aiohttp is now the default. use disable_aiohttp_transport instead.
)
aiohttp_trust_env: bool = False # set to true to use HTTP_ Proxy settings
disable_aiohttp_transport: bool = False # Set this to true to use httpx instead
disable_aiohttp_trust_env: bool = (
False # When False, aiohttp will respect HTTP(S)_PROXY env vars
)
force_ipv4: bool = False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
force_ipv4: bool = (
False # when True, litellm will force ipv4 for all LLM requests. Some users have seen httpx ConnectionError when using ipv6.
)
network_mock: bool = False # When True, use mock transport — no real network calls
####### STOP SEQUENCE LIMIT #######
@@ -439,13 +451,13 @@ context_window_fallbacks: Optional[List] = None
content_policy_fallbacks: Optional[List] = None
allowed_fails: int = 3
allow_dynamic_callback_disabling: bool = True
num_retries_per_request: Optional[
int
] = None # for the request overall (incl. fallbacks + model retries)
num_retries_per_request: Optional[int] = (
None # for the request overall (incl. fallbacks + model retries)
)
####### SECRET MANAGERS #####################
secret_manager_client: Optional[
Any
] = None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
secret_manager_client: Optional[Any] = (
None # list of instantiated key management clients - e.g. azure kv, infisical, etc.
)
_google_kms_resource_name: Optional[str] = None
_key_management_system: Optional["KeyManagementSystem"] = None
# Note: KeyManagementSettings must be eagerly imported because _key_management_settings
@@ -458,12 +470,12 @@ output_parse_pii: bool = False
from litellm.litellm_core_utils.get_model_cost_map import get_model_cost_map
model_cost = get_model_cost_map(url=model_cost_map_url)
cost_discount_config: Dict[
str, float
] = {} # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_margin_config: Dict[
str, Union[float, Dict[str, float]]
] = {} # Provider-specific or global cost margins. Examples:
cost_discount_config: Dict[str, float] = (
{}
) # Provider-specific cost discounts {"vertex_ai": 0.05} = 5% discount
cost_margin_config: Dict[str, Union[float, Dict[str, float]]] = (
{}
) # Provider-specific or global cost margins. Examples:
# Percentage: {"openai": 0.10} = 10% margin
# Fixed: {"openai": {"fixed_amount": 0.001}} = $0.001 per request
# Global: {"global": 0.05} = 5% global margin on all providers
@@ -1313,12 +1325,12 @@ from . import rag
from .types.llms.custom_llm import CustomLLMItem
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[
str
] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[
bool
] = None # disable huggingface tokenizer download. Defaults to openai clk100
_custom_providers: List[str] = (
[]
) # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
None # disable huggingface tokenizer download. Defaults to openai clk100
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###
litellm/_internal_context.py
+13
View File
@@ -0,0 +1,13 @@
"""
Internal request context for LiteLLM.
Provides a ContextVar-based mechanism for internal signals that must not
be settable from user input. Context variables are scoped to the current
asyncio task and cannot be injected via HTTP request bodies.
"""
from contextvars import ContextVar
# When True, suppresses async logging and billing for internal sub-calls
# (e.g., emulated file-search steps that make nested LLM calls).
is_internal_call: ContextVar[bool] = ContextVar("is_internal_call", default=False)
litellm/_lazy_imports.py
+1
View File
@@ -14,6 +14,7 @@ How it works:
This makes importing litellm much faster because we don't load heavy dependencies
until they're actually needed.
"""
import importlib
import sys
from typing import Any, Optional, cast, Callable
litellm/_logging.py
+2
View File
@@ -86,6 +86,8 @@ _SECRET_RE = _build_secret_patterns()
def _redact_string(value: str) -> str:
if not _ENABLE_SECRET_REDACTION:
return value
return _SECRET_RE.sub(_REDACTED, value)
litellm/a2a_protocol/main.py
+3 -3
View File
@@ -120,9 +120,9 @@ def _get_a2a_model_info(a2a_client: Any, kwargs: Dict[str, Any]) -> str:
litellm_logging_obj.model = model
litellm_logging_obj.custom_llm_provider = custom_llm_provider
litellm_logging_obj.model_call_details["model"] = model
litellm_logging_obj.model_call_details[
"custom_llm_provider"
] = custom_llm_provider
litellm_logging_obj.model_call_details["custom_llm_provider"] = (
custom_llm_provider
)
return agent_name
litellm/a2a_protocol/providers/bedrock_agentcore/handler.py
+1 -3
View File
@@ -99,9 +99,7 @@ class BedrockAgentCoreA2AHandler:
)
)
verbose_logger.info(
f"BedrockAgentCore A2A: Sending streaming request to {url}"
)
verbose_logger.info(f"BedrockAgentCore A2A: Sending streaming request to {url}")
client = get_async_httpx_client(
llm_provider=cast(Any, httpxSpecialProvider.A2AProvider),
litellm/a2a_protocol/streaming_iterator.py
+3 -3
View File
@@ -168,9 +168,9 @@ class A2AStreamingIterator:
result: Dict[str, Any] = {
"id": getattr(self.request, "id", "unknown"),
"jsonrpc": "2.0",
"usage": usage.model_dump()
if hasattr(usage, "model_dump")
else dict(usage),
"usage": (
usage.model_dump() if hasattr(usage, "model_dump") else dict(usage)
),
}
# Add final chunk result if available
litellm/anthropic_beta_headers_config.json
+4 -4
View File
@@ -71,12 +71,12 @@
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"context-management-2025-06-27": null,
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"interleaved-thinking-2025-05-14": null,
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
@@ -102,12 +102,12 @@
"computer-use-2025-01-24": "computer-use-2025-01-24",
"computer-use-2025-11-24": "computer-use-2025-11-24",
"context-1m-2025-08-07": "context-1m-2025-08-07",
"context-management-2025-06-27": "context-management-2025-06-27",
"context-management-2025-06-27": null,
"effort-2025-11-24": null,
"fast-mode-2026-02-01": null,
"files-api-2025-04-14": null,
"fine-grained-tool-streaming-2025-05-14": null,
"interleaved-thinking-2025-05-14": "interleaved-thinking-2025-05-14",
"interleaved-thinking-2025-05-14": null,
"mcp-client-2025-11-20": null,
"mcp-client-2025-04-04": null,
"mcp-servers-2025-12-04": null,
litellm/anthropic_interface/__init__.py
+1
View File
@@ -1,6 +1,7 @@
"""
Anthropic module for LiteLLM
"""
from .messages import acreate, create
__all__ = ["acreate", "create"]
litellm/anthropic_interface/messages/__init__.py
+2 -2
View File
@@ -38,7 +38,7 @@ async def acreate(
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
**kwargs,
) -> Union[AnthropicMessagesResponse, AsyncIterator]:
"""
Async wrapper for Anthropic's messages API
@@ -97,7 +97,7 @@ def create(
top_k: Optional[int] = None,
top_p: Optional[float] = None,
container: Optional[Dict] = None,
**kwargs
**kwargs,
) -> Union[
AnthropicMessagesResponse,
AsyncIterator[Any],
litellm/caching/caching_handler.py
+6 -4
View File
@@ -78,7 +78,9 @@ class CachingHandlerResponse(BaseModel):
cached_result: Optional[Any] = None
final_embedding_cached_response: Optional[EmbeddingResponse] = None
embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
embedding_all_elements_cache_hit: bool = (
False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call
)
in_memory_cache_obj = InMemoryCache()
@@ -1014,9 +1016,9 @@ class LLMCachingHandler:
}
if litellm.cache is not None:
litellm_params[
"preset_cache_key"
] = litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
litellm_params["preset_cache_key"] = (
litellm.cache._get_preset_cache_key_from_kwargs(**kwargs)
)
else:
litellm_params["preset_cache_key"] = None
litellm/caching/gcs_cache.py
+1
View File
@@ -1,6 +1,7 @@
"""GCS Cache implementation
Supports syncing responses to Google Cloud Storage Buckets using HTTP requests.
"""
import json
import asyncio
from typing import Optional
litellm/completion_extras/litellm_responses_transformation/handler.py
+1 -3
View File
@@ -142,9 +142,7 @@ class ResponsesToCompletionBridgeHandler:
custom_llm_provider=custom_llm_provider,
)
def completion(
self, *args, **kwargs
) -> Union[
def completion(self, *args, **kwargs) -> Union[
Coroutine[Any, Any, Union["ModelResponse", "CustomStreamWrapper"]],
"ModelResponse",
"CustomStreamWrapper",
litellm/completion_extras/litellm_responses_transformation/transformation.py
+20 -16
View File
@@ -300,10 +300,10 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
if key in ("max_tokens", "max_completion_tokens"):
responses_api_request["max_output_tokens"] = value
elif key == "tools" and value is not None:
responses_api_request[
"tools"
] = self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
responses_api_request["tools"] = (
self._convert_tools_to_responses_format(
cast(List[Dict[str, Any]], value)
)
)
elif key == "response_format":
text_format = self._transform_response_format_to_text_format(value)
@@ -506,9 +506,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
annotations=annotations,
reasoning_items=cast(
Optional[List[ChatCompletionReasoningItem]],
[pending_reasoning_item]
if pending_reasoning_item is not None
else None,
(
[pending_reasoning_item]
if pending_reasoning_item is not None
else None
),
),
)
@@ -566,9 +568,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge):
reasoning_content=reasoning_content,
reasoning_items=cast(
Optional[List[ChatCompletionReasoningItem]],
[pending_reasoning_item]
if pending_reasoning_item is not None
else None,
(
[pending_reasoning_item]
if pending_reasoning_item is not None
else None
),
),
)
choices.append(
@@ -1154,9 +1158,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
)
if provider_specific_fields:
function_chunk[
"provider_specific_fields"
] = provider_specific_fields
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(
@@ -1229,9 +1233,9 @@ class OpenAiResponsesToChatCompletionStreamIterator(BaseModelResponseIterator):
# Add provider_specific_fields to function if present
if provider_specific_fields:
function_chunk[
"provider_specific_fields"
] = provider_specific_fields
function_chunk["provider_specific_fields"] = (
provider_specific_fields
)
tool_call_index = parsed_chunk.get("output_index", 0)
tool_call_chunk = ChatCompletionToolCallChunk(
litellm/compression/compress.py
+5 -3
View File
@@ -247,9 +247,11 @@ def compress(
messages=compressed_messages,
original_tokens=original_tokens,
compressed_tokens=compressed_tokens,
compression_ratio=round(1 - (compressed_tokens / original_tokens), 4)
if original_tokens > 0
else 0.0,
compression_ratio=(
round(1 - (compressed_tokens / original_tokens), 4)
if original_tokens > 0
else 0.0
),
cache=cache,
tools=tools,
)
litellm/constants.py
+32 -2
View File
@@ -1,6 +1,6 @@
import os
import sys
from typing import List, Literal
from typing import List, Literal, Optional
from litellm.litellm_core_utils.env_utils import get_env_int
@@ -413,7 +413,20 @@ MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
)
DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
#### Networking settings ####
request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds
# Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
# value so longer-running surfaces (Router `timeout or litellm.request_timeout`,
# speech/TTS, responses, vector stores, etc.) get a long HTTP deadline. Chat
# `completion()` maps this sentinel down to 600s when the caller did not set a
# per-request/model timeout—see ``CompletionTimeout.resolve`` in completion_timeout.py. MCP uses
# dedicated timeouts (e.g. `MCP_CLIENT_TIMEOUT`), not `request_timeout`.
DEFAULT_REQUEST_TIMEOUT_SECONDS: float = 6000.0
# Pair used for default httpx clients when no custom timeout is passed: read/write
# deadline and connect handshake (see ``http_handler`` cached handler paths).
COMPLETION_HTTP_FALLBACK_SECONDS: float = 600.0
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS: float = 5.0
request_timeout: float = float(
os.getenv("REQUEST_TIMEOUT", str(int(DEFAULT_REQUEST_TIMEOUT_SECONDS)))
)
DEFAULT_A2A_AGENT_TIMEOUT: float = float(
os.getenv("DEFAULT_A2A_AGENT_TIMEOUT", 6000)
) # 10 minutes
@@ -1113,6 +1126,7 @@ BEDROCK_CONVERSE_MODELS = [
"openai.gpt-oss-120b-1:0",
"anthropic.claude-haiku-4-5-20251001-v1:0",
"anthropic.claude-sonnet-4-5-20250929-v1:0",
"anthropic.claude-opus-4-7",
"anthropic.claude-opus-4-6-v1:0",
"anthropic.claude-opus-4-6-v1",
"anthropic.claude-sonnet-4-6",
@@ -1330,6 +1344,22 @@ BATCH_STATUS_POLL_MAX_ATTEMPTS = int(
HEALTH_CHECK_TIMEOUT_SECONDS = int(
os.getenv("HEALTH_CHECK_TIMEOUT_SECONDS", 60)
) # 60 seconds
_background_health_check_max_tokens_env = os.getenv(
"BACKGROUND_HEALTH_CHECK_MAX_TOKENS"
)
try:
_raw_background_health_check_max_tokens = (
_background_health_check_max_tokens_env.strip()
if _background_health_check_max_tokens_env is not None
else ""
)
BACKGROUND_HEALTH_CHECK_MAX_TOKENS: Optional[int] = (
int(_raw_background_health_check_max_tokens)
if _raw_background_health_check_max_tokens
else None
)
except (ValueError, TypeError):
BACKGROUND_HEALTH_CHECK_MAX_TOKENS = None
LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME = "litellm-internal-health-check"
LITTELM_CLI_SERVICE_ACCOUNT_NAME = "litellm-cli"
LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME = "litellm_internal_jobs"
litellm/containers/endpoint_factory.py
+4 -4
View File
@@ -90,10 +90,10 @@ def create_sync_endpoint_function(endpoint_config: Dict) -> Callable:
custom_llm_provider=resolved_custom_llm_provider,
litellm_params=litellm_params,
)
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
litellm/containers/main.py
+62 -44
View File
@@ -168,7 +168,10 @@ def create_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerObject, Coroutine[Any, Any, ContainerObject],]:
) -> Union[
ContainerObject,
Coroutine[Any, Any, ContainerObject],
]:
"""Create a container using the OpenAI Container API.
Currently supports OpenAI
@@ -208,10 +211,10 @@ def create_container(
**kwargs,
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
@@ -260,7 +263,7 @@ def create_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
if isinstance(container_obj, ContainerObject):
container_obj = ContainerRequestUtils.encode_container_id_in_response(
@@ -269,7 +272,7 @@ def create_container(
litellm_metadata=kwargs.get("litellm_metadata"),
extra_body=extra_body,
)
return container_obj
except Exception as e:
@@ -405,7 +408,10 @@ def list_containers(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerListResponse, Coroutine[Any, Any, ContainerListResponse],]:
) -> Union[
ContainerListResponse,
Coroutine[Any, Any, ContainerListResponse],
]:
"""List containers using the OpenAI Container API.
Currently supports OpenAI
@@ -434,10 +440,10 @@ def list_containers(
**kwargs,
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if container_provider_config is None:
@@ -601,7 +607,10 @@ def retrieve_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerObject, Coroutine[Any, Any, ContainerObject],]:
) -> Union[
ContainerObject,
Coroutine[Any, Any, ContainerObject],
]:
"""Retrieve a container using the OpenAI Container API.
Currently supports OpenAI
@@ -630,7 +639,7 @@ def retrieve_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@@ -643,10 +652,10 @@ def retrieve_container(
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@@ -678,7 +687,7 @@ def retrieve_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(container_obj, ContainerObject):
@@ -691,14 +700,14 @@ def retrieve_container(
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
container_obj = ContainerRequestUtils.encode_container_id_in_response(
response_obj=container_obj,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return container_obj
except Exception as e:
@@ -822,7 +831,10 @@ def delete_container(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[DeleteContainerResult, Coroutine[Any, Any, DeleteContainerResult],]:
) -> Union[
DeleteContainerResult,
Coroutine[Any, Any, DeleteContainerResult],
]:
"""Delete a container using the OpenAI Container API.
Currently supports OpenAI
@@ -851,7 +863,7 @@ def delete_container(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@@ -864,10 +876,10 @@ def delete_container(
was_encoded = original_container_id != container_id
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@@ -899,7 +911,7 @@ def delete_container(
timeout=timeout or DEFAULT_REQUEST_TIMEOUT,
_is_async=_is_async,
)
# Encode container_id in response with provider/model metadata for routing
# If input was encoded, preserve encoding in output using the decoded model_id
if isinstance(delete_result, DeleteContainerResult):
@@ -912,14 +924,14 @@ def delete_container(
if "model_info" not in litellm_metadata:
litellm_metadata["model_info"] = {}
litellm_metadata["model_info"]["id"] = litellm_params["model_id"]
delete_result = ContainerRequestUtils.encode_container_id_in_response(
response_obj=delete_result,
custom_llm_provider=resolved_custom_llm_provider,
litellm_metadata=litellm_metadata,
extra_body=None,
)
return delete_result
except Exception as e:
@@ -1057,7 +1069,10 @@ def list_container_files(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerFileListResponse, Coroutine[Any, Any, ContainerFileListResponse],]:
) -> Union[
ContainerFileListResponse,
Coroutine[Any, Any, ContainerFileListResponse],
]:
"""List files in a container using the OpenAI Container API.
Currently supports OpenAI
@@ -1086,7 +1101,7 @@ def list_container_files(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@@ -1095,12 +1110,12 @@ def list_container_files(
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
@@ -1285,7 +1300,10 @@ def upload_container_file(
extra_query: Optional[Dict[str, Any]] = None,
extra_body: Optional[Dict[str, Any]] = None,
**kwargs,
) -> Union[ContainerFileObject, Coroutine[Any, Any, ContainerFileObject],]:
) -> Union[
ContainerFileObject,
Coroutine[Any, Any, ContainerFileObject],
]:
"""Upload a file to a container using the OpenAI Container API.
This endpoint allows uploading files directly to a container session,
@@ -1343,7 +1361,7 @@ def upload_container_file(
api_version=api_version,
**kwargs,
)
# Decode container ID and extract provider info
original_container_id, resolved_custom_llm_provider, litellm_params = (
decode_managed_container_id_for_request(
@@ -1352,12 +1370,12 @@ def upload_container_file(
litellm_params=litellm_params,
)
)
# get provider config
container_provider_config: Optional[
BaseContainerConfig
] = ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
container_provider_config: Optional[BaseContainerConfig] = (
ProviderConfigManager.get_provider_container_config(
provider=litellm.LlmProviders(resolved_custom_llm_provider),
)
)
if container_provider_config is None:
litellm/containers/utils.py
+5 -4
View File
@@ -32,6 +32,7 @@ def decode_managed_container_id_for_request(
return original_container_id, custom_llm_provider, litellm_params
T = TypeVar("T")
@@ -129,14 +130,14 @@ class ContainerRequestUtils:
litellm_metadata = litellm_metadata or {}
model_info: Dict[str, Any] = litellm_metadata.get("model_info", {}) or {}
model_id = model_info.get("id")
# Check if we should encode based on routing metadata
should_encode = False
# Case 1: Router/proxy usage (model_id from router)
if model_id is not None:
should_encode = True
# Case 2: target_model_names in extra_body (model-specific routing)
if extra_body and "target_model_names" in extra_body:
should_encode = True
@@ -148,7 +149,7 @@ class ContainerRequestUtils:
model_id = target_models.split(",")[0].strip()
elif isinstance(target_models, list) and len(target_models) > 0:
model_id = str(target_models[0]).strip()
# Only encode if we have routing metadata
if should_encode and response_obj and hasattr(response_obj, "id"):
encoded_id = ResponsesAPIRequestUtils._build_container_id(
litellm/cost_calculator.py
+40 -7
View File
@@ -545,10 +545,9 @@ def cost_per_token( # noqa: PLR0915
model=model, custom_llm_provider=custom_llm_provider
)
if (
(model_info.get("input_cost_per_token") or 0.0) > 0
or (model_info.get("output_cost_per_token") or 0.0) > 0
):
if (model_info.get("input_cost_per_token") or 0.0) > 0 or (
model_info.get("output_cost_per_token") or 0.0
) > 0:
return generic_cost_per_token(
model=model,
usage=usage_block,
@@ -966,6 +965,8 @@ def _store_cost_breakdown_in_logging_obj(
margin_percent: Optional[float] = None,
margin_fixed_amount: Optional[float] = None,
margin_total_amount: Optional[float] = None,
cache_read_cost: Optional[float] = None,
cache_creation_cost: Optional[float] = None,
) -> None:
"""
Helper function to store cost breakdown in the logging object.
@@ -1001,6 +1002,8 @@ def _store_cost_breakdown_in_logging_obj(
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
cache_read_cost=cache_read_cost,
cache_creation_cost=cache_creation_cost,
)
except Exception as breakdown_error:
@@ -1137,9 +1140,9 @@ def completion_cost( # noqa: PLR0915
or isinstance(completion_response, dict)
): # tts returns a custom class
if isinstance(completion_response, dict):
usage_obj: Optional[
Union[dict, Usage]
] = completion_response.get("usage", {})
usage_obj: Optional[Union[dict, Usage]] = (
completion_response.get("usage", {})
)
else:
usage_obj = getattr(completion_response, "usage", {})
if isinstance(usage_obj, BaseModel) and not _is_known_usage_objects(
@@ -1599,6 +1602,34 @@ def completion_cost( # noqa: PLR0915
# Store cost breakdown in logging object if available
if litellm_logging_obj is not None:
_cache_read_cost: Optional[float] = None
_cache_creation_cost: Optional[float] = None
if cost_per_token_usage_object is not None:
_cr = getattr(
cost_per_token_usage_object, "cache_read_input_tokens", None
) or (cost_per_token_usage_object.model_extra or {}).get(
"cache_read_input_tokens"
)
_cc = getattr(
cost_per_token_usage_object,
"cache_creation_input_tokens",
None,
) or (cost_per_token_usage_object.model_extra or {}).get(
"cache_creation_input_tokens"
)
if (_cr or _cc) and model:
try:
_mi = litellm.get_model_info(
model=model, custom_llm_provider=custom_llm_provider
)
_cr_rate = _mi.get("cache_read_input_token_cost")
if _cr and _cr_rate is not None:
_cache_read_cost = float(_cr) * float(_cr_rate)
_cc_rate = _mi.get("cache_creation_input_token_cost")
if _cc and _cc_rate is not None:
_cache_creation_cost = float(_cc) * float(_cc_rate)
except Exception:
pass
_store_cost_breakdown_in_logging_obj(
litellm_logging_obj=litellm_logging_obj,
prompt_tokens_cost_usd_dollar=prompt_tokens_cost_usd_dollar,
@@ -1612,6 +1643,8 @@ def completion_cost( # noqa: PLR0915
margin_percent=margin_percent,
margin_fixed_amount=margin_fixed_amount,
margin_total_amount=margin_total_amount,
cache_read_cost=_cache_read_cost,
cache_creation_cost=_cache_creation_cost,
)
return _final_cost
litellm/evals/main.py
+44 -44
View File
@@ -152,10 +152,10 @@ def create_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -343,10 +343,10 @@ def list_evals(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -513,10 +513,10 @@ def get_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -682,10 +682,10 @@ def update_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -893,10 +893,10 @@ def delete_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1047,10 +1047,10 @@ def cancel_eval(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1230,10 +1230,10 @@ def create_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1418,10 +1418,10 @@ def list_runs(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1592,10 +1592,10 @@ def get_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1752,10 +1752,10 @@ def cancel_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
@@ -1921,10 +1921,10 @@ def delete_run(
custom_llm_provider = "openai"
# Get provider config
evals_api_provider_config: Optional[
BaseEvalsAPIConfig
] = ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
evals_api_provider_config: Optional[BaseEvalsAPIConfig] = (
ProviderConfigManager.get_provider_evals_api_config( # type: ignore
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if evals_api_provider_config is None:
litellm/exceptions.py
+2 -1
View File
@@ -281,7 +281,7 @@ class Timeout(openai.APITimeoutError): # type: ignore
return _message
class PermissionDeniedError(openai.PermissionDeniedError): # type:ignore
class PermissionDeniedError(openai.PermissionDeniedError): # type: ignore
def __init__(
self,
message,
@@ -847,6 +847,7 @@ class BudgetExceededError(Exception):
):
self.current_cost = current_cost
self.max_budget = max_budget
self.status_code = 429
message = (
message
or f"Budget has been exceeded! Current cost: {current_cost}, Max budget: {max_budget}"
litellm/experimental_mcp_client/client.py
+8 -1
View File
@@ -221,6 +221,7 @@ class MCPClient:
self.extra_headers: Optional[Dict[str, str]] = extra_headers
self.ssl_verify: Optional[VerifyTypes] = ssl_verify
self._aws_auth: Optional[httpx.Auth] = aws_auth
self._last_initialize_instructions: Optional[str] = None
# handle the basic auth value if provided
if auth_value:
self.update_auth_value(auth_value)
@@ -296,7 +297,12 @@ class MCPClient:
session_ctx = ClientSession(read_stream, write_stream)
session = await session_ctx.__aenter__()
try:
await session.initialize()
init_result = await session.initialize()
self._last_initialize_instructions = None
if init_result is not None:
ins = getattr(init_result, "instructions", None)
if isinstance(ins, str) and ins.strip():
self._last_initialize_instructions = ins.strip()
return await operation(session)
finally:
try:
@@ -315,6 +321,7 @@ class MCPClient:
"""Open a session, run the provided coroutine, and clean up."""
http_client: Optional[httpx.AsyncClient] = None
try:
self._last_initialize_instructions = None
transport_ctx, http_client = self._create_transport_context()
return await self._execute_session_operation(transport_ctx, operation)
except Exception:
litellm/files/main.py
+9 -3
View File
@@ -10,7 +10,7 @@ import contextvars
import time
import uuid as uuid_module
from functools import partial
from typing import Any,Coroutine, Dict, Literal, Optional, Union, cast
from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast
import httpx
@@ -53,7 +53,10 @@ from litellm.types.llms.openai import (
OpenAIFileObject,
)
from litellm.types.router import *
from litellm.types.utils import OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS, LlmProviders
from litellm.types.utils import (
OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS,
LlmProviders,
)
from litellm.utils import (
ProviderConfigManager,
client,
@@ -73,6 +76,8 @@ def _should_sdk_support_streaming(
Return whether file content streaming is supported for the provider.
"""
return custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS
openai_files_instance = OpenAIFilesAPI()
azure_files_instance = AzureOpenAIFilesAPI()
vertex_ai_files_instance = VertexAIFilesHandler()
@@ -1094,9 +1099,10 @@ def file_content_streaming(
)
if asyncio.iscoroutine(response):
async def _await_and_wrap() -> FileContentStreamingResult:
return _wrap_streaming_result(await response)
return _await_and_wrap()
return _wrap_streaming_result(response)
return _wrap_streaming_result(response)
litellm/files/streaming.py
+17 -3
View File
@@ -1,6 +1,15 @@
import datetime
import traceback
from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Iterator, Optional, Union, cast
from typing import (
TYPE_CHECKING,
Any,
AsyncIterator,
Dict,
Iterator,
Optional,
Union,
cast,
)
import anyio
from litellm.files.types import FileContentProvider
@@ -11,6 +20,7 @@ if TYPE_CHECKING:
)
from litellm.types.utils import StandardLoggingHiddenParams, StandardLoggingPayload
class FileContentStreamingResponse:
"""
Iterator wrapper for file content streaming that carries LiteLLM metadata
@@ -84,7 +94,9 @@ class FileContentStreamingResponse:
self._close_completed = True
self._logging_completed = True
stream_to_close = self.stream_iterator
self.stream_iterator = cast(Union[Iterator[bytes], AsyncIterator[bytes]], iter(()))
self.stream_iterator = cast(
Union[Iterator[bytes], AsyncIterator[bytes]], iter(())
)
# Shield cleanup from request cancellation so upstream HTTP connections
# are released promptly on client disconnects.
@@ -103,7 +115,9 @@ class FileContentStreamingResponse:
self._close_completed = True
self._logging_completed = True
stream_to_close = self.stream_iterator
self.stream_iterator = cast(Union[Iterator[bytes], AsyncIterator[bytes]], iter(()))
self.stream_iterator = cast(
Union[Iterator[bytes], AsyncIterator[bytes]], iter(())
)
if hasattr(stream_to_close, "close"):
cast(Iterator[bytes], stream_to_close).close() # type: ignore[attr-defined]
litellm/images/main.py
+21 -18
View File
@@ -210,7 +210,10 @@ def image_generation( # noqa: PLR0915
api_version: Optional[str] = None,
custom_llm_provider=None,
**kwargs,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Maps the https://api.openai.com/v1/images/generations endpoint.
@@ -864,11 +867,11 @@ def image_edit( # noqa: PLR0915
)
# get provider config
image_edit_provider_config: Optional[
BaseImageEditConfig
] = ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
image_edit_provider_config: Optional[BaseImageEditConfig] = (
ProviderConfigManager.get_provider_image_edit_config(
model=model,
provider=litellm.LlmProviders(custom_llm_provider),
)
)
if image_edit_provider_config is None:
@@ -876,20 +879,20 @@ def image_edit( # noqa: PLR0915
local_vars.update(kwargs)
# Get ImageEditOptionalRequestParams with only valid parameters
image_edit_optional_params: ImageEditOptionalRequestParams = (
_get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
local_vars
)
image_edit_optional_params: (
ImageEditOptionalRequestParams
) = _get_ImageEditRequestUtils().get_requested_image_edit_optional_param(
local_vars
)
# Get optional parameters for the responses API
image_edit_request_params: Dict = (
_get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
)
image_edit_request_params: (
Dict
) = _get_ImageEditRequestUtils().get_optional_params_image_edit(
model=model,
image_edit_provider_config=image_edit_provider_config,
image_edit_optional_params=image_edit_optional_params,
drop_params=kwargs.get("drop_params"),
additional_drop_params=kwargs.get("additional_drop_params"),
)
# Pre Call logging
litellm/integrations/SlackAlerting/hanging_request_check.py
+4 -4
View File
@@ -102,10 +102,10 @@ class AlertingHangingRequestCheck:
)
for request_id in hanging_requests:
hanging_request_data: Optional[
HangingRequestData
] = await self.hanging_request_cache.async_get_cache(
key=request_id,
hanging_request_data: Optional[HangingRequestData] = (
await self.hanging_request_cache.async_get_cache(
key=request_id,
)
)
if hanging_request_data is None:
litellm/integrations/SlackAlerting/slack_alerting.py
+9 -9
View File
@@ -852,9 +852,9 @@ class SlackAlerting(CustomBatchLogger):
### UNIQUE CACHE KEY ###
cache_key = provider + region_name
outage_value: Optional[
ProviderRegionOutageModel
] = await self.internal_usage_cache.async_get_cache(key=cache_key)
outage_value: Optional[ProviderRegionOutageModel] = (
await self.internal_usage_cache.async_get_cache(key=cache_key)
)
# Convert deployment_ids back to set if it was stored as a list
if outage_value is not None:
@@ -1443,9 +1443,9 @@ Model Info:
self.alert_to_webhook_url is not None
and alert_type in self.alert_to_webhook_url
):
_digest_webhook: Optional[
Union[str, List[str]]
] = self.alert_to_webhook_url[alert_type]
_digest_webhook: Optional[Union[str, List[str]]] = (
self.alert_to_webhook_url[alert_type]
)
elif self.default_webhook_url is not None:
_digest_webhook = self.default_webhook_url
else:
@@ -1499,9 +1499,9 @@ Model Info:
self.alert_to_webhook_url is not None
and alert_type in self.alert_to_webhook_url
):
slack_webhook_url: Optional[
Union[str, List[str]]
] = self.alert_to_webhook_url[alert_type]
slack_webhook_url: Optional[Union[str, List[str]]] = (
self.alert_to_webhook_url[alert_type]
)
elif self.default_webhook_url is not None:
slack_webhook_url = self.default_webhook_url
else:
litellm/integrations/agentops/agentops.py
+1
View File
@@ -1,6 +1,7 @@
"""
AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
"""
import os
from dataclasses import dataclass
from typing import Optional, Dict, Any
litellm/integrations/anthropic_cache_control_hook.py
+4 -4
View File
@@ -106,10 +106,10 @@ class AnthropicCacheControlHook(CustomPromptManagement):
targetted_index += len(messages)
if 0 <= targetted_index < len(messages):
messages[
targetted_index
] = AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
messages[targetted_index] = (
AnthropicCacheControlHook._safe_insert_cache_control_in_message(
messages[targetted_index], control
)
)
else:
verbose_logger.warning(
litellm/integrations/arize/arize_phoenix.py
+3 -3
View File
@@ -178,9 +178,9 @@ class ArizePhoenixLogger(OpenTelemetry): # type: ignore
start_time_val = kwargs.get("start_time", kwargs.get("api_call_start_time"))
parent_span = self.tracer.start_span(
name="litellm_proxy_request",
start_time=self._to_ns(start_time_val)
if start_time_val is not None
else None,
start_time=(
self._to_ns(start_time_val) if start_time_val is not None else None
),
context=traceparent_ctx,
kind=self.span_kind.SERVER,
)
litellm/integrations/azure_storage/azure_storage.py
+6 -6
View File
@@ -54,12 +54,12 @@ class AzureBlobStorageLogger(CustomBatchLogger):
self._service_client_timeout: Optional[float] = None
# Internal variables used for Token based authentication
self.azure_auth_token: Optional[
str
] = None # the Azure AD token to use for Azure Storage API requests
self.token_expiry: Optional[
datetime
] = None # the expiry time of the currentAzure AD token
self.azure_auth_token: Optional[str] = (
None # the Azure AD token to use for Azure Storage API requests
)
self.token_expiry: Optional[datetime] = (
None # the expiry time of the currentAzure AD token
)
asyncio.create_task(self.periodic_flush())
self.flush_lock = asyncio.Lock()
litellm/integrations/braintrust_logging.py
+3 -3
View File
@@ -52,9 +52,9 @@ class BraintrustLogger(CustomLogger):
"Authorization": "Bearer " + self.api_key,
"Content-Type": "application/json",
}
self._project_id_cache: Dict[
str, str
] = {} # Cache mapping project names to IDs
self._project_id_cache: Dict[str, str] = (
{}
) # Cache mapping project names to IDs
self.global_braintrust_http_handler = get_async_httpx_client(
llm_provider=httpxSpecialProvider.LoggingCallback
)
litellm/integrations/cloudzero/cloudzero.py
+4 -4
View File
@@ -402,10 +402,10 @@ class CloudZeroLogger(CustomLogger):
from litellm.constants import CLOUDZERO_EXPORT_INTERVAL_MINUTES
from litellm.integrations.custom_logger import CustomLogger
prometheus_loggers: List[
CustomLogger
] = litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CloudZeroLogger
prometheus_loggers: List[CustomLogger] = (
litellm.logging_callback_manager.get_custom_loggers_for_type(
callback_type=CloudZeroLogger
)
)
# we need to get the initialized prometheus logger instance(s) and call logger.initialize_remaining_budget_metrics() on them
verbose_logger.debug("found %s cloudzero loggers", len(prometheus_loggers))
litellm/integrations/cloudzero/transform.py
+6 -6
View File
@@ -159,9 +159,9 @@ class CBFTransformer:
# CloudZero CBF format with proper column names
cbf_record = {
# Required CBF fields
"time/usage_start": usage_date.isoformat()
if usage_date
else None, # Required: ISO-formatted UTC datetime
"time/usage_start": (
usage_date.isoformat() if usage_date else None
), # Required: ISO-formatted UTC datetime
"cost/cost": float(row.get("spend", 0.0)), # Required: billed cost
"resource/id": resource_id, # CZRN (CloudZero Resource Name)
# Usage metrics for token consumption
@@ -182,9 +182,9 @@ class CBFTransformer:
# Add CZRN components that don't have direct CBF column mappings as resource tags
cbf_record["resource/tag:provider"] = provider # CZRN provider component
cbf_record[
"resource/tag:model"
] = cloud_local_id # CZRN cloud-local-id component (model)
cbf_record["resource/tag:model"] = (
cloud_local_id # CZRN cloud-local-id component (model)
)
# Add resource tags for all dimensions (using resource/tag:<key> format)
for key, value in dimensions.items():
litellm/integrations/custom_guardrail.py
+7 -2
View File
@@ -417,7 +417,9 @@ class CustomGuardrail(CustomLogger):
"""
requested_guardrails = self.get_guardrail_from_metadata(data)
disable_global_guardrail = self.get_disable_global_guardrail(data)
opted_out_global_guardrails = self.get_opted_out_global_guardrails_from_metadata(data)
opted_out_global_guardrails = (
self.get_opted_out_global_guardrails_from_metadata(data)
)
verbose_logger.debug(
"inside should_run_guardrail for guardrail=%s event_type= %s guardrail_supported_event_hooks= %s requested_guardrails= %s self.default_on= %s",
self.guardrail_name,
@@ -426,7 +428,10 @@ class CustomGuardrail(CustomLogger):
requested_guardrails,
self.default_on,
)
if self.default_on is True and self.guardrail_name in opted_out_global_guardrails:
if (
self.default_on is True
and self.guardrail_name in opted_out_global_guardrails
):
return False
if self.default_on is True and disable_global_guardrail is not True:
litellm/integrations/custom_logger.py
+3 -3
View File
@@ -874,9 +874,9 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac
model_response_dict = model_response.model_dump()
standard_logging_object_copy["response"] = model_response_dict
model_call_details_copy[
"standard_logging_object"
] = standard_logging_object_copy
model_call_details_copy["standard_logging_object"] = (
standard_logging_object_copy
)
return model_call_details_copy
async def get_proxy_server_request_from_cold_storage_with_object_key(
litellm/integrations/datadog/datadog_llm_obs.py
+12 -12
View File
@@ -349,9 +349,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if standard_logging_payload.get("status") == "failure":
# Try to get structured error information first
error_information: Optional[
StandardLoggingPayloadErrorInformation
] = standard_logging_payload.get("error_information")
error_information: Optional[StandardLoggingPayloadErrorInformation] = (
standard_logging_payload.get("error_information")
)
if error_information:
error_info = DDLLMObsError(
@@ -621,9 +621,9 @@ class DataDogLLMObsLogger(CustomBatchLogger):
latency_metrics["litellm_overhead_time_ms"] = litellm_overhead_ms
# Guardrail overhead latency
guardrail_info: Optional[
list[StandardLoggingGuardrailInformation]
] = standard_logging_payload.get("guardrail_information")
guardrail_info: Optional[list[StandardLoggingGuardrailInformation]] = (
standard_logging_payload.get("guardrail_information")
)
if guardrail_info is not None:
total_duration = 0.0
for info in guardrail_info:
@@ -793,15 +793,15 @@ class DataDogLLMObsLogger(CustomBatchLogger):
if function_arguments:
# Store arguments as JSON string for Datadog
if isinstance(function_arguments, str):
kv_pairs[
f"tool_calls.{idx}.function.arguments"
] = function_arguments
kv_pairs[f"tool_calls.{idx}.function.arguments"] = (
function_arguments
)
else:
import json
kv_pairs[
f"tool_calls.{idx}.function.arguments"
] = json.dumps(function_arguments)
kv_pairs[f"tool_calls.{idx}.function.arguments"] = (
json.dumps(function_arguments)
)
except (KeyError, TypeError, ValueError) as e:
verbose_logger.debug(
f"DataDogLLMObs: Error processing tool call {idx}: {str(e)}"
litellm/integrations/gcs_bucket/gcs_bucket_base.py
+3 -3
View File
@@ -150,9 +150,9 @@ class GCSBucketBase(CustomBatchLogger):
if kwargs is None:
kwargs = {}
standard_callback_dynamic_params: Optional[
StandardCallbackDynamicParams
] = kwargs.get("standard_callback_dynamic_params", None)
standard_callback_dynamic_params: Optional[StandardCallbackDynamicParams] = (
kwargs.get("standard_callback_dynamic_params", None)
)
bucket_name: str
path_service_account: Optional[str]
litellm/integrations/humanloop.py
+5 -1
View File
@@ -162,7 +162,11 @@ class HumanloopLogger(CustomLogger):
prompt_version: Optional[int] = None,
ignore_prompt_manager_model: Optional[bool] = False,
ignore_prompt_manager_optional_params: Optional[bool] = False,
) -> Tuple[str, List[AllMessageValues], dict,]:
) -> Tuple[
str,
List[AllMessageValues],
dict,
]:
humanloop_api_key = dynamic_callback_params.get(
"humanloop_api_key"
) or get_secret_str("HUMANLOOP_API_KEY")

Some files were not shown because too many files have changed in this diff Show More