tiennm99/litellm
1
0
Fork 0
mirror of https://github.com/tiennm99/litellm.git synced 2026-08-07 02:25:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into litellm_fixParallel_tool_cal

Browse Source
This commit is contained in:
Sameer Kankute
2026-02-27 13:01:54 +05:30
committed by GitHub
parent d0445e1e33 3369cd9d5d
commit 33e6b9c67d
16 changed files with 3128 additions and 26 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+68
View File
@@ -4100,6 +4100,63 @@ jobs:
path: playwright-report
destination: playwright-report
prisma_schema_sync:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Install Neon CLI
command: |
npm i -g neonctl
- run:
name: Install curl and dockerize
command: |
sudo apt-get update
sudo apt-get install -y curl
sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
sudo rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Sync schema on base e2e database
command: |
BASE_DATABASE_URL=$(neon connection-string \
--project-id $NEON_PROJECT_ID \
--api-key $NEON_API_KEY \
--branch br-fancy-paper-ad1olsb3 \
--database-name yuneng-trial-db \
--role neondb_owner)
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=$BASE_DATABASE_URL \
-e LITELLM_MASTER_KEY="sk-1234" \
--name schema-sync \
-v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--use_prisma_db_push
- run:
name: Start outputting logs
command: docker logs -f schema-sync
background: true
- run:
name: Wait for proxy to be ready (schema sync complete)
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Stop schema sync container
command: docker stop schema-sync
test_nonroot_image:
machine:
image: ubuntu-2204:2023.10.1
@@ -4298,6 +4355,15 @@ workflows:
only:
- main
- /litellm_.*/
- prisma_schema_sync:
context: e2e_ui_tests
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- e2e_ui_testing:
name: e2e_ui_testing_chromium
browser: chromium
@@ -4305,6 +4371,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
@@ -4317,6 +4384,7 @@ workflows:
requires:
- ui_build
- build_docker_database_image
- prisma_schema_sync
filters:
branches:
only:
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz
Vendored
BIN
View File
Binary file not shown.
litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql
+3
View File
@@ -0,0 +1,3 @@
-- AlterTable
ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "agent_id" TEXT;
litellm-proxy-extras/litellm_proxy_extras/schema.prisma
+1
View File
@@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken {
config Json @default("{}")
user_id String?
team_id String?
agent_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
litellm-proxy-extras/pyproject.toml
+2 -2
View File
@@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm-proxy-extras"
version = "0.4.48"
version = "0.4.49"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
authors = ["BerriAI"]
readme = "README.md"
@@ -22,7 +22,7 @@ requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "0.4.48"
version = "0.4.49"
version_files = [
"pyproject.toml:version",
"../requirements.txt:litellm-proxy-extras==",
litellm/llms/bedrock/chat/converse_transformation.py
-8
View File
@@ -511,7 +511,6 @@ class AmazonConverseConfig(BaseConfig):
"response_format",
"requestMetadata",
"service_tier",
"parallel_tool_calls",
]
if (
@@ -914,13 +913,6 @@ class AmazonConverseConfig(BaseConfig):
)
if _tool_choice_value is not None:
optional_params["tool_choice"] = _tool_choice_value
if param == "parallel_tool_calls":
disable_parallel = not value
optional_params["_parallel_tool_use_config"] = {
"tool_choice": {
"disable_parallel_tool_use": disable_parallel
}
}
if param == "thinking":
optional_params["thinking"] = value
elif param == "reasoning_effort" and isinstance(value, str):
litellm/provider_endpoints_support_backup.json
+2748
View File
File diff suppressed because it is too large Load Diff
litellm/proxy/public_endpoints/public_endpoints.py
+122 -6
View File
@@ -1,6 +1,8 @@
import json
import os
from typing import List
import re
from importlib.resources import files
from typing import Any, Dict, List, Optional
import litellm
from fastapi import APIRouter, Depends, HTTPException
@@ -23,11 +25,107 @@ from litellm.types.proxy.public_endpoints.public_endpoints import (
AgentCreateInfo,
ProviderCreateInfo,
PublicModelHubInfo,
SupportedEndpointsResponse,
)
from litellm.types.utils import LlmProviders
router = APIRouter()
# ---------------------------------------------------------------------------
# /public/endpoints — helpers
# ---------------------------------------------------------------------------
_ENDPOINT_METADATA: Dict[str, Dict[str, str]] = {
"chat_completions": {"label": "Chat Completions", "endpoint": "/chat/completions"},
"messages": {"label": "Messages", "endpoint": "/messages"},
"responses": {"label": "Responses", "endpoint": "/responses"},
"embeddings": {"label": "Embeddings", "endpoint": "/embeddings"},
"image_generations": {"label": "Image Generations", "endpoint": "/images/generations"},
"audio_transcriptions": {"label": "Audio Transcriptions", "endpoint": "/audio/transcriptions"},
"audio_speech": {"label": "Audio Speech", "endpoint": "/audio/speech"},
"moderations": {"label": "Moderations", "endpoint": "/moderations"},
"batches": {"label": "Batches", "endpoint": "/batches"},
"rerank": {"label": "Rerank", "endpoint": "/rerank"},
"ocr": {"label": "OCR", "endpoint": "/ocr"},
"search": {"label": "Search", "endpoint": "/search"},
"skills": {"label": "Skills", "endpoint": "/skills"},
"interactions": {"label": "Interactions", "endpoint": "/interactions"},
"a2a": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"},
"container": {"label": "Containers", "endpoint": "/containers"},
"container_files": {"label": "Container Files", "endpoint": "/containers/{id}/files"},
"compact": {"label": "Compact", "endpoint": "/responses/compact"},
"files": {"label": "Files", "endpoint": "/files"},
"image_edits": {"label": "Image Edits", "endpoint": "/images/edits"},
"vector_stores_create": {"label": "Vector Stores (Create)", "endpoint": "/vector_stores"},
"vector_stores_search": {"label": "Vector Stores (Search)", "endpoint": "/vector_stores/{id}/search"},
"vector_store_files": {"label": "Vector Store Files", "endpoint": "/vector_stores/{id}/files"},
"video_generations": {"label": "Video Generations", "endpoint": "/videos/generations"},
"assistants": {"label": "Assistants", "endpoint": "/assistants"},
"fine_tuning": {"label": "Fine Tuning", "endpoint": "/fine_tuning/jobs"},
"text_completion": {"label": "Text Completion", "endpoint": "/completions"},
"realtime": {"label": "Realtime", "endpoint": "/realtime"},
"count_tokens": {"label": "Count Tokens", "endpoint": "/utils/token_counter"},
"image_variations": {"label": "Image Variations", "endpoint": "/images/variations"},
"generateContent": {"label": "Generate Content", "endpoint": "/generateContent"},
"bedrock_invoke": {"label": "Bedrock Invoke", "endpoint": "/bedrock/invoke"},
"bedrock_converse": {"label": "Bedrock Converse", "endpoint": "/bedrock/converse"},
"rag_ingest": {"label": "RAG Ingest", "endpoint": "/rag/ingest"},
"rag_query": {"label": "RAG Query", "endpoint": "/rag/query"},
}
_SLUG_SUFFIX_RE = re.compile(r"\s*\(`[^`]+`\)\s*$")
# Loaded once on first request; never invalidated (local file, no TTL needed).
_cached_endpoints: Optional[List[Dict[str, Any]]] = None
def _clean_display_name(raw: str) -> str:
return _SLUG_SUFFIX_RE.sub("", raw).strip()
def _build_endpoints(raw: Dict[str, Any]) -> List[Dict[str, Any]]:
"""Transform raw provider_endpoints_support_backup.json into the response shape."""
providers: Dict[str, Any] = raw.get("providers", {})
# Collect endpoint keys in insertion order (union across all providers).
seen: set = set()
all_keys: List[str] = []
for provider_data in providers.values():
for key in provider_data.get("endpoints", {}):
if key not in seen:
seen.add(key)
all_keys.append(key)
result: List[Dict[str, Any]] = []
for key in all_keys:
meta = _ENDPOINT_METADATA.get(key)
label = meta["label"] if meta else key.replace("_", " ").title()
path = meta["endpoint"] if meta else "/" + key.replace("_", "/")
supporting: List[Dict[str, str]] = [
{
"slug": slug,
"display_name": _clean_display_name(pd.get("display_name", slug)),
}
for slug, pd in providers.items()
if pd.get("endpoints", {}).get(key)
]
result.append({"key": key, "label": label, "endpoint": path, "providers": supporting})
return result
def _load_endpoints() -> List[Dict[str, Any]]:
raw = json.loads(
files("litellm")
.joinpath("provider_endpoints_support_backup.json")
.read_text(encoding="utf-8")
)
return _build_endpoints(raw)
# ---------------------------------------------------------------------------
@router.get(
"/public/model_hub",
@@ -225,6 +323,24 @@ async def get_litellm_blog_posts():
return BlogPostsResponse(posts=posts)
@router.get(
"/public/endpoints",
tags=["public"],
response_model=SupportedEndpointsResponse,
)
async def get_supported_endpoints() -> SupportedEndpointsResponse:
"""
Return the list of LiteLLM proxy endpoints and which providers support each one.
Reads from the bundled local backup file. Result is cached in-process for
the lifetime of the server process.
"""
global _cached_endpoints
if _cached_endpoints is None:
_cached_endpoints = SupportedEndpointsResponse(endpoints=_load_endpoints())
return _cached_endpoints
@router.get(
"/public/agents/fields",
tags=["public", "[beta] Agents"],
@@ -233,7 +349,7 @@ async def get_litellm_blog_posts():
async def get_agent_fields() -> List[AgentCreateInfo]:
"""
Return agent type metadata required by the dashboard create-agent flow.
If an agent has `inherit_credentials_from_provider`, the provider's credential
fields are automatically appended to the agent's credential_fields.
"""
@@ -242,19 +358,19 @@ async def get_agent_fields() -> List[AgentCreateInfo]:
"proxy",
"public_endpoints",
)
agent_create_fields_path = os.path.join(base_path, "agent_create_fields.json")
provider_create_fields_path = os.path.join(base_path, "provider_create_fields.json")
with open(agent_create_fields_path, "r") as f:
agent_create_fields = json.load(f)
with open(provider_create_fields_path, "r") as f:
provider_create_fields = json.load(f)
# Build a lookup map for providers by name
provider_map = {p["provider"]: p for p in provider_create_fields}
# Merge inherited credential fields
for agent in agent_create_fields:
inherit_from = agent.get("inherit_credentials_from_provider")
litellm/proxy/schema.prisma
+1
View File
@@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken {
config Json @default("{}")
user_id String?
team_id String?
agent_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
litellm/types/proxy/public_endpoints/public_endpoints.py
+16
View File
@@ -52,3 +52,19 @@ class AgentCreateInfo(BaseModel):
credential_fields: List[AgentCredentialField]
litellm_params_template: Optional[Dict[str, str]] = None
model_template: Optional[str] = None
class EndpointProvider(BaseModel):
slug: str
display_name: str
class SupportedEndpoint(BaseModel):
key: str
label: str
endpoint: str
providers: List[EndpointProvider]
class SupportedEndpointsResponse(BaseModel):
endpoints: List[SupportedEndpoint]
pyproject.toml
+1 -1
View File
@@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true }
redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"}
mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"}
a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"}
litellm-proxy-extras = {version = "0.4.48", optional = true}
litellm-proxy-extras = {version = "0.4.49", optional = true}
rich = {version = "13.7.1", optional = true}
litellm-enterprise = {version = "0.1.32", optional = true}
diskcache = {version = "^5.6.1", optional = true}
requirements.txt
+1 -1
View File
@@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14"
sentry_sdk==2.21.0 # for sentry error handling
detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests
tzdata==2025.1 # IANA time zone database
litellm-proxy-extras==0.4.48 # for proxy extras - e.g. prisma migrations
litellm-proxy-extras==0.4.49 # for proxy extras - e.g. prisma migrations
llm-sandbox==0.3.31 # for skill execution in sandbox
### LITELLM PACKAGE DEPENDENCIES
python-dotenv==1.0.1 # for env
schema.prisma
+1
View File
@@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken {
config Json @default("{}")
user_id String?
team_id String?
agent_id String?
project_id String?
permissions Json @default("{}")
max_parallel_requests Int?
tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py
+3 -8
View File
@@ -2616,11 +2616,11 @@ def test_empty_assistant_message_handling():
empty or whitespace-only content with a placeholder to prevent AWS Bedrock
Converse API 400 Bad Request errors.
"""
# Import the litellm module that factory.py uses to ensure we patch the correct reference
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
from litellm.litellm_core_utils.prompt_templates.factory import (
_bedrock_converse_messages_pt,
)
# Import the litellm module that factory.py uses to ensure we patch the correct reference
import litellm.litellm_core_utils.prompt_templates.factory as factory_module
# Test case 1: Empty string content - test with modify_params=True to prevent merging
messages = [
@@ -3135,12 +3135,7 @@ def test_native_structured_output_no_fake_stream():
def test_transform_request_with_output_config():
"""Test that outputConfig flows through _transform_request_helper into the final request."""
from litellm.types.llms.bedrock import (
JsonSchemaDefinition,
OutputConfigBlock,
OutputFormat,
OutputFormatStructure,
)
from litellm.types.llms.bedrock import OutputConfigBlock, OutputFormat, OutputFormatStructure, JsonSchemaDefinition
config = AmazonConverseConfig()
tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py
+161
View File
@@ -357,3 +357,164 @@ def test_public_model_hub_mixed_health_statuses():
assert claude["health_checked_at"] is None
app.dependency_overrides.clear()
# ---------------------------------------------------------------------------
# /public/endpoints
# ---------------------------------------------------------------------------
import litellm.proxy.public_endpoints.public_endpoints as _pe_module
from litellm.proxy.public_endpoints.public_endpoints import _build_endpoints, _clean_display_name
@pytest.fixture(autouse=False)
def reset_endpoints_cache():
"""Reset the module-level cache before and after each cache-related test."""
original = _pe_module._cached_endpoints
_pe_module._cached_endpoints = None
yield
_pe_module._cached_endpoints = original
def _make_client():
app = FastAPI()
app.include_router(router)
return TestClient(app)
def test_get_supported_endpoints_returns_200(reset_endpoints_cache):
response = _make_client().get("/public/endpoints")
assert response.status_code == 200
def test_get_supported_endpoints_response_shape(reset_endpoints_cache):
data = _make_client().get("/public/endpoints").json()
assert "endpoints" in data
assert isinstance(data["endpoints"], list)
assert len(data["endpoints"]) > 0
def test_get_supported_endpoints_item_fields(reset_endpoints_cache):
endpoints = _make_client().get("/public/endpoints").json()["endpoints"]
for item in endpoints:
assert "key" in item
assert "label" in item
assert "endpoint" in item
assert "providers" in item
assert isinstance(item["providers"], list)
def test_get_supported_endpoints_provider_fields(reset_endpoints_cache):
endpoints = _make_client().get("/public/endpoints").json()["endpoints"]
for item in endpoints:
for provider in item["providers"]:
assert "slug" in provider
assert "display_name" in provider
def test_get_supported_endpoints_paths_start_with_slash(reset_endpoints_cache):
endpoints = _make_client().get("/public/endpoints").json()["endpoints"]
for item in endpoints:
assert item["endpoint"].startswith("/"), f"Expected path starting with /, got: {item['endpoint']}"
def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache):
endpoints = _make_client().get("/public/endpoints").json()["endpoints"]
keys = [item["key"] for item in endpoints]
assert "chat_completions" in keys
chat = next(item for item in endpoints if item["key"] == "chat_completions")
assert chat["endpoint"] == "/chat/completions"
assert chat["label"] == "Chat Completions"
assert len(chat["providers"]) > 0
def test_get_supported_endpoints_display_names_have_no_slug_suffix(reset_endpoints_cache):
"""Provider display_names must not contain the raw `` (`slug`) `` suffix."""
import re
suffix_re = re.compile(r"\(`[^`]+`\)")
endpoints = _make_client().get("/public/endpoints").json()["endpoints"]
for item in endpoints:
for provider in item["providers"]:
assert not suffix_re.search(provider["display_name"]), (
f"display_name still contains slug suffix: {provider['display_name']!r}"
)
def test_get_supported_endpoints_is_cached(reset_endpoints_cache):
"""`_load_endpoints` is called only once; subsequent requests use the cache."""
client = _make_client()
with patch(
"litellm.proxy.public_endpoints.public_endpoints._load_endpoints",
wraps=_pe_module._load_endpoints,
) as mock_load:
client.get("/public/endpoints")
client.get("/public/endpoints")
client.get("/public/endpoints")
mock_load.assert_called_once()
# ---------------------------------------------------------------------------
# _build_endpoints unit tests (transformation logic)
# ---------------------------------------------------------------------------
_MINIMAL_RAW = {
"providers": {
"openai": {
"display_name": "OpenAI (`openai`)",
"url": "https://example.com",
"endpoints": {"chat_completions": True, "embeddings": True, "images": False},
},
"anthropic": {
"display_name": "Anthropic (`anthropic`)",
"url": "https://example.com",
"endpoints": {"chat_completions": True, "embeddings": False, "images": False},
},
}
}
def test_build_endpoints_known_key_uses_metadata():
result = _build_endpoints(_MINIMAL_RAW)
chat = next(e for e in result if e["key"] == "chat_completions")
assert chat["label"] == "Chat Completions"
assert chat["endpoint"] == "/chat/completions"
def test_build_endpoints_only_includes_supporting_providers():
result = _build_endpoints(_MINIMAL_RAW)
embeddings = next(e for e in result if e["key"] == "embeddings")
slugs = [p["slug"] for p in embeddings["providers"]]
assert slugs == ["openai"]
def test_build_endpoints_unknown_key_derives_label_and_path():
raw = {
"providers": {
"someprovider": {
"display_name": "Some Provider (`someprovider`)",
"endpoints": {"my_custom_endpoint": True},
}
}
}
result = _build_endpoints(raw)
item = result[0]
assert item["key"] == "my_custom_endpoint"
assert item["label"] == "My Custom Endpoint"
assert item["endpoint"].startswith("/")
def test_build_endpoints_empty_providers_returns_empty():
result = _build_endpoints({"providers": {}})
assert result == []
def test_clean_display_name_strips_suffix():
assert _clean_display_name("OpenAI (`openai`)") == "OpenAI"
assert _clean_display_name("AI/ML API (`aiml`)") == "AI/ML API"
assert _clean_display_name("A2A (Agent-to-Agent) (`a2a`)") == "A2A (Agent-to-Agent)"
def test_clean_display_name_passthrough_when_no_suffix():
assert _clean_display_name("OpenAI") == "OpenAI"
assert _clean_display_name("") == ""