From 3e58fe42b7a4e9ddf334525e18677714d68d3af2 Mon Sep 17 00:00:00 2001 From: Chesars Date: Wed, 19 Nov 2025 17:07:46 -0300 Subject: [PATCH 01/23] fix: Support response_format parameter in completion -> responses bridge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes #16810 ## Problem When using completion() with models that have mode: "responses" (like o3-pro, gpt-5-codex), the response_format parameter with JSON schemas was being ignored or incorrectly handled, causing: - Large schemas (>512 chars) to fail with "metadata.schema_dict_json: string too long" error - Structured outputs to be silently dropped - Users' code to break unexpectedly ## Root Cause The completion -> responses bridge in litellm/completion_extras/litellm_responses_transformation/transformation.py was missing the conversion of response_format (Chat Completion format) to text.format (Responses API format). The inverse bridge (responses -> completion) already had this conversion implemented in commit 29f0ed223a, but the completion -> responses direction was incomplete. ## Solution Added _transform_response_format_to_text_format() method that converts: - response_format with json_schema → text.format with json_schema - response_format with json_object → text.format with json_object - response_format with text → text.format with text Updated transform_request() to detect and convert response_format parameter before sending to litellm.responses(). ## Changes - Added _transform_response_format_to_text_format() method (lines 592-647) - Modified transform_request() to handle response_format (lines 199-203) - Added comprehensive tests to validate the conversion ## Testing - 5 new unit tests covering all conversion scenarios - Real API test with OpenAI confirming large schemas (>512 chars) work - No more metadata.schema_dict_json errors ## Impact Users can now use completion() with models that have mode: "responses" and: - Use large JSON schemas without hitting metadata 512 char limit - Get proper structured outputs - Have their existing code continue working --- .../transformation.py | 62 ++++++++ ...responses_transformation_transformation.py | 136 ++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index 54d32fe346..a6dbd5d0dc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -196,6 +196,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): cast(List[Dict[str, Any]], value) ) ) + elif key == "response_format": + # Convert response_format to text.format + text_format = self._transform_response_format_to_text_format(value) + if text_format: + responses_api_request["text"] = text_format # type: ignore elif key in ResponsesAPIOptionalRequestParams.__annotations__.keys(): responses_api_request[key] = value # type: ignore elif key in ("metadata"): @@ -589,6 +594,63 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): return Reasoning(effort="minimal") return None + def _transform_response_format_to_text_format( + self, response_format: Union[Dict[str, Any], Any] + ) -> Optional[Dict[str, Any]]: + """ + Transform Chat Completion response_format parameter to Responses API text.format parameter. + + Chat Completion response_format structure: + { + "type": "json_schema", + "json_schema": { + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + + Responses API text parameter structure: + { + "format": { + "type": "json_schema", + "name": "schema_name", + "schema": {...}, + "strict": True + } + } + """ + if not response_format: + return None + + if isinstance(response_format, dict): + format_type = response_format.get("type") + + if format_type == "json_schema": + json_schema = response_format.get("json_schema", {}) + return { + "format": { + "type": "json_schema", + "name": json_schema.get("name", "response_schema"), + "schema": json_schema.get("schema", {}), + "strict": json_schema.get("strict", False), + } + } + elif format_type == "json_object": + return { + "format": { + "type": "json_object" + } + } + elif format_type == "text": + return { + "format": { + "type": "text" + } + } + + return None + def _map_responses_status_to_finish_reason(self, status: Optional[str]) -> str: """Map responses API status to chat completion finish_reason""" if not status: diff --git a/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py new file mode 100644 index 0000000000..adbaf21907 --- /dev/null +++ b/tests/test_litellm/completion_extras/test_litellm_responses_transformation_transformation.py @@ -0,0 +1,136 @@ +""" +Test for response_format to text.format conversion in completion -> responses bridge +""" +import pytest +from litellm.completion_extras.litellm_responses_transformation.transformation import ( + LiteLLMResponsesTransformationHandler, +) + + +def test_transform_response_format_to_text_format_json_schema(): + """Test conversion of response_format with json_schema to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + # Chat Completion format + response_format = { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + + # Convert to Responses API format + result = handler._transform_response_format_to_text_format(response_format) + + # Verify conversion + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_schema" + assert result["format"]["name"] == "person_schema" + assert result["format"]["strict"] is True + assert "schema" in result["format"] + assert result["format"]["schema"]["type"] == "object" + assert "properties" in result["format"]["schema"] + + +def test_transform_response_format_to_text_format_json_object(): + """Test conversion of response_format with json_object to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "json_object" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "json_object" + + +def test_transform_response_format_to_text_format_text(): + """Test conversion of response_format with text to text.format""" + handler = LiteLLMResponsesTransformationHandler() + + response_format = { + "type": "text" + } + + result = handler._transform_response_format_to_text_format(response_format) + + assert result is not None + assert "format" in result + assert result["format"]["type"] == "text" + + +def test_transform_response_format_to_text_format_none(): + """Test that None input returns None""" + handler = LiteLLMResponsesTransformationHandler() + + result = handler._transform_response_format_to_text_format(None) + + assert result is None + + +def test_transform_request_with_response_format(): + """Test that transform_request correctly handles response_format parameter""" + handler = LiteLLMResponsesTransformationHandler() + + messages = [ + {"role": "user", "content": "Extract person info: John Doe, 30 years old"} + ] + + optional_params = { + "response_format": { + "type": "json_schema", + "json_schema": { + "name": "person_schema", + "schema": { + "type": "object", + "properties": { + "name": {"type": "string"}, + "age": {"type": "integer"} + }, + "required": ["name", "age"], + "additionalProperties": False + }, + "strict": True + } + } + } + + litellm_params = {} + headers = {} + + # Mock logging object + class MockLoggingObj: + pass + + litellm_logging_obj = MockLoggingObj() + + result = handler.transform_request( + model="o3-pro", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + litellm_logging_obj=litellm_logging_obj, + ) + + # Verify that text parameter was set with converted format + assert "text" in result + assert result["text"] is not None + assert "format" in result["text"] + assert result["text"]["format"]["type"] == "json_schema" + assert result["text"]["format"]["name"] == "person_schema" + assert "schema" in result["text"]["format"] From 67622fb0404877bf07865bc1be60932adc315376 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 26 Nov 2025 00:58:47 +0530 Subject: [PATCH 02/23] Add day 0 support for anthropic new feat (#17091) * Added tool search support for anthropic * Add programtic tool calling support * Add tool use input examples support * Add anthropic effort param support * Add anthropic effort param support * Add blog for new features * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * fix mypy and lint errors * Add better handling * Add better handling --- .../blog/anthropic_advanced_features/index.md | 655 ++++++++++++++++ .../docs/providers/anthropic_effort.md | 276 +++++++ .../anthropic_programmatic_tool_calling.md | 430 ++++++++++ .../anthropic_tool_input_examples.md | 438 +++++++++++ .../docs/providers/anthropic_tool_search.md | 397 ++++++++++ litellm/llms/anthropic/chat/handler.py | 81 +- litellm/llms/anthropic/chat/transformation.py | 266 ++++++- litellm/llms/anthropic/common_utils.py | 101 +++ .../adapters/transformation.py | 2 +- litellm/types/llms/anthropic.py | 74 ++ litellm/types/llms/openai.py | 13 +- litellm/types/utils.py | 3 +- .../test_anthropic_chat_transformation.py | 741 ++++++++++++++++++ 13 files changed, 3420 insertions(+), 57 deletions(-) create mode 100644 docs/my-website/blog/anthropic_advanced_features/index.md create mode 100644 docs/my-website/docs/providers/anthropic_effort.md create mode 100644 docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md create mode 100644 docs/my-website/docs/providers/anthropic_tool_input_examples.md create mode 100644 docs/my-website/docs/providers/anthropic_tool_search.md diff --git a/docs/my-website/blog/anthropic_advanced_features/index.md b/docs/my-website/blog/anthropic_advanced_features/index.md new file mode 100644 index 0000000000..71e5d9b192 --- /dev/null +++ b/docs/my-website/blog/anthropic_advanced_features/index.md @@ -0,0 +1,655 @@ +--- +slug: anthropic_advanced_features +title: "Advanced Anthropic Features in LiteLLM: Tool Search, Programmatic Tool Calling, Input Examples, and Effort Control" +date: 2025-01-25T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://media.licdn.com/dms/image/v2/D4D03AQHB_loQYd5gjg/profile-displayphoto-shrink_800_800/profile-displayphoto-shrink_800_800/0/1719137160975?e=1765411200&v=beta&t=c8396f--_lH6Fb_pVvx_jGholPfcl0bvwmNynbNdnII + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] +hide_table_of_contents: false +--- + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +:::info + +This guide covers Anthropic's latest advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. + +::: + +We're excited to announce support for Anthropic's latest advanced features in LiteLLM! These powerful capabilities enable you to build more efficient, scalable, and cost-effective AI applications with Claude. + +## Table of Contents + +1. [Tool Search](#tool-search) +2. [Programmatic Tool Calling](#programmatic-tool-calling) +3. [Tool Input Examples](#tool-input-examples) +4. [Effort Parameter: Control Token Usage](#effort-parameter) +5. [Cost Tracking: Monitor Tool Search Usage](#cost-tracking) +6. [Combining Features](#combining-features) + +--- + +## Tool Search {#tool-search} + +### Usage Example + +```python +import litellm +import os + +# Configure your API key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +# Define your tools with defer_loading +tools = [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } +] + +# Make a request - Claude will search for and use relevant tools +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message.content) +print("Tool calls:", response.choices[0].message.tool_calls) + +# Check tool search usage +if hasattr(response.usage, 'server_tool_use'): + print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}") +``` + +### BM25 Variant (Natural Language Search) + +For natural language queries instead of regex patterns: + +```python +tools = [ + { + "type": "tool_search_tool_bm25_20251119", # Natural language variant + "name": "tool_search_tool_bm25" + }, + # ... your deferred tools +] +``` + +--- + +## Programmatic Tool Calling {#programmatic-tool-calling} + +### Usage Example + +```python +import litellm +import json + +# Define tools that can be called programmatically +tools = [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } +] + +# First request +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + tools=tools +) + +print("Claude's response:", response.choices[0].message) + +# Handle tool calls +messages = [ + {"role": "user", "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue"}, + {"role": "assistant", "content": response.choices[0].message.content, "tool_calls": response.choices[0].message.tool_calls} +] + +# Process each tool call +for tool_call in response.choices[0].message.tool_calls: + # Check if it's a programmatic call + if hasattr(tool_call, 'caller') and tool_call.caller: + print(f"Programmatic call to {tool_call.function.name}") + print(f"Called from: {tool_call.caller}") + + # Simulate tool execution + if tool_call.function.name == "query_database": + args = json.loads(tool_call.function.arguments) + # Simulate database query + result = json.dumps([ + {"region": "West", "revenue": 150000}, + {"region": "East", "revenue": 180000}, + {"region": "Central", "revenue": 120000} + ]) + + messages.append({ + "role": "user", + "content": [{ + "type": "tool_result", + "tool_use_id": tool_call.id, + "content": result + }] + }) + +# Get final response +final_response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=tools +) + +print("\nFinal answer:", final_response.choices[0].message.content) +``` + +--- + +## Tool Input Examples {#tool-input-examples} + +### Usage Example + +```python +import litellm + +tools = [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + tools=tools +) + +print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments) +``` + +--- + +## Effort Parameter: Control Token Usage {#effort-parameter} + +### Usage Example + +```python +import litellm + +message = "Analyze the trade-offs between microservices and monolithic architectures" + +# High effort (default) - Maximum capability +response_high = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "high"} +) + +print("High effort response:") +print(response_high.choices[0].message.content) +print(f"Tokens used: {response_high.usage.completion_tokens}\n") + +# Medium effort - Balanced approach +response_medium = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "medium"} +) + +print("Medium effort response:") +print(response_medium.choices[0].message.content) +print(f"Tokens used: {response_medium.usage.completion_tokens}\n") + +# Low effort - Maximum efficiency +response_low = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": message}], + output_config={"effort": "low"} +) + +print("Low effort response:") +print(response_low.choices[0].message.content) +print(f"Tokens used: {response_low.usage.completion_tokens}\n") + +# Compare token usage +print("Token Comparison:") +print(f"High: {response_high.usage.completion_tokens} tokens") +print(f"Medium: {response_medium.usage.completion_tokens} tokens") +print(f"Low: {response_low.usage.completion_tokens} tokens") +``` + +### Effort with Tool Use + +Lower effort affects both explanations and tool calls: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + } +] + +# Low effort = fewer tool calls, more direct +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Check weather in San Francisco, New York, and London" + }], + tools=tools, + output_config={"effort": "low"} # May combine into fewer calls +) +``` + +--- + +## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} + +### Understanding Tool Search Costs + +Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. + +### Tracking Example + +```python +import litellm + +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools +] + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + tools=tools +) + +# Standard token usage +print("Token Usage:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +# Tool search specific usage +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f"\nTool Search Usage:") + print(f" Search requests: {response.usage.server_tool_use.tool_search_requests}") + + # Calculate cost (example pricing) + input_cost = response.usage.prompt_tokens * 0.000003 # $3 per 1M tokens + output_cost = response.usage.completion_tokens * 0.000015 # $15 per 1M tokens + search_cost = response.usage.server_tool_use.tool_search_requests * 0.0001 # Example + + total_cost = input_cost + output_cost + search_cost + + print(f"\nCost Breakdown:") + print(f" Input tokens: ${input_cost:.6f}") + print(f" Output tokens: ${output_cost:.6f}") + print(f" Tool searches: ${search_cost:.6f}") + print(f" Total: ${total_cost:.6f}") +``` + +### Cost Optimization Tips + +1. **Keep frequently used tools non-deferred** (3-5 tools) +2. **Use tool search for large catalogs** (10+ tools) +3. **Monitor search requests** to identify optimization opportunities +4. **Combine with effort parameter** for maximum efficiency + +```python +# Optimized for cost +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Simple query"}], + tools=tools_with_search, + output_config={"effort": "low"} # Reduce output tokens +) +``` + +--- + +## Combining Features {#combining-features} + +### The Power of Integration + +These features work together seamlessly. Here's a real-world example combining all of them: + +```python +import litellm +import json + +# Large tool catalog with search, programmatic calling, and examples +tools = [ + # Enable tool search + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Enable programmatic calling + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Database tool with all features + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the analytics database. Returns JSON array of results.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL SELECT statement" + }, + "limit": { + "type": "integer", + "description": "Maximum rows to return" + } + }, + "required": ["sql"] + } + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + { + "sql": "SELECT region, SUM(revenue) as total FROM sales GROUP BY region", + "limit": 100 + } + ] + }, + # ... 50 more tools with defer_loading +] + +# Make request with effort control +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + tools=tools, + output_config={"effort": "medium"} # Balanced efficiency +) + +# Track comprehensive usage +print("Complete Usage Metrics:") +print(f" Input tokens: {response.usage.prompt_tokens}") +print(f" Output tokens: {response.usage.completion_tokens}") +print(f" Total tokens: {response.usage.total_tokens}") + +if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use: + print(f" Tool searches: {response.usage.server_tool_use.tool_search_requests}") + +print(f"\nResponse: {response.choices[0].message.content}") +``` + +### Real-World Benefits + +This combination enables: + +1. **Massive scale** - Handle 1000+ tools efficiently +2. **Low latency** - Programmatic calling reduces round trips +3. **High accuracy** - Input examples ensure correct tool usage +4. **Cost control** - Effort parameter optimizes token spend +5. **Full visibility** - Track all usage metrics + +--- + +## Getting Started + +### Installation + +```bash +pip install litellm --upgrade +``` + +### Configuration + +```python +import os +import litellm + +# Set your API key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +# LiteLLM automatically handles beta headers for all features +``` + +### Supported Models + +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + +### Supported Endpoints + +**Note**: All features are supported on the `/chat/completions` endpoint only. + +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + +### Provider Support + +All features work across: +- ✅ Standard Anthropic API +- ✅ Azure Anthropic +- ✅ Vertex AI Anthropic +- ✅ LiteLLM Proxy + +--- + +## Conclusion + +These advanced Anthropic features in LiteLLM enable you to build more sophisticated, efficient, and cost-effective AI applications: + +- **Tool Search** scales to thousands of tools +- **Programmatic Tool Calling** reduces latency and tokens +- **Input Examples** improve accuracy +- **Effort Parameter** controls costs + +All features work seamlessly together and are supported across all Anthropic providers through LiteLLM's unified interface. + +### Resources + +- [LiteLLM Documentation](https://docs.litellm.ai/) +- [Anthropic Tool Search Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_search) +- [Anthropic Programmatic Tool Calling Docs](https://docs.litellm.ai/docs/providers/anthropic_programmatic_tool_calling) +- [Anthropic Input Examples Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_input_examples) +- [Anthropic Effort Parameter Docs](https://docs.litellm.ai/docs/providers/anthropic_effort) + +### Get Started Today + +```bash +pip install litellm --upgrade +``` + +Happy building! 🚀 + diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md new file mode 100644 index 0000000000..d1116ad5be --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -0,0 +1,276 @@ +# Anthropic Effort Parameter + +Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency. + +## Overview + +The `effort` parameter allows you to control how eager Claude is about spending tokens when responding to requests. This gives you the ability to trade off between response thoroughness and token efficiency, all with a single model. + +**Note**: The effort parameter is currently in beta and only supported by Claude Opus 4.5. You must include the beta header `effort-2025-11-24` when using this feature (LiteLLM automatically adds this header when `output_config` with `effort` is detected). + +## How Effort Works + +By default, Claude uses maximum effort—spending as many tokens as needed for the best possible outcome. By lowering the effort level, you can instruct Claude to be more conservative with token usage, optimizing for speed and cost while accepting some reduction in capability. + +**Tip**: Setting `effort` to `"high"` produces exactly the same behavior as omitting the `effort` parameter entirely. + +The effort parameter affects **all tokens** in the response, including: +- Text responses and explanations +- Tool calls and function arguments +- Extended thinking (when enabled) + +This approach has two major advantages: +1. It doesn't require thinking to be enabled in order to use it. +2. It can affect all token spend including tool calls. For example, lower effort would mean Claude makes fewer tool calls. + +This gives a much greater degree of control over efficiency. + +## Effort Levels + +| Level | Description | Typical use case | +|-------|-------------|------------------| +| `high` | Maximum capability—Claude uses as many tokens as needed for the best possible outcome. Equivalent to not setting the parameter. | Complex reasoning, difficult coding problems, agentic tasks | +| `medium` | Balanced approach with moderate token savings. | Agentic tasks that require a balance of speed, cost, and performance | +| `low` | Most efficient—significant token savings with some capability reduction. | Simpler tasks that need the best speed and lowest costs, such as subagents | + +## Quick Start + +### Using LiteLLM SDK + + + + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config={ + "effort": "medium" + } +) + +print(response.choices[0].message.content) +``` + + + + +```typescript +import Anthropic from "@anthropic-ai/sdk"; + +const client = new Anthropic({ + apiKey: process.env.ANTHROPIC_API_KEY, +}); + +const response = await client.messages.create({ + model: "claude-opus-4-5-20251101", + max_tokens: 4096, + messages: [{ + role: "user", + content: "Analyze the trade-offs between microservices and monolithic architectures" + }], + output_config: { + effort: "medium" + } +}); + +console.log(response.content[0].text); +``` + + + + +### Using LiteLLM Proxy + +```bash +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "anthropic/claude-opus-4-5-20251101", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +### Direct Anthropic API Call + +```bash +curl https://api.anthropic.com/v1/messages \ + --header "x-api-key: $ANTHROPIC_API_KEY" \ + --header "anthropic-version: 2023-06-01" \ + --header "anthropic-beta: effort-2025-11-24" \ + --header "content-type: application/json" \ + --data '{ + "model": "claude-opus-4-5-20251101", + "max_tokens": 4096, + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "medium" + } + }' +``` + +## Model Compatibility + +The effort parameter is currently only supported by: +- **Claude Opus 4.5** (`claude-opus-4-5-20251101`) + +## When Should I Adjust the Effort Parameter? + +- Use **high effort** (the default) when you need Claude's best work—complex reasoning, nuanced analysis, difficult coding problems, or any task where quality is the top priority. + +- Use **medium effort** as a balanced option when you want solid performance without the full token expenditure of high effort. + +- Use **low effort** when you're optimizing for speed (because Claude answers with fewer tokens) or cost—for example, simple classification tasks, quick lookups, or high-volume use cases where marginal quality improvements don't justify additional latency or spend. + +## Effort with Tool Use + +When using tools, the effort parameter affects both the explanations around tool calls and the tool calls themselves. Lower effort levels tend to: +- Combine multiple operations into fewer tool calls +- Make fewer tool calls +- Proceed directly to action + +Example with tools: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Check the weather in multiple cities" + }], + tools=[{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + } + }], + output_config={ + "effort": "low" # Will make fewer tool calls + } +) +``` + +## Effort with Extended Thinking + +The effort parameter works seamlessly with extended thinking. When both are enabled, effort controls the token budget across all response types: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{ + "role": "user", + "content": "Solve this complex problem" + }], + thinking={ + "type": "enabled", + "budget_tokens": 5000 + }, + output_config={ + "effort": "medium" # Affects both thinking and response tokens + } +) +``` + +## Best Practices + +1. **Start with the default (high)** for new tasks, then experiment with lower effort levels if you're looking to optimize costs. + +2. **Use medium effort for production agentic workflows** where you need a balance of quality and efficiency. + +3. **Reserve low effort for high-volume, simple tasks** like classification, routing, or data extraction where speed matters more than nuanced responses. + +4. **Monitor token usage** to understand the actual savings from different effort levels for your specific use cases. + +5. **Test with your specific prompts** as the impact of effort levels can vary based on task complexity. + +## Provider Support + +The effort parameter is supported across all Anthropic-compatible providers: + +- **Standard Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Azure Anthropic**: ✅ Supported (Claude Opus 4.5) +- **Vertex AI Anthropic**: ✅ Supported (Claude Opus 4.5) + +LiteLLM automatically handles the beta header injection for all providers. + +## Usage and Pricing + +Token usage with different effort levels is tracked in the standard usage object. Lower effort levels result in fewer output tokens, which directly reduces costs: + +```python +response = litellm.completion( + model="anthropic/claude-opus-4-5-20251101", + messages=[{"role": "user", "content": "Analyze this"}], + output_config={"effort": "low"} +) + +print(f"Output tokens: {response.usage.completion_tokens}") +print(f"Total tokens: {response.usage.total_tokens}") +``` + +## Troubleshooting + +### Beta header not being added + +LiteLLM automatically adds the `effort-2025-11-24` beta header when `output_config` with `effort` is detected. If you're not seeing the header: + +1. Ensure you're using `output_config` with an `effort` field +2. Verify the model is Claude Opus 4.5 +3. Check that LiteLLM version supports this feature + +### Invalid effort value error + +Only three values are accepted: `"high"`, `"medium"`, `"low"`. Any other value will raise a validation error: + +```python +# ❌ This will raise an error +output_config={"effort": "very_low"} + +# ✅ Use one of the valid values +output_config={"effort": "low"} +``` + +### Model not supported + +Currently, only Claude Opus 4.5 supports the effort parameter. Using it with other models may result in the parameter being ignored or an error. + +## Related Features + +- [Extended Thinking](/docs/providers/anthropic_extended_thinking) - Control Claude's reasoning process +- [Tool Use](/docs/providers/anthropic_tools) - Enable Claude to use tools and functions +- [Programmatic Tool Calling](/docs/providers/anthropic_programmatic_tool_calling) - Let Claude write code that calls tools +- [Prompt Caching](/docs/providers/anthropic_prompt_caching) - Cache prompts to reduce costs + +## Additional Resources + +- [Anthropic Effort Documentation](https://docs.anthropic.com/en/docs/build-with-claude/effort) +- [LiteLLM Anthropic Provider Guide](/docs/providers/anthropic) +- [Cost Optimization Best Practices](/docs/guides/cost_optimization) + diff --git a/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md new file mode 100644 index 0000000000..6d3e15785e --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_programmatic_tool_calling.md @@ -0,0 +1,430 @@ +# Anthropic Programmatic Tool Calling + +Programmatic tool calling allows Claude to write code that calls your tools programmatically within a code execution container, rather than requiring round trips through the model for each tool invocation. This reduces latency for multi-tool workflows and decreases token consumption by allowing Claude to filter or process data before it reaches the model's context window. + +:::info +Programmatic tool calling is currently in public beta. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `allowed_callers` field. + +This feature requires the code execution tool to be enabled. +::: + +## Model Compatibility + +Programmatic tool calling is available on the following models: + +| Model | Tool Version | +|-------|--------------| +| Claude Opus 4.5 (`claude-opus-4-5-20251101`) | `code_execution_20250825` | +| Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) | `code_execution_20250825` | + +## Quick Start + +Here's a simple example where Claude programmatically queries a database multiple times and aggregates results: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + { + "role": "user", + "content": "Query sales data for the West, East, and Central regions, then tell me which region had the highest revenue" + } + ], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) + +print(response) +``` + +## How It Works + +When you configure a tool to be callable from code execution and Claude decides to use that tool: + +1. Claude writes Python code that invokes the tool as a function, potentially including multiple tool calls and pre/post-processing logic +2. Claude runs this code in a sandboxed container via code execution +3. When a tool function is called, code execution pauses and the API returns a `tool_use` block with a `caller` field +4. You provide the tool result, and code execution continues (intermediate results are not loaded into Claude's context window) +5. Once all code execution completes, Claude receives the final output and continues working on the task + +This approach is particularly useful for: + +- **Large data processing**: Filter or aggregate tool results before they reach Claude's context +- **Multi-step workflows**: Save tokens and latency by calling tools serially or in a loop without sampling Claude in-between tool calls +- **Conditional logic**: Make decisions based on intermediate tool results + +## The `allowed_callers` Field + +The `allowed_callers` field specifies which contexts can invoke a tool: + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the database", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"] +} +``` + +**Possible values:** + +- `["direct"]` - Only Claude can call this tool directly (default if omitted) +- `["code_execution_20250825"]` - Only callable from within code execution +- `["direct", "code_execution_20250825"]` - Callable both directly and from code execution + +:::tip +We recommend choosing either `["direct"]` or `["code_execution_20250825"]` for each tool rather than enabling both, as this provides clearer guidance to Claude for how best to use the tool. +::: + +## The `caller` Field in Responses + +Every tool use block includes a `caller` field indicating how it was invoked: + +**Direct invocation (traditional tool use):** + +```python +{ + "type": "tool_use", + "id": "toolu_abc123", + "name": "query_database", + "input": {"sql": ""}, + "caller": {"type": "direct"} +} +``` + +**Programmatic invocation:** + +```python +{ + "type": "tool_use", + "id": "toolu_xyz789", + "name": "query_database", + "input": {"sql": ""}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } +} +``` + +The `tool_id` references the code execution tool that made the programmatic call. + +## Container Lifecycle + +Programmatic tool calling uses code execution containers: + +- **Container creation**: A new container is created for each session unless you reuse an existing one +- **Expiration**: Containers expire after approximately 4.5 minutes of inactivity (subject to change) +- **Container ID**: Pass the `container` parameter to reuse an existing container +- **Reuse**: Pass the container ID to maintain state across requests + +```python +# First request - creates a new container +response1 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Query the database"}], + tools=[...] +) + +# Get container ID from response (if available in response metadata) +container_id = response1.get("container", {}).get("id") + +# Second request - reuse the same container +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[...], + tools=[...], + container=container_id # Reuse container +) +``` + +:::warning +When a tool is called programmatically and the container is waiting for your tool result, you must respond before the container expires. Monitor the `expires_at` field. If the container expires, Claude may treat the tool call as timed out and retry it. +::: + +## Example Workflow + +### Step 1: Initial Request + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{ + "role": "user", + "content": "Query customer purchase history from the last quarter and identify our top 5 customers by revenue" + }], + tools=[ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string", "description": "SQL query to execute"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + ] +) +``` + +### Step 2: API Response with Tool Call + +Claude writes code that calls your tool. The response includes: + +```python +{ + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the purchase history and analyze the results." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_abc123", + "name": "code_execution", + "input": { + "code": "results = await query_database('')\ntop_customers = sorted(results, key=lambda x: x['revenue'], reverse=True)[:5]" + } + }, + { + "type": "tool_use", + "id": "toolu_def456", + "name": "query_database", + "input": {"sql": ""}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc123" + } + } + ], + "stop_reason": "tool_use" +} +``` + +### Step 3: Provide Tool Result + +```python +# Add assistant's response and tool result to conversation +messages = [ + {"role": "user", "content": "Query customer purchase history..."}, + { + "role": "assistant", + "content": response.choices[0].message.content, + "tool_calls": response.choices[0].message.tool_calls + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_def456", + "content": '[{"customer_id": "C1", "revenue": 45000}, ...]' + } + ] + } +] + +# Continue the conversation +response2 = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=messages, + tools=[...] +) +``` + +### Step 4: Final Response + +Once code execution completes, Claude provides the final response: + +```python +{ + "content": [ + { + "type": "code_execution_tool_result", + "tool_use_id": "srvtoolu_abc123", + "content": { + "type": "code_execution_result", + "stdout": "Top 5 customers by revenue:\n1. Customer C1: $45,000\n...", + "stderr": "", + "return_code": 0 + } + }, + { + "type": "text", + "text": "I've analyzed the purchase history from last quarter. Your top 5 customers generated $167,500 in total revenue..." + } + ], + "stop_reason": "end_turn" +} +``` + +## Advanced Patterns + +### Batch Processing with Loops + +Claude can write code that processes multiple items efficiently: + +```python +# Claude writes code like this: +regions = ["West", "East", "Central", "North", "South"] +results = {} +for region in regions: + data = await query_database(f"SELECT SUM(revenue) FROM sales WHERE region='{region}'") + results[region] = data[0]["total"] + +top_region = max(results.items(), key=lambda x: x[1]) +print(f"Top region: {top_region[0]} with ${top_region[1]:,}") +``` + +This pattern: +- Reduces model round-trips from N (one per region) to 1 +- Processes large result sets programmatically before returning to Claude +- Saves tokens by only returning aggregated conclusions + +### Early Termination + +Claude can stop processing as soon as success criteria are met: + +```python +endpoints = ["us-east", "eu-west", "apac"] +for endpoint in endpoints: + status = await check_health(endpoint) + if status == "healthy": + print(f"Found healthy endpoint: {endpoint}") + break # Stop early +``` + +### Data Filtering + +```python +logs = await fetch_logs(server_id) +errors = [log for log in logs if "ERROR" in log] +print(f"Found {len(errors)} errors") +for error in errors[-10:]: # Only return last 10 errors + print(error) +``` + +## Best Practices + +### Tool Design + +- **Provide detailed output descriptions**: Since Claude deserializes tool results in code, clearly document the format (JSON structure, field types, etc.) +- **Return structured data**: JSON or other easily parseable formats work best for programmatic processing +- **Keep responses concise**: Return only necessary data to minimize processing overhead + +### When to Use Programmatic Calling + +**Good use cases:** + +- Processing large datasets where you only need aggregates or summaries +- Multi-step workflows with 3+ dependent tool calls +- Operations requiring filtering, sorting, or transformation of tool results +- Tasks where intermediate data shouldn't influence Claude's reasoning +- Parallel operations across many items (e.g., checking 50 endpoints) + +**Less ideal use cases:** + +- Single tool calls with simple responses +- Tools that need immediate user feedback +- Very fast operations where code execution overhead would outweigh the benefit + +## Token Efficiency + +Programmatic tool calling can significantly reduce token consumption: + +- **Tool results from programmatic calls are not added to Claude's context** - only the final code output is +- **Intermediate processing happens in code** - filtering, aggregation, etc. don't consume model tokens +- **Multiple tool calls in one code execution** - reduces overhead compared to separate model turns + +For example, calling 10 tools directly uses ~10x the tokens of calling them programmatically and returning a summary. + +## Provider Support + +LiteLLM supports programmatic tool calling across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `allowed_callers` field. + +## Limitations + +### Feature Incompatibilities + +- **Structured outputs**: Tools with `strict: true` are not supported with programmatic calling +- **Tool choice**: You cannot force programmatic calling of a specific tool via `tool_choice` +- **Parallel tool use**: `disable_parallel_tool_use: true` is not supported with programmatic calling + +### Tool Restrictions + +The following tools cannot currently be called programmatically: + +- Web search +- Web fetch +- Tools provided by an MCP connector + +## Troubleshooting + +### Common Issues + +**"Tool not allowed" error** + +- Verify your tool definition includes `"allowed_callers": ["code_execution_20250825"]` +- Check that you're using a compatible model (Claude Sonnet 4.5 or Opus 4.5) + +**Container expiration** + +- Ensure you respond to tool calls within the container's lifetime (~4.5 minutes) +- Consider implementing faster tool execution + +**Beta header not added** + +- LiteLLM automatically adds the beta header when it detects `allowed_callers` +- If you're manually setting headers, ensure you include `advanced-tool-use-2025-11-20` + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_input_examples.md b/docs/my-website/docs/providers/anthropic_tool_input_examples.md new file mode 100644 index 0000000000..d0b7cc1762 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_input_examples.md @@ -0,0 +1,438 @@ +# Anthropic Tool Input Examples + +Provide concrete examples of valid tool inputs to help Claude understand how to use your tools more effectively. This is particularly useful for complex tools with nested objects, optional parameters, or format-sensitive inputs. + +:::info +Tool input examples is a beta feature. LiteLLM automatically adds the required `advanced-tool-use-2025-11-20` beta header when it detects tools with the `input_examples` field. +::: + +## When to Use Input Examples + +Input examples are most helpful for: + +- **Complex nested objects**: Tools with deeply nested parameter structures +- **Optional parameters**: Showing when optional parameters should be included +- **Format-sensitive inputs**: Demonstrating expected formats (dates, addresses, etc.) +- **Enum values**: Illustrating valid enum choices in context +- **Edge cases**: Showing how to handle special cases + +:::tip +**Prioritize descriptions first!** Clear, detailed tool descriptions are more important than examples. Use `input_examples` as a supplement for complex tools where descriptions alone may not be sufficient. +::: + +## Quick Start + +Add an `input_examples` field to your tool definition with an array of example input objects: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What's the weather like in San Francisco?"} + ], + tools=[ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "The unit of temperature" + } + }, + "required": ["location"] + } + }, + "input_examples": [ + { + "location": "San Francisco, CA", + "unit": "fahrenheit" + }, + { + "location": "Tokyo, Japan", + "unit": "celsius" + }, + { + "location": "New York, NY" # 'unit' is optional + } + ] + } + ] +) + +print(response) +``` + +## How It Works + +When you provide `input_examples`: + +1. **LiteLLM detects** the `input_examples` field in your tool definition +2. **Beta header added automatically**: The `advanced-tool-use-2025-11-20` header is injected +3. **Examples included in prompt**: Anthropic includes the examples alongside your tool schema +4. **Claude learns patterns**: The model uses examples to understand proper tool usage +5. **Better tool calls**: Claude makes more accurate tool calls with correct parameter formats + +## Example Formats + +### Simple Tool with Examples + +```python +{ + "type": "function", + "function": { + "name": "send_email", + "description": "Send an email to a recipient", + "parameters": { + "type": "object", + "properties": { + "to": {"type": "string", "description": "Email address"}, + "subject": {"type": "string"}, + "body": {"type": "string"} + }, + "required": ["to", "subject", "body"] + } + }, + "input_examples": [ + { + "to": "user@example.com", + "subject": "Meeting Reminder", + "body": "Don't forget our meeting tomorrow at 2 PM." + }, + { + "to": "team@company.com", + "subject": "Weekly Update", + "body": "Here's this week's progress report..." + } + ] +} +``` + +### Complex Nested Objects + +```python +{ + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start": { + "type": "object", + "properties": { + "date": {"type": "string"}, + "time": {"type": "string"} + } + }, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + } + }, + "required": ["title", "start"] + } + }, + "input_examples": [ + { + "title": "Team Standup", + "start": { + "date": "2025-01-15", + "time": "09:00" + }, + "attendees": [ + {"email": "alice@example.com", "optional": False}, + {"email": "bob@example.com", "optional": True} + ] + }, + { + "title": "Lunch Break", + "start": { + "date": "2025-01-15", + "time": "12:00" + } + # No attendees - showing optional field + } + ] +} +``` + +### Format-Sensitive Parameters + +```python +{ + "type": "function", + "function": { + "name": "search_flights", + "description": "Search for available flights", + "parameters": { + "type": "object", + "properties": { + "origin": {"type": "string", "description": "Airport code"}, + "destination": {"type": "string", "description": "Airport code"}, + "date": {"type": "string", "description": "Date in YYYY-MM-DD format"}, + "passengers": {"type": "integer"} + }, + "required": ["origin", "destination", "date"] + } + }, + "input_examples": [ + { + "origin": "SFO", + "destination": "JFK", + "date": "2025-03-15", + "passengers": 2 + }, + { + "origin": "LAX", + "destination": "ORD", + "date": "2025-04-20", + "passengers": 1 + } + ] +} +``` + +## Requirements and Limitations + +### Schema Validation + +- Each example **must be valid** according to the tool's `input_schema` +- Invalid examples will return a **400 error** from Anthropic +- Validation happens server-side (LiteLLM passes examples through) + +### Server-Side Tools Not Supported + +Input examples are **only supported for user-defined tools**. The following server-side tools do NOT support `input_examples`: + +- `web_search` (web search tool) +- `code_execution` (code execution tool) +- `computer_use` (computer use tool) +- `bash_tool` (bash execution tool) +- `text_editor` (text editor tool) + +### Token Costs + +Examples add to your prompt tokens: + +- **Simple examples**: ~20-50 tokens per example +- **Complex nested objects**: ~100-200 tokens per example +- **Trade-off**: Higher token cost for better tool call accuracy + +### Model Compatibility + +Input examples work with all Claude models that support the `advanced-tool-use-2025-11-20` beta header: + +- Claude Opus 4.5 (`claude-opus-4-5-20251101`) +- Claude Sonnet 4.5 (`claude-sonnet-4-5-20250929`) +- Claude Opus 4.1 (`claude-opus-4-1-20250805`) + +:::note +On Google Cloud's Vertex AI and Amazon Bedrock, only Claude Opus 4.5 supports tool input examples. +::: + +## Best Practices + +### 1. Show Diverse Examples + +Include examples that demonstrate different use cases: + +```python +"input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, # US city + {"location": "Tokyo, Japan", "unit": "celsius"}, # International + {"location": "New York, NY"} # Optional param omitted +] +``` + +### 2. Demonstrate Optional Parameters + +Show when optional parameters should and shouldn't be included: + +```python +"input_examples": [ + { + "query": "machine learning", + "filters": {"year": 2024, "category": "research"} # With optional filters + }, + { + "query": "artificial intelligence" # Without optional filters + } +] +``` + +### 3. Illustrate Format Requirements + +Make format expectations clear through examples: + +```python +"input_examples": [ + { + "phone": "+1-555-123-4567", # Shows expected phone format + "date": "2025-01-15", # Shows date format (YYYY-MM-DD) + "time": "14:30" # Shows time format (HH:MM) + } +] +``` + +### 4. Keep Examples Realistic + +Use realistic, production-like examples rather than placeholder data: + +```python +# ✅ Good - realistic examples +"input_examples": [ + {"email": "alice@company.com", "role": "admin"}, + {"email": "bob@company.com", "role": "user"} +] + +# ❌ Bad - placeholder examples +"input_examples": [ + {"email": "test@test.com", "role": "role1"}, + {"email": "example@example.com", "role": "role2"} +] +``` + +### 5. Limit Example Count + +Provide 2-5 examples per tool: + +- **Too few** (1): May not show enough variation +- **Just right** (2-5): Demonstrates patterns without bloating tokens +- **Too many** (10+): Wastes tokens, diminishing returns + +## Integration with Other Features + +Input examples work seamlessly with other Anthropic tool features: + +### With Tool Search + +```python +{ + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "input_examples": [ # Input examples + {"sql": "SELECT * FROM users WHERE id = 1"} + ] +} +``` + +### With Programmatic Tool Calling + +```python +{ + "type": "function", + "function": { + "name": "fetch_data", + "description": "Fetch data from API", + "parameters": {...} + }, + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"endpoint": "/api/users", "method": "GET"} + ] +} +``` + +### All Features Combined + +```python +{ + "type": "function", + "function": { + "name": "advanced_tool", + "description": "A complex tool", + "parameters": {...} + }, + "defer_loading": True, # Tool search + "allowed_callers": ["code_execution_20250825"], # Programmatic calling + "input_examples": [ # Input examples + {"param1": "value1", "param2": "value2"} + ] +} +``` + +## Provider Support + +LiteLLM supports input examples across all Anthropic-compatible providers: + +- **Standard Anthropic API** (`anthropic/claude-sonnet-4-5-20250929`) +- **Azure Anthropic** (`azure/claude-sonnet-4-5-20250929`) +- **Vertex AI Anthropic** (`vertex_ai/claude-sonnet-4-5-20250929`) + +The beta header is automatically added when LiteLLM detects tools with `input_examples` field. + +## Troubleshooting + +### "Invalid request" error with examples + +**Problem**: Receiving 400 error when using input examples + +**Solution**: Ensure each example is valid according to your `input_schema`: + +```python +# Check that: +# 1. All required fields are present in examples +# 2. Field types match the schema +# 3. Enum values are valid +# 4. Nested objects follow the schema structure +``` + +### Examples not improving tool calls + +**Problem**: Adding examples doesn't seem to help + +**Solution**: +1. **Check descriptions first**: Ensure tool descriptions are detailed and clear +2. **Review example quality**: Make sure examples are realistic and diverse +3. **Verify schema**: Confirm examples actually match your schema +4. **Add more variation**: Include examples showing different use cases + +### Token usage too high + +**Problem**: Input examples consuming too many tokens + +**Solution**: +1. **Reduce example count**: Use 2-3 examples instead of 5+ +2. **Simplify examples**: Remove unnecessary fields from examples +3. **Consider descriptions**: If descriptions are clear, examples may not be needed + +## When NOT to Use Input Examples + +Skip input examples if: + +- **Tool is simple**: Single parameter tools with clear descriptions +- **Schema is self-explanatory**: Well-structured schema with good descriptions +- **Token budget is tight**: Examples add 20-200 tokens each +- **Server-side tools**: web_search, code_execution, etc. don't support examples + +## Related Features + +- [Anthropic Tool Search](./anthropic_tool_search.md) - Dynamically discover and load tools on-demand +- [Anthropic Programmatic Tool Calling](./anthropic_programmatic_tool_calling.md) - Call tools from code execution +- [Anthropic Provider](./anthropic.md) - General Anthropic provider documentation + diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md new file mode 100644 index 0000000000..3d61022b26 --- /dev/null +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -0,0 +1,397 @@ +# Anthropic Tool Search + +Tool search enables Claude to dynamically discover and load tools on-demand from large tool catalogs (10,000+ tools). Instead of loading all tool definitions into the context window upfront, Claude searches your tool catalog and loads only the tools it needs. + +## Benefits + +- **Context efficiency**: Avoid consuming massive portions of your context window with tool definitions +- **Better tool selection**: Claude's tool selection accuracy degrades with more than 30-50 tools. Tool search maintains accuracy even with thousands of tools +- **On-demand loading**: Tools are only loaded when Claude needs them + +## Supported Models + +Tool search is available on: +- Claude Opus 4.5 +- Claude Sonnet 4.5 + +## Supported Platforms + +- Anthropic API (direct) +- Azure Anthropic (Microsoft Foundry) +- Google Cloud Vertex AI +- Amazon Bedrock (invoke API only, not converse API) + +## Tool Search Variants + +LiteLLM supports both tool search variants: + +### 1. Regex Tool Search (`tool_search_tool_regex_20251119`) + +Claude constructs regex patterns to search for tools. + +### 2. BM25 Tool Search (`tool_search_tool_bm25_20251119`) + +Claude uses natural language queries to search for tools using the BM25 algorithm. + +## Quick Start + +### Basic Example with Regex Tool Search + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "What is the weather in San Francisco?"} + ], + tools=[ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tool - will be loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather at a specific location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"] + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Mark for deferred loading + }, + # Another deferred tool + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) + +print(response.choices[0].message.content) +``` + +### BM25 Tool Search Example + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Search for Python files containing 'authentication'"} + ], + tools=[ + # Tool search tool (BM25 variant) + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Deferred tools... + { + "type": "function", + "function": { + "name": "search_codebase", + "description": "Search through codebase files by content and filename", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_pattern": {"type": "string"} + }, + "required": ["query"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Azure Anthropic + +```python +import litellm + +response = litellm.completion( + model="azure_anthropic/claude-sonnet-4-5", + api_base="https://.services.ai.azure.com/anthropic", + api_key="your-azure-api-key", + messages=[ + {"role": "user", "content": "What's the weather like?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get current weather", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Using with Vertex AI + +```python +import litellm + +response = litellm.completion( + model="vertex_ai/claude-sonnet-4-5", + vertex_project="your-project-id", + vertex_location="us-central1", + messages=[ + {"role": "user", "content": "Search my documents"} + ], + tools=[ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + }, + # Your deferred tools... + ] +) +``` + +## Streaming Support + +Tool search works with streaming: + +```python +import litellm + +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[ + {"role": "user", "content": "Get the weather"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ], + stream=True +) + +for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +## LiteLLM Proxy + +Tool search works automatically through the LiteLLM proxy: + +### Proxy Config + +```yaml +model_list: + - model_name: claude-sonnet + litellm_params: + model: anthropic/claude-sonnet-4-5-20250929 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +### Client Request + +```python +import openai + +client = openai.OpenAI( + api_key="your-litellm-proxy-key", + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="claude-sonnet", + messages=[ + {"role": "user", "content": "What's the weather?"} + ], + tools=[ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + ] +) +``` + +## Important Notes + +### Beta Header + +LiteLLM automatically adds the `advanced-tool-use-2025-11-20` beta header when tool search tools are detected. You don't need to manually specify it. + +### Deferred Loading + +- Tools with `defer_loading: true` are only loaded when Claude discovers them via search +- At least one tool must be non-deferred (the tool search tool itself) +- Keep your 3-5 most frequently used tools as non-deferred for optimal performance + +### Tool Descriptions + +Write clear, descriptive tool names and descriptions that match how users describe tasks. The search algorithm uses: +- Tool names +- Tool descriptions +- Argument names +- Argument descriptions + +### Usage Tracking + +Tool search requests are tracked in the usage object: + +```python +response = litellm.completion( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": "Search for tools"}], + tools=[...] +) + +# Check tool search usage +if response.usage.server_tool_use: + print(f"Tool search requests: {response.usage.server_tool_use.tool_search_requests}") +``` + +## Error Handling + +### All Tools Deferred + +```python +# ❌ This will fail - at least one tool must be non-deferred +tools = [ + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] + +# ✅ Correct - tool search tool is non-deferred +tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {...}, + "defer_loading": True + } +] +``` + +### Missing Tool Definition + +If Claude references a tool that isn't in your deferred tools list, you'll get an error. Make sure all tools that might be discovered are included in the tools parameter with `defer_loading: true`. + +## Best Practices + +1. **Keep frequently used tools non-deferred**: Your 3-5 most common tools should not have `defer_loading: true` + +2. **Use semantic descriptions**: Tool descriptions should use natural language that matches user queries + +3. **Choose the right variant**: + - Use **regex** for exact pattern matching (faster) + - Use **BM25** for natural language semantic search + +4. **Monitor usage**: Track `tool_search_requests` in the usage object to understand search patterns + +5. **Optimize tool catalog**: Remove unused tools and consolidate similar functionality + +## When to Use Tool Search + +**Good use cases:** +- 10+ tools available in your system +- Tool definitions consuming >10K tokens +- Experiencing tool selection accuracy issues +- Building systems with multiple tool categories +- Tool library growing over time + +**When traditional tool calling is better:** +- Less than 10 tools total +- All tools are frequently used +- Very small tool definitions (<100 tokens total) + +## Limitations + +- Not compatible with tool use examples +- Requires Claude Opus 4.5 or Sonnet 4.5 +- On Bedrock, only available via invoke API (not converse API) +- Maximum 10,000 tools in catalog +- Returns 3-5 most relevant tools per search + +## Additional Resources + +- [Anthropic Tool Search Documentation](https://docs.anthropic.com/en/docs/build-with-claude/tool-use/tool-search) +- [LiteLLM Tool Calling Guide](https://docs.litellm.ai/docs/completion/function_call) + diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index b7b39f1039..b363b747de 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -42,6 +42,7 @@ from litellm.types.llms.openai import ( ChatCompletionRedactedThinkingBlock, ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ) from litellm.types.utils import ( Delta, @@ -550,15 +551,18 @@ class ModelResponseIterator: if "text" in content_block["delta"]: text = content_block["delta"]["text"] elif "partial_json" in content_block["delta"]: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": content_block["delta"]["partial_json"], + tool_use = cast( + ChatCompletionToolCallChunk, + { + "id": None, + "type": "function", + "function": { + "name": None, + "arguments": content_block["delta"]["partial_json"], + }, + "index": self.tool_index, }, - "index": self.tool_index, - } + ) elif "citation" in content_block["delta"]: provider_specific_fields["citation"] = content_block["delta"]["citation"] elif ( @@ -569,7 +573,7 @@ class ModelResponseIterator: ChatCompletionThinkingBlock( type="thinking", thinking=content_block["delta"].get("thinking") or "", - signature=content_block["delta"].get("signature"), + signature=str(content_block["delta"].get("signature") or ""), ) ] provider_specific_fields["thinking_blocks"] = thinking_blocks @@ -625,7 +629,7 @@ class ModelResponseIterator: return content_block_start - def chunk_parser(self, chunk: dict) -> ModelResponseStream: + def chunk_parser(self, chunk: dict) -> ModelResponseStream: # noqa: PLR0915 try: type_chunk = chunk.get("type", "") or "" @@ -672,15 +676,32 @@ class ModelResponseIterator: text = content_block_start["content_block"]["text"] elif content_block_start["content_block"]["type"] == "tool_use": self.tool_index += 1 - tool_use = { - "id": content_block_start["content_block"]["id"], - "type": "function", - "function": { - "name": content_block_start["content_block"]["name"], - "arguments": "", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content_block_start["content_block"]: + caller_data = content_block_start["content_block"]["caller"] + if caller_data: + tool_use["caller"] = cast(Dict[str, Any], caller_data) # type: ignore[typeddict-item] + elif content_block_start["content_block"]["type"] == "server_tool_use": + # Handle server tool use (for tool search) + self.tool_index += 1 + tool_use = ChatCompletionToolCallChunk( + id=content_block_start["content_block"]["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content_block_start["content_block"]["name"], + arguments="", + ), + index=self.tool_index, + ) elif ( content_block_start["content_block"]["type"] == "redacted_thinking" ): @@ -696,17 +717,21 @@ class ModelResponseIterator: # check if tool call content block is_empty = self.check_empty_tool_call_args() if is_empty: - tool_use = { - "id": None, - "type": "function", - "function": { - "name": None, - "arguments": "{}", - }, - "index": self.tool_index, - } + tool_use = ChatCompletionToolCallChunk( + id=None, # type: ignore[typeddict-item] + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=None, # type: ignore[typeddict-item] + arguments="{}", + ), + index=self.tool_index, + ) # Reset response_format tool tracking when block stops self.is_response_format_tool = False + elif type_chunk == "tool_result": + # Handle tool_result blocks (for tool search results with tool_reference) + # These are automatically handled by Anthropic API, we just pass them through + pass elif type_chunk == "message_delta": finish_reason, usage = self._handle_message_delta(chunk) elif type_chunk == "message_start": diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 623a98c132..ac1c9b1e00 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -54,7 +54,10 @@ from litellm.types.utils import ( CompletionTokensDetailsWrapper, ) from litellm.types.utils import Message as LitellmMessage -from litellm.types.utils import PromptTokensDetailsWrapper, ServerToolUse +from litellm.types.utils import ( + PromptTokensDetailsWrapper, + ServerToolUse, +) from litellm.utils import ( ModelResponse, Usage, @@ -187,7 +190,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): ) return _tool_choice - def _map_tool_helper( + def _map_tool_helper( # noqa: PLR0915 self, tool: ChatCompletionToolParam ) -> Tuple[Optional[AllAnthropicToolsValues], Optional[AnthropicMcpServerTool]]: returned_tool: Optional[AllAnthropicToolsValues] = None @@ -250,9 +253,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): returned_tool = _computer_tool elif any(tool["type"].startswith(t) for t in ANTHROPIC_HOSTED_TOOLS): - function_name = tool.get("name", tool.get("function", {}).get("name")) - if function_name is None or not isinstance(function_name, str): + function_name_obj = tool.get("name", tool.get("function", {}).get("name")) + if function_name_obj is None or not isinstance(function_name_obj, str): raise ValueError("Missing required parameter: name") + function_name = function_name_obj additional_tool_params = {} for k, v in tool.items(): @@ -268,6 +272,30 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_server = self._map_openai_mcp_server_tool( cast(OpenAIMcpServerTool, tool) ) + elif tool["type"] == "tool_search_tool_regex_20251119": + # Tool search tool using regex + from litellm.types.llms.anthropic import AnthropicToolSearchToolRegex + + tool_name_obj = tool.get("name", "tool_search_tool_regex") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolRegex( + type="tool_search_tool_regex_20251119", + name=tool_name, + ) + elif tool["type"] == "tool_search_tool_bm25_20251119": + # Tool search tool using BM25 + from litellm.types.llms.anthropic import AnthropicToolSearchToolBM25 + + tool_name_obj = tool.get("name", "tool_search_tool_bm25") + if not isinstance(tool_name_obj, str): + raise ValueError("Tool search tool must have a valid name") + tool_name = tool_name_obj + returned_tool = AnthropicToolSearchToolBM25( + type="tool_search_tool_bm25_20251119", + name=tool_name, + ) if returned_tool is None and mcp_server is None: raise ValueError(f"Unsupported tool type: {tool['type']}") @@ -275,14 +303,67 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): _cache_control = tool.get("cache_control", None) _cache_control_function = tool.get("function", {}).get("cache_control", None) if returned_tool is not None: - if _cache_control is not None: - returned_tool["cache_control"] = _cache_control - elif _cache_control_function is not None and isinstance( - _cache_control_function, dict - ): - returned_tool["cache_control"] = ChatCompletionCachedContent( - **_cache_control_function # type: ignore - ) + # Only set cache_control on tools that support it (not tool search tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"): + if _cache_control is not None: + returned_tool["cache_control"] = _cache_control # type: ignore[typeddict-item] + elif _cache_control_function is not None and isinstance( + _cache_control_function, dict + ): + returned_tool["cache_control"] = ChatCompletionCachedContent( # type: ignore[typeddict-item] + **_cache_control_function # type: ignore + ) + + ## check if defer_loading is set in the tool + _defer_loading = tool.get("defer_loading", None) + _defer_loading_function = tool.get("function", {}).get("defer_loading", None) + if returned_tool is not None: + # Only set defer_loading on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _defer_loading is not None: + if not isinstance(_defer_loading, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading # type: ignore[typeddict-item] + elif _defer_loading_function is not None: + if not isinstance(_defer_loading_function, bool): + raise ValueError("defer_loading must be a boolean") + returned_tool["defer_loading"] = _defer_loading_function # type: ignore[typeddict-item] + + ## check if allowed_callers is set in the tool + _allowed_callers = tool.get("allowed_callers", None) + _allowed_callers_function = tool.get("function", {}).get("allowed_callers", None) + if returned_tool is not None: + # Only set allowed_callers on tools that support it (not tool search tools or computer tools) + tool_type = returned_tool.get("type", "") + if tool_type not in ("tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119", "computer_20241022", "computer_20250124"): + if _allowed_callers is not None: + if not isinstance(_allowed_callers, list) or not all( + isinstance(item, str) for item in _allowed_callers + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers # type: ignore[typeddict-item] + elif _allowed_callers_function is not None: + if not isinstance(_allowed_callers_function, list) or not all( + isinstance(item, str) for item in _allowed_callers_function + ): + raise ValueError("allowed_callers must be a list of strings") + returned_tool["allowed_callers"] = _allowed_callers_function # type: ignore[typeddict-item] + + ## check if input_examples is set in the tool + _input_examples = tool.get("input_examples", None) + _input_examples_function = tool.get("function", {}).get("input_examples", None) + if returned_tool is not None: + # Only set input_examples on user-defined tools (type "custom" or no type) + tool_type = returned_tool.get("type", "") + if tool_type == "custom" or (tool_type == "" and "name" in returned_tool): + if _input_examples is not None and isinstance(_input_examples, list): + returned_tool["input_examples"] = _input_examples # type: ignore[typeddict-item] + elif _input_examples_function is not None and isinstance( + _input_examples_function, list + ): + returned_tool["input_examples"] = _input_examples_function # type: ignore[typeddict-item] return returned_tool, mcp_server @@ -334,6 +415,82 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): mcp_servers.append(mcp_server_tool) return anthropic_tools, mcp_servers + def _detect_tool_search_tools(self, tools: Optional[List]) -> bool: + """Check if tool search tools are present in the tools list.""" + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def _separate_deferred_tools( + self, tools: List + ) -> Tuple[List, List]: + """ + Separate tools into deferred and non-deferred lists. + + Returns: + Tuple of (non_deferred_tools, deferred_tools) + """ + non_deferred = [] + deferred = [] + + for tool in tools: + if tool.get("defer_loading", False): + deferred.append(tool) + else: + non_deferred.append(tool) + + return non_deferred, deferred + + def _expand_tool_references( + self, + content: List, + deferred_tools: List, + ) -> List: + """ + Expand tool_reference blocks to full tool definitions. + + When Anthropic's tool search returns results, it includes tool_reference blocks + that reference tools by name. This method expands those references to full + tool definitions from the deferred_tools catalog. + + Args: + content: Response content that may contain tool_reference blocks + deferred_tools: List of deferred tools that can be referenced + + Returns: + Content with tool_reference blocks expanded to full tool definitions + """ + if not deferred_tools: + return content + + # Create a mapping of tool names to tool definitions + tool_map = {} + for tool in deferred_tools: + tool_name = tool.get("name") or tool.get("function", {}).get("name") + if tool_name: + tool_map[tool_name] = tool + + # Expand tool references in content + expanded_content = [] + for item in content: + if isinstance(item, dict) and item.get("type") == "tool_reference": + tool_name = item.get("tool_name") + if tool_name and tool_name in tool_map: + # Replace reference with full tool definition + expanded_content.append(tool_map[tool_name]) + else: + # Keep the reference if we can't find the tool + expanded_content.append(item) + else: + expanded_content.append(item) + + return expanded_content + def _map_stop_sequences( self, stop: Optional[Union[str, List[str]]] ) -> Optional[List[str]]: @@ -822,6 +979,17 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "messages": anthropic_messages, **optional_params, } + + ## Handle output_config (Anthropic-specific parameter) + if "output_config" in optional_params: + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and effort not in ["high", "medium", "low"]: + raise ValueError( + f"Invalid effort value: {effort}. Must be one of: 'high', 'medium', 'low'" + ) + data["output_config"] = output_config return data @@ -870,18 +1038,40 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): text_content += content["text"] ## TOOL CALLING elif content["type"] == "tool_use": - tool_calls.append( - ChatCompletionToolCallChunk( - id=content["id"], - type="function", - function=ChatCompletionToolCallFunctionChunk( - name=content["name"], - arguments=json.dumps(content["input"]), - ), - index=idx, - ) + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content["input"]), + ), + index=idx, ) - + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## SERVER TOOL USE (for tool search) + elif content["type"] == "server_tool_use": + # Server tool use blocks are for tool search - treat as tool calls + tool_call = ChatCompletionToolCallChunk( + id=content["id"], + type="function", + function=ChatCompletionToolCallFunctionChunk( + name=content["name"], + arguments=json.dumps(content.get("input", {})), + ), + index=idx, + ) + # Include caller information if present (for programmatic tool calling) + if "caller" in content: + tool_call["caller"] = cast(Dict[str, Any], content["caller"]) # type: ignore[typeddict-item] + tool_calls.append(tool_call) + ## TOOL SEARCH TOOL RESULT (skip - this is metadata about tool discovery) + elif content["type"] == "tool_search_tool_result": + # This block contains tool_references that were discovered + # We don't need to include this in the response as it's internal metadata + pass elif content.get("thinking", None) is not None: if thinking_blocks is None: thinking_blocks = [] @@ -916,7 +1106,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): return text_content, citations, thinking_blocks, reasoning_content, tool_calls def calculate_usage( - self, usage_object: dict, reasoning_content: Optional[str] + self, usage_object: dict, reasoning_content: Optional[str], completion_response: Optional[dict] = None ) -> Usage: # NOTE: Sometimes the usage object has None set explicitly for token counts, meaning .get() & key access returns None, and we need to account for this prompt_tokens = usage_object.get("input_tokens", 0) or 0 @@ -926,6 +1116,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens: int = 0 cache_creation_token_details: Optional[CacheCreationTokenDetails] = None web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None if ( "cache_creation_input_tokens" in _usage and _usage["cache_creation_input_tokens"] is not None @@ -946,6 +1137,25 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): web_search_requests = cast( int, _usage["server_tool_use"]["web_search_requests"] ) + if ( + "tool_search_requests" in _usage["server_tool_use"] + and _usage["server_tool_use"]["tool_search_requests"] is not None + ): + tool_search_requests = cast( + int, _usage["server_tool_use"]["tool_search_requests"] + ) + + # Count tool_search_requests from content blocks if not in usage + # Anthropic doesn't always include tool_search_requests in the usage object + if tool_search_requests is None and completion_response is not None: + tool_search_count = 0 + for content in completion_response.get("content", []): + if content.get("type") == "server_tool_use": + tool_name = content.get("name", "") + if "tool_search" in tool_name: + tool_search_count += 1 + if tool_search_count > 0: + tool_search_requests = tool_search_count if "cache_creation" in _usage and _usage["cache_creation"] is not None: cache_creation_token_details = CacheCreationTokenDetails( @@ -982,8 +1192,11 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): cache_read_input_tokens=cache_read_input_tokens, completion_tokens_details=completion_token_details, server_tool_use=( - ServerToolUse(web_search_requests=web_search_requests) - if web_search_requests is not None + ServerToolUse( + web_search_requests=web_search_requests, + tool_search_requests=tool_search_requests, + ) + if (web_search_requests is not None or tool_search_requests is not None) else None ), ) @@ -1077,6 +1290,7 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): usage = self.calculate_usage( usage_object=completion_response["usage"], reasoning_content=reasoning_content, + completion_response=completion_response, ) setattr(model_response, "usage", usage) # type: ignore diff --git a/litellm/llms/anthropic/common_utils.py b/litellm/llms/anthropic/common_utils.py index 0d00a3b463..9f5688f9e0 100644 --- a/litellm/llms/anthropic/common_utils.py +++ b/litellm/llms/anthropic/common_utils.py @@ -88,6 +88,86 @@ class AnthropicModelInfo(BaseLLMModelInfo): return True return False + def is_tool_search_used(self, tools: Optional[List]) -> bool: + """ + Check if tool search tools are present in the tools list. + """ + if not tools: + return False + + for tool in tools: + tool_type = tool.get("type", "") + if tool_type in ["tool_search_tool_regex_20251119", "tool_search_tool_bm25_20251119"]: + return True + return False + + def is_programmatic_tool_calling_used(self, tools: Optional[List]) -> bool: + """ + Check if programmatic tool calling is being used (tools with allowed_callers field). + + Returns True if any tool has allowed_callers containing 'code_execution_20250825'. + """ + if not tools: + return False + + for tool in tools: + # Check top-level allowed_callers + allowed_callers = tool.get("allowed_callers", None) + if allowed_callers and isinstance(allowed_callers, list): + if "code_execution_20250825" in allowed_callers: + return True + + # Check function.allowed_callers for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_allowed_callers = function.get("allowed_callers", None) + if function_allowed_callers and isinstance(function_allowed_callers, list): + if "code_execution_20250825" in function_allowed_callers: + return True + + return False + + def is_input_examples_used(self, tools: Optional[List]) -> bool: + """ + Check if input_examples is being used in any tools. + + Returns True if any tool has input_examples field. + """ + if not tools: + return False + + for tool in tools: + # Check top-level input_examples + input_examples = tool.get("input_examples", None) + if input_examples and isinstance(input_examples, list) and len(input_examples) > 0: + return True + + # Check function.input_examples for OpenAI format tools + function = tool.get("function", {}) + if isinstance(function, dict): + function_input_examples = function.get("input_examples", None) + if function_input_examples and isinstance(function_input_examples, list) and len(function_input_examples) > 0: + return True + + return False + + def is_effort_used(self, optional_params: Optional[dict]) -> bool: + """ + Check if effort parameter is being used via output_config. + + Returns True if output_config with effort field is present. + """ + if not optional_params: + return False + + output_config = optional_params.get("output_config") + if output_config and isinstance(output_config, dict): + effort = output_config.get("effort") + if effort and isinstance(effort, str): + return True + + return False + def _get_user_anthropic_beta_headers( self, anthropic_beta_header: Optional[str] ) -> Optional[List[str]]: @@ -122,6 +202,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): pdf_used: bool = False, file_id_used: bool = False, mcp_server_used: bool = False, + tool_search_used: bool = False, + programmatic_tool_calling_used: bool = False, + input_examples_used: bool = False, + effort_used: bool = False, is_vertex_request: bool = False, user_anthropic_beta_headers: Optional[List[str]] = None, ) -> dict: @@ -138,6 +222,15 @@ class AnthropicModelInfo(BaseLLMModelInfo): betas.add("code-execution-2025-05-22") if mcp_server_used: betas.add("mcp-client-2025-04-04") + # Tool search, programmatic tool calling, and input_examples all use the same beta header + if tool_search_used or programmatic_tool_calling_used or input_examples_used: + from litellm.types.llms.anthropic import ANTHROPIC_TOOL_SEARCH_BETA_HEADER + betas.add(ANTHROPIC_TOOL_SEARCH_BETA_HEADER) + + # Effort parameter uses a separate beta header + if effort_used: + from litellm.types.llms.anthropic import ANTHROPIC_EFFORT_BETA_HEADER + betas.add(ANTHROPIC_EFFORT_BETA_HEADER) headers = { "anthropic-version": anthropic_version or "2023-06-01", @@ -182,6 +275,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): ) pdf_used = self.is_pdf_used(messages=messages) file_id_used = self.is_file_id_used(messages=messages) + tool_search_used = self.is_tool_search_used(tools=tools) + programmatic_tool_calling_used = self.is_programmatic_tool_calling_used(tools=tools) + input_examples_used = self.is_input_examples_used(tools=tools) + effort_used = self.is_effort_used(optional_params=optional_params) user_anthropic_beta_headers = self._get_user_anthropic_beta_headers( anthropic_beta_header=headers.get("anthropic-beta") ) @@ -194,6 +291,10 @@ class AnthropicModelInfo(BaseLLMModelInfo): is_vertex_request=optional_params.get("is_vertex_request", False), user_anthropic_beta_headers=user_anthropic_beta_headers, mcp_server_used=mcp_server_used, + tool_search_used=tool_search_used, + programmatic_tool_calling_used=programmatic_tool_calling_used, + input_examples_used=input_examples_used, + effort_used=effort_used, ) headers = {**headers, **anthropic_headers} diff --git a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py index 0e905014fe..98e57f279c 100644 --- a/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py @@ -645,7 +645,7 @@ class LiteLLMAnthropicMessagesAdapter: type="tool_use", id=choice.delta.tool_calls[0].id or str(uuid.uuid4()), name=choice.delta.tool_calls[0].function.name or "", - input={}, + input={}, # type: ignore[typeddict-item] ) elif isinstance(choice, StreamingChoices) and hasattr( choice.delta, "thinking_blocks" diff --git a/litellm/types/llms/anthropic.py b/litellm/types/llms/anthropic.py index fc210a7d08..507b382f78 100644 --- a/litellm/types/llms/anthropic.py +++ b/litellm/types/llms/anthropic.py @@ -36,12 +36,20 @@ class AnthropicOutputSchema(TypedDict, total=False): schema: Required[dict] +class AnthropicOutputConfig(TypedDict, total=False): + """Configuration for controlling Claude's output behavior.""" + effort: Literal["high", "medium", "low"] + + class AnthropicMessagesTool(TypedDict, total=False): name: Required[str] description: str input_schema: Optional[AnthropicInputSchema] type: Literal["custom"] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: bool + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicComputerTool(TypedDict, total=False): @@ -67,24 +75,78 @@ class AnthropicWebSearchTool(TypedDict, total=False): cache_control: Optional[Union[dict, ChatCompletionCachedContent]] max_uses: Optional[int] user_location: Optional[AnthropicWebSearchUserLocation] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicHostedTools(TypedDict, total=False): # for bash_tool and text_editor type: Required[str] name: Required[str] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicCodeExecutionTool(TypedDict, total=False): type: Required[str] name: Required[Literal["code_execution"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] class AnthropicMemoryTool(TypedDict, total=False): type: Required[str] name: Required[Literal["memory"]] cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class AnthropicToolSearchToolRegex(TypedDict, total=False): + """Tool search tool using regex patterns for tool discovery.""" + type: Required[Literal["tool_search_tool_regex_20251119"]] + name: Required[str] + + +class AnthropicToolSearchToolBM25(TypedDict, total=False): + """Tool search tool using BM25 algorithm for tool discovery.""" + type: Required[Literal["tool_search_tool_bm25_20251119"]] + name: Required[str] + cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + defer_loading: Optional[bool] + allowed_callers: Optional[List[str]] + input_examples: Optional[List[Dict[str, Any]]] + + +class ToolReference(TypedDict, total=False): + """Reference to a tool that should be expanded from deferred tools.""" + type: Required[Literal["tool_reference"]] + tool_name: Required[str] + + +class DirectToolCaller(TypedDict, total=False): + """Indicates a tool was called directly by Claude.""" + type: Required[Literal["direct"]] + + +class CodeExecutionToolCaller(TypedDict, total=False): + """Indicates a tool was called programmatically from code execution.""" + type: Required[Literal["code_execution_20250825"]] + tool_id: Required[str] # ID of the code execution tool that made the call + + +ToolCaller = Union[DirectToolCaller, CodeExecutionToolCaller] + + +class AnthropicContainer(TypedDict, total=False): + """Container metadata for code execution.""" + id: Required[str] + expires_at: Optional[str] # ISO 8601 timestamp AllAnthropicToolsValues = Union[ @@ -94,6 +156,8 @@ AllAnthropicToolsValues = Union[ AnthropicWebSearchTool, AnthropicCodeExecutionTool, AnthropicMemoryTool, + AnthropicToolSearchToolRegex, + AnthropicToolSearchToolBM25, ] @@ -121,6 +185,7 @@ class AnthropicMessagesToolUseParam(TypedDict, total=False): name: str input: dict cache_control: Optional[Union[dict, ChatCompletionCachedContent]] + caller: Optional[ToolCaller] AnthropicMessagesAssistantMessageValues = Union[ @@ -372,6 +437,7 @@ class ToolUseBlock(TypedDict): name: str type: Literal["tool_use"] + caller: Optional[ToolCaller] class TextBlock(TypedDict): @@ -565,3 +631,11 @@ class ANTHROPIC_BETA_HEADER_VALUES(str, Enum): WEB_FETCH_2025_09_10 = "web-fetch-2025-09-10" CONTEXT_MANAGEMENT_2025_06_27 = "context-management-2025-06-27" STRUCTURED_OUTPUT_2025_09_25 = "structured-outputs-2025-11-13" + ADVANCED_TOOL_USE_2025_11_20 = "advanced-tool-use-2025-11-20" + + +# Tool search beta header constant +ANTHROPIC_TOOL_SEARCH_BETA_HEADER = "advanced-tool-use-2025-11-20" + +# Effort beta header constant +ANTHROPIC_EFFORT_BETA_HEADER = "effort-2025-11-24" diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index c29b2f32ea..61d58e4c86 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1,6 +1,17 @@ from enum import Enum from os import PathLike -from typing import IO, Any, Dict, Iterable, List, Literal, Mapping, Optional, Tuple, Union +from typing import ( + IO, + Any, + Dict, + Iterable, + List, + Literal, + Mapping, + Optional, + Tuple, + Union, +) import httpx from openai._legacy_response import ( diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 0f73407610..b0d081d8f8 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -999,7 +999,8 @@ class PromptTokensDetailsWrapper( class ServerToolUse(BaseModel): - web_search_requests: Optional[int] + web_search_requests: Optional[int] = None + tool_search_requests: Optional[int] = None class Usage(CompletionUsage): diff --git a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py index 8ff2f0a447..09c16add77 100644 --- a/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/test_litellm/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -556,3 +556,744 @@ def test_anthropic_structured_output_beta_header(): "structured-outputs-2025-11-13" in response["raw_request_headers"]["anthropic-beta"] ) + + +# ============ Tool Search Tests ============ + + +def test_tool_search_regex_detection(): + """Test that tool search regex tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search regex tool + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + ] + assert config.is_tool_search_used(tools) is True + + # Test without tool search + tools = [ + { + "type": "function", + "function": {"name": "get_weather"} + } + ] + assert config.is_tool_search_used(tools) is False + + +def test_tool_search_bm25_detection(): + """Test that tool search BM25 tools are properly detected""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + # Test with tool search BM25 tool + tools = [ + { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + ] + assert config.is_tool_search_used(tools) is True + + +def test_tool_search_beta_header(): + """Test that tool search beta header is automatically added""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + config = AnthropicModelInfo() + + headers = config.get_anthropic_headers( + api_key="test-key", + tool_search_used=True, + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_tool_search_regex_mapping(): + """Test that tool search regex tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_regex_20251119" + assert mapped_tool["name"] == "tool_search_tool_regex" + assert mcp_server is None + + +def test_tool_search_bm25_mapping(): + """Test that tool search BM25 tools are properly mapped""" + config = AnthropicConfig() + + tool = { + "type": "tool_search_tool_bm25_20251119", + "name": "tool_search_tool_bm25" + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool["type"] == "tool_search_tool_bm25_20251119" + assert mapped_tool["name"] == "tool_search_tool_bm25" + assert mcp_server is None + + +def test_deferred_tools_separation(): + """Test that deferred and non-deferred tools are properly separated""" + config = AnthropicConfig() + + tools = [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + { + "type": "function", + "function": {"name": "get_weather"}, + "defer_loading": True + }, + { + "type": "function", + "function": {"name": "search_files"}, + "defer_loading": False + } + ] + + non_deferred, deferred = config._separate_deferred_tools(tools) + + assert len(non_deferred) == 2 # tool_search and search_files + assert len(deferred) == 1 # get_weather + + +def test_server_tool_use_in_response(): + """Test that server_tool_use blocks are parsed correctly""" + config = AnthropicConfig() + + completion_response = { + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01ABC123", + "name": "tool_search_tool_regex", + "input": {"query": "weather"} + } + ] + } + + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "srvtoolu_01ABC123" + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + + +def test_tool_search_usage_tracking(): + """Test that tool_search_requests are tracked in usage""" + config = AnthropicConfig() + + usage_object = { + "input_tokens": 100, + "output_tokens": 50, + "server_tool_use": { + "tool_search_requests": 2 + } + } + + usage = config.calculate_usage(usage_object=usage_object, reasoning_content=None) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.tool_search_requests == 2 + + +def test_tool_reference_expansion(): + """Test that tool_reference blocks are expanded correctly""" + config = AnthropicConfig() + + deferred_tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather" + } + } + ] + + content = [ + {"type": "text", "text": "I'll search for tools"}, + {"type": "tool_reference", "tool_name": "get_weather"} + ] + + expanded = config._expand_tool_references(content, deferred_tools) + + assert len(expanded) == 2 + assert expanded[0]["type"] == "text" + assert expanded[1]["type"] == "function" + assert expanded[1]["function"]["name"] == "get_weather" + + +def test_defer_loading_preserved_in_transformation(): + """Test that defer_loading parameter is preserved when transforming tools""" + config = AnthropicConfig() + + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "defer_loading": True + } + + mapped_tool, mcp_server = config._map_tool_helper(tool) + + assert mapped_tool is not None + assert mapped_tool.get("defer_loading") is True + assert mapped_tool["name"] == "get_weather" + assert mcp_server is None + + +def test_tool_search_complete_response_parsing(): + """Test parsing a complete tool search response with server_tool_use and tool_search_tool_result blocks""" + config = AnthropicConfig() + + # Simulating actual Anthropic API response with tool search + completion_response = { + "content": [ + { + "type": "text", + "text": "I'll search for weather-related tools that can help you." + }, + { + "type": "server_tool_use", + "id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "name": "tool_search_tool_regex", + "input": {"pattern": "weather", "limit": 5}, + "caller": {"type": "direct"} + }, + { + "type": "tool_search_tool_result", + "tool_use_id": "srvtoolu_015i6aVA2niwzv4RG4DtnxDJ", + "content": { + "type": "tool_search_tool_search_result", + "tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}] + } + }, + { + "type": "text", + "text": "Great! I found a weather tool." + }, + { + "type": "tool_use", + "id": "toolu_01CrCNx4ntSaeeV9iArT4JfQ", + "name": "get_weather", + "input": {"location": "San Francisco"} + } + ], + "usage": { + "input_tokens": 1639, + "output_tokens": 170, + "server_tool_use": {"web_search_requests": 0} + } + } + + # Extract content + text, citations, thinking_blocks, reasoning_content, tool_calls = config.extract_response_content( + completion_response + ) + + # Verify text extraction (should concatenate both text blocks) + assert "I'll search for weather-related tools" in text + assert "Great! I found a weather tool" in text + + # Verify tool calls (should have both server_tool_use and tool_use) + assert len(tool_calls) == 2 + assert tool_calls[0]["function"]["name"] == "tool_search_tool_regex" + assert tool_calls[1]["function"]["name"] == "get_weather" + + # Verify usage calculation counts tool_search_requests from content + usage = config.calculate_usage( + usage_object=completion_response["usage"], + reasoning_content=None, + completion_response=completion_response + ) + + assert usage.server_tool_use is not None + assert usage.server_tool_use.web_search_requests == 0 + assert usage.server_tool_use.tool_search_requests == 1 # Counted from server_tool_use blocks + + +def test_allowed_callers_field_preservation(): + """Test that allowed_callers field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level allowed_callers + tool_with_allowed_callers = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_allowed_callers) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_programmatic_tool_calling_beta_header(): + """Test that beta header is automatically added when programmatic tool calling is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with allowed_callers + tools = [ + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": {"type": "object", "properties": {}} + }, + "allowed_callers": ["code_execution_20250825"] + } + ] + + is_programmatic = model_info.is_programmatic_tool_calling_used(tools) + assert is_programmatic is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + programmatic_tool_calling_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_caller_field_in_response(): + """Test that caller field is correctly parsed from tool_use blocks.""" + config = AnthropicConfig() + + # Mock response with programmatic tool call + completion_response = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "content": [ + { + "type": "text", + "text": "I'll query the database." + }, + { + "type": "tool_use", + "id": "toolu_123", + "name": "query_database", + "input": {"sql": "SELECT * FROM users"}, + "caller": { + "type": "code_execution_20250825", + "tool_id": "srvtoolu_abc" + } + } + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 100, "output_tokens": 50} + } + + text, citations, thinking, reasoning, tool_calls = config.extract_response_content(completion_response) + + assert len(tool_calls) == 1 + assert tool_calls[0]["id"] == "toolu_123" + assert tool_calls[0]["function"]["name"] == "query_database" + assert "caller" in tool_calls[0] + assert tool_calls[0]["caller"]["type"] == "code_execution_20250825" + assert tool_calls[0]["caller"]["tool_id"] == "srvtoolu_abc" + + +def test_code_execution_20250825_tool_type(): + """Test that code_execution_20250825 tool type is handled correctly.""" + config = AnthropicConfig() + + tool = { + "type": "code_execution_20250825", + "name": "code_execution" + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert transformed_tool["type"] == "code_execution_20250825" + assert transformed_tool["name"] == "code_execution" + + +def test_allowed_callers_in_function_field(): + """Test that allowed_callers in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + }, + "allowed_callers": ["code_execution_20250825"] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "allowed_callers" in transformed_tool + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_field_preservation(): + """Test that input_examples field is preserved during tool transformation.""" + config = AnthropicConfig() + + # Test with top-level input_examples + tool_with_examples = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]} + }, + "required": ["location"] + } + }, + "input_examples": [ + {"location": "San Francisco, CA", "unit": "fahrenheit"}, + {"location": "Tokyo, Japan", "unit": "celsius"} + ] + } + + transformed_tool, _ = config._map_tool_helper(tool_with_examples) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + assert transformed_tool["input_examples"][0]["location"] == "San Francisco, CA" + + +def test_input_examples_beta_header(): + """Test that beta header is automatically added when input_examples is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test detection with input_examples + tools = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": {"type": "object", "properties": {}} + }, + "input_examples": [ + {"location": "San Francisco, CA"} + ] + } + ] + + is_examples_used = model_info.is_input_examples_used(tools) + assert is_examples_used is True + + # Test header generation + headers = model_info.get_anthropic_headers( + api_key="test-key", + input_examples_used=True + ) + + assert "anthropic-beta" in headers + assert "advanced-tool-use-2025-11-20" in headers["anthropic-beta"] + + +def test_input_examples_in_function_field(): + """Test that input_examples in function field is also preserved.""" + config = AnthropicConfig() + + # Test with function.input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + }, + "input_examples": [ + {"location": "Paris, France"}, + {"location": "London, UK"} + ] + } + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert len(transformed_tool["input_examples"]) == 2 + + +def test_input_examples_with_other_features(): + """Test that input_examples works alongside other tool features.""" + config = AnthropicConfig() + + # Tool with input_examples, defer_loading, and allowed_callers + tool = { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "input_examples": [ + {"sql": "SELECT * FROM users WHERE id = 1"} + ], + "defer_loading": True, + "allowed_callers": ["code_execution_20250825"] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + assert "input_examples" in transformed_tool + assert "defer_loading" in transformed_tool + assert "allowed_callers" in transformed_tool + assert transformed_tool["defer_loading"] is True + assert transformed_tool["allowed_callers"] == ["code_execution_20250825"] + + +def test_input_examples_empty_list_not_added(): + """Test that empty input_examples list is not added to transformed tool.""" + config = AnthropicConfig() + + # Tool with empty input_examples + tool = { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather information", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string"} + }, + "required": ["location"] + } + }, + "input_examples": [] + } + + transformed_tool, _ = config._map_tool_helper(tool) + assert transformed_tool is not None + # Empty list should not be added + assert "input_examples" not in transformed_tool or len(transformed_tool.get("input_examples", [])) == 0 + + +# ============ Effort Parameter Tests ============ + + +def test_effort_output_config_preservation(): + """Test that output_config with effort is preserved in transformation.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Analyze this code"}] + optional_params = { + "output_config": { + "effort": "medium" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "medium" + + +def test_effort_beta_header_injection(): + """Test that effort beta header is automatically added when output_config is detected.""" + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + model_info = AnthropicModelInfo() + + # Test with effort parameter + optional_params = { + "output_config": { + "effort": "low" + } + } + + effort_used = model_info.is_effort_used(optional_params=optional_params) + assert effort_used is True + + headers = model_info.get_anthropic_headers( + api_key="test-key", + effort_used=effort_used + ) + + assert "anthropic-beta" in headers + assert "effort-2025-11-24" in headers["anthropic-beta"] + + +def test_effort_validation(): + """Test that only valid effort values are accepted.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Test"}] + + # Valid values should work + for effort in ["high", "medium", "low"]: + optional_params = {"output_config": {"effort": effort}} + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + assert result["output_config"]["effort"] == effort + + # Invalid value should raise error + with pytest.raises(ValueError, match="Invalid effort value"): + optional_params = {"output_config": {"effort": "invalid"}} + config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + +def test_effort_with_claude_opus_45(): + """Test effort parameter works with Claude Opus 4.5 model.""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Complex analysis task"}] + optional_params = { + "output_config": { + "effort": "high" + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + assert "output_config" in result + assert result["output_config"]["effort"] == "high" + assert result["model"] == "claude-opus-4-5-20251101" + + +def test_effort_with_other_features(): + """Test effort works alongside other features (thinking, tools).""" + config = AnthropicConfig() + + messages = [{"role": "user", "content": "Use tools efficiently"}] + tools = [ + { + "type": "function", + "function": { + "name": "get_data", + "description": "Get data", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"} + }, + "required": ["query"] + } + } + } + ] + optional_params = { + "output_config": { + "effort": "low" + }, + "tools": tools, + "thinking": { + "type": "enabled", + "budget_tokens": 1000 + } + } + + result = config.transform_request( + model="claude-opus-4-5-20251101", + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={} + ) + + # Verify all features are present + assert "output_config" in result + assert result["output_config"]["effort"] == "low" + assert "tools" in result + assert len(result["tools"]) > 0 + assert "thinking" in result From db2c8e363175b6ca155fd159f166e2a7f17a0565 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 11:57:51 -0800 Subject: [PATCH 03/23] docs: initial doc cleanup --- .../index.md | 205 +++++++++++++++++- .../docs/providers/anthropic_tool_search.md | 2 +- 2 files changed, 194 insertions(+), 13 deletions(-) rename docs/my-website/blog/{anthropic_advanced_features => anthropic_opus_4_5_and_advanced_features}/index.md (79%) diff --git a/docs/my-website/blog/anthropic_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md similarity index 79% rename from docs/my-website/blog/anthropic_advanced_features/index.md rename to docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 71e5d9b192..0b0f4a5416 100644 --- a/docs/my-website/blog/anthropic_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -1,7 +1,7 @@ --- slug: anthropic_advanced_features -title: "Advanced Anthropic Features in LiteLLM: Tool Search, Programmatic Tool Calling, Input Examples, and Effort Control" -date: 2025-01-25T10:00:00 +title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" +date: 2025-11-25T10:00:00 authors: - name: Sameer Kankute title: SWE @ LiteLLM (LLM Translation) @@ -22,24 +22,205 @@ hide_table_of_contents: false import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. + +--- + +## Usage + + + + + +```python +import os +from litellm import completion + +# set env - [OPTIONAL] replace with your anthropic key +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" + +messages = [{"role": "user", "content": "Hey! how's it going?"}] + +## OPENAI /chat/completions API format +response = completion(model="claude-opus-4-5-20251101", messages=messages) +print(response) + +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: claude-opus-4-5-20251101 ### MODEL NAME sent to `litellm.completion()` ### + api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("ANTHROPIC_API_KEY") +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + + + +## Usage - Bedrock + :::info -This guide covers Anthropic's latest advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. +LiteLLM uses the boto3 library to authenticate with Bedrock. + +For more ways to authenticate with Bedrock, see the [Bedrock documentation](../../docs/providers/bedrock#authentication). ::: -We're excited to announce support for Anthropic's latest advanced features in LiteLLM! These powerful capabilities enable you to build more efficient, scalable, and cost-effective AI applications with Claude. + + -## Table of Contents -1. [Tool Search](#tool-search) -2. [Programmatic Tool Calling](#programmatic-tool-calling) -3. [Tool Input Examples](#tool-input-examples) -4. [Effort Parameter: Control Token Usage](#effort-parameter) -5. [Cost Tracking: Monitor Tool Search Usage](#cost-tracking) -6. [Combining Features](#combining-features) +```python +import os +from litellm import completion + +os.environ["AWS_ACCESS_KEY_ID"] = "" +os.environ["AWS_SECRET_ACCESS_KEY"] = "" +os.environ["AWS_REGION_NAME"] = "" + +## OPENAI /chat/completions API format +response = completion( + model="bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0", + messages=[{ "content": "Hello, how are you?","role": "user"}] +) +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input + model: bedrock/us.anthropic.claude-opus-4-5-20251101-v1:0 ### MODEL NAME sent to `litellm.completion()` ### + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + aws_region_name: os.environ/AWS_REGION_NAME +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/invoke' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/converse' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "messages": [{"role": "user", "content": "Hello, how are you?"}] + }' +``` + + + + ---- ## Tool Search {#tool-search} diff --git a/docs/my-website/docs/providers/anthropic_tool_search.md b/docs/my-website/docs/providers/anthropic_tool_search.md index 3d61022b26..7b9e7cfaa7 100644 --- a/docs/my-website/docs/providers/anthropic_tool_search.md +++ b/docs/my-website/docs/providers/anthropic_tool_search.md @@ -380,7 +380,7 @@ If Claude references a tool that isn't in your deferred tools list, you'll get a **When traditional tool calling is better:** - Less than 10 tools total - All tools are frequently used -- Very small tool definitions (<100 tokens total) +- Very small tool definitions (\<100 tokens total) ## Limitations From 44cde2e48fe6d2365f8cf3c972d1be8de7bbceec Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 12:03:01 -0800 Subject: [PATCH 04/23] Disable edit, delete, info, for dynamically generated spend tags --- .../tag_management/TagTable.test.tsx | 101 ++++++++++++++++++ .../components/tag_management/TagTable.tsx | 86 +++++++++++---- .../components/CreateTagModal.test.tsx | 64 +++++++++++ .../components/CreateTagModal.tsx | 44 ++------ .../components/tag_management/tag_info.tsx | 61 ++++++----- 5 files changed, 279 insertions(+), 77 deletions(-) create mode 100644 ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx create mode 100644 ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx new file mode 100644 index 0000000000..a56721787d --- /dev/null +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.test.tsx @@ -0,0 +1,101 @@ +import { render, screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import TagTable from "./TagTable"; +import { Tag } from "./types"; + +describe("TagTable", () => { + const mockOnEdit = vi.fn(); + const mockOnDelete = vi.fn(); + const mockOnSelectTag = vi.fn(); + + const mockTag: Tag = { + name: "test-tag", + description: "Test description", + models: ["model-1", "model-2"], + model_info: { + "model-1": "GPT-4", + "model-2": "Claude-3", + }, + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }; + + const mockDynamicSpendTag: Tag = { + name: "dynamic-spend-tag", + description: + "This is just a spend tag that was passed dynamically in a request. It does not control any LLM models.", + models: [], + created_at: "2024-01-01T00:00:00Z", + updated_at: "2024-01-01T00:00:00Z", + }; + + const defaultProps = { + data: [], + onEdit: mockOnEdit, + onDelete: mockOnDelete, + onSelectTag: mockOnSelectTag, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render", () => { + render(); + expect(screen.getByText("Tag Name")).toBeInTheDocument(); + expect(screen.getByText("Description")).toBeInTheDocument(); + expect(screen.getByText("Allowed Models")).toBeInTheDocument(); + expect(screen.getByText("Created")).toBeInTheDocument(); + expect(screen.getByText("Actions")).toBeInTheDocument(); + }); + + it("should display no tags found message when data is empty", () => { + render(); + expect(screen.getByText("No tags found")).toBeInTheDocument(); + }); + + it("should display tag name", () => { + render(); + expect(screen.getByText("test-tag")).toBeInTheDocument(); + }); + + it("should display tag description", () => { + render(); + expect(screen.getByText("Test description")).toBeInTheDocument(); + }); + + it("should display All Models badge when models array is empty", () => { + const tagWithNoModels: Tag = { + ...mockTag, + models: [], + }; + render(); + expect(screen.getByText("All Models")).toBeInTheDocument(); + }); + + it("should display formatted created date", () => { + render(); + const formattedDate = new Date(mockTag.created_at).toLocaleDateString(); + expect(screen.getByText(formattedDate)).toBeInTheDocument(); + }); + + it("should disable tag name button for dynamic spend tags", () => { + render(); + const tagButton = screen.getByRole("button", { name: "dynamic-spend-tag" }); + expect(tagButton).toBeDisabled(); + }); + + it("should disable edit icon for dynamic spend tags", () => { + render(); + const editIcon = screen.getByLabelText("Edit tag (disabled)"); + expect(editIcon).toBeInTheDocument(); + expect(editIcon).toHaveClass("cursor-not-allowed"); + }); + + it("should disable delete icon for dynamic spend tags", () => { + render(); + const deleteIcon = screen.getByLabelText("Delete tag (disabled)"); + expect(deleteIcon).toBeInTheDocument(); + expect(deleteIcon).toHaveClass("cursor-not-allowed"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx index aa43388893..ce28ac6e6f 100644 --- a/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/TagTable.tsx @@ -1,18 +1,4 @@ -import React from "react"; -import { - Table, - TableBody, - TableCell, - TableHead, - TableHeaderCell, - TableRow, - Icon, - Button, - Badge, - Text, -} from "@tremor/react"; -import { PencilAltIcon, TrashIcon, SwitchVerticalIcon, ChevronUpIcon, ChevronDownIcon } from "@heroicons/react/outline"; -import { Tooltip } from "antd"; +import { ChevronDownIcon, ChevronUpIcon, PencilAltIcon, SwitchVerticalIcon, TrashIcon } from "@heroicons/react/outline"; import { ColumnDef, flexRender, @@ -21,6 +7,20 @@ import { SortingState, useReactTable, } from "@tanstack/react-table"; +import { + Badge, + Button, + Icon, + Table, + TableBody, + TableCell, + TableHead, + TableHeaderCell, + TableRow, + Text, +} from "@tremor/react"; +import { Tooltip } from "antd"; +import React from "react"; import { Tag } from "./types"; interface TagTableProps { @@ -30,6 +30,9 @@ interface TagTableProps { onSelectTag: (tagName: string) => void; } +const DYNAMIC_SPEND_TAG_DESCRIPTION = + "This is just a spend tag that was passed dynamically in a request. It does not control any LLM models."; + const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag }) => { const [sorting, setSorting] = React.useState([{ id: "created_at", desc: true }]); @@ -39,14 +42,20 @@ const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag accessorKey: "name", cell: ({ row }) => { const tag = row.original; + const isDynamicSpendTag = tag.description === DYNAMIC_SPEND_TAG_DESCRIPTION; return (
- + @@ -68,7 +77,7 @@ const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag }, }, { - header: "Allowed LLMs", + header: "Allowed Models", accessorKey: "models", cell: ({ row }) => { const tag = row.original; @@ -102,13 +111,50 @@ const TagTable: React.FC = ({ data, onEdit, onDelete, onSelectTag }, { id: "actions", - header: "", + header: "Actions", cell: ({ row }) => { const tag = row.original; + const isDynamicSpendTag = tag.description === DYNAMIC_SPEND_TAG_DESCRIPTION; return (
- onEdit(tag)} className="cursor-pointer" /> - onDelete(tag.name)} className="cursor-pointer" /> + {isDynamicSpendTag ? ( + + + + ) : ( + + onEdit(tag)} + className="cursor-pointer hover:text-blue-500" + /> + + )} + {isDynamicSpendTag ? ( + + + + ) : ( + + onDelete(tag.name)} + className="cursor-pointer hover:text-red-500" + /> + + )}
); }, diff --git a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx new file mode 100644 index 0000000000..997faf4a00 --- /dev/null +++ b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.test.tsx @@ -0,0 +1,64 @@ +import { render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; +import { beforeEach, describe, expect, it, vi } from "vitest"; +import CreateTagModal from "./CreateTagModal"; + +describe("CreateTagModal", () => { + const mockOnCancel = vi.fn(); + const mockOnSubmit = vi.fn(); + const mockAvailableModels = [ + { + model_name: "GPT-4", + litellm_params: { model: "gpt-4" }, + model_info: { id: "model-1" }, + }, + { + model_name: "Claude-3", + litellm_params: { model: "claude-3" }, + model_info: { id: "model-2" }, + }, + ]; + + const defaultProps = { + visible: true, + onCancel: mockOnCancel, + onSubmit: mockOnSubmit, + availableModels: mockAvailableModels, + }; + + beforeEach(() => { + vi.clearAllMocks(); + }); + + it("should render the modal", () => { + render(); + expect(screen.getByRole("dialog")).toBeInTheDocument(); + expect(screen.getByText("Create New Tag")).toBeInTheDocument(); + }); + + it("should submit form with required tag name", async () => { + const user = userEvent.setup(); + render(); + + const tagNameInput = screen.getByLabelText("Tag Name"); + await user.type(tagNameInput, "test-tag"); + + const submitButton = screen.getByRole("button", { name: /Create Tag/i }); + await user.click(submitButton); + + expect(mockOnSubmit).toHaveBeenCalledWith({ + tag_name: "test-tag", + }); + }); + + it("should not submit form when tag name is missing", async () => { + const user = userEvent.setup(); + render(); + + const submitButton = screen.getByRole("button", { name: /Create Tag/i }); + await user.click(submitButton); + + // Form validation should prevent submission + expect(mockOnSubmit).not.toHaveBeenCalled(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx index 4d1909abd9..3412d68452 100644 --- a/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/components/CreateTagModal.tsx @@ -1,9 +1,9 @@ -import React from "react"; -import { Button, TextInput, Accordion, AccordionHeader, AccordionBody, Title } from "@tremor/react"; -import { Modal, Form, Select as Select2, Tooltip, Input } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; -import NumericalInput from "../../shared/numerical_input"; +import { Accordion, AccordionBody, AccordionHeader, Button, TextInput, Title } from "@tremor/react"; +import { Form, Input, Modal, Select as Select2, Tooltip } from "antd"; +import React from "react"; import BudgetDurationDropdown from "../../common_components/budget_duration_dropdown"; +import NumericalInput from "../../shared/numerical_input"; interface ModelInfo { model_name: string; @@ -22,12 +22,7 @@ interface CreateTagModalProps { availableModels: ModelInfo[]; } -const CreateTagModal: React.FC = ({ - visible, - onCancel, - onSubmit, - availableModels, -}) => { +const CreateTagModal: React.FC = ({ visible, onCancel, onSubmit, availableModels }) => { const [form] = Form.useForm(); const handleFinish = (values: any) => { @@ -41,25 +36,9 @@ const CreateTagModal: React.FC = ({ }; return ( - -
- + + + @@ -70,15 +49,15 @@ const CreateTagModal: React.FC = ({ - Allowed Models{" "} - + Allowed Models + } name="allowed_llms" > - + {availableModels.map((model) => (
@@ -150,4 +129,3 @@ const CreateTagModal: React.FC = ({ }; export default CreateTagModal; - diff --git a/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx b/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx index 60cde134e6..1c66a107db 100644 --- a/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx +++ b/ui/litellm-dashboard/src/components/tag_management/tag_info.tsx @@ -1,5 +1,15 @@ import React, { useState, useEffect } from "react"; -import { Card, Text, Title, Button, Badge, Accordion, AccordionHeader, AccordionBody, Title as TremorTitle } from "@tremor/react"; +import { + Card, + Text, + Title, + Button, + Badge, + Accordion, + AccordionHeader, + AccordionBody, + Title as TremorTitle, +} from "@tremor/react"; import { Form, Input, Select as Select2, Tooltip } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; import { fetchUserModels } from "../organisms/create_key_button"; @@ -131,7 +141,7 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken, - + @@ -141,15 +151,15 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken, - Allowed LLMs{" "} - + Allowed Models + } name="models" > - + {userModels.map((modelId) => ( {getModelDisplayName(modelId)} @@ -228,7 +238,7 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken, {tagDetails.description || "-"}
- Allowed LLMs + Allowed Models
{!tagDetails.models || tagDetails.models.length === 0 ? ( All Models @@ -256,30 +266,33 @@ const TagInfoView: React.FC = ({ tagId, onClose, accessToken, Budget & Rate Limits
- {tagDetails.litellm_budget_table.max_budget !== undefined && tagDetails.litellm_budget_table.max_budget !== null && ( -
- Max Budget - ${tagDetails.litellm_budget_table.max_budget} -
- )} + {tagDetails.litellm_budget_table.max_budget !== undefined && + tagDetails.litellm_budget_table.max_budget !== null && ( +
+ Max Budget + ${tagDetails.litellm_budget_table.max_budget} +
+ )} {tagDetails.litellm_budget_table.budget_duration && (
Budget Duration {tagDetails.litellm_budget_table.budget_duration}
)} - {tagDetails.litellm_budget_table.tpm_limit !== undefined && tagDetails.litellm_budget_table.tpm_limit !== null && ( -
- TPM Limit - {tagDetails.litellm_budget_table.tpm_limit.toLocaleString()} -
- )} - {tagDetails.litellm_budget_table.rpm_limit !== undefined && tagDetails.litellm_budget_table.rpm_limit !== null && ( -
- RPM Limit - {tagDetails.litellm_budget_table.rpm_limit.toLocaleString()} -
- )} + {tagDetails.litellm_budget_table.tpm_limit !== undefined && + tagDetails.litellm_budget_table.tpm_limit !== null && ( +
+ TPM Limit + {tagDetails.litellm_budget_table.tpm_limit.toLocaleString()} +
+ )} + {tagDetails.litellm_budget_table.rpm_limit !== undefined && + tagDetails.litellm_budget_table.rpm_limit !== null && ( +
+ RPM Limit + {tagDetails.litellm_budget_table.rpm_limit.toLocaleString()} +
+ )}
)} From be712908a3ea1f625826b261e5d55ecd51890ee8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 25 Nov 2025 12:20:39 -0800 Subject: [PATCH 05/23] [Feat] Add OpenAI compatible bedrock imported models. - qwen etc (#17097) * test_bedrock_openai_imported_model * AmazonBedrockOpenAIConfig * add openai route for bedrock * docs fix * fix code qa check --- docs/my-website/docs/providers/bedrock.md | 202 +--------- .../docs/providers/bedrock_imported.md | 369 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 3 + .../amazon_openai_transformation.py | 186 +++++++++ litellm/llms/bedrock/common_utils.py | 18 +- .../prompt_security/prompt_security.py | 19 +- litellm/proxy/proxy_config.yaml | 19 +- litellm/utils.py | 12 +- .../test_bedrock_completion.py | 97 +++++ 10 files changed, 699 insertions(+), 227 deletions(-) create mode 100644 docs/my-website/docs/providers/bedrock_imported.md create mode 100644 litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index f0b89615a0..9e22f67527 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -7,7 +7,7 @@ ALL Bedrock models (Anthropic, Meta, Deepseek, Mistral, Amazon, etc.) are Suppor | Property | Details | |-------|-------| | Description | Amazon Bedrock is a fully managed service that offers a choice of high-performing foundation models (FMs). | -| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models) | +| Provider Route on LiteLLM | `bedrock/`, [`bedrock/converse/`](#set-converse--invoke-route), [`bedrock/invoke/`](#set-invoke-route), [`bedrock/converse_like/`](#calling-via-internal-proxy), [`bedrock/llama/`](#deepseek-not-r1), [`bedrock/deepseek_r1/`](#deepseek-r1), [`bedrock/qwen3/`](#qwen3-imported-models), [`bedrock/openai/`](./bedrock_imported.md#openai-compatible-imported-models-qwen-25-vl-etc) | | Provider Doc | [Amazon Bedrock ↗](https://docs.aws.amazon.com/bedrock/latest/userguide/what-is-bedrock.html) | | Supported OpenAI Endpoints | `/chat/completions`, `/completions`, `/embeddings`, `/images/generations` | | Rerank Endpoint | `/rerank` | @@ -1598,206 +1598,6 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ -## Bedrock Imported Models (Deepseek, Deepseek R1) - -### Deepseek R1 - -This is a separate route, as the chat template is different. - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/deepseek_r1/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - - -### Deepseek (not R1) - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/llama/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | - - - -Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec - - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], -) -``` - - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: DeepSeek-R1-Distill-Llama-70B - litellm_params: - model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - -### Qwen3 Imported Models - -| Property | Details | -|----------|---------| -| Provider Route | `bedrock/qwen3/{model_arn}` | -| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | - - - - -```python -from litellm import completion -import os - -response = completion( - model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} - messages=[{"role": "user", "content": "Tell me a joke"}], - max_tokens=100, - temperature=0.7 -) -``` - - - - - -**1. Add to config** - -```yaml -model_list: - - model_name: Qwen3-32B - litellm_params: - model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model - -``` - -**2. Start proxy** - -```bash -litellm --config /path/to/config.yaml - -# RUNNING at http://0.0.0.0:4000 -``` - -**3. Test it!** - -```bash -curl --location 'http://0.0.0.0:4000/chat/completions' \ - --header 'Authorization: Bearer sk-1234' \ - --header 'Content-Type: application/json' \ - --data '{ - "model": "Qwen3-32B", # 👈 the 'model_name' in config - "messages": [ - { - "role": "user", - "content": "what llm are you" - } - ], - }' -``` - - - - ### OpenAI GPT OSS | Property | Details | diff --git a/docs/my-website/docs/providers/bedrock_imported.md b/docs/my-website/docs/providers/bedrock_imported.md new file mode 100644 index 0000000000..8b0dd721c3 --- /dev/null +++ b/docs/my-website/docs/providers/bedrock_imported.md @@ -0,0 +1,369 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Bedrock Imported Models + +Bedrock Imported Models (Deepseek, Deepseek R1, Qwen, OpenAI-compatible models) + +### Deepseek R1 + +This is a separate route, as the chat template is different. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/deepseek_r1/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/deepseek_r1/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/deepseek_r1/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + + +### Deepseek (not R1) + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/llama/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Deepseek Bedrock Imported Model](https://aws.amazon.com/blogs/machine-learning/deploy-deepseek-r1-distilled-llama-models-with-amazon-bedrock-custom-model-import/) | + + + +Use this route to call Bedrock Imported Models that follow the `llama` Invoke Request / Response spec + + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n", # bedrock/llama/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], +) +``` + + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: DeepSeek-R1-Distill-Llama-70B + litellm_params: + model: bedrock/llama/arn:aws:bedrock:us-east-1:086734376398:imported-model/r4c4kewx2s0n + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "DeepSeek-R1-Distill-Llama-70B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### Qwen3 Imported Models + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/qwen3/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html), [Qwen3 Models](https://aws.amazon.com/about-aws/whats-new/2025/09/qwen3-models-fully-managed-amazon-bedrock/) | + + + + +```python +from litellm import completion +import os + +response = completion( + model="bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model", # bedrock/qwen3/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=100, + temperature=0.7 +) +``` + + + + + +**1. Add to config** + +```yaml +model_list: + - model_name: Qwen3-32B + litellm_params: + model: bedrock/qwen3/arn:aws:bedrock:us-east-1:086734376398:imported-model/your-qwen3-model + +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "Qwen3-32B", # 👈 the 'model_name' in config + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + }' +``` + + + + +### OpenAI-Compatible Imported Models (Qwen 2.5 VL, etc.) + +Use this route for Bedrock imported models that follow the **OpenAI Chat Completions API spec**. This includes models like Qwen 2.5 VL that accept OpenAI-formatted messages with support for vision (images), tool calling, and other OpenAI features. + +| Property | Details | +|----------|---------| +| Provider Route | `bedrock/openai/{model_arn}` | +| Provider Documentation | [Bedrock Imported Models](https://docs.aws.amazon.com/bedrock/latest/userguide/model-customization-import-model.html) | +| Supported Features | Vision (images), tool calling, streaming, system messages | + +#### LiteLLMSDK Usage + +**Basic Usage** + +```python +from litellm import completion + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", # bedrock/openai/{your-model-arn} + messages=[{"role": "user", "content": "Tell me a joke"}], + max_tokens=300, + temperature=0.5 +) +``` + +**With Vision (Images)** + +```python +import base64 +from litellm import completion + +# Load and encode image +with open("image.jpg", "rb") as f: + image_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What's in this image?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +**Comparing Multiple Images** + +```python +import base64 +from litellm import completion + +# Load images +with open("image1.jpg", "rb") as f: + image1_base64 = base64.b64encode(f.read()).decode("utf-8") +with open("image2.jpg", "rb") as f: + image2_base64 = base64.b64encode(f.read()).decode("utf-8") + +response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z", + messages=[ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "Spot the difference between these two images?"}, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image1_base64}"} + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{image2_base64}"} + } + ] + } + ], + max_tokens=300, + temperature=0.5 +) +``` + +#### LiteLLM Proxy Usage (AI Gateway) + +**1. Add to config** + +```yaml +model_list: + - model_name: qwen-25vl-72b + litellm_params: + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z +``` + +**2. Start proxy** + +```bash +litellm --config /path/to/config.yaml + +# RUNNING at http://0.0.0.0:4000 +``` + +**3. Test it!** + +Basic text request: + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ], + "max_tokens": 300 + }' +``` + +With vision (image): + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ + --header 'Authorization: Bearer sk-1234' \ + --header 'Content-Type: application/json' \ + --data '{ + "model": "qwen-25vl-72b", + "messages": [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images." + }, + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": "data:image/jpeg;base64,/9j/4AAQSkZ..."} + } + ] + } + ], + "max_tokens": 300, + "temperature": 0.5 + }' +``` \ No newline at end of file diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 6fa5fdeced..104c7541d6 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -530,6 +530,7 @@ const sidebars = { items: [ "providers/bedrock", "providers/bedrock_embedding", + "providers/bedrock_imported", "providers/bedrock_image_gen", "providers/bedrock_rerank", "providers/bedrock_agentcore", diff --git a/litellm/__init__.py b/litellm/__init__.py index 768ba39a47..0048f4c29c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1225,6 +1225,9 @@ from .llms.bedrock.chat.invoke_transformations.amazon_titan_transformation impor from .llms.bedrock.chat.invoke_transformations.base_invoke_transformation import ( AmazonInvokeConfig, ) +from .llms.bedrock.chat.invoke_transformations.amazon_openai_transformation import ( + AmazonBedrockOpenAIConfig, +) from .llms.bedrock.image.amazon_stability1_transformation import AmazonStabilityConfig from .llms.bedrock.image.amazon_stability3_transformation import AmazonStability3Config diff --git a/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py new file mode 100644 index 0000000000..ee07b71ef1 --- /dev/null +++ b/litellm/llms/bedrock/chat/invoke_transformations/amazon_openai_transformation.py @@ -0,0 +1,186 @@ +""" +Transformation for Bedrock imported models that use OpenAI Chat Completions format. + +Use this for models imported into Bedrock that accept the OpenAI API format. +Model format: bedrock/openai/ + +Example: bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123 +""" + +from typing import TYPE_CHECKING, Any, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM +from litellm.llms.bedrock.common_utils import BedrockError +from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig +from litellm.types.llms.openai import AllMessageValues + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class AmazonBedrockOpenAIConfig(OpenAIGPTConfig, BaseAWSLLM): + """ + Configuration for Bedrock imported models that use OpenAI Chat Completions format. + + This class handles the transformation of requests and responses for Bedrock + imported models that accept the OpenAI API format directly. + + Inherits from OpenAIGPTConfig to leverage standard OpenAI parameter handling + and response transformation, while adding Bedrock-specific URL generation + and AWS request signing. + + Usage: + model = "bedrock/openai/arn:aws:bedrock:us-east-1:123456789012:imported-model/abc123" + """ + + def __init__(self, **kwargs): + OpenAIGPTConfig.__init__(self, **kwargs) + BaseAWSLLM.__init__(self, **kwargs) + + @property + def custom_llm_provider(self) -> Optional[str]: + return "bedrock" + + def _get_openai_model_id(self, model: str) -> str: + """ + Extract the actual model ID from the LiteLLM model name. + + Input format: bedrock/openai/ + Returns: + """ + # Remove bedrock/ prefix if present + if model.startswith("bedrock/"): + model = model[8:] + + # Remove openai/ prefix + if model.startswith("openai/"): + model = model[7:] + + return model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the Bedrock invoke endpoint. + + Uses the standard Bedrock invoke endpoint format. + """ + model_id = self._get_openai_model_id(model) + + # Get AWS region + aws_region_name = self._get_aws_region_name( + optional_params=optional_params, model=model + ) + + # Get runtime endpoint + aws_bedrock_runtime_endpoint = optional_params.get( + "aws_bedrock_runtime_endpoint", None + ) + endpoint_url, proxy_endpoint_url = self.get_runtime_endpoint( + api_base=api_base, + aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint, + aws_region_name=aws_region_name, + ) + + # Build the invoke URL + if stream: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke-with-response-stream" + else: + endpoint_url = f"{endpoint_url}/model/{model_id}/invoke" + + return endpoint_url + + def sign_request( + self, + headers: dict, + optional_params: dict, + request_data: dict, + api_base: str, + api_key: Optional[str] = None, + model: Optional[str] = None, + stream: Optional[bool] = None, + fake_stream: Optional[bool] = None, + ) -> Tuple[dict, Optional[bytes]]: + """ + Sign the request using AWS Signature Version 4. + """ + return self._sign_request( + service_name="bedrock", + headers=headers, + optional_params=optional_params, + request_data=request_data, + api_base=api_base, + api_key=api_key, + model=model, + stream=stream, + fake_stream=fake_stream, + ) + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to OpenAI Chat Completions format for Bedrock imported models. + + Removes AWS-specific params and stream param (handled separately in URL), + then delegates to parent class for standard OpenAI request transformation. + """ + # Remove stream from optional_params as it's handled separately in URL + optional_params.pop("stream", None) + + # Remove AWS-specific params that shouldn't be in the request body + inference_params = { + k: v + for k, v in optional_params.items() + if k not in self.aws_authentication_params + } + + # Use parent class transform_request for OpenAI format + return super().transform_request( + model=self._get_openai_model_id(model), + messages=messages, + optional_params=inference_params, + litellm_params=litellm_params, + headers=headers, + ) + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate the environment and return headers. + + For Bedrock, we don't need Bearer token auth since we use AWS SigV4. + """ + return headers + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BedrockError: + """Return the appropriate error class for Bedrock.""" + return BedrockError(status_code=status_code, message=error_message) diff --git a/litellm/llms/bedrock/common_utils.py b/litellm/llms/bedrock/common_utils.py index baaec99653..35d3d736a1 100644 --- a/litellm/llms/bedrock/common_utils.py +++ b/litellm/llms/bedrock/common_utils.py @@ -403,6 +403,9 @@ class BedrockModelInfo(BaseLLMModelInfo): if model.startswith("invoke/"): model = model.split("/", 1)[1] + if model.startswith("openai/"): + model = model.split("/", 1)[1] + return model @staticmethod @@ -446,12 +449,12 @@ class BedrockModelInfo(BaseLLMModelInfo): @staticmethod def get_bedrock_route( model: str, - ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke"]: + ) -> Literal["converse", "invoke", "converse_like", "agent", "agentcore", "async_invoke", "openai"]: """ Get the bedrock route for the given model. """ route_mappings: Dict[ - str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke"] + str, Literal["invoke", "converse_like", "converse", "agent", "agentcore", "async_invoke", "openai"] ] = { "invoke/": "invoke", "converse_like/": "converse_like", @@ -459,6 +462,7 @@ class BedrockModelInfo(BaseLLMModelInfo): "agent/": "agent", "agentcore/": "agentcore", "async_invoke/": "async_invoke", + "openai/": "openai", } # Check explicit routes first @@ -517,6 +521,14 @@ class BedrockModelInfo(BaseLLMModelInfo): """ return "async_invoke/" in model + @staticmethod + def _explicit_openai_route(model: str) -> bool: + """ + Check if the model is an explicit openai route. + Used for Bedrock imported models that use OpenAI Chat Completions format. + """ + return "openai/" in model + @staticmethod def get_bedrock_provider_config_for_messages_api( model: str, @@ -566,6 +578,8 @@ def get_bedrock_chat_config(model: str): # Handle explicit routes first if bedrock_route == "converse" or bedrock_route == "converse_like": return litellm.AmazonConverseConfig() + elif bedrock_route == "openai": + return litellm.AmazonBedrockOpenAIConfig() elif bedrock_route == "agent": from litellm.llms.bedrock.chat.invoke_agent.transformation import ( AmazonInvokeAgentConfig, diff --git a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py index daee50f30c..23b9da4714 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py +++ b/litellm/proxy/guardrails/guardrail_hooks/prompt_security/prompt_security.py @@ -1,13 +1,18 @@ -import os -import re import asyncio import base64 +import os +import re from typing import TYPE_CHECKING, Any, AsyncGenerator, Optional, Type, Union + from fastapi import HTTPException + from litellm import DualCache from litellm._logging import verbose_proxy_logger from litellm.integrations.custom_guardrail import CustomGuardrail -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, httpxSpecialProvider +from litellm.llms.custom_httpx.http_handler import ( + get_async_httpx_client, + httpxSpecialProvider, +) from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ( Choices, @@ -15,7 +20,7 @@ from litellm.types.utils import ( EmbeddingResponse, ImageResponse, ModelResponse, - ModelResponseStream + ModelResponseStream, ) if TYPE_CHECKING: @@ -267,8 +272,10 @@ class PromptSecurityGuardrail(CustomGuardrail): content = msg.get('content', '') # Handle both string and list content types if isinstance(content, str): - if content.startswith('### '): return False - if '"follow_ups": [' in content: return False + if content.startswith('### '): + return False + if '"follow_ups": [' in content: + return False return True messages = list(filter(lambda msg: good_msg(msg), messages)) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 014bcdc167..26e867dc33 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -1,22 +1,7 @@ model_list: - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 + - model_name: qwen-25vl-72b litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-east-1 - custom_llm_provider: bedrock - - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 - litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-west-2 - custom_llm_provider: bedrock - - model_name: bedrock/* - litellm_params: - model: bedrock/* - custom_llm_provider: bedrock - aws_region_name: us-west-2 - - model_name: runwayml/* - litellm_params: - model: runwayml/* + model: bedrock/openai/arn:aws:bedrock:us-east-1:046319184608:imported-model/0m2lasirsp6z diff --git a/litellm/utils.py b/litellm/utils.py index f683a59f50..1b4df68995 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3719,7 +3719,17 @@ def get_optional_params( # noqa: PLR0915 else False ), ) - + elif bedrock_route == "openai": + optional_params = litellm.AmazonBedrockOpenAIConfig().map_openai_params( + model=model, + non_default_params=non_default_params, + optional_params=optional_params, + drop_params=( + drop_params + if drop_params is not None and isinstance(drop_params, bool) + else False + ), + ) elif "anthropic" in bedrock_base_model and bedrock_route == "invoke": if bedrock_base_model.startswith("anthropic.claude-3"): optional_params = ( diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 9242950daa..f43e939c68 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3434,3 +3434,100 @@ async def test_bedrock_streaming_passthrough_test1(monkeypatch): print(mock_callback.call_args.kwargs.keys()) assert "standard_logging_object" in mock_callback.call_args.kwargs["kwargs"] assert "response_cost" in mock_callback.call_args.kwargs["kwargs"] + + +def test_bedrock_openai_imported_model(): + """ + Test that Bedrock imported models using OpenAI format work correctly. + + This test validates: + 1. The request body follows OpenAI Chat Completions format + 2. The URL is correctly constructed for Bedrock invoke endpoint + 3. Messages with system, user roles and image_url content are preserved + """ + from litellm.llms.custom_httpx.http_handler import HTTPHandler + + client = HTTPHandler() + + # Sample base64 image data (truncated for test) + sample_base64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + + messages = [ + { + "role": "system", + "content": "You are a helpful assistant that can analyze images.", + }, + { + "role": "user", + "content": [ + { + "type": "text", + "text": "Spot the difference between the two images?", + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/jpeg;base64,{sample_base64}"}, + }, + ], + }, + ] + + with patch.object(client, "post") as mock_post: + try: + response = completion( + model="bedrock/openai/arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy", + messages=messages, + max_tokens=300, + temperature=0.5, + client=client, + ) + except Exception as e: + print(f"Exception (expected during mock): {e}") + + mock_post.assert_called_once() + + # Validate URL + url = mock_post.call_args.kwargs["url"] + print(f"URL: {url}") + assert "bedrock-runtime.us-east-1.amazonaws.com" in url + assert "arn:aws:bedrock:us-east-1:117159858402:imported-model/m4gc1mrfuddy" in url + assert "/invoke" in url + + # Validate request body follows OpenAI format + request_body = json.loads(mock_post.call_args.kwargs["data"]) + print(f"Request body: {json.dumps(request_body, indent=2)}") + + # Check messages structure + assert "messages" in request_body + assert len(request_body["messages"]) == 2 + + # Check system message + system_msg = request_body["messages"][0] + assert system_msg["role"] == "system" + assert "helpful assistant" in system_msg["content"] + + # Check user message with image content + user_msg = request_body["messages"][1] + assert user_msg["role"] == "user" + assert isinstance(user_msg["content"], list) + assert len(user_msg["content"]) == 3 + + # Check text content + assert user_msg["content"][0]["type"] == "text" + assert "Spot the difference" in user_msg["content"][0]["text"] + + # Check image_url content + assert user_msg["content"][1]["type"] == "image_url" + assert "image_url" in user_msg["content"][1] + assert user_msg["content"][1]["image_url"]["url"].startswith("data:image/jpeg;base64,") + + assert user_msg["content"][2]["type"] == "image_url" + assert "image_url" in user_msg["content"][2] + + # Check max_tokens and temperature + assert request_body["max_tokens"] == 300 + assert request_body["temperature"] == 0.5 From 52f1bf1a800bf76d3896fdf9d714f160f45d408b Mon Sep 17 00:00:00 2001 From: YutaSaito <36355491+uc4w6c@users.noreply.github.com> Date: Wed, 26 Nov 2025 07:33:38 +0900 Subject: [PATCH 06/23] fix: missing await (#17103) --- litellm/proxy/_experimental/mcp_server/mcp_server_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 54c79fc696..25b5211464 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1976,7 +1976,7 @@ class MCPServerManager: verbose_logger.debug( f"Adding server to registry: {server.server_id} ({server.server_name})" ) - self.add_update_server(server) + await self.add_update_server(server) verbose_logger.debug( f"Registry now contains {len(self.get_registry())} servers" @@ -2270,7 +2270,7 @@ class MCPServerManager: server.status = "unhealthy" ## try adding server to registry to get error try: - self.add_update_server(server) + await self.add_update_server(server) except Exception as e: server.health_check_error = str(e) server.health_check_error = "Server is not in in memory registry yet. This could be a temporary sync issue." From f3d577592023d374ce532058c1634e63a69f5009 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 14:40:17 -0800 Subject: [PATCH 07/23] fix: fix doc load issue --- docs/my-website/docs/providers/anthropic_effort.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/my-website/docs/providers/anthropic_effort.md b/docs/my-website/docs/providers/anthropic_effort.md index d1116ad5be..0015162a95 100644 --- a/docs/my-website/docs/providers/anthropic_effort.md +++ b/docs/my-website/docs/providers/anthropic_effort.md @@ -1,3 +1,6 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + # Anthropic Effort Parameter Control how many tokens Claude uses when responding with the `effort` parameter, trading off between response thoroughness and token efficiency. From db587926a473f51a21bc96935d10495ae7fdab7e Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 14:46:46 -0800 Subject: [PATCH 08/23] Sorting changes, pending tests and loading state --- .../src/components/view_users/columns.tsx | 14 +++++++-- .../src/components/view_users/table.tsx | 31 +++++++++---------- 2 files changed, 27 insertions(+), 18 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_users/columns.tsx b/ui/litellm-dashboard/src/components/view_users/columns.tsx index 32bfa0ed6d..20df4fc246 100644 --- a/ui/litellm-dashboard/src/components/view_users/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_users/columns.tsx @@ -22,10 +22,12 @@ export const columns = ( handleUserClick: (userId: string, openInEditMode?: boolean) => void, selectionOptions?: SelectionOptions, ): ColumnDef[] => { + // Backend sortable columns: user_id, user_email, created_at, spend, user_alias, user_role const baseColumns: ColumnDef[] = [ { header: "User ID", accessorKey: "user_id", + enableSorting: true, cell: ({ row }) => ( {row.original.user_id ? `${row.original.user_id.slice(0, 7)}...` : "-"} @@ -35,16 +37,19 @@ export const columns = ( { header: "Email", accessorKey: "user_email", + enableSorting: true, cell: ({ row }) => {row.original.user_email || "-"}, }, { header: "Global Proxy Role", accessorKey: "user_role", + enableSorting: true, cell: ({ row }) => {possibleUIRoles?.[row.original.user_role]?.ui_label || "-"}, }, { header: "Spend (USD)", accessorKey: "spend", + enableSorting: true, cell: ({ row }) => ( {row.original.spend ? formatNumberWithCommas(row.original.spend, 4) : "-"} ), @@ -52,6 +57,7 @@ export const columns = ( { header: "Budget (USD)", accessorKey: "max_budget", + enableSorting: false, cell: ({ row }) => ( {row.original.max_budget !== null ? row.original.max_budget : "Unlimited"} ), @@ -66,6 +72,7 @@ export const columns = (
), accessorKey: "sso_user_id", + enableSorting: false, cell: ({ row }) => ( {row.original.sso_user_id !== null ? row.original.sso_user_id : "-"} ), @@ -73,6 +80,7 @@ export const columns = ( { header: "API Keys", accessorKey: "key_count", + enableSorting: false, cell: ({ row }) => ( {row.original.key_count > 0 ? ( @@ -90,7 +98,7 @@ export const columns = ( { header: "Created At", accessorKey: "created_at", - sortingFn: "datetime", + enableSorting: true, cell: ({ row }) => ( {row.original.created_at ? new Date(row.original.created_at).toLocaleDateString() : "-"} @@ -100,7 +108,7 @@ export const columns = ( { header: "Updated At", accessorKey: "updated_at", - sortingFn: "datetime", + enableSorting: false, cell: ({ row }) => ( {row.original.updated_at ? new Date(row.original.updated_at).toLocaleDateString() : "-"} @@ -110,6 +118,7 @@ export const columns = ( { id: "actions", header: "Actions", + enableSorting: false, cell: ({ row }) => (
@@ -148,6 +157,7 @@ export const columns = ( return [ { id: "select", + enableSorting: false, header: () => ( { + onSortingChange: (updaterOrValue: any) => { + const newSorting = typeof updaterOrValue === "function" ? updaterOrValue(sorting) : updaterOrValue; setSorting(newSorting); - if (newSorting.length > 0) { + if (newSorting && Array.isArray(newSorting) && newSorting.length > 0 && newSorting[0]) { const sortState = newSorting[0]; - const sortBy = sortState.id; - const sortOrder = sortState.desc ? "desc" : "asc"; - onSortChange?.(sortBy, sortOrder); + if (sortState.id) { + const sortBy = sortState.id; + const sortOrder = sortState.desc ? "desc" : "asc"; + onSortChange?.(sortBy, sortOrder); + } + } else { + // Reset to default sort when no sorting is selected + onSortChange?.("created_at", "desc"); } }, getCoreRowModel: getCoreRowModel(), - getSortedRowModel: getSortedRowModel(), + manualSorting: true, enableSorting: true, }); @@ -403,7 +402,7 @@ export function UserDataTable({ header.id === "actions" ? "sticky right-0 bg-white shadow-[-4px_0_8px_-6px_rgba(0,0,0,0.1)]" : "" - }`} + } ${header.column.getCanSort() ? "cursor-pointer hover:bg-gray-50" : ""}`} onClick={header.column.getToggleSortingHandler()} >
@@ -412,7 +411,7 @@ export function UserDataTable({ ? null : flexRender(header.column.columnDef.header, header.getContext())}
- {header.id !== "actions" && ( + {header.id !== "actions" && header.column.getCanSort() && (
{header.column.getIsSorted() ? ( { From c0288d81aa4ef31b9e1f529ce032c827c3087c9f Mon Sep 17 00:00:00 2001 From: Sam Chou Date: Tue, 25 Nov 2025 14:49:12 -0800 Subject: [PATCH 09/23] Fix bedrock claude opus 4.5 inference profile - only global currently (#17101) --- .../model_prices_and_context_window_backup.json | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index b3dc11e206..f9cc8bfa06 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2687,7 +2687,7 @@ "/v1/audio/transcriptions" ] }, - "azure/gpt-5.1-2025-11-13": { + "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2723,7 +2723,7 @@ "supports_service_tier": true, "supports_vision": true }, - "azure/gpt-5.1-chat-2025-11-13": { + "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2758,7 +2758,7 @@ "supports_tool_choice": false, "supports_vision": true }, - "azure/gpt-5.1-codex-2025-11-13": { + "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -19521,7 +19521,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/deepseek-v3.1:671b-cloud" : { + "ollama/deepseek-v3.1:671b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 163840, @@ -19531,7 +19531,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:120b-cloud" : { + "ollama/gpt-oss:120b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -19541,7 +19541,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:20b-cloud" : { + "ollama/gpt-oss:20b-cloud": { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -22037,7 +22037,6 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, - "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, @@ -23232,7 +23231,7 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "us.anthropic.claude-opus-4-5-20251101-v1:0": { + "global.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, From 8637d74e170b52c23af6d780f9d7d20eea167069 Mon Sep 17 00:00:00 2001 From: Kerem Turgutlu Date: Wed, 26 Nov 2025 01:50:17 +0300 Subject: [PATCH 10/23] include `server_tool_use` in streaming usage (#16826) * include server_tool_use in streaming usage * add test --- .../streaming_chunk_builder_utils.py | 12 ++- .../streaming_chunk_builder_utils.py | 3 +- .../test_streaming_chunk_builder_utils.py | 81 +++++++++++++++++++ 3 files changed, 94 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py index ddcf81b5ba..c332e5f88f 100644 --- a/litellm/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/litellm_core_utils/streaming_chunk_builder_utils.py @@ -18,6 +18,7 @@ from litellm.types.utils import ( ModelResponseStream, PromptTokensDetailsWrapper, Usage, + ServerToolUse ) from litellm.utils import print_verbose, token_counter @@ -418,7 +419,8 @@ class ChunkProcessor: ## anthropic prompt caching information ## cache_creation_input_tokens: Optional[int] = None cache_read_input_tokens: Optional[int] = None - + + server_tool_use: Optional[ServerToolUse] = None web_search_requests: Optional[int] = None completion_tokens_details: Optional[CompletionTokensDetails] = None prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None @@ -462,6 +464,8 @@ class ChunkProcessor: completion_tokens_details = usage_chunk_dict[ "completion_tokens_details" ] + if hasattr(usage_chunk, 'server_tool_use') and usage_chunk.server_tool_use is not None: + server_tool_use = usage_chunk.server_tool_use if ( usage_chunk_dict["prompt_tokens_details"] is not None and getattr( @@ -483,6 +487,7 @@ class ChunkProcessor: completion_tokens=completion_tokens, cache_creation_input_tokens=cache_creation_input_tokens, cache_read_input_tokens=cache_read_input_tokens, + server_tool_use=server_tool_use, web_search_requests=web_search_requests, completion_tokens_details=completion_tokens_details, prompt_tokens_details=prompt_tokens_details, @@ -513,6 +518,9 @@ class ChunkProcessor: "cache_read_input_tokens" ] + server_tool_use: Optional[ServerToolUse] = calculated_usage_per_chunk[ + "server_tool_use" + ] web_search_requests: Optional[int] = calculated_usage_per_chunk[ "web_search_requests" ] @@ -576,6 +584,8 @@ class ChunkProcessor: if prompt_tokens_details is not None: returned_usage.prompt_tokens_details = prompt_tokens_details + if server_tool_use is not None: + returned_usage.server_tool_use = server_tool_use if web_search_requests is not None: if returned_usage.prompt_tokens_details is None: returned_usage.prompt_tokens_details = PromptTokensDetailsWrapper( diff --git a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py index aa879e14c3..a1f89dac5c 100644 --- a/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py +++ b/litellm/types/litellm_core_utils/streaming_chunk_builder_utils.py @@ -2,7 +2,7 @@ from typing import TYPE_CHECKING, Optional from typing_extensions import TypedDict -from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper +from ..utils import CompletionTokensDetails, PromptTokensDetailsWrapper, ServerToolUse class UsagePerChunk(TypedDict): @@ -10,6 +10,7 @@ class UsagePerChunk(TypedDict): completion_tokens: int cache_creation_input_tokens: Optional[int] cache_read_input_tokens: Optional[int] + server_tool_use: Optional[ServerToolUse] web_search_requests: Optional[int] completion_tokens_details: Optional[CompletionTokensDetails] prompt_tokens_details: Optional[PromptTokensDetailsWrapper] diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py index f663687433..2164a3b82e 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_chunk_builder_utils.py @@ -16,6 +16,7 @@ from litellm.types.utils import ( Function, ModelResponseStream, PromptTokensDetails, + ServerToolUse, StreamingChoices, Usage, ) @@ -325,3 +326,83 @@ def test_stream_chunk_builder_litellm_usage_chunks(): assert usage.prompt_tokens == 50 assert usage.completion_tokens == 27 assert usage.total_tokens == 77 + + +def test_stream_chunk_builder_anthropic_web_search(): + # Prepare two mocked streaming chunks with usage split across them + chunk1 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513206, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + provider_specific_fields=None, + content="", + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=0, + prompt_tokens=50, + total_tokens=50, + completion_tokens_details=None, + server_tool_use=ServerToolUse(web_search_requests=2), + prompt_tokens_details=None, + ), + ) + + chunk2 = ModelResponseStream( + id="chatcmpl-mocked-usage-1", + created=1745513207, + model="claude-sonnet-4-5-20250929", + object="chat.completion.chunk", + system_fingerprint=None, + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + provider_specific_fields=None, + content=None, + role=None, + function_call=None, + tool_calls=None, + audio=None, + ), + logprobs=None, + ) + ], + provider_specific_fields=None, + stream_options={"include_usage": True}, + usage=Usage( + completion_tokens=27, + prompt_tokens=0, + total_tokens=27, + completion_tokens_details=None, + prompt_tokens_details=None, + ), + ) + + chunks = [chunk1, chunk2] + processor = ChunkProcessor(chunks=chunks) + + usage = processor.calculate_usage( + chunks=chunks, model="claude-sonnet-4-5-20250929", completion_output="" + ) + + assert usage.prompt_tokens == 50 + assert usage.completion_tokens == 27 + assert usage.total_tokens == 77 + assert usage.server_tool_use['web_search_requests'] == 2 \ No newline at end of file From 70a13258477f2868cbae00919205af433fa68625 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 15:01:15 -0800 Subject: [PATCH 11/23] docs: more doc cleanup --- .../index.md | 188 ++++++++++-------- ...odel_prices_and_context_window_backup.json | 52 +++++ model_prices_and_context_window.json | 52 +++++ 3 files changed, 209 insertions(+), 83 deletions(-) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 0b0f4a5416..051235bc74 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -26,6 +26,13 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe --- +| Feature | Supported Models | +|---------|-----------------| +| Tool Search | Claude Opus 4.5, Sonnet 4.5 | +| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | +| Input Examples | Claude Opus 4.5, Sonnet 4.5 | +| Effort Parameter | Claude Opus 4.5 only | + ## Usage @@ -222,6 +229,104 @@ curl --location 'http://0.0.0.0:4000/bedrock/model/claude-4/converse' \ +## Usage - Vertex AI + + + + + +```python +from litellm import completion +import json + +## GET CREDENTIALS +## RUN ## +# !gcloud auth application-default login - run this to add vertex credentials to your env +## OR ## +file_path = 'path/to/vertex_ai_service_account.json' + +# Load the JSON file +with open(file_path, 'r') as file: + vertex_credentials = json.load(file) + +# Convert to JSON string +vertex_credentials_json = json.dumps(vertex_credentials) + +## COMPLETION CALL +response = completion( + model="vertex_ai/claude-opus-4-5@20251101", + messages=[{ "content": "Hello, how are you?","role": "user"}], + vertex_credentials=vertex_credentials_json, + vertex_project="your-project-id", + vertex_location="us-east5" +) +``` + + + + +**1. Setup config.yaml** + +```yaml +model_list: + - model_name: claude-4 ### RECEIVED MODEL NAME ### + litellm_params: + model: vertex_ai/claude-opus-4-5@20251101 + vertex_credentials: "/path/to/service_account.json" + vertex_project: "your-project-id" + vertex_location: "us-east5" +``` + +**2. Start the proxy** + +```bash +litellm --config /path/to/config.yaml +``` + +**3. Test it!** + + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + +```bash +curl --location 'http://0.0.0.0:4000/v1/messages' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "max_tokens": 1024, + "messages": [ + { + "role": "user", + "content": "what llm are you" + } + ] + } +' +``` + + + + + + + ## Tool Search {#tool-search} ### Usage Example @@ -336,8 +441,6 @@ tools = [ ## Programmatic Tool Calling {#programmatic-tool-calling} -### Usage Example - ```python import litellm import json @@ -428,8 +531,6 @@ print("\nFinal answer:", final_response.choices[0].message.content) ## Tool Input Examples {#tool-input-examples} -### Usage Example - ```python import litellm @@ -755,82 +856,3 @@ This combination enables: 4. **Cost control** - Effort parameter optimizes token spend 5. **Full visibility** - Track all usage metrics ---- - -## Getting Started - -### Installation - -```bash -pip install litellm --upgrade -``` - -### Configuration - -```python -import os -import litellm - -# Set your API key -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -# LiteLLM automatically handles beta headers for all features -``` - -### Supported Models - -| Feature | Supported Models | -|---------|-----------------| -| Tool Search | Claude Opus 4.5, Sonnet 4.5 | -| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | -| Input Examples | Claude Opus 4.5, Sonnet 4.5 | -| Effort Parameter | Claude Opus 4.5 only | - -### Supported Endpoints - -**Note**: All features are supported on the `/chat/completions` endpoint only. - -| Feature | Supported Models | -|---------|-----------------| -| Tool Search | Claude Opus 4.5, Sonnet 4.5 | -| Programmatic Tool Calling | Claude Opus 4.5, Sonnet 4.5 | -| Input Examples | Claude Opus 4.5, Sonnet 4.5 | -| Effort Parameter | Claude Opus 4.5 only | - -### Provider Support - -All features work across: -- ✅ Standard Anthropic API -- ✅ Azure Anthropic -- ✅ Vertex AI Anthropic -- ✅ LiteLLM Proxy - ---- - -## Conclusion - -These advanced Anthropic features in LiteLLM enable you to build more sophisticated, efficient, and cost-effective AI applications: - -- **Tool Search** scales to thousands of tools -- **Programmatic Tool Calling** reduces latency and tokens -- **Input Examples** improve accuracy -- **Effort Parameter** controls costs - -All features work seamlessly together and are supported across all Anthropic providers through LiteLLM's unified interface. - -### Resources - -- [LiteLLM Documentation](https://docs.litellm.ai/) -- [Anthropic Tool Search Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_search) -- [Anthropic Programmatic Tool Calling Docs](https://docs.litellm.ai/docs/providers/anthropic_programmatic_tool_calling) -- [Anthropic Input Examples Docs](https://docs.litellm.ai/docs/providers/anthropic_tool_input_examples) -- [Anthropic Effort Parameter Docs](https://docs.litellm.ai/docs/providers/anthropic_effort) - -### Get Started Today - -```bash -pip install litellm --upgrade -``` - -Happy building! 🚀 - diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f9cc8bfa06..e51e5bb4b2 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -24604,6 +24604,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index b3dc11e206..ff8766003c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -24605,6 +24605,58 @@ "supports_tool_choice": true, "supports_vision": true }, + "vertex_ai/claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, + "vertex_ai/claude-opus-4-5@20251101": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "vertex_ai-anthropic_models", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "vertex_ai/claude-sonnet-4-5": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, From 3da9974a8770a5d05f839d78a7e15739b0664217 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 15:54:55 -0800 Subject: [PATCH 12/23] Tests --- .../src/components/view_users/table.test.tsx | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/ui/litellm-dashboard/src/components/view_users/table.test.tsx b/ui/litellm-dashboard/src/components/view_users/table.test.tsx index 5b612b2732..278a42e896 100644 --- a/ui/litellm-dashboard/src/components/view_users/table.test.tsx +++ b/ui/litellm-dashboard/src/components/view_users/table.test.tsx @@ -1,6 +1,5 @@ -import { render } from "@testing-library/react"; +import { act, fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; -import React from "react"; import { UserDataTable } from "./table"; @@ -21,7 +20,7 @@ describe("UserDataTable", () => { const updateFilters = vi.fn(); - const { getByText } = render( + render( { />, ); - expect(getByText("Filters")).toBeInTheDocument(); + expect(screen.getByText("Filters")).toBeInTheDocument(); + }); + + it("should call onSortChange when clicking a sortable header", () => { + const filters = { + email: "", + user_id: "", + user_role: "", + sso_user_id: "", + team: "", + model: "", + min_spend: null, + max_spend: null, + sort_by: "created_at", + sort_order: "desc" as const, + }; + + const updateFilters = vi.fn(); + const onSortChange = vi.fn(); + + const possibleUIRoles = { + admin: { ui_label: "Admin" }, + user: { ui_label: "User" }, + }; + + render( + , + ); + + const emailHeader = screen.getByRole("columnheader", { name: /email/i }); + act(() => { + fireEvent.click(emailHeader); + }); + + expect(onSortChange).toHaveBeenCalledWith("user_email", "desc"); }); }); From 8ee6812edff5d1e79d5efbabd5c801a45439b1cf Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 15:58:51 -0800 Subject: [PATCH 13/23] docs: cleanup launch post --- .../index.md | 503 ++++++++++++++++-- ...odel_prices_and_context_window_backup.json | 15 +- 2 files changed, 470 insertions(+), 48 deletions(-) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 051235bc74..9753529f57 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -329,8 +329,13 @@ curl --location 'http://0.0.0.0:4000/v1/messages' \ ## Tool Search {#tool-search} +This lets Claude work with thousands of tools, by dynamically loading tools on-demand, instead of loading all tools into the context window upfront. + ### Usage Example + + + ```python import litellm import os @@ -407,7 +412,7 @@ tools = [ # Make a request - Claude will search for and use relevant tools response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", + model="anthropic/claude-opus-4-5-20251101", messages=[{ "role": "user", "content": "What's the weather like in San Francisco?" @@ -422,6 +427,108 @@ print("Tool calls:", response.choices[0].message.tool_calls) if hasattr(response.usage, 'server_tool_use'): print(f"Tool searches performed: {response.usage.server_tool_use.tool_search_requests}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "What's the weather like in San Francisco?" + }], + "tools": [ + # Tool search tool (regex variant) + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # Deferred tools - loaded on-demand + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the current weather in a given location. Returns temperature and conditions.", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA" + }, + "unit": { + "type": "string", + "enum": ["celsius", "fahrenheit"], + "description": "Temperature unit" + } + }, + "required": ["location"] + } + }, + "defer_loading": True # Load on-demand + }, + { + "type": "function", + "function": { + "name": "search_files", + "description": "Search through files in the workspace using keywords", + "parameters": { + "type": "object", + "properties": { + "query": {"type": "string"}, + "file_types": { + "type": "array", + "items": {"type": "string"} + } + }, + "required": ["query"] + } + }, + "defer_loading": True + }, + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute SQL queries against the database", + "parameters": { + "type": "object", + "properties": { + "sql": {"type": "string"} + }, + "required": ["sql"] + } + }, + "defer_loading": True + } + ] +} +' +``` + + ### BM25 Variant (Natural Language Search) @@ -441,6 +548,11 @@ tools = [ ## Programmatic Tool Calling {#programmatic-tool-calling} +Programmatic tool calling allows Claude to write code that calls your tools programmatically. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/programmatic-tool-calling) + + + + ```python import litellm import json @@ -527,10 +639,80 @@ final_response = litellm.completion( print("\nFinal answer:", final_response.choices[0].message.content) ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Query sales data for West, East, and Central regions, then tell me which had the highest revenue" + }], + "tools": [ + # Code execution tool (required for programmatic calling) + { + "type": "code_execution_20250825", + "name": "code_execution" + }, + # Tool that can be called from code + { + "type": "function", + "function": { + "name": "query_database", + "description": "Execute a SQL query against the sales database. Returns a list of rows as JSON objects.", + "parameters": { + "type": "object", + "properties": { + "sql": { + "type": "string", + "description": "SQL query to execute" + } + }, + "required": ["sql"] + } + }, + "allowed_callers": ["code_execution_20250825"] # Enable programmatic calling + } + ] +} +' +``` + + + --- ## Tool Input Examples {#tool-input-examples} +You can now provide Claude with examples of how to use your tools. [Learn more](https://platform.claude.com/docs/en/agents-and-tools/tool-use/tool-input-examples) + + + + + ```python import litellm @@ -609,12 +791,124 @@ response = litellm.completion( print("Tool call:", response.choices[0].message.tool_calls[0].function.arguments) ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Schedule a team meeting for tomorrow at 2pm for 45 minutes with john@company.com and sarah@company.com" + }], + "tools": [ + { + "type": "function", + "function": { + "name": "create_calendar_event", + "description": "Create a new calendar event with attendees and reminders", + "parameters": { + "type": "object", + "properties": { + "title": {"type": "string"}, + "start_time": { + "type": "string", + "description": "ISO 8601 format: YYYY-MM-DDTHH:MM:SS" + }, + "duration_minutes": {"type": "integer"}, + "attendees": { + "type": "array", + "items": { + "type": "object", + "properties": { + "email": {"type": "string"}, + "optional": {"type": "boolean"} + } + } + }, + "reminders": { + "type": "array", + "items": { + "type": "object", + "properties": { + "minutes_before": {"type": "integer"}, + "method": {"type": "string", "enum": ["email", "popup"]} + } + } + } + }, + "required": ["title", "start_time", "duration_minutes"] + } + }, + # Provide concrete examples + "input_examples": [ + { + "title": "Team Standup", + "start_time": "2025-01-15T09:00:00", + "duration_minutes": 30, + "attendees": [ + {"email": "alice@company.com", "optional": False}, + {"email": "bob@company.com", "optional": False} + ], + "reminders": [ + {"minutes_before": 15, "method": "popup"} + ] + }, + { + "title": "Lunch Break", + "start_time": "2025-01-15T12:00:00", + "duration_minutes": 60 + # Demonstrates optional fields can be omitted + } + ] + } +] +} +' +``` + + + --- ## Effort Parameter: Control Token Usage {#effort-parameter} +Controls aspects like how much effort the model puts into its response, via `output_config={"effort": ..}`. + +:::info + +Soon, we will map OpenAI's `reasoning_effort` parameter to this. +::: + +Potential Values for `effort` parameter: `"high"`, `"medium"`, `"low"`. + ### Usage Example + + + ```python import litellm @@ -660,41 +954,46 @@ print(f"Medium: {response_medium.usage.completion_tokens} tokens") print(f"Low: {response_low.usage.completion_tokens} tokens") ``` -### Effort with Tool Use + + -Lower effort affects both explanations and tool calls: +1. Setup config.yaml -```python -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": {"type": "string"} - }, - "required": ["location"] - } - } - } -] - -# Low effort = fewer tool calls, more direct -response = litellm.completion( - model="anthropic/claude-opus-4-5-20251101", - messages=[{ - "role": "user", - "content": "Check weather in San Francisco, New York, and London" - }], - tools=tools, - output_config={"effort": "low"} # May combine into fewer calls -) +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY ``` ---- +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze the trade-offs between microservices and monolithic architectures" + }], + "output_config": { + "effort": "high" + } + } +' +``` + + + ## Cost Tracking: Monitor Tool Search Usage {#cost-tracking} @@ -702,8 +1001,15 @@ response = litellm.completion( Tool search operations are tracked separately in the usage object, allowing you to monitor and optimize costs. +It is available in the `usage` object, under `server_tool_use.tool_search_requests`. + +Anthropic charges $0.0001 per tool search request. + ### Tracking Example + + + ```python import litellm @@ -749,6 +1055,65 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use print(f" Total: ${total_cost:.6f}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Find and use the weather tool for San Francisco" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ] + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + ### Cost Optimization Tips 1. **Keep frequently used tools non-deferred** (3-5 tools) @@ -756,15 +1121,6 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use 3. **Monitor search requests** to identify optimization opportunities 4. **Combine with effort parameter** for maximum efficiency -```python -# Optimized for cost -response = litellm.completion( - model="anthropic/claude-sonnet-4-5-20250929", - messages=[{"role": "user", "content": "Simple query"}], - tools=tools_with_search, - output_config={"effort": "low"} # Reduce output tokens -) -``` --- @@ -774,6 +1130,9 @@ response = litellm.completion( These features work together seamlessly. Here's a real-world example combining all of them: + + + ```python import litellm import json @@ -846,6 +1205,68 @@ if hasattr(response.usage, 'server_tool_use') and response.usage.server_tool_use print(f"\nResponse: {response.choices[0].message.content}") ``` + + + +1. Setup config.yaml + +```yaml +model_list: + - model_name: claude-4 + litellm_params: + model: anthropic/claude-opus-4-5-20251101 + api_key: os.environ/ANTHROPIC_API_KEY +``` + +2. Start the proxy + +```bash +litellm --config /path/to/config.yaml +``` + +3. Test it! + +```bash +curl --location 'http://0.0.0.0:4000/chat/completions' \ +--header 'Content-Type: application/json' \ +--header 'Authorization: Bearer $LITELLM_KEY' \ +--data ' { + "model": "claude-4", + "messages": [{ + "role": "user", + "content": "Analyze sales by region for the last quarter and identify top performers" + }], + "tools": [ + { + "type": "tool_search_tool_regex_20251119", + "name": "tool_search_tool_regex" + }, + # ... 100 deferred tools + ], + "output_config": { + "effort": "medium" + } + } +' +``` + +Expected Response: + +```json +{ + ..., + "usage": { + ..., + "server_tool_use": { + "tool_search_requests": 1 + } + } +} +``` + + + + ### Real-World Benefits This combination enables: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e51e5bb4b2..ff8766003c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -2687,7 +2687,7 @@ "/v1/audio/transcriptions" ] }, - "azure/gpt-5.1-2025-11-13": { + "azure/gpt-5.1-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2723,7 +2723,7 @@ "supports_service_tier": true, "supports_vision": true }, - "azure/gpt-5.1-chat-2025-11-13": { + "azure/gpt-5.1-chat-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -2758,7 +2758,7 @@ "supports_tool_choice": false, "supports_vision": true }, - "azure/gpt-5.1-codex-2025-11-13": { + "azure/gpt-5.1-codex-2025-11-13": { "cache_read_input_token_cost": 1.25e-07, "cache_read_input_token_cost_priority": 2.5e-07, "input_cost_per_token": 1.25e-06, @@ -19521,7 +19521,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/deepseek-v3.1:671b-cloud": { + "ollama/deepseek-v3.1:671b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 163840, @@ -19531,7 +19531,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:120b-cloud": { + "ollama/gpt-oss:120b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -19541,7 +19541,7 @@ "output_cost_per_token": 0.0, "supports_function_calling": true }, - "ollama/gpt-oss:20b-cloud": { + "ollama/gpt-oss:20b-cloud" : { "input_cost_per_token": 0.0, "litellm_provider": "ollama", "max_input_tokens": 131072, @@ -22037,6 +22037,7 @@ "supports_reasoning": true, "source": "https://cloud.sambanova.ai/plans/pricing" }, + "snowflake/claude-3-5-sonnet": { "litellm_provider": "snowflake", "max_input_tokens": 18000, @@ -23231,7 +23232,7 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, - "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "us.anthropic.claude-opus-4-5-20251101-v1:0": { "cache_creation_input_token_cost": 6.25e-06, "cache_read_input_token_cost": 5e-07, "input_cost_per_token": 5e-06, From 5cb5c2a7b7fcd80de1caa803701af2d596965fbb Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 25 Nov 2025 16:04:27 -0800 Subject: [PATCH 14/23] docs: more doc cleanup --- .../blog/anthropic_opus_4_5_and_advanced_features/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 9753529f57..b545e93618 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -33,6 +33,8 @@ This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced fe | Input Examples | Claude Opus 4.5, Sonnet 4.5 | | Effort Parameter | Claude Opus 4.5 only | +Supported Providers: [Anthropic](../../docs/providers/anthropic), [Bedrock](../../docs/providers/bedrock), [Vertex AI](../../docs/providers/vertex_partner#vertex-ai---anthropic-claude). + ## Usage From 6e5c7c0008f6c157a27ffc4530fe509b57687976 Mon Sep 17 00:00:00 2001 From: Otavio Brito <69211663+otaviofbrito@users.noreply.github.com> Date: Tue, 25 Nov 2025 21:41:35 -0300 Subject: [PATCH 15/23] fix transcription exception handling - /audio/transcriptions (#16791) * fix transcription exception handling * reraise the exception --- litellm/proxy/proxy_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1330774286..7c415b8106 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -5483,6 +5483,7 @@ async def audio_transcriptions( file_object = io.BytesIO(file_content) file_object.name = file.filename data["file"] = file_object + try: ### CALL HOOKS ### - modify incoming data / reject request before calling the model data = await proxy_logging_obj.pre_call_hook( @@ -5500,7 +5501,7 @@ async def audio_transcriptions( ) response = await llm_call except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + raise e finally: file_object.close() # close the file read in by io library From 5ec3f19a53dbf6028df7279468552b9db9442320 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Tue, 25 Nov 2025 16:57:38 -0800 Subject: [PATCH 16/23] Make model select required for team, add checks for all-proxy-models --- .../src/components/OldTeams.test.tsx | 103 ++++++++++++++---- .../src/components/OldTeams.tsx | 14 ++- .../src/components/team/team_info.test.tsx | 88 ++++++++++++++- .../src/components/team/team_info.tsx | 90 ++++++++------- 4 files changed, 225 insertions(+), 70 deletions(-) diff --git a/ui/litellm-dashboard/src/components/OldTeams.test.tsx b/ui/litellm-dashboard/src/components/OldTeams.test.tsx index 261178191f..7f4ec3b09c 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.test.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.test.tsx @@ -1,5 +1,6 @@ -import { act, fireEvent, render, screen } from "@testing-library/react"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { beforeEach, describe, expect, it, vi } from "vitest"; +import { fetchAvailableModelsForTeamOrKey } from "./key_team_helpers/fetch_available_models_team_key"; import { teamCreateCall } from "./networking"; import OldTeams from "./OldTeams"; @@ -23,6 +24,28 @@ vi.mock("./molecules/notifications_manager", () => ({ }, })); +vi.mock("./key_team_helpers/fetch_available_models_team_key", () => ({ + fetchAvailableModelsForTeamOrKey: vi.fn(), + getModelDisplayName: vi.fn((model: string) => model), + unfurlWildcardModelsInList: vi.fn((teamModels: string[], allModels: string[]) => { + const wildcardDisplayNames: string[] = []; + const expandedModels: string[] = []; + + teamModels.forEach((teamModel) => { + if (teamModel.endsWith("/*")) { + const provider = teamModel.replace("/*", ""); + const matchingModels = allModels.filter((model) => model.startsWith(provider + "/")); + expandedModels.push(...matchingModels); + wildcardDisplayNames.push(teamModel); + } else { + expandedModels.push(teamModel); + } + }); + + return [...wildcardDisplayNames, ...expandedModels].filter((item, index, array) => array.indexOf(item) === index); + }), +})); + describe("OldTeams - handleCreate organization handling", () => { beforeEach(() => { vi.clearAllMocks(); @@ -236,7 +259,7 @@ describe("OldTeams - handleCreate organization handling", () => { }); it("should clear the delete modal when the cancel button is clicked", async () => { - const { getByRole, getByTestId } = render( + render( { organizations={[]} />, ); - const deleteTeamButton = getByTestId("delete-team-button"); + const deleteTeamButton = screen.getByTestId("delete-team-button"); act(() => { fireEvent.click(deleteTeamButton); }); @@ -275,7 +298,7 @@ describe("OldTeams - empty state", () => { }); it("should display empty state message when teams array is empty", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should display empty state message when teams is null", () => { - const { getByText } = render( + render( { />, ); - expect(getByText("No teams found")).toBeInTheDocument(); - expect(getByText("Adjust your filters or create a new team")).toBeInTheDocument(); + expect(screen.getByText("No teams found")).toBeInTheDocument(); + expect(screen.getByText("Adjust your filters or create a new team")).toBeInTheDocument(); }); it("should not display empty state when teams array has items", () => { - const { queryByText, getByText } = render( + render( { />, ); - expect(queryByText("No teams found")).not.toBeInTheDocument(); - expect(queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); - expect(getByText("Test Team")).toBeInTheDocument(); + expect(screen.queryByText("No teams found")).not.toBeInTheDocument(); + expect(screen.queryByText("Adjust your filters or create a new team")).not.toBeInTheDocument(); + expect(screen.getByText("Test Team")).toBeInTheDocument(); }); }); @@ -473,7 +496,7 @@ describe("OldTeams - Default Team Settings tab visibility", () => { }); it("should show Default Team Settings tab for Admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should show Default Team Settings tab for proxy_admin role", () => { - const { getByRole } = render( + render( { />, ); - expect(getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); + expect(screen.getByRole("tab", { name: "Default Team Settings" })).toBeInTheDocument(); }); it("should not show Default Team Settings tab for proxy_admin_viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); }); it("should not show Default Team Settings tab for Admin Viewer role", () => { - const { queryByRole } = render( + render( { />, ); - expect(queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + expect(screen.queryByRole("tab", { name: "Default Team Settings" })).not.toBeInTheDocument(); + }); +}); + +describe("OldTeams - all-proxy-models dropdown visibility", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + }); + + it("should not show all-proxy-models option when user has no access to it", async () => { + vi.mocked(fetchAvailableModelsForTeamOrKey).mockResolvedValue(["gpt-4", "gpt-3.5-turbo"]); + + render( + , + ); + + await waitFor(() => { + expect(fetchAvailableModelsForTeamOrKey).toHaveBeenCalled(); + }); + + const createButton = screen.getByRole("button", { name: /create new team/i }); + act(() => { + fireEvent.click(createButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText(/models/i)).toBeInTheDocument(); + }); + const allProxyModelsOption = screen.queryByText("All Proxy Models"); + expect(allProxyModelsOption).not.toBeInTheDocument(); }); }); diff --git a/ui/litellm-dashboard/src/components/OldTeams.tsx b/ui/litellm-dashboard/src/components/OldTeams.tsx index cc66a23eb4..83ec28a517 100644 --- a/ui/litellm-dashboard/src/components/OldTeams.tsx +++ b/ui/litellm-dashboard/src/components/OldTeams.tsx @@ -1139,12 +1139,20 @@ const Teams: React.FC = ({ } + rules={[ + { + required: true, + message: "Please select at least one model", + }, + ]} name="models" > - - All Proxy Models - + {(isProxyAdminRole(userRole || "") || userModels.includes("all-proxy-models")) && ( + + All Proxy Models + + )} No Default Models diff --git a/ui/litellm-dashboard/src/components/team/team_info.test.tsx b/ui/litellm-dashboard/src/components/team/team_info.test.tsx index 526f0972d9..17041659ce 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.test.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.test.tsx @@ -1,7 +1,7 @@ +import * as networking from "@/components/networking"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; import { afterEach, describe, expect, it, vi } from "vitest"; import TeamInfoView from "./team_info"; -import { render, waitFor } from "@testing-library/react"; -import * as networking from "@/components/networking"; // Mock the networking module vi.mock("@/components/networking", () => ({ @@ -61,7 +61,7 @@ describe("TeamInfoView", () => { vi.mocked(networking.getGuardrailsList).mockResolvedValue([]); vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); - const { getByText } = render( + render( {}} @@ -75,7 +75,87 @@ describe("TeamInfoView", () => { />, ); await waitFor(() => { - expect(getByText("User ID")).toBeInTheDocument(); + expect(screen.queryByText("User ID")).not.toBeNull(); }); }); + + it("should not show all-proxy-models option when user has no access to it", async () => { + vi.mocked(networking.teamInfoCall).mockResolvedValue({ + team_id: "123", + team_info: { + team_alias: "Test Team", + team_id: "123", + organization_id: null, + admins: ["admin@test.com"], + members: ["user1@test.com", "user2@test.com"], + members_with_roles: [ + { + user_id: "user1@test.com", + user_email: "user1@test.com", + role: "member", + spend: 0, + budget_id: "budget1", + }, + ], + metadata: {}, + tpm_limit: null, + rpm_limit: null, + max_budget: null, + budget_duration: null, + models: ["gpt-4"], + blocked: false, + spend: 0, + max_parallel_requests: null, + budget_reset_at: null, + model_id: null, + litellm_model_table: null, + created_at: "2024-01-01T00:00:00Z", + team_member_budget_table: null, + }, + keys: [], + team_memberships: [], + }); + + vi.mocked(networking.getGuardrailsList).mockResolvedValue([]); + vi.mocked(networking.fetchMCPAccessGroups).mockResolvedValue([]); + + render( + {}} + onClose={() => {}} + accessToken="123" + is_team_admin={true} + is_proxy_admin={true} + userModels={["gpt-4", "gpt-3.5-turbo"]} + editTeam={false} + premiumUser={false} + />, + ); + + await waitFor(() => { + expect(screen.getAllByText("Test Team")).not.toBeNull(); + }); + + const settingsTab = screen.getByRole("tab", { name: "Settings" }); + act(() => { + fireEvent.click(settingsTab); + }); + + await waitFor(() => { + expect(screen.getByText("Team Settings")).toBeInTheDocument(); + }); + + const editButton = screen.getByRole("button", { name: "Edit Settings" }); + act(() => { + fireEvent.click(editButton); + }); + + await waitFor(() => { + expect(screen.getByLabelText("Models")).toBeInTheDocument(); + }); + + const allProxyModelsOption = screen.queryByText("All Proxy Models"); + expect(allProxyModelsOption).not.toBeInTheDocument(); + }); }); diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index bd52b5aef4..1c6ba629ef 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -1,50 +1,50 @@ -import React, { useState, useEffect } from "react"; -import NumericalInput from "../shared/numerical_input"; +import UserSearchModal from "@/components/common_components/user_search_modal"; import { - Card, - Title, - Text, - Tab, - TabList, - TabGroup, - TabPanel, - TabPanels, - Grid, - Badge, - Button as TremorButton, - TextInput, -} from "@tremor/react"; -import TeamMembersComponent from "./team_member_view"; -import MemberPermissions from "./member_permissions"; -import { - teamInfoCall, - teamMemberDeleteCall, - teamMemberAddCall, - teamMemberUpdateCall, - Member, - teamUpdateCall, getGuardrailsList, + Member, + teamInfoCall, + teamMemberAddCall, + teamMemberDeleteCall, + teamMemberUpdateCall, + teamUpdateCall, } from "@/components/networking"; -import { Button, Form, Input, Select, Switch, message, Tooltip } from "antd"; +import { formatNumberWithCommas } from "@/utils/dataUtils"; +import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; import { InfoCircleOutlined } from "@ant-design/icons"; import { ArrowLeftIcon } from "@heroicons/react/outline"; -import MemberModal from "./edit_membership"; -import UserSearchModal from "@/components/common_components/user_search_modal"; +import { + Badge, + Card, + Grid, + Tab, + TabGroup, + TabList, + TabPanel, + TabPanels, + Text, + TextInput, + Title, + Button as TremorButton, +} from "@tremor/react"; +import { Button, Form, Input, message, Select, Switch, Tooltip } from "antd"; +import { CheckIcon, CopyIcon } from "lucide-react"; +import React, { useEffect, useState } from "react"; +import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; +import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; import { getModelDisplayName } from "../key_team_helpers/fetch_available_models_team_key"; -import ObjectPermissionsView from "../object_permissions_view"; -import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import LoggingSettingsView from "../logging_settings_view"; import MCPServerSelector from "../mcp_server_management/MCPServerSelector"; import MCPToolPermissions from "../mcp_server_management/MCPToolPermissions"; -import { formatNumberWithCommas } from "@/utils/dataUtils"; -import EditLoggingSettings from "./EditLoggingSettings"; -import LoggingSettingsView from "../logging_settings_view"; -import { fetchMCPAccessGroups } from "../networking"; -import { CheckIcon, CopyIcon } from "lucide-react"; -import { copyToClipboard as utilCopyToClipboard } from "../../utils/dataUtils"; import NotificationsManager from "../molecules/notifications_manager"; -import PassThroughRoutesSelector from "../common_components/PassThroughRoutesSelector"; -import { mapEmptyStringToNull } from "@/utils/keyUpdateUtils"; -import DeleteResourceModal from "../common_components/DeleteResourceModal"; +import { fetchMCPAccessGroups } from "../networking"; +import ObjectPermissionsView from "../object_permissions_view"; +import NumericalInput from "../shared/numerical_input"; +import VectorStoreSelector from "../vector_store_management/VectorStoreSelector"; +import MemberModal from "./edit_membership"; +import EditLoggingSettings from "./EditLoggingSettings"; +import MemberPermissions from "./member_permissions"; +import TeamMembersComponent from "./team_member_view"; export interface TeamMembership { user_id: string; @@ -586,11 +586,17 @@ const TeamInfoView: React.FC = ({ - +