diff --git a/docs/my-website/docs/tutorials/claude_responses_api.md b/docs/my-website/docs/tutorials/claude_responses_api.md new file mode 100644 index 0000000000..d95f75c7d7 --- /dev/null +++ b/docs/my-website/docs/tutorials/claude_responses_api.md @@ -0,0 +1,62 @@ +import Image from '@theme/IdealImage'; + +# Call Responses API models on Claude Code + +This tutorial shows how to call the Responses API models like `codex-mini` and `o3-pro` from the Claude Code endpoint on LiteLLM. + + +Pre-requisites: + +- [Claude Code](https://docs.anthropic.com/en/docs/claude-code/overview) installed +- LiteLLM v1.72.6-stable or higher + + +### 1. Setup config.yaml + +```yaml +model_list: + - model_name: codex-mini + litellm_params: + model: codex-mini + api_key: sk-proj-1234567890 + api_base: https://api.openai.com/v1 +``` + +### 2. Start proxy + +```bash +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Test it! (Curl) + +```bash +curl -X POST http://0.0.0.0:4000/v1/messages \ +-H "Authorization: Bearer sk-proj-1234567890" \ +-H "Content-Type: application/json" \ +-d '{ + "model": "codex-mini", + "messages": [{"role": "user", "content": "What is the capital of France?"}] +}' +``` + +### 4. Test it! (Claude Code) + +- Setup environment variables + +```bash +export ANTHROPIC_API_BASE="http://0.0.0.0:4000" +export ANTHROPIC_API_KEY="sk-1234" # replace with your LiteLLM key +``` + +- Start a Claude Code session + +```bash +claude --model codex-mini-latest +``` + +- Send a message + + \ No newline at end of file diff --git a/docs/my-website/img/release_notes/claude_code_demo.png b/docs/my-website/img/release_notes/claude_code_demo.png new file mode 100644 index 0000000000..ffde286c8f Binary files /dev/null and b/docs/my-website/img/release_notes/claude_code_demo.png differ diff --git a/docs/my-website/img/release_notes/codex_on_claude_code.jpg b/docs/my-website/img/release_notes/codex_on_claude_code.jpg new file mode 100644 index 0000000000..f728737b8d Binary files /dev/null and b/docs/my-website/img/release_notes/codex_on_claude_code.jpg differ diff --git a/docs/my-website/img/release_notes/mcp_permissions.png b/docs/my-website/img/release_notes/mcp_permissions.png new file mode 100644 index 0000000000..6818804a84 Binary files /dev/null and b/docs/my-website/img/release_notes/mcp_permissions.png differ diff --git a/docs/my-website/release_notes/v1.72.6-stable/index.md b/docs/my-website/release_notes/v1.72.6-stable/index.md index 41da1ec1ac..4006f02291 100644 --- a/docs/my-website/release_notes/v1.72.6-stable/index.md +++ b/docs/my-website/release_notes/v1.72.6-stable/index.md @@ -31,17 +31,52 @@ This version is not out yet. ## TLDR + * **Why Upgrade** - + - Codex-mini on Claude Code: You can now use `codex-mini` (OpenAI’s code assistant model) via Claude Code. + - MCP Permissions Management: Manage permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. + - UI: Turn on/off auto refresh on logs view. + - Rate Limiting: Support for output token-only rate limiting. * **Who Should Read** + - Teams using `/v1/messages` API (Claude Code) + - Teams using **MCP** + - Teams giving access to self-hosted models and setting rate limits * **Risk of Upgrade** - + - **Low** + - No major changes to existing functionality or package updates. --- ## Key Highlights + +### MCP Permissions Management + +This release brings support for managing permissions for MCP Servers by Keys, Teams, Organizations (entities) on LiteLLM. When a MCP client attempts to list tools, LiteLLM will only return the tools the entity has permissions to access. + +This is great for use cases that require access to restricted data (e.g Jira MCP) that you don't want everyone to use. + +For Proxy Admins, this enables centralized management of all MCP Servers with access control. For developers, this means you'll only see the MCP tools assigned to you. + + + + +### Codex-mini on Claude Code + +This release brings support for calling `codex-mini` (OpenAI’s code assistant model) via Claude Code. + +This is done by LiteLLM enabling any Responses API model (including `o3-pro`) to be called via `/chat/completions` and `/v1/messages` endpoints. This includes: + +- Streaming calls +- Non-streaming calls +- Cost Tracking on success + failure for Responses API models + +Here's how to use it [today](../../docs/tutorials/claude_responses_api) + + + + --- @@ -202,7 +237,7 @@ This version is not out yet. - Make all commands show server URL - [PR](https://github.com/BerriAI/litellm/pull/10801) - **Unicorn** - Allow setting keep alive timeout - [PR](https://github.com/BerriAI/litellm/pull/11594) -- **Experimental Rate Limiting v2** +- **Experimental Rate Limiting v2** (enable via `EXPERIMENTAL_MULTI_INSTANCE_RATE_LIMITING="True"`) - Support specifying rate limit by output_tokens only - [PR](https://github.com/BerriAI/litellm/pull/11646) - Decrement parallel requests on call failure - [PR](https://github.com/BerriAI/litellm/pull/11646) - In-memory only rate limiting support - [PR](https://github.com/BerriAI/litellm/pull/11646) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index b71abf9bd0..3344981e17 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -507,6 +507,7 @@ const sidebars = { "tutorials/tag_management", 'tutorials/litellm_proxy_aporia', "tutorials/gemini_realtime_with_audio", + "tutorials/claude_responses_api", { type: "category", label: "LiteLLM Python SDK Tutorials", diff --git a/litellm/completion_extras/litellm_responses_transformation/handler.py b/litellm/completion_extras/litellm_responses_transformation/handler.py index a799bca437..ea5e8b4c8d 100644 --- a/litellm/completion_extras/litellm_responses_transformation/handler.py +++ b/litellm/completion_extras/litellm_responses_transformation/handler.py @@ -107,6 +107,7 @@ class ResponsesToCompletionBridgeHandler: headers=headers, litellm_logging_obj=logging_obj, ) + result = responses( **request_data, ) @@ -156,14 +157,17 @@ class ResponsesToCompletionBridgeHandler: logging_obj = validated_kwargs["logging_obj"] custom_llm_provider = validated_kwargs["custom_llm_provider"] - request_data = self.transformation_handler.transform_request( - model=model, - messages=messages, - optional_params=optional_params, - litellm_params=litellm_params, - headers=headers, - litellm_logging_obj=logging_obj, - ) + try: + request_data = self.transformation_handler.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers=headers, + litellm_logging_obj=logging_obj, + ) + except Exception as e: + raise e result = await aresponses( **request_data, diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index b9ef00b693..a108d367cc 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -1,6 +1,7 @@ """ Handler for transforming /chat/completions api requests to litellm.responses requests """ + import json from typing import ( TYPE_CHECKING, @@ -62,7 +63,15 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): if isinstance(content, str): instructions = content else: - raise ValueError(f"System message must be a string: {content}") + input_items.append( + { + "type": "message", + "role": role, + "content": self._convert_content_to_responses_format( + content, role # type: ignore + ), + } + ) elif role == "tool": # Convert tool message to function call output format input_items.append( @@ -93,7 +102,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): { "type": "message", "role": role, - "content": self._convert_content_to_responses_format(content), + "content": self._convert_content_to_responses_format( + content, cast(str, role) + ), } ) @@ -301,6 +312,14 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): streaming_response, sync_stream, json_mode ) + def _convert_content_str_to_input_text( + self, content: str, role: str + ) -> Dict[str, Any]: + if role == "user" or role == "system": + return {"type": "input_text", "text": content} + else: + return {"type": "output_text", "text": content} + def _convert_content_to_responses_format( self, content: Union[ @@ -309,6 +328,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): Union["OpenAIMessageContentListBlock", "ChatCompletionThinkingBlock"] ], ], + role: str, ) -> List[Dict[str, Any]]: """Convert chat completion content to responses API format""" verbose_logger.debug( @@ -316,7 +336,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) if isinstance(content, str): - result = [{"type": "input_text", "text": content}] + result = [self._convert_content_str_to_input_text(content, role)] verbose_logger.debug(f"Chat provider: String content -> {result}") return result elif isinstance(content, list): @@ -326,14 +346,16 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): f"Chat provider: Processing content item {i}: {type(item)} = {item}" ) if isinstance(item, str): - converted = {"type": "input_text", "text": item} + converted = self._convert_content_str_to_input_text(item, role) result.append(converted) verbose_logger.debug(f"Chat provider: -> {converted}") elif isinstance(item, dict): # Handle multimodal content original_type = item.get("type") if original_type == "text": - converted = {"type": "input_text", "text": item.get("text", "")} + converted = self._convert_content_str_to_input_text( + item.get("text", ""), role + ) result.append(converted) verbose_logger.debug(f"Chat provider: text -> {converted}") elif original_type == "image_url": @@ -371,10 +393,9 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) else: # Default to input_text for unknown types - converted = { - "type": "input_text", - "text": str(item.get("text", item)), - } + converted = self._convert_content_str_to_input_text( + str(item.get("text", item)), role + ) result.append(converted) verbose_logger.debug( f"Chat provider: unknown({original_type}) -> {converted}" @@ -382,7 +403,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): verbose_logger.debug(f"Chat provider: Final converted content: {result}") return result else: - result = [{"type": "input_text", "text": str(content)}] + result = [self._convert_content_str_to_input_text(str(content), role)] verbose_logger.debug(f"Chat provider: Other content type -> {result}") return result diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 7ea414d262..763d984ca4 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,4 +1,8 @@ model_list: + - model_name: codex-mini + litellm_params: + model: codex-mini-latest + api_key: os.environ/OPENAI_API_KEY - model_name: "gpt-4o-mini-openai" litellm_params: model: gpt-4o-mini diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 8775157ec8..61d160c7a7 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -233,6 +233,7 @@ def responses( # get llm provider logic litellm_params = GenericLiteLLMParams(**kwargs) + ## MOCK RESPONSE LOGIC if litellm_params.mock_response and isinstance( litellm_params.mock_response, str